Add export and import as tar file, Implements #128

src/dgenies/lib/functions.py

@@ -14,8 +14,9 @@ from dgenies.config_reader import AppConfigReader
 import dgenies.lib.validators as validators
 
 ALLOWED_EXTENSIONS = {"fasta": ['fa', 'fasta', 'fna', 'fa.gz', 'fasta.gz', 'fna.gz'],
-                      "idx": ['idx'],
-                      "map": [o[0] for o in getmembers(validators) if isfunction(o[1]) and not o[0].startswith("_")]}
+                      "idx": ['idx',],
+                      "map": [o[0] for o in getmembers(validators) if isfunction(o[1]) and not o[0].startswith("_")],
+                      "backup": ['tar']}
 # map: all functions of validators which does not starts with an underscore.
 
 

src/dgenies/lib/job_manager.py

@@ -21,6 +21,7 @@ from jinja2 import Template
 import traceback
 from pathlib import Path
 from urllib import request, parse
+import tarfile
 from dgenies.bin.split_fa import Splitter
 from dgenies.bin.index import index_file, Index
 from dgenies.bin.filter_contigs import Filter
@@ -40,7 +41,7 @@ if MODE == "webserver":
 class JobManager:
 
     def __init__(self, id_job: str, email: str=None, query: Fasta=None, target: Fasta=None, mailer=None,
-                 tool="minimap2", align: Fasta=None):
+                 tool="minimap2", align: Fasta=None, backup: Fasta=None):
         self.id_job = id_job
         self.email = email
         self.query = query
@@ -48,6 +49,7 @@ class JobManager:
         self.align = align
         if align is not None:
             self.aln_format = os.path.splitext(align.get_path())[1][1:]
+        self.backup = backup
         self.error = ""
         self.id_process = "-1"
         # Get configs:
@@ -716,6 +718,15 @@ class JobManager:
                     files_to_download.append([self.align, "align"])
                 else:
                     return False, True, True
+            if correct and self.backup is not None:
+                if self.backup.get_type() == "local":
+                    self.backup.set_path(self.__getting_local_file(self.backup, "backup"))
+                    correct, error_set, should_be_local = self.check_file("backup", should_be_local,
+                                                                          max_upload_size_readable)
+                elif self.__check_url(self.backup):
+                    files_to_download.append([self.backup, "backup"])
+                else:
+                    return False, True, True
 
             all_downloaded = True
             if correct :
@@ -870,7 +881,8 @@ class JobManager:
         sorter = Sorter(self.paf_raw, self.paf)
         sorter.sort()
         os.remove(self.paf_raw)
-        if self.target is not None and os.path.exists(self.target.get_path()):
+        if self.target is not None and os.path.exists(self.target.get_path()) and not \
+                self.target.get_path().endswith(".idx"):
             os.remove(self.target.get_path())
 
         self.align.set_path(self.paf)
@@ -1076,9 +1088,48 @@ class JobManager:
                 batch_type=job.batch_type)
             log.save()
 
+    def unpack_backup(self):
+        try:
+            with tarfile.open(self.backup.get_path(), "r") as tar:
+                names = tar.getnames()
+                if len(names) != 3:
+                    return False
+                for name in ["map.paf", "query.idx", "target.idx"]:
+                    if name not in names:
+                        return False
+                tar.extractall(self.output_dir)
+                shutil.move(self.paf, self.paf_raw)
+                if not validators.paf(self.paf_raw):
+                    return False
+                self.align = Fasta(name="map", path=self.paf_raw, type_f="local")
+                self.aln_format = "paf"
+                with open(os.path.join(self.output_dir, ".align"), "w") as aln:
+                    aln.write(self.paf_raw)
+                target_path = os.path.join(self.output_dir, "target.idx")
+                self.target = Fasta(name="target", path=target_path, type_f="local")
+                with open(os.path.join(self.output_dir, ".target"), "w") as trgt:
+                    trgt.write(target_path)
+                query_path = os.path.join(self.output_dir, "query.idx")
+                self.query = Fasta(name="query", path=query_path, type_f="local")
+                with open(os.path.join(self.output_dir, ".query"), "w") as qr:
+                    qr.write(query_path)
+                os.remove(self.backup.get_path())
+            return True
+        except:
+            traceback.print_exc()
+            return False
+
     def _after_start(self, success, error_set):
         with Job.connect():
             if success:
+                if self.backup is not None:
+                    success = self.unpack_backup()
+                    if not success:
+                        self.set_job_status("fail", "Backup file is not valid. If it is unattended, please contact the "
+                                                    "support.")
+                        if MODE == "webserver" and self.config.send_mail_status:
+                            self.send_mail()
+                        return False
                 status = "waiting"
                 if MODE == "webserver":
                     job = Job.get(Job.id_job == self.id_job)

src/dgenies/static/css/dgenies.css

@@ -416,6 +416,13 @@ form#export {
     right: 0;
 }
 
+.orsep {
+    font-size: 15pt;
+    margin-top: 8px;
+    margin-bottom: 8px;
+    text-align: center;
+}
+
 /* Tooltip*/
 
 span.tip {

src/dgenies/static/js/dgenies.result.export.js

@@ -136,6 +136,19 @@ dgenies.result.export.export_association_table = function () {
     document.getElementById('my-download').click();
 };
 
+dgenies.result.export.export_backup_file = function() {
+    dgenies.show_loading("Building file...", 180);
+    window.setTimeout(() => {
+        let export_div = $("div#export-pict");
+        export_div.html("");
+        export_div.append($("<a>").attr("href", `/backup/${dgenies.result.id_res}`)
+            .attr("download", dgenies.result.id_res + ".tar").attr("id", "my-download")
+            .text("download"));
+        dgenies.hide_loading();
+        document.getElementById('my-download').click();
+    });
+};
+
 dgenies.result.export.export_no_association_file = function (to) {
     window.setTimeout(() => {
         dgenies.show_loading("Building file...", 180);
@@ -230,6 +243,10 @@ dgenies.result.export.export = function () {
                     async = true;
                 }
             }
+            else if (selection === 9) {
+                dgenies.result.export.export_backup_file();
+                async = true;
+            }
             else
                 dgenies.notify("Not supported yet!", "danger", 2000);
             if (!async)

src/dgenies/static/js/dgenies.run.js

@@ -7,7 +7,15 @@ dgenies.run = {};
 dgenies.run.s_id = null;
 dgenies.run.allowed_ext = [];
 dgenies.run.max_upload_file_size = -1
-dgenies.run.files = [undefined, undefined, undefined, undefined, undefined];
+dgenies.run.files = [undefined, undefined, undefined, undefined, undefined, undefined];
+dgenies.run.files_nb = {
+    "query": 0,
+    "target": 1,
+    "queryidx": 2,
+    "targetidx": 3,
+    "alignfile": 4,
+    "backup": 5
+};
 dgenies.run.allow_upload = false;
 dgenies.run.ping_interval = null;
 dgenies.run.target_example = "";
@@ -29,7 +37,7 @@ dgenies.run.init = function (s_id, allowed_ext, max_upload_file_size=1073741824,
 };
 
 dgenies.run.restore_form = function () {
-    let ftypes = ["query", "target", "alignfile", "queryidx", "targetidx"];
+    let ftypes = ["query", "target", "alignfile", "queryidx", "targetidx", "backup"];
     for (let f in ftypes) {
         let ftype = ftypes[f];
         dgenies.run.change_fasta_type(ftype, $(`select.${ftype}`).find(":selected").text().toLowerCase(), true);
@@ -111,14 +119,15 @@ dgenies.run._init_fileupload = function(ftype, formats, position) {
 };
 
 dgenies.run.init_fileuploads = function () {
-    let ftypes = {"query": {"formats": ["fasta",], "position": 0},
-                  "target": {"formats": ["fasta",], "position": 1},
-                  "queryidx": {"formats": ["fasta", "idx"], "position": 2},
-                  "targetidx": {"formats": ["fasta", "idx"], "position": 3},
-                  "alignfile": {"formats": ["map"], "position": 4},};
+    let ftypes = {"query": {"formats": ["fasta",]},
+                  "target": {"formats": ["fasta",]},
+                  "queryidx": {"formats": ["fasta", "idx"]},
+                  "targetidx": {"formats": ["fasta", "idx"]},
+                  "alignfile": {"formats": ["map"]},
+                  "backup": {"formats": ["backup",]},};
     $.each(ftypes, function(ftype, data) {
         let formats = data["formats"];
-        let position = data["position"];
+        let position = dgenies.run.files_nb[ftype];
         dgenies.run._init_fileupload(ftype, formats, position);
         //Trigger events on hidden file inputs:
         $(`button#button-${ftype}`).click(function() {
@@ -158,6 +167,12 @@ dgenies.run._set_file_event = function(ftype) {
             if (this.files[0].size <= dgenies.run.max_upload_file_size) {
                 file_size.html(dgenies.run.get_file_size_str(this.files[0].size));
                 dgenies.run.set_filename(this.files[0].name, ftype);
+                if (["alignfile", "targetidx", "queryidx"].indexOf(ftype) > -1) {
+                    dgenies.run.reset_file_input("backup");
+                }
+                else if (ftype === "backup") {
+                    dgenies.run.reset_file_form("tab2", true);
+                }
             }
             else {
                 $(this).val("");
@@ -186,7 +201,7 @@ dgenies.run.show_tab = function(tab) {
 };
 
 dgenies.run.set_events = function() {
-    let ftypes = ["query", "target", "alignfile", "queryidx", "targetidx"];
+    let ftypes = ["query", "target", "alignfile", "queryidx", "targetidx", "backup"];
     $.each(ftypes, function (i, ftype) {
         dgenies.run._set_file_event(ftype);
         dgenies.run._set_file_select_event(ftype);
@@ -240,7 +255,7 @@ dgenies.run.enable_form = function () {
     $("input, select, button").prop("disabled", false);
     $("div#uploading-loading").hide();
     $("button#submit").show();
-    let ftypes = ["query", "target", "alignfile", "targetidx", "queryidx"];
+    let ftypes = ["query", "target", "alignfile", "targetidx", "queryidx", "backup"];
     for (let f in ftypes) {
         let ftype = ftypes[f];
         dgenies.run.hide_loading(ftype);
@@ -251,23 +266,26 @@ dgenies.run.enable_form = function () {
     dgenies.run.enabled = true;
 };
 
-dgenies.run.reset_file_form = function(tab) {
+dgenies.run.reset_file_input = function(inp_name) {
+    dgenies.run.change_fasta_type(inp_name, $(`select.${inp_name}`).find(":selected").text().toLowerCase(), true);
+    dgenies.run.files[dgenies.run.files_nb[inp_name]] = undefined;
+};
+
+dgenies.run.reset_file_form = function(tab, except_backup=false) {
     let ftypes = [];
     let i = 0;
     if (tab === "tab2") {
         ftypes = ["alignfile", "queryidx", "targetidx"];
-        i = 2;
+        if (!except_backup) {
+            ftypes.push("backup");
+        }
     }
     else {
         ftypes = ["query", "target"];
-        i = 0;
-    }
-    for (let f in ftypes) {
-        let ftype = ftypes[f];
-        dgenies.run.change_fasta_type(ftype, $(`select.${ftype}`).find(":selected").text().toLowerCase(), true);
-        dgenies.run.files[i] = undefined;
-        i++;
     }
+    $.each(ftypes, function(i, ftype) {
+        dgenies.run.reset_file_input(ftype);
+    });
 };
 
 dgenies.run.do_submit = function () {
@@ -294,6 +312,8 @@ dgenies.run.do_submit = function () {
             "query_type": $("select.queryidx").find(":selected").text().toLowerCase(),
             "target": $("input#targetidx").val(),
             "target_type": $("select.targetidx").find(":selected").text().toLowerCase(),
+            "backup": $("input#backup").val(),
+            "backup_type": $("select.backup").find(":selected").text().toLowerCase(),
         });
     }
     $("div#uploading-loading").html("Submitting form...");
@@ -372,15 +392,20 @@ dgenies.run.valid_form = function () {
 
     /* TAB 2 */
     else {
-        if ($("input#targetidx").val().length === 0) {
-            $("label.file-targetidx").addClass("error");
-            dgenies.run.add_error("Target file is required!");
-            has_errors = true;
+        if ($("input#backup").val().length !== 0) {
+            dgenies.run.reset_file_form("tab2", true);
         }
-        if ($("input#alignfile").val().length === 0) {
-            $("label.file-align").addClass("error");
-            dgenies.run.add_error("Alignment file is required!");
-            has_errors = true;
+        else {
+            if ($("input#targetidx").val().length === 0) {
+                $("label.file-targetidx").addClass("error");
+                dgenies.run.add_error("Target file is required!");
+                has_errors = true;
+            }
+            if ($("input#alignfile").val().length === 0) {
+                $("label.file-align").addClass("error");
+                dgenies.run.add_error("Alignment file is required!");
+                has_errors = true;
+            }
         }
     }
 
@@ -449,23 +474,6 @@ dgenies.run.check_url = function (url) {
         url.startsWith("example://");
 };
 
-dgenies.run.reset_other_tab = function(tab) {
-    if (tab === "tab1") {
-        $("input#alignfile").val("");
-        dgenies.run.files[2] = undefined;
-        $("input#targetidx").val("");
-        dgenies.run.files[3] = undefined;
-        $("input#queryidx").val("");
-        dgenies.run.files[4] = undefined;
-    }
-    else {
-        $("input#target").val("");
-        dgenies.run.files[0] = undefined;
-        $("input#query").val("");
-        dgenies.run.files[1] = undefined;
-    }
-};
-
 dgenies.run._start_upload = function(ftype, fname) {
     let has_uploads = false;
     let fasta_type = parseInt($(`select.${ftype}`).val());
@@ -496,13 +504,12 @@ dgenies.run.start_uploads = function() {
     }
     else {
         dgenies.run.reset_file_form("tab1");
-        inputs = [["queryidx", "Query"], ["targetidx", "Target"], ["alignfile", "Alignment"]]
+        inputs = [["queryidx", "Query"], ["targetidx", "Target"], ["alignfile", "Alignment"], ["backup", "Backup"]]
     }
-    for (let i in inputs) {
-        let input = inputs[i];
+    $.each(inputs, function(i, input) {
         let test_has_uploads = dgenies.run._start_upload(input[0], input[1]);
         has_uploads = has_uploads || test_has_uploads;
-    }
+    });
     if (has_uploads) {
         $("div#uploading-loading").html("Asking for upload...");
         dgenies.run.ask_for_upload();

src/dgenies/templates/result.html

@@ -55,6 +55,7 @@
                 <option value="5">Association table</option>
                 <option value="6">No match queries</option>
                 <option value="7">No match targets</option>
+                <option value="9">Backup file</option>
             </select>
         </form>
     </div>

src/dgenies/templates/run.html

@@ -225,6 +225,36 @@
                                     <div class="file-size queryidx"></div>
                                 </td>
                             </tr>
+                            <tr class="tabx tab2">
+                                <td colspan="2">
+                                    <h3 class="orsep">- or -</h3>
+                                </td>
+                            </tr>
+                            <tr class="backup tabx tab2">
+                                <td>
+                                    <label for="file_backup" class="file-backup">Backup file<sup><a href="#fn{{ fnote }}" id="ref{{ fnote }}">{{ fnote }}</a></sup>{% set fnote = fnote + 1 %}</label>
+                                </td>
+                                <td class="input-group">
+                                    <select class="select-type-input backup">
+                                        <option value="0">Local</option>
+                                        <option value="1">Url</option>
+                                    </select>
+
+                                    <div class="backup-label in-label">
+                                        <div id="progress-backup" class="progress">
+                                            <div class="bar" style="width: 0%;"></div>
+                                        </div>
+                                        <input id="backup" name="backup" class="form-control show-file" type="text" value="" readonly/>
+                                    </div>
+
+                                    <button class="btn btn-file" type="button" id="button-backup" title="Select a file">
+                                        <span class="glyphicon glyphicon-folder-open"></span>&nbsp
+                                    </button>
+                                    <div class="loading-file backup" style="display: none"></div>
+                                    <div class="upload-success backup" style="display: none"></div>
+                                    <div class="file-size backup"></div>
+                                </td>
+                            </tr>
                         </tbody>
                     </table>
                 </div>
@@ -252,6 +282,8 @@
                     {% set fnote_tab = fnote_tab + 1 %}
                     <p class="afterworlff"><sup id="fn{{ fnote_tab }}"><a href="#fn{{ fnote_tab }}" id="ref{{ fnote_tab }}">{{ fnote_tab }}</a></sup> Fasta file or index built by <a href="#">our tool</a>.</p>
                     {% set fnote_tab = fnote_tab + 1 %}
+                    <p class="afterworlff"><sup id="fn{{ fnote_tab }}"><a href="#fn{{ fnote_tab }}" id="ref{{ fnote_tab }}">{{ fnote_tab }}</a></sup> From a previous run.</p>
+                    {% set fnote_tab = fnote_tab + 1 %}
                 </div>
             </div>
         </div>
@@ -262,6 +294,7 @@
             <input type="file" name="file-alignfile" class="file-alignfile" data-url="/upload"/>
             <input type="file" name="file-targetidx" class="file-targetidx" data-url="/upload"/>
             <input type="file" name="file-queryidx" class="file-queryidx" data-url="/upload"/>
+            <input type="file" name="file-backup" class="file-backup" data-url="/upload"/>
         </div>
     </form>
 {% endblock %}
\ No newline at end of file

src/dgenies/views.py

@@ -17,6 +17,7 @@ from dgenies.lib.fasta import Fasta
 from dgenies.tools import Tools
 from markdown import Markdown
 from markdown.extensions.toc import TocExtension
+import tarfile
 if MODE == "webserver":
     from dgenies.database import Session, Gallery
     from peewee import DoesNotExist
@@ -114,6 +115,8 @@ def launch_analysis():
     tool = request.form["tool"] if "tool" in request.form else None
     alignfile = request.form["alignfile"] if "alignfile" in request.form else None
     alignfile_type = request.form["alignfile_type"] if "alignfile_type" in request.form else None
+    backup = request.form["backup"] if "backup" in request.form else None
+    backup_type = request.form["backup_type"] if "backup_type" in request.form else None
 
     # Check form:
     form_pass = True
@@ -123,6 +126,20 @@ def launch_analysis():
         errors.append("Server error: no alignfile_type in form. Please contact the support")
         form_pass = False
 
+    if backup is not None and backup_type is None:
+        errors.append("Server error: no backup_type in form. Please contact the support")
+        form_pass = False
+
+    if backup is not None:
+        alignfile = ""
+        file_query = ""
+        file_target = ""
+
+    else:
+        if file_target == "":
+            errors.append("No target fasta selected")
+            form_pass = False
+
     if tool is not None and tool not in Tools().tools:
         errors.append("Tool unavailable: %s" % tool)
         form_pass = False
@@ -138,9 +155,6 @@ def launch_analysis():
         elif not re.match(r"^[\w.\-]+@[\w\-.]{2,}\.[a-z]{2,4}$", email):
             errors.append("Email is invalid")
             form_pass = False
-    if file_target == "":
-        errors.append("No target fasta selected")
-        form_pass = False
 
     # Form pass
     if form_pass:
@@ -173,23 +187,27 @@ def launch_analysis():
                     form_pass = False
             query = Fasta(name=query_name, path=query_path, type_f=file_query_type, example=example)
         example = False
-        if file_target.startswith("example://"):
-            example = True
-            target_path = config_reader.example_target
-            target_name = os.path.basename(target_path)
-            file_target_type = "local"
-        else:
-            target_name = os.path.splitext(file_target.replace(".gz", ""))[0] if file_target_type == "local" else None
-            target_path = os.path.join(app.config["UPLOAD_FOLDER"], upload_folder, file_target) \
-                if file_target_type == "local" else file_target
-            if file_target_type == "local" and not os.path.exists(target_path):
-                errors.append("Target file not correct!")
-                form_pass = False
-        target = Fasta(name=target_name, path=target_path, type_f=file_target_type, example=example)
+        target = None
+        if file_target != "":
+            if file_target.startswith("example://"):
+                example = True
+                target_path = config_reader.example_target
+                target_name = os.path.basename(target_path)
+                file_target_type = "local"
+            else:
+                target_name = os.path.splitext(file_target.replace(".gz", ""))[0] if file_target_type == "local" else None
+                target_path = os.path.join(app.config["UPLOAD_FOLDER"], upload_folder, file_target) \
+                    if file_target_type == "local" else file_target
+                if file_target_type == "local" and not os.path.exists(target_path):
+                    errors.append("Target file not correct!")
+                    form_pass = False
+            target = Fasta(name=target_name, path=target_path, type_f=file_target_type, example=example)
 
-        align = None
-        if alignfile is not None:
+        if alignfile is not None and alignfile != "" and backup is not None:
             Path(os.path.join(folder_files, ".align")).touch()
+
+        align = None
+        if alignfile is not None and alignfile != "":
             alignfile_name = os.path.splitext(alignfile)[0] if alignfile_type == "local" else None
             alignfile_path = os.path.join(app.config["UPLOAD_FOLDER"], upload_folder, alignfile) \
                 if alignfile_type == "local" else alignfile
@@ -198,6 +216,16 @@ def launch_analysis():
                 form_pass = False
             align = Fasta(name=alignfile_name, path=alignfile_path, type_f=alignfile_type)
 
+        bckp = None
+        if backup is not None:
+            backup_name = os.path.splitext(backup)[0] if backup_type == "local" else None
+            backup_path = os.path.join(app.config["UPLOAD_FOLDER"], upload_folder, backup) \
+                if backup_type == "local" else backup
+            if backup_type == "local" and not os.path.exists(backup_path):
+                errors.append("Backup file not correct!")
+                form_pass = False
+            bckp = Fasta(name=backup_name, path=backup_path, type_f=backup_type)
+
         if form_pass:
             # Launch job:
             job = JobManager(id_job=id_job,
@@ -205,6 +233,7 @@ def launch_analysis():
                              query=query,
                              target=target,
                              align=align,
+                             backup=bckp,
                              mailer=mailer,
                              tool=tool)
             if MODE == "webserver":
@@ -638,6 +667,16 @@ def summary(id_res):
     })
 
 
+@app.route('/backup/<id_res>')
+def get_backup_file(id_res):
+    res_dir = os.path.join(APP_DATA, id_res)
+    tar = os.path.join(res_dir, "%s.tar" % id_res)
+    with tarfile.open(tar, "w") as tarf:
+        for file in ("map.paf", "target.idx", "query.idx"):
+            tarf.add(os.path.join(res_dir, file), arcname=file)
+    return send_file(tar)
+
+
 def get_filter_out(id_res, type_f):
     filter_file = os.path.join(APP_DATA, id_res, ".filter-" + type_f)
     return Response(get_file(filter_file), mimetype="text/plain")