Skip to content
GitLab
Commit 00f1cfcf authored by Floreal Cabanettes's avatar Floreal Cabanettes
Browse files

Add export and import as tar file 

Closes #128

See merge request !4
parents 94fcf64a 0ec9ff23
Expand all Hide whitespace changes
Inline Side-by-side
src/dgenies/lib/functions.py
View file @ 00f1cfcf
......@@ -14,8 +14,9 @@ from dgenies.config_reader import AppConfigReader
import dgenies.lib.validators as validators
ALLOWED_EXTENSIONS = {"fasta": ['fa', 'fasta', 'fna', 'fa.gz', 'fasta.gz', 'fna.gz'],
"idx": ['idx'],
"map": [o[0] for o in getmembers(validators) if isfunction(o[1]) and not o[0].startswith("_")]}
"idx": ['idx',],
"map": [o[0] for o in getmembers(validators) if isfunction(o[1]) and not o[0].startswith("_")],
"backup": ['tar']}
# map: all functions of validators which does not starts with an underscore.
......
src/dgenies/lib/job_manager.py
View file @ 00f1cfcf
......@@ -21,6 +21,7 @@ from jinja2 import Template
import traceback
from pathlib import Path
from urllib import request, parse
import tarfile
from dgenies.bin.split_fa import Splitter
from dgenies.bin.index import index_file, Index
from dgenies.bin.filter_contigs import Filter
......@@ -40,7 +41,7 @@ if MODE == "webserver":
class JobManager:
def __init__(self, id_job: str, email: str=None, query: Fasta=None, target: Fasta=None, mailer=None,
tool="minimap2", align: Fasta=None):
tool="minimap2", align: Fasta=None, backup: Fasta=None):
self.id_job = id_job
self.email = email
self.query = query
......@@ -48,6 +49,7 @@ class JobManager:
self.align = align
if align is not None:
self.aln_format = os.path.splitext(align.get_path())[1][1:]
self.backup = backup
self.error = ""
self.id_process = "-1"
# Get configs:
......@@ -99,7 +101,8 @@ class JobManager:
with open(query_file) as q_f:
file_path = q_f.readline()
self.query = Fasta(
name=os.path.splitext(os.path.basename(file_path.replace(".gz", "")).split("_", 1)[1])[0],
name="target" if file_path.endswith(".idx") else
os.path.splitext(os.path.basename(file_path.replace(".gz", "")).split("_", 1)[1])[0],
path=file_path,
type_f="local"
)
......@@ -108,7 +111,8 @@ class JobManager:
with open(target_file) as t_f:
file_path = t_f.readline()
self.target = Fasta(
name=os.path.splitext(os.path.basename(file_path.replace(".gz", "")).split("_", 1)[1])[0],
name="query" if file_path.endswith(".idx") else
os.path.splitext(os.path.basename(file_path.replace(".gz", "")).split("_", 1)[1])[0],
path=file_path,
type_f="local"
)
......@@ -151,11 +155,12 @@ class JobManager:
else:
message += "Your job %s has failed. You can try again. " \
"If the problem persists, please contact the support.\n\n" % self.id_job
message += "Sequences compared in this analysis:\n"
if query_name is not None:
message += "Target: %s\nQuery: %s\n\n" % (target_name, query_name)
else:
message += "Target: %s\n\n" % target_name
if target_name is not None:
message += "Sequences compared in this analysis:\n"
if query_name is not None:
message += "Target: %s\nQuery: %s\n\n" % (target_name, query_name)
else:
message += "Target: %s\n\n" % target_name
if status == "success":
if self.is_target_filtered():
message += str("Note: target fasta has been filtered because it contains too small contigs."
......@@ -175,7 +180,7 @@ class JobManager:
template = Template(t_file.read())
return template.render(job_name=self.id_job, status=status, url_base=self.config.web_url,
query_name=query_name if query_name is not None else "",
target_name=target_name,
target_name=target_name if target_name is not None else "",
error=self.error,
target_filtered=self.is_target_filtered(), query_filtered=self.is_query_filtered())
......@@ -194,12 +199,16 @@ class JobManager:
status = job.status
self.error = job.error
with open(self.idx_t, "r") as idxt:
target_name = idxt.readline().rstrip()
with open(self.idx_q, "r") as idxq:
query_name = idxq.readline().rstrip()
if query_name == target_name:
query_name = None
target_name = None
if os.path.exists(self.idx_t):
with open(self.idx_t, "r") as idxt:
target_name = idxt.readline().rstrip()
query_name = None
if os.path.exists(self.idx_q):
with open(self.idx_q, "r") as idxq:
query_name = idxq.readline().rstrip()
if query_name == target_name:
query_name = None
# Send:
self.mailer.send_mail(recipients=[self.email],
......@@ -499,7 +508,7 @@ class JobManager:
save_file.write(finale_path)
return True, False, finale_path, name
def __check_url(self, fasta: Fasta):
def __check_url(self, fasta: Fasta, formats: tuple):
url = fasta.get_path()
try:
filename = self.__get_filename_from_url(url)
......@@ -517,11 +526,28 @@ class JobManager:
self.set_status_standalone(status, error)
return False
if filename is not None:
allowed = Functions.allowed_file(filename)
allowed = Functions.allowed_file(filename, formats)
if not allowed:
status = "fail"
error = "<p>File <b>%s</b> downloaded from <b>%s</b> is not a Fasta file!</p>" \
"<p>If this is unattended, please contact the support.</p>" % (filename, url)
format_txt = ""
if len(formats) == 1:
if formats[0] == "fasta":
format_txt = "a Fasta file"
elif formats[0] == "idx":
format_txt = "an index file"
elif formats[0] == "map":
format_txt = "an alignment file"
elif formats[0] == "backup":
format_txt = "a backup file"
else:
format_txt = "a valid file"
else:
if "fasta" in formats and "idx" in formats:
format_txt = "a Fasta file or an index file"
else:
format_txt = "a valid file"
error = "<p>File <b>%s</b> downloaded from <b>%s</b> is not %s!</p>" \
"<p>If this is unattended, please contact the support.</p>" % (filename, url, format_txt)
if MODE == "webserver":
with Job.connect():
job = Job.get(Job.id_job == self.id_job)
......@@ -600,7 +626,7 @@ class JobManager:
if not getattr(validators, self.aln_format)(self.align.get_path()):
self.set_job_status("fail", "Alignment file is invalid. Please check your file.")
return False, True, None
else:
elif input_type != "backup":
if self.config.batch_system_type != "local" and file_size >= getattr(self.config,
"min_%s_size" % input_type):
should_be_local = False
......@@ -692,7 +718,7 @@ class JobManager:
max_upload_size_readable)
if not correct:
return False, error_set, True
elif self.__check_url(self.query):
elif self.__check_url(self.query, ("fasta",) if self.align is None else ("fasta", "idx")):
files_to_download.append([self.query, "query"])
else:
return False, True, True
......@@ -703,7 +729,7 @@ class JobManager:
max_upload_size_readable)
if not correct:
return False, error_set, True
elif self.__check_url(self.target):
elif self.__check_url(self.target, ("fasta",) if self.align is None else ("fasta", "idx")):
files_to_download.append([self.target, "target"])
else:
return False, True, True
......@@ -712,10 +738,19 @@ class JobManager:
self.align.set_path(self.__getting_local_file(self.align, "align"))
correct, error_set, should_be_local = self.check_file("align", should_be_local,
max_upload_size_readable)
elif self.__check_url(self.align):
elif self.__check_url(self.align, ("map",)):
files_to_download.append([self.align, "align"])
else:
return False, True, True
if correct and self.backup is not None:
if self.backup.get_type() == "local":
self.backup.set_path(self.__getting_local_file(self.backup, "backup"))
correct, error_set, should_be_local = self.check_file("backup", should_be_local,
max_upload_size_readable)
elif self.__check_url(self.backup, ("backup",)):
files_to_download.append([self.backup, "backup"])
else:
return False, True, True
all_downloaded = True
if correct :
......@@ -870,7 +905,8 @@ class JobManager:
sorter = Sorter(self.paf_raw, self.paf)
sorter.sort()
os.remove(self.paf_raw)
if self.target is not None and os.path.exists(self.target.get_path()):
if self.target is not None and os.path.exists(self.target.get_path()) and not \
self.target.get_path().endswith(".idx"):
os.remove(self.target.get_path())
self.align.set_path(self.paf)
......@@ -1076,9 +1112,48 @@ class JobManager:
batch_type=job.batch_type)
log.save()
def unpack_backup(self):
try:
with tarfile.open(self.backup.get_path(), "r") as tar:
names = tar.getnames()
if len(names) != 3:
return False
for name in ["map.paf", "query.idx", "target.idx"]:
if name not in names:
return False
tar.extractall(self.output_dir)
shutil.move(self.paf, self.paf_raw)
if not validators.paf(self.paf_raw):
return False
self.align = Fasta(name="map", path=self.paf_raw, type_f="local")
self.aln_format = "paf"
with open(os.path.join(self.output_dir, ".align"), "w") as aln:
aln.write(self.paf_raw)
target_path = os.path.join(self.output_dir, "target.idx")
self.target = Fasta(name="target", path=target_path, type_f="local")
with open(os.path.join(self.output_dir, ".target"), "w") as trgt:
trgt.write(target_path)
query_path = os.path.join(self.output_dir, "query.idx")
self.query = Fasta(name="query", path=query_path, type_f="local")
with open(os.path.join(self.output_dir, ".query"), "w") as qr:
qr.write(query_path)
os.remove(self.backup.get_path())
return True
except:
traceback.print_exc()
return False
def _after_start(self, success, error_set):
with Job.connect():
if success:
if self.backup is not None:
success = self.unpack_backup()
if not success:
self.set_job_status("fail", "Backup file is not valid. If it is unattended, please contact the "
"support.")
if MODE == "webserver" and self.config.send_mail_status:
self.send_mail()
return False
status = "waiting"
if MODE == "webserver":
job = Job.get(Job.id_job == self.id_job)
......@@ -1137,7 +1212,7 @@ class JobManager:
print("Old job found without result dir existing: delete it from BDD!")
for j11 in j1:
j11.delete_instance()
if self.target is not None:
if self.target is not None or self.backup is not None:
job = Job.create(id_job=self.id_job, email=self.email, batch_type=self.config.batch_system_type,
date_created=datetime.now(), tool=self.tool.name if self.tool is not None else None)
job.save()
......
src/dgenies/lib/mail_templates/job_notification.html
View file @ 00f1cfcf
......@@ -43,11 +43,13 @@
{% endif %}
{% endif %}
<p>Sequences compared in this analysis:</p>
<p><em>Target:</em> {{ target_name }}<br/>
{% if query_name != "" %}
<em>Query:</em> {{ query_name }}
</p>
{% if target_name != "" %}
<p>Sequences compared in this analysis:</p>
<p><em>Target:</em> {{ target_name }}
{% if query_name != "" %}
<br/><em>Query:</em> {{ query_name }}
{% endif %}
</p>
{% endif %}
{% if status == "success" %}
......
src/dgenies/static/css/dgenies.css
View file @ 00f1cfcf
......@@ -416,6 +416,13 @@ form#export {
right: 0;
}
.orsep {
font-size: 15pt;
margin-top: 8px;
margin-bottom: 8px;
text-align: center;
}
/* Tooltip*/
span.tip {
......
src/dgenies/static/css/dgenies.min.css
View file @ 00f1cfcf
This diff is collapsed.
src/dgenies/static/js/dgenies.result.export.js
View file @ 00f1cfcf
......@@ -136,6 +136,19 @@ dgenies.result.export.export_association_table = function () {
document.getElementById('my-download').click();
};
dgenies.result.export.export_backup_file = function() {
dgenies.show_loading("Building file...", 180);
window.setTimeout(() => {
let export_div = $("div#export-pict");
export_div.html("");
export_div.append($("<a>").attr("href", `/backup/${dgenies.result.id_res}`)
.attr("download", dgenies.result.id_res + ".tar").attr("id", "my-download")
.text("download"));
dgenies.hide_loading();
document.getElementById('my-download').click();
});
};
dgenies.result.export.export_no_association_file = function (to) {
window.setTimeout(() => {
dgenies.show_loading("Building file...", 180);
......@@ -230,6 +243,10 @@ dgenies.result.export.export = function () {
async = true;
}
}
else if (selection === 9) {
dgenies.result.export.export_backup_file();
async = true;
}
else
dgenies.notify("Not supported yet!", "danger", 2000);
if (!async)
......
src/dgenies/static/js/dgenies.result.min.js
View file @ 00f1cfcf
This diff is collapsed.
src/dgenies/static/js/dgenies.run.js
View file @ 00f1cfcf
......@@ -7,7 +7,15 @@ dgenies.run = {};
dgenies.run.s_id = null;
dgenies.run.allowed_ext = [];
dgenies.run.max_upload_file_size = -1
dgenies.run.files = [undefined, undefined, undefined, undefined, undefined];
dgenies.run.files = [undefined, undefined, undefined, undefined, undefined, undefined];
dgenies.run.files_nb = {
"query": 0,
"target": 1,
"queryidx": 2,
"targetidx": 3,
"alignfile": 4,
"backup": 5
};
dgenies.run.allow_upload = false;
dgenies.run.ping_interval = null;
dgenies.run.target_example = "";
......@@ -29,7 +37,7 @@ dgenies.run.init = function (s_id, allowed_ext, max_upload_file_size=1073741824,
};
dgenies.run.restore_form = function () {
let ftypes = ["query", "target", "alignfile", "queryidx", "targetidx"];
let ftypes = ["query", "target", "alignfile", "queryidx", "targetidx", "backup"];
for (let f in ftypes) {
let ftype = ftypes[f];
dgenies.run.change_fasta_type(ftype, $(`select.${ftype}`).find(":selected").text().toLowerCase(), true);
......@@ -111,14 +119,15 @@ dgenies.run._init_fileupload = function(ftype, formats, position) {
};
dgenies.run.init_fileuploads = function () {
let ftypes = {"query": {"formats": ["fasta",], "position": 0},
"target": {"formats": ["fasta",], "position": 1},
"queryidx": {"formats": ["fasta", "idx"], "position": 2},
"targetidx": {"formats": ["fasta", "idx"], "position": 3},
"alignfile": {"formats": ["map"], "position": 4},};
let ftypes = {"query": {"formats": ["fasta",]},
"target": {"formats": ["fasta",]},
"queryidx": {"formats": ["fasta", "idx"]},
"targetidx": {"formats": ["fasta", "idx"]},
"alignfile": {"formats": ["map"]},
"backup": {"formats": ["backup",]},};
$.each(ftypes, function(ftype, data) {
let formats = data["formats"];
let position = data["position"];
let position = dgenies.run.files_nb[ftype];
dgenies.run._init_fileupload(ftype, formats, position);
//Trigger events on hidden file inputs:
$(`button#button-${ftype}`).click(function() {
......@@ -158,6 +167,12 @@ dgenies.run._set_file_event = function(ftype) {
if (this.files[0].size <= dgenies.run.max_upload_file_size) {
file_size.html(dgenies.run.get_file_size_str(this.files[0].size));
dgenies.run.set_filename(this.files[0].name, ftype);
if (["alignfile", "targetidx", "queryidx"].indexOf(ftype) > -1) {
dgenies.run.reset_file_input("backup");
}
else if (ftype === "backup") {
dgenies.run.reset_file_form("tab2", true);
}
}
else {
$(this).val("");
......@@ -186,7 +201,7 @@ dgenies.run.show_tab = function(tab) {
};
dgenies.run.set_events = function() {
let ftypes = ["query", "target", "alignfile", "queryidx", "targetidx"];
let ftypes = ["query", "target", "alignfile", "queryidx", "targetidx", "backup"];
$.each(ftypes, function (i, ftype) {
dgenies.run._set_file_event(ftype);
dgenies.run._set_file_select_event(ftype);
......@@ -240,7 +255,7 @@ dgenies.run.enable_form = function () {
$("input, select, button").prop("disabled", false);
$("div#uploading-loading").hide();
$("button#submit").show();
let ftypes = ["query", "target", "alignfile", "targetidx", "queryidx"];
let ftypes = ["query", "target", "alignfile", "targetidx", "queryidx", "backup"];
for (let f in ftypes) {
let ftype = ftypes[f];
dgenies.run.hide_loading(ftype);
......@@ -251,23 +266,26 @@ dgenies.run.enable_form = function () {
dgenies.run.enabled = true;
};
dgenies.run.reset_file_form = function(tab) {
dgenies.run.reset_file_input = function(inp_name) {
dgenies.run.change_fasta_type(inp_name, $(`select.${inp_name}`).find(":selected").text().toLowerCase(), true);
dgenies.run.files[dgenies.run.files_nb[inp_name]] = undefined;
};
dgenies.run.reset_file_form = function(tab, except_backup=false) {
let ftypes = [];
let i = 0;
if (tab === "tab2") {
ftypes = ["alignfile", "queryidx", "targetidx"];
i = 2;
if (!except_backup) {
ftypes.push("backup");
}
}
else {
ftypes = ["query", "target"];
i = 0;
}
for (let f in ftypes) {
let ftype = ftypes[f];
dgenies.run.change_fasta_type(ftype, $(`select.${ftype}`).find(":selected").text().toLowerCase(), true);
dgenies.run.files[i] = undefined;
i++;
}
$.each(ftypes, function(i, ftype) {
dgenies.run.reset_file_input(ftype);
});
};
dgenies.run.do_submit = function () {
......@@ -294,6 +312,8 @@ dgenies.run.do_submit = function () {
"query_type": $("select.queryidx").find(":selected").text().toLowerCase(),
"target": $("input#targetidx").val(),
"target_type": $("select.targetidx").find(":selected").text().toLowerCase(),
"backup": $("input#backup").val(),
"backup_type": $("select.backup").find(":selected").text().toLowerCase(),
});
}
$("div#uploading-loading").html("Submitting form...");
......@@ -372,15 +392,20 @@ dgenies.run.valid_form = function () {
/* TAB 2 */
else {
if ($("input#targetidx").val().length === 0) {
$("label.file-targetidx").addClass("error");
dgenies.run.add_error("Target file is required!");
has_errors = true;
if ($("input#backup").val().length !== 0) {
dgenies.run.reset_file_form("tab2", true);
}
if ($("input#alignfile").val().length === 0) {
$("label.file-align").addClass("error");
dgenies.run.add_error("Alignment file is required!");
has_errors = true;
else {
if ($("input#targetidx").val().length === 0) {
$("label.file-targetidx").addClass("error");
dgenies.run.add_error("Target file is required!");
has_errors = true;
}
if ($("input#alignfile").val().length === 0) {
$("label.file-align").addClass("error");
dgenies.run.add_error("Alignment file is required!");
has_errors = true;
}
}
}
......@@ -449,23 +474,6 @@ dgenies.run.check_url = function (url) {
url.startsWith("example://");
};
dgenies.run.reset_other_tab = function(tab) {
if (tab === "tab1") {
$("input#alignfile").val("");
dgenies.run.files[2] = undefined;
$("input#targetidx").val("");
dgenies.run.files[3] = undefined;
$("input#queryidx").val("");
dgenies.run.files[4] = undefined;
}
else {
$("input#target").val("");
dgenies.run.files[0] = undefined;
$("input#query").val("");
dgenies.run.files[1] = undefined;
}
};
dgenies.run._start_upload = function(ftype, fname) {
let has_uploads = false;
let fasta_type = parseInt($(`select.${ftype}`).val());
......@@ -496,13 +504,12 @@ dgenies.run.start_uploads = function() {
}
else {
dgenies.run.reset_file_form("tab1");
inputs = [["queryidx", "Query"], ["targetidx", "Target"], ["alignfile", "Alignment"]]
inputs = [["queryidx", "Query"], ["targetidx", "Target"], ["alignfile", "Alignment"], ["backup", "Backup"]]
}
for (let i in inputs) {
let input = inputs[i];
$.each(inputs, function(i, input) {
let test_has_uploads = dgenies.run._start_upload(input[0], input[1]);
has_uploads = has_uploads || test_has_uploads;
}
});
if (has_uploads) {
$("div#uploading-loading").html("Asking for upload...");
dgenies.run.ask_for_upload();
......
src/dgenies/static/js/dgenies.run.min.js
View file @ 00f1cfcf
This diff is collapsed.
src/dgenies/templates/result.html
View file @ 00f1cfcf
......@@ -55,6 +55,7 @@
<option value="5">Association table</option>
<option value="6">No match queries</option>
<option value="7">No match targets</option>
<option value="9">Backup file</option>
</select>
</form>
</div>
......
src/dgenies/templates/run.html
View file @ 00f1cfcf
......@@ -225,6 +225,36 @@
<div class="file-size queryidx"></div>
</td>
</tr>
<tr class="tabx tab2">
<td colspan="2">
<h3 class="orsep">- or -</h3>
</td>
</tr>
<tr class="backup tabx tab2">
<td>
<label for="file_backup" class="file-backup">Backup file<sup><a href="#fn{{ fnote }}" id="ref{{ fnote }}">{{ fnote }}</a></sup>{% set fnote = fnote + 1 %}</label>
</td>
<td class="input-group">
<select class="select-type-input backup">
<option value="0">Local</option>
<option value="1">Url</option>
</select>
<div class="backup-label in-label">
<div id="progress-backup" class="progress">
<div class="bar" style="width: 0%;"></div>
</div>
<input id="backup" name="backup" class="form-control show-file" type="text" value="" readonly/>
</div>
<button class="btn btn-file" type="button" id="button-backup" title="Select a file">
<span class="glyphicon glyphicon-folder-open"></span>&nbsp
</button>
<div class="loading-file backup" style="display: none"></div>
<div class="upload-success backup" style="display: none"></div>
<div class="file-size backup"></div>
</td>
</tr>
</tbody>
</table>
</div>
......@@ -252,6 +282,8 @@
{% set fnote_tab = fnote_tab + 1 %}
<p class="afterworlff"><sup id="fn{{ fnote_tab }}"><a href="#fn{{ fnote_tab }}" id="ref{{ fnote_tab }}">{{ fnote_tab }}</a></sup> Fasta file or index built by <a href="https://raw.githubusercontent.com/genotoul-bioinfo/dgenies/v{{version}}/src/dgenies/bin/index.py" download="index.py">our tool</a>.</p>
{% set fnote_tab = fnote_tab + 1 %}
<p class="afterworlff"><sup id="fn{{ fnote_tab }}"><a href="#fn{{ fnote_tab }}" id="ref{{ fnote_tab }}">{{ fnote_tab }}</a></sup> From a previous run.</p>
{% set fnote_tab = fnote_tab + 1 %}
</div>
</div>
</div>
......@@ -262,6 +294,7 @@
<input type="file" name="file-alignfile" class="file-alignfile" data-url="/upload"/>
<input type="file" name="file-targetidx" class="file-targetidx" data-url="/upload"/>
<input type="file" name="file-queryidx" class="file-queryidx" data-url="/upload"/>
<input type="file" name="file-backup" class="file-backup" data-url=