Commit a1a6968e authored by Floréal Cabanettes's avatar Floréal Cabanettes
Browse files

Finish implementation of ajax uploading, Implements #23

parent 0c8d0b5a
<?xml version="1.0" ?><svg id="Layer_1" style="enable-background:new 0 0 512 512;" version="1.1" viewBox="0 0 512 512" xml:space="preserve" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink"><style type="text/css">
.st0{fill:#41AD49;}
</style><g><polygon class="st0" points="434.8,49 174.2,309.7 76.8,212.3 0,289.2 174.1,463.3 196.6,440.9 196.6,440.9 511.7,125.8 434.8,49 "/></g></svg>
\ No newline at end of file
......@@ -397,6 +397,8 @@ div.navbar {
background-image: linear-gradient(to bottom,#2B3044 0, #141a30 100%);
}
/* Run form: */
.input-group {
display: inline-block;
}
......@@ -407,6 +409,27 @@ div.navbar {
float: none !important;
}
.input-group button {
padding-top: 0;
}
div.input-group {
padding-left: 15px !important;
}
.upload-btn-wrapper {
position: relative;
height: 30px;
width: 42px;
overflow: hidden;
display: inline-block;
}
button.btn-file {
height: 30px;
width: 42px;
}
div.in-label {
position: relative;
display: inline-block;
......@@ -415,19 +438,19 @@ div.in-label {
}
div.progress {
display: none;
width: 300px;
height: 30px;
position: absolute;
top: 0;
left: 0;
z-index: 10;
opacity: 0.5;
z-index: 5;
background: white;
}
div.progress .bar {
height: 100%;
background: green;
background: #33b234;
z-index: 5;
}
select.select-type-input, .input-group button, .input-group input {
......@@ -435,49 +458,49 @@ select.select-type-input, .input-group button, .input-group input {
line-height: 30px;
}
.input-group button {
padding-top: 0;
input.show-file {
padding-left: 2px;
padding-right: 2px;
width: auto;
z-index: 10 !important;
background: none !important;
color: black;
}
div.input-group {
padding-left: 15px !important;
div.hidden-input-files {
display: none;
}
.upload-btn-wrapper {
position: relative;
height: 30px;
width: 42px;
overflow: hidden;
display: inline-block;
label.error {
color: red;
}
.upload-btn-wrapper button {
position: absolute;
top: 0;
left: 0;
div.loading-file {
display: inline-block;
vertical-align: middle;
width: 30px;
height: 30px;
width: 42px;
background: url("images/loading1.gif") no-repeat 0 0;
background-size: 30px 30px;
}
/*.btn {*/
/*border: 2px solid gray;*/
/*color: black;*/
/*background-color: white;*/
/*border-radius: 8px;*/
/*font-size: 20px;*/
/*font-weight: bold;*/
/*}*/
.upload-btn-wrapper input[type=file] {
font-size: 100px;
position: absolute;
left: 0;
top: 0;
opacity: 0;
div.upload-success {
display: inline-block;
vertical-align: middle;
width: 30px;
height: 30px;
background: url("images/success.svg") no-repeat 0 0;
background-size: 30px 30px;
}
input.show-file {
padding-left: 2px;
padding-right: 2px;
width: auto;
div#uploading-loading {
height: 60px;
line-height: 30px;
width: 200px;
font-size: 14pt;
font-weight: bold;
text-align: center;
color: blue;
background: url("images/loading-cascade.gif") no-repeat 0 25px;
background-size: 200px 30px;
}
\ No newline at end of file
......@@ -4,20 +4,39 @@ if (!dgenies) {
dgenies.run = {};
// Init global variables:
dgenies.run.files = [];
dgenies.run.upload_folder = [];
dgenies.run.files = [undefined, undefined];
dgenies.run.init = function () {
dgenies.run.restore_form();
dgenies.run.set_events();
dgenies.run.init_fileuploads();
};
dgenies.run.restore_form = function () {
if ($("select.query").val() === "0")
$("input#query").val("");
if ($("select.target").val() === "0")
$("input#target").val("");
};
dgenies.run.upload_next = function () {
let next = dgenies.run.files.shift();
while (next === undefined && dgenies.run.files.length > 0) {
next = dgenies.run.files.shift();
}
if (next !== undefined) {
next.submit();
return true;
}
dgenies.run.do_submit();
return false;
};
dgenies.run.init_fileuploads = function () {
$('input.file-query').fileupload({
dataType: 'json',
formData: {folder: dgenies.run.upload_folder},
add: function (e, data) {
dgenies.run.files.push(data);
dgenies.run.files[0] = data;
},
progressall: function (e, data) {
var progress = parseInt(data.loaded / data.total * 100, 10);
......@@ -27,19 +46,17 @@ dgenies.run.init_fileuploads = function () {
);
},
success: function (data, success) {
console.log(data);
dgenies.run.upload_folder[0] = data["folder"];
console.log(dgenies.run.upload_folder);
if (dgenies.run.files.length > 0) {
dgenies.run.files.shift().submit();
}
$("input#query").val(data["files"][0]["name"]);
dgenies.run.hide_loading("query");
dgenies.run.show_success("query");
dgenies.run.upload_next();
}
});
$('input.file-target').fileupload({
dataType: 'json',
formData: {folder: dgenies.run.upload_folder},
add: function (e, data) {
dgenies.run.files.push(data);
dgenies.run.files[1] = data;
},
progressall: function (e, data) {
var progress = parseInt(data.loaded / data.total * 100, 10);
......@@ -49,12 +66,20 @@ dgenies.run.init_fileuploads = function () {
);
},
success: function (data, success) {
dgenies.run.upload_folder[0] = data["folder"];
if (dgenies.run.files.length > 0) {
dgenies.run.files.shift().submit();
}
$("input#target").val(data["files"][0]["name"]);
dgenies.run.hide_loading("target");
dgenies.run.show_success("target");
dgenies.run.upload_next();
}
});
//Trigger events on hidden file inputs:
$("button#button-query").click(function() {
$("input.file-query").trigger("click");
})
$("button#button-target").click(function() {
$("input.file-target").trigger("click");
})
}
dgenies.run.set_events = function() {
......@@ -79,12 +104,121 @@ dgenies.run.set_filename = function (name, fasta) {
$("input#" + fasta).val(name);
};
dgenies.run.submit = function () {
console.log("submit!");
$("div#progress-query").show();
$("div#progress-target").show();
dgenies.run.disable_form = function () {
$("input, select, button").prop("disabled", true);
};
dgenies.run.enable_form = function () {
$("input, select, button").prop("disabled", false);
};
dgenies.run.do_submit = function () {
$("div#uploading-loading").html("Submitting form...");
$.post("/launch_analysis",
{
"id_job": $("input#id_job").val(),
"email": $("input#email").val(),
"query": $("input#query").val(),
"query_type": $("select.query").find(":selected").text().toLowerCase(),
"target": $("input#target").val(),
"target_type": $("select.target").find(":selected").text().toLowerCase()
},
function (data, status) {
if (data["success"]) {
window.location = data["redirect"];
}
}
);
};
dgenies.run.add_error = function (error) {
$("div.errors-submit ul.flashes").append($("<li>").append(error));
};
dgenies.run.valid_form = function () {
let has_errors = false;
// Check name:
if ($("input#id_job").val().length === 0) {
$("label.id-job").addClass("error");
dgenies.run.add_error("Name of your job is required!");
has_errors = true;
}
// Check mail:
let email = $("input#email").val();
let mail_re = /^.+@.+\..+$/;
if (email.match(mail_re) === null) {
$("label.email").addClass("error");
if (email === "")
dgenies.run.add_error("Email is required!");
else
dgenies.run.add_error("Email is not correct!");
has_errors = true;
}
//Check input query:
if ($("input#query").val().length === 0) {
$("label.file-query").addClass("error");
dgenies.run.add_error("Query fasta is required!");
has_errors = true;
}
if (dgenies.run.files.length > 0) {
dgenies.run.files.shift().submit();
// Returns
return !has_errors;
};
dgenies.run.show_loading = function(fasta) {
$(".loading-file." + fasta).show();
};
dgenies.run.hide_loading = function(fasta) {
$(".loading-file." + fasta).hide();
};
dgenies.run.show_success = function(fasta) {
$(".upload-success." + fasta).show()
};
dgenies.run.hide_success = function(fasta) {
$(".upload-success." + fasta).hide()
};
dgenies.run.reset_errors = function() {
$("label").removeClass("error");
$("div.errors-submit ul.flashes").find("li").remove();
};
dgenies.run.start_uploads = function() {
let query_type = parseInt($("select.query").val());
if (query_type === 0) {
$("button#button-query").hide();
dgenies.run.show_loading("query");
}
else {
dgenies.run.files[0] = undefined;
}
let target_type = parseInt($("select.target").val());
if (target_type === 0) {
$("button#button-target").hide();
dgenies.run.show_loading("target");
}
else {
dgenies.run.files[1] = undefined;
}
dgenies.run.upload_next();
};
dgenies.run.show_global_loading = function () {
$("button#submit").hide();
$("div#uploading-loading").show();
};
dgenies.run.submit = function () {
dgenies.run.reset_errors();
if (dgenies.run.valid_form()) {
dgenies.run.disable_form();
dgenies.run.show_global_loading();
dgenies.run.start_uploads();
}
};
\ No newline at end of file
class Fasta:
def __init__(self, name, path, type_f):
self.__name = name
self.__path = path
self.__type = type_f
def set_path(self, path):
self.__path = path
def get_path(self):
return self.__path
def get_name(self):
return self.__name
def get_type(self):
return self.__type
import os
import shutil
import subprocess
import datetime
import threading
import gzip
import urllib.request
import traceback
from config_reader import AppConfigReader
from pony.orm import db_session, select
from database import db, Job
from lib.Fasta import Fasta
class JobManager:
def __init__(self, id_job, email=None, fasta_q=None, fasta_t=None, query_name=None, target_name=None):
def __init__(self, id_job: str, email: str=None, query: Fasta=None, target: Fasta=None):
self.id_job = id_job
self.email = email
self.fasta_q = fasta_q if (fasta_q is None or not fasta_q.endswith(".gz")) else self._decompress(fasta_q)
self.fasta_t = fasta_t if (fasta_t is None or not fasta_t.endswith(".gz")) else self._decompress(fasta_t)
self.query = query_name
self.target = target_name
self.query = query
self.target = target
config_reader = AppConfigReader()
# Get configs:
self.batch_system_type = config_reader.get_batch_system_type()
......@@ -70,8 +69,8 @@ class JobManager:
@db_session
def __launch_local(self):
cmd = ["run_minimap2.sh", self.minimap2, self.samtools, self.threads,
self.fasta_t if self.fasta_t is not None else "NONE", self.fasta_q, self.query,
self.target, self.paf, self.paf_raw, self.output_dir]
self.target.get_path() if self.target is not None else "NONE", self.query.get_path(),
self.query.get_name(), self.target.get_name(), self.paf, self.paf_raw, self.output_dir]
with open(self.logs, "w") as logs:
p = subprocess.Popen(cmd, stdout=logs, stderr=logs)
job = Job.get(id_job=self.id_job)
......@@ -83,21 +82,44 @@ class JobManager:
job.status = status
db.commit()
def __getting_local_file(self, fasta: Fasta):
finale_path = os.path.join(self.output_dir, os.path.basename(fasta.get_path()))
shutil.move(fasta.get_path(), finale_path)
if finale_path.endswith(".gz"):
finale_path = self._decompress(finale_path)
return finale_path
@db_session
def getting_files(self):
job = Job.get(id_job=self.id_job)
job.status = "getfiles"
db.commit()
if self.query is not None:
if self.query.get_type() == "local":
self.query.set_path(self.__getting_local_file(self.query))
if self.target is not None:
if self.target.get_type() == "local":
self.target.set_path(self.__getting_local_file(self.target))
job = Job.get(id_job=self.id_job)
job.status = "waiting"
db.commit()
if self.batch_system_type == "local":
self.__launch_local()
@db_session
def launch(self):
j1 = select(j for j in Job if j.id_job == self.id_job)
if len(j1) > 0:
print("Old job found without result dir existing: delete it from BDD!")
j1.delete()
if self.fasta_q is not None:
if self.query is not None:
job = Job(id_job=self.id_job, email=self.email, batch_type=self.batch_system_type,
date_created=datetime.datetime.now())
db.commit()
if not os.path.exists(self.output_dir):
os.mkdir(self.output_dir)
if self.batch_system_type == "local":
thread = threading.Timer(1, self.__launch_local)
thread.start()
thread = threading.Timer(1, self.getting_files)
thread.start()
else:
job = Job(id_job=self.id_job, email=self.email, batch_type=self.batch_system_type,
date_created=datetime.datetime.now(), status="error")
......
......@@ -2,15 +2,17 @@
import time
import datetime
from flask import Flask, render_template, request, redirect, flash, url_for, jsonify
from flask import Flask, render_template, request, redirect, flash, url_for, jsonify, session
from werkzeug.utils import secure_filename
from lib.paf import Paf
from config_reader import AppConfigReader
from lib.job_manager import JobManager
from lib.functions import *
from lib.upload_file import uploadfile
from lib.Fasta import Fasta
import sys
app_folder = os.path.dirname(os.path.realpath(__file__))
sys.path.insert(0, app_folder)
os.environ["PATH"] = os.path.join(app_folder, "bin") + ":" + os.environ["PATH"]
......@@ -48,6 +50,8 @@ def main():
@app.route("/run", methods=['GET'])
def run():
session["user_tmp_dir"] = random_string(5) + "_" + \
datetime.datetime.fromtimestamp(time.time()).strftime('%Y%m%d%H%M%S')
id_job = random_string(5) + "_" + datetime.datetime.fromtimestamp(time.time()).strftime('%Y%m%d%H%M%S')
if "id_job" in request.args:
id_job = request.args["id_job"]
......@@ -63,60 +67,51 @@ def run():
def launch_analysis():
id_job = request.form["id_job"]
email = request.form["email"]
file_query = request.files["file_query"]
file_target = request.files["file_target"]
file_query = request.form["query"]
file_query_type = request.form["query_type"]
file_target = request.form["target"]
file_target_type = request.form["target_type"]
# Check form:
form_pass = True
errors = []
if id_job == "":
flash("Id of job not given")
errors.append("Id of job not given")
form_pass = False
if email == "":
flash("Email not given")
errors.append("Email not given")
form_pass = False
if file_query.filename == "":
flash("No query fasta selected")
if file_query == "":
errors.append("No query fasta selected")
form_pass = False
# Form pass
if form_pass:
# Check files are correct:
if not allowed_file(file_query.filename):
flash("Format of query fasta must be in fasta format (*.%s)" % ", *.".join(ALLOWED_EXTENSIONS))
form_pass = False
if file_target.filename != "" and not allowed_file(file_target.filename):
flash("Format of target fasta must be in fasta format (*.%s)" % ", *.".join(ALLOWED_EXTENSIONS))
form_pass = False
if form_pass:
# Get final job id:
id_job_orig = id_job
while os.path.exists(os.path.join(app_data, id_job)):
id_job = id_job_orig + "_2"
folder_files = os.path.join(app_data, id_job)
os.makedirs(folder_files)
# Save files:
query_name = os.path.splitext(os.path.basename(file_query.filename))[0]
filename_query = get_valid_uploaded_filename(secure_filename(file_query.filename), folder_files)
target_name = os.path.splitext(os.path.basename(file_target.filename))[0]
query_path = os.path.join(folder_files, filename_query)
file_query.save(query_path)
target_path = None
if file_target.filename != "":
filename_target = get_valid_uploaded_filename(secure_filename(file_target.filename), folder_files)
target_path = os.path.join(folder_files, filename_target)
file_target.save(target_path)
# Launch job:
job = JobManager(id_job, email, query_path, target_path, query_name, target_name)
job.launch()
return redirect(url_for(".status", id_job=id_job))
else:
return redirect(url_for(".run", id_job=id_job, email=email))
# Get final job id:
id_job_orig = id_job
while os.path.exists(os.path.join(app_data, id_job)):
id_job = id_job_orig + "_2"
folder_files = os.path.join(app_data, id_job)
os.makedirs(folder_files)
# Save files:
query_name = os.path.splitext(file_query.replace(".gz", ""))[0]
query_path = os.path.join(app.config["UPLOAD_FOLDER"], session["user_tmp_dir"], file_query)
query = Fasta(name=query_name, path=query_path, type_f=file_query_type)
target = None
if file_target != "":
target_name = os.path.splitext(file_target.replace(".gz", ""))[0]
target_path = os.path.join(app.config["UPLOAD_FOLDER"], session["user_tmp_dir"], file_target)
target = Fasta(name=target_name, path=target_path, type_f=file_target_type)
# Launch job:
job = JobManager(id_job, email, query, target)
job.launch()
return jsonify({"success": True, "redirect": url_for(".status", id_job=id_job)})
else:
return redirect(url_for(".main", id_job=id_job, email=email))
return jsonify({"success": False, "errors": errors})
# Status of a job
......@@ -174,40 +169,36 @@ def sort_graph(id_res):