views.py 33.1 KB
Newer Older
1
from dgenies import app, app_title, app_folder, config_reader, mailer, APP_DATA, MODE, DEBUG, VERSION
2

Floreal Cabanettes's avatar
Floreal Cabanettes committed
3
import os
4
5
import time
import datetime
6
import shutil
7
import re
8
import threading
Floreal Cabanettes's avatar
Floreal Cabanettes committed
9
import traceback
10
import requests
11
from requests.exceptions import ConnectionError
12
import json
Floreal Cabanettes's avatar
Floreal Cabanettes committed
13
from flask import render_template, request, url_for, jsonify, Response, abort, send_file, Markup
14
from pathlib import Path
15
16
17
18
19
from dgenies.lib.paf import Paf
from dgenies.lib.job_manager import JobManager
from dgenies.lib.functions import Functions, ALLOWED_EXTENSIONS
from dgenies.lib.upload_file import UploadFile
from dgenies.lib.fasta import Fasta
20
from dgenies.tools import Tools
Floreal Cabanettes's avatar
Floreal Cabanettes committed
21
22
from markdown import Markdown
from markdown.extensions.toc import TocExtension
23
from markdown.extensions.tables import TableExtension
24
import tarfile
25
from jinja2 import Environment
26
27
28
if MODE == "webserver":
    from dgenies.database import Session, Gallery
    from peewee import DoesNotExist
29

30

31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
def get_latest_version():
    latest = ""
    win32 = ""
    try:
        call = requests.get("https://api.github.com/repos/genotoul-bioinfo/dgenies/releases/latest")
        if call.ok:
            release = json.loads(call.content.decode("utf-8"))
            if "tag_name" in release:
                latest = release["tag_name"][1:]
                for asset in release["assets"]:
                    if asset["name"].endswith(".exe"):
                        win32 = asset["browser_download_url"]
                        break
    except ConnectionError:
        pass
    return latest, win32


Floreal Cabanettes's avatar
Floreal Cabanettes committed
49
50
51
52
@app.context_processor
def global_templates_variables():
    return {
        "title": app_title,
53
        "mode": MODE,
54
55
        "all_jobs": Functions.get_list_all_jobs(MODE),
        "debug": DEBUG
Floreal Cabanettes's avatar
Floreal Cabanettes committed
56
57
58
    }


59
60
61
# Main
@app.route("/", methods=['GET'])
def main():
62
63
64
65
66
67
    if MODE == "webserver":
        pict = Gallery.select().order_by("id")
        if len(pict) > 0:
            pict = pict[0].picture
        else:
            pict = None
Floreal Cabanettes's avatar
Floreal Cabanettes committed
68
69
    else:
        pict = None
Floreal Cabanettes's avatar
Floreal Cabanettes committed
70
    return render_template("index.html", menu="index", pict=pict)
Floreal Cabanettes's avatar
Floreal Cabanettes committed
71
72
73
74


@app.route("/run", methods=['GET'])
def run():
75
76
77
78
79
    tools = Tools().tools
    tools_names = sorted(list(tools.keys()), key=lambda x: (tools[x].order, tools[x].name))
    tools_ava = {}
    for tool_name, tool in tools.items():
        tools_ava[tool_name] = 1 if tool.all_vs_all is not None else 0
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
    if MODE == "webserver":
        with Session.connect():
            s_id = Session.new()
    else:
        upload_folder = Functions.random_string(20)
        tmp_dir = config_reader.upload_folder
        upload_folder_path = os.path.join(tmp_dir, upload_folder)
        while os.path.exists(upload_folder_path):
            upload_folder = Functions.random_string(20)
            upload_folder_path = os.path.join(tmp_dir, upload_folder)
        s_id = upload_folder
    id_job = Functions.random_string(5) + "_" + datetime.datetime.fromtimestamp(time.time()).strftime('%Y%m%d%H%M%S')
    if "id_job" in request.args:
        id_job = request.args["id_job"]
    email = ""
    if "email" in request.args:
        email = request.args["email"]
Floreal Cabanettes's avatar
Floreal Cabanettes committed
97
    return render_template("run.html", id_job=id_job, email=email,
98
                           menu="run", allowed_ext=ALLOWED_EXTENSIONS, s_id=s_id,
99
100
101
                           max_upload_file_size=config_reader.max_upload_file_size,
                           example=config_reader.example_target != "",
                           target=os.path.basename(config_reader.example_target),
102
                           query=os.path.basename(config_reader.example_query), tools_names=tools_names, tools=tools,
103
                           tools_ava=tools_ava, version=VERSION)
104
105


106
107
@app.route("/run-test", methods=['GET'])
def run_test():
108
109
110
111
112
113
114
    if MODE == "webserver":
        print(config_reader.allowed_ip_tests)
        if request.remote_addr not in config_reader.allowed_ip_tests:
            return abort(404)
        with Session.connect():
            return Session.new()
    return abort(500)
115
116


117
118
119
# Launch analysis
@app.route("/launch_analysis", methods=['POST'])
def launch_analysis():
120
121
122
123
124
125
126
127
128
129
130
131
    if MODE == "webserver":
        try:
            with Session.connect():
                session = Session.get(s_id=request.form["s_id"])
        except DoesNotExist:
            return jsonify({"success": False, "errors": ["Session has expired. Please refresh the page and try again"]})
        upload_folder = session.upload_folder
        # Delete session:
        session.delete_instance()
    else:
        upload_folder = request.form["s_id"]

132
133
    id_job = request.form["id_job"]
    email = request.form["email"]
134
135
136
137
    file_query = request.form["query"]
    file_query_type = request.form["query_type"]
    file_target = request.form["target"]
    file_target_type = request.form["target_type"]
138
139
140
    tool = request.form["tool"] if "tool" in request.form else None
    alignfile = request.form["alignfile"] if "alignfile" in request.form else None
    alignfile_type = request.form["alignfile_type"] if "alignfile_type" in request.form else None
141
142
    backup = request.form["backup"] if "backup" in request.form else None
    backup_type = request.form["backup_type"] if "backup_type" in request.form else None
143
144
145

    # Check form:
    form_pass = True
146
    errors = []
147

148
149
150
151
    if alignfile is not None and alignfile_type is None:
        errors.append("Server error: no alignfile_type in form. Please contact the support")
        form_pass = False

Floreal Cabanettes's avatar
Floreal Cabanettes committed
152
    if backup is not None and backup != "" and (backup_type is None or backup_type == ""):
153
154
155
        errors.append("Server error: no backup_type in form. Please contact the support")
        form_pass = False

Floreal Cabanettes's avatar
Floreal Cabanettes committed
156
    if backup is not None and backup != "":
157
158
159
160
        alignfile = ""
        file_query = ""
        file_target = ""
    else:
Floreal Cabanettes's avatar
Floreal Cabanettes committed
161
        backup = None
162
163
164
165
        if file_target == "":
            errors.append("No target fasta selected")
            form_pass = False

166
    if tool is not None and tool not in Tools().tools:
167
        errors.append("Tool unavailable: %s" % tool)
168
        form_pass = False
169

170
    if id_job == "":
171
        errors.append("Id of job not given")
172
173
        form_pass = False

174
175
176
177
    if MODE == "webserver":
        if email == "":
            errors.append("Email not given")
            form_pass = False
Floreal Cabanettes's avatar
Floreal Cabanettes committed
178
        elif not re.match(r"^[\w.\-]+@[\w\-.]{2,}\.[a-z]{2,4}$", email):
179
180
            errors.append("Email is invalid")
            form_pass = False
181
182
183

    # Form pass
    if form_pass:
184
        # Get final job id:
185
        id_job = re.sub('[^A-Za-z0-9_\-]+', '', id_job.replace(" ", "_"))
186
        id_job_orig = id_job
187
        i = 2
188
        while os.path.exists(os.path.join(APP_DATA, id_job)):
189
190
            id_job = id_job_orig + ("_%d" % i)
            i += 1
191

192
        folder_files = os.path.join(APP_DATA, id_job)
193
194
195
        os.makedirs(folder_files)

        # Save files:
196
197
        query = None
        if file_query != "":
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
            example = False
            if file_query.startswith("example://"):
                example = True
                query_path = config_reader.example_query
                query_name = os.path.basename(query_path)
                file_query_type = "local"
            else:
                query_name = os.path.splitext(file_query.replace(".gz", ""))[0] if file_query_type == "local" else None
                query_path = os.path.join(app.config["UPLOAD_FOLDER"], upload_folder, file_query) \
                    if file_query_type == "local" else file_query
                if file_query_type == "local" and not os.path.exists(query_path):
                    errors.append("Query file not correct!")
                    form_pass = False
            query = Fasta(name=query_name, path=query_path, type_f=file_query_type, example=example)
        example = False
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
        target = None
        if file_target != "":
            if file_target.startswith("example://"):
                example = True
                target_path = config_reader.example_target
                target_name = os.path.basename(target_path)
                file_target_type = "local"
            else:
                target_name = os.path.splitext(file_target.replace(".gz", ""))[0] if file_target_type == "local" else None
                target_path = os.path.join(app.config["UPLOAD_FOLDER"], upload_folder, file_target) \
                    if file_target_type == "local" else file_target
                if file_target_type == "local" and not os.path.exists(target_path):
                    errors.append("Target file not correct!")
                    form_pass = False
            target = Fasta(name=target_name, path=target_path, type_f=file_target_type, example=example)
228

229
        if alignfile is not None and alignfile != "" and backup is not None:
230
            Path(os.path.join(folder_files, ".align")).touch()
231
232
233

        align = None
        if alignfile is not None and alignfile != "":
234
235
236
237
238
239
240
241
            alignfile_name = os.path.splitext(alignfile)[0] if alignfile_type == "local" else None
            alignfile_path = os.path.join(app.config["UPLOAD_FOLDER"], upload_folder, alignfile) \
                if alignfile_type == "local" else alignfile
            if alignfile_type == "local" and not os.path.exists(alignfile_path):
                errors.append("Alignment file not correct!")
                form_pass = False
            align = Fasta(name=alignfile_name, path=alignfile_path, type_f=alignfile_type)

242
243
244
245
246
247
248
249
250
251
        bckp = None
        if backup is not None:
            backup_name = os.path.splitext(backup)[0] if backup_type == "local" else None
            backup_path = os.path.join(app.config["UPLOAD_FOLDER"], upload_folder, backup) \
                if backup_type == "local" else backup
            if backup_type == "local" and not os.path.exists(backup_path):
                errors.append("Backup file not correct!")
                form_pass = False
            bckp = Fasta(name=backup_name, path=backup_path, type_f=backup_type)

252
253
        if form_pass:
            # Launch job:
254
255
256
257
258
            job = JobManager(id_job=id_job,
                             email=email,
                             query=query,
                             target=target,
                             align=align,
259
                             backup=bckp,
260
261
                             mailer=mailer,
                             tool=tool)
262
263
264
265
266
267
            if MODE == "webserver":
                job.launch()
            else:
                job.launch_standalone()
            return jsonify({"success": True, "redirect": url_for(".status", id_job=id_job)})
    if not form_pass:
268
        return jsonify({"success": False, "errors": errors})
269
270
271
272
273
274


# Status of a job
@app.route('/status/<id_job>', methods=['GET'])
def status(id_job):
    job = JobManager(id_job)
275
276
277
278
279
280
281
282
283
284
285
286
    j_status = job.status()
    mem_peak = j_status["mem_peak"] if "mem_peak" in j_status else None
    if mem_peak is not None:
        mem_peak = "%.1f G" % (mem_peak / 1024.0 / 1024.0)
    time_e = j_status["time_elapsed"] if "time_elapsed" in j_status else None
    if time_e is not None:
        if time_e < 60:
            time_e = "%d secs" % time_e
        else:
            minutes = time_e // 60
            seconds = time_e - minutes * 60
            time_e = "%d min %d secs" % (minutes, seconds)
287
288
289
290
291
292
293
294
295
    format = request.args.get("format")
    if format is not None and format == "json":
        return jsonify({
            "status": j_status["status"],
            "error": j_status["error"].replace("#ID#", ""),
            "id_job": id_job,
            "mem_peak": mem_peak,
            "time_elapsed": time_e
        })
Floreal Cabanettes's avatar
Floreal Cabanettes committed
296
    return render_template("status.html", status=j_status["status"],
297
298
                           error=j_status["error"].replace("#ID#", ""),
                           id_job=id_job, menu="results", mem_peak=mem_peak,
299
                           time_elapsed=time_e, do_align=job.do_align(),
Floreal Cabanettes's avatar
Floreal Cabanettes committed
300
                           query_filtered=job.is_query_filtered(), target_filtered=job.is_target_filtered())
301
302


Floreal Cabanettes's avatar
Floreal Cabanettes committed
303
# Results path
304
@app.route("/result/<id_res>", methods=['GET'])
305
def result(id_res):
306
    res_dir = os.path.join(APP_DATA, id_res)
Floreal Cabanettes's avatar
Floreal Cabanettes committed
307
    return render_template("result.html", id=id_res, menu="result", current_result=id_res,
308
309
                           is_gallery=Functions.is_in_gallery(id_res, MODE),
                           fasta_file=Functions.query_fasta_file_exists(res_dir))
310
311


312
313
@app.route("/gallery", methods=['GET'])
def gallery():
314
    if MODE == "webserver":
Floreal Cabanettes's avatar
Floreal Cabanettes committed
315
        return render_template("gallery.html", items=Functions.get_gallery_items(), menu="gallery")
316
    return abort(404)
317
318
319
320


@app.route("/gallery/<filename>", methods=['GET'])
def gallery_file(filename):
321
322
323
324
325
326
    if MODE == "webserver":
        try:
            return send_file(os.path.join(config_reader.app_data, "gallery", filename))
        except FileNotFoundError:
            abort(404)
    return abort(404)
327
328


329
def get_file(file, gzip=False):  # pragma: no cover
Floreal Cabanettes's avatar
Floreal Cabanettes committed
330
331
332
333
334
335
    try:
        # Figure out how flask returns static files
        # Tried:
        # - render_template
        # - send_file
        # This should not be so non-obvious
336
        return open(file, "rb" if gzip else "r").read()
337
338
    except FileNotFoundError:
        abort(404)
Floreal Cabanettes's avatar
Floreal Cabanettes committed
339
    except IOError as exc:
340
        print(exc.__traceback__)
341
        abort(500)
Floreal Cabanettes's avatar
Floreal Cabanettes committed
342
343


Floreal Cabanettes's avatar
Floreal Cabanettes committed
344
345
@app.route("/documentation/run", methods=['GET'])
def documentation_run():
346
347
    version = get_latest_version()[0]
    with open(os.path.join(app_folder, "md", "doc_run.md"), "r",  encoding='utf-8') as install_instr:
Floreal Cabanettes's avatar
Floreal Cabanettes committed
348
        content = install_instr.read()
349
350
351
    env = Environment()
    template = env.from_string(content)
    content = template.render(mode=MODE, version=version)
Floreal Cabanettes's avatar
Floreal Cabanettes committed
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
    md = Markdown(extensions=[TocExtension(baselevel=1)])
    max_upload_file_size = config_reader.max_upload_file_size
    if max_upload_file_size == -1:
        max_upload_file_size = "no limit"
    else:
        max_upload_file_size = Functions.get_readable_size(max_upload_file_size, 0)
    max_upload_size = config_reader.max_upload_size
    if max_upload_size == -1:
        max_upload_size = "no limit"
    else:
        max_upload_size = Functions.get_readable_size(max_upload_size, 0)
    max_upload_size_ava = config_reader.max_upload_size_ava
    if max_upload_size_ava == -1:
        max_upload_size_ava = "no limit"
    else:
        max_upload_size_ava = Functions.get_readable_size(max_upload_size_ava, 0)
    content = Markup(md.convert(content)).replace("###size###", max_upload_file_size)\
                                         .replace("###size_unc###", max_upload_size)\
                                         .replace("###size_ava###", max_upload_size_ava)
    toc = Markup(md.toc)
Floreal Cabanettes's avatar
Floreal Cabanettes committed
372
    return render_template("documentation.html", menu="documentation", content=content, toc=toc)
Floreal Cabanettes's avatar
Floreal Cabanettes committed
373
374


375
376
@app.route("/documentation/result", methods=['GET'])
def documentation_result():
377
    with open(os.path.join(app_folder, "md", "user_manual.md"), "r",
378
379
380
381
382
383
384
385
              encoding='utf-8') as install_instr:
        content = install_instr.read()
    md = Markdown(extensions=[TocExtension(baselevel=1)])
    content = Markup(md.convert(content))
    toc = Markup(md.toc)
    return render_template("documentation.html", menu="documentation", content=content, toc=toc)


386
387
@app.route("/documentation/formats", methods=['GET'])
def documentation_formats():
388
    with open(os.path.join(app_folder, "md", "doc_formats.md"), "r",
389
390
391
392
393
394
395
396
              encoding='utf-8') as install_instr:
        content = install_instr.read()
    md = Markdown(extensions=[TocExtension(baselevel=1), TableExtension()])
    content = Markup(md.convert(content))
    toc = Markup(md.toc)
    return render_template("documentation.html", menu="documentation", content=content, toc=toc)


397
398
399
400
401
402
403
404
405
406
407
@app.route("/documentation/dotplot", methods=['GET'])
def documentation_dotplot():
    with open(os.path.join(app_folder, "md", "doc_dotplot.md"), "r",
              encoding='utf-8') as install_instr:
        content = install_instr.read()
    md = Markdown(extensions=[TocExtension(baselevel=1)])
    content = Markup(md.convert(content))
    toc = Markup(md.toc)
    return render_template("documentation.html", menu="documentation", content=content, toc=toc)


Floreal Cabanettes's avatar
Floreal Cabanettes committed
408
409
@app.route("/install", methods=['GET'])
def install():
410
    latest, win32 = get_latest_version()
411

412
    with open(os.path.join(app_folder, "md", "INSTALL.md"), "r", encoding='utf-8') as install_instr:
Floreal Cabanettes's avatar
Floreal Cabanettes committed
413
        content = install_instr.read()
414
415
    env = Environment()
    template = env.from_string(content)
416
    content = template.render(version=latest, win32=win32)
Floreal Cabanettes's avatar
Floreal Cabanettes committed
417
418
419
    md = Markdown(extensions=[TocExtension(baselevel=1)])
    content = Markup(md.convert(content))
    toc = Markup(md.toc)
Floreal Cabanettes's avatar
Floreal Cabanettes committed
420
    return render_template("documentation.html", menu="install", content=content, toc=toc)
Floreal Cabanettes's avatar
Floreal Cabanettes committed
421

Floreal Cabanettes's avatar
Floreal Cabanettes committed
422

423
424
425
426
@app.route("/contact", methods=['GET'])
def contact():
    return render_template("contact.html", menu="contact")

Floreal Cabanettes's avatar
Floreal Cabanettes committed
427

Floreal Cabanettes's avatar
Floreal Cabanettes committed
428
429
430
431
432
433
434
435
436
437
438
@app.route("/paf/<id_res>", methods=['GET'])
def download_paf(id_res):
    map_file = os.path.join(APP_DATA, id_res, "map.paf.sorted")
    if not os.path.exists(map_file):
        map_file = os.path.join(APP_DATA, id_res, "map.paf")
    if not os.path.exists(map_file):
        abort(404)
    content = get_file(map_file)
    return Response(content, mimetype="text/plain")


Floreal Cabanettes's avatar
Floreal Cabanettes committed
439
# Get graph (ajax request)
440
441
442
@app.route('/get_graph', methods=['POST'])
def get_graph():
    id_f = request.form["id"]
443
444
445
    paf = os.path.join(APP_DATA, id_f, "map.paf")
    idx1 = os.path.join(APP_DATA, id_f, "query.idx")
    idx2 = os.path.join(APP_DATA, id_f, "target.idx")
446

447
    paf = Paf(paf, idx1, idx2)
448

449
450
    if paf.parsed:
        res = paf.get_d3js_data()
451
        res["success"] = True
452
453
454
455
456
457
        return jsonify(res)
    return jsonify({"success": False, "message": paf.error})


@app.route('/sort/<id_res>', methods=['POST'])
def sort_graph(id_res):
458
    if not os.path.exists(os.path.join(APP_DATA, id_res, ".all-vs-all")):
459
460
461
        paf_file = os.path.join(APP_DATA, id_res, "map.paf")
        idx1 = os.path.join(APP_DATA, id_res, "query.idx")
        idx2 = os.path.join(APP_DATA, id_res, "target.idx")
462
        paf = Paf(paf_file, idx1, idx2, False)
463
464
465
466
467
468
469
        paf.sort()
        if paf.parsed:
            res = paf.get_d3js_data()
            res["success"] = True
            return jsonify(res)
        return jsonify({"success": False, "message": paf.error})
    return jsonify({"success": False, "message": "Sort is not available for All-vs-All mode"})
Floreal Cabanettes's avatar
Floreal Cabanettes committed
470

Floréal Cabanettes's avatar
Floréal Cabanettes committed
471

472
473
474
475
@app.route('/reverse-contig/<id_res>', methods=['POST'])
def reverse_contig(id_res):
    contig_name = request.form["contig"]
    if not os.path.exists(os.path.join(APP_DATA, id_res, ".all-vs-all")):
476
477
478
        paf_file = os.path.join(APP_DATA, id_res, "map.paf")
        idx1 = os.path.join(APP_DATA, id_res, "query.idx")
        idx2 = os.path.join(APP_DATA, id_res, "target.idx")
479
480
481
482
483
484
485
486
487
488
        paf = Paf(paf_file, idx1, idx2, False)
        paf.reverse_contig(contig_name)
        if paf.parsed:
            res = paf.get_d3js_data()
            res["success"] = True
            return jsonify(res)
        return jsonify({"success": False, "message": paf.error})
    return jsonify({"success": False, "message": "Sort is not available for All-vs-All mode"})


489
490
@app.route('/freenoise/<id_res>', methods=['POST'])
def free_noise(id_res):
491
492
493
    paf_file = os.path.join(APP_DATA, id_res, "map.paf")
    idx1 = os.path.join(APP_DATA, id_res, "query.idx")
    idx2 = os.path.join(APP_DATA, id_res, "target.idx")
494
495
496
497
498
499
500
501
502
    paf = Paf(paf_file, idx1, idx2, False)
    paf.parse_paf(noise=request.form["noise"] == "1")
    if paf.parsed:
        res = paf.get_d3js_data()
        res["success"] = True
        return jsonify(res)
    return jsonify({"success": False, "message": paf.error})


503
504
@app.route('/get-fasta-query/<id_res>', methods=['POST'])
def build_fasta(id_res):
505
    res_dir = os.path.join(APP_DATA, id_res)
506
507
    lock_query = os.path.join(res_dir, ".query-fasta-build")
    is_sorted = os.path.exists(os.path.join(res_dir, ".sorted"))
508
    compressed = request.form["gzip"].lower() == "true"
509
510
511
512
513
    query_fasta = Functions.get_fasta_file(res_dir, "query", is_sorted)
    if query_fasta is not None:
        if is_sorted and not query_fasta.endswith(".sorted"):
            # Do the sort
            Path(lock_query).touch()
514
            if not compressed or MODE == "standalone":  # If compressed, it will took a long time, so not wait
515
                Path(lock_query + ".pending").touch()
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
            index_file = os.path.join(res_dir, "query.idx.sorted")
            if MODE == "webserver":
                thread = threading.Timer(1, Functions.sort_fasta, kwargs={
                    "job_name": id_res,
                    "fasta_file": query_fasta,
                    "index_file": index_file,
                    "lock_file": lock_query,
                    "compress": compressed,
                    "mailer": mailer,
                    "mode": MODE
                })
                thread.start()
            else:
                Functions.sort_fasta(job_name=id_res,
                                     fasta_file=query_fasta,
                                     index_file=index_file,
                                     lock_file=lock_query,
                                     compress=compressed,
                                     mailer=None,
                                     mode=MODE)
536
            if not compressed or MODE == "standalone":
537
538
                if MODE == "webserver":
                    i = 0
539
                    time.sleep(5)
540
541
542
                    while os.path.exists(lock_query) and (i < 2 or MODE == "standalone"):
                        i += 1
                        time.sleep(5)
Floreal Cabanettes's avatar
Floreal Cabanettes committed
543
                os.remove(lock_query + ".pending")
544
545
546
                if os.path.exists(lock_query):
                    return jsonify({"success": True, "status": 1, "status_message": "In progress"})
                return jsonify({"success": True, "status": 2, "status_message": "Done",
547
                                "gzip": compressed})
548
549
            else:
                return jsonify({"success": True, "status": 1, "status_message": "In progress"})
550
551
552
553
554
        elif is_sorted and os.path.exists(lock_query):
            # Sort is already in progress
            return jsonify({"success": True, "status": 1, "status_message": "In progress"})
        else:
            # No sort to do or sort done
555
556
557
558
559
560
561
            if compressed and not query_fasta.endswith(".gz.fasta"):
                # If compressed file is asked, we must compress it now if not done before...
                Path(lock_query).touch()
                thread = threading.Timer(1, Functions.compress_and_send_mail, kwargs={
                    "job_name": id_res,
                    "fasta_file": query_fasta,
                    "index_file": os.path.join(res_dir, "query.idx.sorted"),
562
                    "lock_file": lock_query,
563
564
565
566
567
                    "compressed": compressed,
                    "mailer": mailer
                })
                thread.start()
                return jsonify({"success": True, "status": 1, "status_message": "In progress"})
568
569
570
571
572
573
574
            return jsonify({"success": True, "status": 2, "status_message": "Done",
                            "gzip": query_fasta.endswith(".gz") or query_fasta.endswith(".gz.sorted")})
    else:
        return jsonify({"success": False,
                        "message": "Unable to get fasta file for query. Please contact us to report the bug"})


575
def build_query_as_reference(id_res):
576
577
578
579
580
    paf_file = os.path.join(APP_DATA, id_res, "map.paf")
    idx1 = os.path.join(APP_DATA, id_res, "query.idx")
    idx2 = os.path.join(APP_DATA, id_res, "target.idx")
    paf = Paf(paf_file, idx1, idx2, False, mailer=mailer, id_job=id_res)
    paf.parse_paf(False, True)
581
582
583
584
585
586
587
588
589
590
    if MODE == "webserver":
        thread = threading.Timer(0, paf.build_query_chr_as_reference)
        thread.start()
        return True
    return paf.build_query_chr_as_reference()


@app.route('/build-query-as-reference/<id_res>', methods=['POST'])
def post_query_as_reference(id_res):
    build_query_as_reference(id_res)
591
592
593
    return jsonify({"success": True})


594
595
596
597
598
599
600
@app.route('/get-query-as-reference/<id_res>', methods=['GET'])
def get_query_as_reference(id_res):
    if MODE != "standalone":
        return abort(404)
    return send_file(build_query_as_reference(id_res))


601
602
603
604
605
606
607
608
@app.route('/download/<id_res>/<filename>')
def download_file(id_res, filename):
    file_dl = os.path.join(APP_DATA, id_res, filename)
    if os.path.isfile(file_dl):
        return send_file(file_dl)
    return abort(404)


609
610
611
@app.route('/fasta-query/<id_res>', defaults={'filename': ""}, methods=['GET'])
@app.route('/fasta-query/<id_res>/<filename>', methods=['GET'])  # Use fake URL in mail to set download file name
def dl_fasta(id_res, filename):
612
    res_dir = os.path.join(APP_DATA, id_res)
613
614
615
616
617
618
619
620
621
622
623
624
625
    lock_query = os.path.join(res_dir, ".query-fasta-build")
    is_sorted = os.path.exists(os.path.join(res_dir, ".sorted"))
    if not os.path.exists(lock_query) or not is_sorted:
        query_fasta = Functions.get_fasta_file(res_dir, "query", is_sorted)
        if query_fasta is not None:
            if query_fasta.endswith(".gz") or query_fasta.endswith(".gz.sorted"):
                content = get_file(query_fasta, True)
                return Response(content, mimetype="application/gzip")
            content = get_file(query_fasta)
            return Response(content, mimetype="text/plain")
    abort(404)


626
627
@app.route('/qt-assoc/<id_res>', methods=['GET'])
def qt_assoc(id_res):
628
    res_dir = os.path.join(APP_DATA, id_res)
629
    if os.path.exists(res_dir) and os.path.isdir(res_dir):
630
631
632
        paf_file = os.path.join(APP_DATA, id_res, "map.paf")
        idx1 = os.path.join(APP_DATA, id_res, "query.idx")
        idx2 = os.path.join(APP_DATA, id_res, "target.idx")
633
634
635
636
637
638
639
640
641
642
643
644
        try:
            paf = Paf(paf_file, idx1, idx2, False)
            paf.parse_paf(False)
        except FileNotFoundError:
            print("Unable to load data!")
            abort(404)
            return False
        csv_content = paf.build_query_on_target_association_file()
        return Response(csv_content, mimetype="text/plain")
    abort(404)


645
646
647
648
649
650
@app.route('/no-assoc/<id_res>', methods=['POST'])
def no_assoc(id_res):
    """
    Get contigs that match with None target
    :param id_res: id of the result
    """
651
    res_dir = os.path.join(APP_DATA, id_res)
652
    if os.path.exists(res_dir) and os.path.isdir(res_dir):
653
654
655
        paf_file = os.path.join(APP_DATA, id_res, "map.paf")
        idx1 = os.path.join(APP_DATA, id_res, "query.idx")
        idx2 = os.path.join(APP_DATA, id_res, "target.idx")
656
657
658
659
660
661
        try:
            paf = Paf(paf_file, idx1, idx2, False)
        except FileNotFoundError:
            print("Unable to load data!")
            abort(404)
            return False
662
        file_content = paf.build_list_no_assoc(request.form["to"])
663
664
665
666
667
668
669
670
        empty = file_content == "\n"
        return jsonify({
            "file_content": file_content,
            "empty": empty
        })
    abort(404)


Floreal Cabanettes's avatar
Floreal Cabanettes committed
671
672
@app.route('/summary/<id_res>', methods=['POST'])
def summary(id_res):
673
674
675
676
677
678
679
680
681
682
    paf_file = os.path.join(APP_DATA, id_res, "map.paf")
    idx1 = os.path.join(APP_DATA, id_res, "query.idx")
    idx2 = os.path.join(APP_DATA, id_res, "target.idx")
    try:
        paf = Paf(paf_file, idx1, idx2, False)
    except FileNotFoundError:
        return jsonify({
            "success": False,
            "message": "Unable to load data!"
        })
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
    percents = None
    s_status = "waiting"  # Accepted values: waiting, done, fail
    status_file = os.path.join(APP_DATA, id_res, ".summarize")
    fail_file = status_file + ".fail"
    if not os.path.exists(status_file):  # The job is finished or not started
        if not os.path.exists(fail_file):  # The job has not started yet or has successfully ended
            percents = paf.get_summary_stats()
            if percents is None:  # The job has not started yet
                Path(status_file).touch()
                thread = threading.Timer(0, paf.build_summary_stats, kwargs={"status_file": status_file})
                thread.start()
            else:  # The job has successfully ended
                s_status = "done"
        else:  # The job has failed
            s_status = "fail"

    if s_status == "waiting":  # The job is running
        # Check if the job end in the next 30 seconds
        nb_iter = 0
        while os.path.exists(status_file) and not os.path.exists(fail_file) and nb_iter < 10:
            time.sleep(3)
            nb_iter += 1
        if not os.path.exists(status_file):  # The job has ended
            percents = paf.get_summary_stats()
            if percents is None:  # The job has failed
                s_status = "fail"
            else:  # The job has successfully ended
                s_status = "done"

    if s_status == "fail":
713
714
715
716
717
718
        return jsonify({
            "success": False,
            "message": "Build of summary failed. Please contact us to report the bug"
        })
    return jsonify({
        "success": True,
719
720
        "percents": percents,
        "status": s_status
721
    })
Floreal Cabanettes's avatar
Floreal Cabanettes committed
722
723


724
725
726
727
728
729
730
731
732
733
@app.route('/backup/<id_res>')
def get_backup_file(id_res):
    res_dir = os.path.join(APP_DATA, id_res)
    tar = os.path.join(res_dir, "%s.tar" % id_res)
    with tarfile.open(tar, "w") as tarf:
        for file in ("map.paf", "target.idx", "query.idx"):
            tarf.add(os.path.join(res_dir, file), arcname=file)
    return send_file(tar)


734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
def get_filter_out(id_res, type_f):
    filter_file = os.path.join(APP_DATA, id_res, ".filter-" + type_f)
    return Response(get_file(filter_file), mimetype="text/plain")


@app.route('/filter-out/<id_res>/query')
def get_filter_out_query(id_res):
    return get_filter_out(id_res=id_res, type_f="query")


@app.route('/filter-out/<id_res>/target')
def get_filter_out_target(id_res):
    return get_filter_out(id_res=id_res, type_f="target")


749
750
@app.route("/ask-upload", methods=['POST'])
def ask_upload():
751
752
753
754
755
    if MODE == "standalone":
        return jsonify({
            "success": True,
            "allowed": True
        })
756
757
    try:
        s_id = request.form['s_id']
Floreal Cabanettes's avatar
Floreal Cabanettes committed
758
759
760
        with Session.connect():
            session = Session.get(s_id=s_id)
            allowed = session.ask_for_upload(True)
761
762
        return jsonify({
            "success": True,
Floreal Cabanettes's avatar
Floreal Cabanettes committed
763
            "allowed": allowed
764
765
766
        })
    except DoesNotExist:
        return jsonify({"success": False, "message": "Session not initialized. Please refresh the page."})
767
768


769
770
@app.route("/ping-upload", methods=['POST'])
def ping_upload():
771
772
773
774
775
    if MODE == "webserver":
        s_id = request.form['s_id']
        with Session.connect():
            session = Session.get(s_id=s_id)
            session.ping()
776
    return "OK"
777
778


779
780
781
782
@app.route("/upload", methods=['POST'])
def upload():
    try:
        s_id = request.form['s_id']
783
784
785
786
787
788
        if MODE == "webserver":
            try:
                with Session.connect():
                    session = Session.get(s_id=s_id)
                    if session.ask_for_upload(False):
                        folder = session.upload_folder
Floreal Cabanettes's avatar
Floreal Cabanettes committed
789
                    else:
790
791
792
793
794
795
                        return jsonify({"files": [], "success": "ERR", "message": "Not allowed to upload!"})
            except DoesNotExist:
                return jsonify(
                    {"files": [], "success": "ERR", "message": "Session not initialized. Please refresh the page."})
        else:
            folder = s_id
Floreal Cabanettes's avatar
Floreal Cabanettes committed
796

797
        files = request.files[list(request.files.keys())[0]]
Floreal Cabanettes's avatar
Floreal Cabanettes committed
798

799
800
801
802
803
804
805
        if files:
            filename = files.filename
            folder_files = os.path.join(app.config["UPLOAD_FOLDER"], folder)
            if not os.path.exists(folder_files):
                os.makedirs(folder_files)
            filename = Functions.get_valid_uploaded_filename(filename, folder_files)
            mime_type = files.content_type
Floreal Cabanettes's avatar
Floreal Cabanettes committed
806

Floreal Cabanettes's avatar
Floreal Cabanettes committed
807
            if not Functions.allowed_file(files.filename, request.form['formats'].split(",")):
808
809
                result = UploadFile(name=filename, type_f=mime_type, size=0, not_allowed_msg="File type not allowed")
                shutil.rmtree(folder_files)
Floreal Cabanettes's avatar
Floreal Cabanettes committed
810

811
812
813
814
815
816
817
818
819
820
821
822
823
824
            else:
                # save file to disk
                uploaded_file_path = os.path.join(folder_files, filename)
                files.save(uploaded_file_path)

                # get file size after saving
                size = os.path.getsize(uploaded_file_path)

                # return json for js call back
                result = UploadFile(name=filename, type_f=mime_type, size=size)

            return jsonify({"files": [result.get_file()], "success": "OK"})

        return jsonify({"files": [], "success": "404", "message": "No file provided"})
825
    except:  # Except all possible exceptions to prevent crashes
Floreal Cabanettes's avatar
Floreal Cabanettes committed
826
        traceback.print_exc()
827
828
        return jsonify({"files": [], "success": "ERR", "message": "An unexpected error has occurred on upload. "
                                                                  "Please contact the support."})
829
830


831
832
833
834
835
836
@app.route("/send-mail/<id_res>", methods=['POST'])
def send_mail(id_res):
    allowed = False
    key_file = None
    if "key" in request.form:
        key = request.form["key"]
837
        res_dir = os.path.join(APP_DATA, id_res)
838
839
840
841
842
843
844
845
846
847
848
849
        key_file = os.path.join(res_dir, ".key")
        if os.path.exists(key_file):
            with open(key_file) as k_f:
                true_key = k_f.readline().strip("\n")
                allowed = key == true_key
    if allowed:
        os.remove(key_file)
        job_mng = JobManager(id_job=id_res, mailer=mailer)
        job_mng.set_inputs_from_res_dir()
        job_mng.send_mail()
        return "OK"
    else:
850
        abort(403)
851
852
853
854
855
856
857
858
859
860


@app.route("/delete/<id_res>", methods=['POST'])
def delete_job(id_res):
    job = JobManager(id_job=id_res)
    success, error = job.delete()
    return jsonify({
        "success": success,
        "error": error
    })