views.py 32.5 KB
Newer Older
1
from dgenies import app, app_title, app_folder, config_reader, mailer, APP_DATA, MODE, DEBUG, VERSION
2

Floreal Cabanettes's avatar
Floreal Cabanettes committed
3
import os
4
5
import time
import datetime
6
import shutil
7
import re
8
import threading
Floreal Cabanettes's avatar
Floreal Cabanettes committed
9
import traceback
10
import requests
11
from requests.exceptions import ConnectionError
12
import json
Floreal Cabanettes's avatar
Floreal Cabanettes committed
13
from flask import render_template, request, url_for, jsonify, Response, abort, send_file, Markup
14
from pathlib import Path
15
16
17
18
19
from dgenies.lib.paf import Paf
from dgenies.lib.job_manager import JobManager
from dgenies.lib.functions import Functions, ALLOWED_EXTENSIONS
from dgenies.lib.upload_file import UploadFile
from dgenies.lib.fasta import Fasta
20
from dgenies.lib.latest import Latest
21
from dgenies.tools import Tools
Floreal Cabanettes's avatar
Floreal Cabanettes committed
22
23
from markdown import Markdown
from markdown.extensions.toc import TocExtension
24
from markdown.extensions.tables import TableExtension
25
import tarfile
26
from jinja2 import Environment
27
28
29
if MODE == "webserver":
    from dgenies.database import Session, Gallery
    from peewee import DoesNotExist
30

31

Floreal Cabanettes's avatar
Floreal Cabanettes committed
32
33
34
35
@app.context_processor
def global_templates_variables():
    return {
        "title": app_title,
36
        "mode": MODE,
37
38
        "all_jobs": Functions.get_list_all_jobs(MODE),
        "debug": DEBUG
Floreal Cabanettes's avatar
Floreal Cabanettes committed
39
40
41
    }


42
43
44
# Main
@app.route("/", methods=['GET'])
def main():
45
46
47
48
49
50
    if MODE == "webserver":
        pict = Gallery.select().order_by("id")
        if len(pict) > 0:
            pict = pict[0].picture
        else:
            pict = None
Floreal Cabanettes's avatar
Floreal Cabanettes committed
51
52
    else:
        pict = None
Floreal Cabanettes's avatar
Floreal Cabanettes committed
53
    return render_template("index.html", menu="index", pict=pict)
Floreal Cabanettes's avatar
Floreal Cabanettes committed
54
55
56
57


@app.route("/run", methods=['GET'])
def run():
58
59
60
61
62
    tools = Tools().tools
    tools_names = sorted(list(tools.keys()), key=lambda x: (tools[x].order, tools[x].name))
    tools_ava = {}
    for tool_name, tool in tools.items():
        tools_ava[tool_name] = 1 if tool.all_vs_all is not None else 0
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
    if MODE == "webserver":
        with Session.connect():
            s_id = Session.new()
    else:
        upload_folder = Functions.random_string(20)
        tmp_dir = config_reader.upload_folder
        upload_folder_path = os.path.join(tmp_dir, upload_folder)
        while os.path.exists(upload_folder_path):
            upload_folder = Functions.random_string(20)
            upload_folder_path = os.path.join(tmp_dir, upload_folder)
        s_id = upload_folder
    id_job = Functions.random_string(5) + "_" + datetime.datetime.fromtimestamp(time.time()).strftime('%Y%m%d%H%M%S')
    if "id_job" in request.args:
        id_job = request.args["id_job"]
    email = ""
    if "email" in request.args:
        email = request.args["email"]
Floreal Cabanettes's avatar
Floreal Cabanettes committed
80
    return render_template("run.html", id_job=id_job, email=email,
81
                           menu="run", allowed_ext=ALLOWED_EXTENSIONS, s_id=s_id,
82
83
84
                           max_upload_file_size=config_reader.max_upload_file_size,
                           example=config_reader.example_target != "",
                           target=os.path.basename(config_reader.example_target),
85
                           query=os.path.basename(config_reader.example_query), tools_names=tools_names, tools=tools,
86
                           tools_ava=tools_ava, version=VERSION)
87
88


89
90
@app.route("/run-test", methods=['GET'])
def run_test():
91
92
93
94
95
96
97
    if MODE == "webserver":
        print(config_reader.allowed_ip_tests)
        if request.remote_addr not in config_reader.allowed_ip_tests:
            return abort(404)
        with Session.connect():
            return Session.new()
    return abort(500)
98
99


100
101
102
# Launch analysis
@app.route("/launch_analysis", methods=['POST'])
def launch_analysis():
103
104
105
106
107
108
109
110
111
112
113
114
    if MODE == "webserver":
        try:
            with Session.connect():
                session = Session.get(s_id=request.form["s_id"])
        except DoesNotExist:
            return jsonify({"success": False, "errors": ["Session has expired. Please refresh the page and try again"]})
        upload_folder = session.upload_folder
        # Delete session:
        session.delete_instance()
    else:
        upload_folder = request.form["s_id"]

115
116
    id_job = request.form["id_job"]
    email = request.form["email"]
117
118
119
120
    file_query = request.form["query"]
    file_query_type = request.form["query_type"]
    file_target = request.form["target"]
    file_target_type = request.form["target_type"]
121
122
123
    tool = request.form["tool"] if "tool" in request.form else None
    alignfile = request.form["alignfile"] if "alignfile" in request.form else None
    alignfile_type = request.form["alignfile_type"] if "alignfile_type" in request.form else None
124
125
    backup = request.form["backup"] if "backup" in request.form else None
    backup_type = request.form["backup_type"] if "backup_type" in request.form else None
126
127
128

    # Check form:
    form_pass = True
129
    errors = []
130

131
132
133
134
    if alignfile is not None and alignfile_type is None:
        errors.append("Server error: no alignfile_type in form. Please contact the support")
        form_pass = False

Floreal Cabanettes's avatar
Floreal Cabanettes committed
135
    if backup is not None and backup != "" and (backup_type is None or backup_type == ""):
136
137
138
        errors.append("Server error: no backup_type in form. Please contact the support")
        form_pass = False

Floreal Cabanettes's avatar
Floreal Cabanettes committed
139
    if backup is not None and backup != "":
140
141
142
143
        alignfile = ""
        file_query = ""
        file_target = ""
    else:
Floreal Cabanettes's avatar
Floreal Cabanettes committed
144
        backup = None
145
146
147
148
        if file_target == "":
            errors.append("No target fasta selected")
            form_pass = False

149
    if tool is not None and tool not in Tools().tools:
150
        errors.append("Tool unavailable: %s" % tool)
151
        form_pass = False
152

153
    if id_job == "":
154
        errors.append("Id of job not given")
155
156
        form_pass = False

157
158
159
160
    if MODE == "webserver":
        if email == "":
            errors.append("Email not given")
            form_pass = False
Floreal Cabanettes's avatar
Floreal Cabanettes committed
161
        elif not re.match(r"^[\w.\-]+@[\w\-.]{2,}\.[a-z]{2,4}$", email):
162
163
            errors.append("Email is invalid")
            form_pass = False
164
165
166

    # Form pass
    if form_pass:
167
        # Get final job id:
168
        id_job = re.sub('[^A-Za-z0-9_\-]+', '', id_job.replace(" ", "_"))
169
        id_job_orig = id_job
170
        i = 2
171
        while os.path.exists(os.path.join(APP_DATA, id_job)):
172
173
            id_job = id_job_orig + ("_%d" % i)
            i += 1
174

175
        folder_files = os.path.join(APP_DATA, id_job)
176
177
178
        os.makedirs(folder_files)

        # Save files:
179
180
        query = None
        if file_query != "":
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
            example = False
            if file_query.startswith("example://"):
                example = True
                query_path = config_reader.example_query
                query_name = os.path.basename(query_path)
                file_query_type = "local"
            else:
                query_name = os.path.splitext(file_query.replace(".gz", ""))[0] if file_query_type == "local" else None
                query_path = os.path.join(app.config["UPLOAD_FOLDER"], upload_folder, file_query) \
                    if file_query_type == "local" else file_query
                if file_query_type == "local" and not os.path.exists(query_path):
                    errors.append("Query file not correct!")
                    form_pass = False
            query = Fasta(name=query_name, path=query_path, type_f=file_query_type, example=example)
        example = False
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
        target = None
        if file_target != "":
            if file_target.startswith("example://"):
                example = True
                target_path = config_reader.example_target
                target_name = os.path.basename(target_path)
                file_target_type = "local"
            else:
                target_name = os.path.splitext(file_target.replace(".gz", ""))[0] if file_target_type == "local" else None
                target_path = os.path.join(app.config["UPLOAD_FOLDER"], upload_folder, file_target) \
                    if file_target_type == "local" else file_target
                if file_target_type == "local" and not os.path.exists(target_path):
                    errors.append("Target file not correct!")
                    form_pass = False
            target = Fasta(name=target_name, path=target_path, type_f=file_target_type, example=example)
211

212
        if alignfile is not None and alignfile != "" and backup is not None:
213
            Path(os.path.join(folder_files, ".align")).touch()
214
215
216

        align = None
        if alignfile is not None and alignfile != "":
217
218
219
220
221
222
223
224
            alignfile_name = os.path.splitext(alignfile)[0] if alignfile_type == "local" else None
            alignfile_path = os.path.join(app.config["UPLOAD_FOLDER"], upload_folder, alignfile) \
                if alignfile_type == "local" else alignfile
            if alignfile_type == "local" and not os.path.exists(alignfile_path):
                errors.append("Alignment file not correct!")
                form_pass = False
            align = Fasta(name=alignfile_name, path=alignfile_path, type_f=alignfile_type)

225
226
227
228
229
230
231
232
233
234
        bckp = None
        if backup is not None:
            backup_name = os.path.splitext(backup)[0] if backup_type == "local" else None
            backup_path = os.path.join(app.config["UPLOAD_FOLDER"], upload_folder, backup) \
                if backup_type == "local" else backup
            if backup_type == "local" and not os.path.exists(backup_path):
                errors.append("Backup file not correct!")
                form_pass = False
            bckp = Fasta(name=backup_name, path=backup_path, type_f=backup_type)

235
236
        if form_pass:
            # Launch job:
237
238
239
240
241
            job = JobManager(id_job=id_job,
                             email=email,
                             query=query,
                             target=target,
                             align=align,
242
                             backup=bckp,
243
244
                             mailer=mailer,
                             tool=tool)
245
246
247
248
249
250
            if MODE == "webserver":
                job.launch()
            else:
                job.launch_standalone()
            return jsonify({"success": True, "redirect": url_for(".status", id_job=id_job)})
    if not form_pass:
251
        return jsonify({"success": False, "errors": errors})
252
253
254
255
256
257


# Status of a job
@app.route('/status/<id_job>', methods=['GET'])
def status(id_job):
    job = JobManager(id_job)
258
259
260
261
262
263
264
265
266
267
268
269
    j_status = job.status()
    mem_peak = j_status["mem_peak"] if "mem_peak" in j_status else None
    if mem_peak is not None:
        mem_peak = "%.1f G" % (mem_peak / 1024.0 / 1024.0)
    time_e = j_status["time_elapsed"] if "time_elapsed" in j_status else None
    if time_e is not None:
        if time_e < 60:
            time_e = "%d secs" % time_e
        else:
            minutes = time_e // 60
            seconds = time_e - minutes * 60
            time_e = "%d min %d secs" % (minutes, seconds)
270
271
272
273
274
275
276
277
278
    format = request.args.get("format")
    if format is not None and format == "json":
        return jsonify({
            "status": j_status["status"],
            "error": j_status["error"].replace("#ID#", ""),
            "id_job": id_job,
            "mem_peak": mem_peak,
            "time_elapsed": time_e
        })
Floreal Cabanettes's avatar
Floreal Cabanettes committed
279
    return render_template("status.html", status=j_status["status"],
280
281
                           error=j_status["error"].replace("#ID#", ""),
                           id_job=id_job, menu="results", mem_peak=mem_peak,
282
                           time_elapsed=time_e, do_align=job.do_align(),
Floreal Cabanettes's avatar
Floreal Cabanettes committed
283
                           query_filtered=job.is_query_filtered(), target_filtered=job.is_target_filtered())
284
285


Floreal Cabanettes's avatar
Floreal Cabanettes committed
286
# Results path
287
@app.route("/result/<id_res>", methods=['GET'])
288
def result(id_res):
289
    res_dir = os.path.join(APP_DATA, id_res)
Floreal Cabanettes's avatar
Floreal Cabanettes committed
290
    return render_template("result.html", id=id_res, menu="result", current_result=id_res,
291
292
                           is_gallery=Functions.is_in_gallery(id_res, MODE),
                           fasta_file=Functions.query_fasta_file_exists(res_dir))
293
294


295
296
@app.route("/gallery", methods=['GET'])
def gallery():
297
    if MODE == "webserver":
Floreal Cabanettes's avatar
Floreal Cabanettes committed
298
        return render_template("gallery.html", items=Functions.get_gallery_items(), menu="gallery")
299
    return abort(404)
300
301
302
303


@app.route("/gallery/<filename>", methods=['GET'])
def gallery_file(filename):
304
305
306
307
308
309
    if MODE == "webserver":
        try:
            return send_file(os.path.join(config_reader.app_data, "gallery", filename))
        except FileNotFoundError:
            abort(404)
    return abort(404)
310
311


312
def get_file(file, gzip=False):  # pragma: no cover
Floreal Cabanettes's avatar
Floreal Cabanettes committed
313
314
315
316
317
318
    try:
        # Figure out how flask returns static files
        # Tried:
        # - render_template
        # - send_file
        # This should not be so non-obvious
319
        return open(file, "rb" if gzip else "r").read()
320
321
    except FileNotFoundError:
        abort(404)
Floreal Cabanettes's avatar
Floreal Cabanettes committed
322
    except IOError as exc:
323
        print(exc.__traceback__)
324
        abort(500)
Floreal Cabanettes's avatar
Floreal Cabanettes committed
325
326


Floreal Cabanettes's avatar
Floreal Cabanettes committed
327
328
@app.route("/documentation/run", methods=['GET'])
def documentation_run():
329
330
    latest = Latest()
    version = latest.latest
331
    with open(os.path.join(app_folder, "md", "doc_run.md"), "r",  encoding='utf-8') as install_instr:
Floreal Cabanettes's avatar
Floreal Cabanettes committed
332
        content = install_instr.read()
333
334
335
    env = Environment()
    template = env.from_string(content)
    content = template.render(mode=MODE, version=version)
Floreal Cabanettes's avatar
Floreal Cabanettes committed
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
    md = Markdown(extensions=[TocExtension(baselevel=1)])
    max_upload_file_size = config_reader.max_upload_file_size
    if max_upload_file_size == -1:
        max_upload_file_size = "no limit"
    else:
        max_upload_file_size = Functions.get_readable_size(max_upload_file_size, 0)
    max_upload_size = config_reader.max_upload_size
    if max_upload_size == -1:
        max_upload_size = "no limit"
    else:
        max_upload_size = Functions.get_readable_size(max_upload_size, 0)
    max_upload_size_ava = config_reader.max_upload_size_ava
    if max_upload_size_ava == -1:
        max_upload_size_ava = "no limit"
    else:
        max_upload_size_ava = Functions.get_readable_size(max_upload_size_ava, 0)
    content = Markup(md.convert(content)).replace("###size###", max_upload_file_size)\
                                         .replace("###size_unc###", max_upload_size)\
                                         .replace("###size_ava###", max_upload_size_ava)
    toc = Markup(md.toc)
Floreal Cabanettes's avatar
Floreal Cabanettes committed
356
    return render_template("documentation.html", menu="documentation", content=content, toc=toc)
Floreal Cabanettes's avatar
Floreal Cabanettes committed
357
358


359
360
@app.route("/documentation/result", methods=['GET'])
def documentation_result():
361
    with open(os.path.join(app_folder, "md", "user_manual.md"), "r",
362
363
364
365
366
367
368
369
              encoding='utf-8') as install_instr:
        content = install_instr.read()
    md = Markdown(extensions=[TocExtension(baselevel=1)])
    content = Markup(md.convert(content))
    toc = Markup(md.toc)
    return render_template("documentation.html", menu="documentation", content=content, toc=toc)


370
371
@app.route("/documentation/formats", methods=['GET'])
def documentation_formats():
372
    with open(os.path.join(app_folder, "md", "doc_formats.md"), "r",
373
374
375
376
377
378
379
380
              encoding='utf-8') as install_instr:
        content = install_instr.read()
    md = Markdown(extensions=[TocExtension(baselevel=1), TableExtension()])
    content = Markup(md.convert(content))
    toc = Markup(md.toc)
    return render_template("documentation.html", menu="documentation", content=content, toc=toc)


381
382
383
384
385
386
387
388
389
390
391
@app.route("/documentation/dotplot", methods=['GET'])
def documentation_dotplot():
    with open(os.path.join(app_folder, "md", "doc_dotplot.md"), "r",
              encoding='utf-8') as install_instr:
        content = install_instr.read()
    md = Markdown(extensions=[TocExtension(baselevel=1)])
    content = Markup(md.convert(content))
    toc = Markup(md.toc)
    return render_template("documentation.html", menu="documentation", content=content, toc=toc)


Floreal Cabanettes's avatar
Floreal Cabanettes committed
392
393
@app.route("/install", methods=['GET'])
def install():
394
    latest = Latest()
395

396
    with open(os.path.join(app_folder, "md", "INSTALL.md"), "r", encoding='utf-8') as install_instr:
Floreal Cabanettes's avatar
Floreal Cabanettes committed
397
        content = install_instr.read()
398
399
    env = Environment()
    template = env.from_string(content)
400
    content = template.render(version=latest.latest, win32=latest.win32)
Floreal Cabanettes's avatar
Floreal Cabanettes committed
401
402
403
    md = Markdown(extensions=[TocExtension(baselevel=1)])
    content = Markup(md.convert(content))
    toc = Markup(md.toc)
Floreal Cabanettes's avatar
Floreal Cabanettes committed
404
    return render_template("documentation.html", menu="install", content=content, toc=toc)
Floreal Cabanettes's avatar
Floreal Cabanettes committed
405

Floreal Cabanettes's avatar
Floreal Cabanettes committed
406

407
408
409
410
@app.route("/contact", methods=['GET'])
def contact():
    return render_template("contact.html", menu="contact")

Floreal Cabanettes's avatar
Floreal Cabanettes committed
411

Floreal Cabanettes's avatar
Floreal Cabanettes committed
412
413
414
415
416
417
418
419
420
421
422
@app.route("/paf/<id_res>", methods=['GET'])
def download_paf(id_res):
    map_file = os.path.join(APP_DATA, id_res, "map.paf.sorted")
    if not os.path.exists(map_file):
        map_file = os.path.join(APP_DATA, id_res, "map.paf")
    if not os.path.exists(map_file):
        abort(404)
    content = get_file(map_file)
    return Response(content, mimetype="text/plain")


Floreal Cabanettes's avatar
Floreal Cabanettes committed
423
# Get graph (ajax request)
424
425
426
@app.route('/get_graph', methods=['POST'])
def get_graph():
    id_f = request.form["id"]
427
428
429
    paf = os.path.join(APP_DATA, id_f, "map.paf")
    idx1 = os.path.join(APP_DATA, id_f, "query.idx")
    idx2 = os.path.join(APP_DATA, id_f, "target.idx")
430

431
    paf = Paf(paf, idx1, idx2)
432

433
434
    if paf.parsed:
        res = paf.get_d3js_data()
435
        res["success"] = True
436
437
438
439
440
441
        return jsonify(res)
    return jsonify({"success": False, "message": paf.error})


@app.route('/sort/<id_res>', methods=['POST'])
def sort_graph(id_res):
442
    if not os.path.exists(os.path.join(APP_DATA, id_res, ".all-vs-all")):
443
444
445
        paf_file = os.path.join(APP_DATA, id_res, "map.paf")
        idx1 = os.path.join(APP_DATA, id_res, "query.idx")
        idx2 = os.path.join(APP_DATA, id_res, "target.idx")
446
        paf = Paf(paf_file, idx1, idx2, False)
447
448
449
450
451
452
453
        paf.sort()
        if paf.parsed:
            res = paf.get_d3js_data()
            res["success"] = True
            return jsonify(res)
        return jsonify({"success": False, "message": paf.error})
    return jsonify({"success": False, "message": "Sort is not available for All-vs-All mode"})
Floreal Cabanettes's avatar
Floreal Cabanettes committed
454

Floréal Cabanettes's avatar
Floréal Cabanettes committed
455

456
457
458
459
@app.route('/reverse-contig/<id_res>', methods=['POST'])
def reverse_contig(id_res):
    contig_name = request.form["contig"]
    if not os.path.exists(os.path.join(APP_DATA, id_res, ".all-vs-all")):
460
461
462
        paf_file = os.path.join(APP_DATA, id_res, "map.paf")
        idx1 = os.path.join(APP_DATA, id_res, "query.idx")
        idx2 = os.path.join(APP_DATA, id_res, "target.idx")
463
464
465
466
467
468
469
470
471
472
        paf = Paf(paf_file, idx1, idx2, False)
        paf.reverse_contig(contig_name)
        if paf.parsed:
            res = paf.get_d3js_data()
            res["success"] = True
            return jsonify(res)
        return jsonify({"success": False, "message": paf.error})
    return jsonify({"success": False, "message": "Sort is not available for All-vs-All mode"})


473
474
@app.route('/freenoise/<id_res>', methods=['POST'])
def free_noise(id_res):
475
476
477
    paf_file = os.path.join(APP_DATA, id_res, "map.paf")
    idx1 = os.path.join(APP_DATA, id_res, "query.idx")
    idx2 = os.path.join(APP_DATA, id_res, "target.idx")
478
479
480
481
482
483
484
485
486
    paf = Paf(paf_file, idx1, idx2, False)
    paf.parse_paf(noise=request.form["noise"] == "1")
    if paf.parsed:
        res = paf.get_d3js_data()
        res["success"] = True
        return jsonify(res)
    return jsonify({"success": False, "message": paf.error})


487
488
@app.route('/get-fasta-query/<id_res>', methods=['POST'])
def build_fasta(id_res):
489
    res_dir = os.path.join(APP_DATA, id_res)
490
491
    lock_query = os.path.join(res_dir, ".query-fasta-build")
    is_sorted = os.path.exists(os.path.join(res_dir, ".sorted"))
492
    compressed = request.form["gzip"].lower() == "true"
493
494
495
496
497
    query_fasta = Functions.get_fasta_file(res_dir, "query", is_sorted)
    if query_fasta is not None:
        if is_sorted and not query_fasta.endswith(".sorted"):
            # Do the sort
            Path(lock_query).touch()
498
            if not compressed or MODE == "standalone":  # If compressed, it will took a long time, so not wait
499
                Path(lock_query + ".pending").touch()
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
            index_file = os.path.join(res_dir, "query.idx.sorted")
            if MODE == "webserver":
                thread = threading.Timer(1, Functions.sort_fasta, kwargs={
                    "job_name": id_res,
                    "fasta_file": query_fasta,
                    "index_file": index_file,
                    "lock_file": lock_query,
                    "compress": compressed,
                    "mailer": mailer,
                    "mode": MODE
                })
                thread.start()
            else:
                Functions.sort_fasta(job_name=id_res,
                                     fasta_file=query_fasta,
                                     index_file=index_file,
                                     lock_file=lock_query,
                                     compress=compressed,
                                     mailer=None,
                                     mode=MODE)
520
            if not compressed or MODE == "standalone":
521
522
                if MODE == "webserver":
                    i = 0
523
                    time.sleep(5)
524
525
526
                    while os.path.exists(lock_query) and (i < 2 or MODE == "standalone"):
                        i += 1
                        time.sleep(5)
Floreal Cabanettes's avatar
Floreal Cabanettes committed
527
                os.remove(lock_query + ".pending")
528
529
530
                if os.path.exists(lock_query):
                    return jsonify({"success": True, "status": 1, "status_message": "In progress"})
                return jsonify({"success": True, "status": 2, "status_message": "Done",
531
                                "gzip": compressed})
532
533
            else:
                return jsonify({"success": True, "status": 1, "status_message": "In progress"})
534
535
536
537
538
        elif is_sorted and os.path.exists(lock_query):
            # Sort is already in progress
            return jsonify({"success": True, "status": 1, "status_message": "In progress"})
        else:
            # No sort to do or sort done
539
540
541
542
543
544
545
            if compressed and not query_fasta.endswith(".gz.fasta"):
                # If compressed file is asked, we must compress it now if not done before...
                Path(lock_query).touch()
                thread = threading.Timer(1, Functions.compress_and_send_mail, kwargs={
                    "job_name": id_res,
                    "fasta_file": query_fasta,
                    "index_file": os.path.join(res_dir, "query.idx.sorted"),
546
                    "lock_file": lock_query,
547
548
549
550
551
                    "compressed": compressed,
                    "mailer": mailer
                })
                thread.start()
                return jsonify({"success": True, "status": 1, "status_message": "In progress"})
552
553
554
555
556
557
558
            return jsonify({"success": True, "status": 2, "status_message": "Done",
                            "gzip": query_fasta.endswith(".gz") or query_fasta.endswith(".gz.sorted")})
    else:
        return jsonify({"success": False,
                        "message": "Unable to get fasta file for query. Please contact us to report the bug"})


559
def build_query_as_reference(id_res):
560
561
562
563
564
    paf_file = os.path.join(APP_DATA, id_res, "map.paf")
    idx1 = os.path.join(APP_DATA, id_res, "query.idx")
    idx2 = os.path.join(APP_DATA, id_res, "target.idx")
    paf = Paf(paf_file, idx1, idx2, False, mailer=mailer, id_job=id_res)
    paf.parse_paf(False, True)
565
566
567
568
569
570
571
572
573
574
    if MODE == "webserver":
        thread = threading.Timer(0, paf.build_query_chr_as_reference)
        thread.start()
        return True
    return paf.build_query_chr_as_reference()


@app.route('/build-query-as-reference/<id_res>', methods=['POST'])
def post_query_as_reference(id_res):
    build_query_as_reference(id_res)
575
576
577
    return jsonify({"success": True})


578
579
580
581
582
583
584
@app.route('/get-query-as-reference/<id_res>', methods=['GET'])
def get_query_as_reference(id_res):
    if MODE != "standalone":
        return abort(404)
    return send_file(build_query_as_reference(id_res))


585
586
587
588
589
590
591
592
@app.route('/download/<id_res>/<filename>')
def download_file(id_res, filename):
    file_dl = os.path.join(APP_DATA, id_res, filename)
    if os.path.isfile(file_dl):
        return send_file(file_dl)
    return abort(404)


593
594
595
@app.route('/fasta-query/<id_res>', defaults={'filename': ""}, methods=['GET'])
@app.route('/fasta-query/<id_res>/<filename>', methods=['GET'])  # Use fake URL in mail to set download file name
def dl_fasta(id_res, filename):
596
    res_dir = os.path.join(APP_DATA, id_res)
597
598
599
600
601
602
603
604
605
606
607
608
609
    lock_query = os.path.join(res_dir, ".query-fasta-build")
    is_sorted = os.path.exists(os.path.join(res_dir, ".sorted"))
    if not os.path.exists(lock_query) or not is_sorted:
        query_fasta = Functions.get_fasta_file(res_dir, "query", is_sorted)
        if query_fasta is not None:
            if query_fasta.endswith(".gz") or query_fasta.endswith(".gz.sorted"):
                content = get_file(query_fasta, True)
                return Response(content, mimetype="application/gzip")
            content = get_file(query_fasta)
            return Response(content, mimetype="text/plain")
    abort(404)


610
611
@app.route('/qt-assoc/<id_res>', methods=['GET'])
def qt_assoc(id_res):
612
    res_dir = os.path.join(APP_DATA, id_res)
613
    if os.path.exists(res_dir) and os.path.isdir(res_dir):
614
615
616
        paf_file = os.path.join(APP_DATA, id_res, "map.paf")
        idx1 = os.path.join(APP_DATA, id_res, "query.idx")
        idx2 = os.path.join(APP_DATA, id_res, "target.idx")
617
618
619
620
621
622
623
624
625
626
627
628
        try:
            paf = Paf(paf_file, idx1, idx2, False)
            paf.parse_paf(False)
        except FileNotFoundError:
            print("Unable to load data!")
            abort(404)
            return False
        csv_content = paf.build_query_on_target_association_file()
        return Response(csv_content, mimetype="text/plain")
    abort(404)


629
630
631
632
633
634
@app.route('/no-assoc/<id_res>', methods=['POST'])
def no_assoc(id_res):
    """
    Get contigs that match with None target
    :param id_res: id of the result
    """
635
    res_dir = os.path.join(APP_DATA, id_res)
636
    if os.path.exists(res_dir) and os.path.isdir(res_dir):
637
638
639
        paf_file = os.path.join(APP_DATA, id_res, "map.paf")
        idx1 = os.path.join(APP_DATA, id_res, "query.idx")
        idx2 = os.path.join(APP_DATA, id_res, "target.idx")
640
641
642
643
644
645
        try:
            paf = Paf(paf_file, idx1, idx2, False)
        except FileNotFoundError:
            print("Unable to load data!")
            abort(404)
            return False
646
        file_content = paf.build_list_no_assoc(request.form["to"])
647
648
649
650
651
652
653
654
        empty = file_content == "\n"
        return jsonify({
            "file_content": file_content,
            "empty": empty
        })
    abort(404)


Floreal Cabanettes's avatar
Floreal Cabanettes committed
655
656
@app.route('/summary/<id_res>', methods=['POST'])
def summary(id_res):
657
658
659
660
661
662
663
664
665
666
    paf_file = os.path.join(APP_DATA, id_res, "map.paf")
    idx1 = os.path.join(APP_DATA, id_res, "query.idx")
    idx2 = os.path.join(APP_DATA, id_res, "target.idx")
    try:
        paf = Paf(paf_file, idx1, idx2, False)
    except FileNotFoundError:
        return jsonify({
            "success": False,
            "message": "Unable to load data!"
        })
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
    percents = None
    s_status = "waiting"  # Accepted values: waiting, done, fail
    status_file = os.path.join(APP_DATA, id_res, ".summarize")
    fail_file = status_file + ".fail"
    if not os.path.exists(status_file):  # The job is finished or not started
        if not os.path.exists(fail_file):  # The job has not started yet or has successfully ended
            percents = paf.get_summary_stats()
            if percents is None:  # The job has not started yet
                Path(status_file).touch()
                thread = threading.Timer(0, paf.build_summary_stats, kwargs={"status_file": status_file})
                thread.start()
            else:  # The job has successfully ended
                s_status = "done"
        else:  # The job has failed
            s_status = "fail"

    if s_status == "waiting":  # The job is running
        # Check if the job end in the next 30 seconds
        nb_iter = 0
        while os.path.exists(status_file) and not os.path.exists(fail_file) and nb_iter < 10:
            time.sleep(3)
            nb_iter += 1
        if not os.path.exists(status_file):  # The job has ended
            percents = paf.get_summary_stats()
            if percents is None:  # The job has failed
                s_status = "fail"
            else:  # The job has successfully ended
                s_status = "done"

    if s_status == "fail":
697
698
699
700
701
702
        return jsonify({
            "success": False,
            "message": "Build of summary failed. Please contact us to report the bug"
        })
    return jsonify({
        "success": True,
703
704
        "percents": percents,
        "status": s_status
705
    })
Floreal Cabanettes's avatar
Floreal Cabanettes committed
706
707


708
709
710
711
712
713
714
715
716
717
@app.route('/backup/<id_res>')
def get_backup_file(id_res):
    res_dir = os.path.join(APP_DATA, id_res)
    tar = os.path.join(res_dir, "%s.tar" % id_res)
    with tarfile.open(tar, "w") as tarf:
        for file in ("map.paf", "target.idx", "query.idx"):
            tarf.add(os.path.join(res_dir, file), arcname=file)
    return send_file(tar)


718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
def get_filter_out(id_res, type_f):
    filter_file = os.path.join(APP_DATA, id_res, ".filter-" + type_f)
    return Response(get_file(filter_file), mimetype="text/plain")


@app.route('/filter-out/<id_res>/query')
def get_filter_out_query(id_res):
    return get_filter_out(id_res=id_res, type_f="query")


@app.route('/filter-out/<id_res>/target')
def get_filter_out_target(id_res):
    return get_filter_out(id_res=id_res, type_f="target")


733
734
@app.route("/ask-upload", methods=['POST'])
def ask_upload():
735
736
737
738
739
    if MODE == "standalone":
        return jsonify({
            "success": True,
            "allowed": True
        })
740
741
    try:
        s_id = request.form['s_id']
Floreal Cabanettes's avatar
Floreal Cabanettes committed
742
743
744
        with Session.connect():
            session = Session.get(s_id=s_id)
            allowed = session.ask_for_upload(True)
745
746
        return jsonify({
            "success": True,
Floreal Cabanettes's avatar
Floreal Cabanettes committed
747
            "allowed": allowed
748
749
750
        })
    except DoesNotExist:
        return jsonify({"success": False, "message": "Session not initialized. Please refresh the page."})
751
752


753
754
@app.route("/ping-upload", methods=['POST'])
def ping_upload():
755
756
757
758
759
    if MODE == "webserver":
        s_id = request.form['s_id']
        with Session.connect():
            session = Session.get(s_id=s_id)
            session.ping()
760
    return "OK"
761
762


763
764
765
766
@app.route("/upload", methods=['POST'])
def upload():
    try:
        s_id = request.form['s_id']
767
768
769
770
771
772
        if MODE == "webserver":
            try:
                with Session.connect():
                    session = Session.get(s_id=s_id)
                    if session.ask_for_upload(False):
                        folder = session.upload_folder
Floreal Cabanettes's avatar
Floreal Cabanettes committed
773
                    else:
774
775
776
777
778
779
                        return jsonify({"files": [], "success": "ERR", "message": "Not allowed to upload!"})
            except DoesNotExist:
                return jsonify(
                    {"files": [], "success": "ERR", "message": "Session not initialized. Please refresh the page."})
        else:
            folder = s_id
Floreal Cabanettes's avatar
Floreal Cabanettes committed
780

781
        files = request.files[list(request.files.keys())[0]]
Floreal Cabanettes's avatar
Floreal Cabanettes committed
782

783
784
785
786
787
788
789
        if files:
            filename = files.filename
            folder_files = os.path.join(app.config["UPLOAD_FOLDER"], folder)
            if not os.path.exists(folder_files):
                os.makedirs(folder_files)
            filename = Functions.get_valid_uploaded_filename(filename, folder_files)
            mime_type = files.content_type
Floreal Cabanettes's avatar
Floreal Cabanettes committed
790

Floreal Cabanettes's avatar
Floreal Cabanettes committed
791
            if not Functions.allowed_file(files.filename, request.form['formats'].split(",")):
792
793
                result = UploadFile(name=filename, type_f=mime_type, size=0, not_allowed_msg="File type not allowed")
                shutil.rmtree(folder_files)
Floreal Cabanettes's avatar
Floreal Cabanettes committed
794

795
796
797
798
799
800
801
802
803
804
805
806
807
808
            else:
                # save file to disk
                uploaded_file_path = os.path.join(folder_files, filename)
                files.save(uploaded_file_path)

                # get file size after saving
                size = os.path.getsize(uploaded_file_path)

                # return json for js call back
                result = UploadFile(name=filename, type_f=mime_type, size=size)

            return jsonify({"files": [result.get_file()], "success": "OK"})

        return jsonify({"files": [], "success": "404", "message": "No file provided"})
809
    except:  # Except all possible exceptions to prevent crashes
Floreal Cabanettes's avatar
Floreal Cabanettes committed
810
        traceback.print_exc()
811
812
        return jsonify({"files": [], "success": "ERR", "message": "An unexpected error has occurred on upload. "
                                                                  "Please contact the support."})
813
814


815
816
817
818
819
820
@app.route("/send-mail/<id_res>", methods=['POST'])
def send_mail(id_res):
    allowed = False
    key_file = None
    if "key" in request.form:
        key = request.form["key"]
821
        res_dir = os.path.join(APP_DATA, id_res)
822
823
824
825
826
827
828
829
830
831
832
833
        key_file = os.path.join(res_dir, ".key")
        if os.path.exists(key_file):
            with open(key_file) as k_f:
                true_key = k_f.readline().strip("\n")
                allowed = key == true_key
    if allowed:
        os.remove(key_file)
        job_mng = JobManager(id_job=id_res, mailer=mailer)
        job_mng.set_inputs_from_res_dir()
        job_mng.send_mail()
        return "OK"
    else:
834
        abort(403)
835
836
837
838
839
840
841
842
843
844


@app.route("/delete/<id_res>", methods=['POST'])
def delete_job(id_res):
    job = JobManager(id_job=id_res)
    success, error = job.delete()
    return jsonify({
        "success": success,
        "error": error
    })