views.py 32.3 KB
Newer Older
1
from dgenies import app, app_title, app_folder, config_reader, mailer, APP_DATA, MODE, DEBUG, VERSION
2

Floreal Cabanettes's avatar
Floreal Cabanettes committed
3
import os
4
5
import time
import datetime
6
import shutil
7
import re
8
import threading
Floreal Cabanettes's avatar
Floreal Cabanettes committed
9
import traceback
Floreal Cabanettes's avatar
Floreal Cabanettes committed
10
from flask import render_template, request, url_for, jsonify, Response, abort, send_file, Markup
11
from pathlib import Path
12
13
14
15
16
from dgenies.lib.paf import Paf
from dgenies.lib.job_manager import JobManager
from dgenies.lib.functions import Functions, ALLOWED_EXTENSIONS
from dgenies.lib.upload_file import UploadFile
from dgenies.lib.fasta import Fasta
17
from dgenies.tools import Tools
Floreal Cabanettes's avatar
Floreal Cabanettes committed
18
19
from markdown import Markdown
from markdown.extensions.toc import TocExtension
20
from markdown.extensions.tables import TableExtension
21
import tarfile
22
from jinja2 import Environment
23
24
25
if MODE == "webserver":
    from dgenies.database import Session, Gallery
    from peewee import DoesNotExist
26

27

Floreal Cabanettes's avatar
Floreal Cabanettes committed
28
29
30
31
@app.context_processor
def global_templates_variables():
    return {
        "title": app_title,
32
        "mode": MODE,
33
34
        "all_jobs": Functions.get_list_all_jobs(MODE),
        "debug": DEBUG
Floreal Cabanettes's avatar
Floreal Cabanettes committed
35
36
37
    }


38
39
40
# Main
@app.route("/", methods=['GET'])
def main():
41
42
43
44
45
46
    if MODE == "webserver":
        pict = Gallery.select().order_by("id")
        if len(pict) > 0:
            pict = pict[0].picture
        else:
            pict = None
Floreal Cabanettes's avatar
Floreal Cabanettes committed
47
48
    else:
        pict = None
Floreal Cabanettes's avatar
Floreal Cabanettes committed
49
    return render_template("index.html", menu="index", pict=pict)
Floreal Cabanettes's avatar
Floreal Cabanettes committed
50
51
52
53


@app.route("/run", methods=['GET'])
def run():
54
55
56
57
58
    tools = Tools().tools
    tools_names = sorted(list(tools.keys()), key=lambda x: (tools[x].order, tools[x].name))
    tools_ava = {}
    for tool_name, tool in tools.items():
        tools_ava[tool_name] = 1 if tool.all_vs_all is not None else 0
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
    if MODE == "webserver":
        with Session.connect():
            s_id = Session.new()
    else:
        upload_folder = Functions.random_string(20)
        tmp_dir = config_reader.upload_folder
        upload_folder_path = os.path.join(tmp_dir, upload_folder)
        while os.path.exists(upload_folder_path):
            upload_folder = Functions.random_string(20)
            upload_folder_path = os.path.join(tmp_dir, upload_folder)
        s_id = upload_folder
    id_job = Functions.random_string(5) + "_" + datetime.datetime.fromtimestamp(time.time()).strftime('%Y%m%d%H%M%S')
    if "id_job" in request.args:
        id_job = request.args["id_job"]
    email = ""
    if "email" in request.args:
        email = request.args["email"]
Floreal Cabanettes's avatar
Floreal Cabanettes committed
76
    return render_template("run.html", id_job=id_job, email=email,
77
                           menu="run", allowed_ext=ALLOWED_EXTENSIONS, s_id=s_id,
78
79
80
                           max_upload_file_size=config_reader.max_upload_file_size,
                           example=config_reader.example_target != "",
                           target=os.path.basename(config_reader.example_target),
81
                           query=os.path.basename(config_reader.example_query), tools_names=tools_names, tools=tools,
82
                           tools_ava=tools_ava, version=VERSION)
83
84


85
86
@app.route("/run-test", methods=['GET'])
def run_test():
87
88
89
90
91
92
93
    if MODE == "webserver":
        print(config_reader.allowed_ip_tests)
        if request.remote_addr not in config_reader.allowed_ip_tests:
            return abort(404)
        with Session.connect():
            return Session.new()
    return abort(500)
94
95


96
97
98
# Launch analysis
@app.route("/launch_analysis", methods=['POST'])
def launch_analysis():
99
100
101
102
103
104
105
106
107
108
109
110
    if MODE == "webserver":
        try:
            with Session.connect():
                session = Session.get(s_id=request.form["s_id"])
        except DoesNotExist:
            return jsonify({"success": False, "errors": ["Session has expired. Please refresh the page and try again"]})
        upload_folder = session.upload_folder
        # Delete session:
        session.delete_instance()
    else:
        upload_folder = request.form["s_id"]

111
112
    id_job = request.form["id_job"]
    email = request.form["email"]
113
114
115
116
    file_query = request.form["query"]
    file_query_type = request.form["query_type"]
    file_target = request.form["target"]
    file_target_type = request.form["target_type"]
117
118
119
    tool = request.form["tool"] if "tool" in request.form else None
    alignfile = request.form["alignfile"] if "alignfile" in request.form else None
    alignfile_type = request.form["alignfile_type"] if "alignfile_type" in request.form else None
120
121
    backup = request.form["backup"] if "backup" in request.form else None
    backup_type = request.form["backup_type"] if "backup_type" in request.form else None
122
123
124

    # Check form:
    form_pass = True
125
    errors = []
126

127
128
129
130
    if alignfile is not None and alignfile_type is None:
        errors.append("Server error: no alignfile_type in form. Please contact the support")
        form_pass = False

Floreal Cabanettes's avatar
Floreal Cabanettes committed
131
    if backup is not None and backup != "" and (backup_type is None or backup_type == ""):
132
133
134
        errors.append("Server error: no backup_type in form. Please contact the support")
        form_pass = False

Floreal Cabanettes's avatar
Floreal Cabanettes committed
135
    if backup is not None and backup != "":
136
137
138
139
        alignfile = ""
        file_query = ""
        file_target = ""
    else:
Floreal Cabanettes's avatar
Floreal Cabanettes committed
140
        backup = None
141
142
143
144
        if file_target == "":
            errors.append("No target fasta selected")
            form_pass = False

145
    if tool is not None and tool not in Tools().tools:
146
        errors.append("Tool unavailable: %s" % tool)
147
        form_pass = False
148

149
    if id_job == "":
150
        errors.append("Id of job not given")
151
152
        form_pass = False

153
154
155
156
    if MODE == "webserver":
        if email == "":
            errors.append("Email not given")
            form_pass = False
Floreal Cabanettes's avatar
Floreal Cabanettes committed
157
        elif not re.match(r"^[\w.\-]+@[\w\-.]{2,}\.[a-z]{2,4}$", email):
158
159
            errors.append("Email is invalid")
            form_pass = False
160
161
162

    # Form pass
    if form_pass:
163
        # Get final job id:
164
        id_job = re.sub('[^A-Za-z0-9_\-]+', '', id_job.replace(" ", "_"))
165
        id_job_orig = id_job
166
        i = 2
167
        while os.path.exists(os.path.join(APP_DATA, id_job)):
168
169
            id_job = id_job_orig + ("_%d" % i)
            i += 1
170

171
        folder_files = os.path.join(APP_DATA, id_job)
172
173
174
        os.makedirs(folder_files)

        # Save files:
175
176
        query = None
        if file_query != "":
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
            example = False
            if file_query.startswith("example://"):
                example = True
                query_path = config_reader.example_query
                query_name = os.path.basename(query_path)
                file_query_type = "local"
            else:
                query_name = os.path.splitext(file_query.replace(".gz", ""))[0] if file_query_type == "local" else None
                query_path = os.path.join(app.config["UPLOAD_FOLDER"], upload_folder, file_query) \
                    if file_query_type == "local" else file_query
                if file_query_type == "local" and not os.path.exists(query_path):
                    errors.append("Query file not correct!")
                    form_pass = False
            query = Fasta(name=query_name, path=query_path, type_f=file_query_type, example=example)
        example = False
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
        target = None
        if file_target != "":
            if file_target.startswith("example://"):
                example = True
                target_path = config_reader.example_target
                target_name = os.path.basename(target_path)
                file_target_type = "local"
            else:
                target_name = os.path.splitext(file_target.replace(".gz", ""))[0] if file_target_type == "local" else None
                target_path = os.path.join(app.config["UPLOAD_FOLDER"], upload_folder, file_target) \
                    if file_target_type == "local" else file_target
                if file_target_type == "local" and not os.path.exists(target_path):
                    errors.append("Target file not correct!")
                    form_pass = False
            target = Fasta(name=target_name, path=target_path, type_f=file_target_type, example=example)
207

208
        if alignfile is not None and alignfile != "" and backup is not None:
209
            Path(os.path.join(folder_files, ".align")).touch()
210
211
212

        align = None
        if alignfile is not None and alignfile != "":
213
214
215
216
217
218
219
220
            alignfile_name = os.path.splitext(alignfile)[0] if alignfile_type == "local" else None
            alignfile_path = os.path.join(app.config["UPLOAD_FOLDER"], upload_folder, alignfile) \
                if alignfile_type == "local" else alignfile
            if alignfile_type == "local" and not os.path.exists(alignfile_path):
                errors.append("Alignment file not correct!")
                form_pass = False
            align = Fasta(name=alignfile_name, path=alignfile_path, type_f=alignfile_type)

221
222
223
224
225
226
227
228
229
230
        bckp = None
        if backup is not None:
            backup_name = os.path.splitext(backup)[0] if backup_type == "local" else None
            backup_path = os.path.join(app.config["UPLOAD_FOLDER"], upload_folder, backup) \
                if backup_type == "local" else backup
            if backup_type == "local" and not os.path.exists(backup_path):
                errors.append("Backup file not correct!")
                form_pass = False
            bckp = Fasta(name=backup_name, path=backup_path, type_f=backup_type)

231
232
        if form_pass:
            # Launch job:
233
234
235
236
237
            job = JobManager(id_job=id_job,
                             email=email,
                             query=query,
                             target=target,
                             align=align,
238
                             backup=bckp,
239
240
                             mailer=mailer,
                             tool=tool)
241
242
243
244
245
246
            if MODE == "webserver":
                job.launch()
            else:
                job.launch_standalone()
            return jsonify({"success": True, "redirect": url_for(".status", id_job=id_job)})
    if not form_pass:
247
        return jsonify({"success": False, "errors": errors})
248
249
250
251
252
253


# Status of a job
@app.route('/status/<id_job>', methods=['GET'])
def status(id_job):
    job = JobManager(id_job)
254
255
256
257
258
259
260
261
262
263
264
265
    j_status = job.status()
    mem_peak = j_status["mem_peak"] if "mem_peak" in j_status else None
    if mem_peak is not None:
        mem_peak = "%.1f G" % (mem_peak / 1024.0 / 1024.0)
    time_e = j_status["time_elapsed"] if "time_elapsed" in j_status else None
    if time_e is not None:
        if time_e < 60:
            time_e = "%d secs" % time_e
        else:
            minutes = time_e // 60
            seconds = time_e - minutes * 60
            time_e = "%d min %d secs" % (minutes, seconds)
266
267
268
269
270
271
272
273
274
    format = request.args.get("format")
    if format is not None and format == "json":
        return jsonify({
            "status": j_status["status"],
            "error": j_status["error"].replace("#ID#", ""),
            "id_job": id_job,
            "mem_peak": mem_peak,
            "time_elapsed": time_e
        })
Floreal Cabanettes's avatar
Floreal Cabanettes committed
275
    return render_template("status.html", status=j_status["status"],
276
277
                           error=j_status["error"].replace("#ID#", ""),
                           id_job=id_job, menu="results", mem_peak=mem_peak,
278
                           time_elapsed=time_e, do_align=job.do_align(),
Floreal Cabanettes's avatar
Floreal Cabanettes committed
279
                           query_filtered=job.is_query_filtered(), target_filtered=job.is_target_filtered())
280
281


Floreal Cabanettes's avatar
Floreal Cabanettes committed
282
# Results path
283
@app.route("/result/<id_res>", methods=['GET'])
284
def result(id_res):
285
    res_dir = os.path.join(APP_DATA, id_res)
Floreal Cabanettes's avatar
Floreal Cabanettes committed
286
    return render_template("result.html", id=id_res, menu="result", current_result=id_res,
287
288
                           is_gallery=Functions.is_in_gallery(id_res, MODE),
                           fasta_file=Functions.query_fasta_file_exists(res_dir))
289
290


291
292
@app.route("/gallery", methods=['GET'])
def gallery():
293
    if MODE == "webserver":
Floreal Cabanettes's avatar
Floreal Cabanettes committed
294
        return render_template("gallery.html", items=Functions.get_gallery_items(), menu="gallery")
295
    return abort(404)
296
297
298
299


@app.route("/gallery/<filename>", methods=['GET'])
def gallery_file(filename):
300
301
302
303
304
305
    if MODE == "webserver":
        try:
            return send_file(os.path.join(config_reader.app_data, "gallery", filename))
        except FileNotFoundError:
            abort(404)
    return abort(404)
306
307


308
def get_file(file, gzip=False):  # pragma: no cover
Floreal Cabanettes's avatar
Floreal Cabanettes committed
309
310
311
312
313
314
    try:
        # Figure out how flask returns static files
        # Tried:
        # - render_template
        # - send_file
        # This should not be so non-obvious
315
        return open(file, "rb" if gzip else "r").read()
316
317
    except FileNotFoundError:
        abort(404)
Floreal Cabanettes's avatar
Floreal Cabanettes committed
318
    except IOError as exc:
319
        print(exc.__traceback__)
320
        abort(500)
Floreal Cabanettes's avatar
Floreal Cabanettes committed
321
322


Floreal Cabanettes's avatar
Floreal Cabanettes committed
323
324
@app.route("/documentation/run", methods=['GET'])
def documentation_run():
325
    with open(os.path.join(app_folder, "md", "doc_run.md" if MODE == "webserver" else "doc_run_standalone.md"), "r",
Floreal Cabanettes's avatar
Floreal Cabanettes committed
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
              encoding='utf-8') as install_instr:
        content = install_instr.read()
    md = Markdown(extensions=[TocExtension(baselevel=1)])
    max_upload_file_size = config_reader.max_upload_file_size
    if max_upload_file_size == -1:
        max_upload_file_size = "no limit"
    else:
        max_upload_file_size = Functions.get_readable_size(max_upload_file_size, 0)
    max_upload_size = config_reader.max_upload_size
    if max_upload_size == -1:
        max_upload_size = "no limit"
    else:
        max_upload_size = Functions.get_readable_size(max_upload_size, 0)
    max_upload_size_ava = config_reader.max_upload_size_ava
    if max_upload_size_ava == -1:
        max_upload_size_ava = "no limit"
    else:
        max_upload_size_ava = Functions.get_readable_size(max_upload_size_ava, 0)
    content = Markup(md.convert(content)).replace("###size###", max_upload_file_size)\
                                         .replace("###size_unc###", max_upload_size)\
                                         .replace("###size_ava###", max_upload_size_ava)
    toc = Markup(md.toc)
Floreal Cabanettes's avatar
Floreal Cabanettes committed
348
    return render_template("documentation.html", menu="documentation", content=content, toc=toc)
Floreal Cabanettes's avatar
Floreal Cabanettes committed
349
350


351
352
@app.route("/documentation/result", methods=['GET'])
def documentation_result():
353
    with open(os.path.join(app_folder, "md", "user_manual.md"), "r",
354
355
356
357
358
359
360
361
              encoding='utf-8') as install_instr:
        content = install_instr.read()
    md = Markdown(extensions=[TocExtension(baselevel=1)])
    content = Markup(md.convert(content))
    toc = Markup(md.toc)
    return render_template("documentation.html", menu="documentation", content=content, toc=toc)


362
363
@app.route("/documentation/formats", methods=['GET'])
def documentation_formats():
364
    with open(os.path.join(app_folder, "md", "doc_formats.md"), "r",
365
366
367
368
369
370
371
372
              encoding='utf-8') as install_instr:
        content = install_instr.read()
    md = Markdown(extensions=[TocExtension(baselevel=1), TableExtension()])
    content = Markup(md.convert(content))
    toc = Markup(md.toc)
    return render_template("documentation.html", menu="documentation", content=content, toc=toc)


373
374
375
376
377
378
379
380
381
382
383
@app.route("/documentation/dotplot", methods=['GET'])
def documentation_dotplot():
    with open(os.path.join(app_folder, "md", "doc_dotplot.md"), "r",
              encoding='utf-8') as install_instr:
        content = install_instr.read()
    md = Markdown(extensions=[TocExtension(baselevel=1)])
    content = Markup(md.convert(content))
    toc = Markup(md.toc)
    return render_template("documentation.html", menu="documentation", content=content, toc=toc)


Floreal Cabanettes's avatar
Floreal Cabanettes committed
384
385
@app.route("/install", methods=['GET'])
def install():
386
    with open(os.path.join(app_folder, "md", "INSTALL.md"), "r", encoding='utf-8') as install_instr:
Floreal Cabanettes's avatar
Floreal Cabanettes committed
387
        content = install_instr.read()
388
389
390
    env = Environment()
    template = env.from_string(content)
    content = template.render(version=VERSION)
Floreal Cabanettes's avatar
Floreal Cabanettes committed
391
392
393
    md = Markdown(extensions=[TocExtension(baselevel=1)])
    content = Markup(md.convert(content))
    toc = Markup(md.toc)
Floreal Cabanettes's avatar
Floreal Cabanettes committed
394
    return render_template("documentation.html", menu="install", content=content, toc=toc)
Floreal Cabanettes's avatar
Floreal Cabanettes committed
395

396
397
398
399
@app.route("/contact", methods=['GET'])
def contact():
    return render_template("contact.html", menu="contact")

Floreal Cabanettes's avatar
Floreal Cabanettes committed
400

Floreal Cabanettes's avatar
Floreal Cabanettes committed
401
402
403
404
405
406
407
408
409
410
411
@app.route("/paf/<id_res>", methods=['GET'])
def download_paf(id_res):
    map_file = os.path.join(APP_DATA, id_res, "map.paf.sorted")
    if not os.path.exists(map_file):
        map_file = os.path.join(APP_DATA, id_res, "map.paf")
    if not os.path.exists(map_file):
        abort(404)
    content = get_file(map_file)
    return Response(content, mimetype="text/plain")


Floreal Cabanettes's avatar
Floreal Cabanettes committed
412
# Get graph (ajax request)
413
414
415
@app.route('/get_graph', methods=['POST'])
def get_graph():
    id_f = request.form["id"]
416
417
418
    paf = os.path.join(APP_DATA, id_f, "map.paf")
    idx1 = os.path.join(APP_DATA, id_f, "query.idx")
    idx2 = os.path.join(APP_DATA, id_f, "target.idx")
419

420
    paf = Paf(paf, idx1, idx2)
421

422
423
    if paf.parsed:
        res = paf.get_d3js_data()
424
        res["success"] = True
425
426
427
428
429
430
        return jsonify(res)
    return jsonify({"success": False, "message": paf.error})


@app.route('/sort/<id_res>', methods=['POST'])
def sort_graph(id_res):
431
    if not os.path.exists(os.path.join(APP_DATA, id_res, ".all-vs-all")):
432
433
434
        paf_file = os.path.join(APP_DATA, id_res, "map.paf")
        idx1 = os.path.join(APP_DATA, id_res, "query.idx")
        idx2 = os.path.join(APP_DATA, id_res, "target.idx")
435
        paf = Paf(paf_file, idx1, idx2, False)
436
437
438
439
440
441
442
        paf.sort()
        if paf.parsed:
            res = paf.get_d3js_data()
            res["success"] = True
            return jsonify(res)
        return jsonify({"success": False, "message": paf.error})
    return jsonify({"success": False, "message": "Sort is not available for All-vs-All mode"})
Floreal Cabanettes's avatar
Floreal Cabanettes committed
443

Floréal Cabanettes's avatar
Floréal Cabanettes committed
444

445
446
447
448
@app.route('/reverse-contig/<id_res>', methods=['POST'])
def reverse_contig(id_res):
    contig_name = request.form["contig"]
    if not os.path.exists(os.path.join(APP_DATA, id_res, ".all-vs-all")):
449
450
451
        paf_file = os.path.join(APP_DATA, id_res, "map.paf")
        idx1 = os.path.join(APP_DATA, id_res, "query.idx")
        idx2 = os.path.join(APP_DATA, id_res, "target.idx")
452
453
454
455
456
457
458
459
460
461
        paf = Paf(paf_file, idx1, idx2, False)
        paf.reverse_contig(contig_name)
        if paf.parsed:
            res = paf.get_d3js_data()
            res["success"] = True
            return jsonify(res)
        return jsonify({"success": False, "message": paf.error})
    return jsonify({"success": False, "message": "Sort is not available for All-vs-All mode"})


462
463
@app.route('/freenoise/<id_res>', methods=['POST'])
def free_noise(id_res):
464
465
466
    paf_file = os.path.join(APP_DATA, id_res, "map.paf")
    idx1 = os.path.join(APP_DATA, id_res, "query.idx")
    idx2 = os.path.join(APP_DATA, id_res, "target.idx")
467
468
469
470
471
472
473
474
475
    paf = Paf(paf_file, idx1, idx2, False)
    paf.parse_paf(noise=request.form["noise"] == "1")
    if paf.parsed:
        res = paf.get_d3js_data()
        res["success"] = True
        return jsonify(res)
    return jsonify({"success": False, "message": paf.error})


476
477
@app.route('/get-fasta-query/<id_res>', methods=['POST'])
def build_fasta(id_res):
478
    res_dir = os.path.join(APP_DATA, id_res)
479
480
    lock_query = os.path.join(res_dir, ".query-fasta-build")
    is_sorted = os.path.exists(os.path.join(res_dir, ".sorted"))
481
    compressed = request.form["gzip"].lower() == "true"
482
483
484
485
486
    query_fasta = Functions.get_fasta_file(res_dir, "query", is_sorted)
    if query_fasta is not None:
        if is_sorted and not query_fasta.endswith(".sorted"):
            # Do the sort
            Path(lock_query).touch()
487
            if not compressed or MODE == "standalone":  # If compressed, it will took a long time, so not wait
488
                Path(lock_query + ".pending").touch()
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
            index_file = os.path.join(res_dir, "query.idx.sorted")
            if MODE == "webserver":
                thread = threading.Timer(1, Functions.sort_fasta, kwargs={
                    "job_name": id_res,
                    "fasta_file": query_fasta,
                    "index_file": index_file,
                    "lock_file": lock_query,
                    "compress": compressed,
                    "mailer": mailer,
                    "mode": MODE
                })
                thread.start()
            else:
                Functions.sort_fasta(job_name=id_res,
                                     fasta_file=query_fasta,
                                     index_file=index_file,
                                     lock_file=lock_query,
                                     compress=compressed,
                                     mailer=None,
                                     mode=MODE)
509
            if not compressed or MODE == "standalone":
510
511
                if MODE == "webserver":
                    i = 0
512
                    time.sleep(5)
513
514
515
                    while os.path.exists(lock_query) and (i < 2 or MODE == "standalone"):
                        i += 1
                        time.sleep(5)
Floreal Cabanettes's avatar
Floreal Cabanettes committed
516
                os.remove(lock_query + ".pending")
517
518
519
                if os.path.exists(lock_query):
                    return jsonify({"success": True, "status": 1, "status_message": "In progress"})
                return jsonify({"success": True, "status": 2, "status_message": "Done",
520
                                "gzip": compressed})
521
522
            else:
                return jsonify({"success": True, "status": 1, "status_message": "In progress"})
523
524
525
526
527
        elif is_sorted and os.path.exists(lock_query):
            # Sort is already in progress
            return jsonify({"success": True, "status": 1, "status_message": "In progress"})
        else:
            # No sort to do or sort done
528
529
530
531
532
533
534
            if compressed and not query_fasta.endswith(".gz.fasta"):
                # If compressed file is asked, we must compress it now if not done before...
                Path(lock_query).touch()
                thread = threading.Timer(1, Functions.compress_and_send_mail, kwargs={
                    "job_name": id_res,
                    "fasta_file": query_fasta,
                    "index_file": os.path.join(res_dir, "query.idx.sorted"),
535
                    "lock_file": lock_query,
536
537
538
539
540
                    "compressed": compressed,
                    "mailer": mailer
                })
                thread.start()
                return jsonify({"success": True, "status": 1, "status_message": "In progress"})
541
542
543
544
545
546
547
            return jsonify({"success": True, "status": 2, "status_message": "Done",
                            "gzip": query_fasta.endswith(".gz") or query_fasta.endswith(".gz.sorted")})
    else:
        return jsonify({"success": False,
                        "message": "Unable to get fasta file for query. Please contact us to report the bug"})


548
def build_query_as_reference(id_res):
549
550
551
552
553
    paf_file = os.path.join(APP_DATA, id_res, "map.paf")
    idx1 = os.path.join(APP_DATA, id_res, "query.idx")
    idx2 = os.path.join(APP_DATA, id_res, "target.idx")
    paf = Paf(paf_file, idx1, idx2, False, mailer=mailer, id_job=id_res)
    paf.parse_paf(False, True)
554
555
556
557
558
559
560
561
562
563
    if MODE == "webserver":
        thread = threading.Timer(0, paf.build_query_chr_as_reference)
        thread.start()
        return True
    return paf.build_query_chr_as_reference()


@app.route('/build-query-as-reference/<id_res>', methods=['POST'])
def post_query_as_reference(id_res):
    build_query_as_reference(id_res)
564
565
566
    return jsonify({"success": True})


567
568
569
570
571
572
573
@app.route('/get-query-as-reference/<id_res>', methods=['GET'])
def get_query_as_reference(id_res):
    if MODE != "standalone":
        return abort(404)
    return send_file(build_query_as_reference(id_res))


574
575
576
577
578
579
580
581
@app.route('/download/<id_res>/<filename>')
def download_file(id_res, filename):
    file_dl = os.path.join(APP_DATA, id_res, filename)
    if os.path.isfile(file_dl):
        return send_file(file_dl)
    return abort(404)


582
583
584
@app.route('/fasta-query/<id_res>', defaults={'filename': ""}, methods=['GET'])
@app.route('/fasta-query/<id_res>/<filename>', methods=['GET'])  # Use fake URL in mail to set download file name
def dl_fasta(id_res, filename):
585
    res_dir = os.path.join(APP_DATA, id_res)
586
587
588
589
590
591
592
593
594
595
596
597
598
    lock_query = os.path.join(res_dir, ".query-fasta-build")
    is_sorted = os.path.exists(os.path.join(res_dir, ".sorted"))
    if not os.path.exists(lock_query) or not is_sorted:
        query_fasta = Functions.get_fasta_file(res_dir, "query", is_sorted)
        if query_fasta is not None:
            if query_fasta.endswith(".gz") or query_fasta.endswith(".gz.sorted"):
                content = get_file(query_fasta, True)
                return Response(content, mimetype="application/gzip")
            content = get_file(query_fasta)
            return Response(content, mimetype="text/plain")
    abort(404)


599
600
@app.route('/qt-assoc/<id_res>', methods=['GET'])
def qt_assoc(id_res):
601
    res_dir = os.path.join(APP_DATA, id_res)
602
    if os.path.exists(res_dir) and os.path.isdir(res_dir):
603
604
605
        paf_file = os.path.join(APP_DATA, id_res, "map.paf")
        idx1 = os.path.join(APP_DATA, id_res, "query.idx")
        idx2 = os.path.join(APP_DATA, id_res, "target.idx")
606
607
608
609
610
611
612
613
614
615
616
617
        try:
            paf = Paf(paf_file, idx1, idx2, False)
            paf.parse_paf(False)
        except FileNotFoundError:
            print("Unable to load data!")
            abort(404)
            return False
        csv_content = paf.build_query_on_target_association_file()
        return Response(csv_content, mimetype="text/plain")
    abort(404)


618
619
620
621
622
623
@app.route('/no-assoc/<id_res>', methods=['POST'])
def no_assoc(id_res):
    """
    Get contigs that match with None target
    :param id_res: id of the result
    """
624
    res_dir = os.path.join(APP_DATA, id_res)
625
    if os.path.exists(res_dir) and os.path.isdir(res_dir):
626
627
628
        paf_file = os.path.join(APP_DATA, id_res, "map.paf")
        idx1 = os.path.join(APP_DATA, id_res, "query.idx")
        idx2 = os.path.join(APP_DATA, id_res, "target.idx")
629
630
631
632
633
634
        try:
            paf = Paf(paf_file, idx1, idx2, False)
        except FileNotFoundError:
            print("Unable to load data!")
            abort(404)
            return False
635
        file_content = paf.build_list_no_assoc(request.form["to"])
636
637
638
639
640
641
642
643
        empty = file_content == "\n"
        return jsonify({
            "file_content": file_content,
            "empty": empty
        })
    abort(404)


Floreal Cabanettes's avatar
Floreal Cabanettes committed
644
645
@app.route('/summary/<id_res>', methods=['POST'])
def summary(id_res):
646
647
648
649
650
651
652
653
654
655
    paf_file = os.path.join(APP_DATA, id_res, "map.paf")
    idx1 = os.path.join(APP_DATA, id_res, "query.idx")
    idx2 = os.path.join(APP_DATA, id_res, "target.idx")
    try:
        paf = Paf(paf_file, idx1, idx2, False)
    except FileNotFoundError:
        return jsonify({
            "success": False,
            "message": "Unable to load data!"
        })
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
    percents = None
    s_status = "waiting"  # Accepted values: waiting, done, fail
    status_file = os.path.join(APP_DATA, id_res, ".summarize")
    fail_file = status_file + ".fail"
    if not os.path.exists(status_file):  # The job is finished or not started
        if not os.path.exists(fail_file):  # The job has not started yet or has successfully ended
            percents = paf.get_summary_stats()
            if percents is None:  # The job has not started yet
                Path(status_file).touch()
                thread = threading.Timer(0, paf.build_summary_stats, kwargs={"status_file": status_file})
                thread.start()
            else:  # The job has successfully ended
                s_status = "done"
        else:  # The job has failed
            s_status = "fail"

    if s_status == "waiting":  # The job is running
        # Check if the job end in the next 30 seconds
        nb_iter = 0
        while os.path.exists(status_file) and not os.path.exists(fail_file) and nb_iter < 10:
            time.sleep(3)
            nb_iter += 1
        if not os.path.exists(status_file):  # The job has ended
            percents = paf.get_summary_stats()
            if percents is None:  # The job has failed
                s_status = "fail"
            else:  # The job has successfully ended
                s_status = "done"

    if s_status == "fail":
686
687
688
689
690
691
        return jsonify({
            "success": False,
            "message": "Build of summary failed. Please contact us to report the bug"
        })
    return jsonify({
        "success": True,
692
693
        "percents": percents,
        "status": s_status
694
    })
Floreal Cabanettes's avatar
Floreal Cabanettes committed
695
696


697
698
699
700
701
702
703
704
705
706
@app.route('/backup/<id_res>')
def get_backup_file(id_res):
    res_dir = os.path.join(APP_DATA, id_res)
    tar = os.path.join(res_dir, "%s.tar" % id_res)
    with tarfile.open(tar, "w") as tarf:
        for file in ("map.paf", "target.idx", "query.idx"):
            tarf.add(os.path.join(res_dir, file), arcname=file)
    return send_file(tar)


707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
def get_filter_out(id_res, type_f):
    filter_file = os.path.join(APP_DATA, id_res, ".filter-" + type_f)
    return Response(get_file(filter_file), mimetype="text/plain")


@app.route('/filter-out/<id_res>/query')
def get_filter_out_query(id_res):
    return get_filter_out(id_res=id_res, type_f="query")


@app.route('/filter-out/<id_res>/target')
def get_filter_out_target(id_res):
    return get_filter_out(id_res=id_res, type_f="target")


722
723
@app.route("/ask-upload", methods=['POST'])
def ask_upload():
724
725
726
727
728
    if MODE == "standalone":
        return jsonify({
            "success": True,
            "allowed": True
        })
729
730
    try:
        s_id = request.form['s_id']
Floreal Cabanettes's avatar
Floreal Cabanettes committed
731
732
733
        with Session.connect():
            session = Session.get(s_id=s_id)
            allowed = session.ask_for_upload(True)
734
735
        return jsonify({
            "success": True,
Floreal Cabanettes's avatar
Floreal Cabanettes committed
736
            "allowed": allowed
737
738
739
        })
    except DoesNotExist:
        return jsonify({"success": False, "message": "Session not initialized. Please refresh the page."})
740
741


742
743
@app.route("/ping-upload", methods=['POST'])
def ping_upload():
744
745
746
747
748
    if MODE == "webserver":
        s_id = request.form['s_id']
        with Session.connect():
            session = Session.get(s_id=s_id)
            session.ping()
749
    return "OK"
750
751


752
753
754
755
@app.route("/upload", methods=['POST'])
def upload():
    try:
        s_id = request.form['s_id']
756
757
758
759
760
761
        if MODE == "webserver":
            try:
                with Session.connect():
                    session = Session.get(s_id=s_id)
                    if session.ask_for_upload(False):
                        folder = session.upload_folder
Floreal Cabanettes's avatar
Floreal Cabanettes committed
762
                    else:
763
764
765
766
767
768
                        return jsonify({"files": [], "success": "ERR", "message": "Not allowed to upload!"})
            except DoesNotExist:
                return jsonify(
                    {"files": [], "success": "ERR", "message": "Session not initialized. Please refresh the page."})
        else:
            folder = s_id
Floreal Cabanettes's avatar
Floreal Cabanettes committed
769

770
        files = request.files[list(request.files.keys())[0]]
Floreal Cabanettes's avatar
Floreal Cabanettes committed
771

772
773
774
775
776
777
778
        if files:
            filename = files.filename
            folder_files = os.path.join(app.config["UPLOAD_FOLDER"], folder)
            if not os.path.exists(folder_files):
                os.makedirs(folder_files)
            filename = Functions.get_valid_uploaded_filename(filename, folder_files)
            mime_type = files.content_type
Floreal Cabanettes's avatar
Floreal Cabanettes committed
779

Floreal Cabanettes's avatar
Floreal Cabanettes committed
780
            if not Functions.allowed_file(files.filename, request.form['formats'].split(",")):
781
782
                result = UploadFile(name=filename, type_f=mime_type, size=0, not_allowed_msg="File type not allowed")
                shutil.rmtree(folder_files)
Floreal Cabanettes's avatar
Floreal Cabanettes committed
783

784
785
786
787
788
789
790
791
792
793
794
795
796
797
            else:
                # save file to disk
                uploaded_file_path = os.path.join(folder_files, filename)
                files.save(uploaded_file_path)

                # get file size after saving
                size = os.path.getsize(uploaded_file_path)

                # return json for js call back
                result = UploadFile(name=filename, type_f=mime_type, size=size)

            return jsonify({"files": [result.get_file()], "success": "OK"})

        return jsonify({"files": [], "success": "404", "message": "No file provided"})
798
    except:  # Except all possible exceptions to prevent crashes
Floreal Cabanettes's avatar
Floreal Cabanettes committed
799
        traceback.print_exc()
800
801
        return jsonify({"files": [], "success": "ERR", "message": "An unexpected error has occurred on upload. "
                                                                  "Please contact the support."})
802
803


804
805
806
807
808
809
@app.route("/send-mail/<id_res>", methods=['POST'])
def send_mail(id_res):
    allowed = False
    key_file = None
    if "key" in request.form:
        key = request.form["key"]
810
        res_dir = os.path.join(APP_DATA, id_res)
811
812
813
814
815
816
817
818
819
820
821
822
        key_file = os.path.join(res_dir, ".key")
        if os.path.exists(key_file):
            with open(key_file) as k_f:
                true_key = k_f.readline().strip("\n")
                allowed = key == true_key
    if allowed:
        os.remove(key_file)
        job_mng = JobManager(id_job=id_res, mailer=mailer)
        job_mng.set_inputs_from_res_dir()
        job_mng.send_mail()
        return "OK"
    else:
823
        abort(403)
824
825
826
827
828
829
830
831
832
833


@app.route("/delete/<id_res>", methods=['POST'])
def delete_job(id_res):
    job = JobManager(id_job=id_res)
    success, error = job.delete()
    return jsonify({
        "success": success,
        "error": error
    })