views.py 32.5 KB
Newer Older
1
from dgenies import app, app_title, app_folder, config_reader, mailer, APP_DATA, MODE, DEBUG, VERSION
2

Floreal Cabanettes's avatar
Floreal Cabanettes committed
3
import os
4
5
import time
import datetime
6
import shutil
7
import re
8
import threading
Floreal Cabanettes's avatar
Floreal Cabanettes committed
9
import traceback
Floreal Cabanettes's avatar
Floreal Cabanettes committed
10
from flask import render_template, request, url_for, jsonify, Response, abort, send_file, Markup
11
from pathlib import Path
12
13
14
15
16
from dgenies.lib.paf import Paf
from dgenies.lib.job_manager import JobManager
from dgenies.lib.functions import Functions, ALLOWED_EXTENSIONS
from dgenies.lib.upload_file import UploadFile
from dgenies.lib.fasta import Fasta
17
from dgenies.lib.latest import Latest
18
from dgenies.tools import Tools
Floreal Cabanettes's avatar
Floreal Cabanettes committed
19
20
from markdown import Markdown
from markdown.extensions.toc import TocExtension
21
from markdown.extensions.tables import TableExtension
22
import tarfile
23
from jinja2 import Environment
24
25
26
if MODE == "webserver":
    from dgenies.database import Session, Gallery
    from peewee import DoesNotExist
27

28

Floreal Cabanettes's avatar
Floreal Cabanettes committed
29
30
31
32
@app.context_processor
def global_templates_variables():
    return {
        "title": app_title,
33
        "mode": MODE,
34
35
        "all_jobs": Functions.get_list_all_jobs(MODE),
        "debug": DEBUG
Floreal Cabanettes's avatar
Floreal Cabanettes committed
36
37
38
    }


39
40
41
# Main
@app.route("/", methods=['GET'])
def main():
42
43
44
45
46
47
    if MODE == "webserver":
        pict = Gallery.select().order_by("id")
        if len(pict) > 0:
            pict = pict[0].picture
        else:
            pict = None
Floreal Cabanettes's avatar
Floreal Cabanettes committed
48
49
    else:
        pict = None
Floreal Cabanettes's avatar
Floreal Cabanettes committed
50
    return render_template("index.html", menu="index", pict=pict)
Floreal Cabanettes's avatar
Floreal Cabanettes committed
51
52
53
54


@app.route("/run", methods=['GET'])
def run():
55
56
57
58
59
    tools = Tools().tools
    tools_names = sorted(list(tools.keys()), key=lambda x: (tools[x].order, tools[x].name))
    tools_ava = {}
    for tool_name, tool in tools.items():
        tools_ava[tool_name] = 1 if tool.all_vs_all is not None else 0
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
    if MODE == "webserver":
        with Session.connect():
            s_id = Session.new()
    else:
        upload_folder = Functions.random_string(20)
        tmp_dir = config_reader.upload_folder
        upload_folder_path = os.path.join(tmp_dir, upload_folder)
        while os.path.exists(upload_folder_path):
            upload_folder = Functions.random_string(20)
            upload_folder_path = os.path.join(tmp_dir, upload_folder)
        s_id = upload_folder
    id_job = Functions.random_string(5) + "_" + datetime.datetime.fromtimestamp(time.time()).strftime('%Y%m%d%H%M%S')
    if "id_job" in request.args:
        id_job = request.args["id_job"]
    email = ""
    if "email" in request.args:
        email = request.args["email"]
Floreal Cabanettes's avatar
Floreal Cabanettes committed
77
    return render_template("run.html", id_job=id_job, email=email,
78
                           menu="run", allowed_ext=ALLOWED_EXTENSIONS, s_id=s_id,
79
80
81
                           max_upload_file_size=config_reader.max_upload_file_size,
                           example=config_reader.example_target != "",
                           target=os.path.basename(config_reader.example_target),
82
                           query=os.path.basename(config_reader.example_query), tools_names=tools_names, tools=tools,
83
                           tools_ava=tools_ava, version=VERSION)
84
85


86
87
@app.route("/run-test", methods=['GET'])
def run_test():
88
89
90
91
92
93
94
    if MODE == "webserver":
        print(config_reader.allowed_ip_tests)
        if request.remote_addr not in config_reader.allowed_ip_tests:
            return abort(404)
        with Session.connect():
            return Session.new()
    return abort(500)
95
96


97
98
99
# Launch analysis
@app.route("/launch_analysis", methods=['POST'])
def launch_analysis():
100
101
102
103
104
105
106
107
108
109
110
111
    if MODE == "webserver":
        try:
            with Session.connect():
                session = Session.get(s_id=request.form["s_id"])
        except DoesNotExist:
            return jsonify({"success": False, "errors": ["Session has expired. Please refresh the page and try again"]})
        upload_folder = session.upload_folder
        # Delete session:
        session.delete_instance()
    else:
        upload_folder = request.form["s_id"]

112
113
    id_job = request.form["id_job"]
    email = request.form["email"]
114
115
116
117
    file_query = request.form["query"]
    file_query_type = request.form["query_type"]
    file_target = request.form["target"]
    file_target_type = request.form["target_type"]
118
119
120
    tool = request.form["tool"] if "tool" in request.form else None
    alignfile = request.form["alignfile"] if "alignfile" in request.form else None
    alignfile_type = request.form["alignfile_type"] if "alignfile_type" in request.form else None
121
122
    backup = request.form["backup"] if "backup" in request.form else None
    backup_type = request.form["backup_type"] if "backup_type" in request.form else None
123
124
125

    # Check form:
    form_pass = True
126
    errors = []
127

128
129
130
131
    if alignfile is not None and alignfile_type is None:
        errors.append("Server error: no alignfile_type in form. Please contact the support")
        form_pass = False

Floreal Cabanettes's avatar
Floreal Cabanettes committed
132
    if backup is not None and backup != "" and (backup_type is None or backup_type == ""):
133
134
135
        errors.append("Server error: no backup_type in form. Please contact the support")
        form_pass = False

Floreal Cabanettes's avatar
Floreal Cabanettes committed
136
    if backup is not None and backup != "":
137
138
139
140
        alignfile = ""
        file_query = ""
        file_target = ""
    else:
Floreal Cabanettes's avatar
Floreal Cabanettes committed
141
        backup = None
142
143
144
145
        if file_target == "":
            errors.append("No target fasta selected")
            form_pass = False

146
    if tool is not None and tool not in Tools().tools:
147
        errors.append("Tool unavailable: %s" % tool)
148
        form_pass = False
149

150
    if id_job == "":
151
        errors.append("Id of job not given")
152
153
        form_pass = False

154
155
156
157
    if MODE == "webserver":
        if email == "":
            errors.append("Email not given")
            form_pass = False
Floreal Cabanettes's avatar
Floreal Cabanettes committed
158
        elif not re.match(r"^[\w.\-]+@[\w\-.]{2,}\.[a-z]{2,4}$", email):
159
160
            errors.append("Email is invalid")
            form_pass = False
161
162
163

    # Form pass
    if form_pass:
164
        # Get final job id:
165
        id_job = re.sub('[^A-Za-z0-9_\-]+', '', id_job.replace(" ", "_"))
166
        id_job_orig = id_job
167
        i = 2
168
        while os.path.exists(os.path.join(APP_DATA, id_job)):
169
170
            id_job = id_job_orig + ("_%d" % i)
            i += 1
171

172
        folder_files = os.path.join(APP_DATA, id_job)
173
174
175
        os.makedirs(folder_files)

        # Save files:
176
177
        query = None
        if file_query != "":
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
            example = False
            if file_query.startswith("example://"):
                example = True
                query_path = config_reader.example_query
                query_name = os.path.basename(query_path)
                file_query_type = "local"
            else:
                query_name = os.path.splitext(file_query.replace(".gz", ""))[0] if file_query_type == "local" else None
                query_path = os.path.join(app.config["UPLOAD_FOLDER"], upload_folder, file_query) \
                    if file_query_type == "local" else file_query
                if file_query_type == "local" and not os.path.exists(query_path):
                    errors.append("Query file not correct!")
                    form_pass = False
            query = Fasta(name=query_name, path=query_path, type_f=file_query_type, example=example)
        example = False
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
        target = None
        if file_target != "":
            if file_target.startswith("example://"):
                example = True
                target_path = config_reader.example_target
                target_name = os.path.basename(target_path)
                file_target_type = "local"
            else:
                target_name = os.path.splitext(file_target.replace(".gz", ""))[0] if file_target_type == "local" else None
                target_path = os.path.join(app.config["UPLOAD_FOLDER"], upload_folder, file_target) \
                    if file_target_type == "local" else file_target
                if file_target_type == "local" and not os.path.exists(target_path):
                    errors.append("Target file not correct!")
                    form_pass = False
            target = Fasta(name=target_name, path=target_path, type_f=file_target_type, example=example)
208

209
        if alignfile is not None and alignfile != "" and backup is not None:
210
            Path(os.path.join(folder_files, ".align")).touch()
211
212
213

        align = None
        if alignfile is not None and alignfile != "":
214
215
216
217
218
219
220
221
            alignfile_name = os.path.splitext(alignfile)[0] if alignfile_type == "local" else None
            alignfile_path = os.path.join(app.config["UPLOAD_FOLDER"], upload_folder, alignfile) \
                if alignfile_type == "local" else alignfile
            if alignfile_type == "local" and not os.path.exists(alignfile_path):
                errors.append("Alignment file not correct!")
                form_pass = False
            align = Fasta(name=alignfile_name, path=alignfile_path, type_f=alignfile_type)

222
223
224
225
226
227
228
229
230
231
        bckp = None
        if backup is not None:
            backup_name = os.path.splitext(backup)[0] if backup_type == "local" else None
            backup_path = os.path.join(app.config["UPLOAD_FOLDER"], upload_folder, backup) \
                if backup_type == "local" else backup
            if backup_type == "local" and not os.path.exists(backup_path):
                errors.append("Backup file not correct!")
                form_pass = False
            bckp = Fasta(name=backup_name, path=backup_path, type_f=backup_type)

232
233
        if form_pass:
            # Launch job:
234
235
236
237
238
            job = JobManager(id_job=id_job,
                             email=email,
                             query=query,
                             target=target,
                             align=align,
239
                             backup=bckp,
240
241
                             mailer=mailer,
                             tool=tool)
242
243
244
245
246
247
            if MODE == "webserver":
                job.launch()
            else:
                job.launch_standalone()
            return jsonify({"success": True, "redirect": url_for(".status", id_job=id_job)})
    if not form_pass:
248
        return jsonify({"success": False, "errors": errors})
249
250
251
252
253
254


# Status of a job
@app.route('/status/<id_job>', methods=['GET'])
def status(id_job):
    job = JobManager(id_job)
255
256
257
258
259
260
261
262
263
264
265
266
    j_status = job.status()
    mem_peak = j_status["mem_peak"] if "mem_peak" in j_status else None
    if mem_peak is not None:
        mem_peak = "%.1f G" % (mem_peak / 1024.0 / 1024.0)
    time_e = j_status["time_elapsed"] if "time_elapsed" in j_status else None
    if time_e is not None:
        if time_e < 60:
            time_e = "%d secs" % time_e
        else:
            minutes = time_e // 60
            seconds = time_e - minutes * 60
            time_e = "%d min %d secs" % (minutes, seconds)
267
268
269
270
271
272
273
274
275
    format = request.args.get("format")
    if format is not None and format == "json":
        return jsonify({
            "status": j_status["status"],
            "error": j_status["error"].replace("#ID#", ""),
            "id_job": id_job,
            "mem_peak": mem_peak,
            "time_elapsed": time_e
        })
Floreal Cabanettes's avatar
Floreal Cabanettes committed
276
    return render_template("status.html", status=j_status["status"],
277
278
                           error=j_status["error"].replace("#ID#", ""),
                           id_job=id_job, menu="results", mem_peak=mem_peak,
279
                           time_elapsed=time_e, do_align=job.do_align(),
Floreal Cabanettes's avatar
Floreal Cabanettes committed
280
                           query_filtered=job.is_query_filtered(), target_filtered=job.is_target_filtered())
281
282


Floreal Cabanettes's avatar
Floreal Cabanettes committed
283
# Results path
284
@app.route("/result/<id_res>", methods=['GET'])
285
def result(id_res):
286
    res_dir = os.path.join(APP_DATA, id_res)
Floreal Cabanettes's avatar
Floreal Cabanettes committed
287
    return render_template("result.html", id=id_res, menu="result", current_result=id_res,
288
289
                           is_gallery=Functions.is_in_gallery(id_res, MODE),
                           fasta_file=Functions.query_fasta_file_exists(res_dir))
290
291


292
293
@app.route("/gallery", methods=['GET'])
def gallery():
294
    if MODE == "webserver":
Floreal Cabanettes's avatar
Floreal Cabanettes committed
295
        return render_template("gallery.html", items=Functions.get_gallery_items(), menu="gallery")
296
    return abort(404)
297
298
299
300


@app.route("/gallery/<filename>", methods=['GET'])
def gallery_file(filename):
301
302
303
304
305
306
    if MODE == "webserver":
        try:
            return send_file(os.path.join(config_reader.app_data, "gallery", filename))
        except FileNotFoundError:
            abort(404)
    return abort(404)
307
308


309
def get_file(file, gzip=False):  # pragma: no cover
Floreal Cabanettes's avatar
Floreal Cabanettes committed
310
311
312
313
314
315
    try:
        # Figure out how flask returns static files
        # Tried:
        # - render_template
        # - send_file
        # This should not be so non-obvious
316
        return open(file, "rb" if gzip else "r").read()
317
318
    except FileNotFoundError:
        abort(404)
Floreal Cabanettes's avatar
Floreal Cabanettes committed
319
    except IOError as exc:
320
        print(exc.__traceback__)
321
        abort(500)
Floreal Cabanettes's avatar
Floreal Cabanettes committed
322
323


Floreal Cabanettes's avatar
Floreal Cabanettes committed
324
325
@app.route("/documentation/run", methods=['GET'])
def documentation_run():
326
327
    latest = Latest()
    version = latest.latest
Floreal Cabanettes's avatar
Floreal Cabanettes committed
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
    max_upload_file_size = config_reader.max_upload_file_size
    if max_upload_file_size == -1:
        max_upload_file_size = "no limit"
    else:
        max_upload_file_size = Functions.get_readable_size(max_upload_file_size, 0)
    max_upload_size = config_reader.max_upload_size
    if max_upload_size == -1:
        max_upload_size = "no limit"
    else:
        max_upload_size = Functions.get_readable_size(max_upload_size, 0)
    max_upload_size_ava = config_reader.max_upload_size_ava
    if max_upload_size_ava == -1:
        max_upload_size_ava = "no limit"
    else:
        max_upload_size_ava = Functions.get_readable_size(max_upload_size_ava, 0)
Floreal Cabanettes's avatar
Floreal Cabanettes committed
343
344
345
346
347
348
349
    with open(os.path.join(app_folder, "md", "doc_run.md"), "r",  encoding='utf-8') as install_instr:
        content = install_instr.read()
    env = Environment()
    template = env.from_string(content)
    content = template.render(mode=MODE, version=version, size=max_upload_file_size, size_unc=max_upload_size,
                              size_ava=max_upload_size_ava)
    md = Markdown(extensions=[TocExtension(baselevel=1)])
350
    content = Markup(md.convert(content))
Floreal Cabanettes's avatar
Floreal Cabanettes committed
351
    toc = Markup(md.toc)
Floreal Cabanettes's avatar
Floreal Cabanettes committed
352
    return render_template("documentation.html", menu="documentation", content=content, toc=toc)
Floreal Cabanettes's avatar
Floreal Cabanettes committed
353
354


355
356
@app.route("/documentation/result", methods=['GET'])
def documentation_result():
357
    with open(os.path.join(app_folder, "md", "user_manual.md"), "r",
358
359
360
              encoding='utf-8') as install_instr:
        content = install_instr.read()
    md = Markdown(extensions=[TocExtension(baselevel=1)])
361
    content = Markup(md.convert(content))
362
363
364
365
    toc = Markup(md.toc)
    return render_template("documentation.html", menu="documentation", content=content, toc=toc)


366
367
@app.route("/documentation/formats", methods=['GET'])
def documentation_formats():
368
    with open(os.path.join(app_folder, "md", "doc_formats.md"), "r",
369
370
371
              encoding='utf-8') as install_instr:
        content = install_instr.read()
    md = Markdown(extensions=[TocExtension(baselevel=1), TableExtension()])
372
    content = Markup(md.convert(content))
373
374
375
376
    toc = Markup(md.toc)
    return render_template("documentation.html", menu="documentation", content=content, toc=toc)


377
378
379
380
381
382
@app.route("/documentation/dotplot", methods=['GET'])
def documentation_dotplot():
    with open(os.path.join(app_folder, "md", "doc_dotplot.md"), "r",
              encoding='utf-8') as install_instr:
        content = install_instr.read()
    md = Markdown(extensions=[TocExtension(baselevel=1)])
383
    content = Markup(md.convert(content))
384
385
386
387
    toc = Markup(md.toc)
    return render_template("documentation.html", menu="documentation", content=content, toc=toc)


Floreal Cabanettes's avatar
Floreal Cabanettes committed
388
389
@app.route("/install", methods=['GET'])
def install():
390
    latest = Latest()
391

392
    with open(os.path.join(app_folder, "md", "INSTALL.md"), "r", encoding='utf-8') as install_instr:
Floreal Cabanettes's avatar
Floreal Cabanettes committed
393
        content = install_instr.read()
394
395
    env = Environment()
    template = env.from_string(content)
396
    content = template.render(version=latest.latest, win32=latest.win32)
Floreal Cabanettes's avatar
Floreal Cabanettes committed
397
    md = Markdown(extensions=[TocExtension(baselevel=1)])
398
    content = Markup(md.convert(content))
Floreal Cabanettes's avatar
Floreal Cabanettes committed
399
    toc = Markup(md.toc)
Floreal Cabanettes's avatar
Floreal Cabanettes committed
400
    return render_template("documentation.html", menu="install", content=content, toc=toc)
Floreal Cabanettes's avatar
Floreal Cabanettes committed
401

Floreal Cabanettes's avatar
Floreal Cabanettes committed
402

403
404
405
406
@app.route("/contact", methods=['GET'])
def contact():
    return render_template("contact.html", menu="contact")

Floreal Cabanettes's avatar
Floreal Cabanettes committed
407

Floreal Cabanettes's avatar
Floreal Cabanettes committed
408
409
410
411
412
413
414
415
416
417
418
@app.route("/paf/<id_res>", methods=['GET'])
def download_paf(id_res):
    map_file = os.path.join(APP_DATA, id_res, "map.paf.sorted")
    if not os.path.exists(map_file):
        map_file = os.path.join(APP_DATA, id_res, "map.paf")
    if not os.path.exists(map_file):
        abort(404)
    content = get_file(map_file)
    return Response(content, mimetype="text/plain")


Floreal Cabanettes's avatar
Floreal Cabanettes committed
419
# Get graph (ajax request)
420
421
422
@app.route('/get_graph', methods=['POST'])
def get_graph():
    id_f = request.form["id"]
423
424
425
    paf = os.path.join(APP_DATA, id_f, "map.paf")
    idx1 = os.path.join(APP_DATA, id_f, "query.idx")
    idx2 = os.path.join(APP_DATA, id_f, "target.idx")
426

427
    paf = Paf(paf, idx1, idx2)
428

429
    if paf.parsed:
430
431
432
        valid = os.path.join(APP_DATA, id_f, ".valid")
        if not os.path.exists(valid):
            Path(valid).touch()
433
        res = paf.get_d3js_data()
434
        res["success"] = True
435
436
437
438
439
440
        return jsonify(res)
    return jsonify({"success": False, "message": paf.error})


@app.route('/sort/<id_res>', methods=['POST'])
def sort_graph(id_res):
441
    if not os.path.exists(os.path.join(APP_DATA, id_res, ".all-vs-all")):
442
443
444
        paf_file = os.path.join(APP_DATA, id_res, "map.paf")
        idx1 = os.path.join(APP_DATA, id_res, "query.idx")
        idx2 = os.path.join(APP_DATA, id_res, "target.idx")
445
        paf = Paf(paf_file, idx1, idx2, False)
446
447
448
449
450
451
452
        paf.sort()
        if paf.parsed:
            res = paf.get_d3js_data()
            res["success"] = True
            return jsonify(res)
        return jsonify({"success": False, "message": paf.error})
    return jsonify({"success": False, "message": "Sort is not available for All-vs-All mode"})
Floreal Cabanettes's avatar
Floreal Cabanettes committed
453

Floréal Cabanettes's avatar
Floréal Cabanettes committed
454

455
456
457
458
@app.route('/reverse-contig/<id_res>', methods=['POST'])
def reverse_contig(id_res):
    contig_name = request.form["contig"]
    if not os.path.exists(os.path.join(APP_DATA, id_res, ".all-vs-all")):
459
460
461
        paf_file = os.path.join(APP_DATA, id_res, "map.paf")
        idx1 = os.path.join(APP_DATA, id_res, "query.idx")
        idx2 = os.path.join(APP_DATA, id_res, "target.idx")
462
463
464
465
466
467
468
469
470
471
        paf = Paf(paf_file, idx1, idx2, False)
        paf.reverse_contig(contig_name)
        if paf.parsed:
            res = paf.get_d3js_data()
            res["success"] = True
            return jsonify(res)
        return jsonify({"success": False, "message": paf.error})
    return jsonify({"success": False, "message": "Sort is not available for All-vs-All mode"})


472
473
@app.route('/freenoise/<id_res>', methods=['POST'])
def free_noise(id_res):
474
475
476
    paf_file = os.path.join(APP_DATA, id_res, "map.paf")
    idx1 = os.path.join(APP_DATA, id_res, "query.idx")
    idx2 = os.path.join(APP_DATA, id_res, "target.idx")
477
478
479
480
481
482
483
484
485
    paf = Paf(paf_file, idx1, idx2, False)
    paf.parse_paf(noise=request.form["noise"] == "1")
    if paf.parsed:
        res = paf.get_d3js_data()
        res["success"] = True
        return jsonify(res)
    return jsonify({"success": False, "message": paf.error})


486
487
@app.route('/get-fasta-query/<id_res>', methods=['POST'])
def build_fasta(id_res):
488
    res_dir = os.path.join(APP_DATA, id_res)
489
490
    lock_query = os.path.join(res_dir, ".query-fasta-build")
    is_sorted = os.path.exists(os.path.join(res_dir, ".sorted"))
491
    compressed = request.form["gzip"].lower() == "true"
492
493
494
495
496
    query_fasta = Functions.get_fasta_file(res_dir, "query", is_sorted)
    if query_fasta is not None:
        if is_sorted and not query_fasta.endswith(".sorted"):
            # Do the sort
            Path(lock_query).touch()
497
            if not compressed or MODE == "standalone":  # If compressed, it will took a long time, so not wait
498
                Path(lock_query + ".pending").touch()
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
            index_file = os.path.join(res_dir, "query.idx.sorted")
            if MODE == "webserver":
                thread = threading.Timer(1, Functions.sort_fasta, kwargs={
                    "job_name": id_res,
                    "fasta_file": query_fasta,
                    "index_file": index_file,
                    "lock_file": lock_query,
                    "compress": compressed,
                    "mailer": mailer,
                    "mode": MODE
                })
                thread.start()
            else:
                Functions.sort_fasta(job_name=id_res,
                                     fasta_file=query_fasta,
                                     index_file=index_file,
                                     lock_file=lock_query,
                                     compress=compressed,
                                     mailer=None,
                                     mode=MODE)
519
            if not compressed or MODE == "standalone":
520
521
                if MODE == "webserver":
                    i = 0
522
                    time.sleep(5)
523
524
525
                    while os.path.exists(lock_query) and (i < 2 or MODE == "standalone"):
                        i += 1
                        time.sleep(5)
Floreal Cabanettes's avatar
Floreal Cabanettes committed
526
                os.remove(lock_query + ".pending")
527
528
529
                if os.path.exists(lock_query):
                    return jsonify({"success": True, "status": 1, "status_message": "In progress"})
                return jsonify({"success": True, "status": 2, "status_message": "Done",
530
                                "gzip": compressed})
531
532
            else:
                return jsonify({"success": True, "status": 1, "status_message": "In progress"})
533
534
535
536
537
        elif is_sorted and os.path.exists(lock_query):
            # Sort is already in progress
            return jsonify({"success": True, "status": 1, "status_message": "In progress"})
        else:
            # No sort to do or sort done
538
539
540
541
542
543
544
            if compressed and not query_fasta.endswith(".gz.fasta"):
                # If compressed file is asked, we must compress it now if not done before...
                Path(lock_query).touch()
                thread = threading.Timer(1, Functions.compress_and_send_mail, kwargs={
                    "job_name": id_res,
                    "fasta_file": query_fasta,
                    "index_file": os.path.join(res_dir, "query.idx.sorted"),
545
                    "lock_file": lock_query,
546
547
548
549
550
                    "compressed": compressed,
                    "mailer": mailer
                })
                thread.start()
                return jsonify({"success": True, "status": 1, "status_message": "In progress"})
551
552
553
554
555
556
557
            return jsonify({"success": True, "status": 2, "status_message": "Done",
                            "gzip": query_fasta.endswith(".gz") or query_fasta.endswith(".gz.sorted")})
    else:
        return jsonify({"success": False,
                        "message": "Unable to get fasta file for query. Please contact us to report the bug"})


558
def build_query_as_reference(id_res):
559
560
561
562
563
    paf_file = os.path.join(APP_DATA, id_res, "map.paf")
    idx1 = os.path.join(APP_DATA, id_res, "query.idx")
    idx2 = os.path.join(APP_DATA, id_res, "target.idx")
    paf = Paf(paf_file, idx1, idx2, False, mailer=mailer, id_job=id_res)
    paf.parse_paf(False, True)
564
565
566
567
568
569
570
571
572
573
    if MODE == "webserver":
        thread = threading.Timer(0, paf.build_query_chr_as_reference)
        thread.start()
        return True
    return paf.build_query_chr_as_reference()


@app.route('/build-query-as-reference/<id_res>', methods=['POST'])
def post_query_as_reference(id_res):
    build_query_as_reference(id_res)
574
575
576
    return jsonify({"success": True})


577
578
579
580
581
582
583
@app.route('/get-query-as-reference/<id_res>', methods=['GET'])
def get_query_as_reference(id_res):
    if MODE != "standalone":
        return abort(404)
    return send_file(build_query_as_reference(id_res))


584
585
586
587
588
589
590
591
@app.route('/download/<id_res>/<filename>')
def download_file(id_res, filename):
    file_dl = os.path.join(APP_DATA, id_res, filename)
    if os.path.isfile(file_dl):
        return send_file(file_dl)
    return abort(404)


592
593
594
@app.route('/fasta-query/<id_res>', defaults={'filename': ""}, methods=['GET'])
@app.route('/fasta-query/<id_res>/<filename>', methods=['GET'])  # Use fake URL in mail to set download file name
def dl_fasta(id_res, filename):
595
    res_dir = os.path.join(APP_DATA, id_res)
596
597
598
599
600
601
602
603
604
605
606
607
608
    lock_query = os.path.join(res_dir, ".query-fasta-build")
    is_sorted = os.path.exists(os.path.join(res_dir, ".sorted"))
    if not os.path.exists(lock_query) or not is_sorted:
        query_fasta = Functions.get_fasta_file(res_dir, "query", is_sorted)
        if query_fasta is not None:
            if query_fasta.endswith(".gz") or query_fasta.endswith(".gz.sorted"):
                content = get_file(query_fasta, True)
                return Response(content, mimetype="application/gzip")
            content = get_file(query_fasta)
            return Response(content, mimetype="text/plain")
    abort(404)


609
610
@app.route('/qt-assoc/<id_res>', methods=['GET'])
def qt_assoc(id_res):
611
    res_dir = os.path.join(APP_DATA, id_res)
612
    if os.path.exists(res_dir) and os.path.isdir(res_dir):
613
614
615
        paf_file = os.path.join(APP_DATA, id_res, "map.paf")
        idx1 = os.path.join(APP_DATA, id_res, "query.idx")
        idx2 = os.path.join(APP_DATA, id_res, "target.idx")
616
617
618
619
620
621
622
623
624
625
626
627
        try:
            paf = Paf(paf_file, idx1, idx2, False)
            paf.parse_paf(False)
        except FileNotFoundError:
            print("Unable to load data!")
            abort(404)
            return False
        csv_content = paf.build_query_on_target_association_file()
        return Response(csv_content, mimetype="text/plain")
    abort(404)


628
629
630
631
632
633
@app.route('/no-assoc/<id_res>', methods=['POST'])
def no_assoc(id_res):
    """
    Get contigs that match with None target
    :param id_res: id of the result
    """
634
    res_dir = os.path.join(APP_DATA, id_res)
635
    if os.path.exists(res_dir) and os.path.isdir(res_dir):
636
637
638
        paf_file = os.path.join(APP_DATA, id_res, "map.paf")
        idx1 = os.path.join(APP_DATA, id_res, "query.idx")
        idx2 = os.path.join(APP_DATA, id_res, "target.idx")
639
640
641
642
643
644
        try:
            paf = Paf(paf_file, idx1, idx2, False)
        except FileNotFoundError:
            print("Unable to load data!")
            abort(404)
            return False
645
        file_content = paf.build_list_no_assoc(request.form["to"])
646
647
648
649
650
651
652
653
        empty = file_content == "\n"
        return jsonify({
            "file_content": file_content,
            "empty": empty
        })
    abort(404)


Floreal Cabanettes's avatar
Floreal Cabanettes committed
654
655
@app.route('/summary/<id_res>', methods=['POST'])
def summary(id_res):
656
657
658
659
660
661
662
663
664
665
    paf_file = os.path.join(APP_DATA, id_res, "map.paf")
    idx1 = os.path.join(APP_DATA, id_res, "query.idx")
    idx2 = os.path.join(APP_DATA, id_res, "target.idx")
    try:
        paf = Paf(paf_file, idx1, idx2, False)
    except FileNotFoundError:
        return jsonify({
            "success": False,
            "message": "Unable to load data!"
        })
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
    percents = None
    s_status = "waiting"  # Accepted values: waiting, done, fail
    status_file = os.path.join(APP_DATA, id_res, ".summarize")
    fail_file = status_file + ".fail"
    if not os.path.exists(status_file):  # The job is finished or not started
        if not os.path.exists(fail_file):  # The job has not started yet or has successfully ended
            percents = paf.get_summary_stats()
            if percents is None:  # The job has not started yet
                Path(status_file).touch()
                thread = threading.Timer(0, paf.build_summary_stats, kwargs={"status_file": status_file})
                thread.start()
            else:  # The job has successfully ended
                s_status = "done"
        else:  # The job has failed
            s_status = "fail"

    if s_status == "waiting":  # The job is running
        # Check if the job end in the next 30 seconds
        nb_iter = 0
        while os.path.exists(status_file) and not os.path.exists(fail_file) and nb_iter < 10:
            time.sleep(3)
            nb_iter += 1
        if not os.path.exists(status_file):  # The job has ended
            percents = paf.get_summary_stats()
            if percents is None:  # The job has failed
                s_status = "fail"
            else:  # The job has successfully ended
                s_status = "done"

    if s_status == "fail":
696
697
698
699
700
701
        return jsonify({
            "success": False,
            "message": "Build of summary failed. Please contact us to report the bug"
        })
    return jsonify({
        "success": True,
702
703
        "percents": percents,
        "status": s_status
704
    })
Floreal Cabanettes's avatar
Floreal Cabanettes committed
705
706


707
708
709
710
711
712
713
714
715
716
@app.route('/backup/<id_res>')
def get_backup_file(id_res):
    res_dir = os.path.join(APP_DATA, id_res)
    tar = os.path.join(res_dir, "%s.tar" % id_res)
    with tarfile.open(tar, "w") as tarf:
        for file in ("map.paf", "target.idx", "query.idx"):
            tarf.add(os.path.join(res_dir, file), arcname=file)
    return send_file(tar)


717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
def get_filter_out(id_res, type_f):
    filter_file = os.path.join(APP_DATA, id_res, ".filter-" + type_f)
    return Response(get_file(filter_file), mimetype="text/plain")


@app.route('/filter-out/<id_res>/query')
def get_filter_out_query(id_res):
    return get_filter_out(id_res=id_res, type_f="query")


@app.route('/filter-out/<id_res>/target')
def get_filter_out_target(id_res):
    return get_filter_out(id_res=id_res, type_f="target")


732
733
@app.route("/ask-upload", methods=['POST'])
def ask_upload():
734
735
736
737
738
    if MODE == "standalone":
        return jsonify({
            "success": True,
            "allowed": True
        })
739
740
    try:
        s_id = request.form['s_id']
Floreal Cabanettes's avatar
Floreal Cabanettes committed
741
742
743
        with Session.connect():
            session = Session.get(s_id=s_id)
            allowed = session.ask_for_upload(True)
744
745
        return jsonify({
            "success": True,
Floreal Cabanettes's avatar
Floreal Cabanettes committed
746
            "allowed": allowed
747
748
749
        })
    except DoesNotExist:
        return jsonify({"success": False, "message": "Session not initialized. Please refresh the page."})
750
751


752
753
@app.route("/ping-upload", methods=['POST'])
def ping_upload():
754
755
756
757
758
    if MODE == "webserver":
        s_id = request.form['s_id']
        with Session.connect():
            session = Session.get(s_id=s_id)
            session.ping()
759
    return "OK"
760
761


762
763
764
765
@app.route("/upload", methods=['POST'])
def upload():
    try:
        s_id = request.form['s_id']
766
767
768
769
770
771
        if MODE == "webserver":
            try:
                with Session.connect():
                    session = Session.get(s_id=s_id)
                    if session.ask_for_upload(False):
                        folder = session.upload_folder
Floreal Cabanettes's avatar
Floreal Cabanettes committed
772
                    else:
773
774
775
776
777
778
                        return jsonify({"files": [], "success": "ERR", "message": "Not allowed to upload!"})
            except DoesNotExist:
                return jsonify(
                    {"files": [], "success": "ERR", "message": "Session not initialized. Please refresh the page."})
        else:
            folder = s_id
Floreal Cabanettes's avatar
Floreal Cabanettes committed
779

780
        files = request.files[list(request.files.keys())[0]]
Floreal Cabanettes's avatar
Floreal Cabanettes committed
781

782
783
784
785
786
787
788
        if files:
            filename = files.filename
            folder_files = os.path.join(app.config["UPLOAD_FOLDER"], folder)
            if not os.path.exists(folder_files):
                os.makedirs(folder_files)
            filename = Functions.get_valid_uploaded_filename(filename, folder_files)
            mime_type = files.content_type
Floreal Cabanettes's avatar
Floreal Cabanettes committed
789

Floreal Cabanettes's avatar
Floreal Cabanettes committed
790
            if not Functions.allowed_file(files.filename, request.form['formats'].split(",")):
791
792
                result = UploadFile(name=filename, type_f=mime_type, size=0, not_allowed_msg="File type not allowed")
                shutil.rmtree(folder_files)
Floreal Cabanettes's avatar
Floreal Cabanettes committed
793

794
795
796
797
798
799
800
801
802
803
804
805
806
807
            else:
                # save file to disk
                uploaded_file_path = os.path.join(folder_files, filename)
                files.save(uploaded_file_path)

                # get file size after saving
                size = os.path.getsize(uploaded_file_path)

                # return json for js call back
                result = UploadFile(name=filename, type_f=mime_type, size=size)

            return jsonify({"files": [result.get_file()], "success": "OK"})

        return jsonify({"files": [], "success": "404", "message": "No file provided"})
808
    except:  # Except all possible exceptions to prevent crashes
Floreal Cabanettes's avatar
Floreal Cabanettes committed
809
        traceback.print_exc()
810
811
        return jsonify({"files": [], "success": "ERR", "message": "An unexpected error has occurred on upload. "
                                                                  "Please contact the support."})
812
813


814
815
816
817
818
819
@app.route("/send-mail/<id_res>", methods=['POST'])
def send_mail(id_res):
    allowed = False
    key_file = None
    if "key" in request.form:
        key = request.form["key"]
820
        res_dir = os.path.join(APP_DATA, id_res)
821
822
823
824
825
826
827
828
829
830
831
832
        key_file = os.path.join(res_dir, ".key")
        if os.path.exists(key_file):
            with open(key_file) as k_f:
                true_key = k_f.readline().strip("\n")
                allowed = key == true_key
    if allowed:
        os.remove(key_file)
        job_mng = JobManager(id_job=id_res, mailer=mailer)
        job_mng.set_inputs_from_res_dir()
        job_mng.send_mail()
        return "OK"
    else:
833
        abort(403)
834
835
836
837
838
839
840
841
842
843


@app.route("/delete/<id_res>", methods=['POST'])
def delete_job(id_res):
    job = JobManager(id_job=id_res)
    success, error = job.delete()
    return jsonify({
        "success": success,
        "error": error
    })