views.py 32.9 KB
Newer Older
1
from dgenies import app, app_title, app_folder, config_reader, mailer, APP_DATA, MODE, DEBUG, VERSION
2

Floreal Cabanettes's avatar
Floreal Cabanettes committed
3
import os
4
5
import time
import datetime
6
import shutil
7
import re
8
import threading
Floreal Cabanettes's avatar
Floreal Cabanettes committed
9
import traceback
10
import requests
11
from requests.exceptions import ConnectionError
12
import json
Floreal Cabanettes's avatar
Floreal Cabanettes committed
13
from flask import render_template, request, url_for, jsonify, Response, abort, send_file, Markup
14
from pathlib import Path
15
16
17
18
19
from dgenies.lib.paf import Paf
from dgenies.lib.job_manager import JobManager
from dgenies.lib.functions import Functions, ALLOWED_EXTENSIONS
from dgenies.lib.upload_file import UploadFile
from dgenies.lib.fasta import Fasta
20
from dgenies.tools import Tools
Floreal Cabanettes's avatar
Floreal Cabanettes committed
21
22
from markdown import Markdown
from markdown.extensions.toc import TocExtension
23
from markdown.extensions.tables import TableExtension
24
import tarfile
25
from jinja2 import Environment
26
27
28
if MODE == "webserver":
    from dgenies.database import Session, Gallery
    from peewee import DoesNotExist
29

30

Floreal Cabanettes's avatar
Floreal Cabanettes committed
31
32
33
34
@app.context_processor
def global_templates_variables():
    return {
        "title": app_title,
35
        "mode": MODE,
36
37
        "all_jobs": Functions.get_list_all_jobs(MODE),
        "debug": DEBUG
Floreal Cabanettes's avatar
Floreal Cabanettes committed
38
39
40
    }


41
42
43
# Main
@app.route("/", methods=['GET'])
def main():
44
45
46
47
48
49
    if MODE == "webserver":
        pict = Gallery.select().order_by("id")
        if len(pict) > 0:
            pict = pict[0].picture
        else:
            pict = None
Floreal Cabanettes's avatar
Floreal Cabanettes committed
50
51
    else:
        pict = None
Floreal Cabanettes's avatar
Floreal Cabanettes committed
52
    return render_template("index.html", menu="index", pict=pict)
Floreal Cabanettes's avatar
Floreal Cabanettes committed
53
54
55
56


@app.route("/run", methods=['GET'])
def run():
57
58
59
60
61
    tools = Tools().tools
    tools_names = sorted(list(tools.keys()), key=lambda x: (tools[x].order, tools[x].name))
    tools_ava = {}
    for tool_name, tool in tools.items():
        tools_ava[tool_name] = 1 if tool.all_vs_all is not None else 0
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
    if MODE == "webserver":
        with Session.connect():
            s_id = Session.new()
    else:
        upload_folder = Functions.random_string(20)
        tmp_dir = config_reader.upload_folder
        upload_folder_path = os.path.join(tmp_dir, upload_folder)
        while os.path.exists(upload_folder_path):
            upload_folder = Functions.random_string(20)
            upload_folder_path = os.path.join(tmp_dir, upload_folder)
        s_id = upload_folder
    id_job = Functions.random_string(5) + "_" + datetime.datetime.fromtimestamp(time.time()).strftime('%Y%m%d%H%M%S')
    if "id_job" in request.args:
        id_job = request.args["id_job"]
    email = ""
    if "email" in request.args:
        email = request.args["email"]
Floreal Cabanettes's avatar
Floreal Cabanettes committed
79
    return render_template("run.html", id_job=id_job, email=email,
80
                           menu="run", allowed_ext=ALLOWED_EXTENSIONS, s_id=s_id,
81
82
83
                           max_upload_file_size=config_reader.max_upload_file_size,
                           example=config_reader.example_target != "",
                           target=os.path.basename(config_reader.example_target),
84
                           query=os.path.basename(config_reader.example_query), tools_names=tools_names, tools=tools,
85
                           tools_ava=tools_ava, version=VERSION)
86
87


88
89
@app.route("/run-test", methods=['GET'])
def run_test():
90
91
92
93
94
95
96
    if MODE == "webserver":
        print(config_reader.allowed_ip_tests)
        if request.remote_addr not in config_reader.allowed_ip_tests:
            return abort(404)
        with Session.connect():
            return Session.new()
    return abort(500)
97
98


99
100
101
# Launch analysis
@app.route("/launch_analysis", methods=['POST'])
def launch_analysis():
102
103
104
105
106
107
108
109
110
111
112
113
    if MODE == "webserver":
        try:
            with Session.connect():
                session = Session.get(s_id=request.form["s_id"])
        except DoesNotExist:
            return jsonify({"success": False, "errors": ["Session has expired. Please refresh the page and try again"]})
        upload_folder = session.upload_folder
        # Delete session:
        session.delete_instance()
    else:
        upload_folder = request.form["s_id"]

114
115
    id_job = request.form["id_job"]
    email = request.form["email"]
116
117
118
119
    file_query = request.form["query"]
    file_query_type = request.form["query_type"]
    file_target = request.form["target"]
    file_target_type = request.form["target_type"]
120
121
122
    tool = request.form["tool"] if "tool" in request.form else None
    alignfile = request.form["alignfile"] if "alignfile" in request.form else None
    alignfile_type = request.form["alignfile_type"] if "alignfile_type" in request.form else None
123
124
    backup = request.form["backup"] if "backup" in request.form else None
    backup_type = request.form["backup_type"] if "backup_type" in request.form else None
125
126
127

    # Check form:
    form_pass = True
128
    errors = []
129

130
131
132
133
    if alignfile is not None and alignfile_type is None:
        errors.append("Server error: no alignfile_type in form. Please contact the support")
        form_pass = False

Floreal Cabanettes's avatar
Floreal Cabanettes committed
134
    if backup is not None and backup != "" and (backup_type is None or backup_type == ""):
135
136
137
        errors.append("Server error: no backup_type in form. Please contact the support")
        form_pass = False

Floreal Cabanettes's avatar
Floreal Cabanettes committed
138
    if backup is not None and backup != "":
139
140
141
142
        alignfile = ""
        file_query = ""
        file_target = ""
    else:
Floreal Cabanettes's avatar
Floreal Cabanettes committed
143
        backup = None
144
145
146
147
        if file_target == "":
            errors.append("No target fasta selected")
            form_pass = False

148
    if tool is not None and tool not in Tools().tools:
149
        errors.append("Tool unavailable: %s" % tool)
150
        form_pass = False
151

152
    if id_job == "":
153
        errors.append("Id of job not given")
154
155
        form_pass = False

156
157
158
159
    if MODE == "webserver":
        if email == "":
            errors.append("Email not given")
            form_pass = False
Floreal Cabanettes's avatar
Floreal Cabanettes committed
160
        elif not re.match(r"^[\w.\-]+@[\w\-.]{2,}\.[a-z]{2,4}$", email):
161
162
            errors.append("Email is invalid")
            form_pass = False
163
164
165

    # Form pass
    if form_pass:
166
        # Get final job id:
167
        id_job = re.sub('[^A-Za-z0-9_\-]+', '', id_job.replace(" ", "_"))
168
        id_job_orig = id_job
169
        i = 2
170
        while os.path.exists(os.path.join(APP_DATA, id_job)):
171
172
            id_job = id_job_orig + ("_%d" % i)
            i += 1
173

174
        folder_files = os.path.join(APP_DATA, id_job)
175
176
177
        os.makedirs(folder_files)

        # Save files:
178
179
        query = None
        if file_query != "":
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
            example = False
            if file_query.startswith("example://"):
                example = True
                query_path = config_reader.example_query
                query_name = os.path.basename(query_path)
                file_query_type = "local"
            else:
                query_name = os.path.splitext(file_query.replace(".gz", ""))[0] if file_query_type == "local" else None
                query_path = os.path.join(app.config["UPLOAD_FOLDER"], upload_folder, file_query) \
                    if file_query_type == "local" else file_query
                if file_query_type == "local" and not os.path.exists(query_path):
                    errors.append("Query file not correct!")
                    form_pass = False
            query = Fasta(name=query_name, path=query_path, type_f=file_query_type, example=example)
        example = False
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
        target = None
        if file_target != "":
            if file_target.startswith("example://"):
                example = True
                target_path = config_reader.example_target
                target_name = os.path.basename(target_path)
                file_target_type = "local"
            else:
                target_name = os.path.splitext(file_target.replace(".gz", ""))[0] if file_target_type == "local" else None
                target_path = os.path.join(app.config["UPLOAD_FOLDER"], upload_folder, file_target) \
                    if file_target_type == "local" else file_target
                if file_target_type == "local" and not os.path.exists(target_path):
                    errors.append("Target file not correct!")
                    form_pass = False
            target = Fasta(name=target_name, path=target_path, type_f=file_target_type, example=example)
210

211
        if alignfile is not None and alignfile != "" and backup is not None:
212
            Path(os.path.join(folder_files, ".align")).touch()
213
214
215

        align = None
        if alignfile is not None and alignfile != "":
216
217
218
219
220
221
222
223
            alignfile_name = os.path.splitext(alignfile)[0] if alignfile_type == "local" else None
            alignfile_path = os.path.join(app.config["UPLOAD_FOLDER"], upload_folder, alignfile) \
                if alignfile_type == "local" else alignfile
            if alignfile_type == "local" and not os.path.exists(alignfile_path):
                errors.append("Alignment file not correct!")
                form_pass = False
            align = Fasta(name=alignfile_name, path=alignfile_path, type_f=alignfile_type)

224
225
226
227
228
229
230
231
232
233
        bckp = None
        if backup is not None:
            backup_name = os.path.splitext(backup)[0] if backup_type == "local" else None
            backup_path = os.path.join(app.config["UPLOAD_FOLDER"], upload_folder, backup) \
                if backup_type == "local" else backup
            if backup_type == "local" and not os.path.exists(backup_path):
                errors.append("Backup file not correct!")
                form_pass = False
            bckp = Fasta(name=backup_name, path=backup_path, type_f=backup_type)

234
235
        if form_pass:
            # Launch job:
236
237
238
239
240
            job = JobManager(id_job=id_job,
                             email=email,
                             query=query,
                             target=target,
                             align=align,
241
                             backup=bckp,
242
243
                             mailer=mailer,
                             tool=tool)
244
245
246
247
248
249
            if MODE == "webserver":
                job.launch()
            else:
                job.launch_standalone()
            return jsonify({"success": True, "redirect": url_for(".status", id_job=id_job)})
    if not form_pass:
250
        return jsonify({"success": False, "errors": errors})
251
252
253
254
255
256


# Status of a job
@app.route('/status/<id_job>', methods=['GET'])
def status(id_job):
    job = JobManager(id_job)
257
258
259
260
261
262
263
264
265
266
267
268
    j_status = job.status()
    mem_peak = j_status["mem_peak"] if "mem_peak" in j_status else None
    if mem_peak is not None:
        mem_peak = "%.1f G" % (mem_peak / 1024.0 / 1024.0)
    time_e = j_status["time_elapsed"] if "time_elapsed" in j_status else None
    if time_e is not None:
        if time_e < 60:
            time_e = "%d secs" % time_e
        else:
            minutes = time_e // 60
            seconds = time_e - minutes * 60
            time_e = "%d min %d secs" % (minutes, seconds)
269
270
271
272
273
274
275
276
277
    format = request.args.get("format")
    if format is not None and format == "json":
        return jsonify({
            "status": j_status["status"],
            "error": j_status["error"].replace("#ID#", ""),
            "id_job": id_job,
            "mem_peak": mem_peak,
            "time_elapsed": time_e
        })
Floreal Cabanettes's avatar
Floreal Cabanettes committed
278
    return render_template("status.html", status=j_status["status"],
279
280
                           error=j_status["error"].replace("#ID#", ""),
                           id_job=id_job, menu="results", mem_peak=mem_peak,
281
                           time_elapsed=time_e, do_align=job.do_align(),
Floreal Cabanettes's avatar
Floreal Cabanettes committed
282
                           query_filtered=job.is_query_filtered(), target_filtered=job.is_target_filtered())
283
284


Floreal Cabanettes's avatar
Floreal Cabanettes committed
285
# Results path
286
@app.route("/result/<id_res>", methods=['GET'])
287
def result(id_res):
288
    res_dir = os.path.join(APP_DATA, id_res)
Floreal Cabanettes's avatar
Floreal Cabanettes committed
289
    return render_template("result.html", id=id_res, menu="result", current_result=id_res,
290
291
                           is_gallery=Functions.is_in_gallery(id_res, MODE),
                           fasta_file=Functions.query_fasta_file_exists(res_dir))
292
293


294
295
@app.route("/gallery", methods=['GET'])
def gallery():
296
    if MODE == "webserver":
Floreal Cabanettes's avatar
Floreal Cabanettes committed
297
        return render_template("gallery.html", items=Functions.get_gallery_items(), menu="gallery")
298
    return abort(404)
299
300
301
302


@app.route("/gallery/<filename>", methods=['GET'])
def gallery_file(filename):
303
304
305
306
307
308
    if MODE == "webserver":
        try:
            return send_file(os.path.join(config_reader.app_data, "gallery", filename))
        except FileNotFoundError:
            abort(404)
    return abort(404)
309
310


311
def get_file(file, gzip=False):  # pragma: no cover
Floreal Cabanettes's avatar
Floreal Cabanettes committed
312
313
314
315
316
317
    try:
        # Figure out how flask returns static files
        # Tried:
        # - render_template
        # - send_file
        # This should not be so non-obvious
318
        return open(file, "rb" if gzip else "r").read()
319
320
    except FileNotFoundError:
        abort(404)
Floreal Cabanettes's avatar
Floreal Cabanettes committed
321
    except IOError as exc:
322
        print(exc.__traceback__)
323
        abort(500)
Floreal Cabanettes's avatar
Floreal Cabanettes committed
324
325


Floreal Cabanettes's avatar
Floreal Cabanettes committed
326
327
@app.route("/documentation/run", methods=['GET'])
def documentation_run():
328
    with open(os.path.join(app_folder, "md", "doc_run.md" if MODE == "webserver" else "doc_run_standalone.md"), "r",
Floreal Cabanettes's avatar
Floreal Cabanettes committed
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
              encoding='utf-8') as install_instr:
        content = install_instr.read()
    md = Markdown(extensions=[TocExtension(baselevel=1)])
    max_upload_file_size = config_reader.max_upload_file_size
    if max_upload_file_size == -1:
        max_upload_file_size = "no limit"
    else:
        max_upload_file_size = Functions.get_readable_size(max_upload_file_size, 0)
    max_upload_size = config_reader.max_upload_size
    if max_upload_size == -1:
        max_upload_size = "no limit"
    else:
        max_upload_size = Functions.get_readable_size(max_upload_size, 0)
    max_upload_size_ava = config_reader.max_upload_size_ava
    if max_upload_size_ava == -1:
        max_upload_size_ava = "no limit"
    else:
        max_upload_size_ava = Functions.get_readable_size(max_upload_size_ava, 0)
    content = Markup(md.convert(content)).replace("###size###", max_upload_file_size)\
                                         .replace("###size_unc###", max_upload_size)\
                                         .replace("###size_ava###", max_upload_size_ava)
    toc = Markup(md.toc)
Floreal Cabanettes's avatar
Floreal Cabanettes committed
351
    return render_template("documentation.html", menu="documentation", content=content, toc=toc)
Floreal Cabanettes's avatar
Floreal Cabanettes committed
352
353


354
355
@app.route("/documentation/result", methods=['GET'])
def documentation_result():
356
    with open(os.path.join(app_folder, "md", "user_manual.md"), "r",
357
358
359
360
361
362
363
364
              encoding='utf-8') as install_instr:
        content = install_instr.read()
    md = Markdown(extensions=[TocExtension(baselevel=1)])
    content = Markup(md.convert(content))
    toc = Markup(md.toc)
    return render_template("documentation.html", menu="documentation", content=content, toc=toc)


365
366
@app.route("/documentation/formats", methods=['GET'])
def documentation_formats():
367
    with open(os.path.join(app_folder, "md", "doc_formats.md"), "r",
368
369
370
371
372
373
374
375
              encoding='utf-8') as install_instr:
        content = install_instr.read()
    md = Markdown(extensions=[TocExtension(baselevel=1), TableExtension()])
    content = Markup(md.convert(content))
    toc = Markup(md.toc)
    return render_template("documentation.html", menu="documentation", content=content, toc=toc)


376
377
378
379
380
381
382
383
384
385
386
@app.route("/documentation/dotplot", methods=['GET'])
def documentation_dotplot():
    with open(os.path.join(app_folder, "md", "doc_dotplot.md"), "r",
              encoding='utf-8') as install_instr:
        content = install_instr.read()
    md = Markdown(extensions=[TocExtension(baselevel=1)])
    content = Markup(md.convert(content))
    toc = Markup(md.toc)
    return render_template("documentation.html", menu="documentation", content=content, toc=toc)


Floreal Cabanettes's avatar
Floreal Cabanettes committed
387
388
@app.route("/install", methods=['GET'])
def install():
389
    latest = ""
390
    win32 = ""
391
392
393
394
395
396
397
398
399
400
401
402
    try:
        call = requests.get("https://api.github.com/repos/genotoul-bioinfo/dgenies/releases/latest")
        if call.ok:
            release = json.loads(call.content.decode("utf-8"))
            if "tag_name" in release:
                latest = release["tag_name"][1:]
                for asset in release["assets"]:
                    if asset["name"].endswith(".exe"):
                        win32 = asset["browser_download_url"]
                        break
    except ConnectionError:
        pass
403

404
    with open(os.path.join(app_folder, "md", "INSTALL.md"), "r", encoding='utf-8') as install_instr:
Floreal Cabanettes's avatar
Floreal Cabanettes committed
405
        content = install_instr.read()
406
407
    env = Environment()
    template = env.from_string(content)
408
    content = template.render(version=latest, win32=win32)
Floreal Cabanettes's avatar
Floreal Cabanettes committed
409
410
411
    md = Markdown(extensions=[TocExtension(baselevel=1)])
    content = Markup(md.convert(content))
    toc = Markup(md.toc)
Floreal Cabanettes's avatar
Floreal Cabanettes committed
412
    return render_template("documentation.html", menu="install", content=content, toc=toc)
Floreal Cabanettes's avatar
Floreal Cabanettes committed
413

Floreal Cabanettes's avatar
Floreal Cabanettes committed
414

415
416
417
418
@app.route("/contact", methods=['GET'])
def contact():
    return render_template("contact.html", menu="contact")

Floreal Cabanettes's avatar
Floreal Cabanettes committed
419

Floreal Cabanettes's avatar
Floreal Cabanettes committed
420
421
422
423
424
425
426
427
428
429
430
@app.route("/paf/<id_res>", methods=['GET'])
def download_paf(id_res):
    map_file = os.path.join(APP_DATA, id_res, "map.paf.sorted")
    if not os.path.exists(map_file):
        map_file = os.path.join(APP_DATA, id_res, "map.paf")
    if not os.path.exists(map_file):
        abort(404)
    content = get_file(map_file)
    return Response(content, mimetype="text/plain")


Floreal Cabanettes's avatar
Floreal Cabanettes committed
431
# Get graph (ajax request)
432
433
434
@app.route('/get_graph', methods=['POST'])
def get_graph():
    id_f = request.form["id"]
435
436
437
    paf = os.path.join(APP_DATA, id_f, "map.paf")
    idx1 = os.path.join(APP_DATA, id_f, "query.idx")
    idx2 = os.path.join(APP_DATA, id_f, "target.idx")
438

439
    paf = Paf(paf, idx1, idx2)
440

441
442
    if paf.parsed:
        res = paf.get_d3js_data()
443
        res["success"] = True
444
445
446
447
448
449
        return jsonify(res)
    return jsonify({"success": False, "message": paf.error})


@app.route('/sort/<id_res>', methods=['POST'])
def sort_graph(id_res):
450
    if not os.path.exists(os.path.join(APP_DATA, id_res, ".all-vs-all")):
451
452
453
        paf_file = os.path.join(APP_DATA, id_res, "map.paf")
        idx1 = os.path.join(APP_DATA, id_res, "query.idx")
        idx2 = os.path.join(APP_DATA, id_res, "target.idx")
454
        paf = Paf(paf_file, idx1, idx2, False)
455
456
457
458
459
460
461
        paf.sort()
        if paf.parsed:
            res = paf.get_d3js_data()
            res["success"] = True
            return jsonify(res)
        return jsonify({"success": False, "message": paf.error})
    return jsonify({"success": False, "message": "Sort is not available for All-vs-All mode"})
Floreal Cabanettes's avatar
Floreal Cabanettes committed
462

Floréal Cabanettes's avatar
Floréal Cabanettes committed
463

464
465
466
467
@app.route('/reverse-contig/<id_res>', methods=['POST'])
def reverse_contig(id_res):
    contig_name = request.form["contig"]
    if not os.path.exists(os.path.join(APP_DATA, id_res, ".all-vs-all")):
468
469
470
        paf_file = os.path.join(APP_DATA, id_res, "map.paf")
        idx1 = os.path.join(APP_DATA, id_res, "query.idx")
        idx2 = os.path.join(APP_DATA, id_res, "target.idx")
471
472
473
474
475
476
477
478
479
480
        paf = Paf(paf_file, idx1, idx2, False)
        paf.reverse_contig(contig_name)
        if paf.parsed:
            res = paf.get_d3js_data()
            res["success"] = True
            return jsonify(res)
        return jsonify({"success": False, "message": paf.error})
    return jsonify({"success": False, "message": "Sort is not available for All-vs-All mode"})


481
482
@app.route('/freenoise/<id_res>', methods=['POST'])
def free_noise(id_res):
483
484
485
    paf_file = os.path.join(APP_DATA, id_res, "map.paf")
    idx1 = os.path.join(APP_DATA, id_res, "query.idx")
    idx2 = os.path.join(APP_DATA, id_res, "target.idx")
486
487
488
489
490
491
492
493
494
    paf = Paf(paf_file, idx1, idx2, False)
    paf.parse_paf(noise=request.form["noise"] == "1")
    if paf.parsed:
        res = paf.get_d3js_data()
        res["success"] = True
        return jsonify(res)
    return jsonify({"success": False, "message": paf.error})


495
496
@app.route('/get-fasta-query/<id_res>', methods=['POST'])
def build_fasta(id_res):
497
    res_dir = os.path.join(APP_DATA, id_res)
498
499
    lock_query = os.path.join(res_dir, ".query-fasta-build")
    is_sorted = os.path.exists(os.path.join(res_dir, ".sorted"))
500
    compressed = request.form["gzip"].lower() == "true"
501
502
503
504
505
    query_fasta = Functions.get_fasta_file(res_dir, "query", is_sorted)
    if query_fasta is not None:
        if is_sorted and not query_fasta.endswith(".sorted"):
            # Do the sort
            Path(lock_query).touch()
506
            if not compressed or MODE == "standalone":  # If compressed, it will took a long time, so not wait
507
                Path(lock_query + ".pending").touch()
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
            index_file = os.path.join(res_dir, "query.idx.sorted")
            if MODE == "webserver":
                thread = threading.Timer(1, Functions.sort_fasta, kwargs={
                    "job_name": id_res,
                    "fasta_file": query_fasta,
                    "index_file": index_file,
                    "lock_file": lock_query,
                    "compress": compressed,
                    "mailer": mailer,
                    "mode": MODE
                })
                thread.start()
            else:
                Functions.sort_fasta(job_name=id_res,
                                     fasta_file=query_fasta,
                                     index_file=index_file,
                                     lock_file=lock_query,
                                     compress=compressed,
                                     mailer=None,
                                     mode=MODE)
528
            if not compressed or MODE == "standalone":
529
530
                if MODE == "webserver":
                    i = 0
531
                    time.sleep(5)
532
533
534
                    while os.path.exists(lock_query) and (i < 2 or MODE == "standalone"):
                        i += 1
                        time.sleep(5)
Floreal Cabanettes's avatar
Floreal Cabanettes committed
535
                os.remove(lock_query + ".pending")
536
537
538
                if os.path.exists(lock_query):
                    return jsonify({"success": True, "status": 1, "status_message": "In progress"})
                return jsonify({"success": True, "status": 2, "status_message": "Done",
539
                                "gzip": compressed})
540
541
            else:
                return jsonify({"success": True, "status": 1, "status_message": "In progress"})
542
543
544
545
546
        elif is_sorted and os.path.exists(lock_query):
            # Sort is already in progress
            return jsonify({"success": True, "status": 1, "status_message": "In progress"})
        else:
            # No sort to do or sort done
547
548
549
550
551
552
553
            if compressed and not query_fasta.endswith(".gz.fasta"):
                # If compressed file is asked, we must compress it now if not done before...
                Path(lock_query).touch()
                thread = threading.Timer(1, Functions.compress_and_send_mail, kwargs={
                    "job_name": id_res,
                    "fasta_file": query_fasta,
                    "index_file": os.path.join(res_dir, "query.idx.sorted"),
554
                    "lock_file": lock_query,
555
556
557
558
559
                    "compressed": compressed,
                    "mailer": mailer
                })
                thread.start()
                return jsonify({"success": True, "status": 1, "status_message": "In progress"})
560
561
562
563
564
565
566
            return jsonify({"success": True, "status": 2, "status_message": "Done",
                            "gzip": query_fasta.endswith(".gz") or query_fasta.endswith(".gz.sorted")})
    else:
        return jsonify({"success": False,
                        "message": "Unable to get fasta file for query. Please contact us to report the bug"})


567
def build_query_as_reference(id_res):
568
569
570
571
572
    paf_file = os.path.join(APP_DATA, id_res, "map.paf")
    idx1 = os.path.join(APP_DATA, id_res, "query.idx")
    idx2 = os.path.join(APP_DATA, id_res, "target.idx")
    paf = Paf(paf_file, idx1, idx2, False, mailer=mailer, id_job=id_res)
    paf.parse_paf(False, True)
573
574
575
576
577
578
579
580
581
582
    if MODE == "webserver":
        thread = threading.Timer(0, paf.build_query_chr_as_reference)
        thread.start()
        return True
    return paf.build_query_chr_as_reference()


@app.route('/build-query-as-reference/<id_res>', methods=['POST'])
def post_query_as_reference(id_res):
    build_query_as_reference(id_res)
583
584
585
    return jsonify({"success": True})


586
587
588
589
590
591
592
@app.route('/get-query-as-reference/<id_res>', methods=['GET'])
def get_query_as_reference(id_res):
    if MODE != "standalone":
        return abort(404)
    return send_file(build_query_as_reference(id_res))


593
594
595
596
597
598
599
600
@app.route('/download/<id_res>/<filename>')
def download_file(id_res, filename):
    file_dl = os.path.join(APP_DATA, id_res, filename)
    if os.path.isfile(file_dl):
        return send_file(file_dl)
    return abort(404)


601
602
603
@app.route('/fasta-query/<id_res>', defaults={'filename': ""}, methods=['GET'])
@app.route('/fasta-query/<id_res>/<filename>', methods=['GET'])  # Use fake URL in mail to set download file name
def dl_fasta(id_res, filename):
604
    res_dir = os.path.join(APP_DATA, id_res)
605
606
607
608
609
610
611
612
613
614
615
616
617
    lock_query = os.path.join(res_dir, ".query-fasta-build")
    is_sorted = os.path.exists(os.path.join(res_dir, ".sorted"))
    if not os.path.exists(lock_query) or not is_sorted:
        query_fasta = Functions.get_fasta_file(res_dir, "query", is_sorted)
        if query_fasta is not None:
            if query_fasta.endswith(".gz") or query_fasta.endswith(".gz.sorted"):
                content = get_file(query_fasta, True)
                return Response(content, mimetype="application/gzip")
            content = get_file(query_fasta)
            return Response(content, mimetype="text/plain")
    abort(404)


618
619
@app.route('/qt-assoc/<id_res>', methods=['GET'])
def qt_assoc(id_res):
620
    res_dir = os.path.join(APP_DATA, id_res)
621
    if os.path.exists(res_dir) and os.path.isdir(res_dir):
622
623
624
        paf_file = os.path.join(APP_DATA, id_res, "map.paf")
        idx1 = os.path.join(APP_DATA, id_res, "query.idx")
        idx2 = os.path.join(APP_DATA, id_res, "target.idx")
625
626
627
628
629
630
631
632
633
634
635
636
        try:
            paf = Paf(paf_file, idx1, idx2, False)
            paf.parse_paf(False)
        except FileNotFoundError:
            print("Unable to load data!")
            abort(404)
            return False
        csv_content = paf.build_query_on_target_association_file()
        return Response(csv_content, mimetype="text/plain")
    abort(404)


637
638
639
640
641
642
@app.route('/no-assoc/<id_res>', methods=['POST'])
def no_assoc(id_res):
    """
    Get contigs that match with None target
    :param id_res: id of the result
    """
643
    res_dir = os.path.join(APP_DATA, id_res)
644
    if os.path.exists(res_dir) and os.path.isdir(res_dir):
645
646
647
        paf_file = os.path.join(APP_DATA, id_res, "map.paf")
        idx1 = os.path.join(APP_DATA, id_res, "query.idx")
        idx2 = os.path.join(APP_DATA, id_res, "target.idx")
648
649
650
651
652
653
        try:
            paf = Paf(paf_file, idx1, idx2, False)
        except FileNotFoundError:
            print("Unable to load data!")
            abort(404)
            return False
654
        file_content = paf.build_list_no_assoc(request.form["to"])
655
656
657
658
659
660
661
662
        empty = file_content == "\n"
        return jsonify({
            "file_content": file_content,
            "empty": empty
        })
    abort(404)


Floreal Cabanettes's avatar
Floreal Cabanettes committed
663
664
@app.route('/summary/<id_res>', methods=['POST'])
def summary(id_res):
665
666
667
668
669
670
671
672
673
674
    paf_file = os.path.join(APP_DATA, id_res, "map.paf")
    idx1 = os.path.join(APP_DATA, id_res, "query.idx")
    idx2 = os.path.join(APP_DATA, id_res, "target.idx")
    try:
        paf = Paf(paf_file, idx1, idx2, False)
    except FileNotFoundError:
        return jsonify({
            "success": False,
            "message": "Unable to load data!"
        })
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
    percents = None
    s_status = "waiting"  # Accepted values: waiting, done, fail
    status_file = os.path.join(APP_DATA, id_res, ".summarize")
    fail_file = status_file + ".fail"
    if not os.path.exists(status_file):  # The job is finished or not started
        if not os.path.exists(fail_file):  # The job has not started yet or has successfully ended
            percents = paf.get_summary_stats()
            if percents is None:  # The job has not started yet
                Path(status_file).touch()
                thread = threading.Timer(0, paf.build_summary_stats, kwargs={"status_file": status_file})
                thread.start()
            else:  # The job has successfully ended
                s_status = "done"
        else:  # The job has failed
            s_status = "fail"

    if s_status == "waiting":  # The job is running
        # Check if the job end in the next 30 seconds
        nb_iter = 0
        while os.path.exists(status_file) and not os.path.exists(fail_file) and nb_iter < 10:
            time.sleep(3)
            nb_iter += 1
        if not os.path.exists(status_file):  # The job has ended
            percents = paf.get_summary_stats()
            if percents is None:  # The job has failed
                s_status = "fail"
            else:  # The job has successfully ended
                s_status = "done"

    if s_status == "fail":
705
706
707
708
709
710
        return jsonify({
            "success": False,
            "message": "Build of summary failed. Please contact us to report the bug"
        })
    return jsonify({
        "success": True,
711
712
        "percents": percents,
        "status": s_status
713
    })
Floreal Cabanettes's avatar
Floreal Cabanettes committed
714
715


716
717
718
719
720
721
722
723
724
725
@app.route('/backup/<id_res>')
def get_backup_file(id_res):
    res_dir = os.path.join(APP_DATA, id_res)
    tar = os.path.join(res_dir, "%s.tar" % id_res)
    with tarfile.open(tar, "w") as tarf:
        for file in ("map.paf", "target.idx", "query.idx"):
            tarf.add(os.path.join(res_dir, file), arcname=file)
    return send_file(tar)


726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
def get_filter_out(id_res, type_f):
    filter_file = os.path.join(APP_DATA, id_res, ".filter-" + type_f)
    return Response(get_file(filter_file), mimetype="text/plain")


@app.route('/filter-out/<id_res>/query')
def get_filter_out_query(id_res):
    return get_filter_out(id_res=id_res, type_f="query")


@app.route('/filter-out/<id_res>/target')
def get_filter_out_target(id_res):
    return get_filter_out(id_res=id_res, type_f="target")


741
742
@app.route("/ask-upload", methods=['POST'])
def ask_upload():
743
744
745
746
747
    if MODE == "standalone":
        return jsonify({
            "success": True,
            "allowed": True
        })
748
749
    try:
        s_id = request.form['s_id']
Floreal Cabanettes's avatar
Floreal Cabanettes committed
750
751
752
        with Session.connect():
            session = Session.get(s_id=s_id)
            allowed = session.ask_for_upload(True)
753
754
        return jsonify({
            "success": True,
Floreal Cabanettes's avatar
Floreal Cabanettes committed
755
            "allowed": allowed
756
757
758
        })
    except DoesNotExist:
        return jsonify({"success": False, "message": "Session not initialized. Please refresh the page."})
759
760


761
762
@app.route("/ping-upload", methods=['POST'])
def ping_upload():
763
764
765
766
767
    if MODE == "webserver":
        s_id = request.form['s_id']
        with Session.connect():
            session = Session.get(s_id=s_id)
            session.ping()
768
    return "OK"
769
770


771
772
773
774
@app.route("/upload", methods=['POST'])
def upload():
    try:
        s_id = request.form['s_id']
775
776
777
778
779
780
        if MODE == "webserver":
            try:
                with Session.connect():
                    session = Session.get(s_id=s_id)
                    if session.ask_for_upload(False):
                        folder = session.upload_folder
Floreal Cabanettes's avatar
Floreal Cabanettes committed
781
                    else:
782
783
784
785
786
787
                        return jsonify({"files": [], "success": "ERR", "message": "Not allowed to upload!"})
            except DoesNotExist:
                return jsonify(
                    {"files": [], "success": "ERR", "message": "Session not initialized. Please refresh the page."})
        else:
            folder = s_id
Floreal Cabanettes's avatar
Floreal Cabanettes committed
788

789
        files = request.files[list(request.files.keys())[0]]
Floreal Cabanettes's avatar
Floreal Cabanettes committed
790

791
792
793
794
795
796
797
        if files:
            filename = files.filename
            folder_files = os.path.join(app.config["UPLOAD_FOLDER"], folder)
            if not os.path.exists(folder_files):
                os.makedirs(folder_files)
            filename = Functions.get_valid_uploaded_filename(filename, folder_files)
            mime_type = files.content_type
Floreal Cabanettes's avatar
Floreal Cabanettes committed
798

Floreal Cabanettes's avatar
Floreal Cabanettes committed
799
            if not Functions.allowed_file(files.filename, request.form['formats'].split(",")):
800
801
                result = UploadFile(name=filename, type_f=mime_type, size=0, not_allowed_msg="File type not allowed")
                shutil.rmtree(folder_files)
Floreal Cabanettes's avatar
Floreal Cabanettes committed
802

803
804
805
806
807
808
809
810
811
812
813
814
815
816
            else:
                # save file to disk
                uploaded_file_path = os.path.join(folder_files, filename)
                files.save(uploaded_file_path)

                # get file size after saving
                size = os.path.getsize(uploaded_file_path)

                # return json for js call back
                result = UploadFile(name=filename, type_f=mime_type, size=size)

            return jsonify({"files": [result.get_file()], "success": "OK"})

        return jsonify({"files": [], "success": "404", "message": "No file provided"})
817
    except:  # Except all possible exceptions to prevent crashes
Floreal Cabanettes's avatar
Floreal Cabanettes committed
818
        traceback.print_exc()
819
820
        return jsonify({"files": [], "success": "ERR", "message": "An unexpected error has occurred on upload. "
                                                                  "Please contact the support."})
821
822


823
824
825
826
827
828
@app.route("/send-mail/<id_res>", methods=['POST'])
def send_mail(id_res):
    allowed = False
    key_file = None
    if "key" in request.form:
        key = request.form["key"]
829
        res_dir = os.path.join(APP_DATA, id_res)
830
831
832
833
834
835
836
837
838
839
840
841
        key_file = os.path.join(res_dir, ".key")
        if os.path.exists(key_file):
            with open(key_file) as k_f:
                true_key = k_f.readline().strip("\n")
                allowed = key == true_key
    if allowed:
        os.remove(key_file)
        job_mng = JobManager(id_job=id_res, mailer=mailer)
        job_mng.set_inputs_from_res_dir()
        job_mng.send_mail()
        return "OK"
    else:
842
        abort(403)
843
844
845
846
847
848
849
850
851
852


@app.route("/delete/<id_res>", methods=['POST'])
def delete_job(id_res):
    job = JobManager(id_job=id_res)
    success, error = job.delete()
    return jsonify({
        "success": success,
        "error": error
    })