views.py 20.8 KB
Newer Older
1
from dgenies import app, app_title, config_reader, mailer, APP_DATA
2

Floreal Cabanettes's avatar
Floreal Cabanettes committed
3
import os
4
5
import time
import datetime
6
import shutil
7
import re
8
import threading
9
from flask import render_template, request, url_for, jsonify, Response, abort, send_file
10
from pathlib import Path
11
12
13
14
15
16
from dgenies.lib.paf import Paf
from dgenies.lib.job_manager import JobManager
from dgenies.lib.functions import Functions, ALLOWED_EXTENSIONS
from dgenies.lib.upload_file import UploadFile
from dgenies.lib.fasta import Fasta
from dgenies.database import Session
17
from peewee import DoesNotExist
18

19

Floreal Cabanettes's avatar
Floreal Cabanettes committed
20
21
22
23
24
25
@app.context_processor
def get_launched_results():
    cookie = request.cookies.get("results")
    return {"results": cookie.split("|") if cookie is not None else set()}


26
27
28
# Main
@app.route("/", methods=['GET'])
def main():
Floreal Cabanettes's avatar
Floreal Cabanettes committed
29
30
31
32
33
    return render_template("index.html", title=app_title, menu="index")


@app.route("/run", methods=['GET'])
def run():
Floreal Cabanettes's avatar
Floreal Cabanettes committed
34
35
36
37
38
39
40
41
42
43
44
    with Session.connect():
        s_id = Session.new()
        id_job = Functions.random_string(5) + "_" + datetime.datetime.fromtimestamp(time.time()).strftime('%Y%m%d%H%M%S')
        if "id_job" in request.args:
            id_job = request.args["id_job"]
        email = ""
        if "email" in request.args:
            email = request.args["email"]
        return render_template("run.html", title=app_title, id_job=id_job, email=email,
                               menu="run", allowed_ext=ALLOWED_EXTENSIONS, s_id=s_id,
                               max_upload_file_size=config_reader.max_upload_file_size)
45
46


47
48
49
50
51
@app.route("/run-test", methods=['GET'])
def run_test():
    print(config_reader.allowed_ip_tests)
    if request.remote_addr not in config_reader.allowed_ip_tests:
        return abort(404)
Floreal Cabanettes's avatar
Floreal Cabanettes committed
52
53
    with Session.connect():
        return Session.new()
54
55


56
57
58
# Launch analysis
@app.route("/launch_analysis", methods=['POST'])
def launch_analysis():
59
    try:
Floreal Cabanettes's avatar
Floreal Cabanettes committed
60
61
        with Session.connect():
            session = Session.get(s_id=request.form["s_id"])
62
63
64
65
66
67
    except DoesNotExist:
        return jsonify({"success": False, "errors": ["Session has expired. Please refresh the page and try again"]})
    # Reset session upload:
    session.allow_upload = False
    session.position = -1
    session.save()
68
69
    id_job = request.form["id_job"]
    email = request.form["email"]
70
71
72
73
    file_query = request.form["query"]
    file_query_type = request.form["query_type"]
    file_target = request.form["target"]
    file_target_type = request.form["target_type"]
74
75
76

    # Check form:
    form_pass = True
77
    errors = []
78
    if id_job == "":
79
        errors.append("Id of job not given")
80
81
82
        form_pass = False

    if email == "":
83
        errors.append("Email not given")
84
        form_pass = False
85
86
    if file_target == "":
        errors.append("No target fasta selected")
87
88
89
90
        form_pass = False

    # Form pass
    if form_pass:
91
        # Get final job id:
92
        id_job = re.sub('[^A-Za-z0-9_\-]+', '', id_job.replace(" ", "_"))
93
        id_job_orig = id_job
94
        i = 2
95
        while os.path.exists(os.path.join(APP_DATA, id_job)):
96
97
            id_job = id_job_orig + ("_%d" % i)
            i += 1
98

99
        folder_files = os.path.join(APP_DATA, id_job)
100
101
102
        os.makedirs(folder_files)

        # Save files:
103
        query = None
104
        upload_folder = session.upload_folder
105
106
        if file_query != "":
            query_name = os.path.splitext(file_query.replace(".gz", ""))[0] if file_query_type == "local" else None
107
            query_path = os.path.join(app.config["UPLOAD_FOLDER"], upload_folder, file_query) \
108
109
110
                if file_query_type == "local" else file_query
            query = Fasta(name=query_name, path=query_path, type_f=file_query_type)
        target_name = os.path.splitext(file_target.replace(".gz", ""))[0] if file_target_type == "local" else None
111
        target_path = os.path.join(app.config["UPLOAD_FOLDER"], upload_folder, file_target) \
112
113
            if file_target_type == "local" else file_target
        target = Fasta(name=target_name, path=target_path, type_f=file_target_type)
114
115

        # Launch job:
Floreal Cabanettes's avatar
Floreal Cabanettes committed
116
        job = JobManager(id_job, email, query, target, mailer)
117
        job.launch()
118
119
120

        # Delete session:
        session.delete_instance()
121
        return jsonify({"success": True, "redirect": url_for(".status", id_job=id_job)})
122
    else:
123
        return jsonify({"success": False, "errors": errors})
124
125
126
127
128
129


# Status of a job
@app.route('/status/<id_job>', methods=['GET'])
def status(id_job):
    job = JobManager(id_job)
130
131
132
133
134
135
136
137
138
139
140
141
    j_status = job.status()
    mem_peak = j_status["mem_peak"] if "mem_peak" in j_status else None
    if mem_peak is not None:
        mem_peak = "%.1f G" % (mem_peak / 1024.0 / 1024.0)
    time_e = j_status["time_elapsed"] if "time_elapsed" in j_status else None
    if time_e is not None:
        if time_e < 60:
            time_e = "%d secs" % time_e
        else:
            minutes = time_e // 60
            seconds = time_e - minutes * 60
            time_e = "%d min %d secs" % (minutes, seconds)
142
143
144
145
146
147
148
149
150
    format = request.args.get("format")
    if format is not None and format == "json":
        return jsonify({
            "status": j_status["status"],
            "error": j_status["error"].replace("#ID#", ""),
            "id_job": id_job,
            "mem_peak": mem_peak,
            "time_elapsed": time_e
        })
151
152
153
    return render_template("status.html", title=app_title, status=j_status["status"],
                           error=j_status["error"].replace("#ID#", ""),
                           id_job=id_job, menu="results", mem_peak=mem_peak,
154
155
                           time_elapsed=time_e,
                           query_filtered=job.is_query_filtered(), target_filtered=job.is_target_filtered())
156
157


Floreal Cabanettes's avatar
Floreal Cabanettes committed
158
# Results path
159
@app.route("/result/<id_res>", methods=['GET'])
160
def result(id_res):
161
    my_render = render_template("result.html", title=app_title, id=id_res, menu="result", current_result=id_res)
Floreal Cabanettes's avatar
Floreal Cabanettes committed
162
163
164
165
166
167
168
    response = app.make_response(my_render)
    cookie = request.cookies.get("results")
    cookie = cookie.split("|") if cookie is not None else []
    if id_res not in cookie:
        cookie.insert(0, id_res)
    response.set_cookie(key="results", value="|".join(cookie), path="/")
    return response
169
170


171
172
173
174
175
176
177
178
179
180
181
182
183
@app.route("/gallery", methods=['GET'])
def gallery():
    return render_template("gallery.html", items=Functions.get_gallery_items(), menu="gallery")


@app.route("/gallery/<filename>", methods=['GET'])
def gallery_file(filename):
    try:
        return send_file(os.path.join(config_reader.app_data, "gallery", filename))
    except FileNotFoundError:
        abort(404)


184
def get_file(file, gzip=False):  # pragma: no cover
Floreal Cabanettes's avatar
Floreal Cabanettes committed
185
186
187
188
189
190
    try:
        # Figure out how flask returns static files
        # Tried:
        # - render_template
        # - send_file
        # This should not be so non-obvious
191
        return open(file, "rb" if gzip else "r").read()
192
193
    except FileNotFoundError:
        abort(404)
Floreal Cabanettes's avatar
Floreal Cabanettes committed
194
    except IOError as exc:
195
196
        print(exc)
        abort(500)
Floreal Cabanettes's avatar
Floreal Cabanettes committed
197
198
199
200
201
202
203
204
205
206
207
208
209


@app.route("/paf/<id_res>", methods=['GET'])
def download_paf(id_res):
    map_file = os.path.join(APP_DATA, id_res, "map.paf.sorted")
    if not os.path.exists(map_file):
        map_file = os.path.join(APP_DATA, id_res, "map.paf")
    if not os.path.exists(map_file):
        abort(404)
    content = get_file(map_file)
    return Response(content, mimetype="text/plain")


Floreal Cabanettes's avatar
Floreal Cabanettes committed
210
# Get graph (ajax request)
211
212
213
@app.route('/get_graph', methods=['POST'])
def get_graph():
    id_f = request.form["id"]
214
215
216
    paf = os.path.join(APP_DATA, id_f, "map.paf")
    idx1 = os.path.join(APP_DATA, id_f, "query.idx")
    idx2 = os.path.join(APP_DATA, id_f, "target.idx")
217

218
    paf = Paf(paf, idx1, idx2)
219

220
221
    if paf.parsed:
        res = paf.get_d3js_data()
222
        res["success"] = True
223
224
225
226
227
228
        return jsonify(res)
    return jsonify({"success": False, "message": paf.error})


@app.route('/sort/<id_res>', methods=['POST'])
def sort_graph(id_res):
229
    if not os.path.exists(os.path.join(APP_DATA, id_res, ".all-vs-all")):
230
231
232
        paf_file = os.path.join(APP_DATA, id_res, "map.paf")
        idx1 = os.path.join(APP_DATA, id_res, "query.idx")
        idx2 = os.path.join(APP_DATA, id_res, "target.idx")
233
        paf = Paf(paf_file, idx1, idx2, False)
234
235
236
237
238
239
240
        paf.sort()
        if paf.parsed:
            res = paf.get_d3js_data()
            res["success"] = True
            return jsonify(res)
        return jsonify({"success": False, "message": paf.error})
    return jsonify({"success": False, "message": "Sort is not available for All-vs-All mode"})
Floreal Cabanettes's avatar
Floreal Cabanettes committed
241

Floréal Cabanettes's avatar
Floréal Cabanettes committed
242

243
244
245
246
@app.route('/reverse-contig/<id_res>', methods=['POST'])
def reverse_contig(id_res):
    contig_name = request.form["contig"]
    if not os.path.exists(os.path.join(APP_DATA, id_res, ".all-vs-all")):
247
248
249
        paf_file = os.path.join(APP_DATA, id_res, "map.paf")
        idx1 = os.path.join(APP_DATA, id_res, "query.idx")
        idx2 = os.path.join(APP_DATA, id_res, "target.idx")
250
251
252
253
254
255
256
257
258
259
        paf = Paf(paf_file, idx1, idx2, False)
        paf.reverse_contig(contig_name)
        if paf.parsed:
            res = paf.get_d3js_data()
            res["success"] = True
            return jsonify(res)
        return jsonify({"success": False, "message": paf.error})
    return jsonify({"success": False, "message": "Sort is not available for All-vs-All mode"})


260
261
@app.route('/freenoise/<id_res>', methods=['POST'])
def free_noise(id_res):
262
263
264
    paf_file = os.path.join(APP_DATA, id_res, "map.paf")
    idx1 = os.path.join(APP_DATA, id_res, "query.idx")
    idx2 = os.path.join(APP_DATA, id_res, "target.idx")
265
266
267
268
269
270
271
272
273
    paf = Paf(paf_file, idx1, idx2, False)
    paf.parse_paf(noise=request.form["noise"] == "1")
    if paf.parsed:
        res = paf.get_d3js_data()
        res["success"] = True
        return jsonify(res)
    return jsonify({"success": False, "message": paf.error})


274
275
@app.route('/get-fasta-query/<id_res>', methods=['POST'])
def build_fasta(id_res):
276
    res_dir = os.path.join(APP_DATA, id_res)
277
278
    lock_query = os.path.join(res_dir, ".query-fasta-build")
    is_sorted = os.path.exists(os.path.join(res_dir, ".sorted"))
279
    compressed = request.form["gzip"].lower() == "true"
280
281
282
283
284
    query_fasta = Functions.get_fasta_file(res_dir, "query", is_sorted)
    if query_fasta is not None:
        if is_sorted and not query_fasta.endswith(".sorted"):
            # Do the sort
            Path(lock_query).touch()
285
286
            if not compressed:  # If compressed, it will took a long time, so not wait
                Path(lock_query + ".pending").touch()
287
288
289
290
291
            thread = threading.Timer(1, Functions.sort_fasta, kwargs={
                "job_name": id_res,
                "fasta_file": query_fasta,
                "index_file": os.path.join(res_dir, "query.idx.sorted"),
                "lock_file": lock_query,
292
                "compress": compressed,
293
294
295
                "mailer": mailer
            })
            thread.start()
296
297
298
299
300
301
            if not compressed:
                i = 0
                time.sleep(5)
                while os.path.exists(lock_query) and i < 2:
                    i += 1
                    time.sleep(5)
Floreal Cabanettes's avatar
Floreal Cabanettes committed
302
                os.remove(lock_query + ".pending")
303
304
305
                if os.path.exists(lock_query):
                    return jsonify({"success": True, "status": 1, "status_message": "In progress"})
                return jsonify({"success": True, "status": 2, "status_message": "Done",
306
                                "gzip": False})
307
308
            else:
                return jsonify({"success": True, "status": 1, "status_message": "In progress"})
309
310
311
312
313
        elif is_sorted and os.path.exists(lock_query):
            # Sort is already in progress
            return jsonify({"success": True, "status": 1, "status_message": "In progress"})
        else:
            # No sort to do or sort done
314
315
316
317
318
319
320
            if compressed and not query_fasta.endswith(".gz.fasta"):
                # If compressed file is asked, we must compress it now if not done before...
                Path(lock_query).touch()
                thread = threading.Timer(1, Functions.compress_and_send_mail, kwargs={
                    "job_name": id_res,
                    "fasta_file": query_fasta,
                    "index_file": os.path.join(res_dir, "query.idx.sorted"),
321
                    "lock_file": lock_query,
322
323
324
325
326
                    "compressed": compressed,
                    "mailer": mailer
                })
                thread.start()
                return jsonify({"success": True, "status": 1, "status_message": "In progress"})
327
328
329
330
331
332
333
            return jsonify({"success": True, "status": 2, "status_message": "Done",
                            "gzip": query_fasta.endswith(".gz") or query_fasta.endswith(".gz.sorted")})
    else:
        return jsonify({"success": False,
                        "message": "Unable to get fasta file for query. Please contact us to report the bug"})


334
335
336
@app.route('/fasta-query/<id_res>', defaults={'filename': ""}, methods=['GET'])
@app.route('/fasta-query/<id_res>/<filename>', methods=['GET'])  # Use fake URL in mail to set download file name
def dl_fasta(id_res, filename):
337
    res_dir = os.path.join(APP_DATA, id_res)
338
339
340
341
342
343
344
345
346
347
348
349
350
    lock_query = os.path.join(res_dir, ".query-fasta-build")
    is_sorted = os.path.exists(os.path.join(res_dir, ".sorted"))
    if not os.path.exists(lock_query) or not is_sorted:
        query_fasta = Functions.get_fasta_file(res_dir, "query", is_sorted)
        if query_fasta is not None:
            if query_fasta.endswith(".gz") or query_fasta.endswith(".gz.sorted"):
                content = get_file(query_fasta, True)
                return Response(content, mimetype="application/gzip")
            content = get_file(query_fasta)
            return Response(content, mimetype="text/plain")
    abort(404)


351
352
@app.route('/qt-assoc/<id_res>', methods=['GET'])
def qt_assoc(id_res):
353
    res_dir = os.path.join(APP_DATA, id_res)
354
    if os.path.exists(res_dir) and os.path.isdir(res_dir):
355
356
357
        paf_file = os.path.join(APP_DATA, id_res, "map.paf")
        idx1 = os.path.join(APP_DATA, id_res, "query.idx")
        idx2 = os.path.join(APP_DATA, id_res, "target.idx")
358
359
360
361
362
363
364
365
366
367
368
369
        try:
            paf = Paf(paf_file, idx1, idx2, False)
            paf.parse_paf(False)
        except FileNotFoundError:
            print("Unable to load data!")
            abort(404)
            return False
        csv_content = paf.build_query_on_target_association_file()
        return Response(csv_content, mimetype="text/plain")
    abort(404)


370
371
372
373
374
375
@app.route('/no-assoc/<id_res>', methods=['POST'])
def no_assoc(id_res):
    """
    Get contigs that match with None target
    :param id_res: id of the result
    """
376
    res_dir = os.path.join(APP_DATA, id_res)
377
    if os.path.exists(res_dir) and os.path.isdir(res_dir):
378
379
380
        paf_file = os.path.join(APP_DATA, id_res, "map.paf")
        idx1 = os.path.join(APP_DATA, id_res, "query.idx")
        idx2 = os.path.join(APP_DATA, id_res, "target.idx")
381
382
383
384
385
386
        try:
            paf = Paf(paf_file, idx1, idx2, False)
        except FileNotFoundError:
            print("Unable to load data!")
            abort(404)
            return False
387
        file_content = paf.build_list_no_assoc(request.form["to"])
388
389
390
391
392
393
394
395
        empty = file_content == "\n"
        return jsonify({
            "file_content": file_content,
            "empty": empty
        })
    abort(404)


Floreal Cabanettes's avatar
Floreal Cabanettes committed
396
397
@app.route('/summary/<id_res>', methods=['POST'])
def summary(id_res):
398
399
400
401
402
403
404
405
406
407
    paf_file = os.path.join(APP_DATA, id_res, "map.paf")
    idx1 = os.path.join(APP_DATA, id_res, "query.idx")
    idx2 = os.path.join(APP_DATA, id_res, "target.idx")
    try:
        paf = Paf(paf_file, idx1, idx2, False)
    except FileNotFoundError:
        return jsonify({
            "success": False,
            "message": "Unable to load data!"
        })
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
    percents = None
    s_status = "waiting"  # Accepted values: waiting, done, fail
    status_file = os.path.join(APP_DATA, id_res, ".summarize")
    fail_file = status_file + ".fail"
    if not os.path.exists(status_file):  # The job is finished or not started
        if not os.path.exists(fail_file):  # The job has not started yet or has successfully ended
            percents = paf.get_summary_stats()
            if percents is None:  # The job has not started yet
                Path(status_file).touch()
                thread = threading.Timer(0, paf.build_summary_stats, kwargs={"status_file": status_file})
                thread.start()
            else:  # The job has successfully ended
                s_status = "done"
        else:  # The job has failed
            s_status = "fail"

    if s_status == "waiting":  # The job is running
        # Check if the job end in the next 30 seconds
        nb_iter = 0
        while os.path.exists(status_file) and not os.path.exists(fail_file) and nb_iter < 10:
            time.sleep(3)
            nb_iter += 1
        if not os.path.exists(status_file):  # The job has ended
            percents = paf.get_summary_stats()
            if percents is None:  # The job has failed
                s_status = "fail"
            else:  # The job has successfully ended
                s_status = "done"

    if s_status == "fail":
438
439
440
441
442
443
        return jsonify({
            "success": False,
            "message": "Build of summary failed. Please contact us to report the bug"
        })
    return jsonify({
        "success": True,
444
445
        "percents": percents,
        "status": s_status
446
    })
Floreal Cabanettes's avatar
Floreal Cabanettes committed
447
448


449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
def get_filter_out(id_res, type_f):
    filter_file = os.path.join(APP_DATA, id_res, ".filter-" + type_f)
    return Response(get_file(filter_file), mimetype="text/plain")


@app.route('/filter-out/<id_res>/query')
def get_filter_out_query(id_res):
    return get_filter_out(id_res=id_res, type_f="query")


@app.route('/filter-out/<id_res>/target')
def get_filter_out_target(id_res):
    return get_filter_out(id_res=id_res, type_f="target")


464
465
466
467
@app.route("/ask-upload", methods=['POST'])
def ask_upload():
    try:
        s_id = request.form['s_id']
Floreal Cabanettes's avatar
Floreal Cabanettes committed
468
469
470
        with Session.connect():
            session = Session.get(s_id=s_id)
            allowed = session.ask_for_upload(True)
471
472
        return jsonify({
            "success": True,
Floreal Cabanettes's avatar
Floreal Cabanettes committed
473
            "allowed": allowed
474
475
476
        })
    except DoesNotExist:
        return jsonify({"success": False, "message": "Session not initialized. Please refresh the page."})
477
478


479
480
481
@app.route("/ping-upload", methods=['POST'])
def ping_upload():
    s_id = request.form['s_id']
Floreal Cabanettes's avatar
Floreal Cabanettes committed
482
483
484
    with Session.connect():
        session = Session.get(s_id=s_id)
        session.ping()
485
    return "OK"
486
487


488
489
490
491
@app.route("/upload", methods=['POST'])
def upload():
    try:
        s_id = request.form['s_id']
Floreal Cabanettes's avatar
Floreal Cabanettes committed
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
        with Session.connect():
            session = Session.get(s_id=s_id)
            if session.ask_for_upload(False):
                folder = session.upload_folder
                files = request.files[list(request.files.keys())[0]]

                if files:
                    filename = files.filename
                    folder_files = os.path.join(app.config["UPLOAD_FOLDER"], folder)
                    if not os.path.exists(folder_files):
                        os.makedirs(folder_files)
                    filename = Functions.get_valid_uploaded_filename(filename, folder_files)
                    mime_type = files.content_type

                    if not Functions.allowed_file(files.filename):
                        result = UploadFile(name=filename, type_f=mime_type, size=0, not_allowed_msg="File type not allowed")
                        shutil.rmtree(folder_files)

                    else:
                        # save file to disk
                        uploaded_file_path = os.path.join(folder_files, filename)
                        files.save(uploaded_file_path)

                        # get file size after saving
                        size = os.path.getsize(uploaded_file_path)

                        # return json for js call back
                        result = UploadFile(name=filename, type_f=mime_type, size=size)

                    return jsonify({"files": [result.get_file()], "success": "OK"})

                return jsonify({"files": [], "success": "404", "message": "No file provided"})
            return jsonify({"files": [], "success": "ERR", "message": "Not allowed to upload!"})
525
526
527
528
529
    except DoesNotExist:
        return jsonify({"files": [], "success": "ERR", "message": "Session not initialized. Please refresh the page."})
    except:  # Except all possible exceptions to prevent crashes
        return jsonify({"files": [], "success": "ERR", "message": "An unexpected error has occurred on upload. "
                                                                  "Please contact the support."})
530
531


532
533
534
535
536
537
@app.route("/send-mail/<id_res>", methods=['POST'])
def send_mail(id_res):
    allowed = False
    key_file = None
    if "key" in request.form:
        key = request.form["key"]
538
        res_dir = os.path.join(APP_DATA, id_res)
539
540
541
542
543
544
545
546
547
548
549
550
        key_file = os.path.join(res_dir, ".key")
        if os.path.exists(key_file):
            with open(key_file) as k_f:
                true_key = k_f.readline().strip("\n")
                allowed = key == true_key
    if allowed:
        os.remove(key_file)
        job_mng = JobManager(id_job=id_res, mailer=mailer)
        job_mng.set_inputs_from_res_dir()
        job_mng.send_mail()
        return "OK"
    else:
551
        abort(403)