views.py 17.6 KB
Newer Older
1
from dgenies import app, app_title, config_reader, mailer, APP_DATA
2

Floreal Cabanettes's avatar
Floreal Cabanettes committed
3
import os
4
5
import time
import datetime
6
import shutil
7
import re
8
import threading
9
from flask import render_template, request, url_for, jsonify, Response, abort
10
from pathlib import Path
11
12
13
14
15
16
from .lib.paf import Paf
from .lib.job_manager import JobManager
from .lib.functions import Functions, ALLOWED_EXTENSIONS
from .lib.upload_file import UploadFile
from .lib.fasta import Fasta
from .database import Session
17
from peewee import DoesNotExist
18

19

Floreal Cabanettes's avatar
Floreal Cabanettes committed
20
21
22
23
24
25
@app.context_processor
def get_launched_results():
    cookie = request.cookies.get("results")
    return {"results": cookie.split("|") if cookie is not None else set()}


26
27
28
# Main
@app.route("/", methods=['GET'])
def main():
Floreal Cabanettes's avatar
Floreal Cabanettes committed
29
30
31
32
33
    return render_template("index.html", title=app_title, menu="index")


@app.route("/run", methods=['GET'])
def run():
34
    s_id = Session.new()
Floreal Cabanettes's avatar
Floreal Cabanettes committed
35
    id_job = Functions.random_string(5) + "_" + datetime.datetime.fromtimestamp(time.time()).strftime('%Y%m%d%H%M%S')
36
37
38
39
40
    if "id_job" in request.args:
        id_job = request.args["id_job"]
    email = ""
    if "email" in request.args:
        email = request.args["email"]
Floreal Cabanettes's avatar
Floreal Cabanettes committed
41
    return render_template("run.html", title=app_title, id_job=id_job, email=email,
42
43
                           menu="run", allowed_ext=ALLOWED_EXTENSIONS, s_id=s_id,
                           max_upload_file_size=config_reader.max_upload_file_size)
44
45


46
47
48
49
50
51
52
53
@app.route("/run-test", methods=['GET'])
def run_test():
    print(config_reader.allowed_ip_tests)
    if request.remote_addr not in config_reader.allowed_ip_tests:
        return abort(404)
    return Session.new()


54
55
56
# Launch analysis
@app.route("/launch_analysis", methods=['POST'])
def launch_analysis():
57
58
59
60
61
62
63
64
    try:
        session = Session.get(s_id=request.form["s_id"])
    except DoesNotExist:
        return jsonify({"success": False, "errors": ["Session has expired. Please refresh the page and try again"]})
    # Reset session upload:
    session.allow_upload = False
    session.position = -1
    session.save()
65
66
    id_job = request.form["id_job"]
    email = request.form["email"]
67
68
69
70
    file_query = request.form["query"]
    file_query_type = request.form["query_type"]
    file_target = request.form["target"]
    file_target_type = request.form["target_type"]
71
72
73

    # Check form:
    form_pass = True
74
    errors = []
75
    if id_job == "":
76
        errors.append("Id of job not given")
77
78
79
        form_pass = False

    if email == "":
80
        errors.append("Email not given")
81
        form_pass = False
82
83
    if file_target == "":
        errors.append("No target fasta selected")
84
85
86
87
        form_pass = False

    # Form pass
    if form_pass:
88
        # Get final job id:
89
        id_job = re.sub('[^A-Za-z0-9_\-]+', '', id_job.replace(" ", "_"))
90
        id_job_orig = id_job
91
        i = 2
92
        while os.path.exists(os.path.join(APP_DATA, id_job)):
93
94
            id_job = id_job_orig + ("_%d" % i)
            i += 1
95

96
        folder_files = os.path.join(APP_DATA, id_job)
97
98
99
        os.makedirs(folder_files)

        # Save files:
100
        query = None
101
        upload_folder = session.upload_folder
102
103
        if file_query != "":
            query_name = os.path.splitext(file_query.replace(".gz", ""))[0] if file_query_type == "local" else None
104
            query_path = os.path.join(app.config["UPLOAD_FOLDER"], upload_folder, file_query) \
105
106
107
                if file_query_type == "local" else file_query
            query = Fasta(name=query_name, path=query_path, type_f=file_query_type)
        target_name = os.path.splitext(file_target.replace(".gz", ""))[0] if file_target_type == "local" else None
108
        target_path = os.path.join(app.config["UPLOAD_FOLDER"], upload_folder, file_target) \
109
110
            if file_target_type == "local" else file_target
        target = Fasta(name=target_name, path=target_path, type_f=file_target_type)
111
112

        # Launch job:
Floreal Cabanettes's avatar
Floreal Cabanettes committed
113
        job = JobManager(id_job, email, query, target, mailer)
114
        job.launch()
115
116
117

        # Delete session:
        session.delete_instance()
118
        return jsonify({"success": True, "redirect": url_for(".status", id_job=id_job)})
119
    else:
120
        return jsonify({"success": False, "errors": errors})
121
122
123
124
125
126


# Status of a job
@app.route('/status/<id_job>', methods=['GET'])
def status(id_job):
    job = JobManager(id_job)
127
128
129
130
131
132
133
134
135
136
137
138
    j_status = job.status()
    mem_peak = j_status["mem_peak"] if "mem_peak" in j_status else None
    if mem_peak is not None:
        mem_peak = "%.1f G" % (mem_peak / 1024.0 / 1024.0)
    time_e = j_status["time_elapsed"] if "time_elapsed" in j_status else None
    if time_e is not None:
        if time_e < 60:
            time_e = "%d secs" % time_e
        else:
            minutes = time_e // 60
            seconds = time_e - minutes * 60
            time_e = "%d min %d secs" % (minutes, seconds)
139
140
141
142
143
144
145
146
147
    format = request.args.get("format")
    if format is not None and format == "json":
        return jsonify({
            "status": j_status["status"],
            "error": j_status["error"].replace("#ID#", ""),
            "id_job": id_job,
            "mem_peak": mem_peak,
            "time_elapsed": time_e
        })
148
149
150
151
    return render_template("status.html", title=app_title, status=j_status["status"],
                           error=j_status["error"].replace("#ID#", ""),
                           id_job=id_job, menu="results", mem_peak=mem_peak,
                           time_elapsed=time_e)
152
153


Floreal Cabanettes's avatar
Floreal Cabanettes committed
154
# Results path
155
@app.route("/result/<id_res>", methods=['GET'])
156
def result(id_res):
Floreal Cabanettes's avatar
Floreal Cabanettes committed
157
158
159
160
161
162
163
164
    my_render = render_template("results.html", title=app_title, id=id_res, menu="results", current_result=id_res)
    response = app.make_response(my_render)
    cookie = request.cookies.get("results")
    cookie = cookie.split("|") if cookie is not None else []
    if id_res not in cookie:
        cookie.insert(0, id_res)
    response.set_cookie(key="results", value="|".join(cookie), path="/")
    return response
165
166


167
def get_file(file, gzip=False):  # pragma: no cover
Floreal Cabanettes's avatar
Floreal Cabanettes committed
168
169
170
171
172
173
    try:
        # Figure out how flask returns static files
        # Tried:
        # - render_template
        # - send_file
        # This should not be so non-obvious
174
        return open(file, "rb" if gzip else "r").read()
Floreal Cabanettes's avatar
Floreal Cabanettes committed
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
    except IOError as exc:
        return str(exc)


@app.route("/paf/<id_res>", methods=['GET'])
def download_paf(id_res):
    map_file = os.path.join(APP_DATA, id_res, "map.paf.sorted")
    if not os.path.exists(map_file):
        map_file = os.path.join(APP_DATA, id_res, "map.paf")
    if not os.path.exists(map_file):
        abort(404)
    content = get_file(map_file)
    return Response(content, mimetype="text/plain")


Floreal Cabanettes's avatar
Floreal Cabanettes committed
190
# Get graph (ajax request)
191
192
193
@app.route('/get_graph', methods=['POST'])
def get_graph():
    id_f = request.form["id"]
194
195
196
    paf = os.path.join(APP_DATA, id_f, "map.paf")
    idx1 = os.path.join(APP_DATA, id_f, "query.idx")
    idx2 = os.path.join(APP_DATA, id_f, "target.idx")
197

198
    paf = Paf(paf, idx1, idx2)
199

200
201
    if paf.parsed:
        res = paf.get_d3js_data()
202
        res["success"] = True
203
204
205
206
207
208
        return jsonify(res)
    return jsonify({"success": False, "message": paf.error})


@app.route('/sort/<id_res>', methods=['POST'])
def sort_graph(id_res):
209
    if not os.path.exists(os.path.join(APP_DATA, id_res, ".all-vs-all")):
210
211
212
        paf_file = os.path.join(APP_DATA, id_res, "map.paf")
        idx1 = os.path.join(APP_DATA, id_res, "query.idx")
        idx2 = os.path.join(APP_DATA, id_res, "target.idx")
213
        paf = Paf(paf_file, idx1, idx2, False)
214
215
216
217
218
219
220
        paf.sort()
        if paf.parsed:
            res = paf.get_d3js_data()
            res["success"] = True
            return jsonify(res)
        return jsonify({"success": False, "message": paf.error})
    return jsonify({"success": False, "message": "Sort is not available for All-vs-All mode"})
Floreal Cabanettes's avatar
Floreal Cabanettes committed
221

Floréal Cabanettes's avatar
Floréal Cabanettes committed
222

223
224
225
226
@app.route('/reverse-contig/<id_res>', methods=['POST'])
def reverse_contig(id_res):
    contig_name = request.form["contig"]
    if not os.path.exists(os.path.join(APP_DATA, id_res, ".all-vs-all")):
227
228
229
        paf_file = os.path.join(APP_DATA, id_res, "map.paf")
        idx1 = os.path.join(APP_DATA, id_res, "query.idx")
        idx2 = os.path.join(APP_DATA, id_res, "target.idx")
230
231
232
233
234
235
236
237
238
239
        paf = Paf(paf_file, idx1, idx2, False)
        paf.reverse_contig(contig_name)
        if paf.parsed:
            res = paf.get_d3js_data()
            res["success"] = True
            return jsonify(res)
        return jsonify({"success": False, "message": paf.error})
    return jsonify({"success": False, "message": "Sort is not available for All-vs-All mode"})


240
241
@app.route('/freenoise/<id_res>', methods=['POST'])
def free_noise(id_res):
242
243
244
    paf_file = os.path.join(APP_DATA, id_res, "map.paf")
    idx1 = os.path.join(APP_DATA, id_res, "query.idx")
    idx2 = os.path.join(APP_DATA, id_res, "target.idx")
245
246
247
248
249
250
251
252
253
    paf = Paf(paf_file, idx1, idx2, False)
    paf.parse_paf(noise=request.form["noise"] == "1")
    if paf.parsed:
        res = paf.get_d3js_data()
        res["success"] = True
        return jsonify(res)
    return jsonify({"success": False, "message": paf.error})


254
255
@app.route('/get-fasta-query/<id_res>', methods=['POST'])
def build_fasta(id_res):
256
    res_dir = os.path.join(APP_DATA, id_res)
257
258
    lock_query = os.path.join(res_dir, ".query-fasta-build")
    is_sorted = os.path.exists(os.path.join(res_dir, ".sorted"))
259
    compressed = request.form["gzip"].lower() == "true"
260
261
262
263
264
    query_fasta = Functions.get_fasta_file(res_dir, "query", is_sorted)
    if query_fasta is not None:
        if is_sorted and not query_fasta.endswith(".sorted"):
            # Do the sort
            Path(lock_query).touch()
265
266
            if not compressed:  # If compressed, it will took a long time, so not wait
                Path(lock_query + ".pending").touch()
267
268
269
270
271
            thread = threading.Timer(1, Functions.sort_fasta, kwargs={
                "job_name": id_res,
                "fasta_file": query_fasta,
                "index_file": os.path.join(res_dir, "query.idx.sorted"),
                "lock_file": lock_query,
272
                "compress": compressed,
273
274
275
                "mailer": mailer
            })
            thread.start()
276
277
278
279
280
281
            if not compressed:
                i = 0
                time.sleep(5)
                while os.path.exists(lock_query) and i < 2:
                    i += 1
                    time.sleep(5)
Floreal Cabanettes's avatar
Floreal Cabanettes committed
282
                os.remove(lock_query + ".pending")
283
284
285
                if os.path.exists(lock_query):
                    return jsonify({"success": True, "status": 1, "status_message": "In progress"})
                return jsonify({"success": True, "status": 2, "status_message": "Done",
286
                                "gzip": False})
287
288
            else:
                return jsonify({"success": True, "status": 1, "status_message": "In progress"})
289
290
291
292
293
        elif is_sorted and os.path.exists(lock_query):
            # Sort is already in progress
            return jsonify({"success": True, "status": 1, "status_message": "In progress"})
        else:
            # No sort to do or sort done
294
295
296
297
298
299
300
            if compressed and not query_fasta.endswith(".gz.fasta"):
                # If compressed file is asked, we must compress it now if not done before...
                Path(lock_query).touch()
                thread = threading.Timer(1, Functions.compress_and_send_mail, kwargs={
                    "job_name": id_res,
                    "fasta_file": query_fasta,
                    "index_file": os.path.join(res_dir, "query.idx.sorted"),
301
                    "lock_file": lock_query,
302
303
304
305
306
                    "compressed": compressed,
                    "mailer": mailer
                })
                thread.start()
                return jsonify({"success": True, "status": 1, "status_message": "In progress"})
307
308
309
310
311
312
313
            return jsonify({"success": True, "status": 2, "status_message": "Done",
                            "gzip": query_fasta.endswith(".gz") or query_fasta.endswith(".gz.sorted")})
    else:
        return jsonify({"success": False,
                        "message": "Unable to get fasta file for query. Please contact us to report the bug"})


314
315
316
@app.route('/fasta-query/<id_res>', defaults={'filename': ""}, methods=['GET'])
@app.route('/fasta-query/<id_res>/<filename>', methods=['GET'])  # Use fake URL in mail to set download file name
def dl_fasta(id_res, filename):
317
    res_dir = os.path.join(APP_DATA, id_res)
318
319
320
321
322
323
324
325
326
327
328
329
330
    lock_query = os.path.join(res_dir, ".query-fasta-build")
    is_sorted = os.path.exists(os.path.join(res_dir, ".sorted"))
    if not os.path.exists(lock_query) or not is_sorted:
        query_fasta = Functions.get_fasta_file(res_dir, "query", is_sorted)
        if query_fasta is not None:
            if query_fasta.endswith(".gz") or query_fasta.endswith(".gz.sorted"):
                content = get_file(query_fasta, True)
                return Response(content, mimetype="application/gzip")
            content = get_file(query_fasta)
            return Response(content, mimetype="text/plain")
    abort(404)


331
332
@app.route('/qt-assoc/<id_res>', methods=['GET'])
def qt_assoc(id_res):
333
    res_dir = os.path.join(APP_DATA, id_res)
334
    if os.path.exists(res_dir) and os.path.isdir(res_dir):
335
336
337
        paf_file = os.path.join(APP_DATA, id_res, "map.paf")
        idx1 = os.path.join(APP_DATA, id_res, "query.idx")
        idx2 = os.path.join(APP_DATA, id_res, "target.idx")
338
339
340
341
342
343
344
345
346
347
348
349
        try:
            paf = Paf(paf_file, idx1, idx2, False)
            paf.parse_paf(False)
        except FileNotFoundError:
            print("Unable to load data!")
            abort(404)
            return False
        csv_content = paf.build_query_on_target_association_file()
        return Response(csv_content, mimetype="text/plain")
    abort(404)


350
351
352
353
354
355
@app.route('/no-assoc/<id_res>', methods=['POST'])
def no_assoc(id_res):
    """
    Get contigs that match with None target
    :param id_res: id of the result
    """
356
    res_dir = os.path.join(APP_DATA, id_res)
357
    if os.path.exists(res_dir) and os.path.isdir(res_dir):
358
359
360
        paf_file = os.path.join(APP_DATA, id_res, "map.paf")
        idx1 = os.path.join(APP_DATA, id_res, "query.idx")
        idx2 = os.path.join(APP_DATA, id_res, "target.idx")
361
362
363
364
365
366
        try:
            paf = Paf(paf_file, idx1, idx2, False)
        except FileNotFoundError:
            print("Unable to load data!")
            abort(404)
            return False
367
        file_content = paf.build_list_no_assoc(request.form["to"])
368
369
370
371
372
373
374
375
        empty = file_content == "\n"
        return jsonify({
            "file_content": file_content,
            "empty": empty
        })
    abort(404)


Floreal Cabanettes's avatar
Floreal Cabanettes committed
376
377
378
379
380
@app.route('/summary/<id_res>', methods=['POST'])
def summary(id_res):
    pass


381
382
383
384
385
386
387
388
389
390
391
392
393
@app.route("/ask-upload", methods=['POST'])
def ask_upload():
    try:
        s_id = request.form['s_id']
        session = Session.get(s_id=s_id)
        allowed, position = session.ask_for_upload(True)
        return jsonify({
            "success": True,
            "allowed": allowed,
            "position": position
        })
    except DoesNotExist:
        return jsonify({"success": False, "message": "Session not initialized. Please refresh the page."})
394
395


396
397
398
399
400
401
@app.route("/ping-upload", methods=['POST'])
def ping_upload():
    s_id = request.form['s_id']
    session = Session.get(s_id=s_id)
    session.ping()
    return "OK"
402
403


404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
@app.route("/upload", methods=['POST'])
def upload():
    try:
        s_id = request.form['s_id']
        session = Session.get(s_id=s_id)
        if session.ask_for_upload(False)[0]:
            folder = session.upload_folder
            files = request.files[list(request.files.keys())[0]]

            if files:
                filename = files.filename
                folder_files = os.path.join(app.config["UPLOAD_FOLDER"], folder)
                if not os.path.exists(folder_files):
                    os.makedirs(folder_files)
                filename = Functions.get_valid_uploaded_filename(filename, folder_files)
                mime_type = files.content_type

                if not Functions.allowed_file(files.filename):
                    result = UploadFile(name=filename, type_f=mime_type, size=0, not_allowed_msg="File type not allowed")
                    shutil.rmtree(folder_files)

                else:
                    # save file to disk
                    uploaded_file_path = os.path.join(folder_files, filename)
                    files.save(uploaded_file_path)

                    # get file size after saving
                    size = os.path.getsize(uploaded_file_path)

                    # return json for js call back
                    result = UploadFile(name=filename, type_f=mime_type, size=size)

                return jsonify({"files": [result.get_file()], "success": "OK"})

            return jsonify({"files": [], "success": "404", "message": "No file provided"})
        return jsonify({"files": [], "success": "ERR", "message": "Not allowed to upload!"})
    except DoesNotExist:
        return jsonify({"files": [], "success": "ERR", "message": "Session not initialized. Please refresh the page."})
    except:  # Except all possible exceptions to prevent crashes
        return jsonify({"files": [], "success": "ERR", "message": "An unexpected error has occurred on upload. "
                                                                  "Please contact the support."})
445
446


447
448
449
450
451
452
@app.route("/send-mail/<id_res>", methods=['POST'])
def send_mail(id_res):
    allowed = False
    key_file = None
    if "key" in request.form:
        key = request.form["key"]
453
        res_dir = os.path.join(APP_DATA, id_res)
454
455
456
457
458
459
460
461
462
463
464
465
        key_file = os.path.join(res_dir, ".key")
        if os.path.exists(key_file):
            with open(key_file) as k_f:
                true_key = k_f.readline().strip("\n")
                allowed = key == true_key
    if allowed:
        os.remove(key_file)
        job_mng = JobManager(id_job=id_res, mailer=mailer)
        job_mng.set_inputs_from_res_dir()
        job_mng.send_mail()
        return "OK"
    else:
466
        abort(403)