main.py 18.6 KB
Newer Older
1
2
#!/usr/bin/env python3

Floreal Cabanettes's avatar
Floreal Cabanettes committed
3
import os
4
5
import time
import datetime
6
import shutil
7
import re
8
import threading
9
from flask import Flask, render_template, request, url_for, jsonify, Response, abort
10
from pathlib import Path
11
from lib.paf import Paf
12
from config_reader import AppConfigReader
13
from lib.job_manager import JobManager
14
from lib.functions import Functions, ALLOWED_EXTENSIONS
15
from lib.upload_file import UploadFile
Floreal Cabanettes's avatar
Floreal Cabanettes committed
16
from lib.fasta import Fasta
Floreal Cabanettes's avatar
Floreal Cabanettes committed
17
from lib.mailer import Mailer
Floreal Cabanettes's avatar
Floreal Cabanettes committed
18
from lib.crons import Crons
19
20
from database import Session
from peewee import DoesNotExist
21
22

import sys
23

24
25
26
27
28
29
app_folder = os.path.dirname(os.path.realpath(__file__))
sys.path.insert(0, app_folder)
os.environ["PATH"] = os.path.join(app_folder, "bin") + ":" + os.environ["PATH"]

sqlite_file = os.path.join(app_folder, "database.sqlite")

30
31
32
33

# Init config reader:
config_reader = AppConfigReader()

Floreal Cabanettes's avatar
Floreal Cabanettes committed
34
35
UPLOAD_FOLDER = config_reader.upload_folder
APP_DATA = config_reader.app_data
36

Floreal Cabanettes's avatar
Floreal Cabanettes committed
37
app_title = "D-GENIES - Dotplot for Genomes Interactive, E-connected and Speedy"
Floreal Cabanettes's avatar
Floreal Cabanettes committed
38

39
# Init Flask:
40
app = Flask(__name__, static_url_path='/static')
41
app.config['UPLOAD_FOLDER'] = UPLOAD_FOLDER
42
app.config['MAX_CONTENT_LENGTH'] = config_reader.max_upload_file_size
43
app.config['SECRET_KEY'] = 'dsqdsq-255sdA-fHfg52-25Asd5'
44

45
# Init mail:
Floreal Cabanettes's avatar
Floreal Cabanettes committed
46
mailer = Mailer(app)
47

48
# Folder containing data:
Floreal Cabanettes's avatar
Floreal Cabanettes committed
49
app_data = config_reader.app_data
50

51
52
53
if config_reader.debug and config_reader.log_dir != "stdout" and not os.path.exists(config_reader.log_dir):
    os.makedirs(config_reader.log_dir)

Floreal Cabanettes's avatar
Floreal Cabanettes committed
54
# Crons:
55
if os.getenv('DISABLE_CRONS') != "True":
56
57
58
    print("Starting crons...")
    crons = Crons(app_folder)
    crons.start_all()
Floreal Cabanettes's avatar
Floreal Cabanettes committed
59

60

Floreal Cabanettes's avatar
Floreal Cabanettes committed
61
62
63
64
65
66
@app.context_processor
def get_launched_results():
    cookie = request.cookies.get("results")
    return {"results": cookie.split("|") if cookie is not None else set()}


67
68
69
# Main
@app.route("/", methods=['GET'])
def main():
Floreal Cabanettes's avatar
Floreal Cabanettes committed
70
71
72
73
74
    return render_template("index.html", title=app_title, menu="index")


@app.route("/run", methods=['GET'])
def run():
75
    s_id = Session.new()
Floreal Cabanettes's avatar
Floreal Cabanettes committed
76
    id_job = Functions.random_string(5) + "_" + datetime.datetime.fromtimestamp(time.time()).strftime('%Y%m%d%H%M%S')
77
78
79
80
81
    if "id_job" in request.args:
        id_job = request.args["id_job"]
    email = ""
    if "email" in request.args:
        email = request.args["email"]
Floreal Cabanettes's avatar
Floreal Cabanettes committed
82
    return render_template("run.html", title=app_title, id_job=id_job, email=email,
83
84
                           menu="run", allowed_ext=ALLOWED_EXTENSIONS, s_id=s_id,
                           max_upload_file_size=config_reader.max_upload_file_size)
85
86


87
88
89
90
91
92
93
94
@app.route("/run-test", methods=['GET'])
def run_test():
    print(config_reader.allowed_ip_tests)
    if request.remote_addr not in config_reader.allowed_ip_tests:
        return abort(404)
    return Session.new()


95
96
97
# Launch analysis
@app.route("/launch_analysis", methods=['POST'])
def launch_analysis():
98
99
100
101
102
103
104
105
    try:
        session = Session.get(s_id=request.form["s_id"])
    except DoesNotExist:
        return jsonify({"success": False, "errors": ["Session has expired. Please refresh the page and try again"]})
    # Reset session upload:
    session.allow_upload = False
    session.position = -1
    session.save()
106
107
    id_job = request.form["id_job"]
    email = request.form["email"]
108
109
110
111
    file_query = request.form["query"]
    file_query_type = request.form["query_type"]
    file_target = request.form["target"]
    file_target_type = request.form["target_type"]
112
113
114

    # Check form:
    form_pass = True
115
    errors = []
116
    if id_job == "":
117
        errors.append("Id of job not given")
118
119
120
        form_pass = False

    if email == "":
121
        errors.append("Email not given")
122
        form_pass = False
123
124
    if file_target == "":
        errors.append("No target fasta selected")
125
126
127
128
        form_pass = False

    # Form pass
    if form_pass:
129
        # Get final job id:
130
        id_job = re.sub('[^A-Za-z0-9_\-]+', '', id_job.replace(" ", "_"))
131
        id_job_orig = id_job
132
        i = 2
133
        while os.path.exists(os.path.join(app_data, id_job)):
134
135
            id_job = id_job_orig + ("_%d" % i)
            i += 1
136
137
138
139
140

        folder_files = os.path.join(app_data, id_job)
        os.makedirs(folder_files)

        # Save files:
141
        query = None
142
        upload_folder = session.upload_folder
143
144
        if file_query != "":
            query_name = os.path.splitext(file_query.replace(".gz", ""))[0] if file_query_type == "local" else None
145
            query_path = os.path.join(app.config["UPLOAD_FOLDER"], upload_folder, file_query) \
146
147
148
                if file_query_type == "local" else file_query
            query = Fasta(name=query_name, path=query_path, type_f=file_query_type)
        target_name = os.path.splitext(file_target.replace(".gz", ""))[0] if file_target_type == "local" else None
149
        target_path = os.path.join(app.config["UPLOAD_FOLDER"], upload_folder, file_target) \
150
151
            if file_target_type == "local" else file_target
        target = Fasta(name=target_name, path=target_path, type_f=file_target_type)
152
153

        # Launch job:
Floreal Cabanettes's avatar
Floreal Cabanettes committed
154
        job = JobManager(id_job, email, query, target, mailer)
155
        job.launch()
156
157
158

        # Delete session:
        session.delete_instance()
159
        return jsonify({"success": True, "redirect": url_for(".status", id_job=id_job)})
160
    else:
161
        return jsonify({"success": False, "errors": errors})
162
163
164
165
166
167


# Status of a job
@app.route('/status/<id_job>', methods=['GET'])
def status(id_job):
    job = JobManager(id_job)
168
169
170
171
172
173
174
175
176
177
178
179
    j_status = job.status()
    mem_peak = j_status["mem_peak"] if "mem_peak" in j_status else None
    if mem_peak is not None:
        mem_peak = "%.1f G" % (mem_peak / 1024.0 / 1024.0)
    time_e = j_status["time_elapsed"] if "time_elapsed" in j_status else None
    if time_e is not None:
        if time_e < 60:
            time_e = "%d secs" % time_e
        else:
            minutes = time_e // 60
            seconds = time_e - minutes * 60
            time_e = "%d min %d secs" % (minutes, seconds)
180
181
182
183
184
185
186
187
188
    format = request.args.get("format")
    if format is not None and format == "json":
        return jsonify({
            "status": j_status["status"],
            "error": j_status["error"].replace("#ID#", ""),
            "id_job": id_job,
            "mem_peak": mem_peak,
            "time_elapsed": time_e
        })
189
190
191
192
    return render_template("status.html", title=app_title, status=j_status["status"],
                           error=j_status["error"].replace("#ID#", ""),
                           id_job=id_job, menu="results", mem_peak=mem_peak,
                           time_elapsed=time_e)
193
194


Floreal Cabanettes's avatar
Floreal Cabanettes committed
195
# Results path
196
@app.route("/result/<id_res>", methods=['GET'])
197
def result(id_res):
Floreal Cabanettes's avatar
Floreal Cabanettes committed
198
199
200
201
202
203
204
205
    my_render = render_template("results.html", title=app_title, id=id_res, menu="results", current_result=id_res)
    response = app.make_response(my_render)
    cookie = request.cookies.get("results")
    cookie = cookie.split("|") if cookie is not None else []
    if id_res not in cookie:
        cookie.insert(0, id_res)
    response.set_cookie(key="results", value="|".join(cookie), path="/")
    return response
206
207


208
def get_file(file, gzip=False):  # pragma: no cover
Floreal Cabanettes's avatar
Floreal Cabanettes committed
209
210
211
212
213
214
    try:
        # Figure out how flask returns static files
        # Tried:
        # - render_template
        # - send_file
        # This should not be so non-obvious
215
        return open(file, "rb" if gzip else "r").read()
Floreal Cabanettes's avatar
Floreal Cabanettes committed
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
    except IOError as exc:
        return str(exc)


@app.route("/paf/<id_res>", methods=['GET'])
def download_paf(id_res):
    map_file = os.path.join(APP_DATA, id_res, "map.paf.sorted")
    if not os.path.exists(map_file):
        map_file = os.path.join(APP_DATA, id_res, "map.paf")
    if not os.path.exists(map_file):
        abort(404)
    content = get_file(map_file)
    return Response(content, mimetype="text/plain")


Floreal Cabanettes's avatar
Floreal Cabanettes committed
231
# Get graph (ajax request)
232
233
234
@app.route('/get_graph', methods=['POST'])
def get_graph():
    id_f = request.form["id"]
235
    paf = os.path.join(app_data, id_f, "map.paf")
236
237
    idx1 = os.path.join(app_data, id_f, "query.idx")
    idx2 = os.path.join(app_data, id_f, "target.idx")
238

239
    paf = Paf(paf, idx1, idx2)
240

241
242
    if paf.parsed:
        res = paf.get_d3js_data()
243
        res["success"] = True
244
245
246
247
248
249
        return jsonify(res)
    return jsonify({"success": False, "message": paf.error})


@app.route('/sort/<id_res>', methods=['POST'])
def sort_graph(id_res):
250
    if not os.path.exists(os.path.join(APP_DATA, id_res, ".all-vs-all")):
251
        paf_file = os.path.join(app_data, id_res, "map.paf")
252
253
        idx1 = os.path.join(app_data, id_res, "query.idx")
        idx2 = os.path.join(app_data, id_res, "target.idx")
254
        paf = Paf(paf_file, idx1, idx2, False)
255
256
257
258
259
260
261
        paf.sort()
        if paf.parsed:
            res = paf.get_d3js_data()
            res["success"] = True
            return jsonify(res)
        return jsonify({"success": False, "message": paf.error})
    return jsonify({"success": False, "message": "Sort is not available for All-vs-All mode"})
Floreal Cabanettes's avatar
Floreal Cabanettes committed
262

Floréal Cabanettes's avatar
Floréal Cabanettes committed
263

264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
@app.route('/reverse-contig/<id_res>', methods=['POST'])
def reverse_contig(id_res):
    contig_name = request.form["contig"]
    if not os.path.exists(os.path.join(APP_DATA, id_res, ".all-vs-all")):
        paf_file = os.path.join(app_data, id_res, "map.paf")
        idx1 = os.path.join(app_data, id_res, "query.idx")
        idx2 = os.path.join(app_data, id_res, "target.idx")
        paf = Paf(paf_file, idx1, idx2, False)
        paf.reverse_contig(contig_name)
        if paf.parsed:
            res = paf.get_d3js_data()
            res["success"] = True
            return jsonify(res)
        return jsonify({"success": False, "message": paf.error})
    return jsonify({"success": False, "message": "Sort is not available for All-vs-All mode"})


281
282
283
284
285
286
287
288
289
290
291
292
293
294
@app.route('/freenoise/<id_res>', methods=['POST'])
def free_noise(id_res):
    paf_file = os.path.join(app_data, id_res, "map.paf")
    idx1 = os.path.join(app_data, id_res, "query.idx")
    idx2 = os.path.join(app_data, id_res, "target.idx")
    paf = Paf(paf_file, idx1, idx2, False)
    paf.parse_paf(noise=request.form["noise"] == "1")
    if paf.parsed:
        res = paf.get_d3js_data()
        res["success"] = True
        return jsonify(res)
    return jsonify({"success": False, "message": paf.error})


295
296
297
298
299
@app.route('/get-fasta-query/<id_res>', methods=['POST'])
def build_fasta(id_res):
    res_dir = os.path.join(app_data, id_res)
    lock_query = os.path.join(res_dir, ".query-fasta-build")
    is_sorted = os.path.exists(os.path.join(res_dir, ".sorted"))
300
    compressed = request.form["gzip"].lower() == "true"
301
302
303
304
305
    query_fasta = Functions.get_fasta_file(res_dir, "query", is_sorted)
    if query_fasta is not None:
        if is_sorted and not query_fasta.endswith(".sorted"):
            # Do the sort
            Path(lock_query).touch()
306
307
            if not compressed:  # If compressed, it will took a long time, so not wait
                Path(lock_query + ".pending").touch()
308
309
310
311
312
            thread = threading.Timer(1, Functions.sort_fasta, kwargs={
                "job_name": id_res,
                "fasta_file": query_fasta,
                "index_file": os.path.join(res_dir, "query.idx.sorted"),
                "lock_file": lock_query,
313
                "compress": compressed,
314
315
316
                "mailer": mailer
            })
            thread.start()
317
318
319
320
321
322
            if not compressed:
                i = 0
                time.sleep(5)
                while os.path.exists(lock_query) and i < 2:
                    i += 1
                    time.sleep(5)
Floreal Cabanettes's avatar
Floreal Cabanettes committed
323
                os.remove(lock_query + ".pending")
324
325
326
                if os.path.exists(lock_query):
                    return jsonify({"success": True, "status": 1, "status_message": "In progress"})
                return jsonify({"success": True, "status": 2, "status_message": "Done",
327
                                "gzip": False})
328
329
            else:
                return jsonify({"success": True, "status": 1, "status_message": "In progress"})
330
331
332
333
334
        elif is_sorted and os.path.exists(lock_query):
            # Sort is already in progress
            return jsonify({"success": True, "status": 1, "status_message": "In progress"})
        else:
            # No sort to do or sort done
335
336
337
338
339
340
341
            if compressed and not query_fasta.endswith(".gz.fasta"):
                # If compressed file is asked, we must compress it now if not done before...
                Path(lock_query).touch()
                thread = threading.Timer(1, Functions.compress_and_send_mail, kwargs={
                    "job_name": id_res,
                    "fasta_file": query_fasta,
                    "index_file": os.path.join(res_dir, "query.idx.sorted"),
342
                    "lock_file": lock_query,
343
344
345
346
347
                    "compressed": compressed,
                    "mailer": mailer
                })
                thread.start()
                return jsonify({"success": True, "status": 1, "status_message": "In progress"})
348
349
350
351
352
353
354
            return jsonify({"success": True, "status": 2, "status_message": "Done",
                            "gzip": query_fasta.endswith(".gz") or query_fasta.endswith(".gz.sorted")})
    else:
        return jsonify({"success": False,
                        "message": "Unable to get fasta file for query. Please contact us to report the bug"})


355
356
357
@app.route('/fasta-query/<id_res>', defaults={'filename': ""}, methods=['GET'])
@app.route('/fasta-query/<id_res>/<filename>', methods=['GET'])  # Use fake URL in mail to set download file name
def dl_fasta(id_res, filename):
358
359
360
361
362
363
364
365
366
367
368
369
370
371
    res_dir = os.path.join(app_data, id_res)
    lock_query = os.path.join(res_dir, ".query-fasta-build")
    is_sorted = os.path.exists(os.path.join(res_dir, ".sorted"))
    if not os.path.exists(lock_query) or not is_sorted:
        query_fasta = Functions.get_fasta_file(res_dir, "query", is_sorted)
        if query_fasta is not None:
            if query_fasta.endswith(".gz") or query_fasta.endswith(".gz.sorted"):
                content = get_file(query_fasta, True)
                return Response(content, mimetype="application/gzip")
            content = get_file(query_fasta)
            return Response(content, mimetype="text/plain")
    abort(404)


372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
@app.route('/qt-assoc/<id_res>', methods=['GET'])
def qt_assoc(id_res):
    res_dir = os.path.join(app_data, id_res)
    if os.path.exists(res_dir) and os.path.isdir(res_dir):
        paf_file = os.path.join(app_data, id_res, "map.paf")
        idx1 = os.path.join(app_data, id_res, "query.idx")
        idx2 = os.path.join(app_data, id_res, "target.idx")
        try:
            paf = Paf(paf_file, idx1, idx2, False)
            paf.parse_paf(False)
        except FileNotFoundError:
            print("Unable to load data!")
            abort(404)
            return False
        csv_content = paf.build_query_on_target_association_file()
        return Response(csv_content, mimetype="text/plain")
    abort(404)


391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
@app.route('/no-assoc/<id_res>', methods=['POST'])
def no_assoc(id_res):
    """
    Get contigs that match with None target
    :param id_res: id of the result
    """
    res_dir = os.path.join(app_data, id_res)
    if os.path.exists(res_dir) and os.path.isdir(res_dir):
        paf_file = os.path.join(app_data, id_res, "map.paf")
        idx1 = os.path.join(app_data, id_res, "query.idx")
        idx2 = os.path.join(app_data, id_res, "target.idx")
        try:
            paf = Paf(paf_file, idx1, idx2, False)
        except FileNotFoundError:
            print("Unable to load data!")
            abort(404)
            return False
408
        file_content = paf.build_list_no_assoc(request.form["to"])
409
410
411
412
413
414
415
416
        empty = file_content == "\n"
        return jsonify({
            "file_content": file_content,
            "empty": empty
        })
    abort(404)


417
418
419
420
421
422
423
424
425
426
427
428
429
@app.route("/ask-upload", methods=['POST'])
def ask_upload():
    try:
        s_id = request.form['s_id']
        session = Session.get(s_id=s_id)
        allowed, position = session.ask_for_upload(True)
        return jsonify({
            "success": True,
            "allowed": allowed,
            "position": position
        })
    except DoesNotExist:
        return jsonify({"success": False, "message": "Session not initialized. Please refresh the page."})
430
431


432
433
434
435
436
437
@app.route("/ping-upload", methods=['POST'])
def ping_upload():
    s_id = request.form['s_id']
    session = Session.get(s_id=s_id)
    session.ping()
    return "OK"
438
439


440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
@app.route("/upload", methods=['POST'])
def upload():
    try:
        s_id = request.form['s_id']
        session = Session.get(s_id=s_id)
        if session.ask_for_upload(False)[0]:
            folder = session.upload_folder
            files = request.files[list(request.files.keys())[0]]

            if files:
                filename = files.filename
                folder_files = os.path.join(app.config["UPLOAD_FOLDER"], folder)
                if not os.path.exists(folder_files):
                    os.makedirs(folder_files)
                filename = Functions.get_valid_uploaded_filename(filename, folder_files)
                mime_type = files.content_type

                if not Functions.allowed_file(files.filename):
                    result = UploadFile(name=filename, type_f=mime_type, size=0, not_allowed_msg="File type not allowed")
                    shutil.rmtree(folder_files)

                else:
                    # save file to disk
                    uploaded_file_path = os.path.join(folder_files, filename)
                    files.save(uploaded_file_path)

                    # get file size after saving
                    size = os.path.getsize(uploaded_file_path)

                    # return json for js call back
                    result = UploadFile(name=filename, type_f=mime_type, size=size)

                return jsonify({"files": [result.get_file()], "success": "OK"})

            return jsonify({"files": [], "success": "404", "message": "No file provided"})
        return jsonify({"files": [], "success": "ERR", "message": "Not allowed to upload!"})
    except DoesNotExist:
        return jsonify({"files": [], "success": "ERR", "message": "Session not initialized. Please refresh the page."})
    except:  # Except all possible exceptions to prevent crashes
        return jsonify({"files": [], "success": "ERR", "message": "An unexpected error has occurred on upload. "
                                                                  "Please contact the support."})
481
482


483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
@app.route("/send-mail/<id_res>", methods=['POST'])
def send_mail(id_res):
    allowed = False
    key_file = None
    if "key" in request.form:
        key = request.form["key"]
        res_dir = os.path.join(app_data, id_res)
        key_file = os.path.join(res_dir, ".key")
        if os.path.exists(key_file):
            with open(key_file) as k_f:
                true_key = k_f.readline().strip("\n")
                allowed = key == true_key
    if allowed:
        os.remove(key_file)
        job_mng = JobManager(id_job=id_res, mailer=mailer)
        job_mng.set_inputs_from_res_dir()
        job_mng.send_mail()
        return "OK"
    else:
        abort(403)


Floréal Cabanettes's avatar
Floréal Cabanettes committed
505
if __name__ == '__main__':
Floreal Cabanettes's avatar
Floreal Cabanettes committed
506
    app.run()