main.py 16.9 KB
Newer Older
1
2
#!/usr/bin/env python3

Floreal Cabanettes's avatar
Floreal Cabanettes committed
3
import os
4
5
import time
import datetime
6
import shutil
7
import re
8
import threading
9
from flask import Flask, render_template, request, url_for, jsonify, Response, abort
10
from pathlib import Path
11
from lib.paf import Paf
12
from config_reader import AppConfigReader
13
from lib.job_manager import JobManager
14
from lib.functions import Functions, ALLOWED_EXTENSIONS
15
from lib.upload_file import UploadFile
Floreal Cabanettes's avatar
Floreal Cabanettes committed
16
from lib.fasta import Fasta
Floreal Cabanettes's avatar
Floreal Cabanettes committed
17
from lib.mailer import Mailer
Floreal Cabanettes's avatar
Floreal Cabanettes committed
18
from lib.crons import Crons
19
20
from database import Session
from peewee import DoesNotExist
21
22

import sys
23

24
25
26
27
28
29
app_folder = os.path.dirname(os.path.realpath(__file__))
sys.path.insert(0, app_folder)
os.environ["PATH"] = os.path.join(app_folder, "bin") + ":" + os.environ["PATH"]

sqlite_file = os.path.join(app_folder, "database.sqlite")

30
31
32
33

# Init config reader:
config_reader = AppConfigReader()

Floreal Cabanettes's avatar
Floreal Cabanettes committed
34
35
UPLOAD_FOLDER = config_reader.upload_folder
APP_DATA = config_reader.app_data
36

Floreal Cabanettes's avatar
Floreal Cabanettes committed
37
app_title = "D-GENIES - Dotplot for Genomes Interactive, E-connected and Speedy"
Floreal Cabanettes's avatar
Floreal Cabanettes committed
38

39
# Init Flask:
40
app = Flask(__name__, static_url_path='/static')
41
app.config['UPLOAD_FOLDER'] = UPLOAD_FOLDER
42
app.config['SECRET_KEY'] = 'dsqdsq-255sdA-fHfg52-25Asd5'
43

44
# Init mail:
Floreal Cabanettes's avatar
Floreal Cabanettes committed
45
mailer = Mailer(app)
46

47
# Folder containing data:
Floreal Cabanettes's avatar
Floreal Cabanettes committed
48
app_data = config_reader.app_data
49

50
51
52
if config_reader.debug and config_reader.log_dir != "stdout" and not os.path.exists(config_reader.log_dir):
    os.makedirs(config_reader.log_dir)

Floreal Cabanettes's avatar
Floreal Cabanettes committed
53
# Crons:
54
if os.getenv('DISABLE_CRONS') != "True":
55
56
57
    print("Starting crons...")
    crons = Crons(app_folder)
    crons.start_all()
Floreal Cabanettes's avatar
Floreal Cabanettes committed
58

59

Floreal Cabanettes's avatar
Floreal Cabanettes committed
60
61
62
63
64
65
@app.context_processor
def get_launched_results():
    cookie = request.cookies.get("results")
    return {"results": cookie.split("|") if cookie is not None else set()}


66
67
68
# Main
@app.route("/", methods=['GET'])
def main():
Floreal Cabanettes's avatar
Floreal Cabanettes committed
69
70
71
72
73
    return render_template("index.html", title=app_title, menu="index")


@app.route("/run", methods=['GET'])
def run():
74
    s_id = Session.new()
Floreal Cabanettes's avatar
Floreal Cabanettes committed
75
    id_job = Functions.random_string(5) + "_" + datetime.datetime.fromtimestamp(time.time()).strftime('%Y%m%d%H%M%S')
76
77
78
79
80
    if "id_job" in request.args:
        id_job = request.args["id_job"]
    email = ""
    if "email" in request.args:
        email = request.args["email"]
Floreal Cabanettes's avatar
Floreal Cabanettes committed
81
    return render_template("run.html", title=app_title, id_job=id_job, email=email,
82
                           menu="run", allowed_ext=ALLOWED_EXTENSIONS, s_id=s_id)
83
84


85
86
87
88
89
90
91
92
@app.route("/run-test", methods=['GET'])
def run_test():
    print(config_reader.allowed_ip_tests)
    if request.remote_addr not in config_reader.allowed_ip_tests:
        return abort(404)
    return Session.new()


93
94
95
# Launch analysis
@app.route("/launch_analysis", methods=['POST'])
def launch_analysis():
96
97
98
99
100
101
102
103
    try:
        session = Session.get(s_id=request.form["s_id"])
    except DoesNotExist:
        return jsonify({"success": False, "errors": ["Session has expired. Please refresh the page and try again"]})
    # Reset session upload:
    session.allow_upload = False
    session.position = -1
    session.save()
104
105
    id_job = request.form["id_job"]
    email = request.form["email"]
106
107
108
109
    file_query = request.form["query"]
    file_query_type = request.form["query_type"]
    file_target = request.form["target"]
    file_target_type = request.form["target_type"]
110
111
112

    # Check form:
    form_pass = True
113
    errors = []
114
    if id_job == "":
115
        errors.append("Id of job not given")
116
117
118
        form_pass = False

    if email == "":
119
        errors.append("Email not given")
120
        form_pass = False
121
122
    if file_target == "":
        errors.append("No target fasta selected")
123
124
125
126
        form_pass = False

    # Form pass
    if form_pass:
127
        # Get final job id:
128
        id_job = re.sub('[^A-Za-z0-9_\-]+', '', id_job.replace(" ", "_"))
129
        id_job_orig = id_job
130
        i = 2
131
        while os.path.exists(os.path.join(app_data, id_job)):
132
133
            id_job = id_job_orig + ("_%d" % i)
            i += 1
134
135
136
137
138

        folder_files = os.path.join(app_data, id_job)
        os.makedirs(folder_files)

        # Save files:
139
        query = None
140
        upload_folder = session.upload_folder
141
142
        if file_query != "":
            query_name = os.path.splitext(file_query.replace(".gz", ""))[0] if file_query_type == "local" else None
143
            query_path = os.path.join(app.config["UPLOAD_FOLDER"], upload_folder, file_query) \
144
145
146
                if file_query_type == "local" else file_query
            query = Fasta(name=query_name, path=query_path, type_f=file_query_type)
        target_name = os.path.splitext(file_target.replace(".gz", ""))[0] if file_target_type == "local" else None
147
        target_path = os.path.join(app.config["UPLOAD_FOLDER"], upload_folder, file_target) \
148
149
            if file_target_type == "local" else file_target
        target = Fasta(name=target_name, path=target_path, type_f=file_target_type)
150
151

        # Launch job:
Floreal Cabanettes's avatar
Floreal Cabanettes committed
152
        job = JobManager(id_job, email, query, target, mailer)
153
        job.launch()
154
155
156

        # Delete session:
        session.delete_instance()
157
        return jsonify({"success": True, "redirect": url_for(".status", id_job=id_job)})
158
    else:
159
        return jsonify({"success": False, "errors": errors})
160
161
162
163
164
165


# Status of a job
@app.route('/status/<id_job>', methods=['GET'])
def status(id_job):
    job = JobManager(id_job)
166
167
168
169
170
171
172
173
174
175
176
177
    j_status = job.status()
    mem_peak = j_status["mem_peak"] if "mem_peak" in j_status else None
    if mem_peak is not None:
        mem_peak = "%.1f G" % (mem_peak / 1024.0 / 1024.0)
    time_e = j_status["time_elapsed"] if "time_elapsed" in j_status else None
    if time_e is not None:
        if time_e < 60:
            time_e = "%d secs" % time_e
        else:
            minutes = time_e // 60
            seconds = time_e - minutes * 60
            time_e = "%d min %d secs" % (minutes, seconds)
178
179
180
181
182
183
184
185
186
    format = request.args.get("format")
    if format is not None and format == "json":
        return jsonify({
            "status": j_status["status"],
            "error": j_status["error"].replace("#ID#", ""),
            "id_job": id_job,
            "mem_peak": mem_peak,
            "time_elapsed": time_e
        })
187
188
189
190
    return render_template("status.html", title=app_title, status=j_status["status"],
                           error=j_status["error"].replace("#ID#", ""),
                           id_job=id_job, menu="results", mem_peak=mem_peak,
                           time_elapsed=time_e)
191
192


Floreal Cabanettes's avatar
Floreal Cabanettes committed
193
# Results path
194
@app.route("/result/<id_res>", methods=['GET'])
195
def result(id_res):
Floreal Cabanettes's avatar
Floreal Cabanettes committed
196
197
198
199
200
201
202
203
    my_render = render_template("results.html", title=app_title, id=id_res, menu="results", current_result=id_res)
    response = app.make_response(my_render)
    cookie = request.cookies.get("results")
    cookie = cookie.split("|") if cookie is not None else []
    if id_res not in cookie:
        cookie.insert(0, id_res)
    response.set_cookie(key="results", value="|".join(cookie), path="/")
    return response
204
205


206
def get_file(file, gzip=False):  # pragma: no cover
Floreal Cabanettes's avatar
Floreal Cabanettes committed
207
208
209
210
211
212
    try:
        # Figure out how flask returns static files
        # Tried:
        # - render_template
        # - send_file
        # This should not be so non-obvious
213
        return open(file, "rb" if gzip else "r").read()
Floreal Cabanettes's avatar
Floreal Cabanettes committed
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
    except IOError as exc:
        return str(exc)


@app.route("/paf/<id_res>", methods=['GET'])
def download_paf(id_res):
    map_file = os.path.join(APP_DATA, id_res, "map.paf.sorted")
    if not os.path.exists(map_file):
        map_file = os.path.join(APP_DATA, id_res, "map.paf")
    if not os.path.exists(map_file):
        abort(404)
    content = get_file(map_file)
    return Response(content, mimetype="text/plain")


Floreal Cabanettes's avatar
Floreal Cabanettes committed
229
# Get graph (ajax request)
230
231
232
@app.route('/get_graph', methods=['POST'])
def get_graph():
    id_f = request.form["id"]
233
    paf = os.path.join(app_data, id_f, "map.paf")
234
235
    idx1 = os.path.join(app_data, id_f, "query.idx")
    idx2 = os.path.join(app_data, id_f, "target.idx")
236

237
    paf = Paf(paf, idx1, idx2)
238

239
240
    if paf.parsed:
        res = paf.get_d3js_data()
241
        res["success"] = True
242
243
244
245
246
247
        return jsonify(res)
    return jsonify({"success": False, "message": paf.error})


@app.route('/sort/<id_res>', methods=['POST'])
def sort_graph(id_res):
248
    if not os.path.exists(os.path.join(APP_DATA, id_res, ".all-vs-all")):
249
        paf_file = os.path.join(app_data, id_res, "map.paf")
250
251
        idx1 = os.path.join(app_data, id_res, "query.idx")
        idx2 = os.path.join(app_data, id_res, "target.idx")
252
        paf = Paf(paf_file, idx1, idx2, False)
253
254
255
256
257
258
259
        paf.sort()
        if paf.parsed:
            res = paf.get_d3js_data()
            res["success"] = True
            return jsonify(res)
        return jsonify({"success": False, "message": paf.error})
    return jsonify({"success": False, "message": "Sort is not available for All-vs-All mode"})
Floreal Cabanettes's avatar
Floreal Cabanettes committed
260

Floréal Cabanettes's avatar
Floréal Cabanettes committed
261

262
263
264
265
266
267
268
269
270
271
272
273
274
275
@app.route('/freenoise/<id_res>', methods=['POST'])
def free_noise(id_res):
    paf_file = os.path.join(app_data, id_res, "map.paf")
    idx1 = os.path.join(app_data, id_res, "query.idx")
    idx2 = os.path.join(app_data, id_res, "target.idx")
    paf = Paf(paf_file, idx1, idx2, False)
    paf.parse_paf(noise=request.form["noise"] == "1")
    if paf.parsed:
        res = paf.get_d3js_data()
        res["success"] = True
        return jsonify(res)
    return jsonify({"success": False, "message": paf.error})


276
277
278
279
280
@app.route('/get-fasta-query/<id_res>', methods=['POST'])
def build_fasta(id_res):
    res_dir = os.path.join(app_data, id_res)
    lock_query = os.path.join(res_dir, ".query-fasta-build")
    is_sorted = os.path.exists(os.path.join(res_dir, ".sorted"))
281
    compressed = request.form["gzip"].lower() == "true"
282
283
284
285
286
    query_fasta = Functions.get_fasta_file(res_dir, "query", is_sorted)
    if query_fasta is not None:
        if is_sorted and not query_fasta.endswith(".sorted"):
            # Do the sort
            Path(lock_query).touch()
287
288
            if not compressed:  # If compressed, it will took a long time, so not wait
                Path(lock_query + ".pending").touch()
289
290
291
292
293
            thread = threading.Timer(1, Functions.sort_fasta, kwargs={
                "job_name": id_res,
                "fasta_file": query_fasta,
                "index_file": os.path.join(res_dir, "query.idx.sorted"),
                "lock_file": lock_query,
294
                "compress": compressed,
295
296
297
                "mailer": mailer
            })
            thread.start()
298
299
300
301
302
303
            if not compressed:
                i = 0
                time.sleep(5)
                while os.path.exists(lock_query) and i < 2:
                    i += 1
                    time.sleep(5)
Floreal Cabanettes's avatar
Floreal Cabanettes committed
304
                os.remove(lock_query + ".pending")
305
306
307
                if os.path.exists(lock_query):
                    return jsonify({"success": True, "status": 1, "status_message": "In progress"})
                return jsonify({"success": True, "status": 2, "status_message": "Done",
308
                                "gzip": False})
309
310
            else:
                return jsonify({"success": True, "status": 1, "status_message": "In progress"})
311
312
313
314
315
        elif is_sorted and os.path.exists(lock_query):
            # Sort is already in progress
            return jsonify({"success": True, "status": 1, "status_message": "In progress"})
        else:
            # No sort to do or sort done
316
317
318
319
320
321
322
            if compressed and not query_fasta.endswith(".gz.fasta"):
                # If compressed file is asked, we must compress it now if not done before...
                Path(lock_query).touch()
                thread = threading.Timer(1, Functions.compress_and_send_mail, kwargs={
                    "job_name": id_res,
                    "fasta_file": query_fasta,
                    "index_file": os.path.join(res_dir, "query.idx.sorted"),
323
                    "lock_file": lock_query,
324
325
326
327
328
                    "compressed": compressed,
                    "mailer": mailer
                })
                thread.start()
                return jsonify({"success": True, "status": 1, "status_message": "In progress"})
329
330
331
332
333
334
335
            return jsonify({"success": True, "status": 2, "status_message": "Done",
                            "gzip": query_fasta.endswith(".gz") or query_fasta.endswith(".gz.sorted")})
    else:
        return jsonify({"success": False,
                        "message": "Unable to get fasta file for query. Please contact us to report the bug"})


336
337
338
@app.route('/fasta-query/<id_res>', defaults={'filename': ""}, methods=['GET'])
@app.route('/fasta-query/<id_res>/<filename>', methods=['GET'])  # Use fake URL in mail to set download file name
def dl_fasta(id_res, filename):
339
340
341
342
343
344
345
346
347
348
349
350
351
352
    res_dir = os.path.join(app_data, id_res)
    lock_query = os.path.join(res_dir, ".query-fasta-build")
    is_sorted = os.path.exists(os.path.join(res_dir, ".sorted"))
    if not os.path.exists(lock_query) or not is_sorted:
        query_fasta = Functions.get_fasta_file(res_dir, "query", is_sorted)
        if query_fasta is not None:
            if query_fasta.endswith(".gz") or query_fasta.endswith(".gz.sorted"):
                content = get_file(query_fasta, True)
                return Response(content, mimetype="application/gzip")
            content = get_file(query_fasta)
            return Response(content, mimetype="text/plain")
    abort(404)


353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
@app.route('/qt-assoc/<id_res>', methods=['GET'])
def qt_assoc(id_res):
    res_dir = os.path.join(app_data, id_res)
    if os.path.exists(res_dir) and os.path.isdir(res_dir):
        paf_file = os.path.join(app_data, id_res, "map.paf")
        idx1 = os.path.join(app_data, id_res, "query.idx")
        idx2 = os.path.join(app_data, id_res, "target.idx")
        try:
            paf = Paf(paf_file, idx1, idx2, False)
            paf.parse_paf(False)
        except FileNotFoundError:
            print("Unable to load data!")
            abort(404)
            return False
        csv_content = paf.build_query_on_target_association_file()
        return Response(csv_content, mimetype="text/plain")
    abort(404)


372
373
374
375
376
377
378
379
380
381
382
383
384
@app.route("/ask-upload", methods=['POST'])
def ask_upload():
    try:
        s_id = request.form['s_id']
        session = Session.get(s_id=s_id)
        allowed, position = session.ask_for_upload(True)
        return jsonify({
            "success": True,
            "allowed": allowed,
            "position": position
        })
    except DoesNotExist:
        return jsonify({"success": False, "message": "Session not initialized. Please refresh the page."})
385
386


387
388
389
390
391
392
@app.route("/ping-upload", methods=['POST'])
def ping_upload():
    s_id = request.form['s_id']
    session = Session.get(s_id=s_id)
    session.ping()
    return "OK"
393
394


395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
@app.route("/upload", methods=['POST'])
def upload():
    try:
        s_id = request.form['s_id']
        session = Session.get(s_id=s_id)
        if session.ask_for_upload(False)[0]:
            folder = session.upload_folder
            files = request.files[list(request.files.keys())[0]]

            if files:
                filename = files.filename
                folder_files = os.path.join(app.config["UPLOAD_FOLDER"], folder)
                if not os.path.exists(folder_files):
                    os.makedirs(folder_files)
                filename = Functions.get_valid_uploaded_filename(filename, folder_files)
                mime_type = files.content_type

                if not Functions.allowed_file(files.filename):
                    result = UploadFile(name=filename, type_f=mime_type, size=0, not_allowed_msg="File type not allowed")
                    shutil.rmtree(folder_files)

                else:
                    # save file to disk
                    uploaded_file_path = os.path.join(folder_files, filename)
                    files.save(uploaded_file_path)

                    # get file size after saving
                    size = os.path.getsize(uploaded_file_path)

                    # return json for js call back
                    result = UploadFile(name=filename, type_f=mime_type, size=size)

                return jsonify({"files": [result.get_file()], "success": "OK"})

            return jsonify({"files": [], "success": "404", "message": "No file provided"})
        return jsonify({"files": [], "success": "ERR", "message": "Not allowed to upload!"})
    except DoesNotExist:
        return jsonify({"files": [], "success": "ERR", "message": "Session not initialized. Please refresh the page."})
    except:  # Except all possible exceptions to prevent crashes
        return jsonify({"files": [], "success": "ERR", "message": "An unexpected error has occurred on upload. "
                                                                  "Please contact the support."})
436
437


438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
@app.route("/send-mail/<id_res>", methods=['POST'])
def send_mail(id_res):
    allowed = False
    key_file = None
    if "key" in request.form:
        key = request.form["key"]
        res_dir = os.path.join(app_data, id_res)
        key_file = os.path.join(res_dir, ".key")
        if os.path.exists(key_file):
            with open(key_file) as k_f:
                true_key = k_f.readline().strip("\n")
                allowed = key == true_key
    if allowed:
        os.remove(key_file)
        job_mng = JobManager(id_job=id_res, mailer=mailer)
        job_mng.set_inputs_from_res_dir()
        job_mng.send_mail()
        return "OK"
    else:
        abort(403)


Floréal Cabanettes's avatar
Floréal Cabanettes committed
460
if __name__ == '__main__':
Floreal Cabanettes's avatar
Floreal Cabanettes committed
461
    app.run()