main.py 15.1 KB
Newer Older
1
2
#!/usr/bin/env python3

Floreal Cabanettes's avatar
Floreal Cabanettes committed
3
import os
4
5
import time
import datetime
6
import shutil
7
import re
8
import threading
Floreal Cabanettes's avatar
Floreal Cabanettes committed
9
from flask import Flask, render_template, request, url_for, jsonify, session, Response, abort
10
from pathlib import Path
11
from lib.paf import Paf
12
from config_reader import AppConfigReader
13
from lib.job_manager import JobManager
14
from lib.functions import Functions, ALLOWED_EXTENSIONS
15
from lib.upload_file import UploadFile
Floreal Cabanettes's avatar
Floreal Cabanettes committed
16
from lib.fasta import Fasta
Floreal Cabanettes's avatar
Floreal Cabanettes committed
17
from lib.mailer import Mailer
Floreal Cabanettes's avatar
Floreal Cabanettes committed
18
from lib.crons import Crons
19
20

import sys
21

22
23
24
25
26
27
app_folder = os.path.dirname(os.path.realpath(__file__))
sys.path.insert(0, app_folder)
os.environ["PATH"] = os.path.join(app_folder, "bin") + ":" + os.environ["PATH"]

sqlite_file = os.path.join(app_folder, "database.sqlite")

28
29
30
31

# Init config reader:
config_reader = AppConfigReader()

Floreal Cabanettes's avatar
Floreal Cabanettes committed
32
33
UPLOAD_FOLDER = config_reader.upload_folder
APP_DATA = config_reader.app_data
34

Floreal Cabanettes's avatar
Floreal Cabanettes committed
35
app_title = "D-GENIES - Dotplot for Genomes Interactive, E-connected and Speedy"
Floreal Cabanettes's avatar
Floreal Cabanettes committed
36

37
# Init Flask:
38
app = Flask(__name__, static_url_path='/static')
39
app.config['UPLOAD_FOLDER'] = UPLOAD_FOLDER
40
app.config['SECRET_KEY'] = 'dsqdsq-255sdA-fHfg52-25Asd5'
41

42
# Init mail:
Floreal Cabanettes's avatar
Floreal Cabanettes committed
43
mailer = Mailer(app)
44

45
# Folder containing data:
Floreal Cabanettes's avatar
Floreal Cabanettes committed
46
app_data = config_reader.app_data
47

Floreal Cabanettes's avatar
Floreal Cabanettes committed
48
# Crons:
49
if os.getenv('DISABLE_CRONS') != "True":
50
51
52
    print("Starting crons...")
    crons = Crons(app_folder)
    crons.start_all()
Floreal Cabanettes's avatar
Floreal Cabanettes committed
53

54

Floreal Cabanettes's avatar
Floreal Cabanettes committed
55
56
57
58
59
60
@app.context_processor
def get_launched_results():
    cookie = request.cookies.get("results")
    return {"results": cookie.split("|") if cookie is not None else set()}


61
62
63
# Main
@app.route("/", methods=['GET'])
def main():
Floreal Cabanettes's avatar
Floreal Cabanettes committed
64
65
66
67
68
    return render_template("index.html", title=app_title, menu="index")


@app.route("/run", methods=['GET'])
def run():
Floreal Cabanettes's avatar
Floreal Cabanettes committed
69
    session["user_tmp_dir"] = Functions.random_string(5) + "_" + \
70
                              datetime.datetime.fromtimestamp(time.time()).strftime('%Y%m%d%H%M%S')
Floreal Cabanettes's avatar
Floreal Cabanettes committed
71
    id_job = Functions.random_string(5) + "_" + datetime.datetime.fromtimestamp(time.time()).strftime('%Y%m%d%H%M%S')
72
73
74
75
76
    if "id_job" in request.args:
        id_job = request.args["id_job"]
    email = ""
    if "email" in request.args:
        email = request.args["email"]
Floreal Cabanettes's avatar
Floreal Cabanettes committed
77
    return render_template("run.html", title=app_title, id_job=id_job, email=email,
78
                           menu="run", allowed_ext=ALLOWED_EXTENSIONS)
79
80
81
82
83
84
85


# Launch analysis
@app.route("/launch_analysis", methods=['POST'])
def launch_analysis():
    id_job = request.form["id_job"]
    email = request.form["email"]
86
87
88
89
    file_query = request.form["query"]
    file_query_type = request.form["query_type"]
    file_target = request.form["target"]
    file_target_type = request.form["target_type"]
90
91
92

    # Check form:
    form_pass = True
93
    errors = []
94
    if id_job == "":
95
        errors.append("Id of job not given")
96
97
98
        form_pass = False

    if email == "":
99
        errors.append("Email not given")
100
        form_pass = False
101
102
    if file_target == "":
        errors.append("No target fasta selected")
103
104
105
106
        form_pass = False

    # Form pass
    if form_pass:
107
        # Get final job id:
108
        id_job = re.sub('[^A-Za-z0-9_\-]+', '', id_job.replace(" ", "_"))
109
        id_job_orig = id_job
110
        i = 2
111
        while os.path.exists(os.path.join(app_data, id_job)):
112
113
            id_job = id_job_orig + ("_%d" % i)
            i += 1
114
115
116
117
118

        folder_files = os.path.join(app_data, id_job)
        os.makedirs(folder_files)

        # Save files:
119
120
121
122
123
124
125
126
127
128
        query = None
        if file_query != "":
            query_name = os.path.splitext(file_query.replace(".gz", ""))[0] if file_query_type == "local" else None
            query_path = os.path.join(app.config["UPLOAD_FOLDER"], session["user_tmp_dir"], file_query) \
                if file_query_type == "local" else file_query
            query = Fasta(name=query_name, path=query_path, type_f=file_query_type)
        target_name = os.path.splitext(file_target.replace(".gz", ""))[0] if file_target_type == "local" else None
        target_path = os.path.join(app.config["UPLOAD_FOLDER"], session["user_tmp_dir"], file_target) \
            if file_target_type == "local" else file_target
        target = Fasta(name=target_name, path=target_path, type_f=file_target_type)
129
130

        # Launch job:
Floreal Cabanettes's avatar
Floreal Cabanettes committed
131
        job = JobManager(id_job, email, query, target, mailer)
132
133
        job.launch()
        return jsonify({"success": True, "redirect": url_for(".status", id_job=id_job)})
134
    else:
135
        return jsonify({"success": False, "errors": errors})
136
137
138
139
140
141


# Status of a job
@app.route('/status/<id_job>', methods=['GET'])
def status(id_job):
    job = JobManager(id_job)
142
143
144
145
146
147
148
149
150
151
152
153
    j_status = job.status()
    mem_peak = j_status["mem_peak"] if "mem_peak" in j_status else None
    if mem_peak is not None:
        mem_peak = "%.1f G" % (mem_peak / 1024.0 / 1024.0)
    time_e = j_status["time_elapsed"] if "time_elapsed" in j_status else None
    if time_e is not None:
        if time_e < 60:
            time_e = "%d secs" % time_e
        else:
            minutes = time_e // 60
            seconds = time_e - minutes * 60
            time_e = "%d min %d secs" % (minutes, seconds)
154
155
156
157
158
159
160
161
162
    format = request.args.get("format")
    if format is not None and format == "json":
        return jsonify({
            "status": j_status["status"],
            "error": j_status["error"].replace("#ID#", ""),
            "id_job": id_job,
            "mem_peak": mem_peak,
            "time_elapsed": time_e
        })
163
164
165
166
    return render_template("status.html", title=app_title, status=j_status["status"],
                           error=j_status["error"].replace("#ID#", ""),
                           id_job=id_job, menu="results", mem_peak=mem_peak,
                           time_elapsed=time_e)
167
168


Floreal Cabanettes's avatar
Floreal Cabanettes committed
169
# Results path
170
@app.route("/result/<id_res>", methods=['GET'])
171
def result(id_res):
Floreal Cabanettes's avatar
Floreal Cabanettes committed
172
173
174
175
176
177
178
179
    my_render = render_template("results.html", title=app_title, id=id_res, menu="results", current_result=id_res)
    response = app.make_response(my_render)
    cookie = request.cookies.get("results")
    cookie = cookie.split("|") if cookie is not None else []
    if id_res not in cookie:
        cookie.insert(0, id_res)
    response.set_cookie(key="results", value="|".join(cookie), path="/")
    return response
180
181


182
def get_file(file, gzip=False):  # pragma: no cover
Floreal Cabanettes's avatar
Floreal Cabanettes committed
183
184
185
186
187
188
    try:
        # Figure out how flask returns static files
        # Tried:
        # - render_template
        # - send_file
        # This should not be so non-obvious
189
        return open(file, "rb" if gzip else "r").read()
Floreal Cabanettes's avatar
Floreal Cabanettes committed
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
    except IOError as exc:
        return str(exc)


@app.route("/paf/<id_res>", methods=['GET'])
def download_paf(id_res):
    map_file = os.path.join(APP_DATA, id_res, "map.paf.sorted")
    if not os.path.exists(map_file):
        map_file = os.path.join(APP_DATA, id_res, "map.paf")
    if not os.path.exists(map_file):
        abort(404)
    content = get_file(map_file)
    return Response(content, mimetype="text/plain")


Floreal Cabanettes's avatar
Floreal Cabanettes committed
205
# Get graph (ajax request)
206
207
208
@app.route('/get_graph', methods=['POST'])
def get_graph():
    id_f = request.form["id"]
209
    paf = os.path.join(app_data, id_f, "map.paf")
210
211
    idx1 = os.path.join(app_data, id_f, "query.idx")
    idx2 = os.path.join(app_data, id_f, "target.idx")
212

213
    paf = Paf(paf, idx1, idx2)
214

215
216
    if paf.parsed:
        res = paf.get_d3js_data()
217
        res["success"] = True
218
219
220
221
222
223
        return jsonify(res)
    return jsonify({"success": False, "message": paf.error})


@app.route('/sort/<id_res>', methods=['POST'])
def sort_graph(id_res):
224
    if not os.path.exists(os.path.join(APP_DATA, id_res, ".all-vs-all")):
225
        paf_file = os.path.join(app_data, id_res, "map.paf")
226
227
        idx1 = os.path.join(app_data, id_res, "query.idx")
        idx2 = os.path.join(app_data, id_res, "target.idx")
228
        paf = Paf(paf_file, idx1, idx2, False)
229
230
231
232
233
234
235
        paf.sort()
        if paf.parsed:
            res = paf.get_d3js_data()
            res["success"] = True
            return jsonify(res)
        return jsonify({"success": False, "message": paf.error})
    return jsonify({"success": False, "message": "Sort is not available for All-vs-All mode"})
Floreal Cabanettes's avatar
Floreal Cabanettes committed
236

Floréal Cabanettes's avatar
Floréal Cabanettes committed
237

238
239
240
241
242
243
244
245
246
247
248
249
250
251
@app.route('/freenoise/<id_res>', methods=['POST'])
def free_noise(id_res):
    paf_file = os.path.join(app_data, id_res, "map.paf")
    idx1 = os.path.join(app_data, id_res, "query.idx")
    idx2 = os.path.join(app_data, id_res, "target.idx")
    paf = Paf(paf_file, idx1, idx2, False)
    paf.parse_paf(noise=request.form["noise"] == "1")
    if paf.parsed:
        res = paf.get_d3js_data()
        res["success"] = True
        return jsonify(res)
    return jsonify({"success": False, "message": paf.error})


252
253
254
255
256
@app.route('/get-fasta-query/<id_res>', methods=['POST'])
def build_fasta(id_res):
    res_dir = os.path.join(app_data, id_res)
    lock_query = os.path.join(res_dir, ".query-fasta-build")
    is_sorted = os.path.exists(os.path.join(res_dir, ".sorted"))
257
    compressed = request.form["gzip"].lower() == "true"
258
259
260
261
262
    query_fasta = Functions.get_fasta_file(res_dir, "query", is_sorted)
    if query_fasta is not None:
        if is_sorted and not query_fasta.endswith(".sorted"):
            # Do the sort
            Path(lock_query).touch()
263
264
            if not compressed:  # If compressed, it will took a long time, so not wait
                Path(lock_query + ".pending").touch()
265
266
267
268
269
            thread = threading.Timer(1, Functions.sort_fasta, kwargs={
                "job_name": id_res,
                "fasta_file": query_fasta,
                "index_file": os.path.join(res_dir, "query.idx.sorted"),
                "lock_file": lock_query,
270
                "compress": compressed,
271
272
273
                "mailer": mailer
            })
            thread.start()
274
275
276
277
278
279
            if not compressed:
                i = 0
                time.sleep(5)
                while os.path.exists(lock_query) and i < 2:
                    i += 1
                    time.sleep(5)
Floreal Cabanettes's avatar
Floreal Cabanettes committed
280
                os.remove(lock_query + ".pending")
281
282
283
                if os.path.exists(lock_query):
                    return jsonify({"success": True, "status": 1, "status_message": "In progress"})
                return jsonify({"success": True, "status": 2, "status_message": "Done",
284
                                "gzip": False})
285
286
            else:
                return jsonify({"success": True, "status": 1, "status_message": "In progress"})
287
288
289
290
291
        elif is_sorted and os.path.exists(lock_query):
            # Sort is already in progress
            return jsonify({"success": True, "status": 1, "status_message": "In progress"})
        else:
            # No sort to do or sort done
292
293
294
295
296
297
298
            if compressed and not query_fasta.endswith(".gz.fasta"):
                # If compressed file is asked, we must compress it now if not done before...
                Path(lock_query).touch()
                thread = threading.Timer(1, Functions.compress_and_send_mail, kwargs={
                    "job_name": id_res,
                    "fasta_file": query_fasta,
                    "index_file": os.path.join(res_dir, "query.idx.sorted"),
299
                    "lock_file": lock_query,
300
301
302
303
304
                    "compressed": compressed,
                    "mailer": mailer
                })
                thread.start()
                return jsonify({"success": True, "status": 1, "status_message": "In progress"})
305
306
307
308
309
310
311
            return jsonify({"success": True, "status": 2, "status_message": "Done",
                            "gzip": query_fasta.endswith(".gz") or query_fasta.endswith(".gz.sorted")})
    else:
        return jsonify({"success": False,
                        "message": "Unable to get fasta file for query. Please contact us to report the bug"})


312
313
314
@app.route('/fasta-query/<id_res>', defaults={'filename': ""}, methods=['GET'])
@app.route('/fasta-query/<id_res>/<filename>', methods=['GET'])  # Use fake URL in mail to set download file name
def dl_fasta(id_res, filename):
315
316
317
318
319
320
321
322
323
324
325
326
327
328
    res_dir = os.path.join(app_data, id_res)
    lock_query = os.path.join(res_dir, ".query-fasta-build")
    is_sorted = os.path.exists(os.path.join(res_dir, ".sorted"))
    if not os.path.exists(lock_query) or not is_sorted:
        query_fasta = Functions.get_fasta_file(res_dir, "query", is_sorted)
        if query_fasta is not None:
            if query_fasta.endswith(".gz") or query_fasta.endswith(".gz.sorted"):
                content = get_file(query_fasta, True)
                return Response(content, mimetype="application/gzip")
            content = get_file(query_fasta)
            return Response(content, mimetype="text/plain")
    abort(404)


329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
@app.route('/qt-assoc/<id_res>', methods=['GET'])
def qt_assoc(id_res):
    res_dir = os.path.join(app_data, id_res)
    if os.path.exists(res_dir) and os.path.isdir(res_dir):
        paf_file = os.path.join(app_data, id_res, "map.paf")
        idx1 = os.path.join(app_data, id_res, "query.idx")
        idx2 = os.path.join(app_data, id_res, "target.idx")
        try:
            paf = Paf(paf_file, idx1, idx2, False)
            paf.parse_paf(False)
        except FileNotFoundError:
            print("Unable to load data!")
            abort(404)
            return False
        csv_content = paf.build_query_on_target_association_file()
        return Response(csv_content, mimetype="text/plain")
    abort(404)


348
349
@app.route("/upload", methods=['POST'])
def upload():
350
351
352
353
354
355
356
357
358
    if "user_tmp_dir" in session and session["user_tmp_dir"] != "":
        folder = session["user_tmp_dir"]
        files = request.files[list(request.files.keys())[0]]

        if files:
            filename = files.filename
            folder_files = os.path.join(app.config["UPLOAD_FOLDER"], folder)
            if not os.path.exists(folder_files):
                os.makedirs(folder_files)
Floreal Cabanettes's avatar
Floreal Cabanettes committed
359
            filename = Functions.get_valid_uploaded_filename(filename, folder_files)
360
361
            mime_type = files.content_type

Floreal Cabanettes's avatar
Floreal Cabanettes committed
362
            if not Functions.allowed_file(files.filename):
363
                result = UploadFile(name=filename, type_f=mime_type, size=0, not_allowed_msg="File type not allowed")
364
                shutil.rmtree(folder_files)
365
366
367
368
369
370
371
372
373
374

            else:
                # save file to disk
                uploaded_file_path = os.path.join(folder_files, filename)
                files.save(uploaded_file_path)

                # get file size after saving
                size = os.path.getsize(uploaded_file_path)

                # return json for js call back
375
                result = UploadFile(name=filename, type_f=mime_type, size=size)
376
377
378
379
380

            return jsonify({"files": [result.get_file()], "success": "OK"})

        return jsonify({"files": [], "success": "404", "message": "No file provided"})
    return jsonify({"files": [], "success": "ERR", "message": "Session not initialized. Please refresh the page."})
381
382


383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
@app.route("/send-mail/<id_res>", methods=['POST'])
def send_mail(id_res):
    allowed = False
    key_file = None
    if "key" in request.form:
        key = request.form["key"]
        res_dir = os.path.join(app_data, id_res)
        key_file = os.path.join(res_dir, ".key")
        if os.path.exists(key_file):
            with open(key_file) as k_f:
                true_key = k_f.readline().strip("\n")
                allowed = key == true_key
    if allowed:
        os.remove(key_file)
        job_mng = JobManager(id_job=id_res, mailer=mailer)
        job_mng.set_inputs_from_res_dir()
        job_mng.send_mail()
        return "OK"
    else:
        abort(403)


Floréal Cabanettes's avatar
Floréal Cabanettes committed
405
if __name__ == '__main__':
Floreal Cabanettes's avatar
Floreal Cabanettes committed
406
    app.run()