main.py 13 KB
Newer Older
1
2
#!/usr/bin/env python3

Floreal Cabanettes's avatar
Floreal Cabanettes committed
3
import os
4
5
import time
import datetime
6
import shutil
7
import re
8
import threading
Floreal Cabanettes's avatar
Floreal Cabanettes committed
9
from flask import Flask, render_template, request, url_for, jsonify, session, Response, abort
10
from pathlib import Path
11
from lib.paf import Paf
12
from config_reader import AppConfigReader
13
from lib.job_manager import JobManager
14
from lib.functions import Functions, ALLOWED_EXTENSIONS
15
from lib.upload_file import UploadFile
16
from lib.Fasta import Fasta
Floreal Cabanettes's avatar
Floreal Cabanettes committed
17
from lib.mailer import Mailer
Floreal Cabanettes's avatar
Floreal Cabanettes committed
18
from lib.crons import Crons
19
20

import sys
21

22
23
24
25
26
27
app_folder = os.path.dirname(os.path.realpath(__file__))
sys.path.insert(0, app_folder)
os.environ["PATH"] = os.path.join(app_folder, "bin") + ":" + os.environ["PATH"]

sqlite_file = os.path.join(app_folder, "database.sqlite")

28
29
30
31

# Init config reader:
config_reader = AppConfigReader()

32
UPLOAD_FOLDER = config_reader.get_upload_folder()
Floreal Cabanettes's avatar
Floreal Cabanettes committed
33
APP_DATA = config_reader.get_app_data()
34

Floreal Cabanettes's avatar
Floreal Cabanettes committed
35
app_title = "D-GENIES - Dotplot for Genomes Interactive, E-connected and Speedy"
Floreal Cabanettes's avatar
Floreal Cabanettes committed
36

37
# Init Flask:
38
app = Flask(__name__, static_url_path='/static')
39
app.config['UPLOAD_FOLDER'] = UPLOAD_FOLDER
40
app.config['SECRET_KEY'] = 'dsqdsq-255sdA-fHfg52-25Asd5'
41

42
# Init mail:
Floreal Cabanettes's avatar
Floreal Cabanettes committed
43
mailer = Mailer(app)
44

45
# Folder containing data:
46
47
app_data = config_reader.get_app_data()

Floreal Cabanettes's avatar
Floreal Cabanettes committed
48
49
# Crons:
crons = Crons(app_folder)
Floreal Cabanettes's avatar
Floreal Cabanettes committed
50
crons.start_all()
Floreal Cabanettes's avatar
Floreal Cabanettes committed
51

52

Floreal Cabanettes's avatar
Floreal Cabanettes committed
53
54
55
56
57
58
@app.context_processor
def get_launched_results():
    cookie = request.cookies.get("results")
    return {"results": cookie.split("|") if cookie is not None else set()}


59
60
61
# Main
@app.route("/", methods=['GET'])
def main():
Floreal Cabanettes's avatar
Floreal Cabanettes committed
62
63
64
65
66
    return render_template("index.html", title=app_title, menu="index")


@app.route("/run", methods=['GET'])
def run():
Floreal Cabanettes's avatar
Floreal Cabanettes committed
67
    session["user_tmp_dir"] = Functions.random_string(5) + "_" + \
68
                              datetime.datetime.fromtimestamp(time.time()).strftime('%Y%m%d%H%M%S')
Floreal Cabanettes's avatar
Floreal Cabanettes committed
69
    id_job = Functions.random_string(5) + "_" + datetime.datetime.fromtimestamp(time.time()).strftime('%Y%m%d%H%M%S')
70
71
72
73
74
    if "id_job" in request.args:
        id_job = request.args["id_job"]
    email = ""
    if "email" in request.args:
        email = request.args["email"]
Floreal Cabanettes's avatar
Floreal Cabanettes committed
75
    return render_template("run.html", title=app_title, id_job=id_job, email=email,
76
                           menu="run", allowed_ext=ALLOWED_EXTENSIONS)
77
78
79
80
81
82
83


# Launch analysis
@app.route("/launch_analysis", methods=['POST'])
def launch_analysis():
    id_job = request.form["id_job"]
    email = request.form["email"]
84
85
86
87
    file_query = request.form["query"]
    file_query_type = request.form["query_type"]
    file_target = request.form["target"]
    file_target_type = request.form["target_type"]
88
89
90

    # Check form:
    form_pass = True
91
    errors = []
92
    if id_job == "":
93
        errors.append("Id of job not given")
94
95
96
        form_pass = False

    if email == "":
97
        errors.append("Email not given")
98
        form_pass = False
99
100
    if file_target == "":
        errors.append("No target fasta selected")
101
102
103
104
        form_pass = False

    # Form pass
    if form_pass:
105
        # Get final job id:
106
        id_job = re.sub('[^A-Za-z0-9_\-]+', '', id_job.replace(" ", "_"))
107
108
109
110
111
112
113
114
        id_job_orig = id_job
        while os.path.exists(os.path.join(app_data, id_job)):
            id_job = id_job_orig + "_2"

        folder_files = os.path.join(app_data, id_job)
        os.makedirs(folder_files)

        # Save files:
115
116
117
118
119
120
121
122
123
124
        query = None
        if file_query != "":
            query_name = os.path.splitext(file_query.replace(".gz", ""))[0] if file_query_type == "local" else None
            query_path = os.path.join(app.config["UPLOAD_FOLDER"], session["user_tmp_dir"], file_query) \
                if file_query_type == "local" else file_query
            query = Fasta(name=query_name, path=query_path, type_f=file_query_type)
        target_name = os.path.splitext(file_target.replace(".gz", ""))[0] if file_target_type == "local" else None
        target_path = os.path.join(app.config["UPLOAD_FOLDER"], session["user_tmp_dir"], file_target) \
            if file_target_type == "local" else file_target
        target = Fasta(name=target_name, path=target_path, type_f=file_target_type)
125
126

        # Launch job:
Floreal Cabanettes's avatar
Floreal Cabanettes committed
127
        job = JobManager(id_job, email, query, target, mailer)
128
129
        job.launch()
        return jsonify({"success": True, "redirect": url_for(".status", id_job=id_job)})
130
    else:
131
        return jsonify({"success": False, "errors": errors})
132
133
134
135
136
137


# Status of a job
@app.route('/status/<id_job>', methods=['GET'])
def status(id_job):
    job = JobManager(id_job)
138
    j_status, error = job.status()
139
140
    return render_template("status.html", title=app_title, status=j_status, error=error.replace("#ID#", ""),
                           id_job=id_job, menu="results")
141
142


Floreal Cabanettes's avatar
Floreal Cabanettes committed
143
# Results path
144
@app.route("/result/<id_res>", methods=['GET'])
145
def result(id_res):
Floreal Cabanettes's avatar
Floreal Cabanettes committed
146
147
148
149
150
151
152
153
    my_render = render_template("results.html", title=app_title, id=id_res, menu="results", current_result=id_res)
    response = app.make_response(my_render)
    cookie = request.cookies.get("results")
    cookie = cookie.split("|") if cookie is not None else []
    if id_res not in cookie:
        cookie.insert(0, id_res)
    response.set_cookie(key="results", value="|".join(cookie), path="/")
    return response
154
155


156
def get_file(file, gzip=False):  # pragma: no cover
Floreal Cabanettes's avatar
Floreal Cabanettes committed
157
158
159
160
161
162
    try:
        # Figure out how flask returns static files
        # Tried:
        # - render_template
        # - send_file
        # This should not be so non-obvious
163
        return open(file, "rb" if gzip else "r").read()
Floreal Cabanettes's avatar
Floreal Cabanettes committed
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
    except IOError as exc:
        return str(exc)


@app.route("/paf/<id_res>", methods=['GET'])
def download_paf(id_res):
    map_file = os.path.join(APP_DATA, id_res, "map.paf.sorted")
    if not os.path.exists(map_file):
        map_file = os.path.join(APP_DATA, id_res, "map.paf")
    if not os.path.exists(map_file):
        abort(404)
    content = get_file(map_file)
    return Response(content, mimetype="text/plain")


Floreal Cabanettes's avatar
Floreal Cabanettes committed
179
# Get graph (ajax request)
180
181
182
@app.route('/get_graph', methods=['POST'])
def get_graph():
    id_f = request.form["id"]
183
    paf = os.path.join(app_data, id_f, "map.paf")
184
185
    idx1 = os.path.join(app_data, id_f, "query.idx")
    idx2 = os.path.join(app_data, id_f, "target.idx")
186

187
    paf = Paf(paf, idx1, idx2)
188

189
190
    if paf.parsed:
        res = paf.get_d3js_data()
191
        res["success"] = True
192
193
194
195
196
197
        return jsonify(res)
    return jsonify({"success": False, "message": paf.error})


@app.route('/sort/<id_res>', methods=['POST'])
def sort_graph(id_res):
198
    if not os.path.exists(os.path.join(APP_DATA, id_res, ".all-vs-all")):
199
        paf_file = os.path.join(app_data, id_res, "map.paf")
200
201
        idx1 = os.path.join(app_data, id_res, "query.idx")
        idx2 = os.path.join(app_data, id_res, "target.idx")
202
        paf = Paf(paf_file, idx1, idx2, False)
203
204
205
206
207
208
209
        paf.sort()
        if paf.parsed:
            res = paf.get_d3js_data()
            res["success"] = True
            return jsonify(res)
        return jsonify({"success": False, "message": paf.error})
    return jsonify({"success": False, "message": "Sort is not available for All-vs-All mode"})
Floreal Cabanettes's avatar
Floreal Cabanettes committed
210

Floréal Cabanettes's avatar
Floréal Cabanettes committed
211

212
213
214
215
216
@app.route('/get-fasta-query/<id_res>', methods=['POST'])
def build_fasta(id_res):
    res_dir = os.path.join(app_data, id_res)
    lock_query = os.path.join(res_dir, ".query-fasta-build")
    is_sorted = os.path.exists(os.path.join(res_dir, ".sorted"))
217
    compressed = request.form["gzip"].lower() == "true"
218
219
220
221
222
    query_fasta = Functions.get_fasta_file(res_dir, "query", is_sorted)
    if query_fasta is not None:
        if is_sorted and not query_fasta.endswith(".sorted"):
            # Do the sort
            Path(lock_query).touch()
223
224
            if not compressed:  # If compressed, it will took a long time, so not wait
                Path(lock_query + ".pending").touch()
225
226
227
228
229
            thread = threading.Timer(1, Functions.sort_fasta, kwargs={
                "job_name": id_res,
                "fasta_file": query_fasta,
                "index_file": os.path.join(res_dir, "query.idx.sorted"),
                "lock_file": lock_query,
230
                "compress": compressed,
231
232
233
                "mailer": mailer
            })
            thread.start()
234
235
236
237
238
239
            if not compressed:
                i = 0
                time.sleep(5)
                while os.path.exists(lock_query) and i < 2:
                    i += 1
                    time.sleep(5)
Floreal Cabanettes's avatar
Floreal Cabanettes committed
240
                os.remove(lock_query + ".pending")
241
242
243
                if os.path.exists(lock_query):
                    return jsonify({"success": True, "status": 1, "status_message": "In progress"})
                return jsonify({"success": True, "status": 2, "status_message": "Done",
244
                                "gzip": False})
245
246
            else:
                return jsonify({"success": True, "status": 1, "status_message": "In progress"})
247
248
249
250
251
        elif is_sorted and os.path.exists(lock_query):
            # Sort is already in progress
            return jsonify({"success": True, "status": 1, "status_message": "In progress"})
        else:
            # No sort to do or sort done
252
253
254
255
256
257
258
            if compressed and not query_fasta.endswith(".gz.fasta"):
                # If compressed file is asked, we must compress it now if not done before...
                Path(lock_query).touch()
                thread = threading.Timer(1, Functions.compress_and_send_mail, kwargs={
                    "job_name": id_res,
                    "fasta_file": query_fasta,
                    "index_file": os.path.join(res_dir, "query.idx.sorted"),
259
                    "lock_file": lock_query,
260
261
262
263
264
                    "compressed": compressed,
                    "mailer": mailer
                })
                thread.start()
                return jsonify({"success": True, "status": 1, "status_message": "In progress"})
265
266
267
268
269
270
271
            return jsonify({"success": True, "status": 2, "status_message": "Done",
                            "gzip": query_fasta.endswith(".gz") or query_fasta.endswith(".gz.sorted")})
    else:
        return jsonify({"success": False,
                        "message": "Unable to get fasta file for query. Please contact us to report the bug"})


272
273
274
@app.route('/fasta-query/<id_res>', defaults={'filename': ""}, methods=['GET'])
@app.route('/fasta-query/<id_res>/<filename>', methods=['GET'])  # Use fake URL in mail to set download file name
def dl_fasta(id_res, filename):
275
276
277
278
279
280
281
282
283
284
285
286
287
288
    res_dir = os.path.join(app_data, id_res)
    lock_query = os.path.join(res_dir, ".query-fasta-build")
    is_sorted = os.path.exists(os.path.join(res_dir, ".sorted"))
    if not os.path.exists(lock_query) or not is_sorted:
        query_fasta = Functions.get_fasta_file(res_dir, "query", is_sorted)
        if query_fasta is not None:
            if query_fasta.endswith(".gz") or query_fasta.endswith(".gz.sorted"):
                content = get_file(query_fasta, True)
                return Response(content, mimetype="application/gzip")
            content = get_file(query_fasta)
            return Response(content, mimetype="text/plain")
    abort(404)


289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
@app.route('/qt-assoc/<id_res>', methods=['GET'])
def qt_assoc(id_res):
    res_dir = os.path.join(app_data, id_res)
    if os.path.exists(res_dir) and os.path.isdir(res_dir):
        paf_file = os.path.join(app_data, id_res, "map.paf")
        idx1 = os.path.join(app_data, id_res, "query.idx")
        idx2 = os.path.join(app_data, id_res, "target.idx")
        try:
            paf = Paf(paf_file, idx1, idx2, False)
            paf.parse_paf(False)
        except FileNotFoundError:
            print("Unable to load data!")
            abort(404)
            return False
        csv_content = paf.build_query_on_target_association_file()
        print(csv_content)
        return Response(csv_content, mimetype="text/plain")
    print("PASS")
    abort(404)


310
311
@app.route("/upload", methods=['POST'])
def upload():
312
313
314
315
316
317
318
319
320
    if "user_tmp_dir" in session and session["user_tmp_dir"] != "":
        folder = session["user_tmp_dir"]
        files = request.files[list(request.files.keys())[0]]

        if files:
            filename = files.filename
            folder_files = os.path.join(app.config["UPLOAD_FOLDER"], folder)
            if not os.path.exists(folder_files):
                os.makedirs(folder_files)
Floreal Cabanettes's avatar
Floreal Cabanettes committed
321
            filename = Functions.get_valid_uploaded_filename(filename, folder_files)
322
323
            mime_type = files.content_type

Floreal Cabanettes's avatar
Floreal Cabanettes committed
324
            if not Functions.allowed_file(files.filename):
325
                result = UploadFile(name=filename, type_f=mime_type, size=0, not_allowed_msg="File type not allowed")
326
                shutil.rmtree(folder_files)
327
328
329
330
331
332
333
334
335
336

            else:
                # save file to disk
                uploaded_file_path = os.path.join(folder_files, filename)
                files.save(uploaded_file_path)

                # get file size after saving
                size = os.path.getsize(uploaded_file_path)

                # return json for js call back
337
                result = UploadFile(name=filename, type_f=mime_type, size=size)
338
339
340
341
342

            return jsonify({"files": [result.get_file()], "success": "OK"})

        return jsonify({"files": [], "success": "404", "message": "No file provided"})
    return jsonify({"files": [], "success": "ERR", "message": "Session not initialized. Please refresh the page."})
343
344


Floréal Cabanettes's avatar
Floréal Cabanettes committed
345
if __name__ == '__main__':
Floreal Cabanettes's avatar
Floreal Cabanettes committed
346
    app.run()