main.py 12.2 KB
Newer Older
1
2
#!/usr/bin/env python3

Floreal Cabanettes's avatar
Floreal Cabanettes committed
3
import os
4
5
import time
import datetime
6
import shutil
7
import re
8
import threading
Floreal Cabanettes's avatar
Floreal Cabanettes committed
9
from flask import Flask, render_template, request, url_for, jsonify, session, Response, abort
10
from pathlib import Path
11
from lib.paf import Paf
12
from config_reader import AppConfigReader
13
from lib.job_manager import JobManager
14
from lib.functions import Functions, ALLOWED_EXTENSIONS
15
from lib.upload_file import UploadFile
16
from lib.Fasta import Fasta
Floreal Cabanettes's avatar
Floreal Cabanettes committed
17
from lib.mailer import Mailer
18
19

import sys
20

21
22
23
24
25
26
app_folder = os.path.dirname(os.path.realpath(__file__))
sys.path.insert(0, app_folder)
os.environ["PATH"] = os.path.join(app_folder, "bin") + ":" + os.environ["PATH"]

sqlite_file = os.path.join(app_folder, "database.sqlite")

27
28
29
30

# Init config reader:
config_reader = AppConfigReader()

31
UPLOAD_FOLDER = config_reader.get_upload_folder()
Floreal Cabanettes's avatar
Floreal Cabanettes committed
32
APP_DATA = config_reader.get_app_data()
33

Floreal Cabanettes's avatar
Floreal Cabanettes committed
34
app_title = "D-GENIES - Dotplot for Genomes Interactive, E-connected and Speedy"
Floreal Cabanettes's avatar
Floreal Cabanettes committed
35

36
# Init Flask:
37
app = Flask(__name__, static_url_path='/static')
38
app.config['UPLOAD_FOLDER'] = UPLOAD_FOLDER
39
app.config['SECRET_KEY'] = 'dsqdsq-255sdA-fHfg52-25Asd5'
40

41
# Init mail:
Floreal Cabanettes's avatar
Floreal Cabanettes committed
42
mailer = Mailer(app)
43

44
# Folder containing data:
45
46
47
app_data = config_reader.get_app_data()


Floreal Cabanettes's avatar
Floreal Cabanettes committed
48
49
50
51
52
53
@app.context_processor
def get_launched_results():
    cookie = request.cookies.get("results")
    return {"results": cookie.split("|") if cookie is not None else set()}


54
55
56
# Main
@app.route("/", methods=['GET'])
def main():
Floreal Cabanettes's avatar
Floreal Cabanettes committed
57
58
59
60
61
    return render_template("index.html", title=app_title, menu="index")


@app.route("/run", methods=['GET'])
def run():
Floreal Cabanettes's avatar
Floreal Cabanettes committed
62
    session["user_tmp_dir"] = Functions.random_string(5) + "_" + \
63
                              datetime.datetime.fromtimestamp(time.time()).strftime('%Y%m%d%H%M%S')
Floreal Cabanettes's avatar
Floreal Cabanettes committed
64
    id_job = Functions.random_string(5) + "_" + datetime.datetime.fromtimestamp(time.time()).strftime('%Y%m%d%H%M%S')
65
66
67
68
69
    if "id_job" in request.args:
        id_job = request.args["id_job"]
    email = ""
    if "email" in request.args:
        email = request.args["email"]
Floreal Cabanettes's avatar
Floreal Cabanettes committed
70
    return render_template("run.html", title=app_title, id_job=id_job, email=email,
71
                           menu="run", allowed_ext=ALLOWED_EXTENSIONS)
72
73
74
75
76
77
78


# Launch analysis
@app.route("/launch_analysis", methods=['POST'])
def launch_analysis():
    id_job = request.form["id_job"]
    email = request.form["email"]
79
80
81
82
    file_query = request.form["query"]
    file_query_type = request.form["query_type"]
    file_target = request.form["target"]
    file_target_type = request.form["target_type"]
83
84
85

    # Check form:
    form_pass = True
86
    errors = []
87
    if id_job == "":
88
        errors.append("Id of job not given")
89
90
91
        form_pass = False

    if email == "":
92
        errors.append("Email not given")
93
        form_pass = False
94
95
    if file_target == "":
        errors.append("No target fasta selected")
96
97
98
99
        form_pass = False

    # Form pass
    if form_pass:
100
        # Get final job id:
101
        id_job = re.sub('[^A-Za-z0-9_\-]+', '', id_job.replace(" ", "_"))
102
103
104
105
106
107
108
109
        id_job_orig = id_job
        while os.path.exists(os.path.join(app_data, id_job)):
            id_job = id_job_orig + "_2"

        folder_files = os.path.join(app_data, id_job)
        os.makedirs(folder_files)

        # Save files:
110
111
112
113
114
115
116
117
118
119
        query = None
        if file_query != "":
            query_name = os.path.splitext(file_query.replace(".gz", ""))[0] if file_query_type == "local" else None
            query_path = os.path.join(app.config["UPLOAD_FOLDER"], session["user_tmp_dir"], file_query) \
                if file_query_type == "local" else file_query
            query = Fasta(name=query_name, path=query_path, type_f=file_query_type)
        target_name = os.path.splitext(file_target.replace(".gz", ""))[0] if file_target_type == "local" else None
        target_path = os.path.join(app.config["UPLOAD_FOLDER"], session["user_tmp_dir"], file_target) \
            if file_target_type == "local" else file_target
        target = Fasta(name=target_name, path=target_path, type_f=file_target_type)
120
121

        # Launch job:
Floreal Cabanettes's avatar
Floreal Cabanettes committed
122
        job = JobManager(id_job, email, query, target, mailer)
123
124
        job.launch()
        return jsonify({"success": True, "redirect": url_for(".status", id_job=id_job)})
125
    else:
126
        return jsonify({"success": False, "errors": errors})
127
128
129
130
131
132


# Status of a job
@app.route('/status/<id_job>', methods=['GET'])
def status(id_job):
    job = JobManager(id_job)
133
    j_status, error = job.status()
134
135
    return render_template("status.html", title=app_title, status=j_status, error=error.replace("#ID#", ""),
                           id_job=id_job, menu="results")
136
137


Floreal Cabanettes's avatar
Floreal Cabanettes committed
138
# Results path
139
@app.route("/result/<id_res>", methods=['GET'])
140
def result(id_res):
Floreal Cabanettes's avatar
Floreal Cabanettes committed
141
142
143
144
145
146
147
148
    my_render = render_template("results.html", title=app_title, id=id_res, menu="results", current_result=id_res)
    response = app.make_response(my_render)
    cookie = request.cookies.get("results")
    cookie = cookie.split("|") if cookie is not None else []
    if id_res not in cookie:
        cookie.insert(0, id_res)
    response.set_cookie(key="results", value="|".join(cookie), path="/")
    return response
149
150


151
def get_file(file, gzip=False):  # pragma: no cover
Floreal Cabanettes's avatar
Floreal Cabanettes committed
152
153
154
155
156
157
    try:
        # Figure out how flask returns static files
        # Tried:
        # - render_template
        # - send_file
        # This should not be so non-obvious
158
        return open(file, "rb" if gzip else "r").read()
Floreal Cabanettes's avatar
Floreal Cabanettes committed
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
    except IOError as exc:
        return str(exc)


@app.route("/paf/<id_res>", methods=['GET'])
def download_paf(id_res):
    map_file = os.path.join(APP_DATA, id_res, "map.paf.sorted")
    if not os.path.exists(map_file):
        map_file = os.path.join(APP_DATA, id_res, "map.paf")
    if not os.path.exists(map_file):
        abort(404)
    content = get_file(map_file)
    return Response(content, mimetype="text/plain")


Floreal Cabanettes's avatar
Floreal Cabanettes committed
174
# Get graph (ajax request)
175
176
177
@app.route('/get_graph', methods=['POST'])
def get_graph():
    id_f = request.form["id"]
178
    paf = os.path.join(app_data, id_f, "map.paf")
179
180
    idx1 = os.path.join(app_data, id_f, "query.idx")
    idx2 = os.path.join(app_data, id_f, "target.idx")
181

182
    paf = Paf(paf, idx1, idx2)
183

184
185
    if paf.parsed:
        res = paf.get_d3js_data()
186
        res["success"] = True
187
188
189
190
191
192
        return jsonify(res)
    return jsonify({"success": False, "message": paf.error})


@app.route('/sort/<id_res>', methods=['POST'])
def sort_graph(id_res):
193
194
195
196
197
198
199
200
201
202
203
204
    if not os.path.exists(os.path.join(APP_DATA, id_res, ".all-vs-all")):
        paf = os.path.join(app_data, id_res, "map.paf")
        idx1 = os.path.join(app_data, id_res, "query.idx")
        idx2 = os.path.join(app_data, id_res, "target.idx")
        paf = Paf(paf, idx1, idx2, False)
        paf.sort()
        if paf.parsed:
            res = paf.get_d3js_data()
            res["success"] = True
            return jsonify(res)
        return jsonify({"success": False, "message": paf.error})
    return jsonify({"success": False, "message": "Sort is not available for All-vs-All mode"})
Floreal Cabanettes's avatar
Floreal Cabanettes committed
205

Floréal Cabanettes's avatar
Floréal Cabanettes committed
206

207
208
209
210
211
@app.route('/get-fasta-query/<id_res>', methods=['POST'])
def build_fasta(id_res):
    res_dir = os.path.join(app_data, id_res)
    lock_query = os.path.join(res_dir, ".query-fasta-build")
    is_sorted = os.path.exists(os.path.join(res_dir, ".sorted"))
212
    compressed = request.form["gzip"].lower() == "true"
213
214
215
216
217
    query_fasta = Functions.get_fasta_file(res_dir, "query", is_sorted)
    if query_fasta is not None:
        if is_sorted and not query_fasta.endswith(".sorted"):
            # Do the sort
            Path(lock_query).touch()
218
219
            if not compressed:  # If compressed, it will took a long time, so not wait
                Path(lock_query + ".pending").touch()
220
221
222
223
224
            thread = threading.Timer(1, Functions.sort_fasta, kwargs={
                "job_name": id_res,
                "fasta_file": query_fasta,
                "index_file": os.path.join(res_dir, "query.idx.sorted"),
                "lock_file": lock_query,
225
                "compress": compressed,
226
227
228
                "mailer": mailer
            })
            thread.start()
229
230
231
232
233
234
            if not compressed:
                i = 0
                time.sleep(5)
                while os.path.exists(lock_query) and i < 2:
                    i += 1
                    time.sleep(5)
Floreal Cabanettes's avatar
Floreal Cabanettes committed
235
                os.remove(lock_query + ".pending")
236
237
238
239
240
241
                if os.path.exists(lock_query):
                    return jsonify({"success": True, "status": 1, "status_message": "In progress"})
                return jsonify({"success": True, "status": 2, "status_message": "Done",
                                "gzip": query_fasta.endswith(".gz") or query_fasta.endswith(".gz.sorted")})
            else:
                return jsonify({"success": True, "status": 1, "status_message": "In progress"})
242
243
244
245
246
        elif is_sorted and os.path.exists(lock_query):
            # Sort is already in progress
            return jsonify({"success": True, "status": 1, "status_message": "In progress"})
        else:
            # No sort to do or sort done
247
248
249
250
251
252
253
            if compressed and not query_fasta.endswith(".gz.fasta"):
                # If compressed file is asked, we must compress it now if not done before...
                Path(lock_query).touch()
                thread = threading.Timer(1, Functions.compress_and_send_mail, kwargs={
                    "job_name": id_res,
                    "fasta_file": query_fasta,
                    "index_file": os.path.join(res_dir, "query.idx.sorted"),
254
                    "lock_file": lock_query,
255
256
257
258
259
                    "compressed": compressed,
                    "mailer": mailer
                })
                thread.start()
                return jsonify({"success": True, "status": 1, "status_message": "In progress"})
260
261
262
263
264
265
266
            return jsonify({"success": True, "status": 2, "status_message": "Done",
                            "gzip": query_fasta.endswith(".gz") or query_fasta.endswith(".gz.sorted")})
    else:
        return jsonify({"success": False,
                        "message": "Unable to get fasta file for query. Please contact us to report the bug"})


267
268
269
@app.route('/fasta-query/<id_res>', defaults={'filename': ""}, methods=['GET'])
@app.route('/fasta-query/<id_res>/<filename>', methods=['GET'])  # Use fake URL in mail to set download file name
def dl_fasta(id_res, filename):
270
271
272
273
274
275
276
277
278
279
280
281
282
283
    res_dir = os.path.join(app_data, id_res)
    lock_query = os.path.join(res_dir, ".query-fasta-build")
    is_sorted = os.path.exists(os.path.join(res_dir, ".sorted"))
    if not os.path.exists(lock_query) or not is_sorted:
        query_fasta = Functions.get_fasta_file(res_dir, "query", is_sorted)
        if query_fasta is not None:
            if query_fasta.endswith(".gz") or query_fasta.endswith(".gz.sorted"):
                content = get_file(query_fasta, True)
                return Response(content, mimetype="application/gzip")
            content = get_file(query_fasta)
            return Response(content, mimetype="text/plain")
    abort(404)


284
285
@app.route("/upload", methods=['POST'])
def upload():
286
287
288
289
290
291
292
293
294
    if "user_tmp_dir" in session and session["user_tmp_dir"] != "":
        folder = session["user_tmp_dir"]
        files = request.files[list(request.files.keys())[0]]

        if files:
            filename = files.filename
            folder_files = os.path.join(app.config["UPLOAD_FOLDER"], folder)
            if not os.path.exists(folder_files):
                os.makedirs(folder_files)
Floreal Cabanettes's avatar
Floreal Cabanettes committed
295
            filename = Functions.get_valid_uploaded_filename(filename, folder_files)
296
297
            mime_type = files.content_type

Floreal Cabanettes's avatar
Floreal Cabanettes committed
298
            if not Functions.allowed_file(files.filename):
299
                result = UploadFile(name=filename, type_f=mime_type, size=0, not_allowed_msg="File type not allowed")
300
                shutil.rmtree(folder_files)
301
302
303
304
305
306
307
308
309
310

            else:
                # save file to disk
                uploaded_file_path = os.path.join(folder_files, filename)
                files.save(uploaded_file_path)

                # get file size after saving
                size = os.path.getsize(uploaded_file_path)

                # return json for js call back
311
                result = UploadFile(name=filename, type_f=mime_type, size=size)
312
313
314
315
316

            return jsonify({"files": [result.get_file()], "success": "OK"})

        return jsonify({"files": [], "success": "404", "message": "No file provided"})
    return jsonify({"files": [], "success": "ERR", "message": "Session not initialized. Please refresh the page."})
317
318


Floréal Cabanettes's avatar
Floréal Cabanettes committed
319
if __name__ == '__main__':
Floreal Cabanettes's avatar
Floreal Cabanettes committed
320
    app.run()