Commit b4f720c4 authored by Floreal Cabanettes's avatar Floreal Cabanettes
Browse files

Add launch of jobs for local

parent f050e841
*/__pycache__
.idea
srv/database.sqlite
[global]
upload_folder = /home/fcabanettes/public_html/tmp
data_folder = /home/fcabanettes/public_html/test
# batch system type: local, sge, slurm
batch_system_type = local
threads = 4
[softwares]
minimap2 = /home/fcabanettes/minimap2/minimap2
#!/bin/bash
minimap_exec=$1
samtools_exec=$2
nb_threads=$3
fasta_t=$4
fasta_q=$5
${minimap_exec} -t ${nb_threads} ${fasta_t} ${fasta_q}
${samtools_exec} faidx ${fasta_t}
${samtools_exec} faidx ${fasta_q}
import os
import random
import string
from pony.orm import Database, Required, commit
ALLOWED_EXTENSIONS = {'fa', 'fasta', 'fa.gz', "fasta.gz"}
def allowed_file(filename):
return '.' in filename and \
......@@ -16,12 +21,26 @@ def random_string(s_len):
return ''.join([random.choice(string.ascii_letters + string.digits) for n in range(s_len)])
def get_valid_uploaded_filename(filename):
file_query_s = os.path.join(app.config["UPLOAD_FOLDER"], filename)
def get_valid_uploaded_filename(filename, folder):
file_query_s = os.path.join(folder, filename)
i = 2
filename_orig = filename
while os.path.exists(file_query_s):
filename = str(i) + "_" + filename_orig
file_query_s = os.path.join(app.config["UPLOAD_FOLDER"], filename)
file_query_s = os.path.join(folder, filename)
i += 1
return filename
def generate_database(file_path):
db = Database()
db.bind(provider='sqlite', filename=file_path, create_db=True)
class Job(db.Entity):
id_job = Required(str)
email = Required(str)
id_process = Required(int)
db.generate_mapping(create_tables=True)
commit()
return Job, db
import os
import subprocess
from config_reader import AppConfigReader
from pony.orm import db_session
import threading
class JobManager:
def __init__(self, id_job, email, fasta_q, fasta_t, db, job_db):
self.id_job = id_job
self.email = email
self.fasta_q = fasta_q
self.fasta_t = fasta_t
config_reader = AppConfigReader()
# Get configs:
self.batch_system_type = config_reader.get_batch_system_type()
self.minimap2 = config_reader.get_minimap2_exec()
self.samtools = config_reader.get_samtools_exec()
self.threads = config_reader.get_nb_threads()
self.app_data = config_reader.get_app_data()
# Outputs:
self.output_dir = os.path.join(self.app_data, id_job)
self.paf_raw = os.path.join(self.output_dir, "map_raw.paf")
self.idx_q = os.path.join(self.output_dir, "query.idx")
self.idx_t = os.path.join(self.output_dir, "target.idx")
self.logs = os.path.join(self.output_dir, "logs.err")
self.db = db
self.job_db = job_db
@db_session
def __launch_local(self):
cmd = ["run_minimap2.sh", self.minimap2, self.samtools, self.threads, self.fasta_t, self.fasta_q]
with open(self.paf_raw, "w") as paf, open(self.logs, "w") as logs:
p = subprocess.Popen(cmd)
pid = p.pid
job = self.job_db(id_job=self.id_job, email=self.email, id_process=pid)
self.db.commit()
p.communicate()
def launch(self):
if not os.path.exists(self.output_dir):
os.mkdir(self.output_dir)
if self.batch_system_type == "local":
thread = threading.Timer(1, self.__launch_local)
thread.start()
../bin/
\ No newline at end of file
......@@ -19,10 +19,34 @@ class AppConfigReader(object):
try:
return self.reader.get("global", "upload_folder")
except NoOptionError:
return None
raise Exception("No upload folder found in application.properties (global section)")
def get_app_data(self):
try:
return self.reader.get("global", "data_folder")
except NoOptionError:
return None
raise Exception("No data folder found in application.properties (global section)")
def get_batch_system_type(self):
try:
return self.reader.get("global", "batch_system_type")
except NoOptionError:
return "local"
def get_nb_threads(self):
try:
return self.reader.get("global", "threads")
except NoOptionError:
return "4"
def get_minimap2_exec(self):
try:
return self.reader.get("softwares", "minimap2")
except NoOptionError:
return "minimap2"
def get_samtools_exec(self):
try:
return self.reader.get("softwares", "samtools")
except NoOptionError:
return "samtools"
\ No newline at end of file
......@@ -4,17 +4,30 @@ import os
import json
import time
import datetime
from flask import Flask, render_template, request, redirect, flash, session, url_for
from flask import Flask, render_template, request, redirect, flash, url_for
from werkzeug.utils import secure_filename
from lib.parse_paf import parse_paf
from config_reader import AppConfigReader
from lib.job_manager import JobManager
from lib.functions import *
# try:
# import _preamble
# except ImportError:
# pass
import sys
app_folder = os.path.dirname(os.path.realpath(__file__))
sys.path.insert(0, app_folder)
os.environ["PATH"] = os.path.join(app_folder, "bin") + ":" + os.environ["PATH"]
sqlite_file = os.path.join(app_folder, "database.sqlite")
job_db, db = generate_database(sqlite_file)
# Init config reader:
config_reader = AppConfigReader()
UPLOAD_FOLDER = config_reader.get_upload_folder()
ALLOWED_EXTENSIONS = {'fa', 'fasta', 'fa.gz', "fasta.gz"}
app_title = "ALGECO - A Live GEnome COmparator"
......@@ -74,15 +87,21 @@ def launch_analysis():
form_pass = False
if form_pass:
# Save files:
filename_query = get_valid_uploaded_filename(secure_filename(file_query.filename))
file_query.save(os.path.join(app.config["UPLOAD_FOLDER"], filename_query))
filename_target = get_valid_uploaded_filename(secure_filename(file_target.filename))
file_target.save(os.path.join(app.config["UPLOAD_FOLDER"], filename_target))
filename_query = get_valid_uploaded_filename(secure_filename(file_query.filename), app.config["UPLOAD_FOLDER"])
query_path = os.path.join(app.config["UPLOAD_FOLDER"], filename_query)
file_query.save(query_path)
filename_target = get_valid_uploaded_filename(secure_filename(file_target.filename), app.config["UPLOAD_FOLDER"])
target_path = os.path.join(app.config["UPLOAD_FOLDER"], filename_target)
file_target.save(target_path)
# Get final job id:
id_job_orig = id_job
while os.path.exists(os.path.join(app_data, id_job)):
id_job = id_job_orig + "_2"
# Launch job:
job = JobManager(id_job, email, query_path, target_path, db, job_db)
job.launch()
else:
return redirect(url_for(".main", id_job=id_job, email=email))
else:
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment