Commit 7f5a4b2f authored by Floreal Cabanettes's avatar Floreal Cabanettes
Browse files

Some code refactoring

parent 001ed535
......@@ -18,8 +18,8 @@ from database import Job
config_reader = AppConfigReader()
UPLOAD_FOLDER = config_reader.get_upload_folder()
APP_DATA = config_reader.get_app_data()
UPLOAD_FOLDER = config_reader.upload_folder
APP_DATA = config_reader.app_data
NOW = time.time()
FAKE = False
......
#!/usr/bin/env python3
import os
import threading
import time
import sys
import psutil
......@@ -20,7 +19,7 @@ from config_reader import AppConfigReader
from lib.job_manager import JobManager
config_reader = AppConfigReader()
NB_RUN = config_reader.get_local_nb_runs()
NB_RUN = config_reader.local_nb_runs
@db_session
......@@ -32,9 +31,6 @@ def start_job(id_job):
job_mng = JobManager(id_job=id_job, email=job.email)
job_mng.set_inputs_from_res_dir()
job_mng.run_job_in_thread()
# thread = threading.Timer(1, job_mng.start_job, kwargs={"batch_system_type": "local"})
# thread.daemon = True # Daemonize thread
# thread.start() # Start the execution
@db_session
......
......@@ -32,7 +32,7 @@ def __sort_key_paf_lines(a):
return -sqrt(pow(x2 - x1, 2) + pow(y2 - y1, 2)) * (int(a[9]) / int(a[10]))
def __get_sorted_paf_lines(lines):
def __get_sorted_paf_lines(lines: iter):
paf_lines = []
for line in lines:
parts = line.strip("\n").split("\t")
......@@ -43,8 +43,7 @@ def __get_sorted_paf_lines(lines):
def init(input_f, output_f):
with open(input_f, "r") as paf_file:
paf_lines = paf_file.readlines()
paf_lines = __get_sorted_paf_lines(paf_lines)
paf_lines = __get_sorted_paf_lines(paf_file)
with open(output_f, "w") as out:
out.write("\n".join(["\t".join(x) for x in paf_lines]))
......
......@@ -11,7 +11,7 @@ class Crons:
def __init__(self, base_dir):
self.base_dir = base_dir
self.my_cron = CronTab(user=getpass.getuser())
self.config_reader = AppConfigReader()
self.config = AppConfigReader()
self.local_scheduler_pid_file = os.path.join(self.base_dir, ".local_scheduler_pid")
def clear(self, kill_scheduler=True):
......@@ -36,8 +36,8 @@ class Crons:
"""
Menage cron is launched at 1h00am each day
"""
menage_hour = self.config_reader.get_cron_menage_hour()
menage_freq = self.config_reader.get_cron_menage_freq()
menage_hour = self.config.cron_menage_hour
menage_freq = self.config.cron_menage_freq
if self.base_dir is not None:
job = self.my_cron.new(sys.executable +
" {0}/bin/clean_jobs.py > {0}/logs/menage.log 2>&1".format(self.base_dir),
......
......@@ -165,8 +165,8 @@ class Functions:
@staticmethod
def send_fasta_ready(mailer, job_name, sample_name, compressed=False):
config_reader = AppConfigReader()
web_url = config_reader.get_web_url()
config = AppConfigReader()
web_url = config.web_url
with open(os.path.join(os.path.dirname(os.path.realpath(__file__)), "mail_templates", "dl_fasta.html")) \
as t_file:
template = Template(t_file.read())
......
......@@ -24,20 +24,11 @@ class JobManager:
self.email = email
self.query = query
self.target = target
config_reader = AppConfigReader()
self.error = ""
# Get configs:
self.batch_system_type = config_reader.get_batch_system_type()
self.minimap2 = config_reader.get_minimap2_exec()
self.threads = config_reader.get_nb_threads()
self.app_data = config_reader.get_app_data()
self.web_url = config_reader.get_web_url()
self.mail_status = config_reader.get_mail_status_sender()
self.mail_reply = config_reader.get_mail_reply()
self.mail_org = config_reader.get_mail_org()
self.do_send = config_reader.get_send_mail_status()
self.config = AppConfigReader()
# Outputs:
self.output_dir = os.path.join(self.app_data, id_job)
self.output_dir = os.path.join(self.config.app_data, id_job)
self.paf = os.path.join(self.output_dir, "map.paf")
self.paf_raw = os.path.join(self.output_dir, "map_raw.paf")
self.idx_q = os.path.join(self.output_dir, "query.idx")
......@@ -46,7 +37,7 @@ class JobManager:
self.mailer = mailer
def set_inputs_from_res_dir(self):
res_dir = os.path.join(self.app_data, self.id_job)
res_dir = os.path.join(self.config.app_data, self.id_job)
query_file = os.path.join(res_dir, ".query")
if os.path.exists(query_file):
with open(query_file) as q_f:
......@@ -75,7 +66,7 @@ class JobManager:
return "fail"
def check_job_success(self):
if self.batch_system_type == "local":
if self.config.batch_system_type == "local":
return self.__check_job_success_local()
def get_mail_content(self, status):
......@@ -83,7 +74,7 @@ class JobManager:
if status == "success":
message += "Your job %s was completed successfully!\n\n" % self.id_job
message += str("Your job {0} is finished. You can see the results by clicking on the link below:\n"
"{1}/result/{0}\n\n").format(self.id_job, self.web_url)
"{1}/result/{0}\n\n").format(self.id_job, self.config.web_url)
else:
message += "Your job %s has failed!\n\n" % self.id_job
if self.error != "":
......@@ -104,7 +95,7 @@ class JobManager:
with open(os.path.join(os.path.dirname(os.path.realpath(__file__)), "mail_templates", "job_notification.html"))\
as t_file:
template = Template(t_file.read())
return template.render(job_name=self.id_job, status=status, url_base=self.web_url,
return template.render(job_name=self.id_job, status=status, url_base=self.config.web_url,
query_name=self.query.get_name() if self.query is not None else "",
target_name=self.target.get_name(),
error=self.error)
......@@ -142,7 +133,7 @@ class JobManager:
@db_session
def __launch_local(self):
cmd = ["run_minimap2.sh", self.minimap2, self.threads, self.target.get_path(),
cmd = ["run_minimap2.sh", self.config.minimap2_exec, self.config.nb_threads, self.target.get_path(),
self.query.get_path() if self.query is not None else "NONE", self.paf, self.paf_raw]
with open(self.logs, "w") as logs:
p = subprocess.Popen(cmd, stdout=logs, stderr=logs)
......@@ -236,11 +227,11 @@ class JobManager:
Send mail using POST url (we have no access to mailer)
"""
key = Functions.random_string(15)
key_file = os.path.join(self.app_data, self.id_job, ".key")
key_file = os.path.join(self.config.app_data, self.id_job, ".key")
with open(key_file, "w") as k_f:
k_f.write(key)
data = parse.urlencode({"key": key}).encode()
req = request.Request(self.web_url + "/send-mail/" + self.id_job, data=data)
req = request.Request(self.config.web_url + "/send-mail/" + self.id_job, data=data)
resp = request.urlopen(req)
if resp.getcode() != 200:
print("Job %s: Send mail failed!" % self.id_job)
......@@ -271,9 +262,8 @@ class JobManager:
job = Job.get(id_job=self.id_job)
job.status = "success"
db.commit()
if self.do_send:
if self.config.send_mail_status:
self.send_mail_post()
#self.send_mail(job.status)
@db_session
def start_job(self):
......@@ -288,7 +278,7 @@ class JobManager:
job.status = "fail"
job.error = "<p>Error while getting input files. Please contact the support to report the bug.</p>"
db.commit()
if self.do_send:
if self.config.send_mail_status:
self.send_mail()
except Exception:
......@@ -297,7 +287,7 @@ class JobManager:
job.status = "fail"
job.error = "<p>An unexpected error has occurred. Please contact the support to report the bug.</p>"
db.commit()
if self.do_send:
if self.config.send_mail_status:
self.send_mail()
......@@ -308,7 +298,7 @@ class JobManager:
print("Old job found without result dir existing: delete it from BDD!")
j1.delete()
if self.target is not None:
job = Job(id_job=self.id_job, email=self.email, batch_type=self.batch_system_type,
job = Job(id_job=self.id_job, email=self.email, batch_type=self.config.batch_system_type,
date_created=datetime.datetime.now())
db.commit()
if not os.path.exists(self.output_dir):
......@@ -316,7 +306,7 @@ class JobManager:
thread = threading.Timer(1, self.start_job)
thread.start()
else:
job = Job(id_job=self.id_job, email=self.email, batch_type=self.batch_system_type,
job = Job(id_job=self.id_job, email=self.email, batch_type=self.config.batch_system_type,
date_created=datetime.datetime.now(), status="fail")
db.commit()
......
......@@ -7,20 +7,21 @@ class Mailer:
def __init__(self, app):
self.app = app
self.mail = Mail(app)
config_reader = AppConfigReader()
self.mail_status = config_reader.get_mail_status_sender()
self.mail_reply = config_reader.get_mail_reply()
self.mail_org = config_reader.get_mail_org()
self.disable = config_reader.get_disable_mail()
self.config = AppConfigReader()
# self.mail_status = config_reader.get_mail_status_sender()
# self.mail_reply = config_reader.get_mail_reply()
# self.mail_org = config_reader.get_mail_org()
# self.disable = config_reader.get_disable_mail()
def __send_async_email(self, msg):
with self.app.app_context():
self.mail.send(msg)
def send_mail(self, recipients: list, subject: str, message: str, message_html: str=None):
sender = (self.mail_org, self.mail_status) if self.mail_org is not None else self.mail_status
reply = self.mail_reply
if not self.disable:
sender = (self.config.mail_org, self.config.mail_status_sender) if self.config.mail_org is not None else \
self.config.mail_status_sender
reply = self.config.mail_reply
if not self.config.disable_mail:
msg = Message(
subject= subject,
recipients=recipients,
......@@ -36,7 +37,7 @@ class Mailer:
"################\n\n"
"!!! SEND MAILS DISABLED BY CONFIGURATION !!!\n\n"
"(This might be disabled in production)\n\n")
print("Sender: %s <%s>\n" % sender)
print("Sender: %s <%s>\n" % sender if isinstance(tuple, sender) else ("None", sender))
print("Reply to: %s\n" % reply)
print("Recipients: %s\n" % ", ".join(recipients))
print("Subject: %s\n" % subject)
......
......@@ -141,10 +141,12 @@ class Paf:
try:
with open(self.paf, "r") as paf_file:
paf_lines = paf_file.readlines()
if len(paf_lines) > self.max_nb_lines:
self.sampled = True
for line in paf_lines[:self.max_nb_lines]:
nb_lines = 0
for line in paf_file:
nb_lines += 1
if nb_lines > self.max_nb_lines:
self.sampled = True
break
parts = line.strip("\n").split("\t")
v1 = parts[0]
v6 = parts[5]
......
......@@ -2,16 +2,21 @@ import os
import re
import inspect
from configparser import RawConfigParser, NoOptionError, NoSectionError
from lib.decorators import Singleton
class AppConfigReader(object):
@Singleton
class AppConfigReader:
"""
Store all configs
"""
CONFIG_FILE_PATH = "../application.properties"
def __init__(self):
"""
All "get_*" functions results are stored in the "self.*" corresponding attribute
Example: results of the get_upload_folder function is stored in self.upload_folder
"""
config_file = os.path.join(os.path.dirname(inspect.getfile(self.__class__)), self.CONFIG_FILE_PATH)
if not os.path.exists(config_file):
......@@ -19,6 +24,10 @@ class AppConfigReader(object):
"properties are correct for you!")
self.reader = RawConfigParser()
self.reader.read(config_file)
for attr in dir(self):
attr_o = getattr(self, attr)
if attr.startswith("get_") and callable(attr_o):
setattr(self, attr[4:], attr_o())
@staticmethod
def replace_vars(path):
......
......@@ -2,8 +2,8 @@ import datetime
from config_reader import AppConfigReader
from pony.orm import Database, Required, Optional
config_reader = AppConfigReader()
file_path = config_reader.get_database()
config = AppConfigReader()
file_path = config.database
db = Database()
db.bind(provider='sqlite', filename=file_path, create_db=True)
......
......@@ -29,8 +29,8 @@ sqlite_file = os.path.join(app_folder, "database.sqlite")
# Init config reader:
config_reader = AppConfigReader()
UPLOAD_FOLDER = config_reader.get_upload_folder()
APP_DATA = config_reader.get_app_data()
UPLOAD_FOLDER = config_reader.upload_folder
APP_DATA = config_reader.app_data
app_title = "D-GENIES - Dotplot for Genomes Interactive, E-connected and Speedy"
......@@ -43,7 +43,7 @@ app.config['SECRET_KEY'] = 'dsqdsq-255sdA-fHfg52-25Asd5'
mailer = Mailer(app)
# Folder containing data:
app_data = config_reader.get_app_data()
app_data = config_reader.app_data
# Crons:
crons = Crons(app_folder)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment