Some code refactoring

bin/clean_jobs.py

@@ -18,8 +18,8 @@ from database import Job
 
 config_reader = AppConfigReader()
 
-UPLOAD_FOLDER = config_reader.get_upload_folder()
-APP_DATA = config_reader.get_app_data()
+UPLOAD_FOLDER = config_reader.upload_folder
+APP_DATA = config_reader.app_data
 NOW = time.time()
 FAKE = False
 

bin/local_scheduler.py

 #!/usr/bin/env python3
 
 import os
-import threading
 import time
 import sys
 import psutil
@@ -20,7 +19,7 @@ from config_reader import AppConfigReader
 from lib.job_manager import JobManager
 
 config_reader = AppConfigReader()
-NB_RUN = config_reader.get_local_nb_runs()
+NB_RUN = config_reader.local_nb_runs
 
 
 @db_session
@@ -32,9 +31,6 @@ def start_job(id_job):
     job_mng = JobManager(id_job=id_job, email=job.email)
     job_mng.set_inputs_from_res_dir()
     job_mng.run_job_in_thread()
-    # thread = threading.Timer(1, job_mng.start_job, kwargs={"batch_system_type": "local"})
-    # thread.daemon = True  # Daemonize thread
-    # thread.start()  # Start the execution
 
 
 @db_session

bin/sort_paf.py

@@ -32,7 +32,7 @@ def __sort_key_paf_lines(a):
     return -sqrt(pow(x2 - x1, 2) + pow(y2 - y1, 2)) * (int(a[9]) / int(a[10]))
 
 
-def __get_sorted_paf_lines(lines):
+def __get_sorted_paf_lines(lines: iter):
     paf_lines = []
     for line in lines:
         parts = line.strip("\n").split("\t")
@@ -43,8 +43,7 @@ def __get_sorted_paf_lines(lines):
 
 def init(input_f, output_f):
     with open(input_f, "r") as paf_file:
-        paf_lines = paf_file.readlines()
-        paf_lines = __get_sorted_paf_lines(paf_lines)
+        paf_lines = __get_sorted_paf_lines(paf_file)
         with open(output_f, "w") as out:
             out.write("\n".join(["\t".join(x) for x in paf_lines]))
 

lib/crons.py

@@ -11,7 +11,7 @@ class Crons:
     def __init__(self, base_dir):
         self.base_dir = base_dir
         self.my_cron = CronTab(user=getpass.getuser())
-        self.config_reader = AppConfigReader()
+        self.config = AppConfigReader()
         self.local_scheduler_pid_file = os.path.join(self.base_dir, ".local_scheduler_pid")
 
     def clear(self, kill_scheduler=True):
@@ -36,8 +36,8 @@ class Crons:
         """
         Menage cron is launched at 1h00am each day
         """
-        menage_hour = self.config_reader.get_cron_menage_hour()
-        menage_freq = self.config_reader.get_cron_menage_freq()
+        menage_hour = self.config.cron_menage_hour
+        menage_freq = self.config.cron_menage_freq
         if self.base_dir is not None:
             job = self.my_cron.new(sys.executable +
                                    " {0}/bin/clean_jobs.py > {0}/logs/menage.log 2>&1".format(self.base_dir),

lib/functions.py

@@ -165,8 +165,8 @@ class Functions:
 
     @staticmethod
     def send_fasta_ready(mailer, job_name, sample_name, compressed=False):
-        config_reader = AppConfigReader()
-        web_url = config_reader.get_web_url()
+        config = AppConfigReader()
+        web_url = config.web_url
         with open(os.path.join(os.path.dirname(os.path.realpath(__file__)), "mail_templates", "dl_fasta.html")) \
                 as t_file:
             template = Template(t_file.read())

lib/job_manager.py

@@ -24,20 +24,11 @@ class JobManager:
         self.email = email
         self.query = query
         self.target = target
-        config_reader = AppConfigReader()
         self.error = ""
         # Get configs:
-        self.batch_system_type = config_reader.get_batch_system_type()
-        self.minimap2 = config_reader.get_minimap2_exec()
-        self.threads = config_reader.get_nb_threads()
-        self.app_data = config_reader.get_app_data()
-        self.web_url = config_reader.get_web_url()
-        self.mail_status = config_reader.get_mail_status_sender()
-        self.mail_reply = config_reader.get_mail_reply()
-        self.mail_org = config_reader.get_mail_org()
-        self.do_send = config_reader.get_send_mail_status()
+        self.config = AppConfigReader()
         # Outputs:
-        self.output_dir = os.path.join(self.app_data, id_job)
+        self.output_dir = os.path.join(self.config.app_data, id_job)
         self.paf = os.path.join(self.output_dir, "map.paf")
         self.paf_raw = os.path.join(self.output_dir, "map_raw.paf")
         self.idx_q = os.path.join(self.output_dir, "query.idx")
@@ -46,7 +37,7 @@ class JobManager:
         self.mailer = mailer
 
     def set_inputs_from_res_dir(self):
-        res_dir = os.path.join(self.app_data, self.id_job)
+        res_dir = os.path.join(self.config.app_data, self.id_job)
         query_file = os.path.join(res_dir, ".query")
         if os.path.exists(query_file):
             with open(query_file) as q_f:
@@ -75,7 +66,7 @@ class JobManager:
         return "fail"
 
     def check_job_success(self):
-        if self.batch_system_type == "local":
+        if self.config.batch_system_type == "local":
             return self.__check_job_success_local()
 
     def get_mail_content(self, status):
@@ -83,7 +74,7 @@ class JobManager:
         if status == "success":
             message += "Your job %s was completed successfully!\n\n" % self.id_job
             message += str("Your job {0} is finished. You can see  the results by clicking on the link below:\n"
-                           "{1}/result/{0}\n\n").format(self.id_job, self.web_url)
+                           "{1}/result/{0}\n\n").format(self.id_job, self.config.web_url)
         else:
             message += "Your job %s has failed!\n\n" % self.id_job
             if self.error != "":
@@ -104,7 +95,7 @@ class JobManager:
         with open(os.path.join(os.path.dirname(os.path.realpath(__file__)), "mail_templates", "job_notification.html"))\
                 as t_file:
             template = Template(t_file.read())
-            return template.render(job_name=self.id_job, status=status, url_base=self.web_url,
+            return template.render(job_name=self.id_job, status=status, url_base=self.config.web_url,
                                    query_name=self.query.get_name() if self.query is not None else "",
                                    target_name=self.target.get_name(),
                                    error=self.error)
@@ -142,7 +133,7 @@ class JobManager:
 
     @db_session
     def __launch_local(self):
-        cmd = ["run_minimap2.sh", self.minimap2, self.threads, self.target.get_path(),
+        cmd = ["run_minimap2.sh", self.config.minimap2_exec, self.config.nb_threads, self.target.get_path(),
                self.query.get_path() if self.query is not None else "NONE", self.paf, self.paf_raw]
         with open(self.logs, "w") as logs:
             p = subprocess.Popen(cmd, stdout=logs, stderr=logs)
@@ -236,11 +227,11 @@ class JobManager:
         Send mail using POST url (we have no access to mailer)
         """
         key = Functions.random_string(15)
-        key_file = os.path.join(self.app_data, self.id_job, ".key")
+        key_file = os.path.join(self.config.app_data, self.id_job, ".key")
         with open(key_file, "w") as k_f:
             k_f.write(key)
         data = parse.urlencode({"key": key}).encode()
-        req = request.Request(self.web_url + "/send-mail/" + self.id_job, data=data)
+        req = request.Request(self.config.web_url + "/send-mail/" + self.id_job, data=data)
         resp = request.urlopen(req)
         if resp.getcode() != 200:
             print("Job %s: Send mail failed!" % self.id_job)
@@ -271,9 +262,8 @@ class JobManager:
             job = Job.get(id_job=self.id_job)
             job.status = "success"
             db.commit()
-        if self.do_send:
+        if self.config.send_mail_status:
             self.send_mail_post()
-            #self.send_mail(job.status)
 
     @db_session
     def start_job(self):
@@ -288,7 +278,7 @@ class JobManager:
                 job.status = "fail"
                 job.error = "<p>Error while getting input files. Please contact the support to report the bug.</p>"
                 db.commit()
-                if self.do_send:
+                if self.config.send_mail_status:
                     self.send_mail()
 
         except Exception:
@@ -297,7 +287,7 @@ class JobManager:
             job.status = "fail"
             job.error = "<p>An unexpected error has occurred. Please contact the support to report the bug.</p>"
             db.commit()
-            if self.do_send:
+            if self.config.send_mail_status:
                 self.send_mail()
 
 
@@ -308,7 +298,7 @@ class JobManager:
             print("Old job found without result dir existing: delete it from BDD!")
             j1.delete()
         if self.target is not None:
-            job = Job(id_job=self.id_job, email=self.email, batch_type=self.batch_system_type,
+            job = Job(id_job=self.id_job, email=self.email, batch_type=self.config.batch_system_type,
                       date_created=datetime.datetime.now())
             db.commit()
             if not os.path.exists(self.output_dir):
@@ -316,7 +306,7 @@ class JobManager:
             thread = threading.Timer(1, self.start_job)
             thread.start()
         else:
-            job = Job(id_job=self.id_job, email=self.email, batch_type=self.batch_system_type,
+            job = Job(id_job=self.id_job, email=self.email, batch_type=self.config.batch_system_type,
                       date_created=datetime.datetime.now(), status="fail")
             db.commit()
 

lib/mailer.py

@@ -7,20 +7,21 @@ class Mailer:
     def __init__(self, app):
         self.app = app
         self.mail = Mail(app)
-        config_reader = AppConfigReader()
-        self.mail_status = config_reader.get_mail_status_sender()
-        self.mail_reply = config_reader.get_mail_reply()
-        self.mail_org = config_reader.get_mail_org()
-        self.disable = config_reader.get_disable_mail()
+        self.config = AppConfigReader()
+        # self.mail_status = config_reader.get_mail_status_sender()
+        # self.mail_reply = config_reader.get_mail_reply()
+        # self.mail_org = config_reader.get_mail_org()
+        # self.disable = config_reader.get_disable_mail()
 
     def __send_async_email(self, msg):
         with self.app.app_context():
             self.mail.send(msg)
 
     def send_mail(self, recipients: list, subject: str, message: str, message_html: str=None):
-        sender = (self.mail_org, self.mail_status) if self.mail_org is not None else self.mail_status
-        reply = self.mail_reply
-        if not self.disable:
+        sender = (self.config.mail_org, self.config.mail_status_sender) if self.config.mail_org is not None else \
+            self.config.mail_status_sender
+        reply = self.config.mail_reply
+        if not self.config.disable_mail:
             msg = Message(
                 subject= subject,
                 recipients=recipients,
@@ -36,7 +37,7 @@ class Mailer:
                   "################\n\n"
                   "!!! SEND MAILS DISABLED BY CONFIGURATION !!!\n\n"
                   "(This might be disabled in production)\n\n")
-            print("Sender: %s <%s>\n" % sender)
+            print("Sender: %s <%s>\n" % sender if isinstance(tuple, sender) else ("None", sender))
             print("Reply to: %s\n" % reply)
             print("Recipients: %s\n" % ", ".join(recipients))
             print("Subject: %s\n" % subject)

lib/paf.py

@@ -141,10 +141,12 @@ class Paf:
 
         try:
             with open(self.paf, "r") as paf_file:
-                paf_lines = paf_file.readlines()
-                if len(paf_lines) > self.max_nb_lines:
-                    self.sampled = True
-                for line in paf_lines[:self.max_nb_lines]:
+                nb_lines = 0
+                for line in paf_file:
+                    nb_lines += 1
+                    if nb_lines > self.max_nb_lines:
+                        self.sampled = True
+                        break
                     parts = line.strip("\n").split("\t")
                     v1 = parts[0]
                     v6 = parts[5]

srv/config_reader.py

@@ -2,16 +2,21 @@ import os
 import re
 import inspect
 from configparser import RawConfigParser, NoOptionError, NoSectionError
+from lib.decorators import Singleton
 
 
-class AppConfigReader(object):
+@Singleton
+class AppConfigReader:
     """
+    Store all configs
     """
 
     CONFIG_FILE_PATH = "../application.properties"
 
     def __init__(self):
         """
+        All "get_*" functions results are stored in the "self.*" corresponding attribute
+        Example: results of the get_upload_folder function is stored in self.upload_folder
         """
         config_file = os.path.join(os.path.dirname(inspect.getfile(self.__class__)), self.CONFIG_FILE_PATH)
         if not os.path.exists(config_file):
@@ -19,6 +24,10 @@ class AppConfigReader(object):
                                     "properties are correct for you!")
         self.reader = RawConfigParser()
         self.reader.read(config_file)
+        for attr in dir(self):
+            attr_o = getattr(self, attr)
+            if attr.startswith("get_") and callable(attr_o):
+                setattr(self, attr[4:], attr_o())
 
     @staticmethod
     def replace_vars(path):

srv/database.py

@@ -2,8 +2,8 @@ import datetime
 from config_reader import AppConfigReader
 from pony.orm import Database, Required, Optional
 
-config_reader = AppConfigReader()
-file_path = config_reader.get_database()
+config = AppConfigReader()
+file_path = config.database
 
 db = Database()
 db.bind(provider='sqlite', filename=file_path, create_db=True)

srv/main.py

@@ -29,8 +29,8 @@ sqlite_file = os.path.join(app_folder, "database.sqlite")
 # Init config reader:
 config_reader = AppConfigReader()
 
-UPLOAD_FOLDER = config_reader.get_upload_folder()
-APP_DATA = config_reader.get_app_data()
+UPLOAD_FOLDER = config_reader.upload_folder
+APP_DATA = config_reader.app_data
 
 app_title = "D-GENIES - Dotplot for Genomes Interactive, E-connected and Speedy"
 
@@ -43,7 +43,7 @@ app.config['SECRET_KEY'] = 'dsqdsq-255sdA-fHfg52-25Asd5'
 mailer = Mailer(app)
 
 # Folder containing data:
-app_data = config_reader.get_app_data()
+app_data = config_reader.app_data
 
 # Crons:
 crons = Crons(app_folder)