Improve & fix code + add logs usage + add local config file in etc

.gitignore

@@ -7,3 +7,4 @@ logs/*
 !logs/README
 dgenies/.local_scheduler_pid
 dgenies.egg-info
+application.properties.local

bin

-src/dgenies/bin
\ No newline at end of file

lib

-src/dgenies/lib
\ No newline at end of file

logs

-src/dgenies/logs
\ No newline at end of file

setup.py

@@ -15,5 +15,5 @@ setup(
     zip_safe=False,
     install_requires=reqs,
     data_files=[('/etc/dgenies', ['application.properties'])],
-    scripts=['bin/dgenies'],
+    scripts=['src/bin/dgenies'],
 )

setup.py.bk

-from setuptools import setup, find_packages
-from pip.req import parse_requirements
-
-install_reqs = parse_requirements('requirements.txt', session='hack')
-# reqs is a list of requirement
-# e.g. ['django==1.5.1', 'mezzanine==1.4.6']
-reqs = [str(ir.req) for ir in install_reqs]
-
-setup(
-    name='dgenies',
-    version='0.9',
-    packages=find_packages('src'),
-    package_dir={'dgenies': 'src/dgenies'},
-    include_package_data=True,
-    zip_safe=False,
-    install_requires=reqs,
-    package_data={"dgenies": ["bin/*.py", "static/*/*.css", "static/*/*.js", "templates/*.html"]},
-    data_files=[('/etc/dgenies', {'application.properties':'application.properties.example'})]
-)

src/dgenies/bin/dgenies → src/bin/dgenies

@@ -62,6 +62,8 @@ def run(mode="standalone", debug=False, host="127.0.0.1", port=5000, no_crons=Fa
             "port": port
         })
         thread.start()
+    if debug:
+        os.environ['LOGS'] = "True"
     from dgenies import app
     app.run(host=host, port=port, debug=debug)
 

src/dgenies/__init__.py

@@ -9,10 +9,6 @@ from .lib.crons import Crons
 import sys
 
 app_folder = os.path.dirname(os.path.realpath(__file__))
-sys.path.insert(0, app_folder)
-os.environ["PATH"] = os.path.join(app_folder, "bin") + ":" + os.environ["PATH"]
-
-sqlite_file = os.path.join(app_folder, "database.sqlite")
 
 
 # Init config reader:
@@ -38,10 +34,7 @@ if config_reader.debug and config_reader.log_dir != "stdout" and not os.path.exi
 # Crons:
 if os.getenv('DISABLE_CRONS') != "True":
     print("Starting crons...")
-    crons = Crons(app_folder)
+    crons = Crons(app_folder, config_reader.debug or os.getenv('LOGS') == "True")
     crons.start_all()
 
 from dgenies import views
-
-if __name__ == '__main__':
-    app.run()

src/dgenies/bin/local_scheduler.py

@@ -192,8 +192,8 @@ def parse_args():
     global DEBUG, LOG_FILE
 
     parser = argparse.ArgumentParser(description="Start local scheduler")
-    parser.add_argument('-d', '--debug', type=str, required=False, help="File describing input files")
-    parser.add_argument('-l', '--log-dir', type=str, required=False, help="Url of the server")
+    parser.add_argument('-d', '--debug', type=str, required=False, help="Set to True to enable debug")
+    parser.add_argument('-l', '--log-dir', type=str, required=False, help="Folder into store logs")
     args = parser.parse_args()
 
     if args.debug is not None:

src/dgenies/bin/start_local_scheduler.sh

@@ -3,6 +3,7 @@
 prg_dir=$1
 python=$2
 pid_file=$3
+logs=$4
 
 cd ${prg_dir}
 
@@ -16,8 +17,12 @@ if [ -f "${pid_file}" ]; then
 fi
 
 if [ "$is_started" -eq "0" ]; then
+    args="-d False"
+    if [ "${logs}" != "None" ]; then
+        args="-d True -l ${logs}"
+    fi
     echo "Starting scheduler..."
-    ${python} bin/local_scheduler.py > /dev/null &
+    ${python} bin/local_scheduler.py ${args} &> /dev/null &
     echo $! > ${pid_file}
 else
     echo "Already started!"

src/dgenies/config_reader.py

@@ -20,11 +20,16 @@ class AppConfigReader:
         Example: results of the get_upload_folder function is stored in self.upload_folder
         """
         self.app_dir = os.path.dirname(inspect.getfile(self.__class__))
-        if not os.path.exists(self.config_file):
-            raise FileNotFoundError("ERROR: application.properties not found. Please copy the example file and check "
-                                    "properties are correct for you!")
+        config_file = []
+        if os.path.exists(self.config_file):
+            config_file.append(self.config_file)
         if os.path.exists(self.config_file_local):
-            self.config_file = [self.config_file, self.config_file_local]
+            config_file.append(self.config_file_local)
+        if len(config_file) > 0:
+            self.config_file = config_file
+        else:
+            raise FileNotFoundError("ERROR: application.properties not found. Please copy the example file and "
+                                    "check properties are correct for you!")
         self.reader = RawConfigParser()
         self.reader.read(self.config_file)
         for attr in dir(self):
@@ -105,7 +110,7 @@ class AppConfigReader:
     def _get_minimap2_exec(self):
         try:
             entry = self.reader.get("softwares", "minimap2")
-            return entry if entry != "###DEFAULT###" else "minimap2"
+            return entry if entry != "###DEFAULT###" else os.path.join(self.app_dir, "bin", "minimap2")
         except NoOptionError:
             return os.path.join(self.app_dir, "bin", "minimap2")
 
@@ -317,10 +322,14 @@ class AppConfigReader:
 
     def _get_log_dir(self):
         try:
-            return self._replace_vars(self.reader.get("debug", "log_dir"))
+            log_dir = self._replace_vars(self.reader.get("debug", "log_dir"))
         except (NoOptionError, NoSectionError):
-            if self._get_debug():
-                raise Exception("No log dir defined and debug=True")
+            log_dir = self._replace_vars("###USER###/.dgenies/logs")
+        if not os.path.exists(log_dir):
+            os.makedirs(log_dir)
+        elif not os.path.isdir(log_dir):
+            raise TypeError("Log dir must be a directory")
+        return log_dir
 
     def _get_allowed_ip_tests(self):
         allowed_ip = {"127.0.0.1"}

src/dgenies/database.py

 import os
-from .config_reader import AppConfigReader
+from dgenies.config_reader import AppConfigReader
 from peewee import SqliteDatabase, Model, CharField, IntegerField, DateTimeField, BooleanField, MySQLDatabase
 from datetime import datetime
 
@@ -43,7 +43,7 @@ class Session(Model):
 
     @classmethod
     def new(cls, keep_active=False):
-        from lib.functions import Functions
+        from dgenies.lib.functions import Functions
         my_s_id = Functions.random_string(20)
         while len(cls.select().where(cls.s_id == my_s_id)) > 0:
             my_s_id = Functions.random_string(20)

src/dgenies/lib/crons.py

@@ -9,8 +9,9 @@ from dgenies.config_reader import AppConfigReader
 
 class Crons:
 
-    def __init__(self, base_dir):
+    def __init__(self, base_dir, debug):
         self.base_dir = base_dir
+        self.debug = debug
         self.my_cron = CronTab(user=getpass.getuser())
         self.config = AppConfigReader()
         self.local_scheduler_pid_file = os.path.join(os.path.expanduser("~"), ".dgenies", ".local_scheduler_pid")
@@ -41,7 +42,7 @@ class Crons:
         menage_freq = self.config.cron_menage_freq
         if self.base_dir is not None:
             job = self.my_cron.new(sys.executable +
-                                   " {0}/bin/clean_jobs.py > {0}/logs/menage.log 2>&1".format(self.base_dir),
+                                   " {0}/bin/clean_jobs.py > {0}/menage.log 2>&1".format(self.config.log_dir),
                                    comment="dgenies")
             job.day.every(menage_freq)
             job.hour.on(menage_hour[0])
@@ -60,8 +61,9 @@ class Crons:
             match = re.match(r"^(.+)/lib/(python[^/]+)/((site-packages/bin/python)|())$", pyexec)
             if match:
                 pyexec = "%s/bin/%s" % (match.group(1), match.group(2))
-            job = self.my_cron.new("{0}/bin/start_local_scheduler.sh {0} {1} {2} > /dev/null 2>&1 &".
-                                   format(self.base_dir, pyexec, self.local_scheduler_pid_file),
+            logs = os.path.join(self.config.log_dir, "local_scheduler.log") if self.debug else "/dev/null"
+            job = self.my_cron.new("{0}/bin/start_local_scheduler.sh {0} {1} {2} {3} > /dev/null 2>&1 &".
+                                   format(self.base_dir, pyexec, self.local_scheduler_pid_file, logs),
                                    comment="dgenies")
             job.minute.every(1)
             self.my_cron.write()
\ No newline at end of file

src/dgenies/views.py

@@ -8,12 +8,12 @@ import re
 import threading
 from flask import render_template, request, url_for, jsonify, Response, abort
 from pathlib import Path
-from .lib.paf import Paf
-from .lib.job_manager import JobManager
-from .lib.functions import Functions, ALLOWED_EXTENSIONS
-from .lib.upload_file import UploadFile
-from .lib.fasta import Fasta
-from .database import Session
+from dgenies.lib.paf import Paf
+from dgenies.lib.job_manager import JobManager
+from dgenies.lib.functions import Functions, ALLOWED_EXTENSIONS
+from dgenies.lib.upload_file import UploadFile
+from dgenies.lib.fasta import Fasta
+from dgenies.database import Session
 from peewee import DoesNotExist