Fix & improve code

application.properties

 [global]
-upload_folder = /tmp
-data_folder = ###USER###/.dgenies/data
+config_dir = ###USER###/.dgenies
+upload_folder = /tmp/dgenies
+data_folder = ###CONFIG###/data
 
 # batch system type: local, sge, slurm
 batch_system_type = local
@@ -19,7 +20,7 @@ max_upload_file_size = 1G
 [debug]
 # Debug (enable only for tests)
 enable = False
-log_dir = ###USER###/.dgenies/logs
+log_dir = ###CONFIG###/logs
 # List of allowed IPs for tests, comma separated:
 allowed_ip_tests =
 

dgenies.wsgi

-from dgenies import app as application
\ No newline at end of file
+from dgenies import launch
+
+application = launch()
\ No newline at end of file

src/bin/dgenies

@@ -4,6 +4,11 @@ import os
 import argparse
 import webbrowser
 import threading
+from glob import glob
+import time
+from dgenies.lib.crons import Crons
+from dgenies.config_reader import AppConfigReader
+from dgenies.bin.clean_jobs import parse_data_folders, parse_database, parse_upload_folders
 
 runned = False
 
@@ -45,7 +50,7 @@ def parse_args():
         if args.crons == False and args.logs == False and args.jobs == False:
             print("Nothing to do.")
             return "clear", None
-        return args
+        return "clear", args
 
 
 def start_browser(host, port):
@@ -54,7 +59,6 @@ def start_browser(host, port):
 
 
 def run(mode="standalone", debug=False, host="127.0.0.1", port=5000, no_crons=False, no_browser=False):
-    global runned
     os.environ['DISABLE_CRONS'] = "True" if no_crons else "False"
     if not debug and not no_browser:
         thread = threading.Timer(2, start_browser, kwargs={
@@ -64,11 +68,86 @@ def run(mode="standalone", debug=False, host="127.0.0.1", port=5000, no_crons=Fa
         thread.start()
     if debug:
         os.environ['LOGS'] = "True"
-    from dgenies import app
+    from dgenies import launch
+    app = launch()
     app.run(host=host, port=port, debug=debug)
 
 
+def clear_crons():
+    crons = Crons(None, True)
+    crons.clear()
+
+
+def clear_logs():
+    config = AppConfigReader()
+
+    if hasattr(config, "log_dir"):
+        log_files = glob(os.path.join(config.log_dir, "*.log"))
+        for file in log_files:
+            os.remove(file)
+
+    else:
+        print("No log dir defined!")
+
+
+def clear_jobs(max_data_age=7):
+    config_reader = AppConfigReader()
+
+    upload_folder = config_reader.upload_folder
+    app_data = config_reader.app_data
+    now = time.time()
+
+    max_age = {
+        "uploads": 0,
+        "error": 0,
+        "data": max_data_age,
+        "fasta_sorted": 0
+    }
+
+    print("#########################")
+    print("# Parsing Upload folder #")
+    print("#########################")
+    print("")
+    parse_upload_folders(
+        upload_folder=upload_folder,
+        now=now,
+        max_age=max_age
+    )
+    print("")
+
+    print("######################")
+    print("# Parsing Jobs in DB #")
+    print("######################")
+    print("")
+    parse_database(
+        app_data=app_data,
+        max_age=max_age
+    )
+    print("")
+
+    print("#######################")
+    print("# Parsing Data folder #")
+    print("#######################")
+    print("")
+    parse_data_folders(
+        app_data=app_data,
+        now=now,
+        max_age=max_age
+    )
+    print("")
+
+
 if __name__ == "__main__":
     command, args = parse_args()
     if command == "run":
         run(*args)
+    elif command == "clear":
+        if args.crons:
+            print("Cleaning crons...")
+            clear_crons()
+        if args.logs:
+            print("Cleaning logs...")
+            clear_logs()
+        if args.jobs:
+            print("Cleaning jobs...")
+            clear_jobs(args.max_age)

src/dgenies/__init__.py

@@ -6,35 +6,44 @@ from .config_reader import AppConfigReader
 from .lib.mailer import Mailer
 from .lib.crons import Crons
 
-import sys
+app = None
+app_title = None
+APP_DATA = None
+config_reader = None
+mailer = None
 
-app_folder = os.path.dirname(os.path.realpath(__file__))
 
+def launch():
+    global app, app_title, APP_DATA, config_reader, mailer
+    app_folder = os.path.dirname(os.path.realpath(__file__))
 
-# Init config reader:
-config_reader = AppConfigReader()
 
-UPLOAD_FOLDER = config_reader.upload_folder
-APP_DATA = config_reader.app_data
+    # Init config reader:
+    config_reader = AppConfigReader()
 
-app_title = "D-GENIES - Dotplot for Genomes Interactive, E-connected and Speedy"
+    UPLOAD_FOLDER = config_reader.upload_folder
+    APP_DATA = config_reader.app_data
 
-# Init Flask:
-app = Flask(__name__, static_url_path='/static')
-app.config['UPLOAD_FOLDER'] = UPLOAD_FOLDER
-app.config['MAX_CONTENT_LENGTH'] = config_reader.max_upload_file_size
-app.config['SECRET_KEY'] = 'dsqdsq-255sdA-fHfg52-25Asd5'
+    app_title = "D-GENIES - Dotplot for Genomes Interactive, E-connected and Speedy"
 
-# Init mail:
-mailer = Mailer(app)
+    # Init Flask:
+    app = Flask(__name__, static_url_path='/static')
+    app.config['UPLOAD_FOLDER'] = UPLOAD_FOLDER
+    app.config['MAX_CONTENT_LENGTH'] = config_reader.max_upload_file_size
+    app.config['SECRET_KEY'] = 'dsqdsq-255sdA-fHfg52-25Asd5'
 
-if config_reader.debug and config_reader.log_dir != "stdout" and not os.path.exists(config_reader.log_dir):
-    os.makedirs(config_reader.log_dir)
+    # Init mail:
+    mailer = Mailer(app)
 
-# Crons:
-if os.getenv('DISABLE_CRONS') != "True":
-    print("Starting crons...")
-    crons = Crons(app_folder, config_reader.debug or os.getenv('LOGS') == "True")
-    crons.start_all()
+    if config_reader.debug and config_reader.log_dir != "stdout" and not os.path.exists(config_reader.log_dir):
+        os.makedirs(config_reader.log_dir)
 
-from dgenies import views
+    # Crons:
+    if os.getenv('DISABLE_CRONS') != "True":
+        print("Starting crons...")
+        crons = Crons(app_folder, config_reader.debug or os.getenv('LOGS') == "True")
+        crons.start_all()
+
+    from dgenies import views
+
+    return app

src/dgenies/bin/clean_jobs.py

@@ -14,39 +14,27 @@ from dgenies.database import Job
 
 config_reader = AppConfigReader()
 
-UPLOAD_FOLDER = config_reader.upload_folder
-APP_DATA = config_reader.app_data
-NOW = time.time()
-FAKE = False
-
-max_age = {
-    "uploads": 1,
-    "error": 1,
-    "data": 7,
-    "fasta_sorted": 1
-}
-
-
-def parse_upload_folders():
-    for file in os.listdir(UPLOAD_FOLDER):
-        file = os.path.join(UPLOAD_FOLDER, file)
+
+def parse_upload_folders(upload_folder, now, max_age, fake=False):
+    for file in os.listdir(upload_folder):
+        file = os.path.join(upload_folder, file)
         create_date = os.path.getctime(file)
-        age = (NOW - create_date) / 86400  # Age in days
+        age = (now - create_date) / 86400  # Age in days
         if age > max_age["uploads"]:
             try:
                 if os.path.isdir(file):
                     print("Removing folder %s..." % file)
-                    if not FAKE:
+                    if not fake:
                         shutil.rmtree(file)
                 else:
                     print("Removing file %s..." % file)
-                    if not FAKE:
+                    if not fake:
                         os.remove(file)
             except OSError:
                 print(traceback.print_exc(), file=sys.stderr)
 
 
-def parse_database():
+def parse_database(app_data, max_age, fake=False):
     old_jobs = Job.select().where(
                   ((Job.status == "success") & (Job.date_created < datetime.now() - timedelta(days=max_age["data"])))
                   |
@@ -55,30 +43,30 @@ def parse_database():
     for job in old_jobs:
         id_job = job.id_job
         print("Removing job %s..." % id_job)
-        data_dir = os.path.join(APP_DATA, id_job)
+        data_dir = os.path.join(app_data, id_job)
         if os.path.exists(data_dir) and os.path.isdir(data_dir):
-            if not FAKE:
+            if not fake:
                 shutil.rmtree(data_dir)
         else:
             print("Job %s has no data folder!" % id_job)
-        if not FAKE:
+        if not fake:
             job.delete_instance()
 
 
-def parse_data_folders():
-    for file in os.listdir(APP_DATA):
-        file = os.path.join(APP_DATA, file)
+def parse_data_folders(app_data, now, max_age, fake=False):
+    for file in os.listdir(app_data):
+        file = os.path.join(app_data, file)
         create_date = os.path.getctime(file)
-        age = (NOW - create_date) / 86400  # Age in days
+        age = (now - create_date) / 86400  # Age in days
         if age > max_age["data"]:
             try:
                 if os.path.isdir(file):
                     print("Removing folder %s..." % file)
-                    if not FAKE:
+                    if not fake:
                         shutil.rmtree(file)
                 else:
                     print("Removing file %s..." % file)
-                    if not FAKE:
+                    if not fake:
                         os.remove(file)
             except OSError:
                 print(traceback.print_exc())
@@ -91,10 +79,10 @@ def parse_data_folders():
                         sorted_file = None
                     if sorted_file is not None:
                         create_date = os.path.getctime(sorted_file)
-                        age = (NOW - create_date) / 86400  # Age in days
+                        age = (now - create_date) / 86400  # Age in days
                         if age > max_age["fasta_sorted"]:
                             print("Removing fasta file %s..." % sorted_file)
-                            if not FAKE:
+                            if not fake:
                                 os.remove(sorted_file)
 
 
@@ -102,29 +90,52 @@ if __name__ == '__main__':
     parser = argparse.ArgumentParser(description="Clean old jobs and files")
     parser.add_argument('-f', '--fake', type=bool, const=True, nargs="?", required=False, default=False,
                         help="Fake mode: don't really delete the files (ONLY for debug)")
+    parser.add_argument("-d", "--max-age", type=int, required=False, help="Max age of jobs to delete", default=7)
     args = parser.parse_args()
-    FAKE = args.fake
-    if FAKE:
-        print("RUNNING IN FAKE MODE...")
-        print("")
+    fake = args.fake
+
+    upload_folder = config_reader.upload_folder
+    app_data = config_reader.app_data
+    now = time.time()
+
+    max_age = {
+        "uploads": 1,
+        "error": 1,
+        "data": args.max_age,
+        "fasta_sorted": 1
+    }
 
     print("#########################")
     print("# Parsing Upload folder #")
     print("#########################")
     print("")
-    parse_upload_folders()
+    parse_upload_folders(
+        upload_folder=upload_folder,
+        now=now,
+        max_age=max_age,
+        fake=fake
+    )
     print("")
 
     print("######################")
     print("# Parsing Jobs in DB #")
     print("######################")
     print("")
-    parse_database()
+    parse_database(
+        app_data=app_data,
+        max_age=max_age,
+        fake=fake
+    )
     print("")
 
     print("#######################")
     print("# Parsing Data folder #")
     print("#######################")
     print("")
-    parse_data_folders()
+    parse_data_folders(
+        app_data=app_data,
+        now=now,
+        max_age=max_age,
+        fake=fake
+    )
     print("")

src/dgenies/bin/start_local_scheduler.sh

@@ -10,7 +10,7 @@ cd ${prg_dir}
 is_started=0
 
 if [ -f "${pid_file}" ]; then
-    pid=`cat .local_scheduler_pid`
+    pid=`cat ${pid_file}`
     if ps -p"$pid" -o "pid=" > /dev/null; then
         is_started=1
     fi

src/dgenies/config_reader.py

@@ -40,10 +40,21 @@ class AppConfigReader:
                 except Exception as e:
                     print(e)
 
-    @staticmethod
-    def _replace_vars(path):
-        return path.replace("###USER###", os.path.expanduser("~"))\
+    def _replace_vars(self, path, config=False):
+        new_path = path.replace("###USER###", os.path.expanduser("~"))\
             .replace("###PROGRAM###", os.path.dirname(os.path.dirname(os.path.realpath(__file__))))
+        if "###CONFIG###" in new_path:
+            if config:
+                raise Exception("###CONFIG### tag not allowed for config dir")
+            else:
+                return new_path.replace("###CONFIG###", self._get_config_dir())
+        return new_path
+
+    def _get_config_dir(self):
+        try:
+            return self._replace_vars(self.reader.get("global", "config_dir"), True)
+        except NoOptionError:
+            return self._replace_vars("###USER###/.dgenies")
 
     def _get_upload_folder(self):
         try:
@@ -324,7 +335,7 @@ class AppConfigReader:
         try:
             log_dir = self._replace_vars(self.reader.get("debug", "log_dir"))
         except (NoOptionError, NoSectionError):
-            log_dir = self._replace_vars("###USER###/.dgenies/logs")
+            log_dir = self._replace_vars("###CONFIG###/logs")
         if not os.path.exists(log_dir):
             os.makedirs(log_dir)
         elif not os.path.isdir(log_dir):

src/dgenies/lib/crons.py

@@ -14,7 +14,7 @@ class Crons:
         self.debug = debug
         self.my_cron = CronTab(user=getpass.getuser())
         self.config = AppConfigReader()
-        self.local_scheduler_pid_file = os.path.join(os.path.expanduser("~"), ".dgenies", ".local_scheduler_pid")
+        self.local_scheduler_pid_file = os.path.join(self.config.config_dir, ".local_scheduler_pid")
 
     def clear(self, kill_scheduler=True):
         # Remove old crons:

src/dgenies/logs/README

-This folder will contains some logs from dgenies server.

src/dgenies/logs/menage.log

-#########################
-# Parsing Upload folder #
-#########################
-
-Removing folder /home/fcabanettes/public_html/tmp/tNUTXMSya0MnFT1G32uw...
-Removing folder /home/fcabanettes/public_html/tmp/fYs40lzIocGt4jsfjtYF...
-Removing folder /home/fcabanettes/public_html/tmp/uBZgExNkgcry2pLcrG1c...
-Removing folder /home/fcabanettes/public_html/tmp/kjAI9bVRCdVJ94G4us4J...
-Removing folder /home/fcabanettes/public_html/tmp/s8LdyaIvGZlB7yoNhE5e...
-Removing folder /home/fcabanettes/public_html/tmp/hzYJieimzx103eWEZYam...
-Removing folder /home/fcabanettes/public_html/tmp/h6tshcrkas4qcR3asQAk...
-Removing folder /home/fcabanettes/public_html/tmp/xs0ETD3qvEvpcP6gDQBG...
-Removing folder /home/fcabanettes/public_html/tmp/u2asIrDJjAgEvvY6roJH...
-Removing folder /home/fcabanettes/public_html/tmp/fJMHBqoNtBUMtBHbQHso...
-Removing folder /home/fcabanettes/public_html/tmp/LnJkOdfBD2MIazUy1ptU...
-Removing folder /home/fcabanettes/public_html/tmp/LctmxdQKyMkKFaidB3IE...
-Removing folder /home/fcabanettes/public_html/tmp/1vGYeo2UJmH1556AshY7...
-Removing folder /home/fcabanettes/public_html/tmp/lzETqBCEX88KjREmQv8Q...
-Removing folder /home/fcabanettes/public_html/tmp/Fl3b0TMyCVDXQObmhQlz...
-Removing folder /home/fcabanettes/public_html/tmp/kjoipJhxhPLyXN9MFz2S...
-Removing folder /home/fcabanettes/public_html/tmp/zJAdGAomff5Msicubtfg...
-Removing folder /home/fcabanettes/public_html/tmp/istm3xhrFssCJlt38VfH...
-Removing folder /home/fcabanettes/public_html/tmp/ICxrSou5Pz7V1F82nDow...
-Removing folder /home/fcabanettes/public_html/tmp/18G0oprJ9sXiJ0jIXDAB...
-Removing folder /home/fcabanettes/public_html/tmp/qBSkh4v3zqkAeodhvOqF...
-Removing folder /home/fcabanettes/public_html/tmp/32hUyjffhv6CNFpbkkNb...
-Removing folder /home/fcabanettes/public_html/tmp/APSn6ilAGMJ6vON6eCnj...
-Removing folder /home/fcabanettes/public_html/tmp/F5RrXPa7eaotYIL5vap0...
-Removing folder /home/fcabanettes/public_html/tmp/4haGCx58JWjQTF0ftGFO...
-Removing folder /home/fcabanettes/public_html/tmp/OmPusYoAvcQPvujgmc6u...
-Removing folder /home/fcabanettes/public_html/tmp/36dLLqAYMeCZ2PUpW7Rz...
-Removing folder /home/fcabanettes/public_html/tmp/e9JCwR6xNArhqtiC5WaE...
-Removing folder /home/fcabanettes/public_html/tmp/xDlxZE10dN5g1RNh6yGg...
-Removing folder /home/fcabanettes/public_html/tmp/HFcWMfBgpjhX86KFEsQl...
-Removing folder /home/fcabanettes/public_html/tmp/4YpauTO9f5lLHv65FKyp...
-Removing folder /home/fcabanettes/public_html/tmp/bC3Jb8d0eMAc3HmSkQVJ...
-Removing folder /home/fcabanettes/public_html/tmp/1YN3AVEP4mGbwQUNd3t7...
-
-######################
-# Parsing Jobs in DB #
-######################
-
-
-#######################
-# Parsing Data folder #
-#######################
-
-Removing folder /home/fcabanettes/public_html/test/lPNWi_20171215180120...
-Removing folder /home/fcabanettes/public_html/test/i3LtD_20171222163104...
-Removing folder /home/fcabanettes/public_html/test/pb4Fk_20171221171628...
-Removing folder /home/fcabanettes/public_html/test/uVUyI_20171220173251...
-Removing folder /home/fcabanettes/public_html/test/xGr7b_20171222161435...
-Removing folder /home/fcabanettes/public_html/test/3tNGi_20171222163125...
-Removing folder /home/fcabanettes/public_html/test/NfKVy_20171222161433...
-Removing folder /home/fcabanettes/public_html/test/YwKXB_20171222163035...
-Removing folder /home/fcabanettes/public_html/test/E9Tpm_20171222160949...
-Removing folder /home/fcabanettes/public_html/test/1nKXZ_20171222163035...
-Removing folder /home/fcabanettes/public_html/test/MO0UX_20171222160950...
-Removing folder /home/fcabanettes/public_html/test/Dnge0_20171215180950...
-Removing folder /home/fcabanettes/public_html/test/w587w_20171222160947...
-Removing folder /home/fcabanettes/public_html/test/7bxho_20171221172603...
-Removing folder /home/fcabanettes/public_html/test/y7583_20171220151745...
-Removing folder /home/fcabanettes/public_html/test/hYDXZ_20171215181230...
-Removing folder /home/fcabanettes/public_html/test/mh9vi_20171220152019...
-Removing folder /home/fcabanettes/public_html/test/E7Ad6_20171218104637...
-Removing folder /home/fcabanettes/public_html/test/xWA6v_20171215180424...
-Removing folder /home/fcabanettes/public_html/test/OvT1q_20171222163039...
-Removing folder /home/fcabanettes/public_html/test/oJq9Q_20171215160103...
-Removing folder /home/fcabanettes/public_html/test/62wrQ_20171215180826...
-Removing folder /home/fcabanettes/public_html/test/0BY4Z_20171220160657...
-Removing folder /home/fcabanettes/public_html/test/zzd6b_20171218114300...
-Removing folder /home/fcabanettes/public_html/test/D4Lym_20171222160950...
-Removing folder /home/fcabanettes/public_html/test/8dECv_20171222163037...
-Removing folder /home/fcabanettes/public_html/test/nv7tx_20171222163121...
-Removing folder /home/fcabanettes/public_html/test/c4aJx_20171222163122...
-Removing folder /home/fcabanettes/public_html/test/eIqTH_20171220173241...
-Removing folder /home/fcabanettes/public_html/test/Legn7_20171222161432...
-Removing folder /home/fcabanettes/public_html/test/z7zdo_20171220153236...
-Removing folder /home/fcabanettes/public_html/test/SeBfe_20171222163112...
-Removing folder /home/fcabanettes/public_html/test/HJFVv_20171222163034...
-Removing folder /home/fcabanettes/public_html/test/MFRCO_20171220173313...
-Removing folder /home/fcabanettes/public_html/test/EOE7m_20171215154455...
-Removing folder /home/fcabanettes/public_html/test/Cy970_20171222161434...
-Removing folder /home/fcabanettes/public_html/test/G7xSE_20171215180606...
-Removing folder /home/fcabanettes/public_html/test/Asnzg_20171222115215...
-Removing folder /home/fcabanettes/public_html/test/27Oju_20171220173246...
-Removing folder /home/fcabanettes/public_html/test/8K7Vo_20171220173249...
-Removing folder /home/fcabanettes/public_html/test/rGXlo_20171222161434...
-Removing folder /home/fcabanettes/public_html/test/MghSN_20171222160946...
-Removing folder /home/fcabanettes/public_html/test/4Mdqa_20171220160150...
-Removing folder /home/fcabanettes/public_html/test/6ssx0_20171215141828...
-Removing folder /home/fcabanettes/public_html/test/LlmrB_20171215160741...
-Removing folder /home/fcabanettes/public_html/test/buvxa_20171220173104...
-Removing folder /home/fcabanettes/public_html/test/KtzNU_20171221172234...
-Removing folder /home/fcabanettes/public_html/test/hfI8p_20171220173254...
-Removing folder /home/fcabanettes/public_html/test/KjObX_20171220152611...
-Removing folder /home/fcabanettes/public_html/test/7Yyuk_20171222114023...
-Removing folder /home/fcabanettes/public_html/test/dBKww_20171215180357...
-Removing folder /home/fcabanettes/public_html/test/T8BSG_20171218111221...
-Removing folder /home/fcabanettes/public_html/test/AfBJi_20171215181145...
-Removing folder /home/fcabanettes/public_html/test/N0tN5_20171220154608...
-Removing folder /home/fcabanettes/public_html/test/yWFQs_20171215154622...
-