Commit 374e5fed authored by Floreal Cabanettes's avatar Floreal Cabanettes
Browse files

Improve & fix code + add logs usage + add local config file in etc

parent 529e4f75
......@@ -7,3 +7,4 @@ logs/*
!logs/README
dgenies/.local_scheduler_pid
dgenies.egg-info
application.properties.local
src/dgenies/bin
\ No newline at end of file
src/dgenies/lib
\ No newline at end of file
src/dgenies/logs
\ No newline at end of file
......@@ -15,5 +15,5 @@ setup(
zip_safe=False,
install_requires=reqs,
data_files=[('/etc/dgenies', ['application.properties'])],
scripts=['bin/dgenies'],
scripts=['src/bin/dgenies'],
)
from setuptools import setup, find_packages
from pip.req import parse_requirements
install_reqs = parse_requirements('requirements.txt', session='hack')
# reqs is a list of requirement
# e.g. ['django==1.5.1', 'mezzanine==1.4.6']
reqs = [str(ir.req) for ir in install_reqs]
setup(
name='dgenies',
version='0.9',
packages=find_packages('src'),
package_dir={'dgenies': 'src/dgenies'},
include_package_data=True,
zip_safe=False,
install_requires=reqs,
package_data={"dgenies": ["bin/*.py", "static/*/*.css", "static/*/*.js", "templates/*.html"]},
data_files=[('/etc/dgenies', {'application.properties':'application.properties.example'})]
)
......@@ -62,6 +62,8 @@ def run(mode="standalone", debug=False, host="127.0.0.1", port=5000, no_crons=Fa
"port": port
})
thread.start()
if debug:
os.environ['LOGS'] = "True"
from dgenies import app
app.run(host=host, port=port, debug=debug)
......
......@@ -9,10 +9,6 @@ from .lib.crons import Crons
import sys
app_folder = os.path.dirname(os.path.realpath(__file__))
sys.path.insert(0, app_folder)
os.environ["PATH"] = os.path.join(app_folder, "bin") + ":" + os.environ["PATH"]
sqlite_file = os.path.join(app_folder, "database.sqlite")
# Init config reader:
......@@ -38,10 +34,7 @@ if config_reader.debug and config_reader.log_dir != "stdout" and not os.path.exi
# Crons:
if os.getenv('DISABLE_CRONS') != "True":
print("Starting crons...")
crons = Crons(app_folder)
crons = Crons(app_folder, config_reader.debug or os.getenv('LOGS') == "True")
crons.start_all()
from dgenies import views
if __name__ == '__main__':
app.run()
......@@ -192,8 +192,8 @@ def parse_args():
global DEBUG, LOG_FILE
parser = argparse.ArgumentParser(description="Start local scheduler")
parser.add_argument('-d', '--debug', type=str, required=False, help="File describing input files")
parser.add_argument('-l', '--log-dir', type=str, required=False, help="Url of the server")
parser.add_argument('-d', '--debug', type=str, required=False, help="Set to True to enable debug")
parser.add_argument('-l', '--log-dir', type=str, required=False, help="Folder into store logs")
args = parser.parse_args()
if args.debug is not None:
......
......@@ -3,6 +3,7 @@
prg_dir=$1
python=$2
pid_file=$3
logs=$4
cd ${prg_dir}
......@@ -16,8 +17,12 @@ if [ -f "${pid_file}" ]; then
fi
if [ "$is_started" -eq "0" ]; then
args="-d False"
if [ "${logs}" != "None" ]; then
args="-d True -l ${logs}"
fi
echo "Starting scheduler..."
${python} bin/local_scheduler.py > /dev/null &
${python} bin/local_scheduler.py ${args} &> /dev/null &
echo $! > ${pid_file}
else
echo "Already started!"
......
......@@ -20,11 +20,16 @@ class AppConfigReader:
Example: results of the get_upload_folder function is stored in self.upload_folder
"""
self.app_dir = os.path.dirname(inspect.getfile(self.__class__))
if not os.path.exists(self.config_file):
raise FileNotFoundError("ERROR: application.properties not found. Please copy the example file and check "
"properties are correct for you!")
config_file = []
if os.path.exists(self.config_file):
config_file.append(self.config_file)
if os.path.exists(self.config_file_local):
self.config_file = [self.config_file, self.config_file_local]
config_file.append(self.config_file_local)
if len(config_file) > 0:
self.config_file = config_file
else:
raise FileNotFoundError("ERROR: application.properties not found. Please copy the example file and "
"check properties are correct for you!")
self.reader = RawConfigParser()
self.reader.read(self.config_file)
for attr in dir(self):
......@@ -105,7 +110,7 @@ class AppConfigReader:
def _get_minimap2_exec(self):
try:
entry = self.reader.get("softwares", "minimap2")
return entry if entry != "###DEFAULT###" else "minimap2"
return entry if entry != "###DEFAULT###" else os.path.join(self.app_dir, "bin", "minimap2")
except NoOptionError:
return os.path.join(self.app_dir, "bin", "minimap2")
......@@ -317,10 +322,14 @@ class AppConfigReader:
def _get_log_dir(self):
try:
return self._replace_vars(self.reader.get("debug", "log_dir"))
log_dir = self._replace_vars(self.reader.get("debug", "log_dir"))
except (NoOptionError, NoSectionError):
if self._get_debug():
raise Exception("No log dir defined and debug=True")
log_dir = self._replace_vars("###USER###/.dgenies/logs")
if not os.path.exists(log_dir):
os.makedirs(log_dir)
elif not os.path.isdir(log_dir):
raise TypeError("Log dir must be a directory")
return log_dir
def _get_allowed_ip_tests(self):
allowed_ip = {"127.0.0.1"}
......
import os
from .config_reader import AppConfigReader
from dgenies.config_reader import AppConfigReader
from peewee import SqliteDatabase, Model, CharField, IntegerField, DateTimeField, BooleanField, MySQLDatabase
from datetime import datetime
......@@ -43,7 +43,7 @@ class Session(Model):
@classmethod
def new(cls, keep_active=False):
from lib.functions import Functions
from dgenies.lib.functions import Functions
my_s_id = Functions.random_string(20)
while len(cls.select().where(cls.s_id == my_s_id)) > 0:
my_s_id = Functions.random_string(20)
......
......@@ -9,8 +9,9 @@ from dgenies.config_reader import AppConfigReader
class Crons:
def __init__(self, base_dir):
def __init__(self, base_dir, debug):
self.base_dir = base_dir
self.debug = debug
self.my_cron = CronTab(user=getpass.getuser())
self.config = AppConfigReader()
self.local_scheduler_pid_file = os.path.join(os.path.expanduser("~"), ".dgenies", ".local_scheduler_pid")
......@@ -41,7 +42,7 @@ class Crons:
menage_freq = self.config.cron_menage_freq
if self.base_dir is not None:
job = self.my_cron.new(sys.executable +
" {0}/bin/clean_jobs.py > {0}/logs/menage.log 2>&1".format(self.base_dir),
" {0}/bin/clean_jobs.py > {0}/menage.log 2>&1".format(self.config.log_dir),
comment="dgenies")
job.day.every(menage_freq)
job.hour.on(menage_hour[0])
......@@ -60,8 +61,9 @@ class Crons:
match = re.match(r"^(.+)/lib/(python[^/]+)/((site-packages/bin/python)|())$", pyexec)
if match:
pyexec = "%s/bin/%s" % (match.group(1), match.group(2))
job = self.my_cron.new("{0}/bin/start_local_scheduler.sh {0} {1} {2} > /dev/null 2>&1 &".
format(self.base_dir, pyexec, self.local_scheduler_pid_file),
logs = os.path.join(self.config.log_dir, "local_scheduler.log") if self.debug else "/dev/null"
job = self.my_cron.new("{0}/bin/start_local_scheduler.sh {0} {1} {2} {3} > /dev/null 2>&1 &".
format(self.base_dir, pyexec, self.local_scheduler_pid_file, logs),
comment="dgenies")
job.minute.every(1)
self.my_cron.write()
\ No newline at end of file
......@@ -8,12 +8,12 @@ import re
import threading
from flask import render_template, request, url_for, jsonify, Response, abort
from pathlib import Path
from .lib.paf import Paf
from .lib.job_manager import JobManager
from .lib.functions import Functions, ALLOWED_EXTENSIONS
from .lib.upload_file import UploadFile
from .lib.fasta import Fasta
from .database import Session
from dgenies.lib.paf import Paf
from dgenies.lib.job_manager import JobManager
from dgenies.lib.functions import Functions, ALLOWED_EXTENSIONS
from dgenies.lib.upload_file import UploadFile
from dgenies.lib.fasta import Fasta
from dgenies.database import Session
from peewee import DoesNotExist
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment