Commit 3536adc2 authored by Floreal Cabanettes's avatar Floreal Cabanettes
Browse files

Add standalone mode, Implements #114

parent 6ce3b7f3
...@@ -6,11 +6,8 @@ import webbrowser ...@@ -6,11 +6,8 @@ import webbrowser
import threading import threading
from glob import glob from glob import glob
import time import time
from dgenies.lib.crons import Crons
from dgenies.config_reader import AppConfigReader from dgenies.config_reader import AppConfigReader
from dgenies.bin.clean_jobs import parse_data_folders, parse_database, parse_upload_folders from dgenies.bin.clean_jobs import parse_data_folders, parse_database, parse_upload_folders
from dgenies.database import Gallery, Job
from peewee import DoesNotExist
runned = False runned = False
...@@ -45,6 +42,8 @@ def parse_args(): ...@@ -45,6 +42,8 @@ def parse_args():
default=False) default=False)
clear.add_argument("-m", "--max-age", help="Max age for job to delete (0 for all)", type=int, required=False, clear.add_argument("-m", "--max-age", help="Max age for job to delete (0 for all)", type=int, required=False,
default=0) default=0)
clear.add_argument("-w", "--web", help="Add this option with -j option, if you use the webserver mode", type=bool,
const=True, nargs="?", required=False, default=False)
# Gallery: # Gallery:
gallery = subparsers.add_parser("gallery", help="Manage gallery") gallery = subparsers.add_parser("gallery", help="Manage gallery")
...@@ -103,11 +102,12 @@ def run(mode="standalone", debug=False, host="127.0.0.1", port=5000, no_crons=Fa ...@@ -103,11 +102,12 @@ def run(mode="standalone", debug=False, host="127.0.0.1", port=5000, no_crons=Fa
if debug: if debug:
os.environ['LOGS'] = "True" os.environ['LOGS'] = "True"
from dgenies import launch from dgenies import launch
app = launch() app = launch(mode=mode, debug=debug)
app.run(host=host, port=port, debug=debug) app.run(host=host, port=port, debug=debug)
def clear_crons(): def clear_crons():
from dgenies.lib.crons import Crons
crons = Crons(None, True) crons = Crons(None, True)
crons.clear() crons.clear()
...@@ -122,8 +122,7 @@ def clear_logs(): ...@@ -122,8 +122,7 @@ def clear_logs():
print("No log dir defined!") print("No log dir defined!")
def clear_jobs(max_data_age=7): def clear_jobs(max_data_age=7, web=False):
upload_folder = config.upload_folder upload_folder = config.upload_folder
app_data = config.app_data app_data = config.app_data
now = time.time() now = time.time()
...@@ -146,15 +145,18 @@ def clear_jobs(max_data_age=7): ...@@ -146,15 +145,18 @@ def clear_jobs(max_data_age=7):
) )
print("") print("")
print("######################") if web:
print("# Parsing Jobs in DB #") print("######################")
print("######################") print("# Parsing Jobs in DB #")
print("") print("######################")
gallery_jobs = parse_database( print("")
app_data=app_data, gallery_jobs = parse_database(
max_age=max_age app_data=app_data,
) max_age=max_age
print("") )
print("")
else:
gallery_jobs = []
print("#######################") print("#######################")
print("# Parsing Data folder #") print("# Parsing Data folder #")
...@@ -170,6 +172,8 @@ def clear_jobs(max_data_age=7): ...@@ -170,6 +172,8 @@ def clear_jobs(max_data_age=7):
def add_to_gallery(id_job, name, picture, query, target): def add_to_gallery(id_job, name, picture, query, target):
from dgenies.database import Gallery, Job
from peewee import DoesNotExist
try: try:
job = Job.get(id_job=id_job) job = Job.get(id_job=id_job)
except DoesNotExist: except DoesNotExist:
...@@ -185,6 +189,7 @@ def add_to_gallery(id_job, name, picture, query, target): ...@@ -185,6 +189,7 @@ def add_to_gallery(id_job, name, picture, query, target):
def del_from_gallery_by_id(id_job): def del_from_gallery_by_id(id_job):
from dgenies.database import Gallery, Job
items = Gallery.select().join(Job).where(Job.id_job == id_job) items = Gallery.select().join(Job).where(Job.id_job == id_job)
list_pictures = [] list_pictures = []
for item in items: for item in items:
...@@ -194,6 +199,7 @@ def del_from_gallery_by_id(id_job): ...@@ -194,6 +199,7 @@ def del_from_gallery_by_id(id_job):
def del_from_gallery_by_name(name): def del_from_gallery_by_name(name):
from dgenies.database import Gallery
items = Gallery.select().where(Gallery.name == name) items = Gallery.select().where(Gallery.name == name)
list_pictures = [] list_pictures = []
for item in items: for item in items:
...@@ -215,7 +221,7 @@ if __name__ == "__main__": ...@@ -215,7 +221,7 @@ if __name__ == "__main__":
clear_logs() clear_logs()
if args.jobs: if args.jobs:
print("Cleaning jobs...") print("Cleaning jobs...")
clear_jobs(args.max_age) clear_jobs(args.max_age, args.web)
elif command == "gallery_add": elif command == "gallery_add":
add_to_gallery(args.id_job, args.name, args.pict, args.query, args.target) add_to_gallery(args.id_job, args.name, args.pict, args.query, args.target)
elif command == "gallery_del": elif command == "gallery_del":
......
...@@ -3,7 +3,6 @@ ...@@ -3,7 +3,6 @@
import os import os
from flask import Flask from flask import Flask
from .config_reader import AppConfigReader from .config_reader import AppConfigReader
from .lib.mailer import Mailer
from .lib.crons import Crons from .lib.crons import Crons
app = None app = None
...@@ -12,12 +11,16 @@ APP_DATA = None ...@@ -12,12 +11,16 @@ APP_DATA = None
config_reader = None config_reader = None
mailer = None mailer = None
app_folder = None app_folder = None
MODE = "webserver"
DEBUG = False
def launch(): def launch(mode="webserver", debug=False):
global app, app_title, app_folder, APP_DATA, config_reader, mailer global app, app_title, app_folder, APP_DATA, config_reader, mailer, MODE, DEBUG
app_folder = os.path.dirname(os.path.realpath(__file__)) app_folder = os.path.dirname(os.path.realpath(__file__))
MODE = mode
DEBUG = debug
# Init config reader: # Init config reader:
config_reader = AppConfigReader() config_reader = AppConfigReader()
...@@ -34,13 +37,15 @@ def launch(): ...@@ -34,13 +37,15 @@ def launch():
app.config['SECRET_KEY'] = 'dsqdsq-255sdA-fHfg52-25Asd5' app.config['SECRET_KEY'] = 'dsqdsq-255sdA-fHfg52-25Asd5'
# Init mail: # Init mail:
mailer = Mailer(app) if MODE == "webserver":
from .lib.mailer import Mailer
mailer = Mailer(app)
if config_reader.debug and config_reader.log_dir != "stdout" and not os.path.exists(config_reader.log_dir): if config_reader.debug and config_reader.log_dir != "stdout" and not os.path.exists(config_reader.log_dir):
os.makedirs(config_reader.log_dir) os.makedirs(config_reader.log_dir)
# Crons: # Crons:
if os.getenv('DISABLE_CRONS') != "True": if os.getenv('DISABLE_CRONS') != "True" and MODE == "webserver":
print("Starting crons...") print("Starting crons...")
crons = Crons(app_folder, config_reader.debug or os.getenv('LOGS') == "True") crons = Crons(app_folder, config_reader.debug or os.getenv('LOGS') == "True")
crons.start_all() crons.start_all()
......
...@@ -10,7 +10,6 @@ import argparse ...@@ -10,7 +10,6 @@ import argparse
from dgenies.config_reader import AppConfigReader from dgenies.config_reader import AppConfigReader
from dgenies.lib.functions import Functions from dgenies.lib.functions import Functions
from dgenies.database import Job, Gallery
config_reader = AppConfigReader() config_reader = AppConfigReader()
...@@ -35,6 +34,7 @@ def parse_upload_folders(upload_folder, now, max_age, fake=False): ...@@ -35,6 +34,7 @@ def parse_upload_folders(upload_folder, now, max_age, fake=False):
def parse_database(app_data, max_age, fake=False): def parse_database(app_data, max_age, fake=False):
from dgenies.database import Job, Gallery
gallery_jobs = [] gallery_jobs = []
with Job.connect(): with Job.connect():
old_jobs = Job.select().where( old_jobs = Job.select().where(
......
...@@ -6,23 +6,28 @@ from collections import OrderedDict ...@@ -6,23 +6,28 @@ from collections import OrderedDict
class Merger: class Merger:
def __init__(self, paf_in, paf_out, query_in, query_out): def __init__(self, paf_in, paf_out, query_in, query_out, debug=False):
self.paf_in = paf_in self.paf_in = paf_in
self.paf_out = paf_out self.paf_out = paf_out
self.query_in = query_in self.query_in = query_in
self.query_out = query_out self.query_out = query_out
self.debug = debug
def _printer(self, message):
if self.debug:
print(message)
def merge(self): def merge(self):
print("Loading query index...") self._printer("Loading query index...")
contigs, contigs_split, q_name = self.load_query_index(self.query_in) contigs, contigs_split, q_name = self.load_query_index(self.query_in)
print("Merging contigs in PAF file...") self._printer("Merging contigs in PAF file...")
self.merge_paf(self.paf_in, self.paf_out, contigs, contigs_split) self.merge_paf(self.paf_in, self.paf_out, contigs, contigs_split)
print("Writing new query index...") self._printer("Writing new query index...")
self.write_query_index(self.query_out, contigs, q_name) self.write_query_index(self.query_out, contigs, q_name)
print("DONE!") self._printer("DONE!")
@staticmethod @staticmethod
def _get_sorted_splits(contigs_split: dict, all_contigs: dict): def _get_sorted_splits(contigs_split: dict, all_contigs: dict):
......
...@@ -10,7 +10,7 @@ from collections import OrderedDict ...@@ -10,7 +10,7 @@ from collections import OrderedDict
class Splitter: class Splitter:
def __init__(self, input_f, name_f, output_f, size_c=10000000, query_index="query_split.idx"): def __init__(self, input_f, name_f, output_f, size_c=10000000, query_index="query_split.idx", debug=False):
self.input_f = input_f self.input_f = input_f
self.name_f = name_f self.name_f = name_f
self.size_c = size_c self.size_c = size_c
...@@ -20,6 +20,7 @@ class Splitter: ...@@ -20,6 +20,7 @@ class Splitter:
self.out_dir = os.path.dirname(output_f) self.out_dir = os.path.dirname(output_f)
self.index_file = os.path.join(self.out_dir, query_index) self.index_file = os.path.join(self.out_dir, query_index)
self.nb_contigs = 0 self.nb_contigs = 0
self.debug = debug
def split(self): def split(self):
""" """
...@@ -48,7 +49,8 @@ class Splitter: ...@@ -48,7 +49,8 @@ class Splitter:
return False return False
chr_name = re.split("\s", line[1:])[0] chr_name = re.split("\s", line[1:])[0]
fasta_str = "" fasta_str = ""
print("Parsing contig \"%s\"... " % chr_name, end="") if self.debug:
print("Parsing contig \"%s\"... " % chr_name, end="")
elif len(line) > 0: elif len(line) > 0:
if next_header or re.match(r"^[ATGCKMRYSWBVHDXN.\-]+$", line.upper()) is None: if next_header or re.match(r"^[ATGCKMRYSWBVHDXN.\-]+$", line.upper()) is None:
return False return False
...@@ -95,10 +97,11 @@ class Splitter: ...@@ -95,10 +97,11 @@ class Splitter:
self.write_contig(name, seq, enc) self.write_contig(name, seq, enc)
index_f.write("%s\t%d\n" % (name, len(seq))) index_f.write("%s\t%d\n" % (name, len(seq)))
nb_contigs = len(contigs) nb_contigs = len(contigs)
if nb_contigs == 1: if self.debug:
print("Keeped!") if nb_contigs == 1:
else: print("Keeped!")
print("Splited in %d contigs!" % nb_contigs) else:
print("Splited in %d contigs!" % nb_contigs)
def parse_args(): def parse_args():
......
from dgenies import MODE
import os import os
from dgenies.config_reader import AppConfigReader from dgenies.config_reader import AppConfigReader
from peewee import SqliteDatabase, Model, CharField, IntegerField, DateTimeField, BooleanField, MySQLDatabase, \
OperationalError, ForeignKeyField
from playhouse.shortcuts import RetryOperationalError
from datetime import datetime from datetime import datetime
config = AppConfigReader() config = AppConfigReader()
db_url = config.database_url
db_type = config.database_type
if MODE == "webserver":
from peewee import SqliteDatabase, Model, CharField, IntegerField, DateTimeField, BooleanField, MySQLDatabase, \
OperationalError, ForeignKeyField
from playhouse.shortcuts import RetryOperationalError
class MyRetryDB(RetryOperationalError, MySQLDatabase): db_url = config.database_url
pass db_type = config.database_type
if db_type == "sqlite": class MyRetryDB(RetryOperationalError, MySQLDatabase):
db = SqliteDatabase(db_url) pass
elif db_type == "mysql":
db = MyRetryDB(host=config.database_url, port=config.database_port, user=config.database_user,
passwd=config.database_password, database=config.database_db)
else:
raise Exception("Unsupported database type: " + db_type)
class Database: if db_type == "sqlite":
db = SqliteDatabase(db_url)
elif db_type == "mysql":
db = MyRetryDB(host=config.database_url, port=config.database_port, user=config.database_user,
passwd=config.database_password, database=config.database_db)
else:
raise Exception("Unsupported database type: " + db_type)
nb_open = 0
def __init__(self): class Database:
pass
nb_open = 0
def __enter__(self): def __init__(self):
Database.nb_open += 1
try:
db.connect()
except OperationalError:
pass pass
def __exit__(self, exc_type, exc_val, exc_tb): def __enter__(self):
Database.nb_open -= 1 Database.nb_open += 1
if Database.nb_open == 0: try:
db.close() db.connect()
except OperationalError:
pass
def __exit__(self, exc_type, exc_val, exc_tb):
Database.nb_open -= 1
if Database.nb_open == 0:
db.close()
class BaseModel(Model):
class Meta: class BaseModel(Model):
database = db
@classmethod class Meta:
def connect(cls): database = db
return Database()
@classmethod
def connect(cls):
return Database()
class Job(BaseModel):
id_job = CharField(max_length=50, unique=True)
email = CharField()
id_process = IntegerField(null=True)
batch_type = CharField(max_length=20, default="local")
status = CharField(max_length=20, default="submitted")
date_created = DateTimeField()
error = CharField(default="")
mem_peak = IntegerField(null=True)
time_elapsed = IntegerField(null=True)
class Job(BaseModel):
id_job = CharField(max_length=50, unique=True)
email = CharField()
id_process = IntegerField(null=True)
batch_type = CharField(max_length=20, default="local")
status = CharField(max_length=20, default="submitted")
date_created = DateTimeField()
error = CharField(default="")
mem_peak = IntegerField(null=True)
time_elapsed = IntegerField(null=True)
class Gallery(BaseModel):
job = ForeignKeyField(Job)
name = CharField()
query = CharField()
target = CharField()
picture = CharField()
class Gallery(BaseModel):
job = ForeignKeyField(Job)
name = CharField()
query = CharField()
target = CharField()
picture = CharField()
class Session(BaseModel):
s_id = CharField(max_length=20, unique=True)
date_created = DateTimeField()
upload_folder = CharField(max_length=20)
last_ping = DateTimeField()
status = CharField(default="reset")
keep_active = BooleanField(default=False) # Uploads made by the server must be keep active
@classmethod class Session(BaseModel):
def new(cls, keep_active=False): s_id = CharField(max_length=20, unique=True)
from dgenies.lib.functions import Functions date_created = DateTimeField()
my_s_id = Functions.random_string(20) upload_folder = CharField(max_length=20)
while len(cls.select().where(cls.s_id == my_s_id)) > 0: last_ping = DateTimeField()
status = CharField(default="reset")
keep_active = BooleanField(default=False) # Uploads made by the server must be keep active
@classmethod
def new(cls, keep_active=False):
from dgenies.lib.functions import Functions
my_s_id = Functions.random_string(20) my_s_id = Functions.random_string(20)
upload_folder = Functions.random_string(20) while len(cls.select().where(cls.s_id == my_s_id)) > 0:
tmp_dir = config.upload_folder my_s_id = Functions.random_string(20)
upload_folder_path = os.path.join(tmp_dir, upload_folder)
while os.path.exists(upload_folder_path):
upload_folder = Functions.random_string(20) upload_folder = Functions.random_string(20)
tmp_dir = config.upload_folder
upload_folder_path = os.path.join(tmp_dir, upload_folder) upload_folder_path = os.path.join(tmp_dir, upload_folder)
cls.create(s_id=my_s_id, date_created=datetime.now(), upload_folder=upload_folder, last_ping=datetime.now(), while os.path.exists(upload_folder_path):
keep_active=keep_active) upload_folder = Functions.random_string(20)
return my_s_id upload_folder_path = os.path.join(tmp_dir, upload_folder)
cls.create(s_id=my_s_id, date_created=datetime.now(), upload_folder=upload_folder, last_ping=datetime.now(),
keep_active=keep_active)
return my_s_id
def ask_for_upload(self, change_status=False):
all_asked = Session.select().where((Session.status == "pending") | (Session.status == "active")).\
order_by(Session.date_created)
nb_asked = len(all_asked)
if self.status != "reset":
change_status = False
status = "pending"
if self.status == "active" or (change_status and nb_asked < 5):
status = "active"
self.status = status
self.last_ping = datetime.now()
self.save()
return self.status == "active"
def ask_for_upload(self, change_status=False): def ping(self):
all_asked = Session.select().where((Session.status == "pending") | (Session.status == "active")).\ self.last_ping = datetime.now()
order_by(Session.date_created) self.save()
nb_asked = len(all_asked)
if self.status != "reset":
change_status = False
status = "pending"
if self.status == "active" or (change_status and nb_asked < 5):
status = "active"
self.status = status
self.last_ping = datetime.now()
self.save()
return self.status == "active"
def ping(self): if not Job.table_exists():
self.last_ping = datetime.now() Job.create_table()
self.save()
if not Gallery.table_exists():
Gallery.create_table()
if not Job.table_exists(): if not Session.table_exists():
Job.create_table() Session.create_table()
else:
class Database:
nb_open = 0
def __init__(self):
pass
def __enter__(self):
pass
def __exit__(self, exc_type, exc_val, exc_tb):