Commit da53fd08 authored by Celine Noirot's avatar Celine Noirot Committed by Romain Therville

Add retention date management

parent 09e41229
......@@ -134,6 +134,7 @@ class Analysis (Component):
self.project = None
self.is_editable = False
self.space_id = space_id
self.retention_date = None
# Set the temp folder to the ng6 temp folder
ng6conf = NG6ConfigReader()
......@@ -182,8 +183,8 @@ class Analysis (Component):
if not os.path.isdir(save_dir) and not os.path.isdir(work_dir):
break
directory_name = uuid.uuid4().hex[:9]
self.directory = "/" + os.path.join(ng6conf.get_space_directory(self.space_id), self.DIRECTORIES_STRUCTURE, directory_name)
self.retention_date = ng6conf.get_retention_date(self.space_id)
# first create the output directory
if not os.path.isdir(self.output_directory):
......@@ -320,7 +321,8 @@ class Analysis (Component):
logging.getLogger("Analysis.save").debug("Doing the add analysis")
self.id = t3mysql.add_analysis(self.get_template(), self.name, self.description, self.admin_login,
datetime.date.today(), self.directory, self.software, self.version,
self.options, self.is_editable, parent_uid=parent_uid)
self.options, self.retention_date, self.retention_date,
self.is_editable, parent_uid=parent_uid)
logging.getLogger("Analysis.save").debug("add_analysis done")
# Then add all results link to this analysis
for file in self.results:
......
......@@ -26,6 +26,11 @@ import os
from configparser import ConfigParser
from .utils import *
import logging
from dateutil.relativedelta import relativedelta
import datetime
import time
class NG6ConfigReader(object):
"""
Class NG6ConfigReader: this object read the config file and return the different configuration values
......@@ -56,7 +61,7 @@ class NG6ConfigReader(object):
db_params.append(self.reader.get('database', 'dbname'))
return db_params
except :
raise Error("Failed when parsing the config file !")
raise Exception("Failed when parsing the config file - get_typo3_db_params !")
def get_pid(self):
......@@ -75,7 +80,7 @@ class NG6ConfigReader(object):
try:
return self.reader.get('storage', 'work_directory')
except :
raise Error("Failed when parsing the config file !")
raise Exception("Failed when parsing the config file - get_work_directory! ")
def get_tmp_directory(self):
......@@ -86,7 +91,7 @@ class NG6ConfigReader(object):
try:
return self.reader.get('storage', 'tmp_directory')
except :
raise Error("Failed when parsing the config file !")
raise Exception("Failed when parsing the config file get_tmp_directory !")
def get_save_directory(self):
......@@ -97,7 +102,7 @@ class NG6ConfigReader(object):
try:
return self.reader.get('storage', 'save_directory')
except :
raise Error("Failed when parsing the config file !")
raise Exception("Failed when parsing the config file get_save_directory !")
def get_space_directory(self, space_id="default"):
"""
......@@ -106,9 +111,28 @@ class NG6ConfigReader(object):
"""
try:
return self.reader.get('space_'+space_id, 'dir_name').strip("/")
except :
raise Error("Failed when parsing the config file !")
except:
raise Exception("Failed when parsing the config file !")
def get_retention_date(self, space_id="default"):
"""
return the directory corresponding to space_id
@return: space_dir
"""
date = None
try :
nb_month=self.reader.getint('space_'+space_id,"retention_period")
retention_date = time.mktime((datetime.datetime.today()+ relativedelta(months=nb_month)).timetuple())
return retention_date
except:
try :
(d,m,y)=self.reader.get('space_'+space_id,"retention_date").split('/')
date = time.mktime(datetime.date(int(y),int(m),int(d)).timetuple())
return date
except:
raise Exception("Failed when parsing the config file !")
raise ValueError("Failed while gnerating retention date!")
def get_log_file_path(self):
"""
return the log file path
......@@ -117,7 +141,7 @@ class NG6ConfigReader(object):
try:
return self.reader.get('storage', 'log_file')
except :
raise Error("Failed when parsing the config file !")
raise Exception("Failed when parsing the config file !")
def get_log_level(self):
"""
return the level of log
......@@ -126,7 +150,7 @@ class NG6ConfigReader(object):
try:
return self.reader.get('storage', 'log_level')
except :
raise Error("Failed when parsing the config file !")
raise Exception("Failed when parsing the config file !")
def get_454_mids(self):
......@@ -141,7 +165,7 @@ class NG6ConfigReader(object):
mid_array[mid[0]] = mid[1]
return mid_array
except :
raise Error("Failed when parsing the config file !")
raise Exception("Failed when parsing the config file get_454_mids !")
def get_workflow_filters(self):
"""
......@@ -162,7 +186,7 @@ class NG6ConfigReader(object):
try:
return self.reader.get('ui', 'user_base_directory')
except :
raise Error("Could not retrieve server directory !")
raise Exception("Could not retrieve server directory get_user_base_directory !")
def get_server_parameters(self):
......@@ -170,14 +194,15 @@ class NG6ConfigReader(object):
return the server parameters
@return: server_socket_host, server_socket_port, server_socket_protocol
"""
# try:
server_params = []
server_params.append(self.reader.get('global', 'server_socket_host'))
server_params.append(self.reader.get('global', 'server_socket_port'))
server_params.append(self.reader.get("global", 'server_socket_protocol'))
return server_params
# except :
# raise Error("Failed when parsing the config file !")
try:
server_params = []
server_params.append(self.reader.get('global', 'server_socket_host'))
server_params.append(self.reader.get('global', 'server_socket_port'))
server_params.append(self.reader.get("global", 'server_socket_protocol'))
return server_params
except :
raise Exception("Failed when parsing the config file get_server_parameters !")
def get_retention_time(self):
"""
return the retention time parameter
......@@ -186,4 +211,4 @@ class NG6ConfigReader(object):
try:
return self.reader.get('global', 'retention_time')
except :
raise Error("Could not retrieve retention_time !")
raise Exception("Failed when parsing the config file get_retention_time!")
......@@ -67,6 +67,8 @@ class Run(object):
self.raw_files = []
self.admin_login = None
self.space_id = space_id
self.retention_date = None
# Set the temp folder to the ng6 temp folder
ng6conf = NG6ConfigReader()
tempfile.tempdir = ng6conf.get_tmp_directory()
......@@ -95,6 +97,7 @@ class Run(object):
# Then make directories
os.makedirs(work_dir, 0o751)
self.directory = "/" + os.path.join(ng6conf.get_space_directory(self.space_id),self.DIRECTORIES_STRUCTURE, directory_name)
self.retention_date = ng6conf.get_retention_date(self.space_id)
def process_raw_files(self, total_size, total_seq):
"""
......@@ -366,7 +369,7 @@ class Run(object):
# First add the run into the database
t3mysql = t3MySQLdb()
self.id = t3mysql.add_run(self.name, self.date, self.directory, self.species, self.data_nature,
self.type, 0, 0, self.description, self.sequencer, self.admin_login)
self.type, 0, 0, self.description, self.sequencer, self.admin_login, self.retention_date)
return self.id
def __get_work_directory(self):
......
......@@ -22,7 +22,9 @@ __version__ = '1.0'
__email__ = 'support.bioinfo.genotoul@inra.fr'
__status__ = 'beta'
import datetime, time, calendar, logging
import time
import calendar
import logging
import pymysql
import pymysql.cursors
import collections
......@@ -293,7 +295,7 @@ class t3MySQLdb(object):
# Run functions
def add_run(self, name, date, directory, species, data_nature,
type, nb_sequences, full_seq_size, description, sequencer, cruser_login, hidden=1):
type, nb_sequences, full_seq_size, description, sequencer, cruser_login, retention_date, hidden=1):
"""
Add to the database a new run
@param name : the run name
......@@ -307,12 +309,12 @@ class t3MySQLdb(object):
@param description : the run description
@param sequencer : the sequencer used
@param cruser_login : the create user login
@param retention_date: the retention date for storage.
@param hidden : is the analysis hidden
"""
logging.getLogger("t3MySQLdb.add_run").debug("Inserting run with date of run =" + str(date))
cruser_id = self.get_user_id(cruser_login)
crdate = str(time.time()).split(".")[0]
retention_date = time.mktime((datetime.date.today() + datetime.timedelta(days=365*int(self.cfg_reader.get_retention_time()))).timetuple())
sql = "INSERT INTO tx_nG6_run (pid, tstamp, crdate, cruser_id, name, date, directory, species, data_nature,\
type, nb_sequences, full_seq_size, description, hidden, sequencer, retention_date, data_state, purged_date, purged_size, mail_sent_date) VALUES ('%s', '%s', '%s', %s, '%s', '%s','%s',\
'%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s')" % ( self.cfg_reader.get_pid(), crdate, crdate, cruser_id, self.esc_q(name),
......@@ -671,7 +673,7 @@ class t3MySQLdb(object):
#--------------------------------------------------------------------
# Analyze functions
def add_analysis(self, aclass, name, description, cruser_login, date, directory, software, version, params, is_editable, hidden=1, parent_uid=0):
def add_analysis(self, aclass, name, description, cruser_login, date, directory, software, version, params, is_editable, retention_date, hidden=1, parent_uid=0):
"""
Update an Analyze
@param aclass : the class of the analysis
......@@ -683,13 +685,13 @@ class t3MySQLdb(object):
@param software : the software name used for the analysis
@param version : the software version used for the analysis
@param params : the software params used
@param retention_date:the retention date for storage period
@param hidden : is the analysis hidden
@param parent_uid : the analysis parent uid
@param is_editable : True if analysis has been added by an user
"""
cruser_id = self.get_user_id(cruser_login)
crdate = str(time.time()).split(".")[0]
retention_date = time.mktime((datetime.date.today() + datetime.timedelta(days=365*int(self.cfg_reader.get_retention_time()))).timetuple())
if is_editable : analysis_is_editable = 1
else : analysis_is_editable = 0
......
......@@ -19,7 +19,7 @@ from jflow.workflow import Workflow
from ng6.project import Project
from workflows.types import ng6adminlogin, uniqproject, ng6sspace
from workflows.types import ng6adminlogin, uniqproject
class AddProject (Workflow):
......@@ -30,7 +30,7 @@ class AddProject (Workflow):
self.add_parameter("admin_login", "The login of a ng6 administrator", required = True, type = ng6adminlogin)
self.add_parameter("project_name", "Give a name to your project (has to be unique)", flag = "--name", required = True, type = uniqproject)
self.add_parameter("project_description", "Give a description to your project", flag = "--description", required = True)
self.add_parameter("space", "Where to store data", flag = "--space", default="default", type = ng6sspace)
self.add_parameter("space", "Where to store data", flag = "--space", default="default")
def process(self):
project = Project(self.project_name, self.project_description, self.admin_login, None, self.space)
......
......@@ -91,12 +91,6 @@ def existingprojectid(id):
except :
raise argparse.ArgumentTypeError("The project id '%s' does not exists" % id)
def ng6sspace(val):
try :
NG6ConfigReader.get_space_directory(self.val)
except :
raise argparse.ArgumentTypeError("The space name '%s' is not define in application.properties" % val)
def existingrun(id):
try:
t3mysql = t3MySQLdb()
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment