Add retention date management

src/ng6/analysis.py

@@ -134,6 +134,7 @@ class Analysis (Component):
         self.project = None
         self.is_editable = False
         self.space_id = space_id
+        self.retention_date = None
 
         # Set the temp folder to the ng6 temp folder
         ng6conf = NG6ConfigReader()
@@ -182,8 +183,8 @@ class Analysis (Component):
             if not os.path.isdir(save_dir) and not os.path.isdir(work_dir):
                 break
             directory_name = uuid.uuid4().hex[:9]
-
         self.directory = "/" + os.path.join(ng6conf.get_space_directory(self.space_id), self.DIRECTORIES_STRUCTURE, directory_name)
+        self.retention_date = ng6conf.get_retention_date(self.space_id)
 
         # first create the output directory
         if not os.path.isdir(self.output_directory):
@@ -320,7 +321,8 @@ class Analysis (Component):
         logging.getLogger("Analysis.save").debug("Doing the add analysis")
         self.id = t3mysql.add_analysis(self.get_template(), self.name, self.description, self.admin_login,
                                       datetime.date.today(), self.directory, self.software, self.version,
-                                      self.options, self.is_editable, parent_uid=parent_uid)
+                                      self.options, self.retention_date, self.retention_date, 
+                                      self.is_editable, parent_uid=parent_uid)
         logging.getLogger("Analysis.save").debug("add_analysis done")
         # Then add all results link to this analysis
         for file in self.results:

src/ng6/config_reader.py

@@ -26,6 +26,11 @@ import os
 from configparser import ConfigParser
 from .utils import *
 import logging
+
+from dateutil.relativedelta import relativedelta
+import datetime
+import time
+
 class NG6ConfigReader(object):
     """
     Class NG6ConfigReader: this object read the config file and return the different configuration values
@@ -56,7 +61,7 @@ class NG6ConfigReader(object):
             db_params.append(self.reader.get('database', 'dbname'))
             return db_params
         except :
-            raise Error("Failed when parsing the config file !")
+            raise Exception("Failed when parsing the config file - get_typo3_db_params !")
 
 
     def get_pid(self):
@@ -75,7 +80,7 @@ class NG6ConfigReader(object):
         try:
             return self.reader.get('storage', 'work_directory')
         except :
-            raise Error("Failed when parsing the config file !")
+            raise Exception("Failed when parsing the config file - get_work_directory! ")
 
 
     def get_tmp_directory(self):
@@ -86,7 +91,7 @@ class NG6ConfigReader(object):
         try:
             return self.reader.get('storage', 'tmp_directory')
         except :
-            raise Error("Failed when parsing the config file !")
+            raise Exception("Failed when parsing the config file get_tmp_directory !")
 
 
     def get_save_directory(self):
@@ -97,7 +102,7 @@ class NG6ConfigReader(object):
         try:
             return self.reader.get('storage', 'save_directory')
         except :
-            raise Error("Failed when parsing the config file !")
+            raise Exception("Failed when parsing the config file get_save_directory !")
 
     def get_space_directory(self, space_id="default"):
         """
@@ -106,9 +111,28 @@ class NG6ConfigReader(object):
         """
         try:
             return self.reader.get('space_'+space_id, 'dir_name').strip("/")
-        except :
-            raise Error("Failed when parsing the config file !")
+        except:
+            raise Exception("Failed when parsing the config file !")
 
+    def get_retention_date(self, space_id="default"):
+        """
+        return the directory corresponding to space_id
+          @return: space_dir
+        """
+        date = None
+        try :
+            nb_month=self.reader.getint('space_'+space_id,"retention_period")
+            retention_date = time.mktime((datetime.datetime.today()+ relativedelta(months=nb_month)).timetuple())
+            return retention_date
+        except:
+            try :
+                (d,m,y)=self.reader.get('space_'+space_id,"retention_date").split('/')
+                date = time.mktime(datetime.date(int(y),int(m),int(d)).timetuple())
+                return date
+            except:
+                raise Exception("Failed when parsing the config file !")
+        raise ValueError("Failed while gnerating retention date!")
+    
     def get_log_file_path(self):
         """
         return the log file path
@@ -117,7 +141,7 @@ class NG6ConfigReader(object):
         try:
             return self.reader.get('storage', 'log_file')
         except :
-            raise Error("Failed when parsing the config file !")
+            raise Exception("Failed when parsing the config file !")
     def get_log_level(self):
         """
         return the level of log
@@ -126,7 +150,7 @@ class NG6ConfigReader(object):
         try:
             return self.reader.get('storage', 'log_level')
         except :
-            raise Error("Failed when parsing the config file !")
+            raise Exception("Failed when parsing the config file !")
 
 
     def get_454_mids(self):
@@ -141,7 +165,7 @@ class NG6ConfigReader(object):
                 mid_array[mid[0]] = mid[1]
             return mid_array
         except :
-            raise Error("Failed when parsing the config file !")
+            raise Exception("Failed when parsing the config file get_454_mids !")
 
     def get_workflow_filters(self):
         """
@@ -162,7 +186,7 @@ class NG6ConfigReader(object):
         try:
             return self.reader.get('ui', 'user_base_directory')
         except :
-            raise Error("Could not retrieve server directory !")
+            raise Exception("Could not retrieve server directory get_user_base_directory !")
 
 
     def get_server_parameters(self):
@@ -170,14 +194,15 @@ class NG6ConfigReader(object):
         return the server parameters
           @return:    server_socket_host, server_socket_port, server_socket_protocol
         """
-#        try:
-        server_params = []
-        server_params.append(self.reader.get('global', 'server_socket_host'))
-        server_params.append(self.reader.get('global', 'server_socket_port'))
-        server_params.append(self.reader.get("global", 'server_socket_protocol'))
-        return server_params
-#        except :
-#            raise Error("Failed when parsing the config file !")
+        try:
+            server_params = []
+            server_params.append(self.reader.get('global', 'server_socket_host'))
+            server_params.append(self.reader.get('global', 'server_socket_port'))
+            server_params.append(self.reader.get("global", 'server_socket_protocol'))
+            return server_params
+        except :
+            raise Exception("Failed when parsing the config file get_server_parameters !")
+        
     def get_retention_time(self):
         """
         return the retention time parameter
@@ -186,4 +211,4 @@ class NG6ConfigReader(object):
         try:
             return self.reader.get('global', 'retention_time')
         except :
-            raise Error("Could not retrieve retention_time !")
+            raise Exception("Failed when parsing the config file get_retention_time!")

src/ng6/run.py

@@ -67,6 +67,8 @@ class Run(object):
         self.raw_files = []
         self.admin_login = None
         self.space_id = space_id
+        self.retention_date = None
+        
         # Set the temp folder to the ng6 temp folder
         ng6conf = NG6ConfigReader()
         tempfile.tempdir = ng6conf.get_tmp_directory()
@@ -95,6 +97,7 @@ class Run(object):
             # Then make directories
             os.makedirs(work_dir, 0o751)
             self.directory = "/" + os.path.join(ng6conf.get_space_directory(self.space_id),self.DIRECTORIES_STRUCTURE, directory_name)
+            self.retention_date = ng6conf.get_retention_date(self.space_id)
 
     def process_raw_files(self, total_size, total_seq):
         """
@@ -366,7 +369,7 @@ class Run(object):
         # First add the run into the database
         t3mysql = t3MySQLdb()
         self.id = t3mysql.add_run(self.name, self.date, self.directory, self.species, self.data_nature,
-                                  self.type, 0, 0, self.description, self.sequencer, self.admin_login)
+                                  self.type, 0, 0, self.description, self.sequencer, self.admin_login, self.retention_date)
         return self.id
 
     def __get_work_directory(self):

src/ng6/t3MySQLdb.py

@@ -22,7 +22,9 @@ __version__ = '1.0'
 __email__ = 'support.bioinfo.genotoul@inra.fr'
 __status__ = 'beta'
 
-import datetime, time, calendar, logging
+import time
+import calendar
+import logging
 import pymysql
 import pymysql.cursors
 import collections
@@ -293,7 +295,7 @@ class t3MySQLdb(object):
     # Run functions
 
     def add_run(self, name, date, directory, species, data_nature,
-                type, nb_sequences, full_seq_size, description, sequencer, cruser_login, hidden=1):
+                type, nb_sequences, full_seq_size, description, sequencer, cruser_login, retention_date, hidden=1):
         """
         Add to the database a new run
           @param name          : the run name
@@ -307,12 +309,12 @@ class t3MySQLdb(object):
           @param description   : the run description
           @param sequencer     : the sequencer used
           @param cruser_login  : the create user login
+          @param retention_date: the retention date for storage.
           @param hidden        : is the analysis hidden
         """
         logging.getLogger("t3MySQLdb.add_run").debug("Inserting run with date of run =" + str(date))
         cruser_id = self.get_user_id(cruser_login)
         crdate = str(time.time()).split(".")[0]
-        retention_date = time.mktime((datetime.date.today() + datetime.timedelta(days=365*int(self.cfg_reader.get_retention_time()))).timetuple())
         sql = "INSERT INTO tx_nG6_run (pid, tstamp, crdate, cruser_id, name, date, directory, species, data_nature,\
         type, nb_sequences, full_seq_size, description, hidden, sequencer, retention_date, data_state, purged_date, purged_size, mail_sent_date) VALUES ('%s', '%s', '%s', %s, '%s', '%s','%s',\
         '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s')" % ( self.cfg_reader.get_pid(), crdate, crdate, cruser_id, self.esc_q(name),
@@ -671,7 +673,7 @@ class t3MySQLdb(object):
     #--------------------------------------------------------------------
     # Analyze functions
 
-    def add_analysis(self, aclass, name, description, cruser_login, date, directory, software, version, params, is_editable, hidden=1, parent_uid=0):
+    def add_analysis(self, aclass, name, description, cruser_login, date, directory, software, version, params, is_editable, retention_date, hidden=1, parent_uid=0):
         """
         Update an Analyze
           @param aclass       : the class of the analysis
@@ -683,13 +685,13 @@ class t3MySQLdb(object):
           @param software     : the software name used for the analysis
           @param version      : the software version used for the analysis
           @param params       : the software params used
+          @param retention_date:the retention date for storage period 
           @param hidden       : is the analysis hidden
           @param parent_uid   : the analysis parent uid
           @param is_editable : True if analysis has been added by an user
         """
         cruser_id = self.get_user_id(cruser_login)
         crdate = str(time.time()).split(".")[0]
-        retention_date = time.mktime((datetime.date.today() + datetime.timedelta(days=365*int(self.cfg_reader.get_retention_time()))).timetuple())
 
         if is_editable : analysis_is_editable = 1
         else : analysis_is_editable = 0

workflows/addproject/__init__.py

@@ -19,7 +19,7 @@ from jflow.workflow import Workflow
 
 from ng6.project import Project
 
-from workflows.types import ng6adminlogin, uniqproject, ng6sspace
+from workflows.types import ng6adminlogin, uniqproject
 
 class AddProject (Workflow):
 
@@ -30,7 +30,7 @@ class AddProject (Workflow):
         self.add_parameter("admin_login", "The login of a ng6 administrator", required = True, type = ng6adminlogin)
         self.add_parameter("project_name", "Give a name to your project (has to be unique)", flag = "--name", required = True, type = uniqproject)
         self.add_parameter("project_description", "Give a description to your project", flag = "--description", required = True)
-        self.add_parameter("space", "Where to store data", flag = "--space", default="default", type = ng6sspace)
+        self.add_parameter("space", "Where to store data", flag = "--space", default="default")
 
     def process(self):
         project = Project(self.project_name, self.project_description, self.admin_login, None, self.space)

workflows/types.py

@@ -91,12 +91,6 @@ def existingprojectid(id):
     except :
         raise argparse.ArgumentTypeError("The project id '%s' does not exists" % id)
         
-def ng6sspace(val):
-    try :
-        NG6ConfigReader.get_space_directory(self.val)
-    except :
-        raise argparse.ArgumentTypeError("The space name '%s' is not define in application.properties" % val)
-
 def existingrun(id):
     try:
         t3mysql = t3MySQLdb()