Merge branch 'issue#129' into 'master'

Merge branch Issue#129 with master

See merge request !66

src/ng6/analysis.py

@@ -109,7 +109,7 @@ class Analysis (Component):
     ANALYSIS_CONFIG_FILE_NAME = "analysis.cfg"
 
     def __init__(self, name="", description="", software="",
-                 options="", version="", id=None, parent_analysis=None, space_id="default"):
+                 options="", version="", id=None, date=None, parent_analysis=None, space_id="default"):
         """
         Build an Analysis object
           @param name            : the analysis name
@@ -135,7 +135,8 @@ class Analysis (Component):
         self.is_editable = False
         self.space_id = space_id
         self.retention_date = None
-
+        self.date = date
+        
         # Set the temp folder to the ng6 temp folder
         ng6conf = NG6ConfigReader()
         logging.getLogger("Analysis.__init__").debug("ng6conf est chargé")
@@ -677,7 +678,7 @@ class Analysis (Component):
         #try:
         t3mysql = t3MySQLdb()
         [name, date, description, software, options, version] = t3mysql.select_analysis(id)
-        my_analysis = Analysis(name, description, software, options, version, id)
+        my_analysis = Analysis(name, description, software, options, version, id, date)
         if my_analysis == None:
             logging.getLogger("Analysis.get_from_id").error("The analysis id=" + str(id) + " does not exists in the database.")
             raise Exception("The analysis id=" + str(id) + " does not exists in the database.\n")
@@ -702,25 +703,7 @@ class Analysis (Component):
         except :
             pass
 
-    def set_retention(self, retention):
-        """
-        Updates an analysis's retention field
-          @param retention : the new retention value, in (unix) epoch timestamp format
-        """
-        if self.id != None :
-            t3mysql = t3MySQLdb()
-            t3mysql.set_analysis_retention_from_epoch_timestamp(str(self.id), retention)
 
-    
-    def set_directory(self, directory):
-        """
-        Updates an analysis's directory field
-          @param directory : the new directory path
-        """
-        if self.id != None :
-            t3mysql = t3MySQLdb()
-            t3mysql.update_fields('tx_nG6_analyze', str(self.id), ['directory'], [str(directory)] )
-    
     def get_creation_date(self):
         """
         Returns an analysis's crdate value
@@ -736,4 +719,31 @@ class Analysis (Component):
         """
         if self.id != None :
             t3mysql = t3MySQLdb()
-            return t3mysql.select_analysis_directory(str(self.id))
\ No newline at end of file
+            return t3mysql.select_analysis_directory(str(self.id))
+    
+    def change_space (self, space_id ):
+        ng6conf = NG6ConfigReader()
+        
+        old_path = ng6conf.get_save_directory() + self.directory
+        directory_name = os.path.split(old_path)[-1]
+        new_relative_path = os.path.join(ng6conf.get_space_directory(space_id), self.DIRECTORIES_STRUCTURE, directory_name)
+        
+        new_absolute_path =  os.path.join(ng6conf.get_save_directory(), new_relative_path )
+        
+        str_cmd = ""
+        retcode = -1
+        if str(old_path) != str(new_absolute_path):
+            [retcode, str_cmd] = Utils.rsync_getcmd(old_path,new_absolute_path)
+            if retcode != 0 :               
+                raise Exception("Error while trying to rsync " + old_path + " to " + new_absolute_path + "\n" +
+                                "Command : " + str_cmd + "\n" + "Error code : " + retcode + "\n" + str(err) + "\n")
+        else:
+            print ("rsync could not be launched because the source and destination are the same, from " + old_path + " to " + new_absolute_path + "\n")
+        
+        new_retention = ng6conf.get_retention_date(space_id, self.date)
+        
+        t3mysql = t3MySQLdb()
+        t3mysql.update_fields('tx_nG6_analyze', str(self.id), ['directory','retention_date'], [str("/"+new_relative_path),new_retention] )
+            
+        return [retcode, str_cmd]
+    
\ No newline at end of file

src/ng6/config_reader.py

@@ -114,15 +114,20 @@ class NG6ConfigReader(object):
         except:
             raise Exception("Failed when parsing the config file !")
 
-    def get_retention_date(self, space_id="default"):
+    def get_retention_date(self, space_id="default", creation_date = None):
         """
         return the retention corresponding to space_id
           @return: space_dir
         """
+        
         date = None
+        start_date= datetime.datetime.today()
+        if creation_date is not None :
+            start_date=datetime.datetime.fromtimestamp(creation_date)
+            
         try :
             nb_month=self.reader.getint('space_'+space_id,"retention_period")
-            retention_date = time.mktime((datetime.datetime.today()+ relativedelta(months=nb_month)).timetuple())
+            retention_date = time.mktime((start_date+ relativedelta(months=nb_month)).timetuple())
             return retention_date
         except:
             try :
@@ -132,27 +137,7 @@ class NG6ConfigReader(object):
             except:
                 raise Exception("Failed when parsing the config file !")
         raise ValueError("Failed while generating retention date!")
-    
-    def compute_new_retention_date(self, space_id="default", creation_date = None):
-        """
-        return the retention corresponding to space_id and crdate
-          @return: unix epoch timestamp retention value
-        """
-        date = None
-        try :
-            nb_month=self.reader.getint('space_'+space_id,"retention_period")
-            retention_date = time.mktime((datetime.datetime.fromtimestamp(creation_date)+ relativedelta(months=nb_month)).timetuple())
-            logging.getLogger("config_reader.compute_new_retention_date").debug("IN retention period")
-            return retention_date
-        except:
-            try :
-                (d,m,y)=self.reader.get('space_'+space_id,"retention_date").split('/')
-                date = time.mktime(datetime.date(int(y),int(m),int(d)).timetuple())
-                logging.getLogger("config_reader.compute_new_retention_date").debug("IN retention date")
-                return date
-            except:
-                raise Exception("Failed when parsing the config file !")
-        raise ValueError("Failed while generating retention date!")    
+
     
     def get_log_file_path(self):
         """

src/ng6/run.py

@@ -470,4 +470,31 @@ class Run(object):
             t3mysql = t3MySQLdb()
             return t3mysql.select_run_directory(self.id)
         else :
-            raise UnsavedRunError()
\ No newline at end of file
+            raise UnsavedRunError()
+        
+    def change_space (self, space_id ):
+        ng6conf = NG6ConfigReader()
+        
+        old_path = ng6conf.get_save_directory() + self.directory
+        directory_name = os.path.split(old_path)[-1]
+        new_relative_path = os.path.join(ng6conf.get_space_directory(space_id), self.DIRECTORIES_STRUCTURE, directory_name)
+        
+        new_absolute_path =  os.path.join(ng6conf.get_save_directory(), new_relative_path )
+        
+        str_cmd = ""
+        retcode = -1
+        if str(old_path) != str(new_absolute_path):
+            [retcode, str_cmd] = Utils.rsync_getcmd(old_path,new_absolute_path)
+            if retcode != 0 :   
+                raise Exception("Error while trying to rsync " + old_path + " to " + new_absolute_path + "\n" +
+                                "Command : " + str_cmd + "\n" + "Error code : " + retcode + "\n" + str(err) + "\n")
+        else:
+            print ("rsync could not be launched because the source and destination are the same, from " + old_path + " to " + new_absolute_path + "\n")
+        
+        new_retention = ng6conf.get_retention_date(space_id, self.date)
+        
+        t3mysql = t3MySQLdb()
+        t3mysql.update_fields('tx_nG6_run', str(self.id), ['directory','retention_date'], [str("/"+new_relative_path),new_retention] )
+            
+        return [retcode, str_cmd]
+        

src/ng6/utils.py

@@ -32,7 +32,7 @@ import re
 import base64
 import pty
 import sys
-
+from subprocess import call
 
 class Utils(object):
     """
@@ -436,8 +436,24 @@ class Utils(object):
                 sources.append(e)
                 destinations.append(dest_directories[i])
         return sources, destinations 
+    
+    @staticmethod    
+    def rsync_getcmd( source, destination):
+        if not source.endswith("/"):
+            source+="/"
+        cmd = ["rsync", "-avh", "--perms", "--times", "--remove-sent-files", source, destination]
         
+        print("\n".join(cmd))
         
+        retcode = call(cmd , shell=False)
+        #Delete the old directory if empty
+        if retcode >= 0 and os.path.isdir(source) and not os.listdir(source):
+            try:
+                os.rmdir(source)
+            except Exception as err:
+                raise Exception("Error while deleting " + source + "\n" + str(err) + "\n")
+        return [ retcode, " ".join(cmd)]
+    
     def get_purge_demand_directories(demand_ids):
         from ng6.t3MySQLdb import t3MySQLdb
         t3mysql = t3MySQLdb()

workflows/switch_project_space_id/__init__.py

-#
-# Copyright (C) 2015 INRA
-# 
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-# 
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-# 
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-#
-
-import os
-import sys
-import logging
-import shutil
-
-from ng6.ng6workflow import BasicNG6Workflow
-from ng6.config_reader import NG6ConfigReader
-from ng6.utils import Utils
-
-from ng6.project import Project
-from ng6.run import Run
-from ng6.analysis import Analysis
-
-from ng6.t3MySQLdb import t3MySQLdb
-
-class SwitchProjectSpaceId (BasicNG6Workflow):
-
-    def get_description(self):
-        return """This workflow allows you to migrate a project from a space_id to another.  
-        It updates the project's space_id in DB, moves the runs files, updates the runs and analyzes's files paths, and updates the corresponding retentions."""
-    
-    def define_parameters(self, function="process"):
-        self.add_parameter('project_id', 'The project id', type=int, required = True)
-        self.add_parameter('space_id', 'The new space_id', required = True)
-        
-    def process(self):
-        logging.getLogger("SwitchProjectSpaceId.process").debug("Begin SwitchProjectSpaceId workflow.")
-        #self.space_id = self.space_id
-        #self.project_id = self.project_id
-        self.project_id = int(self.project_id)
-        
-        if self.project_id and self.space_id :
-            logging.getLogger("SwitchProjectSpaceId.process").debug("project_id =" + str(self.project_id)  + " , new space_id = " + str(self.space_id) )
-            project = Project.get_from_id(self.project_id)
-            old_space_id = project.space_id
-            
-            #We raise an error and return None if the project already has the given space_id
-            if str(old_space_id) == str(self.space_id) :
-                raise Exception("The project " + project.name + " already belongs to the space_id '" + str(self.space_id) + "'" )
-                return None
-            
-            #Retrieve the old and new directories
-            ng6conf = NG6ConfigReader()
-            old_directory = ng6conf.get_space_directory(old_space_id)
-            try:
-                new_directory = ng6conf.get_space_directory(self.space_id)
-            except Exception as err:
-                logging.getLogger("SwitchProjectSpaceId.process").error("Error while searching space_id '" + str(self.space_id) + "' in application.properties, it certainly doesn't exist. \n" + str(err))
-                raise Exception("Error while searching space_id '" + str(self.space_id) + "' in application.properties, it certainly doesn't exist. \n" + str(err))
-            
-            base_path = ng6conf.get_save_directory()
-            
-            #Process for the project's runs
-            runs_ids = project.get_project_run_ids()
-            for run_id in runs_ids :
-                logging.getLogger("SwitchProjectSpaceId.process").debug("Migrating run " + str(run_id) )
-                run = Run.get_from_id(run_id)
-                
-                path_old_run_directory = base_path + run.get_directory()
-                path_new_run_directory = path_old_run_directory.replace(old_directory,new_directory)
-                
-                #Update the directory in DB
-                new_run_directory = run.get_directory()
-                new_run_directory = new_run_directory.replace(old_directory, new_directory)
-                
-                #Moving the files
-                #if not os.path.isdir(path_new_run_directory) :
-                #    os.mkdir(path_new_run_directory,0o775)
-                
-                logging.getLogger("SwitchProjectSpaceId.process").debug("Migrating run files from " + str(path_old_run_directory) + " to " + str(path_new_run_directory) )
-                try:
-                    shutil.move(path_old_run_directory, path_new_run_directory)
-                except Exception as err:
-                    logging.getLogger("SwitchProjectSpaceId.process").error("Error while moving " + path_old_run_directory + " to " + path_new_run_directory + " \n" + str(err))
-                    raise Exception("Error while moving " + path_old_run_directory + " to " + path_new_run_directory + " \n" + str(err))
-                
-                #Delete the old directory if empty
-                if os.path.isdir(path_old_run_directory) and not os.listdir(path_old_run_directory) :
-                    os.rmdir(path_old_run_directory)
-                
-                #Update the retention in DB
-                old_retention = run.get_creation_date()
-                new_retention = ng6conf.compute_new_retention_date(self.space_id,old_retention)
-                run.set_retention(new_retention)
-                run.set_directory(new_run_directory)                
-                
-            #Process for the project's analyzes
-            analyzes_ids = project.get_project_analyzes_ids()
-            for analysis_id in analyzes_ids :
-                logging.getLogger("SwitchProjectSpaceId.process").debug("Migrating analysis " + str(analysis_id) )
-                analysis = Analysis.get_from_id(analysis_id)
-                
-                path_old_analysis_directory = base_path + analysis.get_directory()
-                path_new_analysis_directory = path_old_analysis_directory.replace(old_directory,new_directory)
-                
-                #Update the directory in DB
-                new_analysis_directory = analysis.get_directory()
-                new_analysis_directory = new_analysis_directory.replace(old_directory, new_directory)
-                
-                #Moving the files
-                #if not os.path.isdir(path_new_analysis_directory) :
-                #    os.mkdir(path_new_analysis_directory,0o775) 
-                
-                logging.getLogger("SwitchProjectSpaceId.process").debug("Migrating analysis files from " + str(path_old_analysis_directory) + " to " + str(path_new_analysis_directory) )
-                try:
-                    shutil.move(path_old_analysis_directory, path_new_analysis_directory)
-                except Exception as err:
-                    logging.getLogger("SwitchProjectSpaceId.process").error("Error while moving " + path_old_analysis_directory + " to " + path_new_analysis_directory + " \n" + str(err))
-                    raise Exception("Error while moving " + path_old_analysis_directory + " to " + path_new_analysis_directory + " \n" + str(err))
-                
-                #Delete the old directory if empty
-                if os.path.isdir(path_old_analysis_directory) and not os.listdir(path_old_analysis_directory) :
-                    os.rmdir(path_old_analysis_directory)
-                
-                #Update the retention in DB
-                old_retention = analysis.get_creation_date()
-                new_retention = ng6conf.compute_new_retention_date(self.space_id,old_retention)
-                analysis.set_retention(new_retention)
-                analysis.set_directory(new_analysis_directory)
-                
-            #Process for the project's run's analyzes
-            run_analyzes_ids = project.get_project_runs_analyzes_ids()
-            for analysis_id in run_analyzes_ids :
-                logging.getLogger("SwitchProjectSpaceId.process").debug("Migrating analysis " + str(analysis_id) )
-                analysis = Analysis.get_from_id(analysis_id)
-                
-                path_old_analysis_directory = base_path + analysis.get_directory()
-                path_new_analysis_directory = path_old_analysis_directory.replace(old_directory,new_directory)
-                
-                #Update the directory in DB
-                new_analysis_directory = analysis.get_directory()
-                new_analysis_directory = new_analysis_directory.replace(old_directory, new_directory)
-                
-                #Moving the files
-                #if not os.path.isdir(path_new_analysis_directory) :
-                #    os.mkdir(path_new_analysis_directory,0o775) 
-                
-                logging.getLogger("SwitchProjectSpaceId.process").debug("Migrating analysis files from " + str(path_old_analysis_directory) + " to " + str(path_new_analysis_directory) )
-                try:
-                    shutil.move(path_old_analysis_directory, path_new_analysis_directory)
-                except Exception as err:
-                    logging.getLogger("SwitchProjectSpaceId.process").error("Error while moving " + path_old_analysis_directory + " to " + path_new_analysis_directory + " \n" + str(err))
-                    raise Exception("Error while moving " + path_old_analysis_directory + " to " + path_new_analysis_directory + " \n" + str(err))
-                
-                #Delete the old directory if empty
-                if os.path.isdir(path_old_analysis_directory) and not os.listdir(path_old_analysis_directory) :
-                    os.rmdir(path_old_analysis_directory)
-                
-                #Update the retention in DB
-                old_retention = analysis.get_creation_date()
-                new_retention = ng6conf.compute_new_retention_date(self.space_id,old_retention)
-                analysis.set_retention(new_retention)
-                analysis.set_directory(new_analysis_directory)
-            
-            #Update to the projcet's space_id
-            project.update_space_id(self.space_id)
\ No newline at end of file

workflows/switch_space_id/components/move_project.py

+#
+# Copyright (C) 2015 INRA
+# 
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+# 
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+# 
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+import logging
+from jflow.component import Component
+from workflows.types import ng6space,existingprojectid
+from ng6.t3MySQLdb import t3MySQLdb
+
+def migrate_project (project_id, new_space_id, output):
+    import os
+    from subprocess import call
+    import logging
+
+    from ng6.config_reader import NG6ConfigReader
+    from ng6.analysis import Analysis
+    from ng6.project import Project
+    from ng6.run import Run
+
+    with open( output,'w') as fh :
+        ng6conf = NG6ConfigReader()
+        base_path = ng6conf.get_save_directory()
+        
+        project = Project.get_from_id(project_id)
+        
+        logging.getLogger("migrate_project").debug("Migrating project '" + str(project.name) + "' (" + str(project.id) + ") to " + str(new_space_id) + "\n")
+        fh.write("Migrating project '" + str(project.name) + "' (" + str(project.id) + ") to " + str(new_space_id) + "\n")
+        
+        old_space_id = project.space_id
+        fh.write("old_space_id = " + old_space_id + ", new_space_id = " + new_space_id + "\n")
+        
+        str_cmds = []
+        runs = project.get_runs()
+        for run in runs :
+            fh.write("Migrating run '" + run.name + "' (" + str(run.id) + ")" + " to " + str(new_space_id) + "\n")
+            str_cmds.append(run.change_space(new_space_id) )
+            fh.write("Migrating successful! \n")
+            
+            run_analyzes = run.get_analysis()
+            for analysis in run_analyzes :
+                fh.write("Migrating analysis '" + analysis.name + "' (" + str(analysis.id) + ")" + " to " + str(new_space_id) + "\n")
+                str_cmds.append(analysis.change_space(new_space_id) )
+                fh.write("Migrating successful! \n")
+            
+        analyzes = project.get_analysis()
+        for analysis in analyzes :
+            fh.write("Migrating analysis '" + analysis.name + "' (" + str(analysis.id) + ")" + " to " + str(new_space_id) + "\n")
+            str_cmds.append(analysis.change_space(new_space_id) )
+            fh.write("Migrating successful! \n")
+        
+        for command in str_cmds:
+            fh.write("Command launched : "+ str(command[1]) + "\n" )
+            fh.write("Returned code : "+ str(command[0]) + "\n" )
+        
+        project.update_space_id(new_space_id)
+        
+
+class MoveProject (Component):
+    
+    def define_parameters(self, project_id, space_id,):
+        self.add_parameter('project_id', 'The project id', type = existingprojectid, required = True, default = project_id )
+        self.add_parameter('space_id', 'The new space_id', type = ng6space, required = True, default = space_id)
+        self.add_output_file("stdout", "MoveProject stdout", filename="MoveProject.stdout")
+        
+    def process(self):
+        logging.getLogger("MoveProject.process").debug(" Begin MoveProject Component.\n")
+        logging.getLogger("MoveProject.process").debug(" self.space_id="+str(self.space_id)+"\n")
+        logging.getLogger("MoveProject.process").debug(" self.project_id="+str(self.project_id)+"\n")
+        logging.getLogger("MoveProject.process").debug(" stdout = "+str(self.stdout)+"\n")
+        
+        self.add_python_execution(migrate_project, cmd_format="{EXE} {ARG} {OUT}",
+                                      outputs = self.stdout, arguments = [self.project_id, self.space_id])
+        #self.add_python_execution(migrate_project, cmd_format="{EXE} {ARG}",
+        #                              arguments = [self.project_id, self.space_id])
+        

workflows/types.py

@@ -97,7 +97,8 @@ def ng6space(val):
         ng6conf.get_space_directory(val)
     except :
         raise argparse.ArgumentTypeError("The space name '%s' is not define in application.properties" % val)
-
+    return val
+    
 def existingrun(id):
     try:
         t3mysql = t3MySQLdb()