Big update for the switch_space_id workflow

-The workflow now handles empty (no data) runs and analyzes, and more
errors.
-A lot of user cases have been tested.

issue #129

src/ng6/analysis.py

@@ -743,7 +743,7 @@ class Analysis (Component):
                 raise Exception("Error while trying to rsync " + old_path + " to " + new_absolute_path + "\n" +
                                 "Command : " + str_cmd + "\n" + "Error code : " + str(retcode) + "\n")
         else:
-            print ("rsync could not be launched because the source and destination are the same, from " + old_path + " to " + new_absolute_path + "\n")
+            str_cmd = "Source and destination directories are the same : " + old_path + " , ignored.\n"
         
         new_retention = ng6conf.get_retention_date(space_id, self.date)
         
@@ -751,4 +751,4 @@ class Analysis (Component):
         t3mysql.update_fields('tx_nG6_analyze', str(self.id), ['directory','retention_date'], [str("/"+new_relative_path),new_retention] )
             
         return [retcode, str_cmd]
-    
\ No newline at end of file
+    

src/ng6/run.py

@@ -494,7 +494,7 @@ class Run(object):
                 raise Exception("Error while trying to rsync " + old_path + " to " + new_absolute_path + "\n" +
                                 "Command : " + str_cmd + "\n" + "Error code : " + str(retcode) + "\n")
         else:
-            print ("rsync could not be launched because the source and destination are the same, from " + old_path + " to " + new_absolute_path + "\n")
+            str_cmd = "Source and destination directories are the same : " + old_path + " , ignored.\n"
         
         new_retention = ng6conf.get_retention_date(space_id, self.date)
         

src/ng6/utils.py

@@ -361,6 +361,17 @@ class Utils(object):
         return "".join(dec)
         """
     
+    
+    @staticmethod
+    def get_project_dirname(project_id, project_name):
+        return "Project_%s.%s" % ( re.sub( r"[\s\/]", "_", project_name), project_id )
+    @staticmethod
+    def get_run_dirname(run_id, run_name):
+        return "Run_%s.%s" % ( re.sub( r"[^A-Za-z0-9]", "_", run_name), run_id)
+    @staticmethod
+    def get_analyse_dirname(analyse_id, analyse_name):
+        return "Analyse_%s.%s"%( re.sub( r"[\s\/]", "_", analyse_name), analyse_id )
+    
     @staticmethod
     def get_directories_structure_and_content(ng6_username, data_folder, output_folder, prefixed_ids):
         from ng6.t3MySQLdb import t3MySQLdb
@@ -372,20 +383,21 @@ class Utils(object):
         try : user_id = t3mysql.get_user_id(ng6_username)
         except : pass
         
+        
         for prefixed_id in prefixed_ids.split(';') :
             vals = prefixed_id.split('_')
             prefix = vals[0]
             id = vals[1]
-            
             if prefix == "data" or prefix == "run" :
                 run = t3mysql.select_run_informations(id)
                 source_dir = data_folder +'/'+ run['directory']
-                project_name = "Project_%s.%s" % ( re.sub( r"[\s\/]", "_", run['project_name']), run['project_id'] )
+                project_name = Utils.get_project_dirname(run['project_id'],run['project_name'])
                 
                 #run_name = "Run_%s.%s" %( run['name'].replace(' ', '_').replace('/', '_'), id )
-                run_name = "Run_%s.%s" % ( re.sub( r"[^A-Za-z0-9]", "_", run['name']), id)
+                run_name = Utils.get_run_dirname(id,run['name'])
                 
                 raw_data_dir = os.path.join( output_folder, project_name, run_name, "RawData"  )
+                print (run, source_dir,project_name,run_name)
                 if source_dir not in src_directories :
                     src_directories.append(source_dir)
                 if raw_data_dir not in dest_directories :
@@ -394,7 +406,7 @@ class Utils(object):
                 if prefix == "run" :
                     for analyse_id, analyse_values in t3mysql.get_user_run_analysis(user_id, id, 'tx_nG6_analyze.name').items() :
                         analysis_dir = data_folder + '/' + analyse_values["directory"]
-                        analysis_name = "Analyse_%s.%s"%( re.sub( r"[\s\/]", "_", analyse_values['name']), analyse_id )
+                        analysis_name = Utils.get_analyse_dirname(analyse_id, analyse_values['name'] )
                         dest_analysis_dir = os.path.join( output_folder, project_name, run_name, analysis_name  )
                         
                         if analysis_dir not in src_directories :
@@ -409,19 +421,19 @@ class Utils(object):
                 if source_dir not in src_directories :
                     src_directories.append(source_dir)
                 
-                project_name = "Project_%s.%s" % ( re.sub( r"[\s\/]", "_", analyse["project_name"]), analyse["project_id"])
-                
+                project_name = Utils.get_project_dirname(analyse['project_id'],analyse['project_name'])
                 # it's a run analysis
-                if analyse["run_id"]:
-                    run_name = "Run_%s.%s" % ( re.sub( r"[\s\/]", "_", analyse["run_name"]), analyse["run_id"])
-                    
-                    analysis_name = "Analyse_%s.%s" % ( re.sub( r"[\s\/]", "_", analyse["name"]), id)
+                if analyse["run_id"]:                    
+                    run_name = Utils.get_run_dirname(analyse["run_id"], analyse["run_name"])
+
+                    analysis_name = Utils.get_analyse_dirname(id, analyse['name'] )
                     dest_analysis_dir = os.path.join( output_folder, project_name, run_name, analysis_name )
+                    
                     if dest_analysis_dir not in dest_directories :
                         dest_directories.append(dest_analysis_dir)
                 # it's a project analysis
                 else :
-                    analysis_name = "Project_analyses/%s.%s" % ( re.sub( r"[\s\/]", "_", analyse["name"]), id)
+                    analysis_name = "Project_"+ Utils.get_analyse_dirname(id, analyse['name'] ) 
                     dest_analysis_dir = os.path.join( output_folder, project_name, analysis_name )
                     if dest_analysis_dir not in dest_directories :
                         dest_directories.append(dest_analysis_dir)
@@ -439,6 +451,9 @@ class Utils(object):
     
     @staticmethod    
     def rsync_getcmd( source, destination):
+        if source == "" or not os.path.isdir(source) :
+             return [ 0, "Source directory '" + source + "' doesn't exist, ignored."]
+        
         if not source.endswith("/"):
             source+="/"
         cmd = ["rsync", "-avh", "--perms", "--times", "--remove-sent-files", source, destination]

workflows/switch_space_id/components/move_project.py

@@ -29,53 +29,54 @@ def migrate_project (project_id, new_space_id, output):
     from ng6.analysis import Analysis
     from ng6.project import Project
     from ng6.run import Run
-
-    with open( output,'w') as fh :
-        ng6conf = NG6ConfigReader()
-        base_path = ng6conf.get_save_directory()
-        
-        project = Project.get_from_id(project_id)
-        
-        logging.getLogger("migrate_project").debug("Migrating project '" + str(project.name) + "' (" + str(project.id) + ") to " + str(new_space_id) + "\n")
-        fh.write("Migrating project '" + str(project.name) + "' (" + str(project.id) + ") to " + str(new_space_id) + "\n")
+    
+    def log_to_output_files (log_text, log_file):
+        fh = open(log_file, "a")        
+        fh.write(log_text)
+        fh.close()
+        logging.getLogger("move_project.py").debug(log_text)
+        print( "os.path.exists("+log_file+")=" + str(os.path.exists(log_file)))
+    
+    ng6conf = NG6ConfigReader()
+    base_path = ng6conf.get_save_directory()
         
-        old_space_id = project.space_id
-        logging.getLogger("migrate_project").debug("old_space_id = " + old_space_id + ", new_space_id = " + new_space_id + "\n")
-        fh.write("old_space_id = " + old_space_id + ", new_space_id = " + new_space_id + "\n")
+    project = Project.get_from_id(project_id)
+    old_space_id = project.space_id        
+       
+    str_log="MIGRATE PROJECT: '" + str(project.name) + "' (" + str(project.id) + ") from " + old_space_id + " to " + str(new_space_id) + "\n"
+    log_to_output_files(str_log, output)
+       
+    runs = project.get_runs()
+    for run in runs :
+        str_log="\t -  ##RUN '" + run.name + "' (" + str(run.id) + ") - " + run.directory + "\n"
+        log_to_output_files(str_log, output)
         
-        str_cmds = []
-        runs = project.get_runs()
-        for run in runs :
-            fh.write("Migrating run '" + run.name + "' (" + str(run.id) + ")" + " to " + str(new_space_id) + "\n")
-            logging.getLogger("migrate_project").debug("Migrating run '" + run.name + "' (" + str(run.id) + ")" + " to " + str(new_space_id) + "\n")
-            str_cmds.append(run.change_space(new_space_id) )
-            fh.write("Migrating successful! \n")
-            logging.getLogger("migrate_project").debug("Migrating successful! \n")
-            
-            run_analyzes = run.get_analysis()
-            for analysis in run_analyzes :
-                fh.write("Migrating analysis '" + analysis.name + "' (" + str(analysis.id) + ")" + " to " + str(new_space_id) + "\n")
-                logging.getLogger("migrate_project").debug("Migrating analysis '" + analysis.name + "' (" + str(analysis.id) + ")" + " to " + str(new_space_id) + "\n")
-                str_cmds.append(analysis.change_space(new_space_id) )
-                fh.write("Migrating successful! \n")
-                logging.getLogger("migrate_project").debug("Migrating successful! \n")
+        (ret_code, ret_command) = run.change_space(new_space_id) 
+        str_log = "\t   RET CODE: " + str(ret_code) + " CMD : " +  ret_command + "\n"
+        log_to_output_files(str_log, output)
+       
+        run_analyzes = run.get_analysis()
+        for analysis in run_analyzes :
+            str_log = "\t\t - ###ANALYSE " + analysis.name+ "' (" + str(analysis.id) + ") - " + analysis.directory + "\n"
+            log_to_output_files(str_log, output)
             
-        analyzes = project.get_analysis()
-        for analysis in analyzes :
-            fh.write("Migrating analysis '" + analysis.name + "' (" + str(analysis.id) + ")" + " to " + str(new_space_id) + "\n")
-            logging.getLogger("migrate_project").debug("Migrating analysis '" + analysis.name + "' (" + str(analysis.id) + ")" + " to " + str(new_space_id) + "\n")
-            str_cmds.append(analysis.change_space(new_space_id) )
-            fh.write("Migrating successful! \n")
-            logging.getLogger("migrate_project").debug("Migrating successful! \n")
-        
-        for command in str_cmds:
-            fh.write("Command launched : "+ str(command[1]) + "\n" )
-            logging.getLogger("migrate_project").debug("Command launched : "+ str(command[1]) + "\n" )
-            fh.write("Returned code : "+ str(command[0]) + "\n" )
-            logging.getLogger("migrate_project").debug("Returned code : "+ str(command[0]) + "\n" )
-        
-        project.update_space_id(new_space_id)
+            (ret_code, ret_command) = analysis.change_space(new_space_id)
+            str_log = "\t\t   RET CODE: " + str(ret_code) + " CMD : " +  ret_command + "\n"
+            log_to_output_files(str_log, output)
         
+    analyzes = project.get_analysis()
+    for analysis in analyzes :
+        str_log="\t - ##ANALYSE PROJECT '" + analysis.name + "' (" + str(analysis.id) + ") - " + analysis.directory + "\n"
+        log_to_output_files(str_log, output)
+
+        (ret_code, ret_command) = analysis.change_space(new_space_id)
+        str_log = "\tRET CODE: " + str(ret_code) + " CMD : " +  ret_command + "\n"
+        log_to_output_files(str_log, output)
+            
+    project.update_space_id(new_space_id)
+    str_log = "project.update_space_id DONE\n"
+    log_to_output_files(str_log, output)
+
 
 class MoveProject (Component):