Commit 1dc8afbe authored by Romain Therville's avatar Romain Therville 🐭

Big update for the switch_space_id workflow

-New better logs, in MoveProject.stdout and (if the workflow fails) in
ng6.log
-The workflow now handles empty analyzes and runs.
-A lot of user cases have been tested.

issue #129
parent 13279d3d
...@@ -743,7 +743,7 @@ class Analysis (Component): ...@@ -743,7 +743,7 @@ class Analysis (Component):
raise Exception("Error while trying to rsync " + old_path + " to " + new_absolute_path + "\n" + raise Exception("Error while trying to rsync " + old_path + " to " + new_absolute_path + "\n" +
"Command : " + str_cmd + "\n" + "Error code : " + str(retcode) + "\n") "Command : " + str_cmd + "\n" + "Error code : " + str(retcode) + "\n")
else: else:
print ("rsync could not be launched because the source and destination are the same, from " + old_path + " to " + new_absolute_path + "\n") str_cmd = "Source and destination directories are the same : " + old_path + " , ignored.\n"
new_retention = ng6conf.get_retention_date(space_id, self.date) new_retention = ng6conf.get_retention_date(space_id, self.date)
...@@ -751,4 +751,4 @@ class Analysis (Component): ...@@ -751,4 +751,4 @@ class Analysis (Component):
t3mysql.update_fields('tx_nG6_analyze', str(self.id), ['directory','retention_date'], [str("/"+new_relative_path),new_retention] ) t3mysql.update_fields('tx_nG6_analyze', str(self.id), ['directory','retention_date'], [str("/"+new_relative_path),new_retention] )
return [retcode, str_cmd] return [retcode, str_cmd]
\ No newline at end of file
...@@ -494,7 +494,7 @@ class Run(object): ...@@ -494,7 +494,7 @@ class Run(object):
raise Exception("Error while trying to rsync " + old_path + " to " + new_absolute_path + "\n" + raise Exception("Error while trying to rsync " + old_path + " to " + new_absolute_path + "\n" +
"Command : " + str_cmd + "\n" + "Error code : " + str(retcode) + "\n") "Command : " + str_cmd + "\n" + "Error code : " + str(retcode) + "\n")
else: else:
print ("rsync could not be launched because the source and destination are the same, from " + old_path + " to " + new_absolute_path + "\n") str_cmd = "Source and destination directories are the same : " + old_path + " , ignored.\n"
new_retention = ng6conf.get_retention_date(space_id, self.date) new_retention = ng6conf.get_retention_date(space_id, self.date)
......
...@@ -361,6 +361,17 @@ class Utils(object): ...@@ -361,6 +361,17 @@ class Utils(object):
return "".join(dec) return "".join(dec)
""" """
@staticmethod
def get_project_dirname(project_id, project_name):
return "Project_%s.%s" % ( re.sub( r"[\s\/]", "_", project_name), project_id )
@staticmethod
def get_run_dirname(run_id, run_name):
return "Run_%s.%s" % ( re.sub( r"[^A-Za-z0-9]", "_", run_name), run_id)
@staticmethod
def get_analyse_dirname(analyse_id, analyse_name):
return "Analyse_%s.%s"%( re.sub( r"[\s\/]", "_", analyse_name), analyse_id )
@staticmethod @staticmethod
def get_directories_structure_and_content(ng6_username, data_folder, output_folder, prefixed_ids): def get_directories_structure_and_content(ng6_username, data_folder, output_folder, prefixed_ids):
from ng6.t3MySQLdb import t3MySQLdb from ng6.t3MySQLdb import t3MySQLdb
...@@ -372,20 +383,21 @@ class Utils(object): ...@@ -372,20 +383,21 @@ class Utils(object):
try : user_id = t3mysql.get_user_id(ng6_username) try : user_id = t3mysql.get_user_id(ng6_username)
except : pass except : pass
for prefixed_id in prefixed_ids.split(';') : for prefixed_id in prefixed_ids.split(';') :
vals = prefixed_id.split('_') vals = prefixed_id.split('_')
prefix = vals[0] prefix = vals[0]
id = vals[1] id = vals[1]
if prefix == "data" or prefix == "run" : if prefix == "data" or prefix == "run" :
run = t3mysql.select_run_informations(id) run = t3mysql.select_run_informations(id)
source_dir = data_folder +'/'+ run['directory'] source_dir = data_folder +'/'+ run['directory']
project_name = "Project_%s.%s" % ( re.sub( r"[\s\/]", "_", run['project_name']), run['project_id'] ) project_name = Utils.get_project_dirname(run['project_id'],run['project_name'])
#run_name = "Run_%s.%s" %( run['name'].replace(' ', '_').replace('/', '_'), id ) #run_name = "Run_%s.%s" %( run['name'].replace(' ', '_').replace('/', '_'), id )
run_name = "Run_%s.%s" % ( re.sub( r"[^A-Za-z0-9]", "_", run['name']), id) run_name = Utils.get_run_dirname(id,run['name'])
raw_data_dir = os.path.join( output_folder, project_name, run_name, "RawData" ) raw_data_dir = os.path.join( output_folder, project_name, run_name, "RawData" )
print (run, source_dir,project_name,run_name)
if source_dir not in src_directories : if source_dir not in src_directories :
src_directories.append(source_dir) src_directories.append(source_dir)
if raw_data_dir not in dest_directories : if raw_data_dir not in dest_directories :
...@@ -394,7 +406,7 @@ class Utils(object): ...@@ -394,7 +406,7 @@ class Utils(object):
if prefix == "run" : if prefix == "run" :
for analyse_id, analyse_values in t3mysql.get_user_run_analysis(user_id, id, 'tx_nG6_analyze.name').items() : for analyse_id, analyse_values in t3mysql.get_user_run_analysis(user_id, id, 'tx_nG6_analyze.name').items() :
analysis_dir = data_folder + '/' + analyse_values["directory"] analysis_dir = data_folder + '/' + analyse_values["directory"]
analysis_name = "Analyse_%s.%s"%( re.sub( r"[\s\/]", "_", analyse_values['name']), analyse_id ) analysis_name = Utils.get_analyse_dirname(analyse_id, analyse_values['name'] )
dest_analysis_dir = os.path.join( output_folder, project_name, run_name, analysis_name ) dest_analysis_dir = os.path.join( output_folder, project_name, run_name, analysis_name )
if analysis_dir not in src_directories : if analysis_dir not in src_directories :
...@@ -409,19 +421,19 @@ class Utils(object): ...@@ -409,19 +421,19 @@ class Utils(object):
if source_dir not in src_directories : if source_dir not in src_directories :
src_directories.append(source_dir) src_directories.append(source_dir)
project_name = "Project_%s.%s" % ( re.sub( r"[\s\/]", "_", analyse["project_name"]), analyse["project_id"]) project_name = Utils.get_project_dirname(analyse['project_id'],analyse['project_name'])
# it's a run analysis # it's a run analysis
if analyse["run_id"]: if analyse["run_id"]:
run_name = "Run_%s.%s" % ( re.sub( r"[\s\/]", "_", analyse["run_name"]), analyse["run_id"]) run_name = Utils.get_run_dirname(analyse["run_id"], analyse["run_name"])
analysis_name = "Analyse_%s.%s" % ( re.sub( r"[\s\/]", "_", analyse["name"]), id) analysis_name = Utils.get_analyse_dirname(id, analyse['name'] )
dest_analysis_dir = os.path.join( output_folder, project_name, run_name, analysis_name ) dest_analysis_dir = os.path.join( output_folder, project_name, run_name, analysis_name )
if dest_analysis_dir not in dest_directories : if dest_analysis_dir not in dest_directories :
dest_directories.append(dest_analysis_dir) dest_directories.append(dest_analysis_dir)
# it's a project analysis # it's a project analysis
else : else :
analysis_name = "Project_analyses/%s.%s" % ( re.sub( r"[\s\/]", "_", analyse["name"]), id) analysis_name = "Project_"+ Utils.get_analyse_dirname(id, analyse['name'] )
dest_analysis_dir = os.path.join( output_folder, project_name, analysis_name ) dest_analysis_dir = os.path.join( output_folder, project_name, analysis_name )
if dest_analysis_dir not in dest_directories : if dest_analysis_dir not in dest_directories :
dest_directories.append(dest_analysis_dir) dest_directories.append(dest_analysis_dir)
...@@ -439,6 +451,9 @@ class Utils(object): ...@@ -439,6 +451,9 @@ class Utils(object):
@staticmethod @staticmethod
def rsync_getcmd( source, destination): def rsync_getcmd( source, destination):
if source == "" or not os.path.isdir(source) :
return [ 0, "Source directory '" + source + "' doesn't exist, ignored."]
if not source.endswith("/"): if not source.endswith("/"):
source+="/" source+="/"
cmd = ["rsync", "-avh", "--perms", "--times", "--remove-sent-files", source, destination] cmd = ["rsync", "-avh", "--perms", "--times", "--remove-sent-files", source, destination]
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment