Commit 7c169bce authored by Romain Therville's avatar Romain Therville 🐭
Browse files

Update to make the switch_space_id work.

- The type ng6space has been debugged.
- The workflow now generates a .stdout trace file.
- The synthax has changed: python3.4
/save/ng6-test/src/ng6-git-rtherville/bin/ng6_cli.py switchspaceid
--project-id 1293 --space-id seqoccin --admin-login CTD

issue #129
parent 9a64afbb
...@@ -109,7 +109,7 @@ class Analysis (Component): ...@@ -109,7 +109,7 @@ class Analysis (Component):
ANALYSIS_CONFIG_FILE_NAME = "analysis.cfg" ANALYSIS_CONFIG_FILE_NAME = "analysis.cfg"
def __init__(self, name="", description="", software="", def __init__(self, name="", description="", software="",
options="", version="", id=None, parent_analysis=None, space_id="default"): options="", version="", id=None, date=None, parent_analysis=None, space_id="default"):
""" """
Build an Analysis object Build an Analysis object
@param name : the analysis name @param name : the analysis name
...@@ -135,7 +135,8 @@ class Analysis (Component): ...@@ -135,7 +135,8 @@ class Analysis (Component):
self.is_editable = False self.is_editable = False
self.space_id = space_id self.space_id = space_id
self.retention_date = None self.retention_date = None
self.date = date
# Set the temp folder to the ng6 temp folder # Set the temp folder to the ng6 temp folder
ng6conf = NG6ConfigReader() ng6conf = NG6ConfigReader()
logging.getLogger("Analysis.__init__").debug("ng6conf est chargé") logging.getLogger("Analysis.__init__").debug("ng6conf est chargé")
...@@ -677,7 +678,7 @@ class Analysis (Component): ...@@ -677,7 +678,7 @@ class Analysis (Component):
#try: #try:
t3mysql = t3MySQLdb() t3mysql = t3MySQLdb()
[name, date, description, software, options, version] = t3mysql.select_analysis(id) [name, date, description, software, options, version] = t3mysql.select_analysis(id)
my_analysis = Analysis(name, description, software, options, version, id) my_analysis = Analysis(name, description, software, options, version, id, date)
if my_analysis == None: if my_analysis == None:
logging.getLogger("Analysis.get_from_id").error("The analysis id=" + str(id) + " does not exists in the database.") logging.getLogger("Analysis.get_from_id").error("The analysis id=" + str(id) + " does not exists in the database.")
raise Exception("The analysis id=" + str(id) + " does not exists in the database.\n") raise Exception("The analysis id=" + str(id) + " does not exists in the database.\n")
...@@ -702,25 +703,7 @@ class Analysis (Component): ...@@ -702,25 +703,7 @@ class Analysis (Component):
except : except :
pass pass
def set_retention(self, retention):
"""
Updates an analysis's retention field
@param retention : the new retention value, in (unix) epoch timestamp format
"""
if self.id != None :
t3mysql = t3MySQLdb()
t3mysql.set_analysis_retention_from_epoch_timestamp(str(self.id), retention)
def set_directory(self, directory):
"""
Updates an analysis's directory field
@param directory : the new directory path
"""
if self.id != None :
t3mysql = t3MySQLdb()
t3mysql.update_fields('tx_nG6_analyze', str(self.id), ['directory'], [str(directory)] )
def get_creation_date(self): def get_creation_date(self):
""" """
Returns an analysis's crdate value Returns an analysis's crdate value
...@@ -736,4 +719,33 @@ class Analysis (Component): ...@@ -736,4 +719,33 @@ class Analysis (Component):
""" """
if self.id != None : if self.id != None :
t3mysql = t3MySQLdb() t3mysql = t3MySQLdb()
return t3mysql.select_analysis_directory(str(self.id)) return t3mysql.select_analysis_directory(str(self.id))
\ No newline at end of file
def change_space (self, space_id ):
ng6conf = NG6ConfigReader()
old_path = ng6conf.get_save_directory() + self.directory
directory_name = os.path.split(old_path)[-1]
new_relative_path = os.path.join(ng6conf.get_space_directory(space_id), self.DIRECTORIES_STRUCTURE, directory_name)
new_absolute_path = os.path.join(ng6conf.get_save_directory(), new_relative_path )
print ("space_id " +space_id + "\nOLD" + old_path)
print ("NEW" + new_absolute_path)
str_cmd = ""
retcode = -1
if str(old_path) != str(new_absolute_path):
[retcode, str_cmd] = Utils.rsync_getcmd(old_path,new_absolute_path)
if retcode != 0 :
raise Exception("Error while trying to rsync " + old_path + " to " + new_absolute_path + "\n" +
"Command : " + str_cmd + "\n" + "Error code : " + retcode + "\n" + str(err) + "\n")
else:
print ("rsync could not be launched because the source and destination are the same, from " + old_path + " to " + new_absolute_path + "\n")
new_retention = ng6conf.get_retention_date(space_id, self.date)
t3mysql = t3MySQLdb()
t3mysql.update_fields('tx_nG6_analyze', str(self.id), ['directory','retention_date'], [str("/"+new_relative_path),new_retention] )
return [retcode, str_cmd]
\ No newline at end of file
...@@ -114,15 +114,20 @@ class NG6ConfigReader(object): ...@@ -114,15 +114,20 @@ class NG6ConfigReader(object):
except: except:
raise Exception("Failed when parsing the config file !") raise Exception("Failed when parsing the config file !")
def get_retention_date(self, space_id="default"): def get_retention_date(self, space_id="default", creation_date = None):
""" """
return the retention corresponding to space_id return the retention corresponding to space_id
@return: space_dir @return: space_dir
""" """
date = None date = None
start_date= datetime.datetime.today()
if creation_date is not None :
start_date=datetime.datetime.fromtimestamp(creation_date)
try : try :
nb_month=self.reader.getint('space_'+space_id,"retention_period") nb_month=self.reader.getint('space_'+space_id,"retention_period")
retention_date = time.mktime((datetime.datetime.today()+ relativedelta(months=nb_month)).timetuple()) retention_date = time.mktime((start_date+ relativedelta(months=nb_month)).timetuple())
return retention_date return retention_date
except: except:
try : try :
...@@ -132,27 +137,7 @@ class NG6ConfigReader(object): ...@@ -132,27 +137,7 @@ class NG6ConfigReader(object):
except: except:
raise Exception("Failed when parsing the config file !") raise Exception("Failed when parsing the config file !")
raise ValueError("Failed while generating retention date!") raise ValueError("Failed while generating retention date!")
def compute_new_retention_date(self, space_id="default", creation_date = None):
"""
return the retention corresponding to space_id and crdate
@return: unix epoch timestamp retention value
"""
date = None
try :
nb_month=self.reader.getint('space_'+space_id,"retention_period")
retention_date = time.mktime((datetime.datetime.fromtimestamp(creation_date)+ relativedelta(months=nb_month)).timetuple())
logging.getLogger("config_reader.compute_new_retention_date").debug("IN retention period")
return retention_date
except:
try :
(d,m,y)=self.reader.get('space_'+space_id,"retention_date").split('/')
date = time.mktime(datetime.date(int(y),int(m),int(d)).timetuple())
logging.getLogger("config_reader.compute_new_retention_date").debug("IN retention date")
return date
except:
raise Exception("Failed when parsing the config file !")
raise ValueError("Failed while generating retention date!")
def get_log_file_path(self): def get_log_file_path(self):
""" """
......
...@@ -470,4 +470,33 @@ class Run(object): ...@@ -470,4 +470,33 @@ class Run(object):
t3mysql = t3MySQLdb() t3mysql = t3MySQLdb()
return t3mysql.select_run_directory(self.id) return t3mysql.select_run_directory(self.id)
else : else :
raise UnsavedRunError() raise UnsavedRunError()
\ No newline at end of file
def change_space (self, space_id ):
ng6conf = NG6ConfigReader()
old_path = ng6conf.get_save_directory() + self.directory
directory_name = os.path.split(old_path)[-1]
new_relative_path = os.path.join(ng6conf.get_space_directory(space_id), self.DIRECTORIES_STRUCTURE, directory_name)
new_absolute_path = os.path.join(ng6conf.get_save_directory(), new_relative_path )
print ("space_id " +space_id + "\nOLD" + old_path)
print ("NEW" + new_absolute_path)
str_cmd = ""
retcode = -1
if str(old_path) != str(new_absolute_path):
[retcode, str_cmd] = Utils.rsync_getcmd(old_path,new_absolute_path)
if retcode != 0 :
raise Exception("Error while trying to rsync " + old_path + " to " + new_absolute_path + "\n" +
"Command : " + str_cmd + "\n" + "Error code : " + retcode + "\n" + str(err) + "\n")
else:
print ("rsync could not be launched because the source and destination are the same, from " + old_path + " to " + new_absolute_path + "\n")
new_retention = ng6conf.get_retention_date(space_id, self.date)
t3mysql = t3MySQLdb()
t3mysql.update_fields('tx_nG6_run', str(self.id), ['directory','retention_date'], [str("/"+new_relative_path),new_retention] )
return [retcode, str_cmd]
...@@ -32,7 +32,7 @@ import re ...@@ -32,7 +32,7 @@ import re
import base64 import base64
import pty import pty
import sys import sys
from subprocess import call
class Utils(object): class Utils(object):
""" """
...@@ -436,8 +436,24 @@ class Utils(object): ...@@ -436,8 +436,24 @@ class Utils(object):
sources.append(e) sources.append(e)
destinations.append(dest_directories[i]) destinations.append(dest_directories[i])
return sources, destinations return sources, destinations
@staticmethod
def rsync_getcmd( source, destination):
if not source.endswith("/"):
source+="/"
cmd = ["rsync", "-avh", "--perms", "--times", "--remove-sent-files", source, destination]
print("\n".join(cmd))
retcode = call(cmd , shell=False)
#Delete the old directory if empty
if retcode >= 0 and os.path.isdir(source) and not os.listdir(source):
try:
os.rmdir(source)
except Exception as err:
raise Exception("Error while deleting " + source + "\n" + str(err) + "\n")
return [ retcode, " ".join(cmd)]
def get_purge_demand_directories(demand_ids): def get_purge_demand_directories(demand_ids):
from ng6.t3MySQLdb import t3MySQLdb from ng6.t3MySQLdb import t3MySQLdb
t3mysql = t3MySQLdb() t3mysql = t3MySQLdb()
......
#
# Copyright (C) 2015 INRA
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
import os
import sys
import logging
import shutil
from ng6.ng6workflow import BasicNG6Workflow
from ng6.config_reader import NG6ConfigReader
from ng6.utils import Utils
from ng6.project import Project
from ng6.run import Run
from ng6.analysis import Analysis
from ng6.t3MySQLdb import t3MySQLdb
class SwitchProjectSpaceId (BasicNG6Workflow):
def get_description(self):
return """This workflow allows you to migrate a project from a space_id to another.
It updates the project's space_id in DB, moves the runs files, updates the runs and analyzes's files paths, and updates the corresponding retentions."""
def define_parameters(self, function="process"):
self.add_parameter('project_id', 'The project id', type=int, required = True)
self.add_parameter('space_id', 'The new space_id', required = True)
def process(self):
from datetime import datetime
#We open a log file
f = open("/work/ng6-test/jflow_rtherville/ng6_python.log", "a")
#f = open(os.path.join(os.path.dirname(__file__), '../../ng6_python.log'), "a")
now = datetime.now()
datetime = now.strftime("%d/%m/%Y %H:%M:%S")
logging.getLogger("SwitchProjectSpaceId.process").debug(datetime + " Begin SwitchProjectSpaceId workflow.")
f.write("\n\n" + datetime + " Begin SwitchProjectSpaceId workflow.\n")
self.project_id = int(self.project_id)
if self.project_id and self.space_id :
logging.getLogger("SwitchProjectSpaceId.process").debug("project_id =" + str(self.project_id) + " , new space_id = " + str(self.space_id) )
f.write("project_id =" + str(self.project_id) + " , new space_id = " + str(self.space_id) +"\n")
project = Project.get_from_id(self.project_id)
old_space_id = project.space_id
#We log a message if the project already has the given space_id
if str(old_space_id) == str(self.space_id) :
logging.getLogger("SwitchProjectSpaceId.process").debug("The project " + project.name + " already belongs to the space_id '" + str(self.space_id) + "'\n" )
f.write("The project " + project.name + " already belongs to the space_id '" + str(self.space_id) + "'\n" )
#Retrieve the old and new directories
ng6conf = NG6ConfigReader()
old_directory = ng6conf.get_space_directory(old_space_id)
try:
new_directory = ng6conf.get_space_directory(self.space_id)
except Exception as err:
logging.getLogger("SwitchProjectSpaceId.process").error("Error while searching space_id '" + str(self.space_id) + "' in application.properties, it certainly doesn't exist. \n" + str(err))
f.write("Error while searching space_id '" + str(self.space_id) + "' in application.properties, it certainly doesn't exist. \n" + str(err) + "\n")
raise Exception("Error while searching space_id '" + str(self.space_id) + "' in application.properties, it certainly doesn't exist. \n" + str(err) + "\n")
base_path = ng6conf.get_save_directory()
#Process for the project's runs
runs_ids = project.get_project_run_ids()
for run_id in runs_ids :
logging.getLogger("SwitchProjectSpaceId.process").debug("Migrating run " + str(run_id) )
f.write("Migrating run " + str(run_id) + "\n")
run = Run.get_from_id(run_id)
path_old_run_directory = base_path + run.get_directory()
path_new_run_directory = path_old_run_directory.replace(old_directory,new_directory)
#Update the directory in DB
new_run_directory = run.get_directory()
new_run_directory = new_run_directory.replace(old_directory, new_directory)
logging.getLogger("SwitchProjectSpaceId.process").debug("Beginning run files migration, from " + str(path_old_run_directory) + " to " + str(path_new_run_directory) +"\n" )
f.write("Beginning run files migration, from " + str(path_old_run_directory) + " to " + str(path_new_run_directory) +"\n" )
try:
shutil.move(path_old_run_directory, path_new_run_directory)
except Exception as err:
logging.getLogger("SwitchProjectSpaceId.process").error("Error while moving " + path_old_run_directory + " to " + path_new_run_directory + " \n" + str(err) + "\n")
f.write("Error while moving " + path_old_run_directory + " to " + path_new_run_directory + " \n" + str(err) + "\n")
raise Exception("Error while moving " + path_old_run_directory + " to " + path_new_run_directory + " \n" + str(err))
#Delete the old directory if empty
if os.path.isdir(path_old_run_directory) and not os.listdir(path_old_run_directory) :
try:
os.rmdir(path_old_run_directory)
except Exception as err:
logging.getLogger("SwitchProjectSpaceId.process").error("Error while deleting " + path_old_run_directory + "\n" + str(err) + "\n")
f.write("Error while deleting " + path_old_run_directory + "\n" + str(err) + "\n")
raise Exception("Error while deleting " + path_old_run_directory + "\n" + str(err) + "\n")
#Update the retention in DB
old_retention = run.get_creation_date()
new_retention = ng6conf.compute_new_retention_date(self.space_id,old_retention)
run.set_retention(new_retention)
run.set_directory(new_run_directory)
#Process for the project's analyzes
analyzes_ids = project.get_project_analyzes_ids()
for analysis_id in analyzes_ids :
logging.getLogger("SwitchProjectSpaceId.process").debug("Migrating analysis " + str(analysis_id) )
f.write("Migrating analysis " + str(analysis_id) + "\n")
analysis = Analysis.get_from_id(analysis_id)
path_old_analysis_directory = base_path + analysis.get_directory()
path_new_analysis_directory = path_old_analysis_directory.replace(old_directory,new_directory)
#Update the directory in DB
new_analysis_directory = analysis.get_directory()
new_analysis_directory = new_analysis_directory.replace(old_directory, new_directory)
logging.getLogger("SwitchProjectSpaceId.process").debug("Migrating analysis files from " + str(path_old_analysis_directory) + " to " + str(path_new_analysis_directory) + "\n" )
f.write( "Beginning analysis files migration, from " + str(path_old_analysis_directory) + " to " + str(path_new_analysis_directory) + "\n" )
try:
shutil.move(path_old_analysis_directory, path_new_analysis_directory)
except Exception as err:
logging.getLogger("SwitchProjectSpaceId.process").error("Error while moving " + path_old_analysis_directory + " to " + path_new_analysis_directory + " \n" + str(err) + "\n")
f.write("Error while moving " + path_old_analysis_directory + " to " + path_new_analysis_directory + "\n" + str(err) + "\n")
raise Exception("Error while moving " + path_old_analysis_directory + " to " + path_new_analysis_directory + " \n" + str(err) + "\n")
#Delete the old directory if empty
if os.path.isdir(path_old_analysis_directory) and not os.listdir(path_old_analysis_directory) :
try:
os.rmdir(path_old_analysis_directory)
except Exception as err:
logging.getLogger("SwitchProjectSpaceId.process").error("Error while deleting " + path_old_analysis_directory + "\n" + str(err) + "\n")
f.write("Error while deleting " + path_old_analysis_directory + "\n" + str(err) + "\n")
raise Exception("Error while deleting " + path_old_analysis_directory + "\n" + str(err) + "\n")
#Update the retention in DB
old_retention = analysis.get_creation_date()
new_retention = ng6conf.compute_new_retention_date(self.space_id,old_retention)
analysis.set_retention(new_retention)
analysis.set_directory(new_analysis_directory)
#Process for the project's run's analyzes
run_analyzes_ids = project.get_project_runs_analyzes_ids()
for analysis_id in run_analyzes_ids :
logging.getLogger("SwitchProjectSpaceId.process").debug("Migrating analysis " + str(analysis_id) + "\n")
f.write("Migrating analysis " + str(analysis_id) + "\n")
analysis = Analysis.get_from_id(analysis_id)
path_old_analysis_directory = base_path + analysis.get_directory()
path_new_analysis_directory = path_old_analysis_directory.replace(old_directory,new_directory)
#Update the directory in DB
new_analysis_directory = analysis.get_directory()
new_analysis_directory = new_analysis_directory.replace(old_directory, new_directory)
logging.getLogger("SwitchProjectSpaceId.process").debug("Migrating analysis files from " + str(path_old_analysis_directory) + " to " + str(path_new_analysis_directory) + "\n" )
f.write( "Beginning analysis files migration, from " + str(path_old_analysis_directory) + " to " + str(path_new_analysis_directory) + "\n" )
try:
shutil.move(path_old_analysis_directory, path_new_analysis_directory)
except Exception as err:
logging.getLogger("SwitchProjectSpaceId.process").error("Error while moving " + path_old_analysis_directory + " to " + path_new_analysis_directory + " \n" + str(err))
f.write("Error while moving " + path_old_analysis_directory + " to " + path_new_analysis_directory + "\n" + str(err) + "\n")
raise Exception("Error while moving " + path_old_analysis_directory + " to " + path_new_analysis_directory + " \n" + str(err))
#Delete the old directory if empty
if os.path.isdir(path_old_analysis_directory) and not os.listdir(path_old_analysis_directory) :
try:
os.rmdir(path_old_analysis_directory)
except Exception as err:
logging.getLogger("SwitchProjectSpaceId.process").error("Error while deleting " + path_old_analysis_directory + "\n" + str(err) + "\n")
f.write("Error while deleting " + path_old_analysis_directory + "\n" + str(err) + "\n")
raise Exception("Error while deleting " + path_old_analysis_directory + "\n" + str(err) + "\n")
#Update the retention in DB
old_retention = analysis.get_creation_date()
new_retention = ng6conf.compute_new_retention_date(self.space_id,old_retention)
analysis.set_retention(new_retention)
analysis.set_directory(new_analysis_directory)
#We close the log file
f.close()
#Update to the projcet's space_id
project.update_space_id(self.space_id)
\ No newline at end of file
#
# Copyright (C) 2015 INRA
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
import logging
from jflow.component import Component
from workflows.types import ng6space,existingprojectid
from ng6.t3MySQLdb import t3MySQLdb
def migrate_project (project_id, new_space_id, output):
import os
from subprocess import call
import logging
from ng6.config_reader import NG6ConfigReader
from ng6.analysis import Analysis
from ng6.project import Project
from ng6.run import Run
with open( output,'w') as fh :
ng6conf = NG6ConfigReader()
base_path = ng6conf.get_save_directory()
project = Project.get_from_id(project_id)
logging.getLogger("migrate_project").debug("Migrating project '" + str(project.name) + "' (" + str(project.id) + ") to " + str(new_space_id) + "\n")
fh.write("Migrating project '" + str(project.name) + "' (" + str(project.id) + ") to " + str(new_space_id) + "\n")
old_space_id = project.space_id
fh.write("old_space_id = " + old_space_id + ", new_space_id = " + new_space_id + "\n")
str_cmds = []
runs = project.get_runs()
for run in runs :
fh.write("Migrating run '" + run.name + "' (" + str(run.id) + ")" + " to " + str(new_space_id) + "\n")
str_cmds.append(run.change_space(new_space_id) )
fh.write("Migrating successful! \n")
run_analyzes = run.get_analysis()
for analysis in run_analyzes :
fh.write("Migrating analysis '" + analysis.name + "' (" + str(analysis.id) + ")" + " to " + str(new_space_id) + "\n")
str_cmds.append(analysis.change_space(new_space_id) )
fh.write("Migrating successful! \n")
analyzes = project.get_analysis()
for analysis in analyzes :
fh.write("Migrating analysis '" + analysis.name + "' (" + str(analysis.id) + ")" + " to " + str(new_space_id) + "\n")
str_cmds.append(analysis.change_space(new_space_id) )
fh.write("Migrating successful! \n")
for command in str_cmds:
fh.write("Command launched : "+ str(command[1]) + "\n" )
fh.write("Returned code : "+ str(command[0]) + "\n" )
project.update_space_id(new_space_id)
class MoveProject (Component):
def define_parameters(self, project_id, space_id,):
self.add_parameter('project_id', 'The project id', type = existingprojectid, required = True, default = project_id )
self.add_parameter('space_id', 'The new space_id', type = ng6space, required = True, default = space_id)
self.add_output_file("stdout", "MoveProject stdout", filename="MoveProject.stdout")
def process(self):
logging.getLogger("MoveProject.process").debug(" Begin MoveProject Component.\n")
logging.getLogger("MoveProject.process").debug(" self.space_id="+str(self.space_id)+"\n")
logging.getLogger("MoveProject.process").debug(" self.project_id="+str(self.project_id)+"\n")
logging.getLogger("MoveProject.process").debug(" stdout = "+str(self.stdout)+"\n")
self.add_python_execution(migrate_project, cmd_format="{EXE} {ARG} {OUT}",
outputs = self.stdout, arguments = [self.project_id, self.space_id])
#self.add_python_execution(migrate_project, cmd_format="{EXE} {ARG}",
# arguments = [self.project_id, self.space_id])
...@@ -97,7 +97,8 @@ def ng6space(val): ...@@ -97,7 +97,8 @@ def ng6space(val):
ng6conf.get_space_directory(val) ng6conf.get_space_directory(val)
except : except :
raise argparse.ArgumentTypeError("The space name '%s' is not define in application.properties" % val) raise argparse.ArgumentTypeError("The space name '%s' is not define in application.properties" % val)
return val
def existingrun(id): def existingrun(id):
try: try:
t3mysql = t3MySQLdb() t3mysql = t3MySQLdb()
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment