Skip to content

GitLab

ng6
ng6
Commit 0677a917 authored by Celine Noirot's avatar Celine Noirot
Browse files
Options

Merge branch 'issue#129' into 'master' 

Merge branch Issue#129 with master

See merge request !66
parents 3fa3d7c8 616f3286
Hide whitespace changes
Inline Side-by-side
Showing with 175 additions and 221 deletions
src/ng6/analysis.py
View file @ 0677a917
......@@ -109,7 +109,7 @@ class Analysis (Component):
ANALYSIS_CONFIG_FILE_NAME = "analysis.cfg"
def __init__(self, name="", description="", software="",
options="", version="", id=None, parent_analysis=None, space_id="default"):
options="", version="", id=None, date=None, parent_analysis=None, space_id="default"):
"""
Build an Analysis object
@param name : the analysis name
......@@ -135,7 +135,8 @@ class Analysis (Component):
self.is_editable = False
self.space_id = space_id
self.retention_date = None
self.date = date
# Set the temp folder to the ng6 temp folder
ng6conf = NG6ConfigReader()
logging.getLogger("Analysis.__init__").debug("ng6conf est chargé")
......@@ -677,7 +678,7 @@ class Analysis (Component):
#try:
t3mysql = t3MySQLdb()
[name, date, description, software, options, version] = t3mysql.select_analysis(id)
my_analysis = Analysis(name, description, software, options, version, id)
my_analysis = Analysis(name, description, software, options, version, id, date)
if my_analysis == None:
logging.getLogger("Analysis.get_from_id").error("The analysis id=" + str(id) + " does not exists in the database.")
raise Exception("The analysis id=" + str(id) + " does not exists in the database.\n")
......@@ -702,25 +703,7 @@ class Analysis (Component):
except :
pass
def set_retention(self, retention):
"""
Updates an analysis's retention field
@param retention : the new retention value, in (unix) epoch timestamp format
"""
if self.id != None :
t3mysql = t3MySQLdb()
t3mysql.set_analysis_retention_from_epoch_timestamp(str(self.id), retention)
def set_directory(self, directory):
"""
Updates an analysis's directory field
@param directory : the new directory path
"""
if self.id != None :
t3mysql = t3MySQLdb()
t3mysql.update_fields('tx_nG6_analyze', str(self.id), ['directory'], [str(directory)] )
def get_creation_date(self):
"""
Returns an analysis's crdate value
......@@ -736,4 +719,31 @@ class Analysis (Component):
"""
if self.id != None :
t3mysql = t3MySQLdb()
return t3mysql.select_analysis_directory(str(self.id))
\ No newline at end of file
return t3mysql.select_analysis_directory(str(self.id))
def change_space (self, space_id ):
ng6conf = NG6ConfigReader()
old_path = ng6conf.get_save_directory() + self.directory
directory_name = os.path.split(old_path)[-1]
new_relative_path = os.path.join(ng6conf.get_space_directory(space_id), self.DIRECTORIES_STRUCTURE, directory_name)
new_absolute_path = os.path.join(ng6conf.get_save_directory(), new_relative_path )
str_cmd = ""
retcode = -1
if str(old_path) != str(new_absolute_path):
[retcode, str_cmd] = Utils.rsync_getcmd(old_path,new_absolute_path)
if retcode != 0 :
raise Exception("Error while trying to rsync " + old_path + " to " + new_absolute_path + "\n" +
"Command : " + str_cmd + "\n" + "Error code : " + retcode + "\n" + str(err) + "\n")
else:
print ("rsync could not be launched because the source and destination are the same, from " + old_path + " to " + new_absolute_path + "\n")
new_retention = ng6conf.get_retention_date(space_id, self.date)
t3mysql = t3MySQLdb()
t3mysql.update_fields('tx_nG6_analyze', str(self.id), ['directory','retention_date'], [str("/"+new_relative_path),new_retention] )
return [retcode, str_cmd]
\ No newline at end of file
src/ng6/config_reader.py
View file @ 0677a917
......@@ -114,15 +114,20 @@ class NG6ConfigReader(object):
except:
raise Exception("Failed when parsing the config file !")
def get_retention_date(self, space_id="default"):
def get_retention_date(self, space_id="default", creation_date = None):
"""
return the retention corresponding to space_id
@return: space_dir
"""
date = None
start_date= datetime.datetime.today()
if creation_date is not None :
start_date=datetime.datetime.fromtimestamp(creation_date)
try :
nb_month=self.reader.getint('space_'+space_id,"retention_period")
retention_date = time.mktime((datetime.datetime.today()+ relativedelta(months=nb_month)).timetuple())
retention_date = time.mktime((start_date+ relativedelta(months=nb_month)).timetuple())
return retention_date
except:
try :
......@@ -132,27 +137,7 @@ class NG6ConfigReader(object):
except:
raise Exception("Failed when parsing the config file !")
raise ValueError("Failed while generating retention date!")
def compute_new_retention_date(self, space_id="default", creation_date = None):
"""
return the retention corresponding to space_id and crdate
@return: unix epoch timestamp retention value
"""
date = None
try :
nb_month=self.reader.getint('space_'+space_id,"retention_period")
retention_date = time.mktime((datetime.datetime.fromtimestamp(creation_date)+ relativedelta(months=nb_month)).timetuple())
logging.getLogger("config_reader.compute_new_retention_date").debug("IN retention period")
return retention_date
except:
try :
(d,m,y)=self.reader.get('space_'+space_id,"retention_date").split('/')
date = time.mktime(datetime.date(int(y),int(m),int(d)).timetuple())
logging.getLogger("config_reader.compute_new_retention_date").debug("IN retention date")
return date
except:
raise Exception("Failed when parsing the config file !")
raise ValueError("Failed while generating retention date!")
def get_log_file_path(self):
"""
......
src/ng6/run.py
View file @ 0677a917
......@@ -470,4 +470,31 @@ class Run(object):
t3mysql = t3MySQLdb()
return t3mysql.select_run_directory(self.id)
else :
raise UnsavedRunError()
\ No newline at end of file
raise UnsavedRunError()
def change_space (self, space_id ):
ng6conf = NG6ConfigReader()
old_path = ng6conf.get_save_directory() + self.directory
directory_name = os.path.split(old_path)[-1]
new_relative_path = os.path.join(ng6conf.get_space_directory(space_id), self.DIRECTORIES_STRUCTURE, directory_name)
new_absolute_path = os.path.join(ng6conf.get_save_directory(), new_relative_path )
str_cmd = ""
retcode = -1
if str(old_path) != str(new_absolute_path):
[retcode, str_cmd] = Utils.rsync_getcmd(old_path,new_absolute_path)
if retcode != 0 :
raise Exception("Error while trying to rsync " + old_path + " to " + new_absolute_path + "\n" +
"Command : " + str_cmd + "\n" + "Error code : " + retcode + "\n" + str(err) + "\n")
else:
print ("rsync could not be launched because the source and destination are the same, from " + old_path + " to " + new_absolute_path + "\n")
new_retention = ng6conf.get_retention_date(space_id, self.date)
t3mysql = t3MySQLdb()
t3mysql.update_fields('tx_nG6_run', str(self.id), ['directory','retention_date'], [str("/"+new_relative_path),new_retention] )
return [retcode, str_cmd]
src/ng6/utils.py
View file @ 0677a917
......@@ -32,7 +32,7 @@ import re
import base64
import pty
import sys
from subprocess import call
class Utils(object):
"""
......@@ -436,8 +436,24 @@ class Utils(object):
sources.append(e)
destinations.append(dest_directories[i])
return sources, destinations
@staticmethod
def rsync_getcmd( source, destination):
if not source.endswith("/"):
source+="/"
cmd = ["rsync", "-avh", "--perms", "--times", "--remove-sent-files", source, destination]
print("\n".join(cmd))
retcode = call(cmd , shell=False)
#Delete the old directory if empty
if retcode >= 0 and os.path.isdir(source) and not os.listdir(source):
try:
os.rmdir(source)
except Exception as err:
raise Exception("Error while deleting " + source + "\n" + str(err) + "\n")
return [ retcode, " ".join(cmd)]
def get_purge_demand_directories(demand_ids):
from ng6.t3MySQLdb import t3MySQLdb
t3mysql = t3MySQLdb()
......
workflows/switch_project_space_id/__init__.py deleted 100644 → 0
View file @ 3fa3d7c8
#
# Copyright (C) 2015 INRA
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
import os
import sys
import logging
import shutil
from ng6.ng6workflow import BasicNG6Workflow
from ng6.config_reader import NG6ConfigReader
from ng6.utils import Utils
from ng6.project import Project
from ng6.run import Run
from ng6.analysis import Analysis
from ng6.t3MySQLdb import t3MySQLdb
class SwitchProjectSpaceId (BasicNG6Workflow):
def get_description(self):
return """This workflow allows you to migrate a project from a space_id to another.
It updates the project's space_id in DB, moves the runs files, updates the runs and analyzes's files paths, and updates the corresponding retentions."""
def define_parameters(self, function="process"):
self.add_parameter('project_id', 'The project id', type=int, required = True)
self.add_parameter('space_id', 'The new space_id', required = True)
def process(self):
logging.getLogger("SwitchProjectSpaceId.process").debug("Begin SwitchProjectSpaceId workflow.")
#self.space_id = self.space_id
#self.project_id = self.project_id
self.project_id = int(self.project_id)
if self.project_id and self.space_id :
logging.getLogger("SwitchProjectSpaceId.process").debug("project_id =" + str(self.project_id) + " , new space_id = " + str(self.space_id) )
project = Project.get_from_id(self.project_id)
old_space_id = project.space_id
#We raise an error and return None if the project already has the given space_id
if str(old_space_id) == str(self.space_id) :
raise Exception("The project " + project.name + " already belongs to the space_id '" + str(self.space_id) + "'" )
return None
#Retrieve the old and new directories
ng6conf = NG6ConfigReader()
old_directory = ng6conf.get_space_directory(old_space_id)
try:
new_directory = ng6conf.get_space_directory(self.space_id)
except Exception as err:
logging.getLogger("SwitchProjectSpaceId.process").error("Error while searching space_id '" + str(self.space_id) + "' in application.properties, it certainly doesn't exist. \n" + str(err))
raise Exception("Error while searching space_id '" + str(self.space_id) + "' in application.properties, it certainly doesn't exist. \n" + str(err))
base_path = ng6conf.get_save_directory()
#Process for the project's runs
runs_ids = project.get_project_run_ids()
for run_id in runs_ids :
logging.getLogger("SwitchProjectSpaceId.process").debug("Migrating run " + str(run_id) )
run = Run.get_from_id(run_id)
path_old_run_directory = base_path + run.get_directory()
path_new_run_directory = path_old_run_directory.replace(old_directory,new_directory)
#Update the directory in DB
new_run_directory = run.get_directory()
new_run_directory = new_run_directory.replace(old_directory, new_directory)
#Moving the files
#if not os.path.isdir(path_new_run_directory) :
# os.mkdir(path_new_run_directory,0o775)
logging.getLogger("SwitchProjectSpaceId.process").debug("Migrating run files from " + str(path_old_run_directory) + " to " + str(path_new_run_directory) )
try:
shutil.move(path_old_run_directory, path_new_run_directory)
except Exception as err:
logging.getLogger("SwitchProjectSpaceId.process").error("Error while moving " + path_old_run_directory + " to " + path_new_run_directory + " \n" + str(err))
raise Exception("Error while moving " + path_old_run_directory + " to " + path_new_run_directory + " \n" + str(err))
#Delete the old directory if empty
if os.path.isdir(path_old_run_directory) and not os.listdir(path_old_run_directory) :
os.rmdir(path_old_run_directory)
#Update the retention in DB
old_retention = run.get_creation_date()
new_retention = ng6conf.compute_new_retention_date(self.space_id,old_retention)
run.set_retention(new_retention)
run.set_directory(new_run_directory)
#Process for the project's analyzes
analyzes_ids = project.get_project_analyzes_ids()
for analysis_id in analyzes_ids :
logging.getLogger("SwitchProjectSpaceId.process").debug("Migrating analysis " + str(analysis_id) )
analysis = Analysis.get_from_id(analysis_id)
path_old_analysis_directory = base_path + analysis.get_directory()
path_new_analysis_directory = path_old_analysis_directory.replace(old_directory,new_directory)
#Update the directory in DB
new_analysis_directory = analysis.get_directory()
new_analysis_directory = new_analysis_directory.replace(old_directory, new_directory)
#Moving the files
#if not os.path.isdir(path_new_analysis_directory) :
# os.mkdir(path_new_analysis_directory,0o775)
logging.getLogger("SwitchProjectSpaceId.process").debug("Migrating analysis files from " + str(path_old_analysis_directory) + " to " + str(path_new_analysis_directory) )
try:
shutil.move(path_old_analysis_directory, path_new_analysis_directory)
except Exception as err:
logging.getLogger("SwitchProjectSpaceId.process").error("Error while moving " + path_old_analysis_directory + " to " + path_new_analysis_directory + " \n" + str(err))
raise Exception("Error while moving " + path_old_analysis_directory + " to " + path_new_analysis_directory + " \n" + str(err))
#Delete the old directory if empty
if os.path.isdir(path_old_analysis_directory) and not os.listdir(path_old_analysis_directory) :
os.rmdir(path_old_analysis_directory)
#Update the retention in DB
old_retention = analysis.get_creation_date()
new_retention = ng6conf.compute_new_retention_date(self.space_id,old_retention)
analysis.set_retention(new_retention)
analysis.set_directory(new_analysis_directory)
#Process for the project's run's analyzes
run_analyzes_ids = project.get_project_runs_analyzes_ids()
for analysis_id in run_analyzes_ids :
logging.getLogger("SwitchProjectSpaceId.process").debug("Migrating analysis " + str(analysis_id) )
analysis = Analysis.get_from_id(analysis_id)
path_old_analysis_directory = base_path + analysis.get_directory()
path_new_analysis_directory = path_old_analysis_directory.replace(old_directory,new_directory)
#Update the directory in DB
new_analysis_directory = analysis.get_directory()
new_analysis_directory = new_analysis_directory.replace(old_directory, new_directory)
#Moving the files
#if not os.path.isdir(path_new_analysis_directory) :
# os.mkdir(path_new_analysis_directory,0o775)
logging.getLogger("SwitchProjectSpaceId.process").debug("Migrating analysis files from " + str(path_old_analysis_directory) + " to " + str(path_new_analysis_directory) )
try:
shutil.move(path_old_analysis_directory, path_new_analysis_directory)
except Exception as err:
logging.getLogger("SwitchProjectSpaceId.process").error("Error while moving " + path_old_analysis_directory + " to " + path_new_analysis_directory + " \n" + str(err))
raise Exception("Error while moving " + path_old_analysis_directory + " to " + path_new_analysis_directory + " \n" + str(err))
#Delete the old directory if empty
if os.path.isdir(path_old_analysis_directory) and not os.listdir(path_old_analysis_directory) :
os.rmdir(path_old_analysis_directory)
#Update the retention in DB
old_retention = analysis.get_creation_date()
new_retention = ng6conf.compute_new_retention_date(self.space_id,old_retention)
analysis.set_retention(new_retention)
analysis.set_directory(new_analysis_directory)
#Update to the projcet's space_id
project.update_space_id(self.space_id)
\ No newline at end of file
workflows/switch_space_id/components/move_project.py 0 → 100644
View file @ 0677a917
#
# Copyright (C) 2015 INRA
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
import logging
from jflow.component import Component
from workflows.types import ng6space,existingprojectid
from ng6.t3MySQLdb import t3MySQLdb
def migrate_project (project_id, new_space_id, output):
import os
from subprocess import call
import logging
from ng6.config_reader import NG6ConfigReader
from ng6.analysis import Analysis
from ng6.project import Project
from ng6.run import Run
with open( output,'w') as fh :
ng6conf = NG6ConfigReader()
base_path = ng6conf.get_save_directory()
project = Project.get_from_id(project_id)
logging.getLogger("migrate_project").debug("Migrating project '" + str(project.name) + "' (" + str(project.id) + ") to " + str(new_space_id) + "\n")
fh.write("Migrating project '" + str(project.name) + "' (" + str(project.id) + ") to " + str(new_space_id) + "\n")
old_space_id = project.space_id
fh.write("old_space_id = " + old_space_id + ", new_space_id = " + new_space_id + "\n")
str_cmds = []
runs = project.get_runs()
for run in runs :
fh.write("Migrating run '" + run.name + "' (" + str(run.id) + ")" + " to " + str(new_space_id) + "\n")
str_cmds.append(run.change_space(new_space_id) )
fh.write("Migrating successful! \n")
run_analyzes = run.get_analysis()
for analysis in run_analyzes :
fh.write("Migrating analysis '" + analysis.name + "' (" + str(analysis.id) + ")" + " to " + str(new_space_id) + "\n")
str_cmds.append(analysis.change_space(new_space_id) )
fh.write("Migrating successful! \n")
analyzes = project.get_analysis()
for analysis in analyzes :
fh.write("Migrating analysis '" + analysis.name + "' (" + str(analysis.id) + ")" + " to " + str(new_space_id) + "\n")
str_cmds.append(analysis.change_space(new_space_id) )
fh.write("Migrating successful! \n")
for command in str_cmds:
fh.write("Command launched : "+ str(command[1]) + "\n" )
fh.write("Returned code : "+ str(command[0]) + "\n" )
project.update_space_id(new_space_id)
class MoveProject (Component):
def define_parameters(self, project_id, space_id,):
self.add_parameter('project_id', 'The project id', type = existingprojectid, required = True, default = project_id )
self.add_parameter('space_id', 'The new space_id', type = ng6space, required = True, default = space_id)
self.add_output_file("stdout", "MoveProject stdout", filename="MoveProject.stdout")
def process(self):
logging.getLogger("MoveProject.process").debug(" Begin MoveProject Component.\n")
logging.getLogger("MoveProject.process").debug(" self.space_id="+str(self.space_id)+"\n")
logging.getLogger("MoveProject.process").debug(" self.project_id="+str(self.project_id)+"\n")
logging.getLogger("MoveProject.process").debug(" stdout = "+str(self.stdout)+"\n")
self.add_python_execution(migrate_project, cmd_format="{EXE} {ARG} {OUT}",
outputs = self.stdout, arguments = [self.project_id, self.space_id])
#self.add_python_execution(migrate_project, cmd_format="{EXE} {ARG}",
# arguments = [self.project_id, self.space_id])
workflows/types.py
View file @ 0677a917
......@@ -97,7 +97,8 @@ def ng6space(val):
ng6conf.get_space_directory(val)
except :
raise argparse.ArgumentTypeError("The space name '%s' is not define in application.properties" % val)
return val
def existingrun(id):
try:
t3mysql = t3MySQLdb()
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment