Commit db1b75b8 authored by Romain Therville's avatar Romain Therville 🐭

Huge update for the new migration workflow:

- New name : switch_space_id.
- The given space id is checked in application.properties.
- Most of the treatments have been moved to a "components" folder,
allowing us to create easily a new "migrate_several_projects" (or
"bulk_project_migration") later.
- The workflow can be launched from the single project view on the nG6
website, when logged a superadmin user.
- The project's space_id is visible to everyone on the single project
view.
- A warning indicating that the migration updates the retention has been
added to the workflow's form in single project view.
- More logs in ng6.log, and the new log file MoveProject.stdout. (path
exemple : [...]/jflow/work/switchspaceid/wf000452/MoveProject_default/MoveProject.stdout)
- CSS updates for the project views.
- For new space_ids, the <space_id>/run and <space_id>/analyze
directories will be created if missing.


Issue#129
parent 616f3286
......@@ -730,13 +730,18 @@ class Analysis (Component):
new_absolute_path = os.path.join(ng6conf.get_save_directory(), new_relative_path )
#We create the /analyze directory if it's missing
path_to_analyze_dir = os.path.join(ng6conf.get_save_directory(),ng6conf.get_space_directory(space_id),self.DIRECTORIES_STRUCTURE)
if not os.path.isdir(path_to_analyze_dir):
os.mkdir(path_to_analyze_dir,0o755)
str_cmd = ""
retcode = -1
if str(old_path) != str(new_absolute_path):
[retcode, str_cmd] = Utils.rsync_getcmd(old_path,new_absolute_path)
if retcode != 0 :
raise Exception("Error while trying to rsync " + old_path + " to " + new_absolute_path + "\n" +
"Command : " + str_cmd + "\n" + "Error code : " + retcode + "\n" + str(err) + "\n")
"Command : " + str_cmd + "\n" + "Error code : " + str(retcode) + "\n")
else:
print ("rsync could not be launched because the source and destination are the same, from " + old_path + " to " + new_absolute_path + "\n")
......
......@@ -139,6 +139,17 @@ class NG6ConfigReader(object):
raise ValueError("Failed while generating retention date!")
def get_available_spaces(self):
"""
return a list of space_ids
@return: a list containing the spaces strings
"""
available_spaces=[]
for each_section in self.reader.sections():
if each_section.startswith("space_") :
available_spaces.append(each_section.replace('space_',''))
return available_spaces
def get_log_file_path(self):
"""
return the log file path
......
......@@ -481,13 +481,18 @@ class Run(object):
new_absolute_path = os.path.join(ng6conf.get_save_directory(), new_relative_path )
#We create the /run directory if it's missing
path_to_run_dir = os.path.join(ng6conf.get_save_directory(),ng6conf.get_space_directory(space_id),self.DIRECTORIES_STRUCTURE)
if not os.path.isdir(path_to_run_dir):
os.mkdir(path_to_run_dir,0o755)
str_cmd = ""
retcode = -1
if str(old_path) != str(new_absolute_path):
[retcode, str_cmd] = Utils.rsync_getcmd(old_path,new_absolute_path)
if retcode != 0 :
raise Exception("Error while trying to rsync " + old_path + " to " + new_absolute_path + "\n" +
"Command : " + str_cmd + "\n" + "Error code : " + retcode + "\n" + str(err) + "\n")
"Command : " + str_cmd + "\n" + "Error code : " + str(retcode) + "\n")
else:
print ("rsync could not be launched because the source and destination are the same, from " + old_path + " to " + new_absolute_path + "\n")
......
......@@ -1148,7 +1148,8 @@ INNER JOIN fe_groups ON fe_groups.uid = fe_users.usergroup',
'hidden' => $vals['hidden'],
'date' => $vals['crdate'],
'public' => $vals['public'],
'description' => $vals['description']);
'description' => $vals['description'],
'space_id' => $vals['space_id']);
}
/**
......
......@@ -43,6 +43,14 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
<div class="ng6-content-header-left project">
<h2>Projet <small>{$projects[key($projects)].name}</small></h2>
<div><p>Current space: <b><span id="current_space_id">{$projects[key($projects)].space_id}</span></b></p></div>
{if $is_ng6_superadmin}
<div id="change_space_id_warning"><p><i>Warning, migrating a project from one space to another will update both the files location and the retention limits.</i></p></div>
<div id="change_space_id_form"></div>
<div id="wf_id"></div>
<div id="change_space_id_status"></div>
<div id="change_space_id_footer" align="right"></div>
{/if}
</div>
<div class="ng6-content-header-right">
{assign var="manag_values" value="{$managment_purged_data[$projects[key($projects)].id]}"}
......@@ -52,7 +60,7 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
<br />
{assign var="project_size" value="<span id='size' class='tx-nG6-mini-wait'></span>"}
Raw data and analysis results use <strong>{$project_size}</strong> on the hard drive for the whole project.<br />
New data added to this project will be kept{$retention_policy}. <br />
New data added to this project will be kept<b>{$retention_policy}</b>. <br />
<br><label>Data overview</label><br>
<table class="table table-striped table-bordered dataTable" id="manag_purged_data_table">
......@@ -290,6 +298,7 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
<div class="sub-content sc-top">
<div class="ng6-content-header-left project">
<h2>Projects list <small> you can access </small></h2>
</div>
<div class="ng6-content-header-right">
You have access to <strong>{$projects|@count}</strong> projects.
......
......@@ -1028,6 +1028,8 @@ $(function () {
});
});
setChangeSpaceForm();
});
......@@ -2182,6 +2184,146 @@ function addFileHandler( event ){
$("#refresh_workflow").bind('click', {'selector' : '#monitoringWorkflow' }, refreshWFStatusHandler);
}
function setChangeSpaceForm(){
var _get_workflow_status = function(workflow_id, callback, error_callback){
$.ajax({
url : $("#server_url").val() + '/get_workflow_status?display=list&workflow_id=' + workflow_id ,
dataType : 'jsonp',
timeout: 20000 ,
success : function(data){
if (callback){
callback(data);
}
},
error : function(jqXHR, textStatus, errorThrown){
if (error_callback){
error_callback(jqXHR, textStatus, errorThrown);
}
}
});
};
var wfform_options = {
workflowClass: "SwitchSpaceId",
serverURL: $("#server_url").val(),
displayRunButton: false,
displayResetButton: false,
parameters : {
"admin_login" : $("#user_login").val(),
"project_id" : $("#ids").val()
}
};
//$("#change_space_id_status").hide();
$('#change_space_id_status').html([
' <div id="myCarousel" class="carousel slide">',
' <div class="carousel-inner">',
' </div>',
' </div>'
].join(''));
$("#change_space_id_form").wfform(wfform_options);
$('#change_space_id_footer').html([
' <button id="run_workflow" class="btn btn-primary"><i class="glyphicon glyphicon-cog"></i> Migrate</button>',
' <button id="refresh_workflow" class="btn btn-default"><i class="glyphicon glyphicon-refresh"></i> Refresh</button>',
].join(''));
//Show the footer, but hide the "refresh"button until the user clicks "Migrate"
$("#change_space_id_footer").show();
$("#refresh_workflow").hide();
//When clicking 'migrate'
$("#run_workflow").click(function(){
$('#change_space_id_form').wfform('run');
});
//The status label has to be display just for a few seconds each time we hit "Migrate" or "Refresh"
$("#change_space_id_status").hide();
//When clicking 'refresh'
$("#refresh_workflow").click(function(){
//The workflow's id has been saved in the hidden div <div id="wf_id">
_get_workflow_status($("#wf_id").html(),
//success
function(data) {
if (data.status=="completed"){
$("#change_space_id_status").show();
$("#change_space_id_status").html("Workflow status : " + data.status);
setTimeout(function(){
$("#change_space_id_status").hide();
},5000);
$("#current_space_id").html($("#space_id").val());
$("#refresh_workflow").hide();
$("#run_workflow").show();
$("#workflow_form").show();
}
else{
$("#change_space_id_status").show();
$("#change_space_id_status").html("Workflow status : " + data.status);
setTimeout(function(){
$("#change_space_id_status").hide();
},5000);
}
},
//error
function() {
$("#change_space_id_status").html("erreur requete ajax");
}
);
});
// The 'run' function
$('#change_space_id_form').on('run.wfform', function(event, running_wf) {
$("#run_workflow").hide();
$("#workflow_form").hide();
//We will need the workflow's id to check its status with the "refresh" button
$("#wf_id").hide();
$("#wf_id").html(running_wf.id);
$("#refresh_workflow").show();
_get_workflow_status(running_wf.id,
//success
function(data) {
if (data.status=="completed"){
$("#change_space_id_status").show();
$("#change_space_id_status").html("Workflow status : " + data.status);
setTimeout(function(){
$("#change_space_id_status").hide();
},5000);
$("#current_space_id").html($("#space_id").val());
$("#refresh_workflow").hide();
$("#run_workflow").show();
$("#workflow_form").show();
}
else{
$("#change_space_id_status").show();
$("#change_space_id_status").html("Workflow status : " + data.status);
setTimeout(function(){
$("#change_space_id_status").hide();
},5000);
}
},
//error
function() {
$("#change_space_id_status").html("erreur requete ajax");
}
);
});
}
function addAnalysisHandler(){
$('#setAndRunModalLabel').html("Loading");
......
......@@ -71,7 +71,8 @@ a img { border: 0; }
box-shadow: -4px 0 5px -5px grey, 5px 0 5px -6px grey;
margin: 0;
padding: 10px;
width: 300px;
width: 375px;
height: 275px;
text-align: justify;
float: left;
background-color: #FaFaFa;
......
#
# Copyright (C) 2012 INRA
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
import os
import sys
import logging
from ng6.ng6workflow import BasicNG6Workflow
from ng6.project import Project
from jflow.utils import display_error_message
from workflows.types import ng6space,existingprojectid
from ng6.config_reader import NG6ConfigReader
class SwitchSpaceId (BasicNG6Workflow):
def get_description(self):
return """This workflow allows you to migrate a project from a space_id to another.
It updates the project's space_id in DB, moves the runs files, updates the runs and analyzes's files paths, and updates the corresponding retentions."""
def define_parameters(self, function="process"):
ng6conf = NG6ConfigReader()
space_choices = ng6conf.get_available_spaces()
self.add_parameter('project_id', 'The project id', type=existingprojectid, required = True )
self.add_parameter('space_id', 'The new space_id', required = True, type=ng6space, choices = space_choices)
def process(self):
self.project_id = int(self.project_id)
project = Project.get_from_id(self.project_id)
if project is not None :
old_space_id = project.space_id
#We log a message if the project already has the given space_id
if str(old_space_id) == str(self.space_id) :
logging.getLogger("SwitchProjectSpaceId.process").debug("The project " + project.name + " already belongs to the space_id '" + str(self.space_id) + "'\n" )
add = self.add_component("MoveProject", [ self.project_id, self.space_id] )
......@@ -40,30 +40,39 @@ def migrate_project (project_id, new_space_id, output):
fh.write("Migrating project '" + str(project.name) + "' (" + str(project.id) + ") to " + str(new_space_id) + "\n")
old_space_id = project.space_id
logging.getLogger("migrate_project").debug("old_space_id = " + old_space_id + ", new_space_id = " + new_space_id + "\n")
fh.write("old_space_id = " + old_space_id + ", new_space_id = " + new_space_id + "\n")
str_cmds = []
runs = project.get_runs()
for run in runs :
fh.write("Migrating run '" + run.name + "' (" + str(run.id) + ")" + " to " + str(new_space_id) + "\n")
logging.getLogger("migrate_project").debug("Migrating run '" + run.name + "' (" + str(run.id) + ")" + " to " + str(new_space_id) + "\n")
str_cmds.append(run.change_space(new_space_id) )
fh.write("Migrating successful! \n")
logging.getLogger("migrate_project").debug("Migrating successful! \n")
run_analyzes = run.get_analysis()
for analysis in run_analyzes :
fh.write("Migrating analysis '" + analysis.name + "' (" + str(analysis.id) + ")" + " to " + str(new_space_id) + "\n")
logging.getLogger("migrate_project").debug("Migrating analysis '" + analysis.name + "' (" + str(analysis.id) + ")" + " to " + str(new_space_id) + "\n")
str_cmds.append(analysis.change_space(new_space_id) )
fh.write("Migrating successful! \n")
logging.getLogger("migrate_project").debug("Migrating successful! \n")
analyzes = project.get_analysis()
for analysis in analyzes :
fh.write("Migrating analysis '" + analysis.name + "' (" + str(analysis.id) + ")" + " to " + str(new_space_id) + "\n")
logging.getLogger("migrate_project").debug("Migrating analysis '" + analysis.name + "' (" + str(analysis.id) + ")" + " to " + str(new_space_id) + "\n")
str_cmds.append(analysis.change_space(new_space_id) )
fh.write("Migrating successful! \n")
logging.getLogger("migrate_project").debug("Migrating successful! \n")
for command in str_cmds:
fh.write("Command launched : "+ str(command[1]) + "\n" )
logging.getLogger("migrate_project").debug("Command launched : "+ str(command[1]) + "\n" )
fh.write("Returned code : "+ str(command[0]) + "\n" )
logging.getLogger("migrate_project").debug("Returned code : "+ str(command[0]) + "\n" )
project.update_space_id(new_space_id)
......@@ -82,7 +91,7 @@ class MoveProject (Component):
logging.getLogger("MoveProject.process").debug(" stdout = "+str(self.stdout)+"\n")
self.add_python_execution(migrate_project, cmd_format="{EXE} {ARG} {OUT}",
outputs = self.stdout, arguments = [self.project_id, self.space_id])
outputs = self.stdout, arguments = [self.project_id, self.space_id], local = True)
#self.add_python_execution(migrate_project, cmd_format="{EXE} {ARG}",
# arguments = [self.project_id, self.space_id])
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment