Commit bbcdafa4 authored by Penom Nom's avatar Penom Nom
Browse files

new pipeline 'addfile' to add a file to an existing analysis or run

parent 940ecf37
......@@ -195,11 +195,12 @@ class Analysis (Component):
Synchronize data related to the Analysis between temporary folder and long term storage folder.
"""
if os.path.isdir(self.__get_work_directory()):
if not os.path.isdir(self.__get_save_directory()) and len(os.listdir(self.__get_work_directory())):
if len(os.listdir(self.__get_work_directory())):
logging.getLogger("Analysis.sync").debug("Synchronizing analysis id=" + str(self.id) + " from " + self.__get_work_directory() + " to " + self.__get_save_directory())
try :
# First make the save directory
os.makedirs(self.__get_save_directory(), 0751)
if not os.path.isdir(self.__get_save_directory()) :
# First make the save directory
os.makedirs(self.__get_save_directory(), 0751)
if os.path.exists(self.__get_work_directory()) and self.__get_work_directory() != self.__get_save_directory():
cmd = "rsync -avh --remove-sent-files "+ self.__get_work_directory() + "/ " + self.__get_save_directory()
logging.getLogger("Analysis.sync").debug(cmd)
......@@ -537,6 +538,10 @@ class Analysis (Component):
@param gzip : if True the file will be gziped
@return : the web path to the file
"""
if not os.path.isdir(self.__get_work_directory()):
os.makedirs(self.__get_work_directory(), 0751)
# First copy the file into the analysis directory
if os.path.isfile(file) :
if file_name == None:
......
......@@ -148,6 +148,11 @@ class Run(object):
@param archive_name: the archive name if tar is requested
@param delete: delete files
"""
# create workdir if it does not exists
if not os.path.isdir(self.__get_work_directory()) :
os.makedirs(self.__get_work_directory(), 0751)
# First handle if only one file
if len(files) == 1 and mode == "tar.gz":
mode = "gz"
......@@ -231,11 +236,13 @@ class Run(object):
logging.getLogger("Run.sync").error("Error when synchronizing analysis that belongs to run id=" + str(self.id))
raise Exception(str(err))
# If a sync is required
if not os.path.isdir(self.__get_save_directory()) and len(os.listdir(self.__get_work_directory())):
if len(os.listdir(self.__get_work_directory())):
# Finally the Run itself
try :
# First make the save directory
os.makedirs(self.__get_save_directory(), 0751)
# not created for updates
if not os.path.isdir(self.__get_save_directory()) :
# First make the save directory
os.makedirs(self.__get_save_directory(), 0751)
if os.path.exists(self.__get_work_directory()) and self.__get_work_directory() != self.__get_save_directory():
cmd = "rsync -avh --remove-sent-files "+ self.__get_work_directory() + "/ " + self.__get_save_directory()
logging.getLogger("Run.sync").debug(cmd)
......@@ -248,7 +255,7 @@ class Run(object):
logging.getLogger("Run.sync").debug("Synchronization done for run id=" + str(self.id) + "(retcode=" + str(retcode) + ")")
# update the storage size
t3mysql = t3MySQLdb()
t3mysql.update_run_info(self.id, storage_size=Utils.get_storage_size(self.__get_save_directory()))
t3mysql.update_run_info(self.id, storage_size=Utils.get_storage_size(self.__get_save_directory()))
except Exception, err:
raise Exception(str(err))
......
......@@ -97,6 +97,19 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
<div id="modal-foot-tmpl" class="modal-footer"></div>
</div>
{* setAndRunModal *}
<div id="setAndRunModal" class="modal hide fade" tabindex="-1" role="dialog" aria-labelledby="setAndRunModalLabel" aria-hidden="true">
<div class="modal-header">
<button type="button" class="close" data-dismiss="modal" aria-hidden="true">x</button>
<h3 id="setAndRunModalLabel"></h3>
</div>
<div id="setAndRunModalBody" class="modal-body"></div>
<div class="modal-footer">
<button id="reset_workflow" class="btn"><i class="icon-refresh"></i> Reset</button>
<button id="run_workflow" class="btn btn-primary"><i class="icon-cog icon-white"></i> Run</button>
</div>
</div>
{* Error only 1 raw modal *}
<div id="ng6modal-1raw" class="modal hide fade" tabindex="-1" role="dialog" aria-labelledby="modalLabel-1raw" aria-hidden="true">
<div class="modal-header">
......
......@@ -17,6 +17,31 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
{extends file='AnalysisTemplate.tpl'}
{block name=description}
<div class="sub-content sc-top">
<div class="ng6-content-header-left analysis">
<h2>Analysis <small>{$analyse.name}</small></h2>
</div>
<div class="ng6-content-header-right">
{$analyse.description}
<br />
All data related to this analysis use <strong> {$analyse_size} </strong> on the hard drive.
<br/>
<div style="float:right;margin-right:25px">
{if $is_project_admin }
<button id="add_file" type="button" class="btn btn-small btn-primary"><i class="icon-plus icon-white"></i> add file</button>
{/if}
</div>
</div>
<div style="clear:both"></div>
</div>
{/block}
{block name=content}
<ul id="myTab" class="nav nav-tabs">
......
......@@ -286,6 +286,12 @@ class tx_nG6_pi1 extends tslib_pibase {
* Return the analysis view
*/
function pi_analyze_view() {
if (!$GLOBALS['TSFE']->loginUser) { $user_id = null; }
else { $user_id = $GLOBALS['TSFE']->fe_user->user['uid']; }
// is project admin
$is_project_admin = tx_nG6_db::is_project_administrator($GLOBALS['TSFE']->fe_user->user['uid'], $this->piVars['project_id']);
// First select the analyse
$analyse = tx_nG6_db::select_analyse($this->piVars['analyze_id']);
......@@ -312,6 +318,7 @@ class tx_nG6_pi1 extends tslib_pibase {
$smarty->security = true;
$smarty->security_settings['MODIFIER_FUNCS'] = array('count');
$smarty->assign('is_project_admin', $is_project_admin);
$smarty->assign('analyse', $analyse);
$analysis_size = tx_nG6_db::get_analysis_size($this->piVars['analyze_id']);
$smarty->assign('analyse_size', tx_nG6_utils::get_octet_string_representation($analysis_size));
......
......@@ -61,6 +61,14 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
{else}
<strong>{$run_analysis|@count}</strong> analysis has been done on this run.
{/if}
<br/>
<div style="float:right;margin-right:25px">
{if $runs[key($runs)].is_admin }
<button id="add_file" type="button" class="btn btn-small btn-primary"><i class="icon-plus icon-white"></i> add file</button>
{/if}
</div>
</div>
<div style="clear:both"></div>
</div>
......
......@@ -284,7 +284,10 @@ $(function () {
/* add a brand new run option */
$(":button[id=add_run]").on('click', addRunHandler);
/* add a result file to an existing run or analysis*/
$(":button[id=add_file]").on('click', addFileHandler);
/* add a brand new analysis option */
$(":button[id=add_analysis]").on('click', addAnalysisHandler);
......@@ -1633,7 +1636,7 @@ function addRunHandler() {
$('#availableWorkflows').html("wait");
$('#setAndRunModal').modal();
$('#availableWorkflows').availablewf({
filters:["AddAnalysis", "AddProject"],
filters:["AddAnalysis", "AddProject", "AddFiles"],
serverURL: $("#server_url").val()
});
$('#availableWorkflows').on('select', function(event, workflow) {
......@@ -1798,6 +1801,81 @@ function refreshAnalysisTable() {
});
}
function addFileHandler( event ){
$('#setAndRunModalLabel').html("Loading");
$("#reset_workflow").hide();
$("#run_workflow").hide();
var carousel = '<div id="myCarousel" class="carousel slide">';
carousel += '<div class="carousel-inner">';
carousel += '<div id="wfForm" class="active item"></div>';
carousel += '<div id="monitoringWorkflow" class="item"></div>';
carousel += '</div>';
carousel += '</div>';
$('#setAndRunModalBody').html(carousel);
var params = simpleQueryParams(location.href),
page_id = params["id"],
run_id = params["tx_nG6_pi1[run_id]"],
analysis_id = params["tx_nG6_pi1[analyze_id]"],
parameters = {};
// define default parameters so the user do not have to fill them
if (analysis_id) {
parameters["analysis_id"] = analysis_id;
parameters["run_id"] = '';
}
else if (run_id) {
parameters["run_id"] = run_id;
parameters["analysis_id"] = '';
}
if ($("#close").size() == 0) {
$('<button id="close" class="btn" data-dismiss="modal"><i class="icon-remove"></i> Close</button>').insertBefore("#reset_workflow");
}
if ($("#refresh_workflow_status").size() == 0) {
$('<button id="refresh_workflow_status" class="btn"><i class="icon-refresh"></i> Refresh</button>').insertBefore("#reset_workflow");
}
$("#refresh_workflow_status").hide();
$("#close").show();
$('#wfForm').on("loaded", function(event, workflow) {
$('#setAndRunModalLabel').html(workflow["name"] + " <small>" + workflow["help"] + "</small>");
$("#reset_workflow").show();
$("#run_workflow").show();
$("#close").hide();
});
parameters["metadata"] = [];
parameters["admin_login"] = $("#user_login").val();
$('#wfForm').wfform({
workflowClass: "AddFile",
displayRunButton: false,
displayResetButton: false,
parameters: parameters,
serverURL: $("#server_url").val()
});
$('#wfForm').on('run', function(event, running_wf) {
$("#reset_workflow").hide();
$("#run_workflow").hide();
$("#refresh_workflow_status").show();
$("#myCarousel").carousel('next');
$("#myCarousel").carousel('pause');
$('#setAndRunModalLabel').html(running_wf["name"] + " <small>" + running_wf["id"] + "</small>");
$('#monitoringWorkflow').wfstatus({
workflowID: running_wf.id,
forceUsingWorkflow: running_wf,
serverURL: $("#server_url").val()
});
refreshAnalysisTable();
});
$('#setAndRunModal').modal();
unbindAll();
$("#reset_workflow").bind('click', resetAnalysisWorkflowHandler);
$("#run_workflow").bind('click', runAnalysisWorkflowHandler);
$("#refresh_workflow_status").bind('click', refreshAnalysisWorkflowHandler);
}
function addAnalysisHandler(){
$('#setAndRunModalLabel').html("Loading");
$("#reset_workflow").hide();
......
#
# Copyright (C) 2012 INRA
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
import os
import sys
from ng6.ng6workflow import BasicNG6Workflow
from ng6.project import Project
from ng6.run import Run
from ng6.analysis import Analysis
class AddFile (BasicNG6Workflow):
def process(self):
# check if files provided exists
files_to_save = []
for file in self.args["data_file"]:
if os.path.isfile(file):
files_to_save.append(file)
else:
sys.exit("error: %s file does not exists\n" % (file))
self.runobj = None
self.analysisobj = None
if self.args['run_id'] :
self.runobj = Run.get_from_id(self.args["run_id"])
self.metadata.append("run_id="+str(self.args["run_id"]))
add = self.add_component("AddRawFiles", [ self.runobj, files_to_save, self.args["compression"] ] )
elif self.args['analysis_id'] :
self.analysisobj = Analysis.get_from_id(self.args["analysis_id"])
self.metadata.append("analysis_id="+str(self.args["analysis_id"]))
add = self.add_component("AddAnalysisFiles", [ self.analysisobj, files_to_save, True if self.args["compression"] == "gz" else False] )
else :
sys.stderr.write("One of --run-id or --analysis-id parameter is required!\n")
sys.exit()
def post_process(self):
# once everything done, sync directories
if self.runobj:
self.runobj.sync()
elif self.analysisobj:
self.analysisobj.sync()
#
# Copyright (C) 2012 INRA
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
[global]
name = addfile
description = add file to an existing analysis or run
#
# Parameter section
# param.name: the parameter display name
# .flag: the command line flag to use the argument
# .help: a brief description of what the parameter does
# .default [None]: the value produced if the parameter is not provided
# .type [str]: the parameter type that should be tested (str|int|date|localfile|bool|... all types defined in the types.py package)
# .choices [None]: a container of the allowable values for the parameter
# .required [False]: whether or not the command-line option may be omitted
# .action [store]: the basic type of action to be taken (store|append)
# .group [None]: gathers arguments into groups when displaying help messages / forms
# .exclude [None]: will make sure that there is only one arguments provided
#
[parameters]
data_file.name = data_file
data_file.flag = --data-file
data_file.help = Which data files should be added to the analysis or the run
data_file.type = localfile
data_file.required = True
data_file.action = append
compression.name = compression
compression.flag = --compression
compression.help = How should data be compressed once archived
compression.default = none
compression.choices = none|gz|bz2
delete.name = delete
delete.flag = --delete
delete.help = Should the input data be deleted once the process is over
delete.default = False
delete.type = bool
analysis_id.name = analysis_id
analysis_id.flag = --analysis-id
analysis_id.help = The id of the analysis
analysis_id.type = int
analysis_id.exclude = run_id
run_id.name = run_id
run_id.flag = --run-id
run_id.help = The id of the run
run_id.type = int
run_id.exclude = analysis_id
#
# Copyright (C) 2012 INRA
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
import os
import pickle
from jflow.component import Component
from jflow.iotypes import OutputFile, InputFileList
from weaver.function import PythonFunction
from ng6.analysis import Analysis
def add_analysis_file (analysis_dump_path, files_dump_path, gzip):
import pickle
# load the analysis object
analysis_dump = open(analysis_dump_path, "rb")
my_analysis = pickle.load(analysis_dump)
analysis_dump.close()
files_dump = open(files_dump_path, "rb")
files_to_save = pickle.load(files_dump)
files_dump.close()
my_analysis._save_files(files_to_save, gzip)
print my_analysis.id
class AddAnalysisFiles (Component):
def define_parameters(self, analysisobj, files_to_save, gzip = True):
self.analysisobj = analysisobj
self.files_to_save = InputFileList(files_to_save)
self.gzip = gzip
self.stdout = OutputFile(os.path.join(self.output_directory, "addAnalysisFiles.stdout"))
def process(self):
self.analysisobj.raw_files = self.files_to_save
analysis_dump_path = self.get_temporary_file(".dump")
analysis_dump = open(analysis_dump_path, "wb")
pickle.dump(self.analysisobj, analysis_dump)
analysis_dump.close()
files_dump_path = self.get_temporary_file(".dump")
files_dump = open(files_dump_path, "wb")
pickle.dump(self.files_to_save, files_dump)
files_dump.close()
add = PythonFunction(add_analysis_file, cmd_format='{EXE} {ARG} > {OUT}')
add(outputs=self.stdout, includes=self.files_to_save, arguments=[analysis_dump_path, files_dump_path, self.gzip])
\ No newline at end of file
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment