Commit d0e476b9 authored by Penom Nom's avatar Penom Nom
Browse files

Split download into multiple workflows.

parent 7caf1e6c
......@@ -66,7 +66,7 @@ tmp_directory = <path>/tmp
[local]
# This is a space separated list of workflows Class names that will not be available
# for adding a run
runwf_filter = AddAnalysis AddProject AddFiles RADseq Download
runwf_filter = AddAnalysis AddProject AddFiles RADseq DownloadArchive DownloadSymlink
# This must be an absolute path of a directory mounted and accessible by the server
server_directory = /work/
......
......@@ -117,6 +117,17 @@ class BasicNG6Workflow (Workflow):
self.set_to_address(infos['email'])
class DownloadWorkflow(BasicNG6Workflow):
"""
Base class for download workflows classes
"""
def __init__(self, args={}, id=None, function= "process"):
BasicNG6Workflow.__init__(self, args, id, function)
self.add_parameter_list('data_id', 'Ids of a run from which rawdata will be retrieved', type = 'existingrun')
self.add_parameter_list('run_id', 'Ids of run from which all data will be retrieved', type = 'existingrun')
self.add_parameter_list('analysis_id', 'Ids of analysis to retrieve', type = 'existinganalysis')
class AnalysisWorkflow(BasicNG6Workflow, ComponentWorkflow):
"""
A workflow with a single analysis
......
......@@ -30,19 +30,38 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
<h2>Download <small> center </small> </h2>
</div>
<div class="ng6-content-header-right">
Select the data you want to download, select the way you want to get the data by
choosing the format on the right, then click on the download button.
<br /> <br />
<h5>Download Files list :</h5>
<h4>How to download your data</h4>
<p>
<strong>(i)</strong> Use the select tree bellow to select the data you want to download. You can expand the tree using the arrows on the left.
<strong>(ii)</strong> Then select the way you want to download your data using the dropdown menu.<strong>(iii)</strong> Click on the download button
to start the download.
</p>
<form class="form-horizontal">
<div class="form-group">
<label class="col-sm-4 control-label">Available downloads</label>
<div class="col-sm-4">
<select id="available-download-types" class="form-control">
</select>
</div>
</div>
<div class="form-group">
<div class="col-sm-4"></div>
<div class="col-sm-4"></div>
<div class="col-sm-4">
<button id="download_btn" class="btn btn-default disabled" type="button"><i class="glyphicon glyphicon-floppy-save"></i> Download</button>
</div>
</div>
</form>
<h4>Downloaded files : </h4>
<div id="download_list">
<p>
No run neither any analyze selected. <br/>
Please select below the data you want to download...
</p>
</div>
<div style="float:right;margin-top:10px">
<button id="download_btn" class="btn btn-default disabled" type="button">Download</button>
</div>
</div>
<div style="clear:both"></div>
</div>
......
......@@ -406,7 +406,7 @@ $(function () {
$("#modal-label-tmpl").html("Update user <small>" + user_login + "</small>");
var update_user_html = [
'<div id="error_message" class="alert alert-error">',
'<div id="error_message" class="alert alert-danger">',
' <button class="close" data-dismiss="alert" type="button">x</button>',
'</div>',
'<div class="tx-nG6-pi1-update-user" >Update user information. For the laboratory you can use autocomplete to seek for an already existing one or create a new one using the formular<br /> <br />' ,
......@@ -605,7 +605,7 @@ $(function () {
$("#modal-label-tmpl").html("Add a user to the project");
var add_user_html = [
'<div id="error_message" class="alert alert-error">',
'<div id="error_message" class="alert alert-danger">',
' <button class="close" data-dismiss="alert" type="button">x</button>',
'</div>',
'<div class="tx-nG6-pi1-add-new">To add a user to the project, seek the user using the autocompletion fields or fill the form to create a brand new user. <br /> <br /> ',
......@@ -962,7 +962,7 @@ $(function () {
location.assign(location.href);
},
error: function (xhr, ajaxOptions, thrownError) {
var html = '<div id="message" class="alert alert-error"><strong>Error!</strong> An error occured when attempting to finalize installation.<button class="close" data-dismiss="alert" type="button" >x</button></div>';
var html = '<div id="message" class="alert alert-danger"><strong>Error!</strong> An error occured when attempting to finalize installation.<button class="close" data-dismiss="alert" type="button" >x</button></div>';
$("#message").html(html).removeClass("hidden");
}
});
......@@ -970,7 +970,7 @@ $(function () {
});
$("#btn-install").click( function() {
if ($("#username_val").val() == "admin_install") {
var html = '<div id="message" class="alert alert-error"><strong>Login error!</strong> This login is not allowed.<button class="close" data-dismiss="alert" type="button" >x</button></div>';
var html = '<div id="message" class="alert alert-danger"><strong>Login error!</strong> This login is not allowed.<button class="close" data-dismiss="alert" type="button" >x</button></div>';
$("#message").html(html).removeClass("hidden");
} else {
$("#install_form").submit();
......@@ -1826,7 +1826,7 @@ function refreshRunTable() {
url: 'index.php?eID=tx_nG6&type=runs_table&project_id='+$("#current_project_id").val()+'&user_id='+$("#user_id").val()+'&login_user='+$("#login_user").val()+'&page_id='+page_id,
dataType: "html",
error: function (xhr, ajaxOptions, thrownError) {
var html = '<div class="alert alert-error"><strong>Error!</strong> An error occured when attempting to reload the runs table.</div>';
var html = '<div class="alert alert-danger"><strong>Error!</strong> An error occured when attempting to reload the runs table.</div>';
$("#runs").html(html);
},
success: function(data) {
......@@ -1848,7 +1848,7 @@ function refreshRunTable() {
$("#add_run").on('click', addRunHandler);
addWorkflowsStatusOnRun();
} else {
var html = '<div class="alert alert-error"><strong>Error!</strong> An error occured when attempting to reload the runs table.</div>';
var html = '<div class="alert alert-danger"><strong>Error!</strong> An error occured when attempting to reload the runs table.</div>';
$("#runs").html(html);
}
}
......@@ -1894,7 +1894,7 @@ function addRunHandler() {
url: $("#server_url").val() + "/get_workflow_filters?",
dataType: "jsonp",
error: function () {
$('#availableWorkflows').html('<div class="alert alert-error"><strong>Error!</strong> An error occured when attempting to get workflows.</div>');
$('#availableWorkflows').html('<div class="alert alert-danger"><strong>Error!</strong> An error occured when attempting to get workflows.</div>');
},
success: function(data) {
$('#availableWorkflows').availablewf({
......@@ -1985,7 +1985,7 @@ function addProjectHandler(){
url: 'index.php?eID=tx_nG6&type=projects_table&user_id='+$("#user_id").val()+'&login_user='+$("#login_user").val()+'&page_id='+page_id,
dataType: "html",
error: function (xhr, ajaxOptions, thrownError) {
var html = '<div class="alert alert-error"><strong>Error!</strong> An error occured when attempting to reload the projects table.</div>';
var html = '<div class="alert alert-danger"><strong>Error!</strong> An error occured when attempting to reload the projects table.</div>';
$("#projects").html(html);
},
success: function(data) {
......@@ -2009,7 +2009,7 @@ function addProjectHandler(){
$("#add_project").on('click', addProjectHandler);
$('#setAndRunModal').modal('hide');
} else {
var html = '<div class="alert alert-error"><strong>Error!</strong> An error occured when attempting to reload the projects table.</div>';
var html = '<div class="alert alert-danger"><strong>Error!</strong> An error occured when attempting to reload the projects table.</div>';
$("#projects").html(html);
}
}
......@@ -2039,7 +2039,7 @@ function refreshAnalysisTable() {
url: 'index.php?eID=tx_nG6&type=analyses_table&user_id='+$("#user_id").val()+'&login_user='+$("#login_user").val()+'&page_id='+page_id+"&project_id="+parameters["project_id"]+"&run_id="+parameters["run_id"],
dataType: "html",
error: function (xhr, ajaxOptions, thrownError) {
var html = '<div class="alert alert-error"><strong>Error!</strong> An error occured when attempting to reload the analysis table.</div>';
var html = '<div class="alert alert-danger"><strong>Error!</strong> An error occured when attempting to reload the analysis table.</div>';
$("#analyses").html(html);
},
success: function(data) {
......@@ -2060,7 +2060,7 @@ function refreshAnalysisTable() {
$("#unhide_analysis").on('click', unhideHandler);
$("#add_analysis").on('click', addAnalysisHandler);
} else {
var html = '<div class="alert alert-error"><strong>Error!</strong> An error occured when attempting to reload the analysis table.</div>';
var html = '<div class="alert alert-danger"><strong>Error!</strong> An error occured when attempting to reload the analysis table.</div>';
$("#analyses").html(html);
}
}
......
......@@ -145,7 +145,164 @@ $(function () {
}
});
// List of available downloads
var download_options = [
// archive on server
{
label : "archive",
id : "download_archive",
wfName : "DownloadArchive",
onRun : function(event, running_wf, $elementHeader, $elementBody, $elementFooter){
$elementBody.html([
'<div>',
' <div class="alert alert-info">',
' <p>',
' <strong>The creation</strong> of your archive is in progress. You will recieve an email with a downloadable link of your archive when the process is done.',
' You can either close this window, or use the refresh button to view the status. The process can <strong>take a long time</strong> depending on the size of your data.',
' Either way you will recieve an email with the downloadable link.',
' </p>',
' </div>',
' <div id="wfstatus">',
' </div>',
'</div>'
].join(''));
$elementFooter.html([
' <div class="btn-group">',
' <button id="refresh_workflow" class="btn btn-default"><i class="glyphicon glyphicon-refresh"></i> Refresh</button>',
' </div>',
' <div class="btn-group">',
' <button id="close" class="btn btn-default" data-dismiss="modal"><i class="glyphicon glyphicon-remove"></i> Close</button>',
' </div>',
].join(''));
$('#wfstatus').wfstatus({
workflowID : running_wf.id,
display : 'list',
})
$("#refresh_workflow").click(function(){
$('#wfstatus').wfstatus('reload');
// try to get outputs
$.ajax({
url : $("#server_url").val() + '/get_workflow_outputs?workflow_id=' + running_wf.id,
dataType : 'jsonp',
success : function(data){
$elementBody.html("<div class='tx-nG6-wait'>Please wait while retrieving your archive</div>");
$.each(data, function(i, component) {
$.each(component, function(i, file) {
if (file.extension == '.gz'){
$("#myModal").modal('hide');
location.href = file.url;
return;
}
});
});
},
});
});
}
},
// symlink
{
label : "symbolic link",
id : "download_symlink",
wfName : "DownloadSymlink",
onRun : function(event, running_wf, $elementHeader, $elementBody, $elementFooter){
$("#myModalBody").html("<div class='tx-nG6-wait'> <strong>Please</strong> wait while your links are being created.</div>");
var get_workflow_status = function(serverurl, workflow_id, callback, errcb){
$.ajax({
url : serverurl + '/get_workflow_status?display=list&workflow_id=' + workflow_id ,
dataType : 'jsonp',
timeout: 5000 , // More than enough to submit the request
success : function(data){
if (callback){
callback(data);
}
},
error : function(jqXHR, textStatus, errorThrown){
if (errcb){
errcb(jqXHR, textStatus, errorThrown);
}
}
});
};
var cb = function(data){
var wfstatus = data.status
if (wfstatus == "completed"){
$("#myModalBody").html([
'<div class="alert alert-success">',
' <p><strong>Success !</strong> Your links have been created.</p>',
'</div>',
'<div id="symlink_filelist">',
'</div>'
].join(''));
}
else if (wfstatus == 'started' ){
get_workflow_status($("#server_url").val() , running_wf.id, cb, errcb)
}
else {
$("#myModalBody").html([
'<div class="alert alert-danger">',
' <p><strong>Error !</strong> You process is <strong>', wfstatus, '</strong>.</p>',
' <p>', data.errors.msg.join('') ,'</p>',
'</div>',
].join(''));
}
};
/**
* Closure : enable to try to rerun the getworkflow status a
* number of limited time if an error occures
* (because of end EOFError or pickle unexplained errors ...)
*/
function makeErrorCallBackFunc(lim){
var limit = lim;
var errcb = function(jqXHR, textStatus, errorThrown){
while (limit > 0 ){
limit = limit -1 ;
get_workflow_status($("#server_url").val() , running_wf.id, cb, errcb)
return;
}
$("#myModalBody").html([
'<div class="alert alert-danger">',
' <p><strong>Error !</strong> Unable to connect to server.</p>',
'</div>',
'<div class="alert alert-danger">',
' <p><strong>Status</strong> : ', textStatus ,'</p>',
'</div>',
].join(''));
};
return errcb;
}
var errcb = makeErrorCallBackFunc(5);
get_workflow_status($("#server_url").val() , running_wf.id, cb, errcb)
}
}
];
var arrayLength = download_options.length;
for (var i = 0; i < download_options.length; i++) {
$("#available-download-types").append($('<option id="' + download_options[i].id + '" > ' + download_options[i].label + '</option>'));
}
$("#download_btn").click(function(){
var selected_option = $("#available-download-types option:selected" ),
download_option;
var arrayLength = download_options.length;
for (var i = 0; i < download_options.length; i++) {
if ( download_options[i].id === selected_option.attr('id') ) {
download_option = download_options[i];
break;
}
}
$('#myModalLabel').html("Loading");
$('#myModalFooter').html([
' <div class="btn-group">',
......@@ -153,28 +310,32 @@ $(function () {
' </div>',
' <div class="btn-group">',
' <button id="reset_workflow" class="btn btn-default"><i class="glyphicon glyphicon-refresh"></i> Reset</button>',
' <button id="run_workflow" class="btn btn-primary"><i class="glyphicon glyphicon-cog"></i> Run</button>',
' <button id="run_workflow" class="btn btn-primary"><i class="glyphicon glyphicon-floppy-save"></i> Start</button>',
' </div>'
].join(''));
$("#reset_workflow, #run_workflow").hide();
$("#close").show();
$('#myModalBody').html([
'<div >',
' <div>',
' <blockquote>',
' Multiple types of downloads are available, you can switch between them using the arrows on the top right of the formular.',
' </blockquote>',
' </div>',
' <div id="wfform"><div class="tx-nG6-wait">Please wait</div></div>',
'</div>',
'<div >',
' <p id="wf-help" class="text-justify"></p><br/><br/>',
' <div id="wfform"><div class="tx-nG6-wait">Please wait</div></div>',
'</div>',
].join(''));
$('#myModal').modal();
$('#wfform').on("loaded.wfform", function(event, workflow) {
$('#myModalLabel').html(workflow["name"] + " <small>" + workflow["help"] + "</small>");
$("#reset_workflow, #run_workflow").show();
$("#close").hide();
if(workflow){
$('#myModalLabel').html(workflow["name"]) ;
$('#wf-help').html(workflow["help"]);
}
else{
$('#myModalBody').html('<div class="alert alert-warning"> <strong>Warning !</strong> The selected donwload type "' + download_option.label + '" is not available</div>');
}
});
var run_ids = [],
......@@ -194,13 +355,13 @@ $(function () {
}
});
$('#wfform').removeData();
$('#wfform').wfform({
workflowClass: "Download",
var wfform_options = {
workflowClass: download_option.wfName,
serverURL: $("#server_url").val(),
displayRunButton: false,
displayResetButton: false,
parameters : {
"login" : $("#user_login").val(),
"admin_login" : $("#user_login").val(),
"data_id" : data_ids,
"run_id" : run_ids,
"analysis_id" : analysis_ids,
......@@ -217,15 +378,17 @@ $(function () {
' <input id="${param.name}" name="${param.name}" value="${param.default}" class="form-control" type="text">',
' {{/if}}',
'</div>'
].join(''),
serverURL: $("#server_url").val()
});
].join('')
};
$('#wfform').removeData();
$('#wfform').wfform(wfform_options);
$('#wfform').on('run.wfform', function(event, running_wf) {
$("#reset_workflow, #run_workflow").hide();
$("#close").show();
$("#myModalBody").html("<div class='alert alert-info'>The dowload is in progress, it may take a while depending on the server availability."+
" You can close this window and wait for the validation mail.</div>");
$("#myModalBody").html("<div class='alert alert-info'>Workflow started successfully</div>");
download_option.onRun(event, running_wf, $('#myModalLabel'), $('#myModalBody'), $('#myModalFooter'));
});
$("#reset_workflow").click(function(){
......@@ -233,7 +396,6 @@ $(function () {
});
$("#run_workflow").click(function(){
$('#wfform').wfform('run');
});
});
......
......@@ -116,7 +116,7 @@ $(function () {
$("#modal-label-tmpl").html("Add a new ng6 admin user");
var add_ng6_admin = '<div id="error_message" class="alert alert-error"><button class="close" data-dismiss="alert" type="button">x</button></div>';
var add_ng6_admin = '<div id="error_message" class="alert alert-danger"><button class="close" data-dismiss="alert" type="button">x</button></div>';
add_ng6_admin += '<div class="tx-nG6-pi1-add-new">Seek the user using the autocompletion fields<br /> <br /> ';
add_ng6_admin += '<form class="form-horizontal" id="new_ng6_admin_user">';
......
#
# Copyright (C) 2015 INRA
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from ng6.ng6workflow import DownloadWorkflow
from ng6.config_reader import NG6ConfigReader
class DownloadArchive (DownloadWorkflow):
def get_description(self):
return """This workflow allows you to create a downloadable archive of the selected data. The link to the archive will be sent to the user via the provided email"""
def define_parameters(self, function="process"):
config = NG6ConfigReader()
self.save_directory = config.get_save_directory()
self.add_parameter('archive_mail', 'Provide a valid email address to be warned when the archive is created', display_name = "User mail", type="email", required = True)
def process(self):
self.add_component('CreateArchive', [self.admin_login, self.save_directory, self.archive_mail, self.data_id, self.run_id, self.analysis_id])
#
# Copyright (C) 2015 INRA
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
\ No newline at end of file
#
# Copyright (C) 2015 INRA
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
import os
import pickle
from jflow.component import Component
from weaver.function import PythonFunction
def create_archive( ng6_username, email, data_folder, output_folder, ids_dump_path, web_archive, archive_file ):
import os
import pickle
import shutil
import subprocess
import tarfile
from jflow.utils import robust_rmtree
from workflows.download.lib.utils import get_directories_structure_and_content
fh = open(ids_dump_path, "rb")
prefixed_ids = pickle.load(fh)
fh.close()
prefixed_ids = ";".join(prefixed_ids)
src_directories, dest_directories = get_directories_structure_and_content(ng6_username, data_folder, output_folder, prefixed_ids)
for i, source_dir in enumerate(src_directories):
dest_dir = dest_directories[i]
if not os.path.exists(dest_dir):
os.makedirs(dest_dir, 0755)
for filename in os.listdir(source_dir):
source_file_path = os.path.join(source_dir, filename)
dest_file_path = os.path.join(dest_dir, filename)
if os.path.isfile(source_file_path) and filename not in [ "analyse.xml", "index.html", "run.cfg"] :
shutil.copyfile(source_file_path, dest_file_path)
# create archive
with tarfile.open(archive_file, "w:gz") as tar :
tar.add(output_folder, os.path.basename(output_folder))
# send mail
try :
message = 'Dear user, your archive is ready for download at the following link : %s'%web_archive
subject = '[NG6] Archive ready to be downloaded'
subprocess.check_call( "echo '%s' | mail -s '%s' %s"%(message, subject, email) , shell = True)
except :
raise Exception('Unable to send mail')
# delete output folder
robust_rmtree(output_folder)
class CreateArchive (Component):
def define_parameters(self, login, data_directory, email, data_ids = [], run_ids = [], analysis_ids = []):
self.add_parameter('login', 'The login of the user in ng6', type = 'ng6userlogin', default = login)
self.add_parameter('data_directory', 'Where are stored all the datas', default = data_directory, required = True)
self.add_parameter('email', 'The email where the url to the archive will be sent', default = email, required = True)
self.add_parameter_list('data_ids', 'Ids of a run from which rawdata will be retrieved', default = data_ids)
self.add_parameter_list('run_ids', 'Ids of run from which all data will be retrieved', default = run_ids)
self.add_parameter_list('analysis_ids', 'Ids of analysis to retrieve', default = analysis_ids)
self.add_output_file("archive", "created archive name", filename = "ng6_data_archive.tar.gz")
def process(self):
run_ids = [ "run_%s"%i for i in self.run_ids ]
data_ids = [ "data_%s"%i for i in self.data_ids ]
analysis_ids = [ "analyse_%s"%i for i in self.analysis_ids ]
ids_dump_path = self.get_temporary_file(".dump")
fh = open(ids_dump_path, "wb")
pickle.dump( run_ids + data_ids + analysis_ids, fh)
output_folder = os.path.join(self.output_directory, 'ng6_data')
fh.close()
web_archive = self._webify_outputs(self.archive)
archive = PythonFunction(create_archive, cmd_format = "{EXE} {ARG} {IN} {OUT} ")
archive(arguments = [ self.login, self.email, self.data_directory, output_folder, ids_dump_path, web_archive], outputs = self.archive)
def _webify_outputs(self, path):
work_dir = self.config_reader.get_work_directory()
if work_dir.endswith("/"): work_dir = work_dir[:-1]
socket_opt = self.config_reader.get_socket_options()
return 'http://' + socket_opt[0] + ':' + str(socket_opt[1]) + '/' + path.replace(work_dir, 'data'),