Commit a566eb06 authored by Penom Nom's avatar Penom Nom

update download. create a new download workflow with urls

parent f254bdb7
......@@ -51,7 +51,7 @@ if __name__ == '__main__':
app_conf = {
'/':
{'tools.staticdir.root': WEB_DIR},
{'tools.staticdir.root': WEB_DIR, 'tools.CORS.on': True},
os.path.join('/', 'css'):
{'tools.staticdir.on' : True, 'tools.staticdir.dir' : './css/'},
os.path.join('/', 'js'):
......
......@@ -145,6 +145,61 @@ $(function () {
}
});
/**
* Retrieve workflow status
*/
var get_workflow_status = function(serverurl, workflow_id, callback, errcb){
$.ajax({
url : serverurl + '/get_workflow_status?display=list&workflow_id=' + workflow_id ,
dataType : 'jsonp',
timeout: 20000 ,
success : function(data){
if (callback){
callback(data);
}
},
error : function(jqXHR, textStatus, errorThrown){
if (errcb){
errcb(jqXHR, textStatus, errorThrown);
}
}
});
};
/**
* Closure : enable to try to rerun the getworkflow status a
* number of limited time if an error occures
* (because of end EOFError or pickle unexplained errors ...)
*/
function makeErrorCallBackFunc(lim, surl, running_wf, success_cb, $element){
var limit = lim,
wf_id = running_wf.id,
cb = success_cb,
server_url = surl,
$elmt = $element;
var errcb = function(jqXHR, textStatus, errorThrown){
while (limit > 0 ){
limit = limit -1 ;
get_workflow_status(server_url , wf_id, cb, errcb)
return;
}
$elmt.html([
'<div class="alert alert-danger">',
' <p><strong>Error !</strong> Unable to connect to server.</p>',
'</div>',
'<div class="alert alert-danger">',
' <p><strong>Status</strong> : ', textStatus ,'</p>',
'</div>',
].join(''));
};
return errcb;
}
// List of available downloads
var download_options = [
// archive on server
......@@ -231,24 +286,6 @@ $(function () {
$("#myModalBody").html("<div class='tx-nG6-wait'> <strong>Please</strong> wait while your links are being created.</div>");
var get_workflow_status = function(serverurl, workflow_id, callback, errcb){
$.ajax({
url : serverurl + '/get_workflow_status?display=list&workflow_id=' + workflow_id ,
dataType : 'jsonp',
timeout: 5000 , // More than enough to submit the request
success : function(data){
if (callback){
callback(data);
}
},
error : function(jqXHR, textStatus, errorThrown){
if (errcb){
errcb(jqXHR, textStatus, errorThrown);
}
}
});
};
var cb = function(data){
var wfstatus = data.status
if (wfstatus == "completed"){
......@@ -273,39 +310,92 @@ $(function () {
}
};
/**
* Closure : enable to try to rerun the getworkflow status a
* number of limited time if an error occures
* (because of end EOFError or pickle unexplained errors ...)
*/
function makeErrorCallBackFunc(lim){
var limit = lim;
var errcb = function(jqXHR, textStatus, errorThrown){
while (limit > 0 ){
limit = limit -1 ;
get_workflow_status($("#server_url").val() , running_wf.id, cb, errcb)
return;
}
$("#myModalBody").html([
'<div class="alert alert-danger">',
' <p><strong>Error !</strong> Unable to connect to server.</p>',
'</div>',
'<div class="alert alert-danger">',
' <p><strong>Status</strong> : ', textStatus ,'</p>',
'</div>',
].join(''));
};
return errcb;
}
var errcb = makeErrorCallBackFunc(5);
var errcb = makeErrorCallBackFunc(5, $("#server_url").val(), running_wf, cb, $("#myModalBody"))
get_workflow_status($("#server_url").val() , running_wf.id, cb, errcb)
}
},
// urls
{
label : "url",
id : "download_url",
wfName : "DownloadURL",
parameters : {
'url_base' : location.href.replace(/index\.php.+/, "" ) + 'fileadmin'
},
onRun : function(event, running_wf, $elementHeader, $elementBody, $elementFooter){
$("#myModalBody").html("<div class='tx-nG6-wait'> <strong>Please</strong> wait while your urls are being created.</div>");
var cb = function(data){
var wfstatus = data.status
if (wfstatus == "completed"){
$.ajax({
url : $("#server_url").val() + '/get_workflow_outputs?workflow_id=' + running_wf.id ,
dataType : 'jsonp',
timeout: 20000 , // More than enough to submit the request
success : function(data){
$.each(data, function(i, component) {
$.each(component, function(filename, file) {
if (filename == 'output_urls.txt'){
var link_prefix = location.href.replace(/index\.php.+/, "" ) + 'fileadmin' ;
$.get(file.url, function(data){
var file_list = [];
$.each(data.split('\n'), function(i, path){
file_list.push(link_prefix + path);
});
$("#myModalBody").html([
'<div class="alert alert-success">',
' <p><strong>Success !</strong> You will find bellow the list of ursl for your data</p>',
'</div>',
'<div>',
' <textarea id="url_filelist" rows="' + file_list.length + '" cols="70">',
file_list,
' </textarea>',
'</div>'
].join(''));
});
}
});
});
$("#myModalBody").html([
'<div class="alert alert-success">',
' <p><strong>Success !</strong> faut prendre les output maintenant </p>',
'</div>',
'<div id="url_filelist">',
'</div>'
].join(''));
},
error : function(jqXHR, textStatus, errorThrown){
$("#myModalBody").html([
'<div class="alert alert-danger">',
' <p><strong>Failed !</strong> to get outputs from ' + $("#server_url").val() + ' </p>',
'</div>',
].join(''));
}
});
}
else if (wfstatus == 'started' ){
get_workflow_status($("#server_url").val() , running_wf.id, cb, errcb)
}
else {
$("#myModalBody").html([
'<div class="alert alert-danger">',
' <p><strong>Error !</strong> You process is <strong>', wfstatus, '</strong>.</p>',
' <p>', data.errors.msg.join('') ,'</p>',
'</div>',
].join(''));
}
};
var errcb = makeErrorCallBackFunc(5, $("#server_url").val(), running_wf, cb, $("#myModalBody"))
get_workflow_status($("#server_url").val() , running_wf.id, cb, errcb)
}
}
];
var arrayLength = download_options.length;
for (var i = 0; i < download_options.length; i++) {
$("#available-download-types").append($('<option id="' + download_options[i].id + '" > ' + download_options[i].label + '</option>'));
}
......@@ -400,14 +490,23 @@ $(function () {
].join('')
};
if ('parameters' in download_option){
wfform_options.parameters = $.extend({}, wfform_options.parameters, download_option.parameters);
}
$('#wfform').removeData();
$('#wfform').wfform(wfform_options);
$('#wfform').on('run.wfform', function(event, running_wf) {
$("#reset_workflow, #run_workflow").hide();
$("#close").show();
$("#myModalBody").html("<div class='alert alert-info'>Workflow started successfully</div>");
download_option.onRun(event, running_wf, $('#myModalLabel'), $('#myModalBody'), $('#myModalFooter'));
if ('onRun' in download_option){
download_option.onRun(event, running_wf, $('#myModalLabel'), $('#myModalBody'), $('#myModalFooter'));
}
else {
$("#myModalBody").html("<div class='alert alert-info'>Workflow started successfully</div>");
}
});
$("#reset_workflow").click(function(){
......
......@@ -98,5 +98,5 @@ class CreateArchive (Component):
work_dir = self.config_reader.get_work_directory()
if work_dir.endswith("/"): work_dir = work_dir[:-1]
socket_opt = self.config_reader.get_socket_options()
return 'http://' + socket_opt[0] + ':' + str(socket_opt[1]) + '/' + path.replace(work_dir, 'data'),
return 'http://' + socket_opt[0] + ':' + str(socket_opt[1]) + '/' + path.replace(work_dir, 'data')
\ No newline at end of file
#
# Copyright (C) 2015 INRA
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from ng6.ng6workflow import DownloadWorkflow
class DownloadURL (DownloadWorkflow):
def get_description(self):
return """This workflow will create a list of url of each file of your project"""
def define_parameters(self, function="process"):
pass
def process(self):
self.add_component('GetURLS', [self.admin_login, self.data_id, self.run_id, self.analysis_id])
#
# Copyright (C) 2015 INRA
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
\ No newline at end of file
#
# Copyright (C) 2015 INRA
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
import os
import pickle
from jflow.component import Component
from weaver.function import PythonFunction
def get_urls( ng6_username, ids_dump_path, output_url):
import os
import pickle
import subprocess
import sys
from jflow.utils import *
from ng6.utils import Utils
from ng6.config_reader import NG6ConfigReader
ng6config = NG6ConfigReader()
data_directory = ng6config.get_save_directory()
fh = open(ids_dump_path, "rb")
prefixed_ids = pickle.load(fh)
fh.close()
prefixed_ids = ";".join(prefixed_ids)
src_directories, dest_directories = Utils.get_directories_structure_and_content(ng6_username, data_directory, '/fake/output/', prefixed_ids)
files = []
for i, source_dir in enumerate(src_directories):
for filename in os.listdir(source_dir):
source_file_path = os.path.join(source_dir, filename)
if os.path.isfile(source_file_path) and filename not in [ "analyse.xml", "index.html", "run.cfg"] :
files.append(source_file_path.replace(data_directory, ""))
with open(output_url, "w") as fh :
for f in files :
fh.write(f+"\n")
class GetURLS (Component):
def define_parameters(self, login, data_ids = [], run_ids = [], analysis_ids = []):
self.add_parameter('login', 'The login of the user in ng6', type = 'ng6userlogin', default = login)
self.add_parameter_list('data_ids', 'Ids of a run from which rawdata will be retrieved', default = data_ids)
self.add_parameter_list('run_ids', 'Ids of run from which all data will be retrieved', default = run_ids)
self.add_parameter_list('analysis_ids', 'Ids of analysis to retrieve', default = analysis_ids)
self.add_output_file("output_url", "A file with a list of created urls", filename = "output_urls.txt")
def process(self):
run_ids = [ "run_%s"%i for i in self.run_ids ]
data_ids = [ "data_%s"%i for i in self.data_ids ]
analysis_ids = [ "analyse_%s"%i for i in self.analysis_ids ]
ids_dump_path = self.get_temporary_file(".dump")
fh = open(ids_dump_path, "wb")
pickle.dump( run_ids + data_ids + analysis_ids, fh)
fh.close()
fn = PythonFunction(get_urls, cmd_format = "{EXE} {ARG} {IN} {OUT} ")
fn(arguments = [ self.login, ids_dump_path], outputs = self.output_url, local = True)
\ No newline at end of file
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment