Commit 1d3c7294 authored by Penom Nom's avatar Penom Nom
Browse files

update download workflow

parent 0bb7b52e
#
# Copyright (C) 2012 INRA
# Copyright (C) 2015 INRA
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
......@@ -29,8 +29,6 @@ server_socket_host = 127.0.0.1
server_socket_port = 8080
# date format
date_format = %d/%m/%Y
# A space separated list of workfow that will not be available in the web interface
availablewf_filter = AddAnalysis AddProject AddFiles RADseq
[email]
# if you want an email to be sent at the end of the workflow execution
......@@ -65,6 +63,13 @@ work_directory = <path>/work
# accessible by all cluster nodes
tmp_directory = <path>/tmp
[local]
# This is a space separated list of workflows Class names that will not be available
# for adding a run
runwf_filter = AddAnalysis AddProject AddFiles RADseq Download
# This must be an absolute path of a directory mounted and accessible by the server
server_directory = /save/scratch/
# where could be find binaries
[softwares]
# uncomment and set if not in the PATH
......
# Copyright 1999-2000 by Jeffrey Chang. All rights reserved.
# This code is part of the Biopython distribution and governed by its
# license. Please see the LICENSE file that should have been included
# as part of this package.
"""Record classes to hold BLAST output.
Classes:
Blast Holds all the information from a blast search.
PSIBlast Holds all the information from a psi-blast search.
Header Holds information from the header.
Description Holds information about one hit description.
Alignment Holds information about one alignment hit.
HSP Holds information about one HSP.
MultipleAlignment Holds information about a multiple alignment.
DatabaseReport Holds information from the database report.
Parameters Holds information from the parameters.
"""
# XXX finish printable BLAST output
class Header(object):
"""Saves information from a blast header.
Members:
application The name of the BLAST flavor that generated this data.
version Version of blast used.
date Date this data was generated.
reference Reference for blast.
query Name of query sequence.
query_letters Number of letters in the query sequence. (int)
database Name of the database.
database_sequences Number of sequences in the database. (int)
database_letters Number of letters in the database. (int)
"""
def __init__(self):
self.application = ''
self.version = ''
self.date = ''
self.reference = ''
self.query = ''
self.query_letters = None
self.database = ''
self.database_sequences = None
self.database_letters = None
class Description(object):
"""Stores information about one hit in the descriptions section.
Members:
title Title of the hit.
score Number of bits. (int)
bits Bit score. (float)
e E value. (float)
num_alignments Number of alignments for the same subject. (int)
"""
def __init__(self):
self.title = ''
self.score = None
self.bits = None
self.e = None
self.num_alignments = None
def __str__(self):
return "%-66s %5s %s" % (self.title, self.score, self.e)
class Alignment(object):
"""Stores information about one hit in the alignments section.
Members:
title Name.
hit_id Hit identifier. (str)
hit_def Hit definition. (str)
length Length. (int)
hsps A list of HSP objects.
"""
def __init__(self):
self.title = ''
self.hit_id = ''
self.hit_def = ''
self.length = None
self.hsps = []
def __str__(self):
lines = self.title.split('\n')
lines.append("Length = %s\n" % self.length)
return '\n '.join(lines)
class HSP(object):
"""Stores information about one hsp in an alignment hit.
Members:
score BLAST score of hit. (float)
bits Number of bits for that score. (float)
expect Expect value. (float)
num_alignments Number of alignments for same subject. (int)
identities Number of identities (int) if using the XML parser.
Tuple of numer of identities/total aligned (int, int)
if using the (obsolete) plain text parser.
positives Number of positives (int) if using the XML parser.
Tuple of numer of positives/total aligned (int, int)
if using the (obsolete) plain text parser.
gaps Number of gaps (int) if using the XML parser.
Tuple of numer of gaps/total aligned (int, int) if
using the (obsolete) plain text parser.
align_length Length of the alignment. (int)
strand Tuple of (query, target) strand.
frame Tuple of 1 or 2 frame shifts, depending on the flavor.
query The query sequence.
query_start The start residue for the query sequence. (1-based)
query_end The end residue for the query sequence. (1-based)
match The match sequence.
sbjct The sbjct sequence.
sbjct_start The start residue for the sbjct sequence. (1-based)
sbjct_end The end residue for the sbjct sequence. (1-based)
Not all flavors of BLAST return values for every attribute:
score expect identities positives strand frame
BLASTP X X X X
BLASTN X X X X X
BLASTX X X X X X
TBLASTN X X X X X
TBLASTX X X X X X/X
Note: for BLASTX, the query sequence is shown as a protein sequence,
but the numbering is based on the nucleotides. Thus, the numbering
is 3x larger than the number of amino acid residues. A similar effect
can be seen for the sbjct sequence in TBLASTN, and for both sequences
in TBLASTX.
Also, for negative frames, the sequence numbering starts from
query_start and counts down.
"""
def __init__(self):
self.score = None
self.bits = None
self.expect = None
self.num_alignments = None
self.identities = (None, None)
self.positives = (None, None)
self.gaps = (None, None)
self.align_length = None
self.strand = (None, None)
self.frame = ()
self.query = ''
self.query_start = None
self.query_end = None
self.match = ''
self.sbjct = ''
self.sbjct_start = None
self.sbjct_end = None
def __str__(self):
lines = ["Score %i (%i bits), expectation %0.1e, alignment length %i" \
% (self.score, self.bits, self.expect, self.align_length)]
if self.align_length < 50:
lines.append("Query:%s %s %s" % (str(self.query_start).rjust(8),
str(self.query),
str(self.query_end)))
lines.append(" %s" \
% (str(self.match)))
lines.append("Sbjct:%s %s %s" % (str(self.sbjct_start).rjust(8),
str(self.sbjct),
str(self.sbjct_end)))
else:
lines.append("Query:%s %s...%s %s" \
% (str(self.query_start).rjust(8),
str(self.query)[:45],
str(self.query)[-3:],
str(self.query_end)))
lines.append(" %s...%s" \
% (str(self.match)[:45],
str(self.match)[-3:]))
lines.append("Sbjct:%s %s...%s %s" \
% (str(self.sbjct_start).rjust(8),
str(self.sbjct)[:45],
str(self.sbjct)[-3:],
str(self.sbjct_end)))
return "\n".join(lines)
class MultipleAlignment(object):
"""Holds information about a multiple alignment.
Members:
alignment A list of tuples (name, start residue, sequence, end residue).
The start residue is 1-based. It may be blank, if that sequence is
not aligned in the multiple alignment.
"""
def __init__(self):
self.alignment = []
class Round(object):
"""Holds information from a PSI-BLAST round.
Members:
number Round number. (int)
reused_seqs Sequences in model, found again. List of Description objects.
new_seqs Sequences not found, or below threshold. List of Description.
alignments A list of Alignment objects.
multiple_alignment A MultipleAlignment object.
"""
def __init__(self):
self.number = None
self.reused_seqs = []
self.new_seqs = []
self.alignments = []
self.multiple_alignment = None
class DatabaseReport(object):
"""Holds information about a database report.
Members:
database_name List of database names. (can have multiple dbs)
num_letters_in_database Number of letters in the database. (int)
num_sequences_in_database List of number of sequences in the database.
posted_date List of the dates the databases were posted.
ka_params A tuple of (lambda, k, h) values. (floats)
gapped # XXX this isn't set right!
ka_params_gap A tuple of (lambda, k, h) values. (floats)
"""
def __init__(self):
self.database_name = []
self.posted_date = []
self.num_letters_in_database = []
self.num_sequences_in_database = []
self.ka_params = (None, None, None)
self.gapped = 0
self.ka_params_gap = (None, None, None)
class Parameters(object):
"""Holds information about the parameters.
Members:
matrix Name of the matrix.
gap_penalties Tuple of (open, extend) penalties. (floats)
sc_match Match score for nucleotide-nucleotide comparison
sc_mismatch Mismatch penalty for nucleotide-nucleotide comparison
num_hits Number of hits to the database. (int)
num_sequences Number of sequences. (int)
num_good_extends Number of extensions. (int)
num_seqs_better_e Number of sequences better than e-value. (int)
hsps_no_gap Number of HSP's better, without gapping. (int)
hsps_prelim_gapped Number of HSP's gapped in prelim test. (int)
hsps_prelim_gapped_attemped Number of HSP's attempted in prelim. (int)
hsps_gapped Total number of HSP's gapped. (int)
query_length Length of the query. (int)
query_id Identifier of the query sequence. (str)
database_length Number of letters in the database. (int)
effective_hsp_length Effective HSP length. (int)
effective_query_length Effective length of query. (int)
effective_database_length Effective length of database. (int)
effective_search_space Effective search space. (int)
effective_search_space_used Effective search space used. (int)
frameshift Frameshift window. Tuple of (int, float)
threshold Threshold. (int)
window_size Window size. (int)
dropoff_1st_pass Tuple of (score, bits). (int, float)
gap_x_dropoff Tuple of (score, bits). (int, float)
gap_x_dropoff_final Tuple of (score, bits). (int, float)
gap_trigger Tuple of (score, bits). (int, float)
blast_cutoff Tuple of (score, bits). (int, float)
"""
def __init__(self):
self.matrix = ''
self.gap_penalties = (None, None)
self.sc_match = None
self.sc_mismatch = None
self.num_hits = None
self.num_sequences = None
self.num_good_extends = None
self.num_seqs_better_e = None
self.hsps_no_gap = None
self.hsps_prelim_gapped = None
self.hsps_prelim_gapped_attemped = None
self.hsps_gapped = None
self.query_id = None
self.query_length = None
self.database_length = None
self.effective_hsp_length = None
self.effective_query_length = None
self.effective_database_length = None
self.effective_search_space = None
self.effective_search_space_used = None
self.frameshift = (None, None)
self.threshold = None
self.window_size = None
self.dropoff_1st_pass = (None, None)
self.gap_x_dropoff = (None, None)
self.gap_x_dropoff_final = (None, None)
self.gap_trigger = (None, None)
self.blast_cutoff = (None, None)
#TODO - Add a friendly __str__ method to BLAST results
class Blast(Header, DatabaseReport, Parameters):
"""Saves the results from a blast search.
Members:
descriptions A list of Description objects.
alignments A list of Alignment objects.
multiple_alignment A MultipleAlignment object.
+ members inherited from base classes
"""
def __init__(self):
Header.__init__(self)
DatabaseReport.__init__(self)
Parameters.__init__(self)
self.descriptions = []
self.alignments = []
self.multiple_alignment = None
class PSIBlast(Header, DatabaseReport, Parameters):
"""Saves the results from a blastpgp search.
Members:
rounds A list of Round objects.
converged Whether the search converged.
+ members inherited from base classes
"""
def __init__(self):
Header.__init__(self)
DatabaseReport.__init__(self)
Parameters.__init__(self)
self.rounds = []
self.converged = 0
......@@ -126,12 +126,27 @@ class NG6ConfigReader(object):
raise Error("Failed when parsing the config file !")
def get_workflow_filters(self):
filters = self.reader.get("global", 'availablewf_filter')
"""
Return a list of workflow class names
@return: list
"""
filters = self.reader.get("local", 'runwf_filter')
if filters :
return filters.split()
return []
def get_server_directory(self):
"""
Return an absolute path of a directory mounted on the web server
@return: string
"""
try:
return self.reader.get('local', 'server_directory')
except :
raise Error("Could not retrieve server directory !")
def get_server_parameters(self):
"""
return the server parameters
......
......@@ -145,43 +145,6 @@ $(function () {
}
});
/**
*
* @param workflow_id
* @param callback
* @returns
*/
function get_workflow_status(serverurl, workflow_id, callback){
$.ajax({
url : serverurl + '/get_workflow_status?display=list&workflow_id=' + workflow_id,
dataType : 'jsonp',
success : function(data){
if (callback){
callback(data);
}
}
})
}
/**
*
* @param workflow_id
* @param callback
* @returns
*/
function get_workflow_outputs(serverurl, workflow_id, callback){
$.ajax({
url : serverurl + '/get_workflow_outputs?workflow_id=' + workflow_id,
dataType : 'jsonp',
success : function(data){
if (callback){
callback(data);
}
}
})
}
$("#download_btn").click(function(){
$('#myModalLabel').html("Loading");
$('#myModalFooter').html([
......@@ -196,10 +159,19 @@ $(function () {
$("#reset_workflow, #run_workflow").hide();
$("#close").show();
$('#myModalBody').html('<div class="tx-nG6-wait">Please wait</div>');
$('#myModalBody').html([
'<div >',
' <div>',
' <blockquote>',
' Multiple types of downloads are available, you can switch between them using the arrows on the top right of the formular.',
' </blockquote>',
' </div>',
' <div id="wfform"><div class="tx-nG6-wait">Please wait</div></div>',
'</div>',
].join(''));
$('#myModal').modal();
$('#myModalBody').on("loaded.wfform", function(event, workflow) {
$('#wfform').on("loaded.wfform", function(event, workflow) {
$('#myModalLabel').html(workflow["name"] + " <small>" + workflow["help"] + "</small>");
$("#reset_workflow, #run_workflow").show();
$("#close").hide();
......@@ -222,8 +194,8 @@ $(function () {
}
});
$('#myModalBody').removeData();
$('#myModalBody').wfform({
$('#wfform').removeData();
$('#wfform').wfform({
workflowClass: "Download",
displayRunButton: false,
displayResetButton: false,
......@@ -249,57 +221,17 @@ $(function () {
serverURL: $("#server_url").val()
});
var download_archive = function(outdata){
var targz = false;
$.each(outdata, function(i, component) {
$.each(component, function(i, file) {
if (file.extension == '.gz'){
$("#myModal").modal('hide');
location.href = file.url;
return;
}
});
});
};
$('#myModalBody').on('run.wfform', function(event, running_wf) {
$('#wfform').on('run.wfform', function(event, running_wf) {
$("#reset_workflow, #run_workflow").hide();
$("#close").show();
$("#myModalBody").html("<div class='tx-nG6-wait'>Download in process ... If the archive option has been chosen, an email with a link to the archive will be sent when done</div>");
var callback = function(wfdata){
if (wfdata.status == "completed"){
get_workflow_outputs($("#server_url").val(), wfdata.id, download_archive);
$('#myModalLabel').html("Success");
$("#myModalBody").html( [
'<div class="alert alert-success">',
' Your data will be accessible in the specified directory',
'</div>',
'<div id="download-outputs"></div>'
].join(''));
}
else if (wfdata.status == "failed"){
$('#myModalBody').wfstatus({
workflowID : wfdata.id,
display : 'list'
})
}
else{
setTimeout(function(){
get_workflow_status($("#server_url").val(), running_wf.id, callback)
}, 3000) ;
}
};
get_workflow_status($("#server_url").val(), running_wf.id, callback);
$("#myModalBody").html("<div class='alert alert-info'>The dowload is in process, you can now close this window and wait for the validation mail.</div>");
});
$("#reset_workflow").click(function(){
$('#myModalBody').wfform('reset');
$('#wfform').wfform('reset');
});
$("#run_workflow").click(function(){
$('#myModalBody').wfform('run');
$('#wfform').wfform('run');
});
......
......@@ -24,19 +24,25 @@ class Download (Workflow):
return "Download selected data"
def define_parameters(self, function="process"):
config = NG6ConfigReader()
self.save_directory = config.get_save_directory()
self.add_parameter('login', 'The login of the user in ng6', required = True, type = 'ng6userlogin')
self.add_parameter_list('data_id', 'Ids of a run from which rawdata will be retrieved', type = 'existingrun')
self.add_parameter_list('run_id', 'Ids of run from which all data will be retrieved', type = 'existingrun')
self.add_parameter_list('analysis_id', 'Ids of analysis to retrieve', type = 'existinganalysis')
self.add_multiple_parameter("home_link", "Create a symbolik link in a home directory")
self.add_parameter('username', 'Valid username on the server', type = "localuser" ,required = True, add_to ="home_link")
self.add_parameter('password', 'A password for the username on the server', type="password", required = True, add_to ="home_link")
self.add_parameter('user_directory', 'Destination', default = "/work/", required = True, add_to ="home_link", type="localdirectory")
self.add_multiple_parameter("home_link", """The home link will create symbolic links of the selected data for the provided user in the destination.
The user must have write permission on the destination directory""")
self.add_parameter('username', 'A valid user username on the server', type = "localuser" ,required = True, add_to ="home_link")
self.add_parameter('password', 'The connection password for the username on the server', type="password", required = True, add_to ="home_link")
self.add_parameter('user_directory', 'The directory where the links will be created.This directory must exists on the server.', default = config.get_server_directory(),
required = True, add_to ="home_link", type="localdirectory")
self.add_parameter('link_mail', 'Provide an email to be warned when the process is over', display_name = "User mail", type = "email" , add_to ="home_link")
self.add_multiple_parameter("local_archive", "Download an archive on your computer")
self.add_parameter('user_email', 'Provide an email to be warned when the process is over', type="email", required = True, add_to ="local_archive")
self.add_multiple_parameter("local_archive", "This option will create an archive that the user can download with a link")
self.add_parameter('archive_mail', 'Provide a valid email address to be warned when the process is over', display_name = "User mail", type="email", required = True, add_to ="local_archive")
self.add_exclusion_rule("home_link", "local_archive")
......@@ -46,7 +52,8 @@ class Download (Workflow):
if self.home_link['username'] and self.home_link['password'] :
self.add_component('CreateSymlink', [self.login, self.home_link['username'], self.home_link['password'],
save_directory, self.home_link['user_directory'], self.data_id, self.run_id, self.analysis_id])
selef.save_directory, self.home_link['user_directory'],self.home_link['link_mail'],
self.data_id, self.run_id, self.analysis_id])
elif self.local_archive['user_email'] :
self.add_component('CreateArchive', [self.login, save_directory, self.local_archive['user_email'], self.data_id, self.run_id, self.analysis_id])
elif self.local_archive['archive_mail'] :
self.add_component('CreateArchive', [self.login, selef.save_directory, self.local_archive['archive_mail'], self.data_id, self.run_id, self.analysis_id])
......@@ -23,9 +23,10 @@ from ng6.utils import Utils
from weaver.function import PythonFunction
def create_symbolik_link (username, password, ng6_username, data_folder, output_folder, ids_dump_path, output_list, user_script):
def create_symbolik_link (username, password, ng6_username, data_folder, output_folder, email, ids_dump_path, output_list, user_script):
import os
import pickle
import subprocess
from ng6.config_reader import NG6ConfigReader
from ng6.utils import Utils
......@@ -40,6 +41,7 @@ def create_symbolik_link (username, password, ng6_username, data_folder, output_
prefixed_ids = ";".join(prefixed_ids)
src_directories, dest_directories = get_directories_structure_and_content(ng6_username, data_folder, output_folder, prefixed_ids)
destinations = []
with open(user_script, "w") as us:
with open(output_list, "w") as ol :
......@@ -51,6 +53,7 @@ def create_symbolik_link (username, password, ng6_username, data_folder, output_
for filename in os.listdir(source_dir):
source_file_path = os.path.join(source_dir, filename)
dest_file_path = os.path.join(dest_dir, filename)
destinations.append(dest_file_path)
if os.path.isfile(source_file_path) and filename not in [ "analyse.xml", "index.html", "run.cfg"] :
if not os.path.exists(dest_file_path):
us.write("ln -s %s %s ;\n"%(source_file_path,dest_file_path))
......@@ -62,18 +65,27 @@ def create_symbolik_link (username, password, ng6_username, data_folder, output_
s = SSH(username, password, server_params[0])
s.cmd('bash %s'%user_script)