Commit 1099ef25 authored by Penom Nom's avatar Penom Nom

jflow update

parent 7814bd15
......@@ -42,6 +42,7 @@ from jflow.parameter import browsefile, localfile, urlfile, inputfile, create_te
from workflows.types import *
import jflow.utils as utils
from cctools.util import time_format
from jflow.utils import get_octet_string_representation
# function in charge to upload large files
class UploadFieldStorage(cgi.FieldStorage):
......@@ -449,7 +450,11 @@ class JFlowServer (object):
work_dir = self.jflow_config_reader.get_work_directory()
if work_dir.endswith("/"): work_dir = work_dir[:-1]
socket_opt = self.jflow_config_reader.get_socket_options()
return "http://" + socket_opt[0] + ":" + str(socket_opt[1]) + "/" + path.replace(work_dir, web_path)
return {
'url':'http://' + socket_opt[0] + ':' + str(socket_opt[1]) + '/' + path.replace(work_dir, web_path),
'size': get_octet_string_representation(os.path.getsize(os.path.abspath(path))),
'extension': os.path.splitext(path)[1]
}
@cherrypy.expose
@jsonify
......
......@@ -289,7 +289,8 @@ class Component(object):
if p:
commandline += " %s " % p.cmd_format
else :
commandline += " %s %s " % (p.cmd_format, p.default)
if p.default :
commandline += " %s %s " % (p.cmd_format, p.default)
abstraction = self.get_abstraction()
......
#
# Copyright (C) 2012 INRA
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
import os
class ExternalParser(object):
def parse_directory(self, component_directory):
components = []
for component_file in os.listdir(component_directory):
try:
components.append(self.parse(os.path.join(component_directory, component_file)))
except: pass
return components
def parse(self, component_file):
raise NotImplementedError
......@@ -185,9 +185,6 @@ class MultipleParameters(object):
parts = arg.split("=")
if not self.types.has_key(parts[0]):
raise argparse.ArgumentTypeError(parts[0] + " is an invalid flag! Available ones are: "+", ".join(self.types.keys()))
if self.choices[parts[0]] != None:
if parts[1] not in self.choices[parts[0]]:
raise argparse.ArgumentTypeError("argument " + parts[0] + ": invalid choice: '" + parts[1] + "' (choose from " + ", ".join(self.choices[parts[0]]) + "))")
if self.types[parts[0]] == types.BooleanType:
return (parts[0], not self.default[parts[0]], self.required, self.excludes)
......@@ -196,6 +193,10 @@ class MultipleParameters(object):
value = self.types[parts[0]](parts[1])
except:
raise argparse.ArgumentTypeError("invalid " + self.types[parts[0]].__name__ + " value: '" + parts[1] + "' for sub parameter '" + parts[0] + "'")
if self.choices[parts[0]] != None:
if value not in self.choices[parts[0]]:
raise argparse.ArgumentTypeError("argument " + parts[0] + ": invalid choice: '" + parts[1] + "' (choose from " + ", ".join(map(str,self.choices[parts[0]])) + ")")
self.index = parts[0]
return (parts[0], value, self.required, self.excludes, self.actions)
......@@ -849,6 +850,8 @@ class InputFile(StrParameter, AbstractInputFile):
def __init__(self, name, help, file_format="any", default="", type="localfile", choices=None,
required=False, flag=None, group="default", display_name=None, size_limit="0", cmd_format="", argpos=-1):
AbstractInputFile.__init__(self, file_format, size_limit)
if issubclass(default.__class__, list):
raise Exception( "The parameter '" + name + "' cannot be set with a list." )
StrParameter.__init__(self, name, help, flag=flag, default=default, type=type, choices=choices,
required=required, group=group, display_name=display_name, cmd_format=cmd_format, argpos=argpos)
......@@ -888,6 +891,8 @@ class OutputFile(StrParameter, AbstractOutputFile):
required=False, flag=None, group="default", display_name=None,
cmd_format="", argpos=-1):
AbstractIOFile.__init__(self, file_format)
if issubclass(default.__class__, list):
raise Exception( "The parameter '" + name + "' cannot be set with a list." )
StrParameter.__init__(self, name, help, flag=flag, default=default, type="localfile", choices=choices,
required=required, group=group, display_name=display_name, cmd_format=cmd_format, argpos=argpos)
......@@ -901,11 +906,16 @@ class ParameterList(list, AbstractParameter):
AbstractParameter.__init__(self, name, help, flag=flag, default=default, type=type, choices=choices, required=required,
action="append", sub_parameters=sub_parameters, group=group, display_name=display_name,
cmd_format=cmd_format, argpos=argpos)
if default.__class__.__name__ == "str":
return list.__init__(self, [default])
elif default.__class__.__name__ == "list":
return list.__init__(self, default)
liste = []
if default.__class__.__name__ == "list":
for val in default :
liste.append(ParameterFactory.factory( name, help, default=val, type=type, choices=choices,
required=required, flag=flag, group=group, display_name=display_name ))
else :
liste.append(ParameterFactory.factory( name, help, default=default, type=type, choices=choices,
required=required, flag=flag, group=group, display_name=display_name ))
return list.__init__(self, liste)
def append(self, item):
raise TypeError('A parameter is immutable.')
......
......@@ -84,8 +84,6 @@ class MINIWorkflow(object):
class Workflow(threading.Thread):
PROPERTIES_FILE_NAME = "workflow.properties"
MAKEFLOW_LOG_FILE_NAME = "Makeflow.makeflowlog"
DUMP_FILE_NAME = ".workflow.dump"
STDERR_FILE_NAME = "wf_stderr.txt"
......@@ -156,7 +154,9 @@ class Workflow(threading.Thread):
self.stderr = self._set_stderr()
self._serialize()
self.comp_pckg = self._import_components()
self.internal_components = self._import_internal_components()
self.external_components = self._import_external_components()
def add_input_directory(self, name, help, default=None, required=False, flag=None,
group="default", display_name=None, get_files_fn=None, add_to=None):
......@@ -349,15 +349,16 @@ class Workflow(threading.Thread):
elif param.__class__ == MultiParameterList:
new_param = MultiParameterList(param.name, param.help, required=param.required, flag=param.flag, group=param.group, display_name=param.display_name)
new_param.sub_parameters = param.sub_parameters
for idx, sargs in enumerate(args[param.name]):
new_multi_param = MultiParameter(param.name + '_' + str(idx), '', required=False, flag=None, group="default", display_name=None)
sub_args = {}
for sarg in sargs:
sub_args[sarg[0]] = sarg[1]
for sub_param in param.sub_parameters:
new_sub_param = self._prepare_parameter(sub_args, sub_param, "flag")
new_multi_param[new_sub_param.name] = new_sub_param
new_param.append(new_multi_param)
if args.has_key(param.name):
for idx, sargs in enumerate(args[param.name]):
new_multi_param = MultiParameter(param.name + '_' + str(idx), '', required=False, flag=None, group="default", display_name=None)
sub_args = {}
for sarg in sargs:
sub_args[sarg[0]] = sarg[1]
for sub_param in param.sub_parameters:
new_sub_param = self._prepare_parameter(sub_args, sub_param, "flag")
new_multi_param[new_sub_param.name] = new_sub_param
new_param.append(new_multi_param)
else:
new_param = self._prepare_parameter(args, param)
self.__setattr__(param.name, new_param)
......@@ -393,6 +394,15 @@ class Workflow(threading.Thread):
elif issubclass(subparam.__class__, InputDirectory):
gr.add_node_attribute(subparam.name, self.INPUTDIRECTORY_GRAPH_LABEL)
gr.add_node_attribute(subparam.name, subparam.display_name)
elif issubclass(ioparameter.__class__, MultiParameterList):
for subparam in ioparameter.sub_parameters:
gr.add_node(subparam.name)
all_nodes[subparam.name] = None
if issubclass(subparam.__class__, InputDirectory):
gr.add_node_attribute(subparam.name, self.INPUTDIRECTORY_GRAPH_LABEL)
else:
gr.add_node_attribute(subparam.name, self.INPUTFILES_GRAPH_LABEL)
gr.add_node_attribute(subparam.name, subparam.display_name)
for cpt in self.components:
gr.add_node(cpt.get_nameid())
gr.add_node_attribute(cpt.get_nameid(), self.COMPONENT_GRAPH_LABEL)
......@@ -559,6 +569,7 @@ class Workflow(threading.Thread):
def __setstate__(self, state):
self.__dict__ = state.copy()
self.external_components = self._import_external_components()
threading.Thread.__init__(self, name=self.name)
def __getstate__(self):
......@@ -569,6 +580,8 @@ class Workflow(threading.Thread):
del odict['_Thread__started']
del odict['_Thread__block']
del odict['_Thread__stderr']
if odict.has_key('external_components') :
del odict['external_components']
return odict
def set_to_address(self, to_address):
......@@ -654,14 +667,23 @@ class Workflow(threading.Thread):
def add_component(self, component_name, args=[], kwargs={}, component_prefix="default"):
# first build and check if this component is OK
if self.comp_pckg.has_key(component_name):
my_pckge = __import__(self.comp_pckg[component_name], globals(), locals(), [component_name], -1)
# build the object and define required field
cmpt_object = getattr(my_pckge, component_name)()
cmpt_object.output_directory = self.get_component_output_directory(component_name, component_prefix)
cmpt_object.prefix = component_prefix
if kwargs: cmpt_object.define_parameters(**kwargs)
else: cmpt_object.define_parameters(*args)
if self.internal_components.has_key(component_name) or self.external_components.has_key(component_name):
if self.internal_components.has_key(component_name) :
my_pckge = __import__(self.internal_components[component_name], globals(), locals(), [component_name], -1)
# build the object and define required field
cmpt_object = getattr(my_pckge, component_name)()
cmpt_object.output_directory = self.get_component_output_directory(component_name, component_prefix)
cmpt_object.prefix = component_prefix
if kwargs: cmpt_object.define_parameters(**kwargs)
else: cmpt_object.define_parameters(*args)
# external components
else :
cmpt_object = self.external_components[component_name]()
cmpt_object.output_directory = self.get_component_output_directory(component_name, component_prefix)
cmpt_object.prefix = component_prefix
# can't use positional arguments with external components
cmpt_object.define_parameters(**kwargs)
# there is a dynamic component
if cmpt_object.is_dynamic():
......@@ -701,7 +723,8 @@ class Workflow(threading.Thread):
return cmpt_object
else:
raise ImportError(component_name + " component cannot be loaded, available components are: {0}".format(", ".join(self.comp_pckg.keys())))
raise ImportError(component_name + " component cannot be loaded, available components are: {0}".format(
", ".join(self.internal_components.keys() + self.external_components.keys())))
def pre_process(self):
pass
......@@ -990,14 +1013,7 @@ class Workflow(threading.Thread):
return True
return False
def _get_property_path(self):
"""
Return(type:string): the path to the workflow properties file, None if does not exist
"""
property_file = os.path.join(os.path.dirname(inspect.getfile(self.__class__)), self.PROPERTIES_FILE_NAME)
return property_file if os.path.isfile(property_file) else None
def _import_components(self):
def _import_internal_components(self):
pckge = {}
# then import pipeline packages
pipeline_dir = os.path.dirname(inspect.getfile(self.__class__))
......@@ -1021,7 +1037,36 @@ class Workflow(threading.Thread):
except Exception as e:
logging.getLogger("wf." + str(self.id)).debug("Component <{0}> cannot be loaded: {1}".format(modname, e))
return pckge
def _import_external_components(self):
pckge = {}
parsers = []
# get exparsers
extparsers_dir = os.path.join( os.path.dirname(os.path.dirname(inspect.getfile(self.__class__))), 'extparsers' )
for importer, modname, ispkg in pkgutil.iter_modules([extparsers_dir], "workflows.extparsers.") :
try :
m = __import__(modname)
for class_name, obj in inspect.getmembers(sys.modules[modname], inspect.isclass):
if issubclass(obj, jflow.extparser.ExternalParser) and obj.__name__ != jflow.extparser.ExternalParser.__name__:
parsers.append(obj())
except Exception as e:
logging.getLogger("wf." + str(self.id)).debug("Parser <{0}> cannot be loaded: {1}".format(modname, e))
for parser in parsers :
# import from pipeline components package ...
pipeline_components_dir = os.path.join( os.path.dirname(inspect.getfile(self.__class__)), "components" )
# ... and from shared components package
workflow_components_dir = os.path.join(os.path.dirname(os.path.dirname(inspect.getfile(self.__class__))), "components" )
try :
comps = parser.parse_directory(pipeline_components_dir) + parser.parse_directory(workflow_components_dir)
for c in comps :
pckge[c.__name__] = c
except :
pass
return pckge
def _import(self, module, symbols):
""" Import ``symbols`` from ``module`` into global namespace. """
# Import module
......
......@@ -41,14 +41,23 @@ class BasicNG6Workflow (Workflow):
def add_component(self, component_name, args=[], kwargs={}, component_prefix="default", parent=None, addto="run"):
# first build and check if this component is OK
if self.comp_pckg.has_key(component_name):
my_pckge = __import__(self.comp_pckg[component_name], globals(), locals(), [component_name], -1)
# build the object and define required field
cmpt_object = getattr(my_pckge, component_name)()
cmpt_object.output_directory = self.get_component_output_directory(component_name, component_prefix)
cmpt_object.prefix = component_prefix
if kwargs: cmpt_object.define_parameters(**kwargs)
else: cmpt_object.define_parameters(*args)
if self.internal_components.has_key(component_name) or self.external_components.has_key(component_name):
if self.internal_components.has_key(component_name) :
my_pckge = __import__(self.internal_components[component_name], globals(), locals(), [component_name], -1)
# build the object and define required field
cmpt_object = getattr(my_pckge, component_name)()
cmpt_object.output_directory = self.get_component_output_directory(component_name, component_prefix)
cmpt_object.prefix = component_prefix
if kwargs: cmpt_object.define_parameters(**kwargs)
else: cmpt_object.define_parameters(*args)
# external components
else :
cmpt_object = self.external_components[component_name]()
cmpt_object.output_directory = self.get_component_output_directory(component_name, component_prefix)
cmpt_object.prefix = component_prefix
# can't use positional arguments with external components
cmpt_object.define_parameters(**kwargs)
# if the built object is an analysis
for derived_class in cmpt_object.__class__.__bases__:
if derived_class.__name__ == "Analysis":
......@@ -96,8 +105,8 @@ class BasicNG6Workflow (Workflow):
return cmpt_object
else:
raise ImportError(component_name + " component cannot be loaded, available components are: {0}".format(", ".join(self.comp_pckg.keys())))
raise ImportError(component_name + " component cannot be loaded, available components are: {0}".format(
", ".join(self.internal_components.keys() + self.external_components.keys())))
def pre_process(self):
t3mysql = t3MySQLdb()
......
......@@ -19,16 +19,36 @@
* This copyright notice MUST APPEAR in all copies of the script!
***************************************************************/
.handsontable {
overflow: auto;
}
.handsontable td.htInvalid {
background-image: linear-gradient(to bottom, #d9534f 0px, #c12e2a 100%) !important;
background-repeat: repeat-x;
border-color: #b92c28 !important;
color: white;
}
.handsontable .handsontableInput {
box-sizing: content-box;
border: 2px solid #66afe9;
outline: 0;
-webkit-box-shadow: inset 0 1px 1px rgba(0,0,0,.075),0 0 8px rgba(102,175,233,.6);
box-shadow: inset 0 1px 1px rgba(0,0,0,.075),0 0 8px rgba(102,175,233,.6);
}
.handsontable .currentRow {
background-color: #E7E8EF;
}
.handsontable .currentCol {
background-color: #F9F9FB;
}
.floatingBarsG{
.floatingBarsG {
position:relative;
width:16px;
height:20px}
height:20px
}
.blockG{
.blockG {
position:absolute;
background-color:#FFFFFF;
width:3px;
......
This diff is collapsed.
......@@ -62,6 +62,65 @@
success: function(data) {
$this.$element.html("");
$.tmpl($this.options.template, {data: data}).appendTo($this.$element);
// Adding select options
var extensions = new Array();
$.each(data, function(i, component) {
$.each(component, function(i, file) {
if( $.inArray( file.extension, extensions ) == -1 &&
$.inArray( file.extension, $this.options.logFile ) == -1 ) {
extensions.push(file.extension);
}
});
});
$.each(extensions, function(i, item) {
$('#jflow_file_type').append($('<option>', {
text: item
}));
});
// Adding class for filters
$(".output-file").each(function() {
var fileExt = $(this).find("A").text().substr( ($(this).find("A").text().lastIndexOf('.') ) );
if ($.inArray( fileExt, $this.options.logFile ) == -1 ) {
$(this).addClass("output");
}
else {
$(this).addClass("log");
$(this).hide();
}
});
// Events
$("#jflow_log_on").click(function() {
$(".log").each( function(){
$(this).fadeIn(500);
});
$("#jflow_log_on").hide();
$("#jflow_log_off").show();
});
$("#jflow_log_off").click(function() {
$(".log").each( function(){
$(this).fadeOut(500);
});
$("#jflow_log_off").hide();
$("#jflow_log_on").show();
});
$("#jflow_file_type").on('change', function(){
var selected = $("#jflow_file_type option:selected").val();
var regex = new RegExp(selected);
$(".output").each( function(){
if(selected == "all") {
$(this).fadeIn(500);
}
else if( regex.test($(this).text())) {
$(this).fadeIn(500);
}
else {
$(this).fadeOut(500);
}
});
});
}
});
}
......@@ -88,13 +147,32 @@
$.fn.wfoutputs.defaults = {
serverURL: "http://localhost:8080",
template: ['<dl class="dl-horizontal">',
'{{each(component_name, files) data}}',
'<dt>${component_name}</dt>',
'{{each(file_name, href) files}}',
'<dd><a href="${href}" download>${file_name}</a></dd>',
'{{/each}}',
'{{/each}}</dl>'].join('\n'),
template: ['<div style="float:right;margin-bottom:14px">',
' <button id="jflow_log_on" class="btn btn-default btn-xs" type="button"><span class="glyphicon glyphicon-eye-open"></span> View log files</button>',
' <button id="jflow_log_off" style="display:none" class="btn btn-default btn-xs" type="button"><span class="glyphicon glyphicon-eye-close"></span> Mask log files</button>',
' <span class="btn-xs">- View:</span>',
' <select id="jflow_file_type" class="btn btn-default btn-xs" style="width:auto">',
' view: <option>all</option>',
' </select><span class="btn-xs"> files.</span>',
'</div>',
'<div style="clear:both;">',
' <dl "style=margin-bottom:15px">',
' {{each(component_name, files) data}}',
' <div style="clear:both"></div>',
' <dt style="margin-top:15px;background-color: #eee">',
' <span class="glyphicon glyphicon-play" style="color:white;left:-3px"></span>',
' ${component_name}',
' </dt>',
' {{each(file_name, href) files}}',
' <dd class="output-file" style="float:left;margin-left:20px;width:265px;overflow:hidden;white-space: nowrap;text-overflow:ellipsis;">',
' <span class="glyphicon glyphicon-file"></span>',
' <a href="${href.url}" download>${file_name}</a>',
' <span style="color:grey">| ${href.size}</span>',
' </dd>',
' {{/each}}',
' {{/each}}</dl>',
'</div>'].join('\n'),
logFile: [".stderr", ".stdout", ".log"],
workflowID: null
}
......
#
# Copyright (C) 2012 INRA
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
This diff is collapsed.
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment