workflow.py 56.5 KB
Newer Older
Jerome Mariette's avatar
Jerome Mariette committed
1
#
Jerome Mariette's avatar
Jerome Mariette committed
2
# Copyright (C) 2015 INRA
Jerome Mariette's avatar
Jerome Mariette committed
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
# 
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
# 
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
# 
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
#

import inspect
import os
20
import re
Jerome Mariette's avatar
Jerome Mariette committed
21
22
23
24
25
26
27
import sys
import uuid
import pkgutil
import tempfile
import pickle
import time
import threading
Jerome Mariette's avatar
Jerome Mariette committed
28
import types
Jerome Mariette's avatar
Jerome Mariette committed
29
import datetime
30
31
import logging
import traceback
Jerome Mariette's avatar
Jerome Mariette committed
32

33
from configparser import ConfigParser, NoOptionError
Jerome Mariette's avatar
Jerome Mariette committed
34
from inspect import getcallargs
35
from datetime import date as ddate
Jerome Mariette's avatar
Jerome Mariette committed
36

Jerome Mariette's avatar
Jerome Mariette committed
37
import jflow
Jerome Mariette's avatar
Jerome Mariette committed
38
import jflow.utils as utils
39
from jflow.utils import validate_email
40
from pygraph.classes.digraph import digraph
Jerome Mariette's avatar
Jerome Mariette committed
41
from jflow.workflows_manager import WorkflowsManager, JFlowConfigReader
42
from jflow.utils import get_octet_string_representation, get_nb_octet
43
from jflow.parameter import *
44
from cctools.util import time_format
Jerome Mariette's avatar
Jerome Mariette committed
45
46
47
48
49
50
51
52
53
54
55
56

from weaver.script import ABSTRACTIONS
from weaver.script import DATASETS
from weaver.script import FUNCTIONS
from weaver.script import NESTS
from weaver.script import OPTIONS
from weaver.script import STACKS
from weaver.nest import Nest
from weaver.options import Options
from cctools.makeflow import MakeflowLog
from cctools.makeflow.log import Node

Jerome Mariette's avatar
Jerome Mariette committed
57

58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
class MINIWorkflow(object):
    
    def __init__(self, id, name, description, status, start_time, end_time, metadata, 
                 component_nameids, compts_status, errors):
        self.id = id
        self.name = name
        self.description = description
        self.status = status
        self.start_time = start_time
        self.end_time = end_time
        self.metadata = metadata
        self.component_nameids = component_nameids
        self.compts_status = compts_status
        self.errors = errors
        
    def get_components_nameid(self):
        return self.component_nameids

76
77
78
    def get_components_status(self):
        return self.compts_status
    
79
80
81
82
83
84
85
86
    def get_component_status(self, component_nameid):
        return self.compts_status[component_nameid]
    
    def get_errors(self):
        return self.errors
    
    def get_status(self):
        return self.status
87

88
89
class Workflow(threading.Thread):
    
Jerome Mariette's avatar
Jerome Mariette committed
90
    MAKEFLOW_LOG_FILE_NAME = "Makeflow.makeflowlog"
91
    DUMP_FILE_NAME = ".workflow.dump"
92
    STDERR_FILE_NAME = "wf_stderr.txt"
Jerome Mariette's avatar
Jerome Mariette committed
93
    WORKING = ".working"
94
    OLD_EXTENSION = ".old"
95
    DEFAULT_GROUP = "default"
96
    
Jerome Mariette's avatar
Jerome Mariette committed
97
98
99
100
    STATUS_STARTED = "started"
    STATUS_COMPLETED = "completed"
    STATUS_FAILED = "failed"
    STATUS_ABORTED = "aborted"
Jerome Mariette's avatar
Jerome Mariette committed
101
    STATUS_RESETED = "reseted"
102
    
Jerome Mariette's avatar
Jerome Mariette committed
103
104
105
    INPUTFILE_GRAPH_LABEL = "inputfile"
    INPUTFILES_GRAPH_LABEL = "inputfiles"
    INPUTDIRECTORY_GRAPH_LABEL = "inputdirectory"
106
107
    COMPONENT_GRAPH_LABEL = "component"
    
108
    
109
    def __init__(self, args={}, id=None, function= "process"):
Jerome Mariette's avatar
Jerome Mariette committed
110
111
112
113
        # define as a thread
        threading.Thread.__init__(self)
        self.jflow_config_reader = JFlowConfigReader()
        self.manager = WorkflowsManager()
114
        self.components_to_exec = []
115
        self.components = []
Jerome Mariette's avatar
Jerome Mariette committed
116
117
118
119
120
121
        self.makes = {}
        self.globals = {}
        self.options = Options()
        self.status = self.STATUS_STARTED
        self.start_time = None
        self.end_time = None
Jerome Mariette's avatar
Jerome Mariette committed
122
        self.__step = None
Frédéric Escudié's avatar
Frédéric Escudié committed
123
        self.stderr = None
Jerome Mariette's avatar
Jerome Mariette committed
124
        self.args = args
125
        self.dynamic_component_present = False
126
127
128
        self.__to_address = None
        self.__subject = None
        self.__message = None
129
        self.function = function
130
        # intruduce --log-verbose to be able to monitor the new version of makeflow >=4.2.2
Jerome Mariette's avatar
Jerome Mariette committed
131
        self.engine_arguments = ' --log-verbose '
132
        self.component_nameids_is_init = False
Jerome Mariette's avatar
Jerome Mariette committed
133
        self.component_nameids = {}
134
        self.reseted_components = []
Jerome Mariette's avatar
Jerome Mariette committed
135
136
        # try to parse engine arguments
        try:
137
138
            type, options, limit_submission = self.jflow_config_reader.get_batch()
            if limit_submission : self.engine_arguments += ' -J ' + str(limit_submission)
Jerome Mariette's avatar
Jerome Mariette committed
139
140
141
            if type: self.engine_arguments += ' -T ' + type
            if options : self.engine_arguments += ' -B "' + options + '"'
        except: self.engine_arguments = None
142

Jerome Mariette's avatar
Jerome Mariette committed
143
        self.id = id
144
145
        self.name = self.get_name()
        self.description = self.get_description()
Ibouniyamine Nabihoudine's avatar
Ibouniyamine Nabihoudine committed
146
        self.__group = self.jflow_config_reader.get_workflow_group(self.__class__.__name__) or self.DEFAULT_GROUP
147
148
149
        
        # define the parameters 
        self.params_order = []
Jerome Mariette's avatar
Jerome Mariette committed
150
151
        if self.function != None:
            self.define_parameters(self.function)
152
        # add the metadata parameter
153
        self.metadata = []
Jerome Mariette's avatar
Jerome Mariette committed
154
        
Jerome Mariette's avatar
Jerome Mariette committed
155
156
157
        if self.id is not None:
            self.directory = self.manager.get_workflow_directory(self.name, self.id)
            if not os.path.isdir(self.directory):
158
                os.makedirs(self.directory, 0o751)
Frédéric Escudié's avatar
Frédéric Escudié committed
159
            if self.stderr is None:
160
                self.stderr = self._set_stderr()
Jerome Mariette's avatar
Jerome Mariette committed
161
            self._serialize()
Jerome Mariette's avatar
Jerome Mariette committed
162
            
163
164
        self.internal_components = self._import_internal_components()
        self.external_components = self._import_external_components()
165

166
167
    def get_workflow_group(self):
        return self.__group
168
            
Jerome Mariette's avatar
Jerome Mariette committed
169
    def add_input_directory(self, name, help, default=None, required=False, flag=None, 
170
171
172
                            group="default", display_name=None, get_files_fn=None, add_to=None):
        new_param = InputDirectory(name, help, flag=flag, default=default, required=required, 
                                   group=group, display_name=display_name, get_files_fn=get_files_fn)
Jerome Mariette's avatar
Jerome Mariette committed
173
174
175
176
177
178
179
180
181
182
183
        new_param.linkTrace_nameid = name
        # if this input should be added to a particular parameter
        if add_to:
            try:
                self.__getattribute__(add_to).add_sub_parameter(new_param)
            except: pass
        # otherwise, add it to the class itself
        else:
            self.params_order.append(name)
            self.__setattr__(name, new_param)
    
184
    def add_input_file(self, name, help, file_format="any", default=None, type="inputfile", 
185
186
187
188
                       required=False, flag=None, group="default", display_name=None, size_limit="0", add_to=None):
        # check if the size provided is correct
        try: int(get_nb_octet(size_limit))
        except: size_limit="0"
189
        new_param = InputFile(name, help, flag=flag, file_format=file_format, default=default, 
190
                              type=type, required=required, group=group, display_name=display_name, size_limit=size_limit)
Frédéric Escudié's avatar
Frédéric Escudié committed
191
        new_param.linkTrace_nameid = name
192
193
194
195
196
197
        # if this input should be added to a particular parameter
        if add_to:
            try:
                self.__getattribute__(add_to).add_sub_parameter(new_param)
            except: pass
        # otherwise, add it to the class itself
198
        else:
199
200
201
202
            self.params_order.append(name)
            self.__setattr__(name, new_param)
            
    def add_input_file_list(self, name, help, file_format="any", default=None, type="inputfile", 
203
204
                            required=False, flag=None, group="default", display_name=None, size_limit="0", add_to=None):
        # check if the size provided is correct
205
        if default == None: default = []
206
207
        try: int(get_nb_octet(size_limit))
        except: size_limit="0"
208
209
210
211
212
213
214
        if default == None:
            inputs = []
        elif issubclass(default.__class__, list):
            inputs = [IOFile(file, file_format, name, None) for file in default]
        else:
            inputs = [IOFile(default, file_format, name, None)]
        new_param = InputFileList(name, help, flag=flag, file_format=file_format, default=inputs, 
215
                                  type=type, required=required, group=group, display_name=display_name, size_limit=size_limit)
Frédéric Escudié's avatar
Frédéric Escudié committed
216
        new_param.linkTrace_nameid = name
217
218
219
        # if this input should be added to a particular parameter
        if add_to:
            try:
Jerome Mariette's avatar
help ok    
Jerome Mariette committed
220
                self.__getattribute__(add_to).add_sub_parameter(new_param)
221
222
223
224
225
            except: pass
        # otherwise, add it to the class itself
        else:
            self.params_order.append(name)
            self.__setattr__(name, new_param)
226
227
            
    def add_multiple_parameter(self, name, help, required=False, flag=None, group="default", display_name=None):
228
        self.params_order.append(name)
229
230
231
232
233
234
        new_param = MultiParameter(name, help, flag=flag, required=required, group=group, display_name=display_name)
        self.__setattr__(name, new_param)

    def add_multiple_parameter_list(self, name, help, required=False, flag=None, group="default", display_name=None):
        self.params_order.append(name)
        new_param = MultiParameterList(name, help, flag=flag, required=required, group=group, display_name=display_name)
235
236
        self.__setattr__(name, new_param)
    
237
    def add_parameter(self, name, help, default=None, type=str, choices=None, 
238
                      required=False, flag=None, group="default", display_name=None, add_to=None):
239
        new_param = ParameterFactory.factory(name, help, flag=flag, default=default, type=type, choices=choices, 
240
241
242
243
244
245
246
                              required=required, group=group, display_name=display_name)
        # if this input should be added to a particular parameter
        if add_to:
            try:
                self.__getattribute__(add_to).add_sub_parameter(new_param)
            except: pass
        # otherwise, add it to the class itself
247
        else:
248
249
250
            self.params_order.append(name)
            self.__setattr__(name, new_param)
    
251
    def add_parameter_list(self, name, help, default=None, type=str, choices=None, 
252
253
254
                           required=False, flag=None, group="default", display_name=None, add_to=None):
        if default == None: default = []
        new_param = ParameterList(name, help, flag=flag, default=default, type=type, choices=choices, 
255
256
257
258
                                  required=required, group=group, display_name=display_name)
        # if this input should be added to a particular parameter
        if add_to:
            try:
Jerome Mariette's avatar
help ok    
Jerome Mariette committed
259
                self.__getattribute__(add_to).add_sub_parameter(new_param)
260
261
262
263
264
265
            except: pass
        # otherwise, add it to the class itself
        else:
            self.params_order.append(name)
            self.__setattr__(name, new_param)
    
Jerome Mariette's avatar
Jerome Mariette committed
266
    def add_exclusion_rule(self, *args2exclude):
Jerome Mariette's avatar
Jerome Mariette committed
267
        # first of all, does this parameter exist
Jerome Mariette's avatar
Jerome Mariette committed
268
269
270
271
272
273
274
275
276
277
278
279
        params2exclude = []
        for arg2exclude in args2exclude:
            try:
                params2exclude.append(self.__getattribute__(arg2exclude))
            except: pass
        # everything is ok, let's go
        if len(params2exclude) == len(args2exclude):
            new_group = "exclude-"+uuid.uuid4().hex[:5]
            for paramsexclude in params2exclude:
                paramsexclude.group = new_group
        # it might be a mutliple param rule
        else:
280
            self._log("Exclusion rule cannot be applied within a MultiParameter or a MultiParameterList", raisee=True)
Jerome Mariette's avatar
Jerome Mariette committed
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
        # save this for MultiParameter internal exclusion rules, works on command line, not supported on gui
#             for attribute_value in self.__dict__.values():
#                 if issubclass(attribute_value.__class__, MultiParameter) or issubclass(attribute_value.__class__, MultiParameterList):
#                     params2exclude = []
#                     for sub_param in attribute_value.sub_parameters:
#                         if sub_param.name in args2exclude:
#                             params2exclude.append(sub_param)
#                     if len(params2exclude) == len(args2exclude):
#                         new_group = "exclude-"+uuid.uuid4().hex[:5]
#                         flags2exclude = []
#                         for paramsexclude in params2exclude:
#                             paramsexclude.group = new_group
#                             flags2exclude.append(paramsexclude.flag)
#                         attribute_value.type.excludes[new_group] = flags2exclude
#                         break
296

Frédéric Escudié's avatar
Frédéric Escudié committed
297
    def _prepare_parameter(self, args, parameter, key="name"):
298
299
        new_param = None
        # Retrieve value
300
        if parameter.__getattribute__(key) in args:
Frédéric Escudié's avatar
Frédéric Escudié committed
301
            value = args[parameter.__getattribute__(key)]
Jerome Mariette's avatar
Jerome Mariette committed
302
        elif parameter.default != None:
303
304
305
306
            value = parameter.default
        else:
            value = None
        # Set new parameter
Frédéric Escudié's avatar
Frédéric Escudié committed
307
308
        if parameter.__class__ in [StrParameter, IntParameter, FloatParameter, BoolParameter, DateParameter]:
            if value == "" and parameter.__class__ in [IntParameter, FloatParameter, BoolParameter, DateParameter] : value = None # from GUI
309
310
311
312
            new_param = ParameterFactory.factory( parameter.name, parameter.help, default=value, type=parameter.type, choices=parameter.choices, 
                                                  required=parameter.required, flag=parameter.flag, group=parameter.group, 
                                                  display_name=parameter.display_name )
        elif parameter.__class__ ==  ParameterList:
313
            if value == "" : value = [] # from GUI
314
315
316
317
            new_param = ParameterList( parameter.name, parameter.help, default=value, type=parameter.type, choices=parameter.choices,
                                       required=parameter.required, flag=parameter.flag, sub_parameters=parameter.sub_parameters,
                                       group=parameter.group, display_name=parameter.display_name )
        elif parameter.__class__ == InputFileList:
318
            if value == "" : value = [] # from GUI
319
            iovalues = []
Ibouniyamine Nabihoudine's avatar
Ibouniyamine Nabihoudine committed
320
            prepared_files = parameter.prepare(value)
321
            for file in prepared_files:
322
323
                iovalues.append(IOFile(file, parameter.file_format, parameter.linkTrace_nameid, None))
            new_param = InputFileList( parameter.name, parameter.help, file_format=parameter.file_format, default=iovalues,
324
325
                                       type=parameter.type, choices=parameter.choices, required=parameter.required, flag=parameter.flag,
                                       group=parameter.group, display_name=parameter.display_name, size_limit=parameter.size_limit )
326
            new_param.linkTrace_nameid = parameter.linkTrace_nameid            
327
        elif parameter.__class__ == InputFile:
328
            if value == "" : value = None # from GUI
Ibouniyamine Nabihoudine's avatar
Ibouniyamine Nabihoudine committed
329
            prepared_file = parameter.prepare(value)
330
            new_param = InputFile( parameter.name, parameter.help, file_format=parameter.file_format, default=prepared_file,
331
332
                                   type=parameter.type, choices=parameter.choices, required=parameter.required, flag=parameter.flag, 
                                   group=parameter.group, display_name=parameter.display_name )
333
            new_param.linkTrace_nameid = parameter.linkTrace_nameid
Jerome Mariette's avatar
Jerome Mariette committed
334
335
        elif parameter.__class__ == InputDirectory:
            if value == "" : value = None # from GUI
Ibouniyamine Nabihoudine's avatar
Ibouniyamine Nabihoudine committed
336
            prepared_directory = parameter.prepare(value)
337
            new_param = InputDirectory( parameter.name, parameter.help, default=prepared_directory, choices=parameter.choices, 
Jerome Mariette's avatar
Jerome Mariette committed
338
                                        required=parameter.required, flag=parameter.flag, group=parameter.group, 
339
                                        display_name=parameter.display_name, get_files_fn=parameter.get_files_fn)
Jerome Mariette's avatar
Jerome Mariette committed
340
            new_param.linkTrace_nameid = parameter.linkTrace_nameid
341
342
343
344
        else:
            raise Exception( "Unknown class '" +  parameter.__class__.__name__ + "' for parameter.")
        return new_param

345
346
347
    def _set_parameters(self, args):
        parameters = self.get_parameters()
        for param in parameters:
348
            new_param = None
349
            if param.__class__ == MultiParameter:
350
                new_param = MultiParameter(param.name, param.help, required=param.required, flag=param.flag, group=param.group, display_name=param.display_name)
351
                new_param.sub_parameters = param.sub_parameters
352
                if param.name in args:
353
354
355
356
357
358
                    sub_args = {}
                    for sarg in args[param.name]:
                        sub_args[sarg[0]] = sarg[1]
                    for sub_param in param.sub_parameters:
                        new_sub_parameter = self._prepare_parameter(sub_args, sub_param, "flag")
                        new_param[new_sub_parameter.name] = new_sub_parameter
359
            elif param.__class__ == MultiParameterList:
360
                new_param = MultiParameterList(param.name, param.help, required=param.required, flag=param.flag, group=param.group, display_name=param.display_name)
361
                new_param.sub_parameters = param.sub_parameters
362
                if param.name in args:
363
364
365
366
367
368
369
370
371
                    for idx, sargs in enumerate(args[param.name]):
                        new_multi_param = MultiParameter(param.name + '_' + str(idx), '', required=False, flag=None, group="default", display_name=None)
                        sub_args = {}
                        for sarg in sargs:
                            sub_args[sarg[0]] = sarg[1]
                        for sub_param in param.sub_parameters:
                            new_sub_param = self._prepare_parameter(sub_args, sub_param, "flag")
                            new_multi_param[new_sub_param.name] = new_sub_param
                        new_param.append(new_multi_param)
372
            else:
373
                new_param = self._prepare_parameter(args, param)
374
            self.__setattr__(param.name, new_param)
Frédéric Escudié's avatar
Frédéric Escudié committed
375

Jerome Mariette's avatar
Jerome Mariette committed
376
    def get_execution_graph(self):
377
        gr = digraph()
378
379
        # build a all_nodes table to store all nodes
        all_nodes = {}
380
        for ioparameter in list(self.__dict__.values()):
381
382
            if issubclass(ioparameter.__class__, InputFile):
                gr.add_node(ioparameter.name)
Jerome Mariette's avatar
Jerome Mariette committed
383
                gr.add_node_attribute(ioparameter.name, self.INPUTFILE_GRAPH_LABEL)
384
                gr.add_node_attribute(ioparameter.name, ioparameter.display_name)
385
                all_nodes[ioparameter.name] = None
386
387
            elif issubclass(ioparameter.__class__, InputFileList):
                gr.add_node(ioparameter.name)
Jerome Mariette's avatar
Jerome Mariette committed
388
389
390
391
392
393
                gr.add_node_attribute(ioparameter.name, self.INPUTFILES_GRAPH_LABEL)
                gr.add_node_attribute(ioparameter.name, ioparameter.display_name)
                all_nodes[ioparameter.name] = None
            elif issubclass(ioparameter.__class__, InputDirectory):
                gr.add_node(ioparameter.name)
                gr.add_node_attribute(ioparameter.name, self.INPUTDIRECTORY_GRAPH_LABEL)
394
                gr.add_node_attribute(ioparameter.name, ioparameter.display_name)
395
                all_nodes[ioparameter.name] = None
Philippe Bardou's avatar
Philippe Bardou committed
396
            elif issubclass(ioparameter.__class__, MultiParameter):
Jerome Mariette's avatar
Jerome Mariette committed
397
398
399
400
401
402
403
404
405
406
                for subparam in ioparameter.sub_parameters:
                    gr.add_node(subparam.name)
                    all_nodes[subparam.name] = None
                    if issubclass(subparam.__class__, InputFile):
                        gr.add_node_attribute(subparam.name, self.INPUTFILE_GRAPH_LABEL)
                    elif issubclass(subparam.__class__, InputFileList):
                        gr.add_node_attribute(subparam.name, self.INPUTFILES_GRAPH_LABEL)
                    elif issubclass(subparam.__class__, InputDirectory):
                        gr.add_node_attribute(subparam.name, self.INPUTDIRECTORY_GRAPH_LABEL)
                    gr.add_node_attribute(subparam.name, subparam.display_name)
Philippe Bardou's avatar
Philippe Bardou committed
407
408
409
410
411
412
413
414
415
            elif issubclass(ioparameter.__class__, MultiParameterList):
                for subparam in ioparameter.sub_parameters:
                    gr.add_node(subparam.name)
                    all_nodes[subparam.name] = None                        
                    if issubclass(subparam.__class__, InputDirectory):
                        gr.add_node_attribute(subparam.name, self.INPUTDIRECTORY_GRAPH_LABEL)
                    else:
                        gr.add_node_attribute(subparam.name, self.INPUTFILES_GRAPH_LABEL)
                    gr.add_node_attribute(subparam.name, subparam.display_name)
416
        for cpt in self.components:
417
418
            gr.add_node(cpt.get_nameid())
            gr.add_node_attribute(cpt.get_nameid(), self.COMPONENT_GRAPH_LABEL)
419
            gr.add_node_attribute(cpt.get_nameid(), cpt.get_nameid())
420
            all_nodes[cpt.get_nameid()] = None
Jerome Mariette's avatar
Jerome Mariette committed
421
        for cpt in self.components:
422
            for ioparameter in list(cpt.__dict__.values()):
Jerome Mariette's avatar
Jerome Mariette committed
423
                if issubclass( ioparameter.__class__, InputFile ) or issubclass( ioparameter.__class__, InputFileList) or issubclass( ioparameter.__class__, InputDirectory):
Frédéric Escudié's avatar
Frédéric Escudié committed
424
425
                    for parent in ioparameter.parent_linkTrace_nameid:
                        try: gr.add_edge((parent, ioparameter.linkTrace_nameid))
Frédéric Escudié's avatar
Frédéric Escudié committed
426
                        except: pass
427
428
429
430
                elif issubclass( ioparameter.__class__, InputObject) or issubclass( ioparameter.__class__, InputObjectList):
                    for parent in ioparameter.parent_linkTrace_nameid:
                        try: gr.add_edge((parent, ioparameter.linkTrace_nameid))
                        except: pass
431
432
        # check if all nodes are connected
        for edge in gr.edges():
433
            if edge[0] in all_nodes:
434
                del all_nodes[edge[0]]
435
            if edge[1] in all_nodes:
436
437
                del all_nodes[edge[1]]
        # then remove all unconnected nodes: to delete inputs not defined by the user
438
        for orphan_node in list(all_nodes.keys()):
439
            gr.del_node(orphan_node)
440
        return gr
Frédéric Escudié's avatar
Frédéric Escudié committed
441

Jerome Mariette's avatar
Jerome Mariette committed
442
443
    def delete(self):
        if self.get_status() in [self.STATUS_COMPLETED, self.STATUS_FAILED, self.STATUS_ABORTED]:
Jerome Mariette's avatar
Jerome Mariette committed
444
            utils.robust_rmtree(self.directory)
Jerome Mariette's avatar
Jerome Mariette committed
445

446
    @staticmethod
447
    def config_parser(arg_lines):
448
449
        for arg in arg_lines:
            yield arg
450
451
452
453
            
    @staticmethod
    def get_status_under_text_format(workflow, detailed=False, display_errors=False, html=False):
        if workflow.start_time: start_time = time.asctime(time.localtime(workflow.start_time))
454
        else: start_time = "-"
455
456
        if workflow.start_time and workflow.end_time: elapsed_time = str(workflow.end_time-workflow.start_time)
        elif workflow.start_time: elapsed_time = str(time.time()-workflow.start_time)
457
        else: elapsed_time = "-"
Jerome Mariette's avatar
Jerome Mariette committed
458
        elapsed_time = "-" if elapsed_time == "-" else str(datetime.timedelta(seconds=int(str(elapsed_time).split(".")[0])))
459
        if workflow.end_time: end_time = time.asctime(time.localtime(workflow.end_time))
460
461
462
        else: end_time = "-"
        if detailed:
            # Global
463
464
            title = "Workflow #" + utils.get_nb_string(workflow.id) + " (" + workflow.name + ") is " + \
                    workflow.get_status() + ", time elapsed: " + str(elapsed_time) + " (from " + start_time + \
465
466
                    " to " + end_time + ")"
            worflow_errors = ""
467
            error = workflow.get_errors()
468
            if error is not None:
Jerome Mariette's avatar
Jerome Mariette committed
469
470
                if html: worflow_errors = "Workflow Error :\n  <span style='color:#ff0000'>" + error["location"] + "\n    " + "\n    ".join(error["msg"]) + "</span>"
                else: worflow_errors = "Workflow Error :\n  \033[91m" + error["location"] + "\n    " + "\n    ".join(error["msg"]) + "\033[0m"
471
472
473
            # By components
            components_errors = ""
            status = "Components Status :\n"
474
            components_status = workflow.get_components_status()
475
            for i, component in enumerate(workflow.get_components_nameid()):
476
                status_info = components_status[component]
477
478
479
480
481
482
483
484
485
486
487
                try: perc_waiting = (status_info["waiting"]*100.0)/status_info["tasks"]
                except: perc_waiting = 0
                try: perc_running = (status_info["running"]*100.0)/status_info["tasks"]
                except: perc_running = 0
                try: perc_failed = (status_info["failed"]*100.0)/status_info["tasks"]
                except: perc_failed = 0
                try: perc_aborted = (status_info["aborted"]*100.0)/status_info["tasks"]
                except: perc_aborted = 0
                try: perc_completed = (status_info["completed"]*100.0)/status_info["tasks"]
                except: perc_completed = 0
                
Jerome Mariette's avatar
Jerome Mariette committed
488
489
490
                if status_info["running"] > 0: 
                    if html: running = "<span style='color:#3b3bff'>running:" + str(status_info["running"]) + "</span>"
                    else: running = "\033[94mrunning:" + str(status_info["running"]) + "\033[0m"
491
                else: running = "running:" + str(status_info["running"])
Jerome Mariette's avatar
Jerome Mariette committed
492
493
494
                if status_info["waiting"] > 0: 
                    if html: waiting = "<span style='color:#ffea00'>waiting:" + str(status_info["waiting"]) + "</span>"
                    else: waiting = "\033[93mwaiting:" + str(status_info["waiting"]) + "\033[0m"
495
                else: waiting = "waiting:" + str(status_info["waiting"])            
Jerome Mariette's avatar
Jerome Mariette committed
496
497
498
                if status_info["failed"] > 0: 
                    if html: failed = "<span style='color:#ff0000'>failed:" + str(status_info["failed"]) + "</span>"
                    else: failed = "\033[91mfailed:" + str(status_info["failed"]) + "\033[0m"
499
                else: failed = "failed:" + str(status_info["failed"])
Jerome Mariette's avatar
Jerome Mariette committed
500
501
502
                if status_info["aborted"] > 0: 
                    if html: aborted = "<span style='color:#ff01ba'>aborted:" + str(status_info["aborted"]) + "</span>"
                    else: aborted = "\033[95maborted:" + str(status_info["aborted"]) + "\033[0m"
503
                else: aborted = "aborted:" + str(status_info["aborted"])
Jerome Mariette's avatar
Jerome Mariette committed
504
505
506
                if status_info["completed"] == status_info["tasks"] and status_info["completed"] > 0: 
                    if html: completed = "<span style='color:#14ac00'>completed:" + str(status_info["completed"]) + "</span>"
                    else: completed = "\033[92mcompleted:" + str(status_info["completed"]) + "\033[0m"
507
508
509
510
511
                else: completed = "completed:" + str(status_info["completed"])
                
                if display_errors and len(status_info["failed_commands"]) > 0:
                    if components_errors == "" :
                        components_errors = "Failed Commands :\n"  
512
                    components_errors += "  - " + component + " :\n    " + "\n    ".join(status_info["failed_commands"]) + "\n"
513
514
515
                status += "  - " + component + ", time elapsed " + time_format(status_info["time"]) + \
                    " (total:" + str(status_info["tasks"]) + ", " + waiting + ", " + running + ", " + failed + \
                    ", " + aborted + ", " + completed + ")"
516
                if i<len(workflow.get_components_nameid())-1: status += "\n"
517
518
519
            # Format str
            pretty_str = title
            pretty_str += ("\n" + worflow_errors) if worflow_errors != "" else ""
520
521
            if len(workflow.get_components_nameid()) > 0:
                pretty_str += ("\n" + status) if status != "" else ""
522
                pretty_str += ("\n" + components_errors[:-1]) if components_errors != "" else ""
Jerome Mariette's avatar
Jerome Mariette committed
523
524
            if html: return pretty_str.replace("\n", "<br />")
            else: return pretty_str
525
        else:
526
527
            pretty_str = utils.get_nb_string(workflow.id) + "\t" + workflow.name + "\t"
            if workflow.get_status() == Workflow.STATUS_STARTED:
Jerome Mariette's avatar
Jerome Mariette committed
528
                pretty_str += "\033[94m"
529
            elif workflow.get_status() == Workflow.STATUS_COMPLETED:
Jerome Mariette's avatar
Jerome Mariette committed
530
                pretty_str += "\033[92m"
531
            elif workflow.get_status() == Workflow.STATUS_FAILED:
Jerome Mariette's avatar
Jerome Mariette committed
532
                pretty_str += "\033[91m"
533
            elif workflow.get_status() == Workflow.STATUS_ABORTED:
Jerome Mariette's avatar
Jerome Mariette committed
534
                pretty_str += "\033[91m"
535
            elif workflow.get_status() == Workflow.STATUS_RESETED:
Jerome Mariette's avatar
Jerome Mariette committed
536
                pretty_str += "\033[3m"
537
            pretty_str += workflow.get_status() + "\033[0m"
Jerome Mariette's avatar
Jerome Mariette committed
538
539
            pretty_str += "\t" + elapsed_time + "\t" + start_time + "\t" + end_time
            return pretty_str
540
    
541
    def get_errors(self):
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
        if os.path.isfile(self.stderr):
            error = {
                "title"     : "",
                "msg"       : list(),
                "traceback" : list()
            }
            line_idx = 0
            FH_stderr = open( self.stderr )
            lines = FH_stderr.readlines()
            while line_idx < len(lines):
                if lines[line_idx].strip().startswith("##"):
                    error["title"]     = lines[line_idx].rstrip()
                    error["msg"]       = list()
                    error["traceback"] = list()
                    # skip all lines before the traceback
                    while not lines[line_idx].startswith("Traceback"):
                        line_idx += 1
                    # skip : "Traceback (most recent call last):"
                    line_idx += 1    
                    while lines[line_idx] != lines[line_idx].lstrip():
                        error["traceback"].append({ 
                                                   "location" : lines[line_idx].strip(),
                                                   "line"     : lines[line_idx].strip()
                        })
                        line_idx += 2
                    # Error message
                    while line_idx < len(lines) and not lines[line_idx].strip().startswith("##"):
                        try:
                            error["msg"].append( lines[line_idx].strip().split(":", 1)[1][1:] )
                        except:
                            error["msg"].append( lines[line_idx].strip() )
                        line_idx += 1
                    line_idx -= 1
                line_idx += 1
            FH_stderr.close()
            last_stack_location = ""
            if len(error["traceback"]) > 0:
                last_stack_location = error["traceback"][-1]["location"].strip()
                self.status = self.STATUS_FAILED
                return { "msg" : error["msg"], "location" : last_stack_location }
            else:
                return None
Frédéric Escudié's avatar
Frédéric Escudié committed
584
585
        else:
            return None
586
                
587
    def get_outputs_per_components(self):
588
589
        outputs_files = {}
        for current_components in self.components:
Philippe Bardou's avatar
Philippe Bardou committed
590
            #status = self.get_component_status(current_components.get_nameid())
591
            outputs_files[current_components.get_nameid()] = current_components.get_output_files()
Philippe Bardou's avatar
Philippe Bardou committed
592
            #outputs_files["0"] = status["completed"]
593
594
        return outputs_files
    
Jerome Mariette's avatar
Jerome Mariette committed
595
596
    def __setstate__(self, state):
        self.__dict__ = state.copy()
597
        self.external_components = self._import_external_components()
Jerome Mariette's avatar
Jerome Mariette committed
598
599
600
601
602
603
604
        threading.Thread.__init__(self, name=self.name)
        
    def __getstate__(self):
        """
        Threading uses Lock Object, do not consider these objects when serializing a workflow
        """
        odict = self.__dict__.copy()
605
        del odict['_started']
606
607
608
609
        if '_tstate_lock' in odict: # python 3.4
            del odict['_tstate_lock']
        else: # python 3.2
            del odict['_block']
610
611
        del odict['_stderr']
        if 'external_components' in odict:
612
            del odict['external_components']
Jerome Mariette's avatar
Jerome Mariette committed
613
614
        return odict
    
615
616
617
618
619
620
621
622
623
624
625
626
627
    def set_to_address(self, to_address):
        self.__to_address = to_address

    def set_subject(self, subject):
        self.__subject = subject

    def set_message(self, message):
        self.__message = message
    
    def _send_email(self):
        
        import smtplib
        from email.mime.text import MIMEText        
628
        smtps, smtpp, froma, fromp, toa, subject, message = self.jflow_config_reader.get_email_options()
629
630
631
632
633
        
        if self.__to_address: toa = self.__to_address
        if self.__subject: subject = self.__subject
        if self.__message: message = self.__message
        
634
        if smtps and smtpp and froma and fromp:
635
636
            if not toa: toa = froma
            if validate_email(froma) and validate_email(toa):
637
638
639
640
641
                try:
                    # Open a plain text file for reading.  For this example, assume that
                    # the text file contains only ASCII characters.
                    # Create a text/plain message
                    if not message:
642
                        message = Workflow.get_status_under_text_format(self, True, True, True)
Jerome Mariette's avatar
Jerome Mariette committed
643
                    msg = MIMEText(message, 'html')
644
645
646
647
648
649
650
651
652
653
                    me = froma
                    you = toa
                    if not subject: subject = "JFlow - Workflow #" + str(self.id) + " is " + self.status
                    msg['Subject'] = subject
                    msg['From'] = me
                    msg['To'] = you
                    # Send the message via our own SMTP server, but don't include the
                    # envelope header.
                    s = smtplib.SMTP(smtps, smtpp)
                    s.ehlo()
Jerome Mariette's avatar
Jerome Mariette committed
654
655
656
657
658
                    # if the SMTP server does not provides TLS or identification
                    try:
                        s.starttls()
                        s.login(me, fromp)
                    except smtplib.SMTPHeloError:
659
                        self._log("The server didn't reply properly to the HELO greeting.", level="debug", traceback=traceback.format_exc(chain=False))
Jerome Mariette's avatar
Jerome Mariette committed
660
                    except smtplib.SMTPAuthenticationError:
661
                        self._log("The server didn't accept the username/password combination.", level="debug", traceback=traceback.format_exc(chain=False))
Jerome Mariette's avatar
Jerome Mariette committed
662
                    except smtplib.SMTPException:
663
                        self._log("No suitable authentication method was found, or the server does not support the STARTTLS extension.", level="debug", traceback=traceback.format_exc(chain=False))
Jerome Mariette's avatar
Jerome Mariette committed
664
                    except RuntimeError:
665
                        self._log("SSL/TLS support is not available to your Python interpreter.", level="debug", traceback=traceback.format_exc(chain=False))
Jerome Mariette's avatar
Jerome Mariette committed
666
                    except:
667
                        self._log("Unhandled error when sending mail.", level="debug", traceback=traceback.format_exc(chain=False))
Jerome Mariette's avatar
Jerome Mariette committed
668
669
670
                    finally:
                        s.sendmail(me, [you], msg.as_string())
                        s.close()
671
                except:
672
                    self._log("Impossible to connect to smtp server '" + smtps + "'", level="debug", traceback=traceback.format_exc(chain=False))
673
    
674
675
676
677
    def get_parameters_per_groups(self):
        name = self.get_name()
        description = self.get_description()
        parameters = self.get_parameters()
678
679
        pgparameters, parameters_order = {}, []
        for param in parameters:
Jerome Mariette's avatar
Jerome Mariette committed
680
            if param.group not in parameters_order: parameters_order.append(param.group)
681
            if param.group in pgparameters:
682
                pgparameters[param.group].append(param)
Jerome Mariette's avatar
Jerome Mariette committed
683
            else:
684
685
                pgparameters[param.group] = [param]
        return [pgparameters, parameters_order]
Jerome Mariette's avatar
Jerome Mariette committed
686
    
687
688
689
    def get_parameters(self):
        params = []
        for param in self.params_order:
690
            for attribute_value in list(self.__dict__.values()):
691
692
693
694
                if (issubclass(attribute_value.__class__, AbstractParameter)) and param == attribute_value.name:
                    params.append(attribute_value)
        return params
    
695
    def get_exec_path(self, software):
696
697
698
699
700
        exec_path = self.jflow_config_reader.get_exec(software)
        if exec_path is None and os.path.isfile(os.path.join(os.path.dirname(inspect.getfile(self.__class__)), "../bin", software)):
            exec_path = os.path.join(os.path.dirname(inspect.getfile(self.__class__)), "../bin", software)
        elif exec_path is None and os.path.isfile(os.path.join(os.path.dirname(inspect.getfile(self.__class__)), "bin", software)):
            exec_path = os.path.join(os.path.dirname(inspect.getfile(self.__class__)), "bin", software)
701
        elif exec_path is None and utils.which(software) == None:
702
            raise Exception("'" + software + "' path connot be retrieved either in the PATH and in the application.properties file!")
703
704
705
        elif exec_path is None and utils.which(software) != None: 
            exec_path = software
        elif exec_path != None and not os.path.isfile(exec_path):
706
            raise Exception("'" + exec_path + "' set for '" + software + "' does not exists, please provide a valid path!")
707
        return exec_path
708
    
Jerome Mariette's avatar
Jerome Mariette committed
709
710
    def add_component(self, component_name, args=[], kwargs={}, component_prefix="default"):
        # first build and check if this component is OK
711
        if component_name in self.internal_components or component_name in self.external_components:
712
            
713
            if component_name in self.internal_components:
Jerome Mariette's avatar
Jerome Mariette committed
714
                my_pckge = __import__(self.internal_components[component_name], globals(), locals(), [component_name])
715
716
717
718
719
720
721
722
723
724
725
726
727
                # build the object and define required field
                cmpt_object = getattr(my_pckge, component_name)()
                cmpt_object.output_directory = self.get_component_output_directory(component_name, component_prefix)
                cmpt_object.prefix = component_prefix
                if kwargs: cmpt_object.define_parameters(**kwargs)
                else: cmpt_object.define_parameters(*args)
            # external components
            else :
                cmpt_object = self.external_components[component_name]()
                cmpt_object.output_directory = self.get_component_output_directory(component_name, component_prefix)
                cmpt_object.prefix = component_prefix
                # can't use positional arguments with external components
                cmpt_object.define_parameters(**kwargs)
728
729
            
            # there is a dynamic component
730
            if cmpt_object.is_dynamic():
731
732
733
734
                self.dynamic_component_present = True
                # if already init, add the component to the list and check if weaver should be executed
                if self.component_nameids_is_init:
                    # add the component
735
                    self.components_to_exec.append(cmpt_object)
736
                    self.components.append(cmpt_object)
737
738
739
740
741
742
743
744
745
                    self._execute_weaver()
                    # update outputs
                    for output in cmpt_object.get_dynamic_outputs():
                        output.update()
                else:
                    if self._component_is_duplicated(cmpt_object):
                        raise ValueError("Component " + cmpt_object.__class__.__name__ + " with prefix " + 
                                            cmpt_object.prefix + " already exist in this pipeline!")
                    self.component_nameids[cmpt_object.get_nameid()] = None
746
                    self.components_to_exec = []
747
                    self.components = []
748
749
750
            else:
                if self.component_nameids_is_init:
                    # add the component
751
                    self.components_to_exec.append(cmpt_object)
752
                    self.components.append(cmpt_object)
753
                elif not self.component_nameids_is_init and not self.dynamic_component_present:
Jerome Mariette's avatar
Jerome Mariette committed
754
755
756
                    if self._component_is_duplicated(cmpt_object):
                        raise ValueError("Component " + cmpt_object.__class__.__name__ + " with prefix " + 
                                            cmpt_object.prefix + " already exist in this pipeline!")
757
                    self.components_to_exec.append(cmpt_object)
758
                    self.components.append(cmpt_object)
759
760
761
762
763
764
                else:
                    if self._component_is_duplicated(cmpt_object):
                        raise ValueError("Component " + cmpt_object.__class__.__name__ + " with prefix " + 
                                            cmpt_object.prefix + " already exist in this pipeline!")
                    self.component_nameids[cmpt_object.get_nameid()] = None

765
766
            return cmpt_object
        else:
767
            raise ImportError(component_name + " component cannot be loaded, available components are: {0}".format(
768
                                           ", ".join(list(self.internal_components.keys()) + list(self.external_components.keys()))))
Jerome Mariette's avatar
Jerome Mariette committed
769
770
771
772
773
774
775
776
    
    def pre_process(self):
        pass
    
    def process(self):
        """ 
        Run the workflow, has to be implemented by subclasses
        """
777
        raise NotImplementedError( "Workflow.process() must be implemented in " + self.__class__.__name__ )
778
779
780

    def get_name(self):
        """ 
781
        Return the workflow name.
782
        """
Jerome Mariette's avatar
Jerome Mariette committed
783
        return self.__class__.__name__.lower()
784
785
786
787
788
    
    def get_description(self):
        """ 
        Return the workflow description, has to be implemented by subclasses
        """
789
        raise NotImplementedError( "Workflow.get_description() must be implemented in " + self.__class__.__name__ )
790
    
791
    def define_parameters(self, function="process"):
792
793
794
        """ 
        Define the workflow parameters, has to be implemented by subclasses
        """
795
        raise NotImplementedError( "Workflow.define_parameters() must be implemented in " + self.__class__.__name__ )
Jerome Mariette's avatar
Jerome Mariette committed
796
797
798
799
800
801
802
803
804
805
806
807
    
    def post_process(self):
        pass
    
    def get_temporary_file(self, suffix=".txt"):
        tempfile_name = os.path.basename(tempfile.NamedTemporaryFile(suffix=suffix).name)
        return os.path.join(self.jflow_config_reader.get_tmp_directory(), tempfile_name)

    def get_component_output_directory(self, component_name, component_prefix):
        return os.path.join(self.directory, component_name + "_" + component_prefix)
    
    def get_components_nameid(self):
808
        return list(self.component_nameids.keys())
Jerome Mariette's avatar
Jerome Mariette committed
809
    
Jerome Mariette's avatar
Jerome Mariette committed
810
811
    def wf_execution_wrapper(self):
        getattr(self, self.function)()
Jerome Mariette's avatar
Jerome Mariette committed
812
    
Jerome Mariette's avatar
Jerome Mariette committed
813
814
815
816
    def run(self):
        """
        Only require for Threading
        """
817
818
        try:
            # if this is the first time the workflow run
Jerome Mariette's avatar
Jerome Mariette committed
819
            if self.__step == None :
820
                self.start_time = time.time()
Jerome Mariette's avatar
Jerome Mariette committed
821
                self.__step = 0
822
823
                self.status = self.STATUS_STARTED
                self.end_time = None
Jerome Mariette's avatar
Jerome Mariette committed
824
825
                # if some args are provided, let's fill the parameters
                self._set_parameters(self.args)
826
                self._serialize()
827
            # if pre_processing has not been done yet
Jerome Mariette's avatar
Jerome Mariette committed
828
            if self.__step == 0:
829
                self.pre_process()
Jerome Mariette's avatar
Jerome Mariette committed
830
                self.__step = 1
831
832
                self._serialize()
            # if collecting components and running workflow has not been done yet
Jerome Mariette's avatar
Jerome Mariette committed
833
            if self.__step == 1:
834
835
                try:
                    self.reseted_components = []
836
                    self.components = []
Jerome Mariette's avatar
Jerome Mariette committed
837
                    self.status = self.STATUS_STARTED
838
                    self.wf_execution_wrapper()
839
                except SystemExit: 
840
841
842
                    self.status = self.STATUS_FAILED
                    self.end_time = time.time()
                    self._serialize()
843
                    self._send_email()
844
845
846
                    raise
                self.component_nameids_is_init = True
                if self.dynamic_component_present:
Jerome Mariette's avatar
Jerome Mariette committed
847
                    self.__step = 2
848
849
                else:
                    self._execute_weaver()
Jerome Mariette's avatar
Jerome Mariette committed
850
                    self.__step = 3
851
852
                self._serialize() 
            # if the workflow was a dynamic one
Jerome Mariette's avatar
Jerome Mariette committed
853
            if self.__step == 2:
854
855
                try:
                    self.reseted_components = []
856
                    self.components = []
Jerome Mariette's avatar
Jerome Mariette committed
857
                    self.status = self.STATUS_STARTED
858
                    self.wf_execution_wrapper()
859
                except SystemExit: 
860
861
862
                    self.status = self.STATUS_FAILED
                    self.end_time = time.time()
                    self._serialize()
863
                    self._send_email()
864
865
866
                    raise
                if len(self.components_to_exec) > 0:
                    self._execute_weaver()
Jerome Mariette's avatar
Jerome Mariette committed
867
                self.__step = 3
868
869
                self._serialize()
            # if post processing has ne been done yet
Jerome Mariette's avatar
Jerome Mariette committed
870
            if self.__step == 3:
871
                self.post_process()
872
                if self.status == self.STATUS_STARTED: self.status = self.STATUS_COMPLETED
873
874
                self.end_time = time.time()
                self._serialize()
875
                self._send_email()
876
        except Exception as e:
877
            self._log(str(e), traceback=traceback.format_exc(chain=False))