component.py 31.8 KB
Newer Older
Jerome Mariette's avatar
Jerome Mariette committed
1
#
Jerome Mariette's avatar
Jerome Mariette committed
2
# Copyright (C) 2015 INRA
Jerome Mariette's avatar
Jerome Mariette committed
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
# 
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
# 
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
# 
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
#

import os
19
import re
20
import sys
Jerome Mariette's avatar
Jerome Mariette committed
21
22
import inspect
import tempfile
Jerome Mariette's avatar
Jerome Mariette committed
23
import types
Floreal Cabanettes's avatar
Floreal Cabanettes committed
24
import shutil
Jerome Mariette's avatar
Jerome Mariette committed
25

26
27
from operator import attrgetter

Jerome Mariette's avatar
Jerome Mariette committed
28
29
30
from jflow.workflows_manager import WorkflowsManager
from jflow.config_reader import JFlowConfigReader
from jflow.dataset import ArrayList
Jerome Mariette's avatar
Jerome Mariette committed
31
from jflow.utils import which, display_error_message
Jerome Mariette's avatar
Jerome Mariette committed
32
from jflow.parameter import *
33
from jflow.abstraction import MultiMap
Jerome Mariette's avatar
Jerome Mariette committed
34

35
36
from weaver.util import parse_string_list
from weaver.function import ShellFunction
37
from jflow.abstraction import Map
38
39
from weaver.function import PythonFunction

Jerome Mariette's avatar
Jerome Mariette committed
40
41

class Component(object):
Céline Noirot's avatar
Céline Noirot committed
42
    
Jerome Mariette's avatar
Jerome Mariette committed
43
    TRACE_FILE_NAME = "trace.txt"
Céline Noirot's avatar
Céline Noirot committed
44
    
45
    def __init__(self):
Floreal Cabanettes's avatar
Floreal Cabanettes committed
46
        self.__prefix = "default"
Jerome Mariette's avatar
Jerome Mariette committed
47
        self.params_order = []
Jerome Mariette's avatar
Jerome Mariette committed
48
        self.output_directory = None
49
        self.description = None
Jerome Mariette's avatar
Jerome Mariette committed
50
        self.config_reader = JFlowConfigReader()
51
        self.version = self.get_version()
Jerome Mariette's avatar
Jerome Mariette committed
52
53
        if isinstance(self.version, bytes):
            self.version = self.version.decode()
54
        self.batch_options = self.config_reader.get_component_batch_options(self.__class__.__name__)
55
        self.modules = self.config_reader.get_component_modules(self.__class__.__name__)
56
        # in case of SGE, parse the cpu and memory parameter
57
58
        self.__cpu=None
        self.__memory=None
Céline Noirot's avatar
Céline Noirot committed
59
        type, options, limit_submission = self.config_reader.get_batch()
60
        if type.lower() == "sge" :
61
62
            try:
                self.__cpu = int(re.match( r'.*-pe\s+(\w+)\s+(\d+)\s?.*', self.batch_options).group(2))
63
64
            except: pass
            try:
65
                self.__memory = re.match( r'.*-l\s+mem=(\d+\S+)\s?.*', self.batch_options).group(1)
66
            except: pass
Céline Noirot's avatar
Céline Noirot committed
67
68
69
70
71
        elif type.lower() == "local" :
            try:
                self.__cpu = int(re.match( r'.*cpu=(\d+)\s?.*', self.batch_options).group(1))
            except: pass
            try:
72
                self.__memory = re.match( r'.*\s?mem=(\d+\w)\s?.*', self.batch_options).group(1)
Céline Noirot's avatar
Céline Noirot committed
73
            except: pass
74

Floreal Cabanettes's avatar
Floreal Cabanettes committed
75
76
77
78
79
80
    def get_prefix(self):
        return self.__prefix

    def set_prefix(self, prefix):
        self.__prefix = prefix

81
82
83
    def get_description(self):
        return self.description

84
85
86
87
88
    def get_cpu(self):
        return self.__cpu
    
    def get_memory(self):
        return self.__memory
Frédéric Escudié's avatar
Frédéric Escudié committed
89
90

    def is_dynamic(self):
91
92
93
94
95
96
97
        return len(self.get_dynamic_outputs()) != 0

    def get_dynamic_outputs(self):
        """
         @return : the list of outputs updated at the end of component execution.
        """
        dynamic_outputs = list()
98
        for attribute_value in list(self.__dict__.values()):
99
100
101
            if issubclass( attribute_value.__class__, DynamicOutput ):
                dynamic_outputs.append( attribute_value )
        return dynamic_outputs
Frédéric Escudié's avatar
Frédéric Escudié committed
102

103
    def get_output_files(self):
Philippe Bardou's avatar
Philippe Bardou committed
104
        outputs = {}
105
        for attribute_value in list(self.__dict__.values()):
106
            if ( issubclass( attribute_value.__class__, DynamicOutput ) or
107
                 issubclass( attribute_value.__class__, OutputFileList)):
Philippe Bardou's avatar
Philippe Bardou committed
108
                for f in attribute_value:
109
                    outputs[os.path.basename(f)] = f
110
            elif issubclass( attribute_value.__class__, OutputFile):
111
                outputs[os.path.basename(attribute_value)] = attribute_value
112
113
            elif issubclass( attribute_value.__class__, OutputDirectory):
                outputs[os.path.basename(attribute_value)] = attribute_value
114
        return outputs
Frédéric Escudié's avatar
Frédéric Escudié committed
115

116
117
118
    def add_input_directory(self, name, help, default=None, required=False, flag=None, 
                            group="default",  
                            display_name=None, cmd_format="", argpos=-1):
Jerome Mariette's avatar
Jerome Mariette committed
119
120
121
122
123
124
        new_param = InputDirectory(name, help, flag=flag, default=default, required=required, group=group, 
                                   display_name=display_name, cmd_format=cmd_format, argpos=argpos)
        # store where the parameter is coming from
        new_param.linkTrace_nameid = self.get_nameid()
        if issubclass( default.__class__, LinkTraceback ):
            new_param.parent_linkTrace_nameid = [default.linkTrace_nameid]
125
126
127
        # add it to the class itself
        self.params_order.append(name)
        self.__setattr__(name, new_param)
Jerome Mariette's avatar
Jerome Mariette committed
128

129
    def add_input_file(self, name, help, file_format="any", default=None, type="inputfile", 
130
                       required=False, flag=None, group="default", display_name=None, 
131
                       cmd_format="", argpos=-1):
132
        new_param = InputFile(name, help, flag=flag, file_format=file_format, default=default, 
133
134
                              type=type, required=required, group=group, display_name=display_name, 
                              cmd_format=cmd_format, argpos=argpos)
135
        # store where the parameter is coming from
Frédéric Escudié's avatar
Frédéric Escudié committed
136
        new_param.linkTrace_nameid = self.get_nameid()
Frédéric Escudié's avatar
Frédéric Escudié committed
137
        if issubclass( default.__class__, LinkTraceback ):
Frédéric Escudié's avatar
Frédéric Escudié committed
138
            new_param.parent_linkTrace_nameid = [default.linkTrace_nameid]
139
140
141
        # add it to the class itself
        self.params_order.append(name)
        self.__setattr__(name, new_param)
142
143
144
    
    def reset(self):
        for file in os.listdir(self.output_directory):
Floreal Cabanettes's avatar
Floreal Cabanettes committed
145
146
147
148
149
            final_file = os.path.join(self.output_directory, file)
            if os.path.isdir(final_file):
                shutil.rmtree(final_file)
            else:
                os.remove(final_file)
150
    
151
    def add_input_file_list(self, name, help, file_format="any", default=None, type="inputfile", 
152
                            required=False, flag=None, group="default", display_name=None,
153
                            cmd_format="", argpos=-1):
154
155
        if default == None:
            inputs = []
156
        elif issubclass(default.__class__, list):
Frédéric Escudié's avatar
Frédéric Escudié committed
157
            inputs = [IOFile(file, file_format, self.get_nameid(), None) for file in default]
158
159
        else:
            inputs = [IOFile(default, file_format, self.get_nameid(), None)]
160
        new_param = InputFileList(name, help, flag=flag, file_format=file_format, default=inputs, 
161
162
                                  type=type, required=required, group=group, display_name=display_name,
                                  cmd_format=cmd_format, argpos=argpos)
163
        # store where the parameter is coming from
Frédéric Escudié's avatar
Frédéric Escudié committed
164
        new_param.linkTrace_nameid = self.get_nameid()
165
166
        if issubclass( default.__class__, list ):
            for idx, val in enumerate(default):
Frédéric Escudié's avatar
Frédéric Escudié committed
167
                if issubclass( val.__class__, LinkTraceback ):
Frédéric Escudié's avatar
Frédéric Escudié committed
168
                    new_param[idx].parent_linkTrace_nameid = [val.linkTrace_nameid]
Frédéric Escudié's avatar
Frédéric Escudié committed
169
170
                    if not val.linkTrace_nameid in new_param.parent_linkTrace_nameid:
                        new_param.parent_linkTrace_nameid.append( val.linkTrace_nameid )
Frédéric Escudié's avatar
Frédéric Escudié committed
171
172
        else:
            if issubclass( default.__class__, LinkTraceback ):
Frédéric Escudié's avatar
Frédéric Escudié committed
173
174
175
                new_param[0].parent_linkTrace_nameid = [default.linkTrace_nameid]
                if not default.linkTrace_nameid in new_param.parent_linkTrace_nameid:
                    new_param.parent_linkTrace_nameid.append( default.linkTrace_nameid )
176
177
178
        # add it to the class itself
        self.params_order.append(name)
        self.__setattr__(name, new_param)
Frédéric Escudié's avatar
Frédéric Escudié committed
179

180
    def add_parameter(self, name, help, default=None, type=str, choices=None, 
181
                      required=False, flag=None, group="default", display_name=None,   
182
                      cmd_format="", argpos=-1):
Frédéric Escudié's avatar
Frédéric Escudié committed
183
        new_param = ParameterFactory.factory(name, help, flag=flag, default=default, type=type, choices=choices, 
184
185
                              required=required, group=group, display_name=display_name,  
                              cmd_format=cmd_format, argpos=argpos)
186
187
188
        # add it to the class itself
        self.params_order.append(name)
        self.__setattr__(name, new_param)
189

190
    def add_parameter_list(self, name, help, default=None, type=str, choices=None, 
191
                           required=False, flag=None, group="default", display_name=None,
192
                           cmd_format="", argpos=-1):
193
194
        if default == None: default = []
        new_param = ParameterList(name, help, flag=flag, default=default, type=type, choices=choices, 
195
196
                                  required=required, group=group, display_name=display_name,
                                  cmd_format=cmd_format, argpos=argpos)
197
198
199
        # add it to the class itself
        self.params_order.append(name)
        self.__setattr__(name, new_param)
Frédéric Escudié's avatar
Frédéric Escudié committed
200

201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
    def add_input_object(self, name, help, default=None, required=False):
        new_param = InputObject(name, help, default=default, required=required)
        # store where the parameter is coming from
        new_param.linkTrace_nameid = self.get_nameid()
        
        if issubclass( default.__class__, list ):
            for idx, val in enumerate(default):
                if hasattr( val, "linkTrace_nameid" ):
                    if not val.linkTrace_nameid in new_param.parent_linkTrace_nameid:
                        new_param.parent_linkTrace_nameid.append(val.linkTrace_nameid)
                    new_param.default[idx].parent_linkTrace_nameid = [val.linkTrace_nameid]
                    new_param.default[idx].linkTrace_nameid = self.get_nameid()
        elif hasattr( default, "linkTrace_nameid" ):
            new_param.parent_linkTrace_nameid = [default.linkTrace_nameid]
            new_param.default.parent_linkTrace_nameid = [default.linkTrace_nameid]
            new_param.default.linkTrace_nameid = self.get_nameid()
        
        # add it to the class itself
        self.params_order.append(name)
        self.__setattr__(name, new_param)        
    
    def add_input_object_list(self, name, help, default=None, required=False):
        if default == None: default = []
        new_param = InputObjectList(name, help, default=default, required=required)
        
        # store where the parameter is coming from
        new_param.linkTrace_nameid = self.get_nameid()
        for idx, val in enumerate(new_param.default):
            if hasattr( val, "linkTrace_nameid" ):
                if not val.linkTrace_nameid in new_param.parent_linkTrace_nameid:
                    new_param.parent_linkTrace_nameid.append(val.linkTrace_nameid)
                new_param.default[idx].parent_linkTrace_nameid = [val.linkTrace_nameid]
                new_param.default[idx].linkTrace_nameid = self.get_nameid()

        # add it to the class itself
        self.params_order.append(name)
        self.__setattr__(name, new_param)
    
    def add_output_object(self, name, help, required=False):
        new_param = OutputObject(name, help, required=required)
        # store where the parameter is coming from
        new_param.linkTrace_nameid = self.get_nameid()
        new_param.default.linkTrace_nameid = self.get_nameid()
        # add it to the class itself
        self.params_order.append(name)
        self.__setattr__(name, new_param)
    
    def add_output_object_list(self, name, help, nb_items=0, required=False):
        new_param = OutputObjectList(name, help, nb_items=nb_items, required=required)
        # store where the parameter is coming from
        new_param.linkTrace_nameid = self.get_nameid()
        for idx, val in enumerate(new_param.default):
            new_param.default[idx].linkTrace_nameid = self.get_nameid()
        # add it to the class itself
        self.params_order.append(name)
        self.__setattr__(name, new_param)
257
258
259
260
261
262
263
264
265
266
267

    def add_output_directory(self, name, help, dirname=None, group="default", display_name=None,
                            cmd_format="", argpos=-1):
        dirname = os.path.basename(dirname)
        new_param = OutputDirectory(name, help, default=os.path.join(self.output_directory, dirname),
                               group=group, display_name=display_name, cmd_format=cmd_format, argpos=argpos)
        # store where the parameter is coming from
        new_param.linkTrace_nameid = self.get_nameid()
        # add it to the class itself
        self.params_order.append(name)
        self.__setattr__(name, new_param)
268
        
269
    def add_output_file(self, name, help, file_format="any", filename=None, group="default", display_name=None,
270
                         cmd_format="", argpos=-1):
271
        filename = os.path.basename(filename)
Frédéric Escudié's avatar
Frédéric Escudié committed
272
        new_param = OutputFile(name, help, default=os.path.join(self.output_directory, filename),
273
274
                               file_format=file_format, group=group, display_name=display_name,
                               cmd_format=cmd_format, argpos=argpos)
Frédéric Escudié's avatar
Frédéric Escudié committed
275
        # store where the parameter is coming from
Frédéric Escudié's avatar
Frédéric Escudié committed
276
        new_param.linkTrace_nameid = self.get_nameid()
277
278
279
        # add it to the class itself
        self.params_order.append(name)
        self.__setattr__(name, new_param)
Frédéric Escudié's avatar
Frédéric Escudié committed
280

281
    def add_output_file_list(self, name, help, file_format="any", pattern='{basename_woext}.out', 
282
                             items=None, group="default", display_name=None, cmd_format="", argpos=-1):
Frédéric Escudié's avatar
Frédéric Escudié committed
283
        default = [IOFile(file, file_format, self.get_nameid(), None) for file in self.get_outputs(pattern, items)]
284
285
        new_param = OutputFileList(name, help, default=default, file_format=file_format, group=group, display_name=display_name,
                                   cmd_format=cmd_format, argpos=argpos)
Frédéric Escudié's avatar
Frédéric Escudié committed
286
        # store where the parameter is coming from
Frédéric Escudié's avatar
Frédéric Escudié committed
287
        new_param.linkTrace_nameid = self.get_nameid()
288
289
290
        # add it to the class itself
        self.params_order.append(name)
        self.__setattr__(name, new_param)
Frédéric Escudié's avatar
Frédéric Escudié committed
291

Frédéric Escudié's avatar
Frédéric Escudié committed
292
    def add_output_file_endswith(self, name, help, pattern, file_format="any", behaviour="include",
293
                               group="default", display_name=None, cmd_format="", argpos=-1):
294
        new_param = OutputFilesEndsWith(name, help, self.output_directory, pattern, include=(behaviour == "include"), 
295
296
                                       file_format=file_format, group=group, display_name=display_name,
                                       cmd_format=cmd_format, argpos=argpos)
Frédéric Escudié's avatar
Frédéric Escudié committed
297
        # store where the parameter is coming from
Frédéric Escudié's avatar
Frédéric Escudié committed
298
        new_param.linkTrace_nameid = self.get_nameid()
299
300
301
        # add it to the class itself
        self.params_order.append(name)
        self.__setattr__(name, new_param)
Frédéric Escudié's avatar
Frédéric Escudié committed
302

Frédéric Escudié's avatar
Frédéric Escudié committed
303
    def add_output_file_pattern(self, name, help, pattern, file_format="any", behaviour="include",
304
                               group="default", display_name=None, cmd_format="", argpos=-1):
305
        new_param = OutputFilesPattern(name, help, self.output_directory, pattern, include=(behaviour == "exclude"), 
306
307
                                       file_format=file_format, group=group, display_name=display_name,
                                       cmd_format=cmd_format, argpos=argpos)
Frédéric Escudié's avatar
Frédéric Escudié committed
308
        # store where the parameter is coming from
Frédéric Escudié's avatar
Frédéric Escudié committed
309
        new_param.linkTrace_nameid = self.get_nameid()
310
311
312
        # add it to the class itself
        self.params_order.append(name)
        self.__setattr__(name, new_param)
313
    
Jerome Mariette's avatar
Jerome Mariette committed
314
    def _longestCommonSubstr(self, data, clean_end=True):
Jerome Mariette's avatar
Jerome Mariette committed
315
316
317
318
319
320
321
322
        substr = ''
        if len(data) > 1 and len(data[0]) > 0:
            for i in range(len(data[0])):
                for j in range(len(data[0])-i+1):
                    if j > len(substr) and all(data[0][i:i+j] in x for x in data):
                        substr = data[0][i:i+j]
        else:
            substr = data[0]
Jerome Mariette's avatar
Jerome Mariette committed
323
324
325
        if clean_end:
            while substr.endswith("_") or substr.endswith("-") or substr.endswith("."):
                substr = substr[:-1]
Jerome Mariette's avatar
Jerome Mariette committed
326
        return substr
Frédéric Escudié's avatar
Frédéric Escudié committed
327

Jerome Mariette's avatar
Jerome Mariette committed
328
329
330
331
    def get_outputs(self, output_list=None, input_list=None):
        """
        If `output_list` is a string template, then it may have the following
        fields:
Frédéric Escudié's avatar
Frédéric Escudié committed
332

Jerome Mariette's avatar
Jerome Mariette committed
333
334
335
336
        - `{fullpath}`, `{FULL}`         -- Full input file path.
        - `{basename}`, `{BASE}`         -- Base input file name.
        - `{fullpath_woext}`, `{FULLWE}` -- Full input file path without extension
        - `{basename_woext}`, `{BASEWE}` -- Base input file name without extension
Frédéric Escudié's avatar
Frédéric Escudié committed
337
        """
Jerome Mariette's avatar
Jerome Mariette committed
338
339
        if output_list is None:
            return []
Frédéric Escudié's avatar
Frédéric Escudié committed
340

Jerome Mariette's avatar
Jerome Mariette committed
341
        if isinstance(output_list, str):
Jerome Mariette's avatar
Jerome Mariette committed
342
            ilist = []
Jerome Mariette's avatar
Jerome Mariette committed
343
            if not input_list or not '{' in str(output_list):
344
345
346
347
                if input_list is not None and len(input_list) == 0:
                    return []
                else:
                    return [output_list]
Jerome Mariette's avatar
Jerome Mariette committed
348
349
350
351
352
353
354
355
356
357
            # if multiple list of inputs is used
            elif isinstance(input_list[0], list):
                for i, val in enumerate(input_list[0]):
                    iter_values = []
                    for j, ingroup in enumerate(input_list):
                        iter_values.append(os.path.basename(input_list[j][i]))
                    ilist.append(self._longestCommonSubstr(iter_values))
            else:
                ilist = parse_string_list(input_list)
                            
Jerome Mariette's avatar
Jerome Mariette committed
358
359
360
361
362
363
364
365
366
367
368
            return [os.path.join(self.output_directory, str(output_list).format(
                        fullpath       = input,
                        FULL           = input,
                        i              = '{0:05X}'.format(i),
                        NUMBER         = '{0:05X}'.format(i),
                        fullpath_woext = os.path.splitext(input)[0],
                        FULL_WOEXT     = os.path.splitext(input)[0],
                        basename       = os.path.basename(input),
                        BASE           = os.path.basename(input),
                        basename_woext = os.path.splitext(os.path.basename(input))[0] if os.path.splitext(os.path.basename(input))[1] != ".gz" else os.path.splitext(os.path.splitext(os.path.basename(input))[0])[0],
                        BASE_WOEXT     = os.path.splitext(os.path.basename(input))[0] if os.path.splitext(os.path.basename(input))[1] != ".gz" else os.path.splitext(os.path.splitext(os.path.basename(input))[0])[0]))
Jerome Mariette's avatar
Jerome Mariette committed
369
                    for i, input in enumerate(ilist)]
Jerome Mariette's avatar
Jerome Mariette committed
370
371
372
    
    def execute(self):
        # first create the output directory
Jerome Mariette's avatar
Jerome Mariette committed
373
        if not os.path.isdir(self.output_directory):
374
            os.makedirs(self.output_directory, 0o751)
Jerome Mariette's avatar
Jerome Mariette committed
375
        # then run the component
Jerome Mariette's avatar
Jerome Mariette committed
376
        self.process()
Jerome Mariette's avatar
Jerome Mariette committed
377
    
Jerome Mariette's avatar
Jerome Mariette committed
378
    def process(self):
Jerome Mariette's avatar
Jerome Mariette committed
379
        """ 
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
        Run the component, can be implemented by subclasses for a 
        more complex process 
        """
        # get all parameters
        parameters = []
        inputs = []
        outputs = []
        for param_name in self.params_order:
            param = self.__getattribute__(param_name)
            if isinstance(param, AbstractParameter) :
                if isinstance(param, AbstractInputFile):
                    inputs.append(param)
                elif isinstance(param, AbstractOutputFile):
                    outputs.append(param)
                else :
                    parameters.append(param)
        
        # sort parameters using argpos
        parameters = sorted(parameters, key=attrgetter('argpos'))
        inputs = sorted(inputs, key=attrgetter('argpos'))
        outputs = sorted(outputs, key=attrgetter('argpos'))
        filteredparams = []
402
        commandline = self.get_exec_path(self.get_command())
403
404
405
406
407
408

        for p in parameters :
            if isinstance(p, BoolParameter) :
                if p:
                    commandline += " %s " % p.cmd_format
            else :
409
410
                if p.default :
                    commandline += " %s %s " % (p.cmd_format, p.default)
411
412
413
414
415
416
417
418
419
420
421
422
423
424
        
        abstraction = self.get_abstraction()
        
        if abstraction == None:
            cpt = 1
            for file in inputs + outputs :
                if isinstance(file, InputFile) or isinstance(file, OutputFile):
                    commandline +=  ' %s $%s ' % (file.cmd_format, cpt)
                    cpt+=1
                # input file list or output file list / pattern / ends with
                else :
                    for e in file :
                        commandline += ' %s $%s ' % (file.cmd_format, cpt)
                        cpt+=1
425
            function = ShellFunction( commandline,  cmd_format='{EXE} {IN} {OUT}', modules=self.modules)
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
            function(inputs=inputs, outputs=outputs) 
        # weaver map abstraction
        elif abstraction == 'map' :
            if not(len(inputs) == len(outputs) == 1) :
                display_error_message("You can only have one type of input and one type of output for the map abstraction")

            for file in inputs :
                commandline += ' %s $1 ' % file.cmd_format
                if isinstance(file, ParameterList) :
                    inputs = file
            
            for file in outputs :
                commandline += ' %s $2 ' % file.cmd_format
                if isinstance(file, ParameterList) :
                    outputs = file 
            
442
            function = ShellFunction( commandline,  cmd_format='{EXE} {IN} {OUT}', modules=self.modules)
443
444
445
446
447
448
449
450
451
452
453
            exe = Map(function, inputs=inputs, outputs=outputs)
        
        # jflow multimap
        elif abstraction == 'multimap' :
            cpt = 1
            for file in inputs + outputs:
                if not(isinstance(file, ParameterList)):
                    display_error_message("Multimap abstraction can be used only with ParameterList")
                commandline += ' %s $%s ' % (file.cmd_format, cpt)
                cpt+=1
            
454
            function  = ShellFunction( commandline,  cmd_format='{EXE} {IN} {OUT}', modules=self.modules)
455
456
457
458
459
460
461
462
463
            exe = MultiMap(function, inputs=inputs, outputs=outputs)
        # anything other than that will be considered errored
        else :
            raise Exception('Unsupported abstraction %s ' % abstraction)
        
    def get_command(self):
        """
            get a path to an executable. Has to be implemented by subclasses 
            if the process has not been implemented  
Jerome Mariette's avatar
Jerome Mariette committed
464
        """
465
        raise NotImplementedError("Either the Component.get_command() function or the Component.process() function has to be implemented!")
Jerome Mariette's avatar
Jerome Mariette committed
466
    
467
468
469
470
471
    def get_abstraction(self):
        """
            get the abstraction. Has to be implemented by subclasses 
            if the process has not been implemented  
        """ 
472
        raise NotImplementedError("Either the Component.get_abstraction() function or the Component.process() function has to be implemented!")
473

474
475
476
477
    def get_version(self):
        """ 
        Return the tool version, has to be implemented by subclasses
        """
Jerome Mariette's avatar
Jerome Mariette committed
478
        return None
479
    
Jerome Mariette's avatar
Jerome Mariette committed
480
481
482
    def get_temporary_file(self, suffix=".txt"):
        # first check if tmp directory exists
        if not os.path.isdir(self.config_reader.get_tmp_directory()):
483
            os.makedirs(self.config_reader.get_tmp_directory(), 0o751)
Jerome Mariette's avatar
Jerome Mariette committed
484
485
486
487
488
489
490
491
        tempfile_name = os.path.basename(tempfile.NamedTemporaryFile(suffix=suffix).name)
        return os.path.join(self.config_reader.get_tmp_directory(), tempfile_name)
    
    def define_parameters(self, *args):
        """ 
        Define the component parameters, has to be implemented by subclasses
        """
        raise NotImplementedError
492
493
494

    def get_resource(self, resource):
        return self.config_reader.get_resource(resource)
Jerome Mariette's avatar
Jerome Mariette committed
495
496
    
    def get_exec_path(self, software):
497
        exec_path = self.config_reader.get_exec(software)
498
499
500
        if exec_path is None and os.path.isfile(os.path.join(os.path.dirname(inspect.getfile(self.__class__)), "../../bin", software)):
            exec_path = os.path.join(os.path.dirname(inspect.getfile(self.__class__)), "../../bin", software)
        elif exec_path is None and os.path.isfile(os.path.join(os.path.dirname(inspect.getfile(self.__class__)), "../bin", software)):
501
502
503
            exec_path = os.path.join(os.path.dirname(inspect.getfile(self.__class__)), "../bin", software)
        elif exec_path is None and os.path.isfile(os.path.join(os.path.dirname(inspect.getfile(self.__class__)), "bin", software)):
            exec_path = os.path.join(os.path.dirname(inspect.getfile(self.__class__)), "bin", software)
504
        elif exec_path is None and which(software) == None:
505
506
            logging.getLogger("jflow").exception("'" + software + "' path connot be retrieved either in the PATH and in the application.properties file!")
            raise Exception("'" + software + "' path connot be retrieved either in the PATH and in the application.properties file!")
507
508
509
        elif exec_path is None and which(software) != None: 
            exec_path = software
        elif exec_path != None and not os.path.isfile(exec_path):
510
511
            logging.getLogger("jflow").exception("'" + exec_path + "' set for '" + software + "' does not exists, please provide a valid path!")
            raise Exception("'" + exec_path + "' set for '" + software + "' does not exists, please provide a valid path!")
512
        return exec_path
Jerome Mariette's avatar
Jerome Mariette committed
513
    
Jerome Mariette's avatar
Jerome Mariette committed
514
    def get_nameid(self):
Floreal Cabanettes's avatar
Floreal Cabanettes committed
515
        return self.__class__.__name__ + "." + self.__prefix
Jerome Mariette's avatar
Jerome Mariette committed
516
    
Jerome Mariette's avatar
Jerome Mariette committed
517
    def __eq__(self, other):
Floreal Cabanettes's avatar
Floreal Cabanettes committed
518
        return self.__class__ == other.__class__ and self.__prefix == other.get_prefix()
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
    
    def __getattribute__(self, attr):
        # an IOobject is a specific object defined by the presence of the dump_path attribute
        if hasattr(object.__getattribute__(self, attr), "default"):
            if isinstance (object.__getattribute__(self, attr).default, OObject) and os.path.exists(object.__getattribute__(self, attr).default.dump_path):
                object.__getattribute__(self, attr).default=object.__getattribute__(self, attr).default.load()
            if hasattr(object.__getattribute__(self, attr).default, "is_ioobject"):
                return object.__getattribute__(self, attr).default
            elif isinstance(object.__getattribute__(self, attr).default, list) and len(object.__getattribute__(self, attr).default)>0:
                if isinstance(object.__getattribute__(self, attr).default[0], OObject):
                    for i, val in enumerate (object.__getattribute__(self, attr).default):
                       if os.path.exists(val.dump_path):
                           object.__getattribute__(self, attr).default[i]=val.load()
                if hasattr(object.__getattribute__(self, attr).default[0], "is_ioobject"):
                    return object.__getattribute__(self, attr).default
            
                
        return object.__getattribute__(self, attr)
    
    def __generate_iolist (self, ioparameter, map):
        new_ios = []
        includes = []
        if map :
            if len (ioparameter) >0 :
                if isinstance(ioparameter[0], list):
                    for cin in ioparameter:
                        if hasattr(cin[0], "is_ioobject"):
                            new_ios.append([i.dump_path for i in cin])
                        else:
                            new_ios.append(cin)
                else:
                    for cin in ioparameter:
                        if hasattr(cin, "is_ioobject"):
                            new_ios.append(cin.dump_path)
                        else:
                            new_ios.append(cin)
        else :
            new_ios = []
            if hasattr(ioparameter, "is_ioobject"):
                includes.extend(ioparameter.includes)
                new_ios.append(ioparameter.dump_path)
            elif isinstance(ioparameter, list):
                for cin in ioparameter:
                    if hasattr(cin, "is_ioobject"):
                        includes.extend(cin.includes)
                        new_ios.append(cin.dump_path)
                    else:
                        new_ios.append(cin)
            else:
                new_ios = ioparameter
        return new_ios,includes
    
    def add_python_execution(self, function, inputs=[], outputs=[], arguments=[], includes=[], 
                             add_path=None, collect=False, local=False, map=False, cmd_format=""):
               
        if map:
            if not issubclass(inputs.__class__, list) or not issubclass(outputs.__class__, list):
                logging.getLogger("jflow").exception("add_python_execution: '" + function.__name__ + "' map requires a list as inputs and output!")
                raise Exception("add_python_execution: '" + function.__name__ + "' map requires a list as inputs and output!")
        #Command format to build
        if cmd_format == "" :
            cmd_format = "{EXE} "
            if len(arguments)>0:
                cmd_format += " {ARG}"
            if (isinstance(inputs, list) and len(inputs)>0) or (inputs is not None and inputs != []):
                cmd_format += " {IN}"
            if (isinstance(outputs, list) and len(outputs)>0) or (outputs is not None and outputs != []):
                cmd_format += " {OUT}"
587
        py_function = PythonFunction(function, add_path=add_path, cmd_format=cmd_format, modules=self.modules)
588
589
590
591
592
593
594

        
        new_inputs,includes_in = self.__generate_iolist(inputs, map)
        new_outputs,includes_out = self.__generate_iolist(outputs, map)
        if not isinstance(includes, list):
            includes=[includes]
        if map:
595
596
            MultiMap(py_function, inputs=new_inputs, outputs=new_outputs, includes=includes+includes_in,
                     collect=collect, local=local, arguments=arguments)
597
598
        else:
            py_function(inputs=new_inputs, outputs=new_outputs, arguments=arguments, includes=includes+includes_in)
599

Jerome Mariette's avatar
Jerome Mariette committed
600
        self.__write_trace(function.__name__, inputs, outputs, arguments, cmd_format, map, "PythonFunction")
Céline Noirot's avatar
Céline Noirot committed
601
        
602
603
    def add_shell_execution(self, source, inputs=[], outputs=[], arguments=[], includes=[], 
                            cmd_format=None, map=False, shell=None, collect=False, local=False):
604
        shell_function  = ShellFunction(source, shell=shell, cmd_format=cmd_format, modules=self.modules)
605
606
607
608
609
       
        # if abstraction is map or multimap
        if map :
            # if input and output are list or filelist
            if issubclass(inputs.__class__, list) and issubclass(outputs.__class__, list) :
610
                MultiMap(shell_function,inputs=inputs, outputs=outputs, includes=includes, collect=collect, local=local, arguments=arguments)
611
612
613
614
615
616
            else :
                logging.getLogger("jflow").exception("add_shell_execution: '" + source + "' map requires a list as inputs and output")
                raise Exception("add_shell_execution: '" + source + "'  map requires a list as inputs and output")
            
        else :
            shell_function( inputs=inputs, outputs=outputs, arguments=arguments, includes=includes )   
Jerome Mariette's avatar
Jerome Mariette committed
617
        self.__write_trace(source, inputs, outputs, arguments, cmd_format, map, "Shell")
Céline Noirot's avatar
Céline Noirot committed
618
        
Jerome Mariette's avatar
Jerome Mariette committed
619
620
    def __write_trace(self, name,  inputs, outputs, arguments, cmd_format, map, type):
        trace_fh=open(os.path.join(self.output_directory, Component.TRACE_FILE_NAME), "a")
Céline Noirot's avatar
Céline Noirot committed
621
622
623
624
625
626
        trace_fh.write("###\n###Execution trace of "+type+": "+name+"\n")
        if map :
            trace_fh.write("MODE MAP\n")
        
        if cmd_format != "" and type == "Shell":
            trace_fh.write("COMMAND: " + str(cmd_format) + "\n")
Jerome Mariette's avatar
Jerome Mariette committed
627
628
629
        self.__write_element(trace_fh,"ARGCUMENTS",arguments)
        self.__write_element(trace_fh,"INPUTS",inputs)
        self.__write_element(trace_fh,"OUTPUTS",outputs)
Céline Noirot's avatar
Céline Noirot committed
630
631
        trace_fh.close()
        
Jerome Mariette's avatar
Jerome Mariette committed
632
    def __write_element(self,fh, title, element):
Céline Noirot's avatar
Céline Noirot committed
633
634
635
636
637
638
        to_write=''
        if isinstance(element, list):
            if len (element)> 0 :
                if isinstance(element[0], list):
                    for i in range(len(element)) : 
                        to_write+="List"+str(i+1)+": \n"
639
                        to_write+="\n".join([str(x) for x in element[i]])+"\n"
Céline Noirot's avatar
Céline Noirot committed
640
                else : 
641
                    to_write+="\n".join([str(x) for x in element])+"\n"
Céline Noirot's avatar
Céline Noirot committed
642
        else :
643
            to_write+= str(element)+"\n"
Céline Noirot's avatar
Céline Noirot committed
644
645
        if to_write != "" :
            fh.write(title+" :\n")
Jerome Mariette's avatar
Jerome Mariette committed
646
            fh.write(to_write)