component.py 15.7 KB
Newer Older
Jerome Mariette's avatar
Jerome Mariette committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
#
# Copyright (C) 2012 INRA
# 
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
# 
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
# 
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
#

import os
19
import sys
Jerome Mariette's avatar
Jerome Mariette committed
20
21
import inspect
import tempfile
Jerome Mariette's avatar
Jerome Mariette committed
22
import types
Jerome Mariette's avatar
Jerome Mariette committed
23
24
25
26

from jflow.workflows_manager import WorkflowsManager
from jflow.config_reader import JFlowConfigReader
from jflow.dataset import ArrayList
Jerome Mariette's avatar
Jerome Mariette committed
27
from jflow.utils import which, display_error_message
Jerome Mariette's avatar
Jerome Mariette committed
28
from jflow.parameter import *
Jerome Mariette's avatar
Jerome Mariette committed
29
30
31
32
from weaver.util import parse_string_list


class Component(object):
Frédéric Escudié's avatar
Frédéric Escudié committed
33

34
    def __init__(self):
Jerome Mariette's avatar
Jerome Mariette committed
35
        self.prefix = "default"
Jerome Mariette's avatar
Jerome Mariette committed
36
        self.params_order = []
Jerome Mariette's avatar
Jerome Mariette committed
37
38
        self.output_directory = None
        self.config_reader = JFlowConfigReader()
39
        self.version = self.get_version()
40
        self.batch_options = self.config_reader.get_component_batch_options(self.__class__.__name__)
Frédéric Escudié's avatar
Frédéric Escudié committed
41
42

    def is_dynamic(self):
43
44
45
46
47
48
49
50
51
52
53
        return len(self.get_dynamic_outputs()) != 0

    def get_dynamic_outputs(self):
        """
         @return : the list of outputs updated at the end of component execution.
        """
        dynamic_outputs = list()
        for attribute_value in self.__dict__.values():
            if issubclass( attribute_value.__class__, DynamicOutput ):
                dynamic_outputs.append( attribute_value )
        return dynamic_outputs
Frédéric Escudié's avatar
Frédéric Escudié committed
54

55
    def get_output_files(self):
Philippe Bardou's avatar
Philippe Bardou committed
56
        outputs = {}
57
58
        for attribute_value in self.__dict__.values():
            if ( issubclass( attribute_value.__class__, DynamicOutput ) or
Philippe Bardou's avatar
Philippe Bardou committed
59
60
                 issubclass( attribute_value.__class__, OutputFileList) ):
                for f in attribute_value:
61
                    outputs[os.path.basename(f)] = f
62
            elif issubclass( attribute_value.__class__, OutputFile):
63
                outputs[os.path.basename(attribute_value)] = attribute_value
64
        return outputs
Frédéric Escudié's avatar
Frédéric Escudié committed
65

66
67
68
69
    def add_input_file(self, name, help, file_format="any", default=None, type="inputfile", 
                       required=False, flag=None, group="default", display_name=None, add_to=None):
        new_param = InputFile(name, help, flag=flag, file_format=file_format, default=default, 
                              type=type, required=required, group=group, display_name=display_name)
70
        # store where the parameter is coming from
Frédéric Escudié's avatar
Frédéric Escudié committed
71
        new_param.linkTrace_nameid = self.get_nameid()
Frédéric Escudié's avatar
Frédéric Escudié committed
72
        if issubclass( default.__class__, LinkTraceback ):
Frédéric Escudié's avatar
Frédéric Escudié committed
73
            new_param.parent_linkTrace_nameid = [default.linkTrace_nameid]
74
75
76
77
78
79
        # if this input should be added to a particular parameter
        if add_to:
            try:
                self.__getattribute__(add_to).add_sub_parameter(new_param)
            except: pass
        # otherwise, add it to the class itself
Jerome Mariette's avatar
Jerome Mariette committed
80
        else:
81
82
            self.params_order.append(name)
            self.__setattr__(name, new_param)
83
84
85
86
87
    
    def reset(self):
        for file in os.listdir(self.output_directory):
            os.remove(os.path.join(self.output_directory, file))
    
88
89
    def add_input_file_list(self, name, help, file_format="any", default=None, type="inputfile", 
                            required=False, flag=None, group="default", display_name=None, add_to=None):
90
91
        if default == None:
            inputs = []
92
        elif issubclass(default.__class__, list):
Frédéric Escudié's avatar
Frédéric Escudié committed
93
            inputs = [IOFile(file, file_format, self.get_nameid(), None) for file in default]
94
95
        else:
            inputs = [IOFile(default, file_format, self.get_nameid(), None)]
96
        new_param = InputFileList(name, help, flag=flag, file_format=file_format, default=inputs, 
Jerome Mariette's avatar
Jerome Mariette committed
97
                                  type=type, required=required, group=group, display_name=display_name)
98
        # store where the parameter is coming from
Frédéric Escudié's avatar
Frédéric Escudié committed
99
        new_param.linkTrace_nameid = self.get_nameid()
100
101
        if issubclass( default.__class__, list ):
            for idx, val in enumerate(default):
Frédéric Escudié's avatar
Frédéric Escudié committed
102
                if issubclass( val.__class__, LinkTraceback ):
Frédéric Escudié's avatar
Frédéric Escudié committed
103
104
105
                    new_param[idx].parent_linkTrace_nameid = [val.linkTrace_nameid]
                if not val.linkTrace_nameid in new_param.parent_linkTrace_nameid:
                    new_param.parent_linkTrace_nameid.append( val.linkTrace_nameid )
Frédéric Escudié's avatar
Frédéric Escudié committed
106
107
        else:
            if issubclass( default.__class__, LinkTraceback ):
Frédéric Escudié's avatar
Frédéric Escudié committed
108
109
110
                new_param[0].parent_linkTrace_nameid = [default.linkTrace_nameid]
                if not default.linkTrace_nameid in new_param.parent_linkTrace_nameid:
                    new_param.parent_linkTrace_nameid.append( default.linkTrace_nameid )
Jerome Mariette's avatar
Jerome Mariette committed
111
112
113
114
115
116
117
118
119
        # if this input should be added to a particular parameter
        if add_to:
            try:
                self.__getattribute__(add_to).add_sub_parameter(new_param)
            except: pass
        # otherwise, add it to the class itself
        else:
            self.params_order.append(name)
            self.__setattr__(name, new_param)
Frédéric Escudié's avatar
Frédéric Escudié committed
120

121
122
    def add_parameter(self, name, help, default=None, type=types.StringType, choices=None, 
                      required=False, flag=None, group="default", display_name=None, add_to=None):
Frédéric Escudié's avatar
Frédéric Escudié committed
123
        new_param = ParameterFactory.factory(name, help, flag=flag, default=default, type=type, choices=choices, 
124
125
126
127
128
129
130
                              required=required, group=group, display_name=display_name)
        # if this input should be added to a particular parameter
        if add_to:
            try:
                self.__getattribute__(add_to).add_sub_parameter(new_param)
            except: pass
        # otherwise, add it to the class itself
Jerome Mariette's avatar
Jerome Mariette committed
131
        else:
132
133
134
135
136
137
138
            self.params_order.append(name)
            self.__setattr__(name, new_param)

    def add_parameter_list(self, name, help, default=None, type=types.StringType, choices=None, 
                           required=False, flag=None, group="default", display_name=None, add_to=None):
        if default == None: default = []
        new_param = ParameterList(name, help, flag=flag, default=default, type=type, choices=choices, 
Jerome Mariette's avatar
Jerome Mariette committed
139
140
141
142
143
144
145
146
147
148
                                  required=required, group=group, display_name=display_name)
        # if this input should be added to a particular parameter
        if add_to:
            try:
                self.__getattribute__(add_to).add_sub_parameter(new_param)
            except: pass
        # otherwise, add it to the class itself
        else:
            self.params_order.append(name)
            self.__setattr__(name, new_param)
Frédéric Escudié's avatar
Frédéric Escudié committed
149

150
151
    def add_output_file(self, name, help, file_format="any", filename=None, group="default", display_name=None, add_to=None):
        filename = os.path.basename(filename)
Frédéric Escudié's avatar
Frédéric Escudié committed
152
        new_param = OutputFile(name, help, default=os.path.join(self.output_directory, filename),
153
                               file_format=file_format, group=group, display_name=display_name)
Frédéric Escudié's avatar
Frédéric Escudié committed
154
        # store where the parameter is coming from
Frédéric Escudié's avatar
Frédéric Escudié committed
155
        new_param.linkTrace_nameid = self.get_nameid()
156
157
158
159
160
161
        # if this input should be added to a particular parameter
        if add_to:
            try:
                self.__getattribute__(add_to).add_sub_parameter(new_param)
            except: pass
        # otherwise, add it to the class itself
Jerome Mariette's avatar
Jerome Mariette committed
162
        else:
163
164
            self.params_order.append(name)
            self.__setattr__(name, new_param)
Frédéric Escudié's avatar
Frédéric Escudié committed
165

166
167
    def add_output_file_list(self, name, help, file_format="any", pattern='{basename_woext}.out', 
                             items=None, group="default", display_name=None, add_to=None):
Frédéric Escudié's avatar
Frédéric Escudié committed
168
        default = [IOFile(file, file_format, self.get_nameid(), None) for file in self.get_outputs(pattern, items)]
169
        new_param = OutputFileList(name, help, default=default, file_format=file_format, group=group, display_name=display_name)
Frédéric Escudié's avatar
Frédéric Escudié committed
170
        # store where the parameter is coming from
Frédéric Escudié's avatar
Frédéric Escudié committed
171
        new_param.linkTrace_nameid = self.get_nameid()
Jerome Mariette's avatar
Jerome Mariette committed
172
173
174
175
176
177
178
179
180
        # if this input should be added to a particular parameter
        if add_to:
            try:
                self.__getattribute__(add_to).add_sub_parameter(new_param)
            except: pass
        # otherwise, add it to the class itself
        else:
            self.params_order.append(name)
            self.__setattr__(name, new_param)
Frédéric Escudié's avatar
Frédéric Escudié committed
181

Frédéric Escudié's avatar
Frédéric Escudié committed
182
    def add_output_file_endswith(self, name, help, pattern, file_format="any", behaviour="include",
Frédéric Escudié's avatar
Frédéric Escudié committed
183
                               group="default", display_name=None, add_to=None):
184
        new_param = OutputFilesEndsWith(name, help, self.output_directory, pattern, include=(behaviour == "include"), 
185
                                       file_format=file_format, group=group, display_name=display_name)
Frédéric Escudié's avatar
Frédéric Escudié committed
186
        # store where the parameter is coming from
Frédéric Escudié's avatar
Frédéric Escudié committed
187
        new_param.linkTrace_nameid = self.get_nameid()
Frédéric Escudié's avatar
Frédéric Escudié committed
188
189
190
191
192
193
194
195
196
197
        # if this input should be added to a particular parameter
        if add_to:
            try:
                self.__getattribute__(add_to).add_sub_parameter(new_param)
            except: pass
        # otherwise, add it to the class itself
        else:
            self.params_order.append(name)
            self.__setattr__(name, new_param)

Frédéric Escudié's avatar
Frédéric Escudié committed
198
    def add_output_file_pattern(self, name, help, pattern, file_format="any", behaviour="include",
Frédéric Escudié's avatar
Frédéric Escudié committed
199
                               group="default", display_name=None, add_to=None):
200
        new_param = OutputFilesPattern(name, help, self.output_directory, pattern, include=(behaviour == "exclude"), 
201
                                       file_format=file_format, group=group, display_name=display_name)
Frédéric Escudié's avatar
Frédéric Escudié committed
202
        # store where the parameter is coming from
Frédéric Escudié's avatar
Frédéric Escudié committed
203
        new_param.linkTrace_nameid = self.get_nameid()
Frédéric Escudié's avatar
Frédéric Escudié committed
204
205
206
207
208
209
210
211
212
213
        # if this input should be added to a particular parameter
        if add_to:
            try:
                self.__getattribute__(add_to).add_sub_parameter(new_param)
            except: pass
        # otherwise, add it to the class itself
        else:
            self.params_order.append(name)
            self.__setattr__(name, new_param)

Jerome Mariette's avatar
Jerome Mariette committed
214
    def _longestCommonSubstr(self, data, clean_end=True):
Jerome Mariette's avatar
Jerome Mariette committed
215
216
217
218
219
220
221
222
        substr = ''
        if len(data) > 1 and len(data[0]) > 0:
            for i in range(len(data[0])):
                for j in range(len(data[0])-i+1):
                    if j > len(substr) and all(data[0][i:i+j] in x for x in data):
                        substr = data[0][i:i+j]
        else:
            substr = data[0]
Jerome Mariette's avatar
Jerome Mariette committed
223
224
225
        if clean_end:
            while substr.endswith("_") or substr.endswith("-") or substr.endswith("."):
                substr = substr[:-1]
Jerome Mariette's avatar
Jerome Mariette committed
226
        return substr
Frédéric Escudié's avatar
Frédéric Escudié committed
227

Jerome Mariette's avatar
Jerome Mariette committed
228
229
230
231
    def get_outputs(self, output_list=None, input_list=None):
        """
        If `output_list` is a string template, then it may have the following
        fields:
Frédéric Escudié's avatar
Frédéric Escudié committed
232

Jerome Mariette's avatar
Jerome Mariette committed
233
234
235
236
        - `{fullpath}`, `{FULL}`         -- Full input file path.
        - `{basename}`, `{BASE}`         -- Base input file name.
        - `{fullpath_woext}`, `{FULLWE}` -- Full input file path without extension
        - `{basename_woext}`, `{BASEWE}` -- Base input file name without extension
Frédéric Escudié's avatar
Frédéric Escudié committed
237
        """
Jerome Mariette's avatar
Jerome Mariette committed
238
239
        if output_list is None:
            return []
Frédéric Escudié's avatar
Frédéric Escudié committed
240

Jerome Mariette's avatar
Jerome Mariette committed
241
        if isinstance(output_list, str):
Jerome Mariette's avatar
Jerome Mariette committed
242
            ilist = []
Jerome Mariette's avatar
Jerome Mariette committed
243
            if not input_list or not '{' in str(output_list):
244
245
246
247
                if input_list is not None and len(input_list) == 0:
                    return []
                else:
                    return [output_list]
Jerome Mariette's avatar
Jerome Mariette committed
248
249
250
251
252
253
254
255
256
257
            # if multiple list of inputs is used
            elif isinstance(input_list[0], list):
                for i, val in enumerate(input_list[0]):
                    iter_values = []
                    for j, ingroup in enumerate(input_list):
                        iter_values.append(os.path.basename(input_list[j][i]))
                    ilist.append(self._longestCommonSubstr(iter_values))
            else:
                ilist = parse_string_list(input_list)
                            
Jerome Mariette's avatar
Jerome Mariette committed
258
259
260
261
262
263
264
265
266
267
268
            return [os.path.join(self.output_directory, str(output_list).format(
                        fullpath       = input,
                        FULL           = input,
                        i              = '{0:05X}'.format(i),
                        NUMBER         = '{0:05X}'.format(i),
                        fullpath_woext = os.path.splitext(input)[0],
                        FULL_WOEXT     = os.path.splitext(input)[0],
                        basename       = os.path.basename(input),
                        BASE           = os.path.basename(input),
                        basename_woext = os.path.splitext(os.path.basename(input))[0] if os.path.splitext(os.path.basename(input))[1] != ".gz" else os.path.splitext(os.path.splitext(os.path.basename(input))[0])[0],
                        BASE_WOEXT     = os.path.splitext(os.path.basename(input))[0] if os.path.splitext(os.path.basename(input))[1] != ".gz" else os.path.splitext(os.path.splitext(os.path.basename(input))[0])[0]))
Jerome Mariette's avatar
Jerome Mariette committed
269
                    for i, input in enumerate(ilist)]
Jerome Mariette's avatar
Jerome Mariette committed
270
271
272
    
    def execute(self):
        # first create the output directory
Jerome Mariette's avatar
Jerome Mariette committed
273
274
        if not os.path.isdir(self.output_directory):
            os.makedirs(self.output_directory, 0751)
Jerome Mariette's avatar
Jerome Mariette committed
275
        # then run the component
Jerome Mariette's avatar
Jerome Mariette committed
276
        self.process()
Jerome Mariette's avatar
Jerome Mariette committed
277
    
Jerome Mariette's avatar
Jerome Mariette committed
278
    def process(self):
Jerome Mariette's avatar
Jerome Mariette committed
279
280
281
282
283
        """ 
        Run the component, has to be implemented by subclasses
        """
        raise NotImplementedError
    
284
285
286
287
    def get_version(self):
        """ 
        Return the tool version, has to be implemented by subclasses
        """
Jerome Mariette's avatar
Jerome Mariette committed
288
        return None
289
    
Jerome Mariette's avatar
Jerome Mariette committed
290
291
292
293
294
295
296
297
298
299
300
301
    def get_temporary_file(self, suffix=".txt"):
        # first check if tmp directory exists
        if not os.path.isdir(self.config_reader.get_tmp_directory()):
            os.makedirs(self.config_reader.get_tmp_directory(), 0751)
        tempfile_name = os.path.basename(tempfile.NamedTemporaryFile(suffix=suffix).name)
        return os.path.join(self.config_reader.get_tmp_directory(), tempfile_name)
    
    def define_parameters(self, *args):
        """ 
        Define the component parameters, has to be implemented by subclasses
        """
        raise NotImplementedError
302
303
304

    def get_resource(self, resource):
        return self.config_reader.get_resource(resource)
Jerome Mariette's avatar
Jerome Mariette committed
305
306
    
    def get_exec_path(self, software):
307
        exec_path = self.config_reader.get_exec(software)
308
309
310
        if exec_path is None and os.path.isfile(os.path.join(os.path.dirname(inspect.getfile(self.__class__)), "../../bin", software)):
            exec_path = os.path.join(os.path.dirname(inspect.getfile(self.__class__)), "../../bin", software)
        elif exec_path is None and os.path.isfile(os.path.join(os.path.dirname(inspect.getfile(self.__class__)), "../bin", software)):
311
312
313
            exec_path = os.path.join(os.path.dirname(inspect.getfile(self.__class__)), "../bin", software)
        elif exec_path is None and os.path.isfile(os.path.join(os.path.dirname(inspect.getfile(self.__class__)), "bin", software)):
            exec_path = os.path.join(os.path.dirname(inspect.getfile(self.__class__)), "bin", software)
314
        elif exec_path is None and which(software) == None:
Jerome Mariette's avatar
Jerome Mariette committed
315
            display_error_message("'" + software + "' path connot be retrieved either in the PATH and in the application.properties file!")
316
317
318
        elif exec_path is None and which(software) != None: 
            exec_path = software
        elif exec_path != None and not os.path.isfile(exec_path):
Jerome Mariette's avatar
Jerome Mariette committed
319
            display_error_message("'" + exec_path + "' set for '" + software + "' does not exists, please provide a valid path!")
320
        return exec_path
Jerome Mariette's avatar
Jerome Mariette committed
321
    
Jerome Mariette's avatar
Jerome Mariette committed
322
323
324
    def get_nameid(self):
        return self.__class__.__name__ + "." + self.prefix
    
Jerome Mariette's avatar
Jerome Mariette committed
325
326
    def __eq__(self, other):
        return self.__class__ == other.__class__ and self.prefix == other.prefix