mobyle.py 16 KB
Newer Older
1
#
Jerome Mariette's avatar
Jerome Mariette committed
2
# Copyright (C) 2015 INRA
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
# 
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
# 
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
# 
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
#

import os
import re
import copy
import fnmatch
import xml.etree.ElementTree as ET

from jflow.extparser import ExternalParser
Ibouniyamine Nabihoudine's avatar
Ibouniyamine Nabihoudine committed
25
from jflow.parameter import OutputFile
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
import argparse

class MobyleParser(ExternalParser):
    
    def parse(self, component_file):
        parameters = []
        parameters_names = []
        tree = ET.parse(component_file)
        root = tree.getroot()
        
        # get command string
        command = root.findtext(".//head/command", None)
        
        # retrieve all parameters from xml file 
        for parameterNode in root.findall('.//parameters/parameter[name]'):
            attrs = parameterNode.attrib
            param = self._parseParameter(parameterNode)
            if param['name'] in parameters_names :
                raise Exception('Duplicated parameter (%s)'%param['name'])
            parameters.append(param)
            parameters_names.append(param['name'])

        def fn_get_command(self):
            if command is None:
                return self._command
            return command 
    
        def fn_get_abstraction(self):
            return
    
        def fn_define_parameters(self, **kwargs):
57
            for userarg in list(kwargs.keys()):
58
59
60
                if userarg not in parameters_names :
                    raise Exception("Invalid argument '%s' for %s"%(userarg, self.get_nameid()))
            
Ibouniyamine Nabihoudine's avatar
Ibouniyamine Nabihoudine committed
61
            filenames_parameters = []
62
63
64
65
            for param in parameters:
                pname = param['name']
                param['value'] = param.get('vdef')
                
66
                if pname in kwargs:
67
68
69
70
71
72
                    param['value'] = kwargs[pname]
                
                printval = param['value']
                if param['type'] == 'file' or type(param['value']) == str :
                     printval = "'" + str(param['value']) + "'"
                
73
                exec ("%s = %s"%(pname, printval), globals(), locals())
Ibouniyamine Nabihoudine's avatar
Ibouniyamine Nabihoudine committed
74
75
76
                
                if param['isfilename'] :
                    filenames_parameters.append(param)
77
78
79
80
81
82
83
84
85
86
87
88
89
            
            # resolve format
            for param in parameters:
                pname = param['name']
                value = param.get('value')
                vdef = param.get('vdef')
                
                if param['format'] is not None :
                    param['format'] = eval( param['format'] )
                    param['format'] = re.sub(r""+str(value)+"", "", param['format'])
                    # TODO : do I need to remove the '=' ?????
                    param['format'] = re.sub(r"=$", "", param['format'])
                    
Ibouniyamine Nabihoudine's avatar
Ibouniyamine Nabihoudine committed
90
                    # empty format here means no value
91
                    if not param['format'].strip() :
Ibouniyamine Nabihoudine's avatar
Ibouniyamine Nabihoudine committed
92
                        param['format'] = None
93
94
95
                        param['value'] = None
                
                elif param['flist'] :
96
                    vlist = list(param['flist'].keys())
97
98
                    # custom string type depending on vlist for keys and convert to flist
                    
99
                    class flist_type(jflow.extparser._serializable_nested_function):
100
101
102
                        def __call__(self, vvv):
                            if vvv in vlist :
                                return  eval(param['flist'][vvv])
Ibouniyamine Nabihoudine's avatar
Ibouniyamine Nabihoudine committed
103
                            raise argparse.ArgumentTypeError("%s is an invalid argument, valid ones are (%s)"%(vvv, ', '.join( map(str,vlist) )) )
104
105
106
107
108
109
110
111
112
113
114
115
116
                    
                    param['type'] = flist_type
                    param['vlist'] = vlist
                    if param['value'] :
                        param['value'] = flist_type(param['value'])
                
            
            # add parameters
            for param in parameters:
                pname = param['name']
                if param['iscommand'] :
                    self._command = param['value'] or param['format']
                    continue
Ibouniyamine Nabihoudine's avatar
Ibouniyamine Nabihoudine committed
117
                
118
119
120
121
122
123
124
125
126
127
128
                arguments = {}
                arguments['required'] = param.get('required', False)
                if param['value'] :
                    arguments['default'] = param['value']
                if param['vlist'] :
                    arguments['choices'] = param['vlist'] 
                if param['format'] :
                    arguments['cmd_format'] = param['format']
                if param['argpos'] :
                    arguments['argpos'] = param['argpos']
                
Ibouniyamine Nabihoudine's avatar
Ibouniyamine Nabihoudine committed
129
130
131
                if param['preconditions'] :
                    all_precond_true = True
                    for precond in param['preconditions'] :
132
                        try :
Ibouniyamine Nabihoudine's avatar
Ibouniyamine Nabihoudine committed
133
                            eval(precond)
134
                        except :
Ibouniyamine Nabihoudine's avatar
Ibouniyamine Nabihoudine committed
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
                            all_precond_true = False
                            break
                    if not  all_precond_true :
                        arguments.pop('default', None)
                        if param['type'] == 'file' :
                            if not param['isoutput'] :
                                arguments.pop('choices', None)
                                self.add_input_file(pname, param['help'], **arguments)
                            # do not add output files
                        else :
                            self.add_parameter(pname, param['help'], **arguments)
                        continue # --> next parameter
                
                if param['isfilename'] :
                    arguments.pop('default', None)
                    arguments.pop('required', None)
                    arguments.pop('choices', None)    
                    arguments['filename'] = param['value']
                    self.add_output_file( pname, param['help'], **arguments )
                
                elif param['type'] == 'file' :
                    if not param['isoutput'] :
                        arguments.pop('choices', None)
                        self.add_input_file(pname, param['help'], **arguments)
                    # output file
                    else :
                        arguments.pop('choices', None)
                        arguments.pop('default', None)
                        arguments.pop('required', None) 
164
                        
Ibouniyamine Nabihoudine's avatar
Ibouniyamine Nabihoudine committed
165
166
167
168
169
170
171
172
                        nfilenames = []
                        for expression in param['filenames'] :
                            try :
                                expression = eval(expression)
                            except : pass
                            nfilenames.append(expression)
                        param['filenames'] =  nfilenames          

173
                        if len(param['filenames']) == 1 :
Ibouniyamine Nabihoudine's avatar
Ibouniyamine Nabihoudine committed
174
175
176
177
178
179
180
181
182
183
184
185
                            filename = self.get_outputs("{basename}", param['filenames'])[0]
                            in_parameters = False
                            for param_name in self.params_order :
                                obj = self.__getattribute__(param_name)
                                if isinstance(obj, OutputFile) :
                                    if obj.default == filename :
                                        in_parameters = True
                                        break
                            if in_parameters :
                                continue # next parameter, no need to add this one 
                        # regexp to match all filenames
                        regexp = "|".join([ fnmatch.translate(e) for e in param['filenames']])
186
                        print((">>regex for %s    %s"%(pname,regexp)))
Ibouniyamine Nabihoudine's avatar
Ibouniyamine Nabihoudine committed
187
                        self.add_output_file_pattern( pname, param['help'], regexp, **arguments )
188
189
                        
                else :
Ibouniyamine Nabihoudine's avatar
Ibouniyamine Nabihoudine committed
190
191
192
                    arguments['type'] = param['type']
                    self.add_parameter(pname, param['help'], **arguments)
                
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
            # controls
            for param in parameters :
                pname = param['name']
                if param.get("controls") :
                    value = param.get('value')
                    vdef = param.get('vdef')
                    if value :
                        for ctrl in  param.get("controls") :
                            if not eval(ctrl[0]) :
                                raise Exception('The parameter %s does not respect the following condition : %s'%(pname, ctrl[1]))
                
                
        component_name = root.find(".//head/name").text.replace('-', ' ').replace('_', ' ')
        component_name = "".join(component_name.title().split())
        
208
        return self.build_component(component_name, fn_define_parameters, get_command = fn_get_command, get_abstraction = fn_get_abstraction)
209
210
211
212
213
214
215

    def _parseParameter(self, parameterNode):    
        attrs = parameterNode.attrib
        PYTHON_TYPES = {
            'String'    : { 'func' : str, 'str' : 'str' }, 
            'Integer'   : { 'func' : int, 'str' : 'int' },
            'Float'     : { 'func' : float, 'str' : 'float'}, 
Ibouniyamine Nabihoudine's avatar
Ibouniyamine Nabihoudine committed
216
            'Boolean'   : { 'func' : lambda x : bool(int(x)), 'str': 'bool'}
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
        }               
        
        SIMPLE_TYPES = [ 'Choice', 'MultipleChoice']
    
        param = {
          'name'            : parameterNode.findtext('./name'),
          'help'            : parameterNode.findtext('./prompt'),
          'iscommand'       : False,
          'format'          : None,
          'type'            : None,
          'isfilename'      : False,
          'required'        : False,
          'multiple'        : False,
          'isoutput'        : False,
          'isstdout'        : False,
          'vdef'            : None,
          'argpos'          : -1,
          'vlist'           : None, # list of choices
          'flist'           : None, # list of format
          'controls'        : None,
          'preconditions'   : None,
          'filenames'        : None
        }

        if  'iscommand' in attrs and attrs['iscommand'] in ["1","true"] :
            param['iscommand'] = True
        if  ('ismandatory' in attrs and  attrs['ismandatory'] in ["1","true"]) or \
            ('ismaininput' in attrs and  attrs['ismaininput'] in ["1","true"]) :
            param['required'] = True
        if 'isout' in attrs and  attrs[ 'isout' ] in ["1","true"]:
            param['isoutput'] = True
        if 'isstdout' in attrs and  attrs[ 'isstdout' ] in ["1","true"]:
            param['isstdout'] = True 
            param['isoutput'] = True
        
        casting = None
Ibouniyamine Nabihoudine's avatar
Ibouniyamine Nabihoudine committed
253
254
255
256
257
258
        klass = parameterNode.findtext("./type/datatype/class", '')
        klass = parameterNode.findtext("./type/datatype/superclass", '').strip() or klass.strip()
        
        if not klass :
            raise Exception('Mobyle parameter parsing error, Missing <type> in param %s '%param['name'])
        
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
        if klass in PYTHON_TYPES :
            casting = PYTHON_TYPES[klass]['func']
            param['type'] =  PYTHON_TYPES[klass]['str']
        elif klass == "Filename" :
            casting = PYTHON_TYPES['String']['func']
            param['type'] =  PYTHON_TYPES['String']['str']
            param['isfilename']  = True
        elif klass not in SIMPLE_TYPES :
            param['type'] = 'file'
        # TODO : missing the case of choice in SIMPLE_TYPES ....
        
        # format represent the flag of the command
        formatCode = parameterNode.findtext('./format/code[@proglang="python"]', '').strip() 
        if formatCode :
            param['format'] = formatCode 
        
        try :
            param['argpos'] = int( parameterNode.findtext( './argpos' ) )
Ibouniyamine Nabihoudine's avatar
Ibouniyamine Nabihoudine committed
277
        except : pass
278
279
280
281
282
283
284
285
286
            
        # default value    
        vdefs = [] 
        for vdefNode in parameterNode.findall( './vdef/value' ):
            vdefs.append( casting(vdefNode.text) if casting else vdefNode.text)
        if vdefs:
            if len(vdefs) == 1 :
                param['vdef'] =  casting(vdefs[0]) if casting else vdefs[0]
            else :
287
                param['vdef'] = list(map(casting, vdefs)) if casting else vdefs
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
        
        # vlist : choices
        if parameterNode.find( './vlist' ) is not None :
            vlist = []
            for velem in parameterNode.findall( './vlist/velem' ) :
                label = velem.findtext('./value' , '' ).strip()
                val = velem.findtext('./value' , '' ).strip()
                if velem.attrib.get('undef', '') in ['1', "true"] :
                    if param['vdef'] == label :
                        param['vdef'] = None
                else :
                    vlist.append( casting(val) if casting else val)
            if vlist :
                param['vlist'] = vlist

        # flist : list of format
        if parameterNode.find( './flist' ) is not None :
            flist = {}
            for felem in parameterNode.findall( './flist/felem' ) :
                label = felem.findtext('./value' , '' ).strip()
                format = felem.findtext('./code[@proglang="python"]' , '' ).strip()
                if felem.attrib.get('undef', '') in ['1', "true"] :
                    if param['vdef'] == label :
                        param['vdef'] = None 
                else :
                    flist[label] = format
            if flist:
                param['flist'] = flist
        
        # controls
        controls = []
        for ctrl in parameterNode.findall( './ctrl' ):
            message = ' '.join([ n.text for n in  ctrl.findall( './message/text' )])
            for codeNode in ctrl.findall( './code[@proglang="python"]'):
                code = codeNode.text
                controls.append( (code,message) )
        if controls :
            param['controls'] = controls
        
        # preconditions
        preconditions = []
        for precondNode in parameterNode.findall( './precond/code[@proglang="python"]' ) :
            precond = precondNode.text.strip() 
            if precond != '':
                preconditions.append(precond)
        if preconditions:
            param['preconditions'] = preconditions
        
        # paramfile, I still don't know what this is used for ....
        paramfile = parameterNode.find( './paramfile' )
        if paramfile is not None:
            param['paramfile'] = paramfile.text 
    
        # output filenames mask
        filenamesCode = parameterNode.findall( './filenames/code[@proglang="python"]')
        fnames = []
        for codeNode in filenamesCode :
            fnames.append(codeNode.text.strip())
        if fnames :
            param['filenames'] = fnames

       
        # another property I didn't get ... I think it is used to test the range of possible
        # values for a defined value ... (a custom type ?)
        scaleNode = parameterNode.find( './scale')
        if scaleNode is not None:
            minNode = scaleNode.find('./min' )
            maxNode = scaleNode.find('./max' )
            if minNode is not None or maxNode is not None:
                try:
                    inc = scaleNode.find('./inc' ).text
                except:
                    inc = None
                try:
                    max = maxNode.find( './value' ).text
                    min = minNode.find( './value' ).text
                    param['scale'] = ( min , max , inc )
                except:
                    try:
                        minCodes = {}
                        maxCodes = {}
                        max = maxNode.find( './code[@proglang="python"]' ).text
                        min = minNode.find( './code[@proglang="python"]' ).text
                        param['scale'] = ( min , max , inc )
                    except : pass
        
        return param