parameter.py 39.2 KB
Newer Older
Jerome Mariette's avatar
Jerome Mariette committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
#
# Copyright (C) 2012 INRA
# 
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
# 
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
# 
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
#

Frédéric Escudié's avatar
Frédéric Escudié committed
18
import re
19
import sys
Jerome Mariette's avatar
Jerome Mariette committed
20
import types
Jerome Mariette's avatar
Jerome Mariette committed
21
import datetime
Jerome Mariette's avatar
Jerome Mariette committed
22
import logging
Jerome Mariette's avatar
Jerome Mariette committed
23
import argparse
Jerome Mariette's avatar
Jerome Mariette committed
24
25
import os
import tempfile
26
from argparse import _ensure_value
Jerome Mariette's avatar
Jerome Mariette committed
27
import urllib2
28
import copy as _copy
Jerome Mariette's avatar
Jerome Mariette committed
29
from urlparse import urlparse
30

Jerome Mariette's avatar
Jerome Mariette committed
31
from jflow.config_reader import JFlowConfigReader
32
from jflow.utils import get_octet_string_representation, get_nb_octet
Jerome Mariette's avatar
Jerome Mariette committed
33

Jerome Mariette's avatar
Jerome Mariette committed
34
# import custom types and custom formats
Jerome Mariette's avatar
Jerome Mariette committed
35
from workflows.types import *
Jerome Mariette's avatar
Jerome Mariette committed
36
from workflows.formats import *
Jerome Mariette's avatar
Jerome Mariette committed
37

Jerome Mariette's avatar
Jerome Mariette committed
38
39
40
41
42
43
44
45
46
47
48
49
50

# define all input type available
INPUTFILE_TYPES = ["inputfile", "localfile", "urlfile", "browsefile"]

def browsefile(file):
    # browsefile are not available from command line, considere it as a localfile
    # from the gui, this will not been tested this way
    return localfile(file)

def localfile(file):
    if os.path.isfile(file):
        return os.path.abspath(file)
    else:
Jerome Mariette's avatar
Jerome Mariette committed
51
        raise argparse.ArgumentTypeError("File '" + file + "' does not exists! Please provide a valid file path!")
52

Jerome Mariette's avatar
Jerome Mariette committed
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
def urlfile(file):
    uri_object = urlparse(file)
    try:
        opener = urllib2.urlopen(file)
    except:
        raise argparse.ArgumentTypeError("URL '" + file + "' is invalid!")
    file_name = os.path.basename(uri_object.path)
    if file_name is not None and file_name != "":
        metadata = opener.info()
        file_size = int(metadata.getheaders("Content-Length")[0])
        if file_size == 0:
            raise argparse.ArgumentTypeError("The URL file '" + file + "' is empty!")
        return file
    else:
        raise argparse.ArgumentTypeError("URL '" + file + "' does not contain any file name!")
68

Jerome Mariette's avatar
Jerome Mariette committed
69
70
71
72
73
74
75
76
def inputfile(file):
    # test the format
    uri_object = urlparse(file)
    # check the file
    if uri_object.scheme == '':
        return localfile(file)
    else:
        return urlfile(file)
77

78
79
def create_test_function(itype):
    try: itype = itype.encode('ascii','ignore')
Jerome Mariette's avatar
Jerome Mariette committed
80
    except: pass
81
    try:
82
        ctype, csizel = itype.split(AbstractInputFile.SIZE_LIMIT_SPLITER)
83
        def inner_function(ifile):
Jerome Mariette's avatar
Jerome Mariette committed
84
85
            # first eval the asked type
            returned_value = eval(ctype)(ifile)
86
87
88
89
90
91
            # if 0 unlimited size
            if csizel != "0":
                # first test the size of the file
                uri_object = urlparse(ifile)
                if uri_object.scheme == '':
                    isize = os.path.getsize(ifile)
92
93
                    if isize > int(get_nb_octet(csizel)):
                        raise argparse.ArgumentTypeError("File '" + ifile + "' (size=" + get_octet_string_representation(isize) + ") exceeds size limits: " + csizel + ".")
94
95
96
97
98
99
100
101
102
                else:
                    try:
                        opener = urllib2.urlopen(file)
                        metadata = opener.info()
                        isize = int(metadata.getheaders("Content-Length")[0])
                        if isize > int(get_nb_octet(csizel)):
                            raise argparse.ArgumentTypeError("File '" + ifile + "' (size=" + get_octet_string_representation(isize) + ") exceeds size limits: " + csizel + ".")
                    except:
                        raise argparse.ArgumentTypeError("URL '" + file + "' is invalid!")
103
            # then test the type
Jerome Mariette's avatar
Jerome Mariette committed
104
            return returned_value
105
106
107
        inner_function.__name__ = ctype+AbstractInputFile.SIZE_LIMIT_SPLITER+csizel
        return inner_function
    except:
108
109
110
111
        if type(itype) == str:
            return eval(itype)
        else:
            return itype
112

Jerome Mariette's avatar
Jerome Mariette committed
113
class MultipleParameters(object):
114
    def __init__(self, types, required, choices, excludes, default, actions):
Jerome Mariette's avatar
Jerome Mariette committed
115
        self.types = types
116
        self.choices = choices
Jerome Mariette's avatar
Jerome Mariette committed
117
        self.excludes = excludes
118
        self.default = default
119
        self.actions = actions
Jerome Mariette's avatar
Jerome Mariette committed
120
        self.index = None
121
        self.required = required
Jerome Mariette's avatar
Jerome Mariette committed
122
        self.__name__ = "MultipleParameters"
123

Jerome Mariette's avatar
help ok    
Jerome Mariette committed
124
125
126
127
128
    def get_help(self):
        help = " ("
        for flag in self.types.keys():
            help += flag + "=<" + self.types[flag].__name__.upper() + ">, "
        return help[:-2] + ")"
129

130
    def __call__(self, arg):
Jerome Mariette's avatar
Jerome Mariette committed
131
132
133
        parts = arg.split("=")
        if not self.types.has_key(parts[0]):
            raise argparse.ArgumentTypeError(parts[0] + " is an invalid flag! Available ones are: "+", ".join(self.types.keys()))
134
135
136
        if self.choices[parts[0]] != None:
            if parts[1] not in self.choices[parts[0]]:
                raise argparse.ArgumentTypeError("argument " + parts[0] + ": invalid choice: '" + parts[1] + "' (choose from " + ", ".join(self.choices[parts[0]]) + "))")
137

138
139
140
141
142
143
144
        if self.types[parts[0]] == types.BooleanType:
            return (parts[0], not self.default[parts[0]], self.required, self.excludes)
        else:
            try:
                value = self.types[parts[0]](parts[1])
            except:
                raise argparse.ArgumentTypeError("invalid " + self.types[parts[0]].__name__ + " value: '" + parts[1] + "' for sub parameter '" + parts[0] + "'")
145

146
            self.index = parts[0]
147
            return (parts[0], value, self.required, self.excludes, self.actions)
Jerome Mariette's avatar
Jerome Mariette committed
148

149
150
class MiltipleAction(argparse.Action):
    def __call__(self, parser, namespace, values, option_string=None):
151
152
153
154
155
156
157
        # what is commun within required and excludes
        exclusif_required = {}
        for exclude_group in values[0][3]:
            exclusif_required[exclude_group] = False
            for param in values[0][3][exclude_group]:
                if param in values[0][2]:
                    exclusif_required[exclude_group] = True
Jerome Mariette's avatar
Jerome Mariette committed
158
159
        given_params = []
        # first check for required parameters
160
161
        try:
            required = _copy.copy(values[0][2])
162
163
164
165
166
167
            # delete required that are exclusive
            for group in exclusif_required:
                if exclusif_required[group]:
                    for val in values[0][3][group]:
                        if val in required:
                            del required[required.index(val)]
168
            for val in values:
Jerome Mariette's avatar
Jerome Mariette committed
169
                given_params.append(val[0])
170
171
172
173
174
                if val[0] in required:
                    del required[required.index(val[0])]
        except: pass
        if len(required) == 1: parser.error(", ".join(required) + " is a required parameter!")
        elif len(required) > 1: parser.error(", ".join(required) + " are required parameters!")
Jerome Mariette's avatar
Jerome Mariette committed
175
176
177
178
179
180
181
182
        # then for exclude ones    
        for exclude_group in values[0][3]:
            found = None
            for param in values[0][3][exclude_group]:
                if param in given_params and found != None:
                    parser.error("argument '" + found + "': not allowed with argument '" + param + "'")
                    break
                elif param in given_params: found = param
183

184
185
186
187
188
189
190
191
        # check for required exclusive if one of them is in
        if len(exclusif_required.keys()) > 0:
            for group in exclusif_required:
                if exclusif_required[group]:
                    rfound = False
                    for param in values[0][3][group]:
                        if param in given_params: rfound = True
            if not rfound: parser.error("one of the arguments: " + ", ".join(values[0][3][group]) + " is required")
192

Jerome Mariette's avatar
Jerome Mariette committed
193
        # if ok add the value
194
195
196
197
198
199
200
201
202
203
204
        final_hash, final_values = {}, []
        for value in values:
            if values[0][4][value[0]] == "append" and final_hash.has_key(value[0]):
                final_hash[value[0]].append(value[1])
            elif values[0][4][value[0]] == "append":
                final_hash[value[0]]= [value[1]]
            else:
                final_hash[value[0]]= value[1]
        for param in final_hash:
            final_values.append((param, final_hash[param]))
        setattr(namespace, self.dest, final_values)
205

206

207
208
class MiltipleAppendAction(argparse.Action):
    def __call__(self, parser, namespace, values, option_string=None):
209
210
211
212
213
214
215
        # what is commun within required and excludes
        exclusif_required = {}
        for exclude_group in values[0][3]:
            exclusif_required[exclude_group] = False
            for param in values[0][3][exclude_group]:
                if param in values[0][2]:
                    exclusif_required[exclude_group] = True
Jerome Mariette's avatar
Jerome Mariette committed
216
217
        given_params = []
        # first check for required parameters
218
219
        try:
            required = _copy.copy(values[0][2])
220
221
222
223
224
225
            # delete required that are exclusive
            for group in exclusif_required:
                if exclusif_required[group]:
                    for val in values[0][3][group]:
                        if val in required:
                            del required[required.index(val)]
226
            for val in values:
Jerome Mariette's avatar
Jerome Mariette committed
227
                given_params.append(val[0])
228
229
230
231
232
                if val[0] in required:
                    del required[required.index(val[0])]
        except: pass
        if len(required) == 1: parser.error(", ".join(required) + " is a required parameter!")
        elif len(required) > 1: parser.error(", ".join(required) + " are required parameters!")
Jerome Mariette's avatar
Jerome Mariette committed
233
234
235
236
237
238
239
240
        # then for exclude ones    
        for exclude_group in values[0][3]:
            found = None
            for param in values[0][3][exclude_group]:
                if param in given_params and found != None:
                    parser.error("argument '" + found + "': not allowed with argument '" + param + "'")
                    break
                elif param in given_params: found = param
241

242
243
244
245
246
247
248
249
        # check for required exclusive if one of them is in
        if len(exclusif_required.keys()) > 0:
            for group in exclusif_required:
                if exclusif_required[group]:
                    rfound = False
                    for param in values[0][3][group]:
                        if param in given_params: rfound = True
            if not rfound: parser.error("one of the arguments: " + ", ".join(values[0][3][group]) + " is required")
Jerome Mariette's avatar
Jerome Mariette committed
250
        # if ok add the value
251
        items = _copy.copy(_ensure_value(namespace, self.dest, []))
252
253
254
255
256
257
258
259
260
261
262
        final_hash, final_values = {}, []
        for value in values:
            if values[0][4][value[0]] == "append" and final_hash.has_key(value[0]):
                final_hash[value[0]].append(value[1])
            elif values[0][4][value[0]] == "append":
                final_hash[value[0]]= [value[1]]
            else:
                final_hash[value[0]]= value[1]
        for param in final_hash:
            final_values.append((param, final_hash[param]))
        items.append(final_values)
263
        setattr(namespace, self.dest, items)
264
265
266


class AbstractParameter(object):
267

268
269
    def __init__(self, name, help, default=None, type=types.StringType, choices=None, required=False,
                 flag=None, action="store", sub_parameters=None, group="default", display_name=None):
Jerome Mariette's avatar
Jerome Mariette committed
270

Jerome Mariette's avatar
Jerome Mariette committed
271
272
        self.name = name
        self.help = help
273
        self.action = action
Jerome Mariette's avatar
Jerome Mariette committed
274
        self.nargs = None
275
276
277
        if sub_parameters:
            self.sub_parameters = sub_parameters
        else: self.sub_parameters = []
Jerome Mariette's avatar
Jerome Mariette committed
278
        self.group = group
279
280
281
        if flag == None:
            self.flag = "--"+name.replace("_", "-")
        else: self.flag = flag
282
        if display_name == None:
283
            self.display_name = name.replace("_", " ").title()
Jerome Mariette's avatar
Jerome Mariette committed
284
        else: self.display_name = display_name
Jerome Mariette's avatar
help ok    
Jerome Mariette committed
285
        self.required = required
286
        self.choices = choices
287
288

        # Set parameter type
Jerome Mariette's avatar
Jerome Mariette committed
289
        if type == "date":
Frédéric Escudié's avatar
Frédéric Escudié committed
290
            self.type = date
Jerome Mariette's avatar
Jerome Mariette committed
291
        elif type == "multiple":
Jerome Mariette's avatar
help ok    
Jerome Mariette committed
292
            self.type = "multiple"
Jerome Mariette's avatar
Jerome Mariette committed
293
294
        elif isinstance(type, types.FunctionType):
            self.type = type
295
        elif type in [types.StringType, types.IntType, types.FloatType,  types.BooleanType]:
Jerome Mariette's avatar
Jerome Mariette committed
296
            self.type = type
Jerome Mariette's avatar
Jerome Mariette committed
297
298
299
        else:
            try: self.type = eval(type)
            except: self.type = types.StringType
300
301

        # Set parameter value
Jerome Mariette's avatar
Jerome Mariette committed
302
        self.default = default
303

304
    def export_to_argparse(self):
Jerome Mariette's avatar
Jerome Mariette committed
305
        if self.type == types.BooleanType and str(self.default).lower() in (False, "false",  "f", "0"):
306
307
308
            return {"help": self.help, "required": self.required, "dest": self.name, 
                    "default": False, "action": "store_true"}
        elif self.type == types.BooleanType:
309
            return {"help": self.help, "required": self.required, "dest": self.name, 
310
                    "default": True, "action": "store_false"}
Jerome Mariette's avatar
Jerome Mariette committed
311
        elif self.nargs > 1:
312
            return {"type": self.get_test_function(), "help": self.help, "required": self.required,
Jerome Mariette's avatar
Jerome Mariette committed
313
                    "dest": self.name, "default": self.default,
Jerome Mariette's avatar
Jerome Mariette committed
314
                    "action": self.action, "choices": self.choices, "nargs": "+"}
315
        else:
316
            return {"type": self.get_test_function(), "help": self.help, "required": self.required,
Jerome Mariette's avatar
Jerome Mariette committed
317
                    "dest": self.name, "default": self.default,
318
                    "action": self.action, "choices": self.choices}
319

Jerome Mariette's avatar
Jerome Mariette committed
320
321
    def get_type(self):
        return self.type.__name__
322

323
324
    def get_test_function(self):
        return create_test_function(self.type)
325

Jerome Mariette's avatar
Jerome Mariette committed
326
class AbstractIOFile(object):
327

Jerome Mariette's avatar
Jerome Mariette committed
328
329
    def __init__(self, file_format="any"):
        self.file_format = file_format
Jerome Mariette's avatar
Jerome Mariette committed
330
        self.component_nameid = None
331
        self.parent_component_nameid = []
Jerome Mariette's avatar
Jerome Mariette committed
332
333
334


class MultiParameter(dict, AbstractParameter):
Frédéric Escudié's avatar
Frédéric Escudié committed
335

Jerome Mariette's avatar
Jerome Mariette committed
336
337
338
339
340
    def __init__(self, name, help, required=False, flag=None, group="default", display_name=None):
        AbstractParameter.__init__(self, name, help, required=required, type="multiple", flag=flag, group=group, 
                                   display_name=display_name)
        return dict.__init__(self, {})

Jerome Mariette's avatar
help ok    
Jerome Mariette committed
341
342
343
    def add_sub_parameter(self, param):
        param_flag = param.flag[2:]
        if self.type == "multiple":
Jerome Mariette's avatar
Jerome Mariette committed
344
345
346
            if param.required: req = [param_flag] 
            else: req = []
            self.type = MultipleParameters({param_flag: param.type}, req, 
Jerome Mariette's avatar
help ok    
Jerome Mariette committed
347
348
349
350
351
352
353
354
                                           {param_flag: param.choices}, {}, {param_flag: param.default},
                                           {param_flag: param.action})
            if self.action == "append":
                self.action = MiltipleAppendAction
            else:
                self.action = MiltipleAction
            self.global_help = self.help
            self.help = self.global_help + " (" + param_flag + "=<" + param.type.__name__.upper() + ">)"
Jerome Mariette's avatar
Jerome Mariette committed
355
            self.default = {}
Jerome Mariette's avatar
Jerome Mariette committed
356
            self.nargs = "+"
Jerome Mariette's avatar
help ok    
Jerome Mariette committed
357
358
359
        elif self.type.__class__ == MultipleParameters:
            self.type.types[param_flag] = param.type
            self.type.choices[param_flag] = param.choices
360
            self.type.default[param_flag] = param.default if param != None else None
Jerome Mariette's avatar
help ok    
Jerome Mariette committed
361
            self.type.actions[param_flag] = param.action
Jerome Mariette's avatar
Jerome Mariette committed
362
363
            if param.required:
                self.type.required.append(param_flag)
Jerome Mariette's avatar
help ok    
Jerome Mariette committed
364
            self.help = self.global_help + self.type.get_help()
Jerome Mariette's avatar
Jerome Mariette committed
365
        param.flag = param_flag
Jerome Mariette's avatar
help ok    
Jerome Mariette committed
366
        self.sub_parameters.append(param)
367

368
369

class MultiParameterList(list, AbstractParameter):
Frédéric Escudié's avatar
Frédéric Escudié committed
370

371
    def __init__(self, name, help, required=False, flag=None, group="default", display_name=None):
Jerome Mariette's avatar
help ok    
Jerome Mariette committed
372
373
        AbstractParameter.__init__(self, name, help, required=required, type="multiple", flag=flag, 
                                   action="append", group=group, display_name=display_name)
374
        return list.__init__(self, [])
Jerome Mariette's avatar
Jerome Mariette committed
375

Jerome Mariette's avatar
Jerome Mariette committed
376
377
378
    def add_sub_parameter(self, param):
        param_flag = param.flag[2:]
        if self.type == "multiple":
Frédéric Escudié's avatar
Frédéric Escudié committed
379
            if param.required: req = [param_flag]
Jerome Mariette's avatar
Jerome Mariette committed
380
            else: req = []
Frédéric Escudié's avatar
Frédéric Escudié committed
381
            self.type = MultipleParameters({param_flag: param.type}, req,
Jerome Mariette's avatar
Jerome Mariette committed
382
383
384
385
386
387
388
389
390
391
392
393
394
                                           {param_flag: param.choices}, {}, {param_flag: param.default},
                                           {param_flag: param.action})
            if self.action == "append":
                self.action = MiltipleAppendAction
            else:
                self.action = MiltipleAction
            self.global_help = self.help
            self.help = self.global_help + " (" + param_flag + "=<" + param.type.__name__.upper() + ">)"
            self.default = []
            self.nargs = "+"
        elif self.type.__class__ == MultipleParameters:
            self.type.types[param_flag] = param.type
            self.type.choices[param_flag] = param.choices
395
            self.type.default[param_flag] = param.default if param != None else None
Jerome Mariette's avatar
Jerome Mariette committed
396
397
398
399
400
401
402
            self.type.actions[param_flag] = param.action
            if param.required:
                self.type.required.append(param_flag)
            self.help = self.global_help + self.type.get_help()
        param.flag = param_flag
        self.sub_parameters.append(param)

403

Frédéric Escudié's avatar
Frédéric Escudié committed
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
class ParameterFactory(object):
    @staticmethod
    def factory(*args, **kwargs):
        if not kwargs.has_key( "type" ):
            return StrParameter( *args, **kwargs )

        if kwargs["type"] == "int" or kwargs["type"] is types.IntType:
            return IntParameter( *args, **kwargs )
        elif kwargs["type"] == "bool" or kwargs["type"] is types.BooleanType:
            return BoolParameter( *args, **kwargs )
        elif kwargs["type"] == "float" or kwargs["type"] is types.FloatType:
            return FloatParameter( *args, **kwargs )
        elif kwargs["type"] == "date" or kwargs["type"] == date:
            return DateParameter( *args, **kwargs )
        else:
            return StrParameter( *args, **kwargs )


422
423
424
425
426
427
def noneException(*args, **kwargs):
    raise Exception( "The parameter value is None." )


class BoolParameter(int, AbstractParameter):

Frédéric Escudié's avatar
Frédéric Escudié committed
428
    def __new__(self, name, help, default=False, type=types.BooleanType, choices=None, required=False,
429
                flag=None, sub_parameters=None, group="default", display_name=None):
430
431
432
433
434
435
436
437
438
439
440
441
442
        bool_default = False if default == None else bool(default)
        val = int.__new__(self, bool_default)
        val.is_None = False
        if default == None:
            val.is_None = True
            for attr in val.__dict__:
                value = getattr(val, attr)
                if callable(value) and attr not in ["__new__", "__init__", "__int__", "__getattribute__", "__eq__", "__ne__", "__nonzero__"]:
                    setattr(val, attr, noneException)
        return val

    def __init__(self, name, help, default=None, type=types.BooleanType, choices=None, required=False,
                 flag=None, sub_parameters=None, group="default", display_name=None):
Frédéric Escudié's avatar
Frédéric Escudié committed
443
        AbstractParameter.__init__(self, name, help, flag=flag, default=bool(default), type=type, choices=choices, required=required, 
444
                                   action="store", sub_parameters=sub_parameters, group=group, display_name=display_name)
445

Frédéric Escudié's avatar
Frédéric Escudié committed
446
447
448
449
450
    def __str__(self):
        if self.is_None:
            noneException()
        return str(bool(self))

451
452
    def __eq__(self, other):
        if other.__class__.__name__ == "NoneType":
453
454
455
456
            return self.is_None
        elif self.is_None:
            return False
        else:
Frédéric Escudié's avatar
Frédéric Escudié committed
457
            return int(self) == int(bool(other))
458

459
    def __ne__(self, other):
Frédéric Escudié's avatar
Frédéric Escudié committed
460
        return not (self == other)
461

462
    def __nonzero__(self):
463
        if self.is_None:
464
            return False
465
466
467
        else:
            return self != 0

468

469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
class IntParameter(int, AbstractParameter):

    def __new__(self, name, help, default=None, type=types.IntType, choices=None, required=False,
                flag=None, sub_parameters=None, group="default", display_name=None):
        int_default = 0 if default == None else int(default)
        val = int.__new__(self, int_default)
        val.is_None = False
        if default == None:
            val.is_None = True
            for attr in val.__dict__:
                value = getattr(val, attr)
                if callable(value) and attr not in ["__new__", "__init__", "__int__", "__getattribute__", "__eq__", "__ne__", "__nonzero__"]:
                    setattr(val, attr, noneException)
        return val

    def __init__(self, name, help, default=None, type=types.IntType, choices=None, required=False,
485
                 flag=None, sub_parameters=None, group="default", display_name=None):
Jerome Mariette's avatar
Jerome Mariette committed
486
        AbstractParameter.__init__(self, name, help, flag=flag, default=default, type=type, choices=choices, required=required, 
487
488
                                   action="store", sub_parameters=sub_parameters, group=group, display_name=display_name)

489
490
491
492
493
494
495
496
497
    def __eq__(self, other):
        if other.__class__.__name__ == "NoneType":
            return self.is_None
        elif self.is_None:
            return False
        else:
            return int(self) == int(other)

    def __ne__(self, other):
Frédéric Escudié's avatar
Frédéric Escudié committed
498
        return not (self == other)
499
500
501
502
503
504
505
506

    def __nonzero__(self):
        if self.is_None:
            return False
        else:
            return self != 0


Frédéric Escudié's avatar
Frédéric Escudié committed
507
class FloatParameter(float, AbstractParameter):
508
509
510
511

    def __new__(self, name, help, default=None, type=types.FloatType, choices=None, required=False,
                flag=None, sub_parameters=None, group="default", display_name=None):
        float_default = 0.0 if default == None else float(default)
Frédéric Escudié's avatar
Frédéric Escudié committed
512
        val = float.__new__(self, float_default)
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
        val.is_None = False
        if default == None:
            val.is_None = True
            for attr in val.__dict__:
                value = getattr(val, attr)
                if callable(value) and attr not in ["__new__", "__init__", "__float__", "__getattribute__", "__eq__", "__ne__", "__nonzero__"]:
                    setattr(val, attr, noneException)
        return val

    def __init__(self, name, help, default=None, type=types.FloatType, choices=None, required=False,
                 flag=None, sub_parameters=None, group="default", display_name=None):
        AbstractParameter.__init__(self, name, help, flag=flag, default=default, type=type, choices=choices, required=required, 
                                   action="store", sub_parameters=sub_parameters, group=group, display_name=display_name)

    def __eq__(self, other):
        if other.__class__.__name__ == "NoneType":
            return self.is_None
        elif self.is_None:
            return False
        else:
            return float(self) == float(other)

    def __ne__(self, other):
Frédéric Escudié's avatar
Frédéric Escudié committed
536
        return not (self == other)
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573

    def __nonzero__(self):
        if self.is_None:
            return False
        else:
            return self != 0.0


class StrParameter(str, AbstractParameter):

    def __new__(self, name, help, default=None, type=types.StringType, choices=None, required=False,
                flag=None, sub_parameters=None, group="default", display_name=None):
        str_default = "" if default == None else str(default)
        val = str.__new__(self, str_default)
        val.is_None = False
        if default == None:
            val.is_None = True
            for attr in val.__dict__:
                value = getattr(val, attr)
                if callable(value) and attr not in ["__new__", "__init__", "__str__", "__getattribute__", "__eq__", "__ne__", "__nonzero__"]:
                    setattr(val, attr, noneException)
        return val

    def __init__(self, name, help, default=None, type=types.StringType, choices=None, required=False,
                 flag=None, sub_parameters=None, group="default", display_name=None):
        AbstractParameter.__init__(self, name, help, flag=flag, default=default, type=type, choices=choices, required=required, 
                                   action="store", sub_parameters=sub_parameters, group=group, display_name=display_name)

    def __eq__(self, other):
        if other.__class__.__name__ == "NoneType":
            return self.is_None
        elif self.is_None:
            return False
        else:
            return str(self) == str(other)

    def __ne__(self, other):
Frédéric Escudié's avatar
Frédéric Escudié committed
574
        return not(self == other)
575
576
577
578
579
580
581
582

    def __nonzero__(self):
        if self.is_None:
            return False
        else:
            return (True if str(self) else False)


Frédéric Escudié's avatar
Frédéric Escudié committed
583
class DateParameter(datetime.datetime, AbstractParameter):
584

Frédéric Escudié's avatar
Frédéric Escudié committed
585
586
587
588
589
590
591
592
593
594
595
    def __new__(self, name, help, default=None, type=date, choices=None, required=False,
                flag=None, sub_parameters=None, group="default", display_name=None):
        date_default = datetime.datetime.today()
        if default != None and issubclass(default.__class__, datetime.datetime):
            date_default = default
        elif default != None:
            date_default = date(default)

        val = datetime.datetime.__new__(self, date_default.year, date_default.month, date_default.day)
        if default != None:
            val.is_None = False
596
        else:
Frédéric Escudié's avatar
Frédéric Escudié committed
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
            val.is_None = True
            for attr in val.__dict__:
                value = getattr(val, attr)
                if callable(value) and attr not in ["__new__", "__init__", "__getattribute__", "__eq__", "__ne__", "__nonzero__"]:
                    setattr(val, attr, noneException)
        return val

    def __init__(self, name, help, default=None, type=date, choices=None, required=False,
                 flag=None, sub_parameters=None, group="default", display_name=None):
        if default != None and not issubclass(default.__class__, datetime.datetime):
            date_default = date(default)
            default = datetime.datetime(date_default.year, date_default.month, date_default.day)
        AbstractParameter.__init__(self, name, help, flag=flag, default=default, type=type, choices=choices, required=required, 
                                   action="store", sub_parameters=sub_parameters, group=group, display_name=display_name)

    def __eq__(self, other):
        if other.__class__.__name__ == "NoneType":
            return self.is_None
        elif self.is_None:
            return False
        else:
            return datetime.date(self.year, self.month, self.day) == other

    def __ne__(self, other):
        return not( self == other )

    def __nonzero__(self):
        if self.is_None:
            return False
        else:
            return (True if datetime.date(self.year, self.month, self.day) else False)

629
630
631
632
633
634
635
636
    def __reduce__(self):
        return (DateParameter, (self.name, self.help, self.default, date, self.choices, self.required,
                 self.flag, self.sub_parameters, self.group, self.display_name), None, None, None)

    def __reduce_ex__(self, protocol):
        return (DateParameter, (self.name, self.help, self.default, date, self.choices, self.required,
                 self.flag, self.sub_parameters, self.group, self.display_name), None, None, None)

Frédéric Escudié's avatar
Frédéric Escudié committed
637
638
639
640
641

class AbstractInputFile(AbstractIOFile):
    """
     @summary : Parent of all InputFile(s) parameters.
    """
642
643
644
645
646
647
648
    
    SIZE_LIMIT_SPLITER = "__sl"
    
    def __init__(self, file_format="any", size_limit="0"):
        AbstractIOFile.__init__(self, file_format)
        self.size_limit = size_limit
    
Jerome Mariette's avatar
Jerome Mariette committed
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
    def _download_urlfile(self, input):
        try:
            uri_object = urlparse(input)
            opener = urllib2.urlopen(input)
            block_size = 8000
            jflowconf = JFlowConfigReader()
            tmp_directory = os.path.join(jflowconf.get_tmp_directory(), os.path.basename(tempfile.NamedTemporaryFile().name))
            os.mkdir(tmp_directory) 
            file_path = os.path.join(tmp_directory, os.path.basename(uri_object.path))
            local_file = open(file_path, 'wb')
            if os.path.basename(uri_object.path) is not None and os.path.basename(uri_object.path) != "":
                metadata = opener.info()
                file_size = int(metadata.getheaders("Content-Length")[0])
                while True:
                    buffer = opener.read(block_size)
                    # End of download
                    if not buffer: break
                    # Parts of download
                    local_file.write(buffer)
                local_file.close()
                logging.getLogger("AbstractInputFile.download_urlfile").debug("URL file '{0}' successfully downloaded as: {1}".format(input, file_path))
            return [file_path, True]
        except:
            return [input, False]
673

Jerome Mariette's avatar
Jerome Mariette committed
674
    def check(self, ifile):
Jerome Mariette's avatar
Jerome Mariette committed
675
676
677
678
        try:
            eval(self.file_format)
            function_exists = True
        except: function_exists = False
679
680
681
682
683
684
685
686
687
        if function_exists:
            try: 
                eval(self.file_format)(ifile)
            except jflow.InvalidFormatError as e:
                sys.stderr.write(str(e)+"\n")
                sys.exit(1)
        else:
            sys.stderr.write("Error: Invalid file format '" + self.file_format + "'!\n")
            sys.exit(1)
Frédéric Escudié's avatar
Frédéric Escudié committed
688
689
690
691
692
693
694

class AbstractOutputFile(AbstractIOFile):
    """
     @summary : Parent of all OutputFile(s) parameters.
    """
    pass

695
696
697

class InputFile(StrParameter, AbstractInputFile):

Jerome Mariette's avatar
Jerome Mariette committed
698
    def __new__(self, name, help, file_format="any", default="", type="localfile", choices=None, 
699
                required=False, flag=None, group="default", display_name=None, size_limit="0"):
700

Jerome Mariette's avatar
Jerome Mariette committed
701
702
703
        if hasattr(type, '__call__'):
            type2test = type.__name__
        else: type2test = type
704

Jerome Mariette's avatar
Jerome Mariette committed
705
706
707
        if type2test not in INPUTFILE_TYPES:
            raise ValueError("InputFile.__new__: wrong type provided: '"+type2test+"', this should be choosen between '" 
                             + "', '".join(INPUTFILE_TYPES)+"'")
708
709
710
711

        return StrParameter.__new__(self, name, help, flag=flag, default=default, type=type, choices=choices, 
                           required=required, group=group, display_name=display_name)

Jerome Mariette's avatar
Jerome Mariette committed
712
    def __init__(self, name, help, file_format="any", default="", type="localfile", choices=None, 
713
714
                required=False, flag=None, group="default", display_name=None, size_limit="0"):
        AbstractInputFile.__init__(self, file_format, size_limit)
715
        StrParameter.__init__(self, name, help, flag=flag, default=default, type=type, choices=choices, 
716
                           required=required, group=group, display_name=display_name)
Jerome Mariette's avatar
Jerome Mariette committed
717

Jerome Mariette's avatar
Jerome Mariette committed
718
719
720
    def get_type(self):
        return self.type.__name__+AbstractInputFile.SIZE_LIMIT_SPLITER+self.size_limit

721
722
    def get_test_function(self):
        if (self.size_limit == "0"): ctype = self.type
Jerome Mariette's avatar
Jerome Mariette committed
723
        else: ctype = self.get_type()
724
725
        return create_test_function(ctype)

Jerome Mariette's avatar
Jerome Mariette committed
726
727
728
729
730
731
732
733
734
735
    def prepare_input_file(self, input):
        # handle url inputs
        new_path, is_uri = self._download_urlfile(input)
        # handle upload inputs
        try: is_local = os.path.isfile(input)
        except: is_local = False
        if not is_uri and not is_local and self.type.__name__ == "inputfile" or self.type.__name__ == "browsefile":
            jflow_config_reader = JFlowConfigReader()
            new_path = os.path.join(jflow_config_reader.get_tmp_directory(), input)
        if is_local: new_path = input
Jerome Mariette's avatar
Jerome Mariette committed
736
        self.check(new_path)
Jerome Mariette's avatar
Jerome Mariette committed
737
        return new_path
738

Jerome Mariette's avatar
Jerome Mariette committed
739

740
741
class OutputFile(StrParameter, AbstractOutputFile):

Frédéric Escudié's avatar
Frédéric Escudié committed
742
    def __new__(self, name, help, file_format="any", default="", choices=None,
743
                required=False, flag=None, group="default", display_name=None):
Frédéric Escudié's avatar
Frédéric Escudié committed
744
        return StrParameter.__new__(self, name, help, flag=flag, default=default, type="localfile", choices=choices,
745
746
                           required=required, group=group, display_name=display_name)

Frédéric Escudié's avatar
Frédéric Escudié committed
747
    def __init__(self, name, help, file_format="any", default="", choices=None,
748
                required=False, flag=None, group="default", display_name=None):
Jerome Mariette's avatar
Jerome Mariette committed
749
        AbstractIOFile.__init__(self, file_format)
750
        StrParameter.__init__(self, name, help, flag=flag, default=default, type="localfile", choices=choices, 
751
752
                           required=required, group=group, display_name=display_name)

753

754
class ParameterList(list, AbstractParameter):
755

Jerome Mariette's avatar
Jerome Mariette committed
756
    def __init__(self, name, help, default=None, type=types.StringType, choices=None, required=False,
757
                 flag=None, sub_parameters=None, group="default", display_name=None):
Jerome Mariette's avatar
Jerome Mariette committed
758
        if default == None: default = []
Frédéric Escudié's avatar
Frédéric Escudié committed
759
        AbstractParameter.__init__(self, name, help, flag=flag, default=default, type=type, choices=choices, required=required,
760
761
762
763
764
                                   action="append", sub_parameters=sub_parameters, group=group, display_name=display_name)
        if default.__class__.__name__ == "str":
            return list.__init__(self, [default])
        elif default.__class__.__name__ == "list":
            return list.__init__(self, default)
765

766

Frédéric Escudié's avatar
Frédéric Escudié committed
767
class InputFileList(ParameterList, AbstractInputFile):
768

Jerome Mariette's avatar
Jerome Mariette committed
769
    def __init__(self, name, help, file_format="any", default=None, type="localfile", choices=None, 
770
                 required=False, flag=None, group="default", display_name=None, size_limit="0"):
771

Jerome Mariette's avatar
Jerome Mariette committed
772
        if default == None: default = []   
Jerome Mariette's avatar
Jerome Mariette committed
773
774
775
        if hasattr(type, '__call__'):
            type2test = type.__name__
        else: type2test = type
776

Jerome Mariette's avatar
Jerome Mariette committed
777
778
779
        if type2test not in INPUTFILE_TYPES:
            raise ValueError("InputFile.__new__: wrong type provided: '"+type2test+"', this should be choosen between '" 
                             + "', '".join(INPUTFILE_TYPES)+"'")
780

781
        AbstractInputFile.__init__(self, file_format, size_limit)
Jerome Mariette's avatar
Jerome Mariette committed
782
        ParameterList.__init__(self, name, help, flag=flag, default=default, type=type, choices=choices, 
783
                               required=required, group=group, display_name=display_name)
784

785
786
787
788
        if default.__class__.__name__ == "str":
            return list.__init__(self, [default])
        elif default.__class__.__name__ == "list":
            return list.__init__(self, default)
789
790
        elif issubclass( default.__class__, InputFile ):
            return list.__init__(self, [default])
Frédéric Escudié's avatar
Frédéric Escudié committed
791
        elif issubclass( default.__class__, AbstractInputFile ):
Jerome Mariette's avatar
Jerome Mariette committed
792
            return list.__init__(self, default)
793
794
        elif issubclass( default.__class__, OutputFile ):
            return list.__init__(self, [default])
Frédéric Escudié's avatar
Frédéric Escudié committed
795
        elif issubclass( default.__class__, AbstractOutputFile ):
Jerome Mariette's avatar
Jerome Mariette committed
796
            return list.__init__(self, default)
Jerome Mariette's avatar
Jerome Mariette committed
797
798
799
800
    
    def get_type(self):
        return self.type.__name__+AbstractInputFile.SIZE_LIMIT_SPLITER+self.size_limit
    
801
802
    def get_test_function(self):
        if (self.size_limit == "0"): ctype = self.type
Jerome Mariette's avatar
Jerome Mariette committed
803
        else: ctype = self.get_type()
804
805
        return create_test_function(ctype)

Jerome Mariette's avatar
Jerome Mariette committed
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
    def prepare_input_files(self, inputs):
        path2test = inputs
        if not path2test.__class__.__name__ == "list":
            path2test = [path2test]
        new_vals, done = [], []
        # handle url inputs
        for path in path2test:
            new_url, is_uri = self._download_urlfile(path)
            if is_uri:
                new_vals.append(new_url)
                done.append(path)
        for d in done: path2test.remove(d)
        done = []
        # handle localfile
        for path in path2test:
            if os.path.isfile(path):
                new_vals.append(path)
                done.append(path)
        for d in done: path2test.remove(d)        
        # handle upload inputs
        if self.type.__name__ == "inputfile" or self.type.__name__ == "browsefile":
            for path in path2test:
                jflow_config_reader = JFlowConfigReader()
                new_vals.append(os.path.join(jflow_config_reader.get_tmp_directory(), path))
Jerome Mariette's avatar
Jerome Mariette committed
830
831
832
        # now that all files are downloaded and ok, check the format
        for cfile in new_vals:
            self.check(cfile)
Jerome Mariette's avatar
Jerome Mariette committed
833
        return new_vals
834

Frédéric Escudié's avatar
Frédéric Escudié committed
835
836
class OutputFileList(ParameterList, AbstractOutputFile):

Jerome Mariette's avatar
Jerome Mariette committed
837
838
839
    def __init__(self, name, help, file_format="any", default=None, choices=None, 
                 required=False, flag=None, group="default", display_name=None):
        if default == None: default = []
Jerome Mariette's avatar
Jerome Mariette committed
840
        AbstractIOFile.__init__(self, file_format)
Jerome Mariette's avatar
Jerome Mariette committed
841
        ParameterList.__init__(self, name, help, flag=flag, default=default, type="localfile", choices=choices, 
842
843
844
845
846
                               required=required, group=group, display_name=display_name)
        if default.__class__.__name__ == "str":
            return list.__init__(self, [default])
        elif default.__class__.__name__ == "list":
            return list.__init__(self, default)
Frédéric Escudié's avatar
Frédéric Escudié committed
847

848

Frédéric Escudié's avatar
Frédéric Escudié committed
849
850
851
852
853
854
855
856
857
858
class DynamicOutput(ParameterList, AbstractOutputFile):
    """
     @warning : with this class of output, the component become dynamic.
    """
    def update(self):
        """
         This method is used at the end of component execution to update output list.
        """
        raise NotImplementedError

859

Frédéric Escudié's avatar
Frédéric Escudié committed
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
class OutputFilesEndsWith(DynamicOutput):

    def __init__(self, name, help, output_directory, end_str, include=True, file_format="any", choices=None, 
                 required=False, flag=None, group="default", display_name=None):
        """
         @warning : with this class of output, the component become dynamic.
         @param output_directory : path to the directory where outputs will be created.
         @param end_str : the end of the files names.
         @param include : if true, the files with name terminated by end_str are added into output files.
                          If false, the files with name not terminated by end_str are added into output files.
        """
        AbstractIOFile.__init__(self, file_format)
        default = []
        ParameterList.__init__(self, name, help, flag=flag, default=default, type="localfile", choices=choices, 
                               required=required, group=group, display_name=display_name)
        self.output_directory = output_directory
        self.end_str = end_str
        self.include = include
        return list.__init__(self, default)

    def update(self):
        output_files = list()
        for file in os.listdir( self.output_directory ):
            if file.endswith( self.end_str ) and self.include :
                output_files.append( os.path.join(self.output_directory, file) )
            elif not file.endswith( self.end_str ) and not self.include:
                output_files.append( os.path.join(self.output_directory, file) )
        list.__init__(self, output_files)

889

Frédéric Escudié's avatar
Frédéric Escudié committed
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
class OutputFilesPattern(DynamicOutput):

    def __init__(self, name, help, output_directory, pattern, include=True, file_format="any", choices=None, 
                  required=False, flag=None, group="default", display_name=None):
        """
         @warning : with this class of output, the component become dynamic.
         @param output_directory : path to the directory where outputs will be created.
         @param pattern : the pattern of (a part) the file names.
         @param include : if true, the files with the pattern in file name are added into output files.
                          If false, the files with the pattern in file name are added into output files.
        """
        AbstractIOFile.__init__(self, file_format)
        default = []
        ParameterList.__init__(self, name, help, flag=flag, default=default, type="localfile", choices=choices, 
                               required=required, group=group, display_name=display_name)
        self.output_directory = output_directory
        self.pattern = pattern
        self.include = include
        return list.__init__(self, default)

    def update(self):
        output_files = list()
        for file in os.listdir( self.output_directory ):
            if self.include and re.search( self.pattern, file ) is not None:
                output_files.append( os.path.join(self.output_directory, file) )
            elif not self.include and re.search( self.pattern, file ) is None:
                output_files.append( os.path.join(self.output_directory, file) )
        return list.__init__(self, output_files)