parameter.py 36.1 KB
Newer Older
Jerome Mariette's avatar
Jerome Mariette committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
#
# Copyright (C) 2012 INRA
# 
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
# 
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
# 
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
#

Frédéric Escudié's avatar
Frédéric Escudié committed
18
import re
19
import sys
Jerome Mariette's avatar
Jerome Mariette committed
20
import types
Jerome Mariette's avatar
Jerome Mariette committed
21
import datetime
Jerome Mariette's avatar
Jerome Mariette committed
22
import logging
Jerome Mariette's avatar
Jerome Mariette committed
23
import argparse
Jerome Mariette's avatar
Jerome Mariette committed
24
25
import os
import tempfile
26
from argparse import _ensure_value
Jerome Mariette's avatar
Jerome Mariette committed
27
import urllib2
28
import copy as _copy
Jerome Mariette's avatar
Jerome Mariette committed
29
30
31
from urlparse import urlparse
from jflow.config_reader import JFlowConfigReader

Jerome Mariette's avatar
Jerome Mariette committed
32
# import custom types and custom formats
Jerome Mariette's avatar
Jerome Mariette committed
33
from workflows.types import *
Jerome Mariette's avatar
Jerome Mariette committed
34
from workflows.formats import *
Jerome Mariette's avatar
Jerome Mariette committed
35

Jerome Mariette's avatar
Jerome Mariette committed
36
37
38
39
40
41
42
43
44
45
46
47
48
49

# define all input type available
INPUTFILE_TYPES = ["inputfile", "localfile", "urlfile", "browsefile"]

def browsefile(file):
    # browsefile are not available from command line, considere it as a localfile
    # from the gui, this will not been tested this way
    return localfile(file)

def localfile(file):
    if os.path.isfile(file):
        return os.path.abspath(file)
    else:
        raise argparse.ArgumentTypeError("Local file '" + file + "' does not exists! Please provide a valid file path!")
50

Jerome Mariette's avatar
Jerome Mariette committed
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
def urlfile(file):
    uri_object = urlparse(file)
    try:
        opener = urllib2.urlopen(file)
    except:
        raise argparse.ArgumentTypeError("URL '" + file + "' is invalid!")
    file_name = os.path.basename(uri_object.path)
    if file_name is not None and file_name != "":
        metadata = opener.info()
        file_size = int(metadata.getheaders("Content-Length")[0])
        if file_size == 0:
            raise argparse.ArgumentTypeError("The URL file '" + file + "' is empty!")
        return file
    else:
        raise argparse.ArgumentTypeError("URL '" + file + "' does not contain any file name!")
66

Jerome Mariette's avatar
Jerome Mariette committed
67
68
69
70
71
72
73
74
def inputfile(file):
    # test the format
    uri_object = urlparse(file)
    # check the file
    if uri_object.scheme == '':
        return localfile(file)
    else:
        return urlfile(file)
75

Jerome Mariette's avatar
Jerome Mariette committed
76
class MultipleParameters(object):
77
    def __init__(self, types, required, choices, excludes, default, actions):
Jerome Mariette's avatar
Jerome Mariette committed
78
        self.types = types
79
        self.choices = choices
Jerome Mariette's avatar
Jerome Mariette committed
80
        self.excludes = excludes
81
        self.default = default
82
        self.actions = actions
Jerome Mariette's avatar
Jerome Mariette committed
83
        self.index = None
84
        self.required = required
Jerome Mariette's avatar
Jerome Mariette committed
85
        self.__name__ = "MultipleParameters"
86

Jerome Mariette's avatar
help ok    
Jerome Mariette committed
87
88
89
90
91
    def get_help(self):
        help = " ("
        for flag in self.types.keys():
            help += flag + "=<" + self.types[flag].__name__.upper() + ">, "
        return help[:-2] + ")"
92

93
    def __call__(self, arg):
Jerome Mariette's avatar
Jerome Mariette committed
94
95
96
        parts = arg.split("=")
        if not self.types.has_key(parts[0]):
            raise argparse.ArgumentTypeError(parts[0] + " is an invalid flag! Available ones are: "+", ".join(self.types.keys()))
97
98
99
        if self.choices[parts[0]] != None:
            if parts[1] not in self.choices[parts[0]]:
                raise argparse.ArgumentTypeError("argument " + parts[0] + ": invalid choice: '" + parts[1] + "' (choose from " + ", ".join(self.choices[parts[0]]) + "))")
100

101
102
103
104
105
106
107
        if self.types[parts[0]] == types.BooleanType:
            return (parts[0], not self.default[parts[0]], self.required, self.excludes)
        else:
            try:
                value = self.types[parts[0]](parts[1])
            except:
                raise argparse.ArgumentTypeError("invalid " + self.types[parts[0]].__name__ + " value: '" + parts[1] + "' for sub parameter '" + parts[0] + "'")
108

109
            self.index = parts[0]
110
            return (parts[0], value, self.required, self.excludes, self.actions)
Jerome Mariette's avatar
Jerome Mariette committed
111

112
113
class MiltipleAction(argparse.Action):
    def __call__(self, parser, namespace, values, option_string=None):
114
115
116
117
118
119
120
        # what is commun within required and excludes
        exclusif_required = {}
        for exclude_group in values[0][3]:
            exclusif_required[exclude_group] = False
            for param in values[0][3][exclude_group]:
                if param in values[0][2]:
                    exclusif_required[exclude_group] = True
Jerome Mariette's avatar
Jerome Mariette committed
121
122
        given_params = []
        # first check for required parameters
123
124
        try:
            required = _copy.copy(values[0][2])
125
126
127
128
129
130
            # delete required that are exclusive
            for group in exclusif_required:
                if exclusif_required[group]:
                    for val in values[0][3][group]:
                        if val in required:
                            del required[required.index(val)]
131
            for val in values:
Jerome Mariette's avatar
Jerome Mariette committed
132
                given_params.append(val[0])
133
134
135
136
137
                if val[0] in required:
                    del required[required.index(val[0])]
        except: pass
        if len(required) == 1: parser.error(", ".join(required) + " is a required parameter!")
        elif len(required) > 1: parser.error(", ".join(required) + " are required parameters!")
Jerome Mariette's avatar
Jerome Mariette committed
138
139
140
141
142
143
144
145
        # then for exclude ones    
        for exclude_group in values[0][3]:
            found = None
            for param in values[0][3][exclude_group]:
                if param in given_params and found != None:
                    parser.error("argument '" + found + "': not allowed with argument '" + param + "'")
                    break
                elif param in given_params: found = param
146

147
148
149
150
151
152
153
154
        # check for required exclusive if one of them is in
        if len(exclusif_required.keys()) > 0:
            for group in exclusif_required:
                if exclusif_required[group]:
                    rfound = False
                    for param in values[0][3][group]:
                        if param in given_params: rfound = True
            if not rfound: parser.error("one of the arguments: " + ", ".join(values[0][3][group]) + " is required")
155

Jerome Mariette's avatar
Jerome Mariette committed
156
        # if ok add the value
157
158
159
160
161
162
163
164
165
166
167
        final_hash, final_values = {}, []
        for value in values:
            if values[0][4][value[0]] == "append" and final_hash.has_key(value[0]):
                final_hash[value[0]].append(value[1])
            elif values[0][4][value[0]] == "append":
                final_hash[value[0]]= [value[1]]
            else:
                final_hash[value[0]]= value[1]
        for param in final_hash:
            final_values.append((param, final_hash[param]))
        setattr(namespace, self.dest, final_values)
168

169

170
171
class MiltipleAppendAction(argparse.Action):
    def __call__(self, parser, namespace, values, option_string=None):
172
173
174
175
176
177
178
        # what is commun within required and excludes
        exclusif_required = {}
        for exclude_group in values[0][3]:
            exclusif_required[exclude_group] = False
            for param in values[0][3][exclude_group]:
                if param in values[0][2]:
                    exclusif_required[exclude_group] = True
Jerome Mariette's avatar
Jerome Mariette committed
179
180
        given_params = []
        # first check for required parameters
181
182
        try:
            required = _copy.copy(values[0][2])
183
184
185
186
187
188
            # delete required that are exclusive
            for group in exclusif_required:
                if exclusif_required[group]:
                    for val in values[0][3][group]:
                        if val in required:
                            del required[required.index(val)]
189
            for val in values:
Jerome Mariette's avatar
Jerome Mariette committed
190
                given_params.append(val[0])
191
192
193
194
195
                if val[0] in required:
                    del required[required.index(val[0])]
        except: pass
        if len(required) == 1: parser.error(", ".join(required) + " is a required parameter!")
        elif len(required) > 1: parser.error(", ".join(required) + " are required parameters!")
Jerome Mariette's avatar
Jerome Mariette committed
196
197
198
199
200
201
202
203
        # then for exclude ones    
        for exclude_group in values[0][3]:
            found = None
            for param in values[0][3][exclude_group]:
                if param in given_params and found != None:
                    parser.error("argument '" + found + "': not allowed with argument '" + param + "'")
                    break
                elif param in given_params: found = param
204

205
206
207
208
209
210
211
212
        # check for required exclusive if one of them is in
        if len(exclusif_required.keys()) > 0:
            for group in exclusif_required:
                if exclusif_required[group]:
                    rfound = False
                    for param in values[0][3][group]:
                        if param in given_params: rfound = True
            if not rfound: parser.error("one of the arguments: " + ", ".join(values[0][3][group]) + " is required")
Jerome Mariette's avatar
Jerome Mariette committed
213
        # if ok add the value
214
        items = _copy.copy(_ensure_value(namespace, self.dest, []))
215
216
217
218
219
220
221
222
223
224
225
        final_hash, final_values = {}, []
        for value in values:
            if values[0][4][value[0]] == "append" and final_hash.has_key(value[0]):
                final_hash[value[0]].append(value[1])
            elif values[0][4][value[0]] == "append":
                final_hash[value[0]]= [value[1]]
            else:
                final_hash[value[0]]= value[1]
        for param in final_hash:
            final_values.append((param, final_hash[param]))
        items.append(final_values)
226
        setattr(namespace, self.dest, items)
227
228
229


class AbstractParameter(object):
230

231
232
    def __init__(self, name, help, default=None, type=types.StringType, choices=None, required=False,
                 flag=None, action="store", sub_parameters=None, group="default", display_name=None):
Jerome Mariette's avatar
Jerome Mariette committed
233

Jerome Mariette's avatar
Jerome Mariette committed
234
235
        self.name = name
        self.help = help
236
        self.action = action
Jerome Mariette's avatar
Jerome Mariette committed
237
        self.nargs = None
238
239
240
        if sub_parameters:
            self.sub_parameters = sub_parameters
        else: self.sub_parameters = []
Jerome Mariette's avatar
Jerome Mariette committed
241
        self.group = group
242
243
244
        if flag == None:
            self.flag = "--"+name.replace("_", "-")
        else: self.flag = flag
245
        if display_name == None:
246
            self.display_name = name.replace("_", " ").title()
Jerome Mariette's avatar
Jerome Mariette committed
247
        else: self.display_name = display_name
Jerome Mariette's avatar
help ok    
Jerome Mariette committed
248
        self.required = required
249
        self.choices = choices
250
251

        # Set parameter type
Jerome Mariette's avatar
Jerome Mariette committed
252
        if type == "date":
Frédéric Escudié's avatar
Frédéric Escudié committed
253
            self.type = date
Jerome Mariette's avatar
Jerome Mariette committed
254
        elif type == "multiple":
Jerome Mariette's avatar
help ok    
Jerome Mariette committed
255
            self.type = "multiple"
Jerome Mariette's avatar
Jerome Mariette committed
256
257
        elif isinstance(type, types.FunctionType):
            self.type = type
258
        elif type in [types.StringType, types.IntType, types.FloatType,  types.BooleanType]:
Jerome Mariette's avatar
Jerome Mariette committed
259
            self.type = type
Jerome Mariette's avatar
Jerome Mariette committed
260
261
262
        else:
            try: self.type = eval(type)
            except: self.type = types.StringType
263
264

        # Set parameter value
Jerome Mariette's avatar
Jerome Mariette committed
265
        self.default = default
266

267
    def export_to_argparse(self):
Jerome Mariette's avatar
Jerome Mariette committed
268
        if self.type == types.BooleanType and str(self.default).lower() in (False, "false",  "f", "0"):
269
270
271
            return {"help": self.help, "required": self.required, "dest": self.name, 
                    "default": False, "action": "store_true"}
        elif self.type == types.BooleanType:
272
            return {"help": self.help, "required": self.required, "dest": self.name, 
273
                    "default": True, "action": "store_false"}
Jerome Mariette's avatar
Jerome Mariette committed
274
275
276
        elif self.nargs > 1:
            return {"type": self.type, "help": self.help, "required": self.required,
                    "dest": self.name, "default": self.default,
Jerome Mariette's avatar
Jerome Mariette committed
277
                    "action": self.action, "choices": self.choices, "nargs": "+"}
278
279
        else:
            return {"type": self.type, "help": self.help, "required": self.required,
Jerome Mariette's avatar
Jerome Mariette committed
280
                    "dest": self.name, "default": self.default,
281
                    "action": self.action, "choices": self.choices}
282

Jerome Mariette's avatar
Jerome Mariette committed
283
284
    def get_type(self):
        return self.type.__name__
285
286


Jerome Mariette's avatar
Jerome Mariette committed
287
class AbstractIOFile(object):
288

Jerome Mariette's avatar
Jerome Mariette committed
289
290
    def __init__(self, file_format="any"):
        self.file_format = file_format
Jerome Mariette's avatar
Jerome Mariette committed
291
        self.component_nameid = None
Jerome Mariette's avatar
Jerome Mariette committed
292
293
294
295
        self.parent_component_nameid = None


class MultiParameter(dict, AbstractParameter):
Frédéric Escudié's avatar
Frédéric Escudié committed
296

Jerome Mariette's avatar
Jerome Mariette committed
297
298
299
300
301
    def __init__(self, name, help, required=False, flag=None, group="default", display_name=None):
        AbstractParameter.__init__(self, name, help, required=required, type="multiple", flag=flag, group=group, 
                                   display_name=display_name)
        return dict.__init__(self, {})

Jerome Mariette's avatar
help ok    
Jerome Mariette committed
302
303
304
    def add_sub_parameter(self, param):
        param_flag = param.flag[2:]
        if self.type == "multiple":
Jerome Mariette's avatar
Jerome Mariette committed
305
306
307
            if param.required: req = [param_flag] 
            else: req = []
            self.type = MultipleParameters({param_flag: param.type}, req, 
Jerome Mariette's avatar
help ok    
Jerome Mariette committed
308
309
310
311
312
313
314
315
                                           {param_flag: param.choices}, {}, {param_flag: param.default},
                                           {param_flag: param.action})
            if self.action == "append":
                self.action = MiltipleAppendAction
            else:
                self.action = MiltipleAction
            self.global_help = self.help
            self.help = self.global_help + " (" + param_flag + "=<" + param.type.__name__.upper() + ">)"
Jerome Mariette's avatar
Jerome Mariette committed
316
            self.default = {}
Jerome Mariette's avatar
Jerome Mariette committed
317
            self.nargs = "+"
Jerome Mariette's avatar
help ok    
Jerome Mariette committed
318
319
320
        elif self.type.__class__ == MultipleParameters:
            self.type.types[param_flag] = param.type
            self.type.choices[param_flag] = param.choices
321
            self.type.default[param_flag] = param.default if param != None else None
Jerome Mariette's avatar
help ok    
Jerome Mariette committed
322
            self.type.actions[param_flag] = param.action
Jerome Mariette's avatar
Jerome Mariette committed
323
324
            if param.required:
                self.type.required.append(param_flag)
Jerome Mariette's avatar
help ok    
Jerome Mariette committed
325
            self.help = self.global_help + self.type.get_help()
Jerome Mariette's avatar
Jerome Mariette committed
326
        param.flag = param_flag
Jerome Mariette's avatar
help ok    
Jerome Mariette committed
327
        self.sub_parameters.append(param)
328

329
330

class MultiParameterList(list, AbstractParameter):
Frédéric Escudié's avatar
Frédéric Escudié committed
331

332
    def __init__(self, name, help, required=False, flag=None, group="default", display_name=None):
Jerome Mariette's avatar
help ok    
Jerome Mariette committed
333
334
        AbstractParameter.__init__(self, name, help, required=required, type="multiple", flag=flag, 
                                   action="append", group=group, display_name=display_name)
335
        return list.__init__(self, [])
Jerome Mariette's avatar
Jerome Mariette committed
336

Jerome Mariette's avatar
Jerome Mariette committed
337
338
339
    def add_sub_parameter(self, param):
        param_flag = param.flag[2:]
        if self.type == "multiple":
Frédéric Escudié's avatar
Frédéric Escudié committed
340
            if param.required: req = [param_flag]
Jerome Mariette's avatar
Jerome Mariette committed
341
            else: req = []
Frédéric Escudié's avatar
Frédéric Escudié committed
342
            self.type = MultipleParameters({param_flag: param.type}, req,
Jerome Mariette's avatar
Jerome Mariette committed
343
344
345
346
347
348
349
350
351
352
353
354
355
                                           {param_flag: param.choices}, {}, {param_flag: param.default},
                                           {param_flag: param.action})
            if self.action == "append":
                self.action = MiltipleAppendAction
            else:
                self.action = MiltipleAction
            self.global_help = self.help
            self.help = self.global_help + " (" + param_flag + "=<" + param.type.__name__.upper() + ">)"
            self.default = []
            self.nargs = "+"
        elif self.type.__class__ == MultipleParameters:
            self.type.types[param_flag] = param.type
            self.type.choices[param_flag] = param.choices
356
            self.type.default[param_flag] = param.default if param != None else None
Jerome Mariette's avatar
Jerome Mariette committed
357
358
359
360
361
362
363
            self.type.actions[param_flag] = param.action
            if param.required:
                self.type.required.append(param_flag)
            self.help = self.global_help + self.type.get_help()
        param.flag = param_flag
        self.sub_parameters.append(param)

364

Frédéric Escudié's avatar
Frédéric Escudié committed
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
class ParameterFactory(object):
    @staticmethod
    def factory(*args, **kwargs):
        if not kwargs.has_key( "type" ):
            return StrParameter( *args, **kwargs )

        if kwargs["type"] == "int" or kwargs["type"] is types.IntType:
            return IntParameter( *args, **kwargs )
        elif kwargs["type"] == "bool" or kwargs["type"] is types.BooleanType:
            return BoolParameter( *args, **kwargs )
        elif kwargs["type"] == "float" or kwargs["type"] is types.FloatType:
            return FloatParameter( *args, **kwargs )
        elif kwargs["type"] == "date" or kwargs["type"] == date:
            return DateParameter( *args, **kwargs )
        else:
            return StrParameter( *args, **kwargs )


383
384
385
386
387
388
def noneException(*args, **kwargs):
    raise Exception( "The parameter value is None." )


class BoolParameter(int, AbstractParameter):

Frédéric Escudié's avatar
Frédéric Escudié committed
389
    def __new__(self, name, help, default=False, type=types.BooleanType, choices=None, required=False,
390
                flag=None, sub_parameters=None, group="default", display_name=None):
391
392
393
394
395
396
397
398
399
400
401
402
403
        bool_default = False if default == None else bool(default)
        val = int.__new__(self, bool_default)
        val.is_None = False
        if default == None:
            val.is_None = True
            for attr in val.__dict__:
                value = getattr(val, attr)
                if callable(value) and attr not in ["__new__", "__init__", "__int__", "__getattribute__", "__eq__", "__ne__", "__nonzero__"]:
                    setattr(val, attr, noneException)
        return val

    def __init__(self, name, help, default=None, type=types.BooleanType, choices=None, required=False,
                 flag=None, sub_parameters=None, group="default", display_name=None):
Frédéric Escudié's avatar
Frédéric Escudié committed
404
        AbstractParameter.__init__(self, name, help, flag=flag, default=bool(default), type=type, choices=choices, required=required, 
405
                                   action="store", sub_parameters=sub_parameters, group=group, display_name=display_name)
406

Frédéric Escudié's avatar
Frédéric Escudié committed
407
408
409
410
411
    def __str__(self):
        if self.is_None:
            noneException()
        return str(bool(self))

412
413
    def __eq__(self, other):
        if other.__class__.__name__ == "NoneType":
414
415
416
417
            return self.is_None
        elif self.is_None:
            return False
        else:
Frédéric Escudié's avatar
Frédéric Escudié committed
418
            return int(self) == int(bool(other))
419

420
    def __ne__(self, other):
Frédéric Escudié's avatar
Frédéric Escudié committed
421
        return not (self == other)
422

423
    def __nonzero__(self):
424
        if self.is_None:
425
            return False
426
427
428
        else:
            return self != 0

429

430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
class IntParameter(int, AbstractParameter):

    def __new__(self, name, help, default=None, type=types.IntType, choices=None, required=False,
                flag=None, sub_parameters=None, group="default", display_name=None):
        int_default = 0 if default == None else int(default)
        val = int.__new__(self, int_default)
        val.is_None = False
        if default == None:
            val.is_None = True
            for attr in val.__dict__:
                value = getattr(val, attr)
                if callable(value) and attr not in ["__new__", "__init__", "__int__", "__getattribute__", "__eq__", "__ne__", "__nonzero__"]:
                    setattr(val, attr, noneException)
        return val

    def __init__(self, name, help, default=None, type=types.IntType, choices=None, required=False,
446
                 flag=None, sub_parameters=None, group="default", display_name=None):
Jerome Mariette's avatar
Jerome Mariette committed
447
        AbstractParameter.__init__(self, name, help, flag=flag, default=default, type=type, choices=choices, required=required, 
448
449
                                   action="store", sub_parameters=sub_parameters, group=group, display_name=display_name)

450
451
452
453
454
455
456
457
458
    def __eq__(self, other):
        if other.__class__.__name__ == "NoneType":
            return self.is_None
        elif self.is_None:
            return False
        else:
            return int(self) == int(other)

    def __ne__(self, other):
Frédéric Escudié's avatar
Frédéric Escudié committed
459
        return not (self == other)
460
461
462
463
464
465
466
467

    def __nonzero__(self):
        if self.is_None:
            return False
        else:
            return self != 0


Frédéric Escudié's avatar
Frédéric Escudié committed
468
class FloatParameter(float, AbstractParameter):
469
470
471
472

    def __new__(self, name, help, default=None, type=types.FloatType, choices=None, required=False,
                flag=None, sub_parameters=None, group="default", display_name=None):
        float_default = 0.0 if default == None else float(default)
Frédéric Escudié's avatar
Frédéric Escudié committed
473
        val = float.__new__(self, float_default)
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
        val.is_None = False
        if default == None:
            val.is_None = True
            for attr in val.__dict__:
                value = getattr(val, attr)
                if callable(value) and attr not in ["__new__", "__init__", "__float__", "__getattribute__", "__eq__", "__ne__", "__nonzero__"]:
                    setattr(val, attr, noneException)
        return val

    def __init__(self, name, help, default=None, type=types.FloatType, choices=None, required=False,
                 flag=None, sub_parameters=None, group="default", display_name=None):
        AbstractParameter.__init__(self, name, help, flag=flag, default=default, type=type, choices=choices, required=required, 
                                   action="store", sub_parameters=sub_parameters, group=group, display_name=display_name)

    def __eq__(self, other):
        if other.__class__.__name__ == "NoneType":
            return self.is_None
        elif self.is_None:
            return False
        else:
            return float(self) == float(other)

    def __ne__(self, other):
Frédéric Escudié's avatar
Frédéric Escudié committed
497
        return not (self == other)
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534

    def __nonzero__(self):
        if self.is_None:
            return False
        else:
            return self != 0.0


class StrParameter(str, AbstractParameter):

    def __new__(self, name, help, default=None, type=types.StringType, choices=None, required=False,
                flag=None, sub_parameters=None, group="default", display_name=None):
        str_default = "" if default == None else str(default)
        val = str.__new__(self, str_default)
        val.is_None = False
        if default == None:
            val.is_None = True
            for attr in val.__dict__:
                value = getattr(val, attr)
                if callable(value) and attr not in ["__new__", "__init__", "__str__", "__getattribute__", "__eq__", "__ne__", "__nonzero__"]:
                    setattr(val, attr, noneException)
        return val

    def __init__(self, name, help, default=None, type=types.StringType, choices=None, required=False,
                 flag=None, sub_parameters=None, group="default", display_name=None):
        AbstractParameter.__init__(self, name, help, flag=flag, default=default, type=type, choices=choices, required=required, 
                                   action="store", sub_parameters=sub_parameters, group=group, display_name=display_name)

    def __eq__(self, other):
        if other.__class__.__name__ == "NoneType":
            return self.is_None
        elif self.is_None:
            return False
        else:
            return str(self) == str(other)

    def __ne__(self, other):
Frédéric Escudié's avatar
Frédéric Escudié committed
535
        return not(self == other)
536
537
538
539
540
541
542
543

    def __nonzero__(self):
        if self.is_None:
            return False
        else:
            return (True if str(self) else False)


Frédéric Escudié's avatar
Frédéric Escudié committed
544
class DateParameter(datetime.datetime, AbstractParameter):
545

Frédéric Escudié's avatar
Frédéric Escudié committed
546
547
548
549
550
551
552
553
554
555
556
    def __new__(self, name, help, default=None, type=date, choices=None, required=False,
                flag=None, sub_parameters=None, group="default", display_name=None):
        date_default = datetime.datetime.today()
        if default != None and issubclass(default.__class__, datetime.datetime):
            date_default = default
        elif default != None:
            date_default = date(default)

        val = datetime.datetime.__new__(self, date_default.year, date_default.month, date_default.day)
        if default != None:
            val.is_None = False
557
        else:
Frédéric Escudié's avatar
Frédéric Escudié committed
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
            val.is_None = True
            for attr in val.__dict__:
                value = getattr(val, attr)
                if callable(value) and attr not in ["__new__", "__init__", "__getattribute__", "__eq__", "__ne__", "__nonzero__"]:
                    setattr(val, attr, noneException)
        return val

    def __init__(self, name, help, default=None, type=date, choices=None, required=False,
                 flag=None, sub_parameters=None, group="default", display_name=None):
        if default != None and not issubclass(default.__class__, datetime.datetime):
            date_default = date(default)
            default = datetime.datetime(date_default.year, date_default.month, date_default.day)
        AbstractParameter.__init__(self, name, help, flag=flag, default=default, type=type, choices=choices, required=required, 
                                   action="store", sub_parameters=sub_parameters, group=group, display_name=display_name)

    def __eq__(self, other):
        if other.__class__.__name__ == "NoneType":
            return self.is_None
        elif self.is_None:
            return False
        else:
            return datetime.date(self.year, self.month, self.day) == other

    def __ne__(self, other):
        return not( self == other )

    def __nonzero__(self):
        if self.is_None:
            return False
        else:
            return (True if datetime.date(self.year, self.month, self.day) else False)


class AbstractInputFile(AbstractIOFile):
    """
     @summary : Parent of all InputFile(s) parameters.
    """
Jerome Mariette's avatar
Jerome Mariette committed
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
    def _download_urlfile(self, input):
        try:
            uri_object = urlparse(input)
            opener = urllib2.urlopen(input)
            block_size = 8000
            jflowconf = JFlowConfigReader()
            tmp_directory = os.path.join(jflowconf.get_tmp_directory(), os.path.basename(tempfile.NamedTemporaryFile().name))
            os.mkdir(tmp_directory) 
            file_path = os.path.join(tmp_directory, os.path.basename(uri_object.path))
            local_file = open(file_path, 'wb')
            if os.path.basename(uri_object.path) is not None and os.path.basename(uri_object.path) != "":
                metadata = opener.info()
                file_size = int(metadata.getheaders("Content-Length")[0])
                while True:
                    buffer = opener.read(block_size)
                    # End of download
                    if not buffer: break
                    # Parts of download
                    local_file.write(buffer)
                local_file.close()
                logging.getLogger("AbstractInputFile.download_urlfile").debug("URL file '{0}' successfully downloaded as: {1}".format(input, file_path))
            return [file_path, True]
        except:
            return [input, False]
        
Jerome Mariette's avatar
Jerome Mariette committed
620
    def check(self, ifile):
Jerome Mariette's avatar
Jerome Mariette committed
621
622
623
624
        try:
            eval(self.file_format)
            function_exists = True
        except: function_exists = False
625
626
627
628
629
630
631
632
633
        if function_exists:
            try: 
                eval(self.file_format)(ifile)
            except jflow.InvalidFormatError as e:
                sys.stderr.write(str(e)+"\n")
                sys.exit(1)
        else:
            sys.stderr.write("Error: Invalid file format '" + self.file_format + "'!\n")
            sys.exit(1)
Frédéric Escudié's avatar
Frédéric Escudié committed
634
635
636
637
638
639
640

class AbstractOutputFile(AbstractIOFile):
    """
     @summary : Parent of all OutputFile(s) parameters.
    """
    pass

641
642
643

class InputFile(StrParameter, AbstractInputFile):

Jerome Mariette's avatar
Jerome Mariette committed
644
    def __new__(self, name, help, file_format="any", default="", type="localfile", choices=None, 
645
                required=False, flag=None, group="default", display_name=None):
646

Jerome Mariette's avatar
Jerome Mariette committed
647
648
649
        if hasattr(type, '__call__'):
            type2test = type.__name__
        else: type2test = type
650

Jerome Mariette's avatar
Jerome Mariette committed
651
652
653
        if type2test not in INPUTFILE_TYPES:
            raise ValueError("InputFile.__new__: wrong type provided: '"+type2test+"', this should be choosen between '" 
                             + "', '".join(INPUTFILE_TYPES)+"'")
654
655
656
657

        return StrParameter.__new__(self, name, help, flag=flag, default=default, type=type, choices=choices, 
                           required=required, group=group, display_name=display_name)

Jerome Mariette's avatar
Jerome Mariette committed
658
    def __init__(self, name, help, file_format="any", default="", type="localfile", choices=None, 
659
                required=False, flag=None, group="default", display_name=None):
Jerome Mariette's avatar
Jerome Mariette committed
660
        AbstractIOFile.__init__(self, file_format)
661
        StrParameter.__init__(self, name, help, flag=flag, default=default, type=type, choices=choices, 
662
                           required=required, group=group, display_name=display_name)
Jerome Mariette's avatar
Jerome Mariette committed
663

Jerome Mariette's avatar
Jerome Mariette committed
664
665
666
667
668
669
670
671
672
673
    def prepare_input_file(self, input):
        # handle url inputs
        new_path, is_uri = self._download_urlfile(input)
        # handle upload inputs
        try: is_local = os.path.isfile(input)
        except: is_local = False
        if not is_uri and not is_local and self.type.__name__ == "inputfile" or self.type.__name__ == "browsefile":
            jflow_config_reader = JFlowConfigReader()
            new_path = os.path.join(jflow_config_reader.get_tmp_directory(), input)
        if is_local: new_path = input
Jerome Mariette's avatar
Jerome Mariette committed
674
        self.check(new_path)
Jerome Mariette's avatar
Jerome Mariette committed
675
676
677
        return new_path
            

678
679
class OutputFile(StrParameter, AbstractOutputFile):

Frédéric Escudié's avatar
Frédéric Escudié committed
680
    def __new__(self, name, help, file_format="any", default="", choices=None,
681
                required=False, flag=None, group="default", display_name=None):
Frédéric Escudié's avatar
Frédéric Escudié committed
682
        return StrParameter.__new__(self, name, help, flag=flag, default=default, type="localfile", choices=choices,
683
684
                           required=required, group=group, display_name=display_name)

Frédéric Escudié's avatar
Frédéric Escudié committed
685
    def __init__(self, name, help, file_format="any", default="", choices=None,
686
                required=False, flag=None, group="default", display_name=None):
Jerome Mariette's avatar
Jerome Mariette committed
687
        AbstractIOFile.__init__(self, file_format)
688
        StrParameter.__init__(self, name, help, flag=flag, default=default, type="localfile", choices=choices, 
689
690
                           required=required, group=group, display_name=display_name)

691

692
class ParameterList(list, AbstractParameter):
693

Jerome Mariette's avatar
Jerome Mariette committed
694
    def __init__(self, name, help, default=None, type=types.StringType, choices=None, required=False,
695
                 flag=None, sub_parameters=None, group="default", display_name=None):
Jerome Mariette's avatar
Jerome Mariette committed
696
        if default == None: default = []
Frédéric Escudié's avatar
Frédéric Escudié committed
697
        AbstractParameter.__init__(self, name, help, flag=flag, default=default, type=type, choices=choices, required=required,
698
699
700
701
702
                                   action="append", sub_parameters=sub_parameters, group=group, display_name=display_name)
        if default.__class__.__name__ == "str":
            return list.__init__(self, [default])
        elif default.__class__.__name__ == "list":
            return list.__init__(self, default)
703

704

Frédéric Escudié's avatar
Frédéric Escudié committed
705
class InputFileList(ParameterList, AbstractInputFile):
706

Jerome Mariette's avatar
Jerome Mariette committed
707
    def __init__(self, name, help, file_format="any", default=None, type="localfile", choices=None, 
708
                 required=False, flag=None, group="default", display_name=None):
709

Jerome Mariette's avatar
Jerome Mariette committed
710
        if default == None: default = []   
Jerome Mariette's avatar
Jerome Mariette committed
711
712
713
        if hasattr(type, '__call__'):
            type2test = type.__name__
        else: type2test = type
714

Jerome Mariette's avatar
Jerome Mariette committed
715
716
717
        if type2test not in INPUTFILE_TYPES:
            raise ValueError("InputFile.__new__: wrong type provided: '"+type2test+"', this should be choosen between '" 
                             + "', '".join(INPUTFILE_TYPES)+"'")
718

Jerome Mariette's avatar
Jerome Mariette committed
719
        AbstractIOFile.__init__(self, file_format)
Jerome Mariette's avatar
Jerome Mariette committed
720
        ParameterList.__init__(self, name, help, flag=flag, default=default, type=type, choices=choices, 
721
                               required=required, group=group, display_name=display_name)
722

723
724
725
726
        if default.__class__.__name__ == "str":
            return list.__init__(self, [default])
        elif default.__class__.__name__ == "list":
            return list.__init__(self, default)
727
728
        elif issubclass( default.__class__, InputFile ):
            return list.__init__(self, [default])
Frédéric Escudié's avatar
Frédéric Escudié committed
729
        elif issubclass( default.__class__, AbstractInputFile ):
Jerome Mariette's avatar
Jerome Mariette committed
730
            return list.__init__(self, default)
731
732
        elif issubclass( default.__class__, OutputFile ):
            return list.__init__(self, [default])
Frédéric Escudié's avatar
Frédéric Escudié committed
733
        elif issubclass( default.__class__, AbstractOutputFile ):
Jerome Mariette's avatar
Jerome Mariette committed
734
            return list.__init__(self, default)
735

Jerome Mariette's avatar
Jerome Mariette committed
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
    def prepare_input_files(self, inputs):
        path2test = inputs
        if not path2test.__class__.__name__ == "list":
            path2test = [path2test]
        new_vals, done = [], []
        # handle url inputs
        for path in path2test:
            new_url, is_uri = self._download_urlfile(path)
            if is_uri:
                new_vals.append(new_url)
                done.append(path)
        for d in done: path2test.remove(d)
        done = []
        # handle localfile
        for path in path2test:
            if os.path.isfile(path):
                new_vals.append(path)
                done.append(path)
        for d in done: path2test.remove(d)        
        # handle upload inputs
        if self.type.__name__ == "inputfile" or self.type.__name__ == "browsefile":
            for path in path2test:
                jflow_config_reader = JFlowConfigReader()
                new_vals.append(os.path.join(jflow_config_reader.get_tmp_directory(), path))
Jerome Mariette's avatar
Jerome Mariette committed
760
761
762
        # now that all files are downloaded and ok, check the format
        for cfile in new_vals:
            self.check(cfile)
Jerome Mariette's avatar
Jerome Mariette committed
763
        return new_vals
764

Frédéric Escudié's avatar
Frédéric Escudié committed
765
766
class OutputFileList(ParameterList, AbstractOutputFile):

Jerome Mariette's avatar
Jerome Mariette committed
767
768
769
    def __init__(self, name, help, file_format="any", default=None, choices=None, 
                 required=False, flag=None, group="default", display_name=None):
        if default == None: default = []
Jerome Mariette's avatar
Jerome Mariette committed
770
        AbstractIOFile.__init__(self, file_format)
Jerome Mariette's avatar
Jerome Mariette committed
771
        ParameterList.__init__(self, name, help, flag=flag, default=default, type="localfile", choices=choices, 
772
773
774
775
776
                               required=required, group=group, display_name=display_name)
        if default.__class__.__name__ == "str":
            return list.__init__(self, [default])
        elif default.__class__.__name__ == "list":
            return list.__init__(self, default)
Frédéric Escudié's avatar
Frédéric Escudié committed
777

778

Frédéric Escudié's avatar
Frédéric Escudié committed
779
780
781
782
783
784
785
786
787
788
class DynamicOutput(ParameterList, AbstractOutputFile):
    """
     @warning : with this class of output, the component become dynamic.
    """
    def update(self):
        """
         This method is used at the end of component execution to update output list.
        """
        raise NotImplementedError

789

Frédéric Escudié's avatar
Frédéric Escudié committed
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
class OutputFilesEndsWith(DynamicOutput):

    def __init__(self, name, help, output_directory, end_str, include=True, file_format="any", choices=None, 
                 required=False, flag=None, group="default", display_name=None):
        """
         @warning : with this class of output, the component become dynamic.
         @param output_directory : path to the directory where outputs will be created.
         @param end_str : the end of the files names.
         @param include : if true, the files with name terminated by end_str are added into output files.
                          If false, the files with name not terminated by end_str are added into output files.
        """
        AbstractIOFile.__init__(self, file_format)
        default = []
        ParameterList.__init__(self, name, help, flag=flag, default=default, type="localfile", choices=choices, 
                               required=required, group=group, display_name=display_name)
        self.output_directory = output_directory
        self.end_str = end_str
        self.include = include
        return list.__init__(self, default)

    def update(self):
        output_files = list()
        for file in os.listdir( self.output_directory ):
            if file.endswith( self.end_str ) and self.include :
                output_files.append( os.path.join(self.output_directory, file) )
            elif not file.endswith( self.end_str ) and not self.include:
                output_files.append( os.path.join(self.output_directory, file) )
        list.__init__(self, output_files)

819

Frédéric Escudié's avatar
Frédéric Escudié committed
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
class OutputFilesPattern(DynamicOutput):

    def __init__(self, name, help, output_directory, pattern, include=True, file_format="any", choices=None, 
                  required=False, flag=None, group="default", display_name=None):
        """
         @warning : with this class of output, the component become dynamic.
         @param output_directory : path to the directory where outputs will be created.
         @param pattern : the pattern of (a part) the file names.
         @param include : if true, the files with the pattern in file name are added into output files.
                          If false, the files with the pattern in file name are added into output files.
        """
        AbstractIOFile.__init__(self, file_format)
        default = []
        ParameterList.__init__(self, name, help, flag=flag, default=default, type="localfile", choices=choices, 
                               required=required, group=group, display_name=display_name)
        self.output_directory = output_directory
        self.pattern = pattern
        self.include = include
        return list.__init__(self, default)

    def update(self):
        output_files = list()
        for file in os.listdir( self.output_directory ):
            if self.include and re.search( self.pattern, file ) is not None:
                output_files.append( os.path.join(self.output_directory, file) )
            elif not self.include and re.search( self.pattern, file ) is None:
                output_files.append( os.path.join(self.output_directory, file) )
        return list.__init__(self, output_files)