parameter.py 41.9 KB
Newer Older
Jerome Mariette's avatar
Jerome Mariette committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
#
# Copyright (C) 2012 INRA
# 
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
# 
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
# 
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
#

Frédéric Escudié's avatar
Frédéric Escudié committed
18
import re
19
import sys
Jerome Mariette's avatar
Jerome Mariette committed
20
import types
Jerome Mariette's avatar
Jerome Mariette committed
21
import datetime
Jerome Mariette's avatar
Jerome Mariette committed
22
import logging
Jerome Mariette's avatar
Jerome Mariette committed
23
import argparse
Jerome Mariette's avatar
Jerome Mariette committed
24
import os
25
import fnmatch
Jerome Mariette's avatar
Jerome Mariette committed
26
import tempfile
27
from argparse import _ensure_value
Jerome Mariette's avatar
Jerome Mariette committed
28
import urllib2
29
import copy as _copy
Jerome Mariette's avatar
Jerome Mariette committed
30
from urlparse import urlparse
31

Jerome Mariette's avatar
Jerome Mariette committed
32
from jflow.config_reader import JFlowConfigReader
33
from jflow.utils import get_octet_string_representation, get_nb_octet
Jerome Mariette's avatar
Jerome Mariette committed
34

Jerome Mariette's avatar
Jerome Mariette committed
35
# import custom types and custom formats
Jerome Mariette's avatar
Jerome Mariette committed
36
from workflows.types import *
Jerome Mariette's avatar
Jerome Mariette committed
37
from workflows.formats import *
Jerome Mariette's avatar
Jerome Mariette committed
38

Jerome Mariette's avatar
Jerome Mariette committed
39
40

# define all input type available
Frédéric Escudié's avatar
Frédéric Escudié committed
41
INPUTFILE_TYPES = ["inputfile", "localfile", "urlfile", "browsefile"]
Jerome Mariette's avatar
Jerome Mariette committed
42
43
44
45
46
47
48
49
50
51

def browsefile(file):
    # browsefile are not available from command line, considere it as a localfile
    # from the gui, this will not been tested this way
    return localfile(file)

def localfile(file):
    if os.path.isfile(file):
        return os.path.abspath(file)
    else:
Jerome Mariette's avatar
Jerome Mariette committed
52
        raise argparse.ArgumentTypeError("File '" + file + "' does not exists! Please provide a valid file path!")
53

Jerome Mariette's avatar
Jerome Mariette committed
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
def urlfile(file):
    uri_object = urlparse(file)
    try:
        opener = urllib2.urlopen(file)
    except:
        raise argparse.ArgumentTypeError("URL '" + file + "' is invalid!")
    file_name = os.path.basename(uri_object.path)
    if file_name is not None and file_name != "":
        metadata = opener.info()
        file_size = int(metadata.getheaders("Content-Length")[0])
        if file_size == 0:
            raise argparse.ArgumentTypeError("The URL file '" + file + "' is empty!")
        return file
    else:
        raise argparse.ArgumentTypeError("URL '" + file + "' does not contain any file name!")
69

Jerome Mariette's avatar
Jerome Mariette committed
70
71
72
73
74
def inputfile(file):
    # test the format
    uri_object = urlparse(file)
    # check the file
    if uri_object.scheme == '':
Frédéric Escudié's avatar
Frédéric Escudié committed
75
76
77
78
79
        try:
            regexpfiles(file)
            return file
        except:
            return localfile(file)
Jerome Mariette's avatar
Jerome Mariette committed
80
81
    else:
        return urlfile(file)
82

Frédéric Escudié's avatar
Frédéric Escudié committed
83
def regexpfiles(files_pattern):
84
85
86
87
88
89
90
    try:
        if ':' in files_pattern:
            folder, pattern = files_pattern.rsplit(':')
        else:
            folder, pattern = os.path.split(files_pattern)
    except:
        raise argparse.ArgumentTypeError("Pattern '" + file + "' is invalid!")
Frédéric Escudié's avatar
Frédéric Escudié committed
91
92
93
94
95
    if not os.path.exists(folder):
        raise argparse.ArgumentTypeError("The folder '" + folder + "' doesn't exist!")
    if not os.access(folder, os.R_OK):
        raise argparse.ArgumentTypeError("You do not have permission to read '" + folder + "'!")
    return files_pattern
96

97
98
def create_test_function(itype):
    try: itype = itype.encode('ascii','ignore')
Jerome Mariette's avatar
Jerome Mariette committed
99
    except: pass
100
    try:
101
        ctype, csizel = itype.split(AbstractInputFile.SIZE_LIMIT_SPLITER)
102
        def inner_function(ifile):
Jerome Mariette's avatar
Jerome Mariette committed
103
104
            # first eval the asked type
            returned_value = eval(ctype)(ifile)
105
106
107
108
109
            # if 0 unlimited size
            if csizel != "0":
                # first test the size of the file
                uri_object = urlparse(ifile)
                if uri_object.scheme == '':
Frédéric Escudié's avatar
Frédéric Escudié committed
110
111
112
113
114
115
116
117
118
119
120
121
                    isize = 0
                    try:
                        regexpfiles(ifile)
                        if ':' in returned_value:
                            folder, pattern = returned_value.rsplit(':')
                        else:
                            folder, pattern = os.path.split(returned_value)
                        for item in os.listdir(folder):
                            if os.path.isfile(os.path.join(folder, item)) and fnmatch.fnmatch(item, pattern):
                                isize += os.path.getsize(os.path.abspath(os.path.join(folder, item)))
                    except:
                        isize = os.path.getsize(ifile)
122
123
                    if isize > int(get_nb_octet(csizel)):
                        raise argparse.ArgumentTypeError("File '" + ifile + "' (size=" + get_octet_string_representation(isize) + ") exceeds size limits: " + csizel + ".")
124
125
                else:
                    try:
Frédéric Escudié's avatar
Frédéric Escudié committed
126
                        opener = urllib2.urlopen(ifile)
127
128
129
130
131
132
                        metadata = opener.info()
                        isize = int(metadata.getheaders("Content-Length")[0])
                        if isize > int(get_nb_octet(csizel)):
                            raise argparse.ArgumentTypeError("File '" + ifile + "' (size=" + get_octet_string_representation(isize) + ") exceeds size limits: " + csizel + ".")
                    except:
                        raise argparse.ArgumentTypeError("URL '" + file + "' is invalid!")
133
            # then test the type
Jerome Mariette's avatar
Jerome Mariette committed
134
            return returned_value
135
136
137
        inner_function.__name__ = ctype+AbstractInputFile.SIZE_LIMIT_SPLITER+csizel
        return inner_function
    except:
138
139
140
141
        if type(itype) == str:
            return eval(itype)
        else:
            return itype
142

Jerome Mariette's avatar
Jerome Mariette committed
143
class MultipleParameters(object):
144
    def __init__(self, types, required, choices, excludes, default, actions):
Jerome Mariette's avatar
Jerome Mariette committed
145
        self.types = types
146
        self.choices = choices
Jerome Mariette's avatar
Jerome Mariette committed
147
        self.excludes = excludes
148
        self.default = default
149
        self.actions = actions
Jerome Mariette's avatar
Jerome Mariette committed
150
        self.index = None
151
        self.required = required
Jerome Mariette's avatar
Jerome Mariette committed
152
        self.__name__ = "MultipleParameters"
153

Jerome Mariette's avatar
help ok    
Jerome Mariette committed
154
155
156
157
158
    def get_help(self):
        help = " ("
        for flag in self.types.keys():
            help += flag + "=<" + self.types[flag].__name__.upper() + ">, "
        return help[:-2] + ")"
159

160
    def __call__(self, arg):
Jerome Mariette's avatar
Jerome Mariette committed
161
162
163
        parts = arg.split("=")
        if not self.types.has_key(parts[0]):
            raise argparse.ArgumentTypeError(parts[0] + " is an invalid flag! Available ones are: "+", ".join(self.types.keys()))
164
165
166
        if self.choices[parts[0]] != None:
            if parts[1] not in self.choices[parts[0]]:
                raise argparse.ArgumentTypeError("argument " + parts[0] + ": invalid choice: '" + parts[1] + "' (choose from " + ", ".join(self.choices[parts[0]]) + "))")
167

168
169
170
171
172
173
174
        if self.types[parts[0]] == types.BooleanType:
            return (parts[0], not self.default[parts[0]], self.required, self.excludes)
        else:
            try:
                value = self.types[parts[0]](parts[1])
            except:
                raise argparse.ArgumentTypeError("invalid " + self.types[parts[0]].__name__ + " value: '" + parts[1] + "' for sub parameter '" + parts[0] + "'")
175

176
            self.index = parts[0]
177
            return (parts[0], value, self.required, self.excludes, self.actions)
Jerome Mariette's avatar
Jerome Mariette committed
178

179
180
class MiltipleAction(argparse.Action):
    def __call__(self, parser, namespace, values, option_string=None):
181
182
183
184
185
186
187
        # what is commun within required and excludes
        exclusif_required = {}
        for exclude_group in values[0][3]:
            exclusif_required[exclude_group] = False
            for param in values[0][3][exclude_group]:
                if param in values[0][2]:
                    exclusif_required[exclude_group] = True
Jerome Mariette's avatar
Jerome Mariette committed
188
189
        given_params = []
        # first check for required parameters
190
191
        try:
            required = _copy.copy(values[0][2])
192
193
194
195
196
197
            # delete required that are exclusive
            for group in exclusif_required:
                if exclusif_required[group]:
                    for val in values[0][3][group]:
                        if val in required:
                            del required[required.index(val)]
198
            for val in values:
Jerome Mariette's avatar
Jerome Mariette committed
199
                given_params.append(val[0])
200
201
202
203
204
                if val[0] in required:
                    del required[required.index(val[0])]
        except: pass
        if len(required) == 1: parser.error(", ".join(required) + " is a required parameter!")
        elif len(required) > 1: parser.error(", ".join(required) + " are required parameters!")
Jerome Mariette's avatar
Jerome Mariette committed
205
206
207
208
209
210
211
212
        # then for exclude ones    
        for exclude_group in values[0][3]:
            found = None
            for param in values[0][3][exclude_group]:
                if param in given_params and found != None:
                    parser.error("argument '" + found + "': not allowed with argument '" + param + "'")
                    break
                elif param in given_params: found = param
213

214
215
216
217
218
219
220
221
        # check for required exclusive if one of them is in
        if len(exclusif_required.keys()) > 0:
            for group in exclusif_required:
                if exclusif_required[group]:
                    rfound = False
                    for param in values[0][3][group]:
                        if param in given_params: rfound = True
            if not rfound: parser.error("one of the arguments: " + ", ".join(values[0][3][group]) + " is required")
222

Jerome Mariette's avatar
Jerome Mariette committed
223
        # if ok add the value
224
225
226
227
228
229
230
231
232
233
234
        final_hash, final_values = {}, []
        for value in values:
            if values[0][4][value[0]] == "append" and final_hash.has_key(value[0]):
                final_hash[value[0]].append(value[1])
            elif values[0][4][value[0]] == "append":
                final_hash[value[0]]= [value[1]]
            else:
                final_hash[value[0]]= value[1]
        for param in final_hash:
            final_values.append((param, final_hash[param]))
        setattr(namespace, self.dest, final_values)
235

236

237
238
class MiltipleAppendAction(argparse.Action):
    def __call__(self, parser, namespace, values, option_string=None):
239
240
241
242
243
244
245
        # what is commun within required and excludes
        exclusif_required = {}
        for exclude_group in values[0][3]:
            exclusif_required[exclude_group] = False
            for param in values[0][3][exclude_group]:
                if param in values[0][2]:
                    exclusif_required[exclude_group] = True
Jerome Mariette's avatar
Jerome Mariette committed
246
247
        given_params = []
        # first check for required parameters
248
249
        try:
            required = _copy.copy(values[0][2])
250
251
252
253
254
255
            # delete required that are exclusive
            for group in exclusif_required:
                if exclusif_required[group]:
                    for val in values[0][3][group]:
                        if val in required:
                            del required[required.index(val)]
256
            for val in values:
Jerome Mariette's avatar
Jerome Mariette committed
257
                given_params.append(val[0])
258
259
260
261
262
                if val[0] in required:
                    del required[required.index(val[0])]
        except: pass
        if len(required) == 1: parser.error(", ".join(required) + " is a required parameter!")
        elif len(required) > 1: parser.error(", ".join(required) + " are required parameters!")
Jerome Mariette's avatar
Jerome Mariette committed
263
264
265
266
267
268
269
270
        # then for exclude ones    
        for exclude_group in values[0][3]:
            found = None
            for param in values[0][3][exclude_group]:
                if param in given_params and found != None:
                    parser.error("argument '" + found + "': not allowed with argument '" + param + "'")
                    break
                elif param in given_params: found = param
271

272
273
274
275
276
277
278
279
        # check for required exclusive if one of them is in
        if len(exclusif_required.keys()) > 0:
            for group in exclusif_required:
                if exclusif_required[group]:
                    rfound = False
                    for param in values[0][3][group]:
                        if param in given_params: rfound = True
            if not rfound: parser.error("one of the arguments: " + ", ".join(values[0][3][group]) + " is required")
Jerome Mariette's avatar
Jerome Mariette committed
280
        # if ok add the value
281
        items = _copy.copy(_ensure_value(namespace, self.dest, []))
282
283
284
285
286
287
288
289
290
291
292
        final_hash, final_values = {}, []
        for value in values:
            if values[0][4][value[0]] == "append" and final_hash.has_key(value[0]):
                final_hash[value[0]].append(value[1])
            elif values[0][4][value[0]] == "append":
                final_hash[value[0]]= [value[1]]
            else:
                final_hash[value[0]]= value[1]
        for param in final_hash:
            final_values.append((param, final_hash[param]))
        items.append(final_values)
293
        setattr(namespace, self.dest, items)
294
295
296


class AbstractParameter(object):
297

298
299
    def __init__(self, name, help, default=None, type=types.StringType, choices=None, required=False,
                 flag=None, action="store", sub_parameters=None, group="default", display_name=None):
Jerome Mariette's avatar
Jerome Mariette committed
300

Jerome Mariette's avatar
Jerome Mariette committed
301
302
        self.name = name
        self.help = help
303
        self.action = action
Jerome Mariette's avatar
Jerome Mariette committed
304
        self.nargs = None
305
306
307
        if sub_parameters:
            self.sub_parameters = sub_parameters
        else: self.sub_parameters = []
Jerome Mariette's avatar
Jerome Mariette committed
308
        self.group = group
309
310
311
        if flag == None:
            self.flag = "--"+name.replace("_", "-")
        else: self.flag = flag
312
        if display_name == None:
313
            self.display_name = name.replace("_", " ").title()
Jerome Mariette's avatar
Jerome Mariette committed
314
        else: self.display_name = display_name
Jerome Mariette's avatar
help ok    
Jerome Mariette committed
315
        self.required = required
316
        self.choices = choices
317
318

        # Set parameter type
Jerome Mariette's avatar
Jerome Mariette committed
319
        if type == "date":
Frédéric Escudié's avatar
Frédéric Escudié committed
320
            self.type = date
Jerome Mariette's avatar
Jerome Mariette committed
321
        elif type == "multiple":
Jerome Mariette's avatar
help ok    
Jerome Mariette committed
322
            self.type = "multiple"
Jerome Mariette's avatar
Jerome Mariette committed
323
324
        elif isinstance(type, types.FunctionType):
            self.type = type
325
        elif type in [types.StringType, types.IntType, types.FloatType,  types.BooleanType]:
Jerome Mariette's avatar
Jerome Mariette committed
326
            self.type = type
Jerome Mariette's avatar
Jerome Mariette committed
327
328
329
        else:
            try: self.type = eval(type)
            except: self.type = types.StringType
330
331

        # Set parameter value
Jerome Mariette's avatar
Jerome Mariette committed
332
        self.default = default
333

334
    def export_to_argparse(self):
Jerome Mariette's avatar
Jerome Mariette committed
335
        if self.type == types.BooleanType and str(self.default).lower() in (False, "false",  "f", "0"):
336
337
338
            return {"help": self.help, "required": self.required, "dest": self.name, 
                    "default": False, "action": "store_true"}
        elif self.type == types.BooleanType:
339
            return {"help": self.help, "required": self.required, "dest": self.name, 
340
                    "default": True, "action": "store_false"}
Jerome Mariette's avatar
Jerome Mariette committed
341
        elif self.nargs > 1:
342
            return {"type": self.get_test_function(), "help": self.help, "required": self.required,
Jerome Mariette's avatar
Jerome Mariette committed
343
                    "dest": self.name, "default": self.default,
Jerome Mariette's avatar
Jerome Mariette committed
344
                    "action": self.action, "choices": self.choices, "nargs": "+"}
345
        else:
346
            return {"type": self.get_test_function(), "help": self.help, "required": self.required,
Jerome Mariette's avatar
Jerome Mariette committed
347
                    "dest": self.name, "default": self.default,
348
                    "action": self.action, "choices": self.choices}
349

Jerome Mariette's avatar
Jerome Mariette committed
350
351
    def get_type(self):
        return self.type.__name__
352

353
354
    def get_test_function(self):
        return create_test_function(self.type)
355

Jerome Mariette's avatar
Jerome Mariette committed
356
class AbstractIOFile(object):
357

Jerome Mariette's avatar
Jerome Mariette committed
358
359
    def __init__(self, file_format="any"):
        self.file_format = file_format
Jerome Mariette's avatar
Jerome Mariette committed
360
        self.component_nameid = None
361
        self.parent_component_nameid = []
Jerome Mariette's avatar
Jerome Mariette committed
362
363
364


class MultiParameter(dict, AbstractParameter):
Frédéric Escudié's avatar
Frédéric Escudié committed
365

Jerome Mariette's avatar
Jerome Mariette committed
366
367
368
369
370
    def __init__(self, name, help, required=False, flag=None, group="default", display_name=None):
        AbstractParameter.__init__(self, name, help, required=required, type="multiple", flag=flag, group=group, 
                                   display_name=display_name)
        return dict.__init__(self, {})

Jerome Mariette's avatar
help ok    
Jerome Mariette committed
371
372
373
    def add_sub_parameter(self, param):
        param_flag = param.flag[2:]
        if self.type == "multiple":
Jerome Mariette's avatar
Jerome Mariette committed
374
375
376
            if param.required: req = [param_flag] 
            else: req = []
            self.type = MultipleParameters({param_flag: param.type}, req, 
Jerome Mariette's avatar
help ok    
Jerome Mariette committed
377
378
379
380
381
382
383
384
                                           {param_flag: param.choices}, {}, {param_flag: param.default},
                                           {param_flag: param.action})
            if self.action == "append":
                self.action = MiltipleAppendAction
            else:
                self.action = MiltipleAction
            self.global_help = self.help
            self.help = self.global_help + " (" + param_flag + "=<" + param.type.__name__.upper() + ">)"
Jerome Mariette's avatar
Jerome Mariette committed
385
            self.default = {}
Jerome Mariette's avatar
Jerome Mariette committed
386
            self.nargs = "+"
Jerome Mariette's avatar
help ok    
Jerome Mariette committed
387
388
389
        elif self.type.__class__ == MultipleParameters:
            self.type.types[param_flag] = param.type
            self.type.choices[param_flag] = param.choices
390
            self.type.default[param_flag] = param.default if param != None else None
Jerome Mariette's avatar
help ok    
Jerome Mariette committed
391
            self.type.actions[param_flag] = param.action
Jerome Mariette's avatar
Jerome Mariette committed
392
393
            if param.required:
                self.type.required.append(param_flag)
Jerome Mariette's avatar
help ok    
Jerome Mariette committed
394
            self.help = self.global_help + self.type.get_help()
Jerome Mariette's avatar
Jerome Mariette committed
395
        param.flag = param_flag
Jerome Mariette's avatar
help ok    
Jerome Mariette committed
396
        self.sub_parameters.append(param)
397

398
399

class MultiParameterList(list, AbstractParameter):
Frédéric Escudié's avatar
Frédéric Escudié committed
400

401
    def __init__(self, name, help, required=False, flag=None, group="default", display_name=None):
Jerome Mariette's avatar
help ok    
Jerome Mariette committed
402
403
        AbstractParameter.__init__(self, name, help, required=required, type="multiple", flag=flag, 
                                   action="append", group=group, display_name=display_name)
404
        return list.__init__(self, [])
Jerome Mariette's avatar
Jerome Mariette committed
405

Jerome Mariette's avatar
Jerome Mariette committed
406
407
408
    def add_sub_parameter(self, param):
        param_flag = param.flag[2:]
        if self.type == "multiple":
Frédéric Escudié's avatar
Frédéric Escudié committed
409
            if param.required: req = [param_flag]
Jerome Mariette's avatar
Jerome Mariette committed
410
            else: req = []
Frédéric Escudié's avatar
Frédéric Escudié committed
411
            self.type = MultipleParameters({param_flag: param.type}, req,
Jerome Mariette's avatar
Jerome Mariette committed
412
413
414
415
416
417
418
419
420
421
422
423
424
                                           {param_flag: param.choices}, {}, {param_flag: param.default},
                                           {param_flag: param.action})
            if self.action == "append":
                self.action = MiltipleAppendAction
            else:
                self.action = MiltipleAction
            self.global_help = self.help
            self.help = self.global_help + " (" + param_flag + "=<" + param.type.__name__.upper() + ">)"
            self.default = []
            self.nargs = "+"
        elif self.type.__class__ == MultipleParameters:
            self.type.types[param_flag] = param.type
            self.type.choices[param_flag] = param.choices
425
            self.type.default[param_flag] = param.default if param != None else None
Jerome Mariette's avatar
Jerome Mariette committed
426
427
428
429
430
431
432
            self.type.actions[param_flag] = param.action
            if param.required:
                self.type.required.append(param_flag)
            self.help = self.global_help + self.type.get_help()
        param.flag = param_flag
        self.sub_parameters.append(param)

433

Frédéric Escudié's avatar
Frédéric Escudié committed
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
class ParameterFactory(object):
    @staticmethod
    def factory(*args, **kwargs):
        if not kwargs.has_key( "type" ):
            return StrParameter( *args, **kwargs )

        if kwargs["type"] == "int" or kwargs["type"] is types.IntType:
            return IntParameter( *args, **kwargs )
        elif kwargs["type"] == "bool" or kwargs["type"] is types.BooleanType:
            return BoolParameter( *args, **kwargs )
        elif kwargs["type"] == "float" or kwargs["type"] is types.FloatType:
            return FloatParameter( *args, **kwargs )
        elif kwargs["type"] == "date" or kwargs["type"] == date:
            return DateParameter( *args, **kwargs )
        else:
            return StrParameter( *args, **kwargs )


452
453
454
455
456
457
458
def none_decorator(fn):
    def new_func(*args, **kwargs):
        if args[0].is_None:
            raise Exception("The parameter '" + args[0].name + "' is None.")
        else:
            return fn(*args, **kwargs)
    return new_func
459
460
461
462


class BoolParameter(int, AbstractParameter):

Frédéric Escudié's avatar
Frédéric Escudié committed
463
    def __new__(self, name, help, default=False, type=types.BooleanType, choices=None, required=False,
464
                flag=None, sub_parameters=None, group="default", display_name=None):
465
466
        bool_default = False if default == None else bool(default)
        val = int.__new__(self, bool_default)
467
468
469
470
471
472
        val.is_None = False if default != None else True
        for attr in bool.__dict__:
            func = getattr(bool, attr)
            if callable(func) and attr not in ["__new__", "__init__", "__bool__", "__getattribute__", "__setattribute__",
                                               "__eq__", "__ne__", "__nonzero__", "__str__", "__repr__"]:
                setattr(BoolParameter, attr, none_decorator(func))
473
474
475
476
        return val

    def __init__(self, name, help, default=None, type=types.BooleanType, choices=None, required=False,
                 flag=None, sub_parameters=None, group="default", display_name=None):
Frédéric Escudié's avatar
Frédéric Escudié committed
477
        AbstractParameter.__init__(self, name, help, flag=flag, default=bool(default), type=type, choices=choices, required=required, 
478
                                   action="store", sub_parameters=sub_parameters, group=group, display_name=display_name)
479

Frédéric Escudié's avatar
Frédéric Escudié committed
480
481
    def __str__(self):
        if self.is_None:
482
            return str(None)
Frédéric Escudié's avatar
Frédéric Escudié committed
483
484
        return str(bool(self))

485
486
487
    def __repr__(self):
        return str(self)

488
489
    def __eq__(self, other):
        if other.__class__.__name__ == "NoneType":
490
491
492
493
            return self.is_None
        elif self.is_None:
            return False
        else:
Frédéric Escudié's avatar
Frédéric Escudié committed
494
            return int(self) == int(bool(other))
495

496
    def __ne__(self, other):
Frédéric Escudié's avatar
Frédéric Escudié committed
497
        return not (self == other)
498

499
    def __nonzero__(self):
500
        if self.is_None:
501
            return False
502
503
504
        else:
            return self != 0

505

506
507
508
509
510
511
class IntParameter(int, AbstractParameter):

    def __new__(self, name, help, default=None, type=types.IntType, choices=None, required=False,
                flag=None, sub_parameters=None, group="default", display_name=None):
        int_default = 0 if default == None else int(default)
        val = int.__new__(self, int_default)
512
513
514
515
516
517
        val.is_None = False if default != None else True
        for attr in int.__dict__:
            func = getattr(int, attr)
            if callable(func) and attr not in ["__new__", "__init__", "__int__", "__getattribute__", "__setattribute__",
                                               "__eq__", "__ne__", "__nonzero__", "__str__", "__repr__"]:
                setattr(IntParameter, attr, none_decorator(func))
518
519
520
        return val

    def __init__(self, name, help, default=None, type=types.IntType, choices=None, required=False,
521
                 flag=None, sub_parameters=None, group="default", display_name=None):
522
523
        AbstractParameter.__init__( self, name, help, flag=flag, default=default, type=type, choices=choices, required=required,
                                    action="store", sub_parameters=sub_parameters, group=group, display_name=display_name )
524

525
526
527
528
529
    def __str__(self):
        if self.is_None:
            return str(None)
        return str(int(self))

530
531
532
    def __repr__(self):
        return str(self)

533
534
535
536
537
538
539
540
541
    def __eq__(self, other):
        if other.__class__.__name__ == "NoneType":
            return self.is_None
        elif self.is_None:
            return False
        else:
            return int(self) == int(other)

    def __ne__(self, other):
Frédéric Escudié's avatar
Frédéric Escudié committed
542
        return not (self == other)
543
544
545
546
547
548
549
550

    def __nonzero__(self):
        if self.is_None:
            return False
        else:
            return self != 0


Frédéric Escudié's avatar
Frédéric Escudié committed
551
class FloatParameter(float, AbstractParameter):
552
553
554
555

    def __new__(self, name, help, default=None, type=types.FloatType, choices=None, required=False,
                flag=None, sub_parameters=None, group="default", display_name=None):
        float_default = 0.0 if default == None else float(default)
Frédéric Escudié's avatar
Frédéric Escudié committed
556
        val = float.__new__(self, float_default)
557
558
559
560
561
562
        val.is_None = False if default != None else True
        for attr in float.__dict__:
            func = getattr(float, attr)
            if callable(func) and attr not in ["__new__", "__init__", "__float__", "__getattribute__", "__setattribute__", 
                                               "__eq__", "__ne__", "__nonzero__", "__str__", "__repr__"]:
                setattr(FloatParameter, attr, none_decorator(func))
563
564
565
566
567
568
569
        return val

    def __init__(self, name, help, default=None, type=types.FloatType, choices=None, required=False,
                 flag=None, sub_parameters=None, group="default", display_name=None):
        AbstractParameter.__init__(self, name, help, flag=flag, default=default, type=type, choices=choices, required=required, 
                                   action="store", sub_parameters=sub_parameters, group=group, display_name=display_name)

570
571
572
573
574
    def __str__(self):
        if self.is_None:
            return str(None)
        return str(float(self))

575
576
577
    def __repr__(self):
        return str(self)

578
579
580
581
582
583
584
585
586
    def __eq__(self, other):
        if other.__class__.__name__ == "NoneType":
            return self.is_None
        elif self.is_None:
            return False
        else:
            return float(self) == float(other)

    def __ne__(self, other):
Frédéric Escudié's avatar
Frédéric Escudié committed
587
        return not (self == other)
588
589
590
591
592
593
594
595
596
597
598
599
600
601

    def __nonzero__(self):
        if self.is_None:
            return False
        else:
            return self != 0.0


class StrParameter(str, AbstractParameter):

    def __new__(self, name, help, default=None, type=types.StringType, choices=None, required=False,
                flag=None, sub_parameters=None, group="default", display_name=None):
        str_default = "" if default == None else str(default)
        val = str.__new__(self, str_default)
602
603
604
605
606
607
        val.is_None = False if default != None else True
        for attr in str.__dict__:
            func = getattr(str, attr)
            if callable(func) and attr not in ["__new__", "__init__", "__str__", "__getattribute__", "__setattribute__",
                                               "__eq__", "__ne__", "__nonzero__", "__repr__"]:
                setattr(StrParameter, attr, none_decorator(func))
608
609
610
611
612
613
614
        return val

    def __init__(self, name, help, default=None, type=types.StringType, choices=None, required=False,
                 flag=None, sub_parameters=None, group="default", display_name=None):
        AbstractParameter.__init__(self, name, help, flag=flag, default=default, type=type, choices=choices, required=required, 
                                   action="store", sub_parameters=sub_parameters, group=group, display_name=display_name)

615
616
617
618
619
    def __str__(self):
        if self.is_None:
            return str(None)
        return str.__str__(self)

620
621
622
623
624
    def __repr__(self):
        if self.is_None:
            return str(None)
        return "'" + str(self) + "'"

625
626
627
628
629
630
631
632
633
    def __eq__(self, other):
        if other.__class__.__name__ == "NoneType":
            return self.is_None
        elif self.is_None:
            return False
        else:
            return str(self) == str(other)

    def __ne__(self, other):
Frédéric Escudié's avatar
Frédéric Escudié committed
634
        return not(self == other)
635
636
637
638
639
640
641
642

    def __nonzero__(self):
        if self.is_None:
            return False
        else:
            return (True if str(self) else False)


Frédéric Escudié's avatar
Frédéric Escudié committed
643
class DateParameter(datetime.datetime, AbstractParameter):
644

Frédéric Escudié's avatar
Frédéric Escudié committed
645
646
647
648
649
650
651
652
    def __new__(self, name, help, default=None, type=date, choices=None, required=False,
                flag=None, sub_parameters=None, group="default", display_name=None):
        date_default = datetime.datetime.today()
        if default != None and issubclass(default.__class__, datetime.datetime):
            date_default = default
        elif default != None:
            date_default = date(default)
        val = datetime.datetime.__new__(self, date_default.year, date_default.month, date_default.day)
653
654
655
656
657
658
        val.is_None = False if default != None else True
        for attr in datetime.datetime.__dict__:
            func = getattr(datetime.datetime, attr)
            if callable(func) and attr not in ["__new__", "__init__", "__getattribute__", "__setattribute__",
                                               "__eq__", "__ne__", "__nonzero__", "__str__", "__repr__"]:
                setattr(DateParameter, attr, none_decorator(func))
Frédéric Escudié's avatar
Frédéric Escudié committed
659
660
661
662
663
664
665
666
667
668
        return val

    def __init__(self, name, help, default=None, type=date, choices=None, required=False,
                 flag=None, sub_parameters=None, group="default", display_name=None):
        if default != None and not issubclass(default.__class__, datetime.datetime):
            date_default = date(default)
            default = datetime.datetime(date_default.year, date_default.month, date_default.day)
        AbstractParameter.__init__(self, name, help, flag=flag, default=default, type=type, choices=choices, required=required, 
                                   action="store", sub_parameters=sub_parameters, group=group, display_name=display_name)

669
670
671
672
673
    def __str__(self):
        if self.is_None:
            return str(None)
        return datetime.datetime.__str__(self)

674
675
676
    def __repr__(self):
        return str(self)

Frédéric Escudié's avatar
Frédéric Escudié committed
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
    def __eq__(self, other):
        if other.__class__.__name__ == "NoneType":
            return self.is_None
        elif self.is_None:
            return False
        else:
            return datetime.date(self.year, self.month, self.day) == other

    def __ne__(self, other):
        return not( self == other )

    def __nonzero__(self):
        if self.is_None:
            return False
        else:
            return (True if datetime.date(self.year, self.month, self.day) else False)

694
695
696
697
698
699
700
701
    def __reduce__(self):
        return (DateParameter, (self.name, self.help, self.default, date, self.choices, self.required,
                 self.flag, self.sub_parameters, self.group, self.display_name), None, None, None)

    def __reduce_ex__(self, protocol):
        return (DateParameter, (self.name, self.help, self.default, date, self.choices, self.required,
                 self.flag, self.sub_parameters, self.group, self.display_name), None, None, None)

Frédéric Escudié's avatar
Frédéric Escudié committed
702
703
704
705
706

class AbstractInputFile(AbstractIOFile):
    """
     @summary : Parent of all InputFile(s) parameters.
    """
707
708
709
710
711
712
713
    
    SIZE_LIMIT_SPLITER = "__sl"
    
    def __init__(self, file_format="any", size_limit="0"):
        AbstractIOFile.__init__(self, file_format)
        self.size_limit = size_limit
    
Jerome Mariette's avatar
Jerome Mariette committed
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
    def _download_urlfile(self, input):
        try:
            uri_object = urlparse(input)
            opener = urllib2.urlopen(input)
            block_size = 8000
            jflowconf = JFlowConfigReader()
            tmp_directory = os.path.join(jflowconf.get_tmp_directory(), os.path.basename(tempfile.NamedTemporaryFile().name))
            os.mkdir(tmp_directory) 
            file_path = os.path.join(tmp_directory, os.path.basename(uri_object.path))
            local_file = open(file_path, 'wb')
            if os.path.basename(uri_object.path) is not None and os.path.basename(uri_object.path) != "":
                metadata = opener.info()
                file_size = int(metadata.getheaders("Content-Length")[0])
                while True:
                    buffer = opener.read(block_size)
                    # End of download
                    if not buffer: break
                    # Parts of download
                    local_file.write(buffer)
                local_file.close()
                logging.getLogger("AbstractInputFile.download_urlfile").debug("URL file '{0}' successfully downloaded as: {1}".format(input, file_path))
            return [file_path, True]
        except:
            return [input, False]
738

Jerome Mariette's avatar
Jerome Mariette committed
739
    def check(self, ifile):
Jerome Mariette's avatar
Jerome Mariette committed
740
741
742
743
        try:
            eval(self.file_format)
            function_exists = True
        except: function_exists = False
744
745
746
747
748
749
750
751
752
        if function_exists:
            try: 
                eval(self.file_format)(ifile)
            except jflow.InvalidFormatError as e:
                sys.stderr.write(str(e)+"\n")
                sys.exit(1)
        else:
            sys.stderr.write("Error: Invalid file format '" + self.file_format + "'!\n")
            sys.exit(1)
Frédéric Escudié's avatar
Frédéric Escudié committed
753
754
755
756
757
758
759

class AbstractOutputFile(AbstractIOFile):
    """
     @summary : Parent of all OutputFile(s) parameters.
    """
    pass

760
761
762

class InputFile(StrParameter, AbstractInputFile):

Jerome Mariette's avatar
Jerome Mariette committed
763
    def __new__(self, name, help, file_format="any", default="", type="localfile", choices=None, 
764
                required=False, flag=None, group="default", display_name=None, size_limit="0"):
Jerome Mariette's avatar
Jerome Mariette committed
765
766
767
        if hasattr(type, '__call__'):
            type2test = type.__name__
        else: type2test = type
768

Jerome Mariette's avatar
Jerome Mariette committed
769
770
771
        if type2test not in INPUTFILE_TYPES:
            raise ValueError("InputFile.__new__: wrong type provided: '"+type2test+"', this should be choosen between '" 
                             + "', '".join(INPUTFILE_TYPES)+"'")
772
773
774
775

        return StrParameter.__new__(self, name, help, flag=flag, default=default, type=type, choices=choices, 
                           required=required, group=group, display_name=display_name)

Jerome Mariette's avatar
Jerome Mariette committed
776
    def __init__(self, name, help, file_format="any", default="", type="localfile", choices=None, 
777
778
                required=False, flag=None, group="default", display_name=None, size_limit="0"):
        AbstractInputFile.__init__(self, file_format, size_limit)
779
        StrParameter.__init__(self, name, help, flag=flag, default=default, type=type, choices=choices, 
780
                           required=required, group=group, display_name=display_name)
Jerome Mariette's avatar
Jerome Mariette committed
781

Jerome Mariette's avatar
Jerome Mariette committed
782
783
784
    def get_type(self):
        return self.type.__name__+AbstractInputFile.SIZE_LIMIT_SPLITER+self.size_limit

785
786
    def get_test_function(self):
        if (self.size_limit == "0"): ctype = self.type
Jerome Mariette's avatar
Jerome Mariette committed
787
        else: ctype = self.get_type()
788
789
        return create_test_function(ctype)

Jerome Mariette's avatar
Jerome Mariette committed
790
    def prepare_input_file(self, input):
791
792
        if input == None:
            return None
Jerome Mariette's avatar
Jerome Mariette committed
793
794
795
796
797
798
799
800
801
        # handle url inputs
        new_path, is_uri = self._download_urlfile(input)
        # handle upload inputs
        try: is_local = os.path.isfile(input)
        except: is_local = False
        if not is_uri and not is_local and self.type.__name__ == "inputfile" or self.type.__name__ == "browsefile":
            jflow_config_reader = JFlowConfigReader()
            new_path = os.path.join(jflow_config_reader.get_tmp_directory(), input)
        if is_local: new_path = input
Jerome Mariette's avatar
Jerome Mariette committed
802
        self.check(new_path)
Jerome Mariette's avatar
Jerome Mariette committed
803
        return new_path
804

Jerome Mariette's avatar
Jerome Mariette committed
805

806
807
class OutputFile(StrParameter, AbstractOutputFile):

Frédéric Escudié's avatar
Frédéric Escudié committed
808
    def __new__(self, name, help, file_format="any", default="", choices=None,
809
                required=False, flag=None, group="default", display_name=None):
Frédéric Escudié's avatar
Frédéric Escudié committed
810
        return StrParameter.__new__(self, name, help, flag=flag, default=default, type="localfile", choices=choices,
811
812
                           required=required, group=group, display_name=display_name)

Frédéric Escudié's avatar
Frédéric Escudié committed
813
    def __init__(self, name, help, file_format="any", default="", choices=None,
814
                required=False, flag=None, group="default", display_name=None):
Jerome Mariette's avatar
Jerome Mariette committed
815
        AbstractIOFile.__init__(self, file_format)
816
        StrParameter.__init__(self, name, help, flag=flag, default=default, type="localfile", choices=choices, 
817
818
                           required=required, group=group, display_name=display_name)

819

820
class ParameterList(list, AbstractParameter):
821

Jerome Mariette's avatar
Jerome Mariette committed
822
    def __init__(self, name, help, default=None, type=types.StringType, choices=None, required=False,
823
                 flag=None, sub_parameters=None, group="default", display_name=None):
Jerome Mariette's avatar
Jerome Mariette committed
824
        if default == None: default = []
Frédéric Escudié's avatar
Frédéric Escudié committed
825
        AbstractParameter.__init__(self, name, help, flag=flag, default=default, type=type, choices=choices, required=required,
826
827
828
829
830
                                   action="append", sub_parameters=sub_parameters, group=group, display_name=display_name)
        if default.__class__.__name__ == "str":
            return list.__init__(self, [default])
        elif default.__class__.__name__ == "list":
            return list.__init__(self, default)
831

832

Frédéric Escudié's avatar
Frédéric Escudié committed
833
class InputFileList(ParameterList, AbstractInputFile):
834

Jerome Mariette's avatar
Jerome Mariette committed
835
    def __init__(self, name, help, file_format="any", default=None, type="localfile", choices=None, 
836
                 required=False, flag=None, group="default", display_name=None, size_limit="0"):
837

Jerome Mariette's avatar
Jerome Mariette committed
838
        if default == None: default = []   
Jerome Mariette's avatar
Jerome Mariette committed
839
840
841
        if hasattr(type, '__call__'):
            type2test = type.__name__
        else: type2test = type
842

Frédéric Escudié's avatar
Frédéric Escudié committed
843
        if type2test not in INPUTFILE_TYPES + ["regexpfiles"]:
Jerome Mariette's avatar
Jerome Mariette committed
844
845
            raise ValueError("InputFile.__new__: wrong type provided: '"+type2test+"', this should be choosen between '" 
                             + "', '".join(INPUTFILE_TYPES)+"'")
846

847
        AbstractInputFile.__init__(self, file_format, size_limit)
Jerome Mariette's avatar
Jerome Mariette committed
848
        ParameterList.__init__(self, name, help, flag=flag, default=default, type=type, choices=choices, 
849
                               required=required, group=group, display_name=display_name)
850

851
852
853
854
        if default.__class__.__name__ == "str":
            return list.__init__(self, [default])
        elif default.__class__.__name__ == "list":
            return list.__init__(self, default)
855
856
        elif issubclass( default.__class__, InputFile ):
            return list.__init__(self, [default])
Frédéric Escudié's avatar
Frédéric Escudié committed
857
        elif issubclass( default.__class__, AbstractInputFile ):
Jerome Mariette's avatar
Jerome Mariette committed
858
            return list.__init__(self, default)
859
860
        elif issubclass( default.__class__, OutputFile ):
            return list.__init__(self, [default])
Frédéric Escudié's avatar
Frédéric Escudié committed
861
        elif issubclass( default.__class__, AbstractOutputFile ):
Jerome Mariette's avatar
Jerome Mariette committed
862
            return list.__init__(self, default)
Jerome Mariette's avatar
Jerome Mariette committed
863
864
865
866
    
    def get_type(self):
        return self.type.__name__+AbstractInputFile.SIZE_LIMIT_SPLITER+self.size_limit
    
867
868
    def get_test_function(self):
        if (self.size_limit == "0"): ctype = self.type
Jerome Mariette's avatar
Jerome Mariette committed
869
        else: ctype = self.get_type()
870
871
        return create_test_function(ctype)

Jerome Mariette's avatar
Jerome Mariette committed
872
    def prepare_input_files(self, inputs):
Jerome Mariette's avatar
Jerome Mariette committed
873
        path2test = _copy.deepcopy(inputs)
Frédéric Escudié's avatar
Frédéric Escudié committed
874
        new_vals = list()
Jerome Mariette's avatar
Jerome Mariette committed
875
876
877
878
        if not path2test.__class__.__name__ == "list":
            path2test = [path2test]
        for path in path2test:
            new_url, is_uri = self._download_urlfile(path)
Frédéric Escudié's avatar
Frédéric Escudié committed
879
            if is_uri: # handle url inputs
Jerome Mariette's avatar
Jerome Mariette committed
880
                new_vals.append(new_url)
Frédéric Escudié's avatar
Frédéric Escudié committed
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
            elif os.path.isfile(path): # handle localfile
                new_vals.append(localfile(path))
            else:
                try: # handle regexp files
                    regexpfiles(path)
                    if ':' in path:
                        folder, pattern = path.rsplit(':')
                    else:
                        folder, pattern = os.path.split(path)
                    for item in sorted(os.listdir(folder)):
                        if os.path.isfile(os.path.join(folder, item)) and fnmatch.fnmatch(item, pattern):
                            new_vals.append( localfile(os.path.join(folder, item)) )
                except: # handle upload inputs
                    jflow_config_reader = JFlowConfigReader()
                    new_vals.append(os.path.join(jflow_config_reader.get_tmp_directory(), path))
Jerome Mariette's avatar
Jerome Mariette committed
896
897
898
        # now that all files are downloaded and ok, check the format
        for cfile in new_vals:
            self.check(cfile)
Jerome Mariette's avatar
Jerome Mariette committed
899
        return new_vals
900

Frédéric Escudié's avatar
Frédéric Escudié committed
901
902
class OutputFileList(ParameterList, AbstractOutputFile):

Jerome Mariette's avatar
Jerome Mariette committed
903
904
905
    def __init__(self, name, help, file_format="any", default=None, choices=None, 
                 required=False, flag=None, group="default", display_name=None):
        if default == None: default = []
Jerome Mariette's avatar
Jerome Mariette committed
906
        AbstractIOFile.__init__(self, file_format)
Jerome Mariette's avatar
Jerome Mariette committed
907
        ParameterList.__init__(self, name, help, flag=flag, default=default, type="localfile", choices=choices, 
908
909
910
911
912
                               required=required, group=group, display_name=display_name)
        if default.__class__.__name__ == "str":
            return list.__init__(self, [default])
        elif default.__class__.__name__ == "list":
            return list.__init__(self, default)
Frédéric Escudié's avatar
Frédéric Escudié committed
913

914

Frédéric Escudié's avatar
Frédéric Escudié committed
915
916
917
918
919
920
921
922
923
924
class DynamicOutput(ParameterList, AbstractOutputFile):
    """
     @warning : with this class of output, the component become dynamic.
    """
    def update(self):
        """
         This method is used at the end of component execution to update output list.
        """
        raise NotImplementedError

925

Frédéric Escudié's avatar
Frédéric Escudié committed
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
class OutputFilesEndsWith(DynamicOutput):

    def __init__(self, name, help, output_directory, end_str, include=True, file_format="any", choices=None, 
                 required=False, flag=None, group="default", display_name=None):
        """
         @warning : with this class of output, the component become dynamic.
         @param output_directory : path to the directory where outputs will be created.
         @param end_str : the end of the files names.
         @param include : if true, the files with name terminated by end_str are added into output files.
                          If false, the files with name not terminated by end_str are added into output files.
        """
        AbstractIOFile.__init__(self, file_format)
        default = []
        ParameterList.__init__(self, name, help, flag=flag, default=default, type="localfile", choices=choices, 
                               required=required, group=group, display_name=display_name)
        self.output_directory = output_directory
        self.end_str = end_str
        self.include = include
        return list.__init__(self, default)

    def update(self):
        output_files = list()
        for file in os.listdir( self.output_directory ):
            if file.endswith( self.end_str ) and self.include :
                output_files.append( os.path.join(self.output_directory, file) )
            elif not file.endswith( self.end_str ) and not self.include:
                output_files.append( os.path.join(self.output_directory, file) )
        list.__init__(self, output_files)

955

Frédéric Escudié's avatar
Frédéric Escudié committed
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
class OutputFilesPattern(DynamicOutput):

    def __init__(self, name, help, output_directory, pattern, include=True, file_format="any", choices=None, 
                  required=False, flag=None, group="default", display_name=None):
        """
         @warning : with this class of output, the component become dynamic.
         @param output_directory : path to the directory where outputs will be created.
         @param pattern : the pattern of (a part) the file names.
         @param include : if true, the files with the pattern in file name are added into output files.
                          If false, the files with the pattern in file name are added into output files.
        """
        AbstractIOFile.__init__(self, file_format)
        default = []
        ParameterList.__init__(self, name, help, flag=flag, default=default, type="localfile", choices=choices, 
                               required=required, group=group, display_name=display_name)
        self.output_directory = output_directory
        self.pattern = pattern
        self.include = include
        return list.__init__(self, default)

    def update(self):
        output_files = list()
        for file in os.listdir( self.output_directory ):
            if self.include and re.search( self.pattern, file ) is not None:
                output_files.append( os.path.join(self.output_directory, file) )
            elif not self.include and re.search( self.pattern, file ) is None:
                output_files.append( os.path.join(self.output_directory, file) )
        return list.__init__(self, output_files)