parameter.py 47.9 KB
Newer Older
1
#
Penom Nom's avatar
Penom Nom committed
2
# Copyright (C) 2015 INRA
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
# 
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
# 
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
# 
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
#

Penom Nom's avatar
Penom Nom committed
18
19
import re
import sys
20
import types
Jerome Mariette's avatar
Jerome Mariette committed
21
import datetime
Penom Nom's avatar
Penom Nom committed
22
import logging
23
import argparse
Penom Nom's avatar
Penom Nom committed
24
import os
25
import fnmatch
Penom Nom's avatar
Penom Nom committed
26
import tempfile
Jerome Mariette's avatar
Jerome Mariette committed
27
from argparse import _ensure_value
Penom Nom's avatar
Penom Nom committed
28
import urllib2
Jerome Mariette's avatar
Jerome Mariette committed
29
import copy as _copy
Penom Nom's avatar
Penom Nom committed
30
31
32
from urlparse import urlparse

from jflow.config_reader import JFlowConfigReader
33
from jflow.utils import get_octet_string_representation, get_nb_octet, display_error_message
Penom Nom's avatar
Penom Nom committed
34
35

# import custom types and custom formats
Jerome Mariette's avatar
Jerome Mariette committed
36
from workflows.types import *
Penom Nom's avatar
Penom Nom committed
37
38
39
40
41
from workflows.formats import *


# define all input type available
INPUTFILE_TYPES = ["inputfile", "localfile", "urlfile", "browsefile"]
42
INPUTFILES_TYPES = ["inputfiles", "localfile", "urlfile", "browsefile", "regexpfiles"]
Penom Nom's avatar
Penom Nom committed
43

Penom Nom's avatar
Penom Nom committed
44
45
46
47
48
49
def inputdirectory(directory):
    if os.path.isdir(directory):
        return directory
    else:
        raise argparse.ArgumentTypeError("'" + directory + "' is not a valid directory!")

Penom Nom's avatar
Penom Nom committed
50
51
52
53
54
55
56
def browsefile(file):
    # browsefile are not available from command line, considere it as a localfile
    # from the gui, this will not been tested this way
    return localfile(file)

def localfile(file):
    if os.path.isfile(file):
57
        return file
Penom Nom's avatar
Penom Nom committed
58
    else:
Penom Nom's avatar
Penom Nom committed
59
        raise argparse.ArgumentTypeError("'" + file + "' is not a valid file!")
60

Penom Nom's avatar
Penom Nom committed
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
def urlfile(file):
    uri_object = urlparse(file)
    try:
        opener = urllib2.urlopen(file)
    except:
        raise argparse.ArgumentTypeError("URL '" + file + "' is invalid!")
    file_name = os.path.basename(uri_object.path)
    if file_name is not None and file_name != "":
        metadata = opener.info()
        file_size = int(metadata.getheaders("Content-Length")[0])
        if file_size == 0:
            raise argparse.ArgumentTypeError("The URL file '" + file + "' is empty!")
        return file
    else:
        raise argparse.ArgumentTypeError("URL '" + file + "' does not contain any file name!")

def inputfile(file):
    # test the format
    uri_object = urlparse(file)
    # check the file
    if uri_object.scheme == '':
        return localfile(file)
    else:
        return urlfile(file)

86
87
88
89
90
91
92
93
94
95
96
97
98
def inputfiles(file):
    # test the format
    uri_object = urlparse(file)
    # check the file
    if uri_object.scheme == '':
        try:
            regexpfiles(file)
            return file
        except:
            return localfile(file)
    else:
        return urlfile(file)

99
100
101
102
103
104
105
def regexpfiles(files_pattern):
    try:
        if ':' in files_pattern:
            folder, pattern = files_pattern.rsplit(':')
        else:
            folder, pattern = os.path.split(files_pattern)
    except:
106
107
108
        raise argparse.ArgumentTypeError("Regexp '" + files_pattern + "' is invalid!")
    if not pattern:
        raise argparse.ArgumentTypeError("Regexp '" + files_pattern + "' is invalid!")
109
110
111
112
    if not os.path.exists(folder):
        raise argparse.ArgumentTypeError("The folder '" + folder + "' doesn't exist!")
    if not os.access(folder, os.R_OK):
        raise argparse.ArgumentTypeError("You do not have permission to read '" + folder + "'!")
113
114
115
116
117
118
    nb_files = 0
    for item in sorted(os.listdir(folder)):
        if os.path.isfile(os.path.join(folder, item)) and fnmatch.fnmatch(item, pattern):
            nb_files += 1
    if nb_files == 0:
        raise argparse.ArgumentTypeError("0 file selected by the regexp!")
119
120
    return files_pattern

Penom Nom's avatar
Penom Nom committed
121
122
123
124
125
126
127
128
129
130
131
132
133
def create_test_function(itype):
    try: itype = itype.encode('ascii','ignore')
    except: pass
    try:
        ctype, csizel = itype.split(AbstractInputFile.SIZE_LIMIT_SPLITER)
        def inner_function(ifile):
            # first eval the asked type
            returned_value = eval(ctype)(ifile)
            # if 0 unlimited size
            if csizel != "0":
                # first test the size of the file
                uri_object = urlparse(ifile)
                if uri_object.scheme == '':
134
135
136
137
138
139
140
141
142
143
144
145
                    isize = 0
                    try:
                        regexpfiles(ifile)
                        if ':' in returned_value:
                            folder, pattern = returned_value.rsplit(':')
                        else:
                            folder, pattern = os.path.split(returned_value)
                        for item in os.listdir(folder):
                            if os.path.isfile(os.path.join(folder, item)) and fnmatch.fnmatch(item, pattern):
                                isize += os.path.getsize(os.path.abspath(os.path.join(folder, item)))
                    except:
                        isize = os.path.getsize(ifile)
Penom Nom's avatar
Penom Nom committed
146
147
148
149
                    if isize > int(get_nb_octet(csizel)):
                        raise argparse.ArgumentTypeError("File '" + ifile + "' (size=" + get_octet_string_representation(isize) + ") exceeds size limits: " + csizel + ".")
                else:
                    try:
150
                        opener = urllib2.urlopen(ifile)
Penom Nom's avatar
Penom Nom committed
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
                        metadata = opener.info()
                        isize = int(metadata.getheaders("Content-Length")[0])
                        if isize > int(get_nb_octet(csizel)):
                            raise argparse.ArgumentTypeError("File '" + ifile + "' (size=" + get_octet_string_representation(isize) + ") exceeds size limits: " + csizel + ".")
                    except:
                        raise argparse.ArgumentTypeError("URL '" + file + "' is invalid!")
            # then test the type
            return returned_value
        inner_function.__name__ = ctype+AbstractInputFile.SIZE_LIMIT_SPLITER+csizel
        return inner_function
    except:
        if type(itype) == str:
            return eval(itype)
        else:
            return itype
166

Celine Noirot's avatar
Celine Noirot committed
167
class MultipleParameters(object):
168
    def __init__(self, types, required, choices, excludes, default, actions):
Jerome Mariette's avatar
Jerome Mariette committed
169
        self.types = types
170
        self.choices = choices
171
        self.excludes = excludes
172
        self.default = default
173
        self.actions = actions
174
        self.index = None
Jerome Mariette's avatar
Jerome Mariette committed
175
        self.required = required
Celine Noirot's avatar
Celine Noirot committed
176
        self.__name__ = "MultipleParameters"
Penom Nom's avatar
Penom Nom committed
177
178
179
180
181
182
183

    def get_help(self):
        help = " ("
        for flag in self.types.keys():
            help += flag + "=<" + self.types[flag].__name__.upper() + ">, "
        return help[:-2] + ")"

184
    def __call__(self, arg):
185
186
187
        parts = arg.split("=")
        if not self.types.has_key(parts[0]):
            raise argparse.ArgumentTypeError(parts[0] + " is an invalid flag! Available ones are: "+", ".join(self.types.keys()))
Penom Nom's avatar
Penom Nom committed
188

189
190
191
192
193
194
195
        if self.types[parts[0]] == types.BooleanType:
            return (parts[0], not self.default[parts[0]], self.required, self.excludes)
        else:
            try:
                value = self.types[parts[0]](parts[1])
            except:
                raise argparse.ArgumentTypeError("invalid " + self.types[parts[0]].__name__ + " value: '" + parts[1] + "' for sub parameter '" + parts[0] + "'")
Penom Nom's avatar
Penom Nom committed
196
197
198
199
            
            if self.choices[parts[0]] != None:
                if value not in self.choices[parts[0]]:
                    raise argparse.ArgumentTypeError("argument " + parts[0] + ": invalid choice: '" + parts[1] + "' (choose from " + ", ".join(map(str,self.choices[parts[0]])) + ")")
Penom Nom's avatar
Penom Nom committed
200

201
            self.index = parts[0]
202
            return (parts[0], value, self.required, self.excludes, self.actions)
Jerome Mariette's avatar
Jerome Mariette committed
203

Jerome Mariette's avatar
Jerome Mariette committed
204
205
class MiltipleAction(argparse.Action):
    def __call__(self, parser, namespace, values, option_string=None):
206
207
208
209
210
211
212
        # what is commun within required and excludes
        exclusif_required = {}
        for exclude_group in values[0][3]:
            exclusif_required[exclude_group] = False
            for param in values[0][3][exclude_group]:
                if param in values[0][2]:
                    exclusif_required[exclude_group] = True
213
214
        given_params = []
        # first check for required parameters
Jerome Mariette's avatar
Jerome Mariette committed
215
216
        try:
            required = _copy.copy(values[0][2])
217
218
219
220
221
222
            # delete required that are exclusive
            for group in exclusif_required:
                if exclusif_required[group]:
                    for val in values[0][3][group]:
                        if val in required:
                            del required[required.index(val)]
Jerome Mariette's avatar
Jerome Mariette committed
223
            for val in values:
224
                given_params.append(val[0])
Jerome Mariette's avatar
Jerome Mariette committed
225
226
227
228
229
                if val[0] in required:
                    del required[required.index(val[0])]
        except: pass
        if len(required) == 1: parser.error(", ".join(required) + " is a required parameter!")
        elif len(required) > 1: parser.error(", ".join(required) + " are required parameters!")
230
231
232
233
234
235
236
237
        # then for exclude ones    
        for exclude_group in values[0][3]:
            found = None
            for param in values[0][3][exclude_group]:
                if param in given_params and found != None:
                    parser.error("argument '" + found + "': not allowed with argument '" + param + "'")
                    break
                elif param in given_params: found = param
Penom Nom's avatar
Penom Nom committed
238

239
240
241
242
243
244
245
246
        # check for required exclusive if one of them is in
        if len(exclusif_required.keys()) > 0:
            for group in exclusif_required:
                if exclusif_required[group]:
                    rfound = False
                    for param in values[0][3][group]:
                        if param in given_params: rfound = True
            if not rfound: parser.error("one of the arguments: " + ", ".join(values[0][3][group]) + " is required")
Penom Nom's avatar
Penom Nom committed
247

248
        # if ok add the value
249
250
251
252
253
254
255
256
257
258
259
        final_hash, final_values = {}, []
        for value in values:
            if values[0][4][value[0]] == "append" and final_hash.has_key(value[0]):
                final_hash[value[0]].append(value[1])
            elif values[0][4][value[0]] == "append":
                final_hash[value[0]]= [value[1]]
            else:
                final_hash[value[0]]= value[1]
        for param in final_hash:
            final_values.append((param, final_hash[param]))
        setattr(namespace, self.dest, final_values)
Jerome Mariette's avatar
Jerome Mariette committed
260

Penom Nom's avatar
Penom Nom committed
261

Jerome Mariette's avatar
Jerome Mariette committed
262
263
class MiltipleAppendAction(argparse.Action):
    def __call__(self, parser, namespace, values, option_string=None):
264
265
266
267
268
269
270
        # what is commun within required and excludes
        exclusif_required = {}
        for exclude_group in values[0][3]:
            exclusif_required[exclude_group] = False
            for param in values[0][3][exclude_group]:
                if param in values[0][2]:
                    exclusif_required[exclude_group] = True
271
272
        given_params = []
        # first check for required parameters
Jerome Mariette's avatar
Jerome Mariette committed
273
274
        try:
            required = _copy.copy(values[0][2])
275
276
277
278
279
280
            # delete required that are exclusive
            for group in exclusif_required:
                if exclusif_required[group]:
                    for val in values[0][3][group]:
                        if val in required:
                            del required[required.index(val)]
Jerome Mariette's avatar
Jerome Mariette committed
281
            for val in values:
282
                given_params.append(val[0])
Jerome Mariette's avatar
Jerome Mariette committed
283
284
285
286
287
                if val[0] in required:
                    del required[required.index(val[0])]
        except: pass
        if len(required) == 1: parser.error(", ".join(required) + " is a required parameter!")
        elif len(required) > 1: parser.error(", ".join(required) + " are required parameters!")
288
289
290
291
292
293
294
295
        # then for exclude ones    
        for exclude_group in values[0][3]:
            found = None
            for param in values[0][3][exclude_group]:
                if param in given_params and found != None:
                    parser.error("argument '" + found + "': not allowed with argument '" + param + "'")
                    break
                elif param in given_params: found = param
Penom Nom's avatar
Penom Nom committed
296

297
298
299
300
301
302
303
304
        # check for required exclusive if one of them is in
        if len(exclusif_required.keys()) > 0:
            for group in exclusif_required:
                if exclusif_required[group]:
                    rfound = False
                    for param in values[0][3][group]:
                        if param in given_params: rfound = True
            if not rfound: parser.error("one of the arguments: " + ", ".join(values[0][3][group]) + " is required")
305
        # if ok add the value
Jerome Mariette's avatar
Jerome Mariette committed
306
        items = _copy.copy(_ensure_value(namespace, self.dest, []))
307
308
309
310
311
312
313
314
315
316
317
        final_hash, final_values = {}, []
        for value in values:
            if values[0][4][value[0]] == "append" and final_hash.has_key(value[0]):
                final_hash[value[0]].append(value[1])
            elif values[0][4][value[0]] == "append":
                final_hash[value[0]]= [value[1]]
            else:
                final_hash[value[0]]= value[1]
        for param in final_hash:
            final_values.append((param, final_hash[param]))
        items.append(final_values)
Penom Nom's avatar
Penom Nom committed
318
319
320
321
322
323
        setattr(namespace, self.dest, items)


class AbstractParameter(object):

    def __init__(self, name, help, default=None, type=types.StringType, choices=None, required=False,
324
325
                 flag=None, action="store", sub_parameters=None, group="default", display_name=None, 
                 cmd_format="", argpos=-1):
Jerome Mariette's avatar
Jerome Mariette committed
326

327
328
        self.name = name
        self.help = help
329
        self.action = action
330
        self.nargs = None
Penom Nom's avatar
Penom Nom committed
331
332
333
        if sub_parameters:
            self.sub_parameters = sub_parameters
        else: self.sub_parameters = []
Jerome Mariette's avatar
Jerome Mariette committed
334
        self.group = group
Penom Nom's avatar
Penom Nom committed
335
336
337
        if flag == None:
            self.flag = "--"+name.replace("_", "-")
        else: self.flag = flag
Jerome Mariette's avatar
Jerome Mariette committed
338
        if display_name == None:
Penom Nom's avatar
Penom Nom committed
339
            self.display_name = name.replace("_", " ").title()
Jerome Mariette's avatar
Jerome Mariette committed
340
        else: self.display_name = display_name
Penom Nom's avatar
Penom Nom committed
341
        self.required = required
342
        self.choices = choices
343
344
345
        self.argpos = argpos
        self.cmd_format = cmd_format
        
Penom Nom's avatar
Penom Nom committed
346
        # Set parameter type
Jerome Mariette's avatar
Jerome Mariette committed
347
348
        if type == "date":
            self.type = date
Jerome Mariette's avatar
Jerome Mariette committed
349
        elif type == "multiple":
Penom Nom's avatar
Penom Nom committed
350
            self.type = "multiple"
Jerome Mariette's avatar
Jerome Mariette committed
351
352
        elif isinstance(type, types.FunctionType):
            self.type = type
Penom Nom's avatar
Penom Nom committed
353
354
        elif type in [types.StringType, types.IntType, types.FloatType,  types.BooleanType]:
            self.type = type
Jerome Mariette's avatar
Jerome Mariette committed
355
356
357
        else:
            try: self.type = eval(type)
            except: self.type = types.StringType
Penom Nom's avatar
Penom Nom committed
358
359

        # Set parameter value
360
361
362
363
        if choices != None and default == None:
            self.default = choices[0]
        else:
            self.default = default
Penom Nom's avatar
Penom Nom committed
364

365
    def export_to_argparse(self):
Penom Nom's avatar
Penom Nom committed
366
        if self.type == types.BooleanType and str(self.default).lower() in (False, "false",  "f", "0"):
367
368
369
            return {"help": self.help, "required": self.required, "dest": self.name, 
                    "default": False, "action": "store_true"}
        elif self.type == types.BooleanType:
370
            return {"help": self.help, "required": self.required, "dest": self.name, 
371
                    "default": True, "action": "store_false"}
Jerome Mariette's avatar
Jerome Mariette committed
372
        elif self.nargs > 1:
Penom Nom's avatar
Penom Nom committed
373
            return {"type": self.get_test_function(), "help": self.help, "required": self.required,
Jerome Mariette's avatar
Jerome Mariette committed
374
                    "dest": self.name, "default": self.default,
375
                    "action": self.action, "choices": self.choices, "nargs": "+"}
376
        else:
Penom Nom's avatar
Penom Nom committed
377
            return {"type": self.get_test_function(), "help": self.help, "required": self.required,
Jerome Mariette's avatar
Jerome Mariette committed
378
                    "dest": self.name, "default": self.default,
379
                    "action": self.action, "choices": self.choices}
Penom Nom's avatar
Penom Nom committed
380

Jerome Mariette's avatar
Jerome Mariette committed
381
382
    def get_type(self):
        return self.type.__name__
Penom Nom's avatar
Penom Nom committed
383
384
385
386
387

    def get_test_function(self):
        return create_test_function(self.type)


388
389
390
391
392
393
394
395
396
397
class LinkTraceback(object):

    def __init__(self, linkTrace_nameid=None, parent_linkTrace_nameid=None):
        self.linkTrace_nameid = linkTrace_nameid
        self.parent_linkTrace_nameid = [] if parent_linkTrace_nameid == None else parent_linkTrace_nameid

class AbstractIOFile(LinkTraceback):

    def __init__(self, file_format="any", linkTrace_nameid=None, parent_linkTrace_nameid=None):
        LinkTraceback.__init__(self, linkTrace_nameid, parent_linkTrace_nameid)
Penom Nom's avatar
Penom Nom committed
398
        self.file_format = file_format
399
400
401
402
403
404
405
406
407


class IOFile(str, AbstractIOFile):

    def __new__(self, val="", file_format="any", linkTrace_nameid=None, parent_linkTrace_nameid=None):
        return str.__new__(self, val)

    def __init__(self, val="", file_format="any", linkTrace_nameid=None, parent_linkTrace_nameid=None):
        AbstractIOFile.__init__(self, file_format, linkTrace_nameid, parent_linkTrace_nameid)
Penom Nom's avatar
Penom Nom committed
408
409
410
411


class MultiParameter(dict, AbstractParameter):

412
    def __init__(self, name, help, required=False, flag=None, group="default", display_name=None, cmd_format="", argpos=-1):
Penom Nom's avatar
Penom Nom committed
413
        AbstractParameter.__init__(self, name, help, required=required, type="multiple", flag=flag, group=group, 
414
                                   display_name=display_name, cmd_format=cmd_format, argpos=argpos)
Penom Nom's avatar
Penom Nom committed
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
        return dict.__init__(self, {})

    def add_sub_parameter(self, param):
        param_flag = param.flag[2:]
        if self.type == "multiple":
            if param.required: req = [param_flag] 
            else: req = []
            self.type = MultipleParameters({param_flag: param.type}, req, 
                                           {param_flag: param.choices}, {}, {param_flag: param.default},
                                           {param_flag: param.action})
            if self.action == "append":
                self.action = MiltipleAppendAction
            else:
                self.action = MiltipleAction
            self.global_help = self.help
            self.help = self.global_help + " (" + param_flag + "=<" + param.type.__name__.upper() + ">)"
            self.default = {}
            self.nargs = "+"
        elif self.type.__class__ == MultipleParameters:
            self.type.types[param_flag] = param.type
            self.type.choices[param_flag] = param.choices
            self.type.default[param_flag] = param.default if param != None else None
            self.type.actions[param_flag] = param.action
            if param.required:
                self.type.required.append(param_flag)
            self.help = self.global_help + self.type.get_help()
        param.flag = param_flag
        self.sub_parameters.append(param)


class MultiParameterList(list, AbstractParameter):

447
    def __init__(self, name, help, required=False, flag=None, group="default", display_name=None, cmd_format="", argpos=-1):
Penom Nom's avatar
Penom Nom committed
448
        AbstractParameter.__init__(self, name, help, required=required, type="multiple", flag=flag, 
449
450
                                   action="append", group=group, display_name=display_name, 
                                   cmd_format=cmd_format, argpos=argpos)
Penom Nom's avatar
Penom Nom committed
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
        return list.__init__(self, [])

    def add_sub_parameter(self, param):
        param_flag = param.flag[2:]
        if self.type == "multiple":
            if param.required: req = [param_flag]
            else: req = []
            self.type = MultipleParameters({param_flag: param.type}, req,
                                           {param_flag: param.choices}, {}, {param_flag: param.default},
                                           {param_flag: param.action})
            if self.action == "append":
                self.action = MiltipleAppendAction
            else:
                self.action = MiltipleAction
            self.global_help = self.help
            self.help = self.global_help + " (" + param_flag + "=<" + param.type.__name__.upper() + ">)"
            self.default = []
            self.nargs = "+"
        elif self.type.__class__ == MultipleParameters:
            self.type.types[param_flag] = param.type
            self.type.choices[param_flag] = param.choices
            self.type.default[param_flag] = param.default if param != None else None
            self.type.actions[param_flag] = param.action
            if param.required:
                self.type.required.append(param_flag)
            self.help = self.global_help + self.type.get_help()
        param.flag = param_flag
        self.sub_parameters.append(param)


class ParameterFactory(object):
    @staticmethod
    def factory(*args, **kwargs):
        if not kwargs.has_key( "type" ):
            return StrParameter( *args, **kwargs )

        if kwargs["type"] == "int" or kwargs["type"] is types.IntType:
            return IntParameter( *args, **kwargs )
        elif kwargs["type"] == "bool" or kwargs["type"] is types.BooleanType:
            return BoolParameter( *args, **kwargs )
        elif kwargs["type"] == "float" or kwargs["type"] is types.FloatType:
            return FloatParameter( *args, **kwargs )
        elif kwargs["type"] == "date" or kwargs["type"] == date:
            return DateParameter( *args, **kwargs )
        else:
            return StrParameter( *args, **kwargs )


def none_decorator(fn):
    def new_func(*args, **kwargs):
        if args[0].is_None:
            raise Exception("The parameter '" + args[0].name + "' is None.")
        else:
            return fn(*args, **kwargs)
    return new_func


class BoolParameter(int, AbstractParameter):

    def __new__(self, name, help, default=False, type=types.BooleanType, choices=None, required=False,
511
                flag=None, sub_parameters=None, group="default", display_name=None,  cmd_format="", argpos=-1):
512
        
513
        bool_default = True
514
        if default == None or default in [False, 0]:
515
            bool_default = False
516
517
518
        elif default.__class__.__name__ == "str" and default in ["False", "F", "false", "f", 0, "0"]:
            bool_default = False
            
Penom Nom's avatar
Penom Nom committed
519
520
521
522
523
524
525
526
527
528
        val = int.__new__(self, bool_default)
        val.is_None = False if default != None else True
        for attr in bool.__dict__:
            func = getattr(bool, attr)
            if callable(func) and attr not in ["__new__", "__init__", "__bool__", "__getattribute__", "__setattribute__",
                                               "__eq__", "__ne__", "__nonzero__", "__str__", "__repr__"]:
                setattr(BoolParameter, attr, none_decorator(func))
        return val

    def __init__(self, name, help, default=None, type=types.BooleanType, choices=None, required=False,
529
                 flag=None, sub_parameters=None, group="default", display_name=None, cmd_format="", argpos=-1):
Penom Nom's avatar
Penom Nom committed
530
        AbstractParameter.__init__(self, name, help, flag=flag, default=bool(default), type=type, choices=choices, required=required, 
531
                                   action="store", sub_parameters=sub_parameters, group=group, display_name=display_name,  cmd_format=cmd_format, argpos=argpos)
Penom Nom's avatar
Penom Nom committed
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561

    def __str__(self):
        if self.is_None:
            return str(None)
        return str(bool(self))

    def __repr__(self):
        return str(self)

    def __eq__(self, other):
        if other.__class__.__name__ == "NoneType":
            return self.is_None
        elif self.is_None:
            return False
        else:
            return int(self) == int(bool(other))

    def __ne__(self, other):
        return not (self == other)

    def __nonzero__(self):
        if self.is_None:
            return False
        else:
            return self != 0


class IntParameter(int, AbstractParameter):

    def __new__(self, name, help, default=None, type=types.IntType, choices=None, required=False,
562
                flag=None, sub_parameters=None, group="default", display_name=None,  cmd_format="", argpos=-1):
Penom Nom's avatar
Penom Nom committed
563
564
565
566
567
568
569
570
571
572
573
        int_default = 0 if default == None else int(default)
        val = int.__new__(self, int_default)
        val.is_None = False if default != None else True
        for attr in int.__dict__:
            func = getattr(int, attr)
            if callable(func) and attr not in ["__new__", "__init__", "__int__", "__getattribute__", "__setattribute__",
                                               "__eq__", "__ne__", "__nonzero__", "__str__", "__repr__"]:
                setattr(IntParameter, attr, none_decorator(func))
        return val

    def __init__(self, name, help, default=None, type=types.IntType, choices=None, required=False,
574
                 flag=None, sub_parameters=None, group="default", display_name=None, cmd_format="", argpos=-1):
Penom Nom's avatar
Penom Nom committed
575
        AbstractParameter.__init__( self, name, help, flag=flag, default=default, type=type, choices=choices, required=required,
576
                                    action="store", sub_parameters=sub_parameters, group=group, display_name=display_name,  cmd_format=cmd_format, argpos=argpos)
Penom Nom's avatar
Penom Nom committed
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606

    def __str__(self):
        if self.is_None:
            return str(None)
        return str(int(self))

    def __repr__(self):
        return str(self)

    def __eq__(self, other):
        if other.__class__.__name__ == "NoneType":
            return self.is_None
        elif self.is_None:
            return False
        else:
            return int(self) == int(other)

    def __ne__(self, other):
        return not (self == other)

    def __nonzero__(self):
        if self.is_None:
            return False
        else:
            return self != 0


class FloatParameter(float, AbstractParameter):

    def __new__(self, name, help, default=None, type=types.FloatType, choices=None, required=False,
607
                flag=None, sub_parameters=None, group="default", display_name=None, cmd_format="", argpos=-1):
Penom Nom's avatar
Penom Nom committed
608
609
610
611
612
613
614
615
616
617
618
        float_default = 0.0 if default == None else float(default)
        val = float.__new__(self, float_default)
        val.is_None = False if default != None else True
        for attr in float.__dict__:
            func = getattr(float, attr)
            if callable(func) and attr not in ["__new__", "__init__", "__float__", "__getattribute__", "__setattribute__", 
                                               "__eq__", "__ne__", "__nonzero__", "__str__", "__repr__"]:
                setattr(FloatParameter, attr, none_decorator(func))
        return val

    def __init__(self, name, help, default=None, type=types.FloatType, choices=None, required=False,
619
                 flag=None, sub_parameters=None, group="default", display_name=None, cmd_format="", argpos=-1):
Penom Nom's avatar
Penom Nom committed
620
        AbstractParameter.__init__(self, name, help, flag=flag, default=default, type=type, choices=choices, required=required, 
621
                                   action="store", sub_parameters=sub_parameters, group=group, display_name=display_name,cmd_format=cmd_format, argpos=argpos )
Penom Nom's avatar
Penom Nom committed
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651

    def __str__(self):
        if self.is_None:
            return str(None)
        return str(float(self))

    def __repr__(self):
        return str(self)

    def __eq__(self, other):
        if other.__class__.__name__ == "NoneType":
            return self.is_None
        elif self.is_None:
            return False
        else:
            return float(self) == float(other)

    def __ne__(self, other):
        return not (self == other)

    def __nonzero__(self):
        if self.is_None:
            return False
        else:
            return self != 0.0


class StrParameter(str, AbstractParameter):

    def __new__(self, name, help, default=None, type=types.StringType, choices=None, required=False,
652
                flag=None, sub_parameters=None, group="default", display_name=None, cmd_format="", argpos=-1):
Penom Nom's avatar
Penom Nom committed
653
654
655
656
657
658
659
660
661
662
663
        str_default = "" if default == None else str(default)
        val = str.__new__(self, str_default)
        val.is_None = False if default != None else True
        for attr in str.__dict__:
            func = getattr(str, attr)
            if callable(func) and attr not in ["__new__", "__init__", "__str__", "__getattribute__", "__setattribute__",
                                               "__eq__", "__ne__", "__nonzero__", "__repr__"]:
                setattr(StrParameter, attr, none_decorator(func))
        return val

    def __init__(self, name, help, default=None, type=types.StringType, choices=None, required=False,
664
                 flag=None, sub_parameters=None, group="default", display_name=None, cmd_format="", argpos=-1):
Penom Nom's avatar
Penom Nom committed
665
        AbstractParameter.__init__(self, name, help, flag=flag, default=default, type=type, choices=choices, required=required, 
666
                                   action="store", sub_parameters=sub_parameters, group=group, display_name=display_name, cmd_format=cmd_format, argpos=argpos)
Penom Nom's avatar
Penom Nom committed
667

668
    def __str__(self):
Penom Nom's avatar
Penom Nom committed
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
        if self.is_None:
            return str(None)
        return str.__str__(self)

    def __repr__(self):
        if self.is_None:
            return str(None)
        return "'" + str(self) + "'"

    def __eq__(self, other):
        if other.__class__.__name__ == "NoneType":
            return self.is_None
        elif self.is_None:
            return False
        else:
            return str(self) == str(other)

    def __ne__(self, other):
        return not(self == other)

    def __nonzero__(self):
        if self.is_None:
            return False
        else:
            return (True if str(self) else False)


class DateParameter(datetime.datetime, AbstractParameter):

    def __new__(self, name, help, default=None, type=date, choices=None, required=False,
699
                flag=None, sub_parameters=None, group="default", display_name=None, cmd_format="", argpos=-1):
Penom Nom's avatar
Penom Nom committed
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
        date_default = datetime.datetime.today()
        if default != None and issubclass(default.__class__, datetime.datetime):
            date_default = default
        elif default != None:
            date_default = date(default)
        val = datetime.datetime.__new__(self, date_default.year, date_default.month, date_default.day)
        val.is_None = False if default != None else True
        for attr in datetime.datetime.__dict__:
            func = getattr(datetime.datetime, attr)
            if callable(func) and attr not in ["__new__", "__init__", "__getattribute__", "__setattribute__",
                                               "__eq__", "__ne__", "__nonzero__", "__str__", "__repr__"]:
                setattr(DateParameter, attr, none_decorator(func))
        return val

    def __init__(self, name, help, default=None, type=date, choices=None, required=False,
715
                 flag=None, sub_parameters=None, group="default", display_name=None, cmd_format="", argpos=-1):
Penom Nom's avatar
Penom Nom committed
716
717
718
719
        if default != None and not issubclass(default.__class__, datetime.datetime):
            date_default = date(default)
            default = datetime.datetime(date_default.year, date_default.month, date_default.day)
        AbstractParameter.__init__(self, name, help, flag=flag, default=default, type=type, choices=choices, required=required, 
720
                                   action="store", sub_parameters=sub_parameters, group=group, display_name=display_name, cmd_format=cmd_format, argpos=argpos)
Penom Nom's avatar
Penom Nom committed
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754

    def __str__(self):
        if self.is_None:
            return str(None)
        return datetime.datetime.__str__(self)

    def __repr__(self):
        return str(self)

    def __eq__(self, other):
        if other.__class__.__name__ == "NoneType":
            return self.is_None
        elif self.is_None:
            return False
        else:
            return datetime.date(self.year, self.month, self.day) == other

    def __ne__(self, other):
        return not( self == other )

    def __nonzero__(self):
        if self.is_None:
            return False
        else:
            return (True if datetime.date(self.year, self.month, self.day) else False)

    def __reduce__(self):
        return (DateParameter, (self.name, self.help, self.default, date, self.choices, self.required,
                 self.flag, self.sub_parameters, self.group, self.display_name), None, None, None)

    def __reduce_ex__(self, protocol):
        return (DateParameter, (self.name, self.help, self.default, date, self.choices, self.required,
                 self.flag, self.sub_parameters, self.group, self.display_name), None, None, None)

Penom Nom's avatar
Penom Nom committed
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
def input_directory_get_files_fn(input):
    return os.listdir(input)

class InputDirectory(StrParameter, LinkTraceback):

    def __new__(self, name, help, default="", choices=None, required=False, flag=None, 
                group="default", display_name=None, get_files_fn=None, cmd_format="", argpos=-1):
        return StrParameter.__new__(self, name, help, flag=flag, default=default, type="inputdirectory", choices=choices, 
                           required=required, group=group, display_name=display_name, cmd_format=cmd_format, argpos=argpos)

    def __init__(self, name, help, default="", choices=None, required=False, flag=None, 
                group="default", display_name=None, get_files_fn=None, cmd_format="", argpos=-1):
        LinkTraceback.__init__(self)
        StrParameter.__init__(self, name, help, flag=flag, default=default, type="inputdirectory", choices=choices, 
                           required=required, group=group, display_name=display_name, cmd_format=cmd_format, argpos=argpos)
        if hasattr(get_files_fn, "__call__"):
            self.get_files_fn = get_files_fn
        else:
            self.get_files_fn = input_directory_get_files_fn
Penom Nom's avatar
Penom Nom committed
774

Penom Nom's avatar
Penom Nom committed
775
776
777
778
779
780
781
782
783
784
785
    def prepare(self, input):
        if input == None:
            return None
        return os.path.abspath(input)
    
    def get_files(self, *args):
        files = []
        for file in self.get_files_fn(self, *args):
            files.append( IOFile(os.path.join(self, file), "any", self.linkTrace_nameid, None) )
        return files
    
Penom Nom's avatar
Penom Nom committed
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
class AbstractInputFile(AbstractIOFile):
    """
     @summary : Parent of all InputFile(s) parameters.
    """
    
    SIZE_LIMIT_SPLITER = "__sl"
    
    def __init__(self, file_format="any", size_limit="0"):
        AbstractIOFile.__init__(self, file_format)
        self.size_limit = size_limit
    
    def _download_urlfile(self, input):
        try:
            uri_object = urlparse(input)
            opener = urllib2.urlopen(input)
            block_size = 8000
            jflowconf = JFlowConfigReader()
            tmp_directory = os.path.join(jflowconf.get_tmp_directory(), os.path.basename(tempfile.NamedTemporaryFile().name))
            os.mkdir(tmp_directory) 
            file_path = os.path.join(tmp_directory, os.path.basename(uri_object.path))
            if os.path.basename(uri_object.path) is not None and os.path.basename(uri_object.path) != "":
807
                local_file = open(file_path, 'wb')
Penom Nom's avatar
Penom Nom committed
808
809
810
811
812
813
814
815
816
                metadata = opener.info()
                file_size = int(metadata.getheaders("Content-Length")[0])
                while True:
                    buffer = opener.read(block_size)
                    # End of download
                    if not buffer: break
                    # Parts of download
                    local_file.write(buffer)
                local_file.close()
817
                logging.getLogger("jflow").debug("URL file '{0}' successfully downloaded as: {1}".format(input, file_path))
Penom Nom's avatar
Penom Nom committed
818
819
820
821
822
823
824
825
826
827
828
829
830
            return [file_path, True]
        except:
            return [input, False]

    def check(self, ifile):
        try:
            eval(self.file_format)
            function_exists = True
        except: function_exists = False
        if function_exists:
            try: 
                eval(self.file_format)(ifile)
            except jflow.InvalidFormatError as e:
831
                raise Exception (str(e))
Penom Nom's avatar
Penom Nom committed
832
        else:
833
            raise Exception("Invalid file format '" + self.file_format + "'!")
Penom Nom's avatar
Penom Nom committed
834
835
836
837
838
839
840
841
842
843
844

class AbstractOutputFile(AbstractIOFile):
    """
     @summary : Parent of all OutputFile(s) parameters.
    """
    pass


class InputFile(StrParameter, AbstractInputFile):

    def __new__(self, name, help, file_format="any", default="", type="localfile", choices=None, 
845
                required=False, flag=None, group="default", display_name=None, size_limit="0",  cmd_format="", argpos=-1):
Penom Nom's avatar
Penom Nom committed
846
847
848
849
850
851
852
853
854
        if hasattr(type, '__call__'):
            type2test = type.__name__
        else: type2test = type

        if type2test not in INPUTFILE_TYPES:
            raise ValueError("InputFile.__new__: wrong type provided: '"+type2test+"', this should be choosen between '" 
                             + "', '".join(INPUTFILE_TYPES)+"'")

        return StrParameter.__new__(self, name, help, flag=flag, default=default, type=type, choices=choices, 
855
                           required=required, group=group, display_name=display_name, cmd_format=cmd_format, argpos=argpos)
Penom Nom's avatar
Penom Nom committed
856
857

    def __init__(self, name, help, file_format="any", default="", type="localfile", choices=None, 
858
                required=False, flag=None, group="default", display_name=None, size_limit="0",  cmd_format="", argpos=-1):
Penom Nom's avatar
Penom Nom committed
859
        AbstractInputFile.__init__(self, file_format, size_limit)
Penom Nom's avatar
Penom Nom committed
860
861
        if issubclass(default.__class__, list):
            raise Exception( "The parameter '" + name + "' cannot be set with a list." )
Penom Nom's avatar
Penom Nom committed
862
        StrParameter.__init__(self, name, help, flag=flag, default=default, type=type, choices=choices, 
863
                           required=required, group=group, display_name=display_name, cmd_format=cmd_format, argpos=argpos)
Penom Nom's avatar
Penom Nom committed
864
865
866
867
868
869
870
871
872

    def get_type(self):
        return self.type.__name__+AbstractInputFile.SIZE_LIMIT_SPLITER+self.size_limit

    def get_test_function(self):
        if (self.size_limit == "0"): ctype = self.type
        else: ctype = self.get_type()
        return create_test_function(ctype)

Penom Nom's avatar
Penom Nom committed
873
    def prepare(self, input):
Penom Nom's avatar
Penom Nom committed
874
875
876
877
878
879
880
881
882
883
        if input == None:
            return None
        # handle url inputs
        new_path, is_uri = self._download_urlfile(input)
        # handle upload inputs
        try: is_local = os.path.isfile(input)
        except: is_local = False
        if not is_uri and not is_local and self.type.__name__ == "inputfile" or self.type.__name__ == "browsefile":
            jflow_config_reader = JFlowConfigReader()
            new_path = os.path.join(jflow_config_reader.get_tmp_directory(), input)
884
        if is_local: new_path = os.path.abspath(input)
Penom Nom's avatar
Penom Nom committed
885
886
887
888
889
890
891
        self.check(new_path)
        return new_path


class OutputFile(StrParameter, AbstractOutputFile):

    def __new__(self, name, help, file_format="any", default="", choices=None,
892
893
                required=False, flag=None, group="default", display_name=None, 
                cmd_format="", argpos=-1):
Penom Nom's avatar
Penom Nom committed
894
        return StrParameter.__new__(self, name, help, flag=flag, default=default, type="localfile", choices=choices,
895
                           required=required, group=group, display_name=display_name, cmd_format=cmd_format, argpos=argpos)
Penom Nom's avatar
Penom Nom committed
896
897

    def __init__(self, name, help, file_format="any", default="", choices=None,
898
899
                required=False, flag=None, group="default", display_name=None,
                cmd_format="", argpos=-1):
Penom Nom's avatar
Penom Nom committed
900
        AbstractIOFile.__init__(self, file_format)
Penom Nom's avatar
Penom Nom committed
901
902
        if issubclass(default.__class__, list):
            raise Exception( "The parameter '" + name + "' cannot be set with a list." )
Penom Nom's avatar
Penom Nom committed
903
        StrParameter.__init__(self, name, help, flag=flag, default=default, type="localfile", choices=choices, 
904
                           required=required, group=group, display_name=display_name, cmd_format=cmd_format, argpos=argpos)
Penom Nom's avatar
Penom Nom committed
905
906
907
908
909


class ParameterList(list, AbstractParameter):

    def __init__(self, name, help, default=None, type=types.StringType, choices=None, required=False,
910
911
                 flag=None, sub_parameters=None, group="default", display_name=None,
                  cmd_format="", argpos=-1):
Penom Nom's avatar
Penom Nom committed
912
913
        if default == None: default = []
        AbstractParameter.__init__(self, name, help, flag=flag, default=default, type=type, choices=choices, required=required,
914
915
                                   action="append", sub_parameters=sub_parameters, group=group, display_name=display_name,
                                   cmd_format=cmd_format, argpos=argpos)
Penom Nom's avatar
Penom Nom committed
916
        liste = []
Penom Nom's avatar
Penom Nom committed
917
        if isinstance( default, list ):
Penom Nom's avatar
Penom Nom committed
918
919
920
921
922
923
924
925
            for val in default :
                liste.append(ParameterFactory.factory( name, help, default=val, type=type, choices=choices, 
                      required=required, flag=flag, group=group, display_name=display_name ))
        else :
            liste.append(ParameterFactory.factory( name, help, default=default, type=type, choices=choices, 
                      required=required, flag=flag, group=group, display_name=display_name ))
        return list.__init__(self, liste)
        
926
927
928
929
930
931
    def append(self, item):
        raise TypeError('A parameter is immutable.')

    def extend(self, items):
        raise TypeError('A parameter is immutable.')

Penom Nom's avatar
Penom Nom committed
932
933
934
935

class InputFileList(ParameterList, AbstractInputFile):

    def __init__(self, name, help, file_format="any", default=None, type="localfile", choices=None, 
936
937
                 required=False, flag=None, group="default", display_name=None, size_limit="0",
                 cmd_format="", argpos=-1):
Penom Nom's avatar
Penom Nom committed
938

939
        if default == None: default = []
Penom Nom's avatar
Penom Nom committed
940
        if hasattr(type, '__call__'):
941
942
            if type.__name__ == "inputfile":
                type = inputfiles
Penom Nom's avatar
Penom Nom committed
943
            type2test = type.__name__
944
945
946
947
948
        else:
            if type == "inputfile":
                type = "inputfiles"
            type2test = type
        if type2test not in INPUTFILES_TYPES:
Penom Nom's avatar
Penom Nom committed
949
950
951
952
953
            raise ValueError("InputFile.__new__: wrong type provided: '"+type2test+"', this should be choosen between '" 
                             + "', '".join(INPUTFILE_TYPES)+"'")

        AbstractInputFile.__init__(self, file_format, size_limit)
        ParameterList.__init__(self, name, help, flag=flag, default=default, type=type, choices=choices, 
954
955
                               required=required, group=group, display_name=display_name,
                               cmd_format=cmd_format, argpos=argpos)
Penom Nom's avatar
Penom Nom committed
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977

        if default.__class__.__name__ == "str":
            return list.__init__(self, [default])
        elif default.__class__.__name__ == "list":
            return list.__init__(self, default)
        elif issubclass( default.__class__, InputFile ):
            return list.__init__(self, [default])
        elif issubclass( default.__class__, AbstractInputFile ):
            return list.__init__(self, default)
        elif issubclass( default.__class__, OutputFile ):
            return list.__init__(self, [default])
        elif issubclass( default.__class__, AbstractOutputFile ):
            return list.__init__(self, default)
    
    def get_type(self):
        return self.type.__name__+AbstractInputFile.SIZE_LIMIT_SPLITER+self.size_limit
    
    def get_test_function(self):
        if (self.size_limit == "0"): ctype = self.type
        else: ctype = self.get_type()
        return create_test_function(ctype)

Penom Nom's avatar
Penom Nom committed
978
    def prepare(self, inputs):
Penom Nom's avatar
Penom Nom committed
979
        path2test = _copy.deepcopy(inputs)
980
        new_vals = list()
Penom Nom's avatar
Penom Nom committed
981
982
983
984
        if not path2test.__class__.__name__ == "list":
            path2test = [path2test]
        for path in path2test:
            new_url, is_uri = self._download_urlfile(path)
985
            if is_uri: # handle url inputs
Penom Nom's avatar
Penom Nom committed
986
                new_vals.append(new_url)
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
            elif os.path.isfile(path): # handle localfile
                new_vals.append(os.path.abspath(path))
            else:
                try: # handle regexp files
                    regexpfiles(path)
                    if ':' in path:
                        folder, pattern = path.rsplit(':')
                    else:
                        folder, pattern = os.path.split(path)
                    for item in sorted(os.listdir(folder)):
                        if os.path.isfile(os.path.join(folder, item)) and fnmatch.fnmatch(item, pattern):
                            new_vals.append( os.path.abspath(os.path.join(folder, item)) )
                except: # handle upload inputs
                    jflow_config_reader = JFlowConfigReader()
For faster browsing, not all history is shown. View entire blame