parameter.py 41.6 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17
#
# Copyright (C) 2012 INRA
# 
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
# 
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
# 
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
#

Penom Nom's avatar
Penom Nom committed
18 19
import re
import sys
20
import types
Jerome Mariette's avatar
Jerome Mariette committed
21
import datetime
Penom Nom's avatar
Penom Nom committed
22
import logging
23
import argparse
Penom Nom's avatar
Penom Nom committed
24
import os
25
import fnmatch
Penom Nom's avatar
Penom Nom committed
26
import tempfile
Jerome Mariette's avatar
Jerome Mariette committed
27
from argparse import _ensure_value
Penom Nom's avatar
Penom Nom committed
28
import urllib2
Jerome Mariette's avatar
Jerome Mariette committed
29
import copy as _copy
Penom Nom's avatar
Penom Nom committed
30 31 32 33 34 35
from urlparse import urlparse

from jflow.config_reader import JFlowConfigReader
from jflow.utils import get_octet_string_representation, get_nb_octet

# import custom types and custom formats
Jerome Mariette's avatar
Jerome Mariette committed
36
from workflows.types import *
Penom Nom's avatar
Penom Nom committed
37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52
from workflows.formats import *


# define all input type available
INPUTFILE_TYPES = ["inputfile", "localfile", "urlfile", "browsefile"]

def browsefile(file):
    # browsefile are not available from command line, considere it as a localfile
    # from the gui, this will not been tested this way
    return localfile(file)

def localfile(file):
    if os.path.isfile(file):
        return os.path.abspath(file)
    else:
        raise argparse.ArgumentTypeError("File '" + file + "' does not exists! Please provide a valid file path!")
53

Penom Nom's avatar
Penom Nom committed
54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78
def urlfile(file):
    uri_object = urlparse(file)
    try:
        opener = urllib2.urlopen(file)
    except:
        raise argparse.ArgumentTypeError("URL '" + file + "' is invalid!")
    file_name = os.path.basename(uri_object.path)
    if file_name is not None and file_name != "":
        metadata = opener.info()
        file_size = int(metadata.getheaders("Content-Length")[0])
        if file_size == 0:
            raise argparse.ArgumentTypeError("The URL file '" + file + "' is empty!")
        return file
    else:
        raise argparse.ArgumentTypeError("URL '" + file + "' does not contain any file name!")

def inputfile(file):
    # test the format
    uri_object = urlparse(file)
    # check the file
    if uri_object.scheme == '':
        return localfile(file)
    else:
        return urlfile(file)

79 80 81 82 83 84 85 86 87 88 89 90 91 92
def regexpfiles(files_pattern):
    try:
        if ':' in files_pattern:
            folder, pattern = files_pattern.rsplit(':')
        else:
            folder, pattern = os.path.split(files_pattern)
    except:
        raise argparse.ArgumentTypeError("Pattern '" + file + "' is invalid!")
    if not os.path.exists(folder):
        raise argparse.ArgumentTypeError("The folder '" + folder + "' doesn't exist!")
    if not os.access(folder, os.R_OK):
        raise argparse.ArgumentTypeError("You do not have permission to read '" + folder + "'!")
    return files_pattern

Penom Nom's avatar
Penom Nom committed
93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126
def create_test_function(itype):
    try: itype = itype.encode('ascii','ignore')
    except: pass
    try:
        ctype, csizel = itype.split(AbstractInputFile.SIZE_LIMIT_SPLITER)
        def inner_function(ifile):
            # first eval the asked type
            returned_value = eval(ctype)(ifile)
            # if 0 unlimited size
            if csizel != "0":
                # first test the size of the file
                uri_object = urlparse(ifile)
                if uri_object.scheme == '':
                    isize = os.path.getsize(ifile)
                    if isize > int(get_nb_octet(csizel)):
                        raise argparse.ArgumentTypeError("File '" + ifile + "' (size=" + get_octet_string_representation(isize) + ") exceeds size limits: " + csizel + ".")
                else:
                    try:
                        opener = urllib2.urlopen(file)
                        metadata = opener.info()
                        isize = int(metadata.getheaders("Content-Length")[0])
                        if isize > int(get_nb_octet(csizel)):
                            raise argparse.ArgumentTypeError("File '" + ifile + "' (size=" + get_octet_string_representation(isize) + ") exceeds size limits: " + csizel + ".")
                    except:
                        raise argparse.ArgumentTypeError("URL '" + file + "' is invalid!")
            # then test the type
            return returned_value
        inner_function.__name__ = ctype+AbstractInputFile.SIZE_LIMIT_SPLITER+csizel
        return inner_function
    except:
        if type(itype) == str:
            return eval(itype)
        else:
            return itype
127

Celine Noirot's avatar
Celine Noirot committed
128
class MultipleParameters(object):
129
    def __init__(self, types, required, choices, excludes, default, actions):
Jerome Mariette's avatar
Jerome Mariette committed
130
        self.types = types
131
        self.choices = choices
132
        self.excludes = excludes
133
        self.default = default
134
        self.actions = actions
135
        self.index = None
Jerome Mariette's avatar
Jerome Mariette committed
136
        self.required = required
Celine Noirot's avatar
Celine Noirot committed
137
        self.__name__ = "MultipleParameters"
Penom Nom's avatar
Penom Nom committed
138 139 140 141 142 143 144

    def get_help(self):
        help = " ("
        for flag in self.types.keys():
            help += flag + "=<" + self.types[flag].__name__.upper() + ">, "
        return help[:-2] + ")"

145
    def __call__(self, arg):
146 147 148
        parts = arg.split("=")
        if not self.types.has_key(parts[0]):
            raise argparse.ArgumentTypeError(parts[0] + " is an invalid flag! Available ones are: "+", ".join(self.types.keys()))
149 150 151
        if self.choices[parts[0]] != None:
            if parts[1] not in self.choices[parts[0]]:
                raise argparse.ArgumentTypeError("argument " + parts[0] + ": invalid choice: '" + parts[1] + "' (choose from " + ", ".join(self.choices[parts[0]]) + "))")
Penom Nom's avatar
Penom Nom committed
152

153 154 155 156 157 158 159
        if self.types[parts[0]] == types.BooleanType:
            return (parts[0], not self.default[parts[0]], self.required, self.excludes)
        else:
            try:
                value = self.types[parts[0]](parts[1])
            except:
                raise argparse.ArgumentTypeError("invalid " + self.types[parts[0]].__name__ + " value: '" + parts[1] + "' for sub parameter '" + parts[0] + "'")
Penom Nom's avatar
Penom Nom committed
160

161
            self.index = parts[0]
162
            return (parts[0], value, self.required, self.excludes, self.actions)
Jerome Mariette's avatar
Jerome Mariette committed
163

Jerome Mariette's avatar
Jerome Mariette committed
164 165
class MiltipleAction(argparse.Action):
    def __call__(self, parser, namespace, values, option_string=None):
166 167 168 169 170 171 172
        # what is commun within required and excludes
        exclusif_required = {}
        for exclude_group in values[0][3]:
            exclusif_required[exclude_group] = False
            for param in values[0][3][exclude_group]:
                if param in values[0][2]:
                    exclusif_required[exclude_group] = True
173 174
        given_params = []
        # first check for required parameters
Jerome Mariette's avatar
Jerome Mariette committed
175 176
        try:
            required = _copy.copy(values[0][2])
177 178 179 180 181 182
            # delete required that are exclusive
            for group in exclusif_required:
                if exclusif_required[group]:
                    for val in values[0][3][group]:
                        if val in required:
                            del required[required.index(val)]
Jerome Mariette's avatar
Jerome Mariette committed
183
            for val in values:
184
                given_params.append(val[0])
Jerome Mariette's avatar
Jerome Mariette committed
185 186 187 188 189
                if val[0] in required:
                    del required[required.index(val[0])]
        except: pass
        if len(required) == 1: parser.error(", ".join(required) + " is a required parameter!")
        elif len(required) > 1: parser.error(", ".join(required) + " are required parameters!")
190 191 192 193 194 195 196 197
        # then for exclude ones    
        for exclude_group in values[0][3]:
            found = None
            for param in values[0][3][exclude_group]:
                if param in given_params and found != None:
                    parser.error("argument '" + found + "': not allowed with argument '" + param + "'")
                    break
                elif param in given_params: found = param
Penom Nom's avatar
Penom Nom committed
198

199 200 201 202 203 204 205 206
        # check for required exclusive if one of them is in
        if len(exclusif_required.keys()) > 0:
            for group in exclusif_required:
                if exclusif_required[group]:
                    rfound = False
                    for param in values[0][3][group]:
                        if param in given_params: rfound = True
            if not rfound: parser.error("one of the arguments: " + ", ".join(values[0][3][group]) + " is required")
Penom Nom's avatar
Penom Nom committed
207

208
        # if ok add the value
209 210 211 212 213 214 215 216 217 218 219
        final_hash, final_values = {}, []
        for value in values:
            if values[0][4][value[0]] == "append" and final_hash.has_key(value[0]):
                final_hash[value[0]].append(value[1])
            elif values[0][4][value[0]] == "append":
                final_hash[value[0]]= [value[1]]
            else:
                final_hash[value[0]]= value[1]
        for param in final_hash:
            final_values.append((param, final_hash[param]))
        setattr(namespace, self.dest, final_values)
Jerome Mariette's avatar
Jerome Mariette committed
220

Penom Nom's avatar
Penom Nom committed
221

Jerome Mariette's avatar
Jerome Mariette committed
222 223
class MiltipleAppendAction(argparse.Action):
    def __call__(self, parser, namespace, values, option_string=None):
224 225 226 227 228 229 230
        # what is commun within required and excludes
        exclusif_required = {}
        for exclude_group in values[0][3]:
            exclusif_required[exclude_group] = False
            for param in values[0][3][exclude_group]:
                if param in values[0][2]:
                    exclusif_required[exclude_group] = True
231 232
        given_params = []
        # first check for required parameters
Jerome Mariette's avatar
Jerome Mariette committed
233 234
        try:
            required = _copy.copy(values[0][2])
235 236 237 238 239 240
            # delete required that are exclusive
            for group in exclusif_required:
                if exclusif_required[group]:
                    for val in values[0][3][group]:
                        if val in required:
                            del required[required.index(val)]
Jerome Mariette's avatar
Jerome Mariette committed
241
            for val in values:
242
                given_params.append(val[0])
Jerome Mariette's avatar
Jerome Mariette committed
243 244 245 246 247
                if val[0] in required:
                    del required[required.index(val[0])]
        except: pass
        if len(required) == 1: parser.error(", ".join(required) + " is a required parameter!")
        elif len(required) > 1: parser.error(", ".join(required) + " are required parameters!")
248 249 250 251 252 253 254 255
        # then for exclude ones    
        for exclude_group in values[0][3]:
            found = None
            for param in values[0][3][exclude_group]:
                if param in given_params and found != None:
                    parser.error("argument '" + found + "': not allowed with argument '" + param + "'")
                    break
                elif param in given_params: found = param
Penom Nom's avatar
Penom Nom committed
256

257 258 259 260 261 262 263 264
        # check for required exclusive if one of them is in
        if len(exclusif_required.keys()) > 0:
            for group in exclusif_required:
                if exclusif_required[group]:
                    rfound = False
                    for param in values[0][3][group]:
                        if param in given_params: rfound = True
            if not rfound: parser.error("one of the arguments: " + ", ".join(values[0][3][group]) + " is required")
265
        # if ok add the value
Jerome Mariette's avatar
Jerome Mariette committed
266
        items = _copy.copy(_ensure_value(namespace, self.dest, []))
267 268 269 270 271 272 273 274 275 276 277
        final_hash, final_values = {}, []
        for value in values:
            if values[0][4][value[0]] == "append" and final_hash.has_key(value[0]):
                final_hash[value[0]].append(value[1])
            elif values[0][4][value[0]] == "append":
                final_hash[value[0]]= [value[1]]
            else:
                final_hash[value[0]]= value[1]
        for param in final_hash:
            final_values.append((param, final_hash[param]))
        items.append(final_values)
Penom Nom's avatar
Penom Nom committed
278 279 280 281 282 283 284
        setattr(namespace, self.dest, items)


class AbstractParameter(object):

    def __init__(self, name, help, default=None, type=types.StringType, choices=None, required=False,
                 flag=None, action="store", sub_parameters=None, group="default", display_name=None):
Jerome Mariette's avatar
Jerome Mariette committed
285

286 287
        self.name = name
        self.help = help
288
        self.action = action
289
        self.nargs = None
Penom Nom's avatar
Penom Nom committed
290 291 292
        if sub_parameters:
            self.sub_parameters = sub_parameters
        else: self.sub_parameters = []
Jerome Mariette's avatar
Jerome Mariette committed
293
        self.group = group
Penom Nom's avatar
Penom Nom committed
294 295 296
        if flag == None:
            self.flag = "--"+name.replace("_", "-")
        else: self.flag = flag
Jerome Mariette's avatar
Jerome Mariette committed
297
        if display_name == None:
Penom Nom's avatar
Penom Nom committed
298
            self.display_name = name.replace("_", " ").title()
Jerome Mariette's avatar
Jerome Mariette committed
299
        else: self.display_name = display_name
Penom Nom's avatar
Penom Nom committed
300
        self.required = required
301
        self.choices = choices
Penom Nom's avatar
Penom Nom committed
302 303

        # Set parameter type
Jerome Mariette's avatar
Jerome Mariette committed
304 305
        if type == "date":
            self.type = date
Jerome Mariette's avatar
Jerome Mariette committed
306
        elif type == "multiple":
Penom Nom's avatar
Penom Nom committed
307
            self.type = "multiple"
Jerome Mariette's avatar
Jerome Mariette committed
308 309
        elif isinstance(type, types.FunctionType):
            self.type = type
Penom Nom's avatar
Penom Nom committed
310 311
        elif type in [types.StringType, types.IntType, types.FloatType,  types.BooleanType]:
            self.type = type
Jerome Mariette's avatar
Jerome Mariette committed
312 313 314
        else:
            try: self.type = eval(type)
            except: self.type = types.StringType
Penom Nom's avatar
Penom Nom committed
315 316

        # Set parameter value
Jerome Mariette's avatar
Jerome Mariette committed
317
        self.default = default
Penom Nom's avatar
Penom Nom committed
318

319
    def export_to_argparse(self):
Penom Nom's avatar
Penom Nom committed
320
        if self.type == types.BooleanType and str(self.default).lower() in (False, "false",  "f", "0"):
321 322 323
            return {"help": self.help, "required": self.required, "dest": self.name, 
                    "default": False, "action": "store_true"}
        elif self.type == types.BooleanType:
324
            return {"help": self.help, "required": self.required, "dest": self.name, 
325
                    "default": True, "action": "store_false"}
Jerome Mariette's avatar
Jerome Mariette committed
326
        elif self.nargs > 1:
Penom Nom's avatar
Penom Nom committed
327
            return {"type": self.get_test_function(), "help": self.help, "required": self.required,
Jerome Mariette's avatar
Jerome Mariette committed
328
                    "dest": self.name, "default": self.default,
329
                    "action": self.action, "choices": self.choices, "nargs": "+"}
330
        else:
Penom Nom's avatar
Penom Nom committed
331
            return {"type": self.get_test_function(), "help": self.help, "required": self.required,
Jerome Mariette's avatar
Jerome Mariette committed
332
                    "dest": self.name, "default": self.default,
333
                    "action": self.action, "choices": self.choices}
Penom Nom's avatar
Penom Nom committed
334

Jerome Mariette's avatar
Jerome Mariette committed
335 336
    def get_type(self):
        return self.type.__name__
Penom Nom's avatar
Penom Nom committed
337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599

    def get_test_function(self):
        return create_test_function(self.type)

class AbstractIOFile(object):

    def __init__(self, file_format="any"):
        self.file_format = file_format
        self.component_nameid = None
        self.parent_component_nameid = []


class MultiParameter(dict, AbstractParameter):

    def __init__(self, name, help, required=False, flag=None, group="default", display_name=None):
        AbstractParameter.__init__(self, name, help, required=required, type="multiple", flag=flag, group=group, 
                                   display_name=display_name)
        return dict.__init__(self, {})

    def add_sub_parameter(self, param):
        param_flag = param.flag[2:]
        if self.type == "multiple":
            if param.required: req = [param_flag] 
            else: req = []
            self.type = MultipleParameters({param_flag: param.type}, req, 
                                           {param_flag: param.choices}, {}, {param_flag: param.default},
                                           {param_flag: param.action})
            if self.action == "append":
                self.action = MiltipleAppendAction
            else:
                self.action = MiltipleAction
            self.global_help = self.help
            self.help = self.global_help + " (" + param_flag + "=<" + param.type.__name__.upper() + ">)"
            self.default = {}
            self.nargs = "+"
        elif self.type.__class__ == MultipleParameters:
            self.type.types[param_flag] = param.type
            self.type.choices[param_flag] = param.choices
            self.type.default[param_flag] = param.default if param != None else None
            self.type.actions[param_flag] = param.action
            if param.required:
                self.type.required.append(param_flag)
            self.help = self.global_help + self.type.get_help()
        param.flag = param_flag
        self.sub_parameters.append(param)


class MultiParameterList(list, AbstractParameter):

    def __init__(self, name, help, required=False, flag=None, group="default", display_name=None):
        AbstractParameter.__init__(self, name, help, required=required, type="multiple", flag=flag, 
                                   action="append", group=group, display_name=display_name)
        return list.__init__(self, [])

    def add_sub_parameter(self, param):
        param_flag = param.flag[2:]
        if self.type == "multiple":
            if param.required: req = [param_flag]
            else: req = []
            self.type = MultipleParameters({param_flag: param.type}, req,
                                           {param_flag: param.choices}, {}, {param_flag: param.default},
                                           {param_flag: param.action})
            if self.action == "append":
                self.action = MiltipleAppendAction
            else:
                self.action = MiltipleAction
            self.global_help = self.help
            self.help = self.global_help + " (" + param_flag + "=<" + param.type.__name__.upper() + ">)"
            self.default = []
            self.nargs = "+"
        elif self.type.__class__ == MultipleParameters:
            self.type.types[param_flag] = param.type
            self.type.choices[param_flag] = param.choices
            self.type.default[param_flag] = param.default if param != None else None
            self.type.actions[param_flag] = param.action
            if param.required:
                self.type.required.append(param_flag)
            self.help = self.global_help + self.type.get_help()
        param.flag = param_flag
        self.sub_parameters.append(param)


class ParameterFactory(object):
    @staticmethod
    def factory(*args, **kwargs):
        if not kwargs.has_key( "type" ):
            return StrParameter( *args, **kwargs )

        if kwargs["type"] == "int" or kwargs["type"] is types.IntType:
            return IntParameter( *args, **kwargs )
        elif kwargs["type"] == "bool" or kwargs["type"] is types.BooleanType:
            return BoolParameter( *args, **kwargs )
        elif kwargs["type"] == "float" or kwargs["type"] is types.FloatType:
            return FloatParameter( *args, **kwargs )
        elif kwargs["type"] == "date" or kwargs["type"] == date:
            return DateParameter( *args, **kwargs )
        else:
            return StrParameter( *args, **kwargs )


def none_decorator(fn):
    def new_func(*args, **kwargs):
        if args[0].is_None:
            raise Exception("The parameter '" + args[0].name + "' is None.")
        else:
            return fn(*args, **kwargs)
    return new_func


class BoolParameter(int, AbstractParameter):

    def __new__(self, name, help, default=False, type=types.BooleanType, choices=None, required=False,
                flag=None, sub_parameters=None, group="default", display_name=None):
        bool_default = False if default == None else bool(default)
        val = int.__new__(self, bool_default)
        val.is_None = False if default != None else True
        for attr in bool.__dict__:
            func = getattr(bool, attr)
            if callable(func) and attr not in ["__new__", "__init__", "__bool__", "__getattribute__", "__setattribute__",
                                               "__eq__", "__ne__", "__nonzero__", "__str__", "__repr__"]:
                setattr(BoolParameter, attr, none_decorator(func))
        return val

    def __init__(self, name, help, default=None, type=types.BooleanType, choices=None, required=False,
                 flag=None, sub_parameters=None, group="default", display_name=None):
        AbstractParameter.__init__(self, name, help, flag=flag, default=bool(default), type=type, choices=choices, required=required, 
                                   action="store", sub_parameters=sub_parameters, group=group, display_name=display_name)

    def __str__(self):
        if self.is_None:
            return str(None)
        return str(bool(self))

    def __repr__(self):
        return str(self)

    def __eq__(self, other):
        if other.__class__.__name__ == "NoneType":
            return self.is_None
        elif self.is_None:
            return False
        else:
            return int(self) == int(bool(other))

    def __ne__(self, other):
        return not (self == other)

    def __nonzero__(self):
        if self.is_None:
            return False
        else:
            return self != 0


class IntParameter(int, AbstractParameter):

    def __new__(self, name, help, default=None, type=types.IntType, choices=None, required=False,
                flag=None, sub_parameters=None, group="default", display_name=None):
        int_default = 0 if default == None else int(default)
        val = int.__new__(self, int_default)
        val.is_None = False if default != None else True
        for attr in int.__dict__:
            func = getattr(int, attr)
            if callable(func) and attr not in ["__new__", "__init__", "__int__", "__getattribute__", "__setattribute__",
                                               "__eq__", "__ne__", "__nonzero__", "__str__", "__repr__"]:
                setattr(IntParameter, attr, none_decorator(func))
        return val

    def __init__(self, name, help, default=None, type=types.IntType, choices=None, required=False,
                 flag=None, sub_parameters=None, group="default", display_name=None):
        AbstractParameter.__init__( self, name, help, flag=flag, default=default, type=type, choices=choices, required=required,
                                    action="store", sub_parameters=sub_parameters, group=group, display_name=display_name )

    def __str__(self):
        if self.is_None:
            return str(None)
        return str(int(self))

    def __repr__(self):
        return str(self)

    def __eq__(self, other):
        if other.__class__.__name__ == "NoneType":
            return self.is_None
        elif self.is_None:
            return False
        else:
            return int(self) == int(other)

    def __ne__(self, other):
        return not (self == other)

    def __nonzero__(self):
        if self.is_None:
            return False
        else:
            return self != 0


class FloatParameter(float, AbstractParameter):

    def __new__(self, name, help, default=None, type=types.FloatType, choices=None, required=False,
                flag=None, sub_parameters=None, group="default", display_name=None):
        float_default = 0.0 if default == None else float(default)
        val = float.__new__(self, float_default)
        val.is_None = False if default != None else True
        for attr in float.__dict__:
            func = getattr(float, attr)
            if callable(func) and attr not in ["__new__", "__init__", "__float__", "__getattribute__", "__setattribute__", 
                                               "__eq__", "__ne__", "__nonzero__", "__str__", "__repr__"]:
                setattr(FloatParameter, attr, none_decorator(func))
        return val

    def __init__(self, name, help, default=None, type=types.FloatType, choices=None, required=False,
                 flag=None, sub_parameters=None, group="default", display_name=None):
        AbstractParameter.__init__(self, name, help, flag=flag, default=default, type=type, choices=choices, required=required, 
                                   action="store", sub_parameters=sub_parameters, group=group, display_name=display_name)

    def __str__(self):
        if self.is_None:
            return str(None)
        return str(float(self))

    def __repr__(self):
        return str(self)

    def __eq__(self, other):
        if other.__class__.__name__ == "NoneType":
            return self.is_None
        elif self.is_None:
            return False
        else:
            return float(self) == float(other)

    def __ne__(self, other):
        return not (self == other)

    def __nonzero__(self):
        if self.is_None:
            return False
        else:
            return self != 0.0


class StrParameter(str, AbstractParameter):

    def __new__(self, name, help, default=None, type=types.StringType, choices=None, required=False,
                flag=None, sub_parameters=None, group="default", display_name=None):
        str_default = "" if default == None else str(default)
        val = str.__new__(self, str_default)
        val.is_None = False if default != None else True
        for attr in str.__dict__:
            func = getattr(str, attr)
            if callable(func) and attr not in ["__new__", "__init__", "__str__", "__getattribute__", "__setattribute__",
                                               "__eq__", "__ne__", "__nonzero__", "__repr__"]:
                setattr(StrParameter, attr, none_decorator(func))
        return val

    def __init__(self, name, help, default=None, type=types.StringType, choices=None, required=False,
                 flag=None, sub_parameters=None, group="default", display_name=None):
        AbstractParameter.__init__(self, name, help, flag=flag, default=default, type=type, choices=choices, required=required, 
                                   action="store", sub_parameters=sub_parameters, group=group, display_name=display_name)

600
    def __str__(self):
Penom Nom's avatar
Penom Nom committed
601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827
        if self.is_None:
            return str(None)
        return str.__str__(self)

    def __repr__(self):
        if self.is_None:
            return str(None)
        return "'" + str(self) + "'"

    def __eq__(self, other):
        if other.__class__.__name__ == "NoneType":
            return self.is_None
        elif self.is_None:
            return False
        else:
            return str(self) == str(other)

    def __ne__(self, other):
        return not(self == other)

    def __nonzero__(self):
        if self.is_None:
            return False
        else:
            return (True if str(self) else False)


class DateParameter(datetime.datetime, AbstractParameter):

    def __new__(self, name, help, default=None, type=date, choices=None, required=False,
                flag=None, sub_parameters=None, group="default", display_name=None):
        date_default = datetime.datetime.today()
        if default != None and issubclass(default.__class__, datetime.datetime):
            date_default = default
        elif default != None:
            date_default = date(default)
        val = datetime.datetime.__new__(self, date_default.year, date_default.month, date_default.day)
        val.is_None = False if default != None else True
        for attr in datetime.datetime.__dict__:
            func = getattr(datetime.datetime, attr)
            if callable(func) and attr not in ["__new__", "__init__", "__getattribute__", "__setattribute__",
                                               "__eq__", "__ne__", "__nonzero__", "__str__", "__repr__"]:
                setattr(DateParameter, attr, none_decorator(func))
        return val

    def __init__(self, name, help, default=None, type=date, choices=None, required=False,
                 flag=None, sub_parameters=None, group="default", display_name=None):
        if default != None and not issubclass(default.__class__, datetime.datetime):
            date_default = date(default)
            default = datetime.datetime(date_default.year, date_default.month, date_default.day)
        AbstractParameter.__init__(self, name, help, flag=flag, default=default, type=type, choices=choices, required=required, 
                                   action="store", sub_parameters=sub_parameters, group=group, display_name=display_name)

    def __str__(self):
        if self.is_None:
            return str(None)
        return datetime.datetime.__str__(self)

    def __repr__(self):
        return str(self)

    def __eq__(self, other):
        if other.__class__.__name__ == "NoneType":
            return self.is_None
        elif self.is_None:
            return False
        else:
            return datetime.date(self.year, self.month, self.day) == other

    def __ne__(self, other):
        return not( self == other )

    def __nonzero__(self):
        if self.is_None:
            return False
        else:
            return (True if datetime.date(self.year, self.month, self.day) else False)

    def __reduce__(self):
        return (DateParameter, (self.name, self.help, self.default, date, self.choices, self.required,
                 self.flag, self.sub_parameters, self.group, self.display_name), None, None, None)

    def __reduce_ex__(self, protocol):
        return (DateParameter, (self.name, self.help, self.default, date, self.choices, self.required,
                 self.flag, self.sub_parameters, self.group, self.display_name), None, None, None)


class AbstractInputFile(AbstractIOFile):
    """
     @summary : Parent of all InputFile(s) parameters.
    """
    
    SIZE_LIMIT_SPLITER = "__sl"
    
    def __init__(self, file_format="any", size_limit="0"):
        AbstractIOFile.__init__(self, file_format)
        self.size_limit = size_limit
    
    def _download_urlfile(self, input):
        try:
            uri_object = urlparse(input)
            opener = urllib2.urlopen(input)
            block_size = 8000
            jflowconf = JFlowConfigReader()
            tmp_directory = os.path.join(jflowconf.get_tmp_directory(), os.path.basename(tempfile.NamedTemporaryFile().name))
            os.mkdir(tmp_directory) 
            file_path = os.path.join(tmp_directory, os.path.basename(uri_object.path))
            local_file = open(file_path, 'wb')
            if os.path.basename(uri_object.path) is not None and os.path.basename(uri_object.path) != "":
                metadata = opener.info()
                file_size = int(metadata.getheaders("Content-Length")[0])
                while True:
                    buffer = opener.read(block_size)
                    # End of download
                    if not buffer: break
                    # Parts of download
                    local_file.write(buffer)
                local_file.close()
                logging.getLogger("AbstractInputFile.download_urlfile").debug("URL file '{0}' successfully downloaded as: {1}".format(input, file_path))
            return [file_path, True]
        except:
            return [input, False]

    def check(self, ifile):
        try:
            eval(self.file_format)
            function_exists = True
        except: function_exists = False
        if function_exists:
            try: 
                eval(self.file_format)(ifile)
            except jflow.InvalidFormatError as e:
                sys.stderr.write(str(e)+"\n")
                sys.exit(1)
        else:
            sys.stderr.write("Error: Invalid file format '" + self.file_format + "'!\n")
            sys.exit(1)

class AbstractOutputFile(AbstractIOFile):
    """
     @summary : Parent of all OutputFile(s) parameters.
    """
    pass


class InputFile(StrParameter, AbstractInputFile):

    def __new__(self, name, help, file_format="any", default="", type="localfile", choices=None, 
                required=False, flag=None, group="default", display_name=None, size_limit="0"):
        if hasattr(type, '__call__'):
            type2test = type.__name__
        else: type2test = type

        if type2test not in INPUTFILE_TYPES:
            raise ValueError("InputFile.__new__: wrong type provided: '"+type2test+"', this should be choosen between '" 
                             + "', '".join(INPUTFILE_TYPES)+"'")

        return StrParameter.__new__(self, name, help, flag=flag, default=default, type=type, choices=choices, 
                           required=required, group=group, display_name=display_name)

    def __init__(self, name, help, file_format="any", default="", type="localfile", choices=None, 
                required=False, flag=None, group="default", display_name=None, size_limit="0"):
        AbstractInputFile.__init__(self, file_format, size_limit)
        StrParameter.__init__(self, name, help, flag=flag, default=default, type=type, choices=choices, 
                           required=required, group=group, display_name=display_name)

    def get_type(self):
        return self.type.__name__+AbstractInputFile.SIZE_LIMIT_SPLITER+self.size_limit

    def get_test_function(self):
        if (self.size_limit == "0"): ctype = self.type
        else: ctype = self.get_type()
        return create_test_function(ctype)

    def prepare_input_file(self, input):
        if input == None:
            return None
        # handle url inputs
        new_path, is_uri = self._download_urlfile(input)
        # handle upload inputs
        try: is_local = os.path.isfile(input)
        except: is_local = False
        if not is_uri and not is_local and self.type.__name__ == "inputfile" or self.type.__name__ == "browsefile":
            jflow_config_reader = JFlowConfigReader()
            new_path = os.path.join(jflow_config_reader.get_tmp_directory(), input)
        if is_local: new_path = input
        self.check(new_path)
        return new_path


class OutputFile(StrParameter, AbstractOutputFile):

    def __new__(self, name, help, file_format="any", default="", choices=None,
                required=False, flag=None, group="default", display_name=None):
        return StrParameter.__new__(self, name, help, flag=flag, default=default, type="localfile", choices=choices,
                           required=required, group=group, display_name=display_name)

    def __init__(self, name, help, file_format="any", default="", choices=None,
                required=False, flag=None, group="default", display_name=None):
        AbstractIOFile.__init__(self, file_format)
        StrParameter.__init__(self, name, help, flag=flag, default=default, type="localfile", choices=choices, 
                           required=required, group=group, display_name=display_name)


class ParameterList(list, AbstractParameter):

    def __init__(self, name, help, default=None, type=types.StringType, choices=None, required=False,
                 flag=None, sub_parameters=None, group="default", display_name=None):
        if default == None: default = []
        AbstractParameter.__init__(self, name, help, flag=flag, default=default, type=type, choices=choices, required=required,
                                   action="append", sub_parameters=sub_parameters, group=group, display_name=display_name)
        if default.__class__.__name__ == "str":
            return list.__init__(self, [default])
        elif default.__class__.__name__ == "list":
            return list.__init__(self, default)


class InputFileList(ParameterList, AbstractInputFile):

    def __init__(self, name, help, file_format="any", default=None, type="localfile", choices=None, 
                 required=False, flag=None, group="default", display_name=None, size_limit="0"):

        if default == None: default = []   
        if hasattr(type, '__call__'):
            type2test = type.__name__
        else: type2test = type

828
        if type2test not in INPUTFILE_TYPES + ["regexpfiles"]:
Penom Nom's avatar
Penom Nom committed
829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860
            raise ValueError("InputFile.__new__: wrong type provided: '"+type2test+"', this should be choosen between '" 
                             + "', '".join(INPUTFILE_TYPES)+"'")

        AbstractInputFile.__init__(self, file_format, size_limit)
        ParameterList.__init__(self, name, help, flag=flag, default=default, type=type, choices=choices, 
                               required=required, group=group, display_name=display_name)

        if default.__class__.__name__ == "str":
            return list.__init__(self, [default])
        elif default.__class__.__name__ == "list":
            return list.__init__(self, default)
        elif issubclass( default.__class__, InputFile ):
            return list.__init__(self, [default])
        elif issubclass( default.__class__, AbstractInputFile ):
            return list.__init__(self, default)
        elif issubclass( default.__class__, OutputFile ):
            return list.__init__(self, [default])
        elif issubclass( default.__class__, AbstractOutputFile ):
            return list.__init__(self, default)
    
    def get_type(self):
        return self.type.__name__+AbstractInputFile.SIZE_LIMIT_SPLITER+self.size_limit
    
    def get_test_function(self):
        if (self.size_limit == "0"): ctype = self.type
        else: ctype = self.get_type()
        return create_test_function(ctype)

    def prepare_input_files(self, inputs):
        path2test = _copy.deepcopy(inputs)
        if not path2test.__class__.__name__ == "list":
            path2test = [path2test]
861 862 863 864 865 866 867 868 869 870 871 872
        # handle regexp files
        new_vals = list()
        if self.type.__name__ == "regexpfiles":
            for files_pattern in path2test:
                if ':' in files_pattern:
                    folder, pattern = files_pattern.rsplit(':')
                else:
                    folder, pattern = os.path.split(files_pattern)
                for item in os.listdir(folder):
                    if os.path.isfile(os.path.join(folder, item)) and fnmatch.fnmatch(item, pattern):
                        new_vals.append( os.path.abspath(os.path.join(folder, item)) )
            path2test = new_vals
Penom Nom's avatar
Penom Nom committed
873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980
        new_vals, done = [], []
        # handle url inputs
        for path in path2test:
            new_url, is_uri = self._download_urlfile(path)
            if is_uri:
                new_vals.append(new_url)
                done.append(path)
        for d in done: path2test.remove(d)
        done = []
        # handle localfile
        for path in path2test:
            if os.path.isfile(path):
                new_vals.append(path)
                done.append(path)
        for d in done: path2test.remove(d)        
        # handle upload inputs
        if self.type.__name__ == "inputfile" or self.type.__name__ == "browsefile":
            for path in path2test:
                jflow_config_reader = JFlowConfigReader()
                new_vals.append(os.path.join(jflow_config_reader.get_tmp_directory(), path))
        # now that all files are downloaded and ok, check the format
        for cfile in new_vals:
            self.check(cfile)
        return new_vals

class OutputFileList(ParameterList, AbstractOutputFile):

    def __init__(self, name, help, file_format="any", default=None, choices=None, 
                 required=False, flag=None, group="default", display_name=None):
        if default == None: default = []
        AbstractIOFile.__init__(self, file_format)
        ParameterList.__init__(self, name, help, flag=flag, default=default, type="localfile", choices=choices, 
                               required=required, group=group, display_name=display_name)
        if default.__class__.__name__ == "str":
            return list.__init__(self, [default])
        elif default.__class__.__name__ == "list":
            return list.__init__(self, default)


class DynamicOutput(ParameterList, AbstractOutputFile):
    """
     @warning : with this class of output, the component become dynamic.
    """
    def update(self):
        """
         This method is used at the end of component execution to update output list.
        """
        raise NotImplementedError


class OutputFilesEndsWith(DynamicOutput):

    def __init__(self, name, help, output_directory, end_str, include=True, file_format="any", choices=None, 
                 required=False, flag=None, group="default", display_name=None):
        """
         @warning : with this class of output, the component become dynamic.
         @param output_directory : path to the directory where outputs will be created.
         @param end_str : the end of the files names.
         @param include : if true, the files with name terminated by end_str are added into output files.
                          If false, the files with name not terminated by end_str are added into output files.
        """
        AbstractIOFile.__init__(self, file_format)
        default = []
        ParameterList.__init__(self, name, help, flag=flag, default=default, type="localfile", choices=choices, 
                               required=required, group=group, display_name=display_name)
        self.output_directory = output_directory
        self.end_str = end_str
        self.include = include
        return list.__init__(self, default)

    def update(self):
        output_files = list()
        for file in os.listdir( self.output_directory ):
            if file.endswith( self.end_str ) and self.include :
                output_files.append( os.path.join(self.output_directory, file) )
            elif not file.endswith( self.end_str ) and not self.include:
                output_files.append( os.path.join(self.output_directory, file) )
        list.__init__(self, output_files)


class OutputFilesPattern(DynamicOutput):

    def __init__(self, name, help, output_directory, pattern, include=True, file_format="any", choices=None, 
                  required=False, flag=None, group="default", display_name=None):
        """
         @warning : with this class of output, the component become dynamic.
         @param output_directory : path to the directory where outputs will be created.
         @param pattern : the pattern of (a part) the file names.
         @param include : if true, the files with the pattern in file name are added into output files.
                          If false, the files with the pattern in file name are added into output files.
        """
        AbstractIOFile.__init__(self, file_format)
        default = []
        ParameterList.__init__(self, name, help, flag=flag, default=default, type="localfile", choices=choices, 
                               required=required, group=group, display_name=display_name)
        self.output_directory = output_directory
        self.pattern = pattern
        self.include = include
        return list.__init__(self, default)

    def update(self):
        output_files = list()
        for file in os.listdir( self.output_directory ):
            if self.include and re.search( self.pattern, file ) is not None:
                output_files.append( os.path.join(self.output_directory, file) )
            elif not self.include and re.search( self.pattern, file ) is None:
                output_files.append( os.path.join(self.output_directory, file) )
        return list.__init__(self, output_files)