Commit f90d1aeb authored by Jerome Mariette's avatar Jerome Mariette
Browse files

size_limit for input file ok from cli

parent 853b28e0
...@@ -27,7 +27,9 @@ from argparse import _ensure_value ...@@ -27,7 +27,9 @@ from argparse import _ensure_value
import urllib2 import urllib2
import copy as _copy import copy as _copy
from urlparse import urlparse from urlparse import urlparse
from jflow.config_reader import JFlowConfigReader from jflow.config_reader import JFlowConfigReader
from jflow.utils import get_octet_string_representation, get_nb_octet
# import custom types and custom formats # import custom types and custom formats
from workflows.types import * from workflows.types import *
...@@ -73,6 +75,32 @@ def inputfile(file): ...@@ -73,6 +75,32 @@ def inputfile(file):
else: else:
return urlfile(file) return urlfile(file)
def create_test_function(type):
try:
ctype, csizel = type.split(AbstractInputFile.SIZE_LIMIT_SPLITER)
def inner_function(ifile):
# first test the size of the file
uri_object = urlparse(ifile)
if uri_object.scheme == '':
isize = os.path.getsize(ifile)
if isize > int(get_nb_octet(csizel)):
raise argparse.ArgumentTypeError("File '" + ifile + "' (size=" + get_octet_string_representation(isize) + ") exceeds size limits: " + csizel + ".")
else:
try:
opener = urllib2.urlopen(file)
metadata = opener.info()
isize = int(metadata.getheaders("Content-Length")[0])
if isize > int(get_nb_octet(csizel)):
raise argparse.ArgumentTypeError("File '" + ifile + "' (size=" + get_octet_string_representation(isize) + ") exceeds size limits: " + csizel + ".")
except:
raise argparse.ArgumentTypeError("URL '" + file + "' is invalid!")
# then test the type
return eval(ctype)(ifile)
inner_function.__name__ = ctype+AbstractInputFile.SIZE_LIMIT_SPLITER+csizel
return inner_function
except:
return type
class MultipleParameters(object): class MultipleParameters(object):
def __init__(self, types, required, choices, excludes, default, actions): def __init__(self, types, required, choices, excludes, default, actions):
self.types = types self.types = types
...@@ -272,17 +300,19 @@ class AbstractParameter(object): ...@@ -272,17 +300,19 @@ class AbstractParameter(object):
return {"help": self.help, "required": self.required, "dest": self.name, return {"help": self.help, "required": self.required, "dest": self.name,
"default": True, "action": "store_false"} "default": True, "action": "store_false"}
elif self.nargs > 1: elif self.nargs > 1:
return {"type": self.type, "help": self.help, "required": self.required, return {"type": self.get_test_function(), "help": self.help, "required": self.required,
"dest": self.name, "default": self.default, "dest": self.name, "default": self.default,
"action": self.action, "choices": self.choices, "nargs": "+"} "action": self.action, "choices": self.choices, "nargs": "+"}
else: else:
return {"type": self.type, "help": self.help, "required": self.required, return {"type": self.get_test_function(), "help": self.help, "required": self.required,
"dest": self.name, "default": self.default, "dest": self.name, "default": self.default,
"action": self.action, "choices": self.choices} "action": self.action, "choices": self.choices}
def get_type(self): def get_type(self):
return self.type.__name__ return self.type.__name__
def get_test_function(self):
return create_test_function(self.type)
class AbstractIOFile(object): class AbstractIOFile(object):
...@@ -600,6 +630,13 @@ class AbstractInputFile(AbstractIOFile): ...@@ -600,6 +630,13 @@ class AbstractInputFile(AbstractIOFile):
""" """
@summary : Parent of all InputFile(s) parameters. @summary : Parent of all InputFile(s) parameters.
""" """
SIZE_LIMIT_SPLITER = "__sl"
def __init__(self, file_format="any", size_limit="0"):
AbstractIOFile.__init__(self, file_format)
self.size_limit = size_limit
def _download_urlfile(self, input): def _download_urlfile(self, input):
try: try:
uri_object = urlparse(input) uri_object = urlparse(input)
...@@ -650,7 +687,7 @@ class AbstractOutputFile(AbstractIOFile): ...@@ -650,7 +687,7 @@ class AbstractOutputFile(AbstractIOFile):
class InputFile(StrParameter, AbstractInputFile): class InputFile(StrParameter, AbstractInputFile):
def __new__(self, name, help, file_format="any", default="", type="localfile", choices=None, def __new__(self, name, help, file_format="any", default="", type="localfile", choices=None,
required=False, flag=None, group="default", display_name=None): required=False, flag=None, group="default", display_name=None, size_limit="0"):
if hasattr(type, '__call__'): if hasattr(type, '__call__'):
type2test = type.__name__ type2test = type.__name__
...@@ -664,11 +701,16 @@ class InputFile(StrParameter, AbstractInputFile): ...@@ -664,11 +701,16 @@ class InputFile(StrParameter, AbstractInputFile):
required=required, group=group, display_name=display_name) required=required, group=group, display_name=display_name)
def __init__(self, name, help, file_format="any", default="", type="localfile", choices=None, def __init__(self, name, help, file_format="any", default="", type="localfile", choices=None,
required=False, flag=None, group="default", display_name=None): required=False, flag=None, group="default", display_name=None, size_limit="0"):
AbstractIOFile.__init__(self, file_format) AbstractInputFile.__init__(self, file_format, size_limit)
StrParameter.__init__(self, name, help, flag=flag, default=default, type=type, choices=choices, StrParameter.__init__(self, name, help, flag=flag, default=default, type=type, choices=choices,
required=required, group=group, display_name=display_name) required=required, group=group, display_name=display_name)
def get_test_function(self):
if (self.size_limit == "0"): ctype = self.type
else: ctype = self.get_type()+AbstractInputFile.SIZE_LIMIT_SPLITER+self.size_limit
return create_test_function(ctype)
def prepare_input_file(self, input): def prepare_input_file(self, input):
# handle url inputs # handle url inputs
new_path, is_uri = self._download_urlfile(input) new_path, is_uri = self._download_urlfile(input)
...@@ -713,7 +755,7 @@ class ParameterList(list, AbstractParameter): ...@@ -713,7 +755,7 @@ class ParameterList(list, AbstractParameter):
class InputFileList(ParameterList, AbstractInputFile): class InputFileList(ParameterList, AbstractInputFile):
def __init__(self, name, help, file_format="any", default=None, type="localfile", choices=None, def __init__(self, name, help, file_format="any", default=None, type="localfile", choices=None,
required=False, flag=None, group="default", display_name=None): required=False, flag=None, group="default", display_name=None, size_limit="0"):
if default == None: default = [] if default == None: default = []
if hasattr(type, '__call__'): if hasattr(type, '__call__'):
...@@ -724,7 +766,7 @@ class InputFileList(ParameterList, AbstractInputFile): ...@@ -724,7 +766,7 @@ class InputFileList(ParameterList, AbstractInputFile):
raise ValueError("InputFile.__new__: wrong type provided: '"+type2test+"', this should be choosen between '" raise ValueError("InputFile.__new__: wrong type provided: '"+type2test+"', this should be choosen between '"
+ "', '".join(INPUTFILE_TYPES)+"'") + "', '".join(INPUTFILE_TYPES)+"'")
AbstractIOFile.__init__(self, file_format) AbstractInputFile.__init__(self, file_format, size_limit)
ParameterList.__init__(self, name, help, flag=flag, default=default, type=type, choices=choices, ParameterList.__init__(self, name, help, flag=flag, default=default, type=type, choices=choices,
required=required, group=group, display_name=display_name) required=required, group=group, display_name=display_name)
...@@ -741,6 +783,11 @@ class InputFileList(ParameterList, AbstractInputFile): ...@@ -741,6 +783,11 @@ class InputFileList(ParameterList, AbstractInputFile):
elif issubclass( default.__class__, AbstractOutputFile ): elif issubclass( default.__class__, AbstractOutputFile ):
return list.__init__(self, default) return list.__init__(self, default)
def get_test_function(self):
if (self.size_limit == "0"): ctype = self.type
else: ctype = self.get_type()+AbstractInputFile.SIZE_LIMIT_SPLITER+self.size_limit
return create_test_function(ctype)
def prepare_input_files(self, inputs): def prepare_input_files(self, inputs):
path2test = inputs path2test = inputs
if not path2test.__class__.__name__ == "list": if not path2test.__class__.__name__ == "list":
......
...@@ -18,6 +18,7 @@ ...@@ -18,6 +18,7 @@
import re import re
import smtplib import smtplib
import socket import socket
import math
try: try:
import DNS import DNS
...@@ -71,6 +72,34 @@ def get_nb_string(value, length=6): ...@@ -71,6 +72,34 @@ def get_nb_string(value, length=6):
s_value = zeros + s_value s_value = zeros + s_value
return s_value return s_value
def get_nb_octet(size):
"""
Return the number of bytes: value has to be formated like this: 5Mb, 20Gb ...
"""
octets_link = ["bytes", "Kb", "Mb", "Gb", "Tb", "Pb", "Eb", "Zb"]
unit = size[len(size)-2:len(size)]
pow_val = int(octets_link.index(unit)) * 10
val = pow(2, pow_val)
nb_octet = int(size[:len(size)-2]) * val
return nb_octet
def get_octet_string_representation(size):
"""
Return the string representation of a byte
"""
octets_link = ["bytes", "Kb", "Mb", "Gb", "Tb", "Pb", "Eb", "Zb"]
p = int(math.ceil(float(len(str(size)))/float(3) - float(1)))
pow_needed = p * 10
pow_needed = pow(2, pow_needed)
value = str(float(size)/float(pow_needed))
tmp = value.split(".")
value = tmp[0] + "." + tmp[1][:2]
try:
value = value + " " + octets_link[p]
except:
raise TypeError("In core.common:project_id unexpected input value for size: " + str(size))
return str(value)
def get_argument_pattern( list, start_number=1 ): def get_argument_pattern( list, start_number=1 ):
""" """
Return the argument pattern for arguments files. Ex : with 3 element in list this function returns ["${1} ${2} ${3}", 4]. Return the argument pattern for arguments files. Ex : with 3 element in list this function returns ["${1} ${2} ${3}", 4].
......
...@@ -37,6 +37,7 @@ import jflow.utils as utils ...@@ -37,6 +37,7 @@ import jflow.utils as utils
from jflow.utils import validate_email from jflow.utils import validate_email
from jflow.graph import Graph from jflow.graph import Graph
from jflow.workflows_manager import WorkflowsManager, JFlowConfigReader from jflow.workflows_manager import WorkflowsManager, JFlowConfigReader
from jflow.utils import get_octet_string_representation, get_nb_octet
from jflow.parameter import * from jflow.parameter import *
from cctools.util import time_format from cctools.util import time_format
...@@ -122,9 +123,12 @@ class Workflow(threading.Thread): ...@@ -122,9 +123,12 @@ class Workflow(threading.Thread):
self._serialize() self._serialize()
def add_input_file(self, name, help, file_format="any", default=None, type="inputfile", def add_input_file(self, name, help, file_format="any", default=None, type="inputfile",
required=False, flag=None, group="default", display_name=None, add_to=None): required=False, flag=None, group="default", display_name=None, size_limit="0", add_to=None):
# check if the size provided is correct
try: int(get_nb_octet(size_limit))
except: size_limit="0"
new_param = InputFile(name, help, flag=flag, file_format=file_format, default=default, new_param = InputFile(name, help, flag=flag, file_format=file_format, default=default,
type=type, required=required, group=group, display_name=display_name) type=type, required=required, group=group, display_name=display_name, size_limit=size_limit)
# if this input should be added to a particular parameter # if this input should be added to a particular parameter
if add_to: if add_to:
try: try:
...@@ -136,10 +140,13 @@ class Workflow(threading.Thread): ...@@ -136,10 +140,13 @@ class Workflow(threading.Thread):
self.__setattr__(name, new_param) self.__setattr__(name, new_param)
def add_input_file_list(self, name, help, file_format="any", default=None, type="inputfile", def add_input_file_list(self, name, help, file_format="any", default=None, type="inputfile",
required=False, flag=None, group="default", display_name=None, add_to=None): required=False, flag=None, group="default", display_name=None, size_limit="0", add_to=None):
# check if the size provided is correct
if default == None: default = [] if default == None: default = []
try: int(get_nb_octet(size_limit))
except: size_limit="0"
new_param = InputFileList(name, help, flag=flag, file_format=file_format, default=default, new_param = InputFileList(name, help, flag=flag, file_format=file_format, default=default,
type=type, required=required, group=group, display_name=display_name) type=type, required=required, group=group, display_name=display_name, size_limit=size_limit)
# if this input should be added to a particular parameter # if this input should be added to a particular parameter
if add_to: if add_to:
try: try:
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment