size_limit for input file ok from cli

src/jflow/parameter.py

@@ -27,7 +27,9 @@ from argparse import _ensure_value
 import urllib2
 import copy as _copy
 from urlparse import urlparse
+
 from jflow.config_reader import JFlowConfigReader
+from jflow.utils import get_octet_string_representation, get_nb_octet
 
 # import custom types and custom formats
 from workflows.types import *
@@ -73,6 +75,32 @@ def inputfile(file):
     else:
         return urlfile(file)
 
+def create_test_function(type):
+    try:
+        ctype, csizel = type.split(AbstractInputFile.SIZE_LIMIT_SPLITER)
+        def inner_function(ifile):
+            # first test the size of the file
+            uri_object = urlparse(ifile)
+            if uri_object.scheme == '':
+                isize = os.path.getsize(ifile)
+                if isize > int(get_nb_octet(csizel)):
+                    raise argparse.ArgumentTypeError("File '" + ifile + "' (size=" + get_octet_string_representation(isize) + ") exceeds size limits: " + csizel + ".")
+            else:
+                try:
+                    opener = urllib2.urlopen(file)
+                    metadata = opener.info()
+                    isize = int(metadata.getheaders("Content-Length")[0])
+                    if isize > int(get_nb_octet(csizel)):
+                        raise argparse.ArgumentTypeError("File '" + ifile + "' (size=" + get_octet_string_representation(isize) + ") exceeds size limits: " + csizel + ".")
+                except:
+                    raise argparse.ArgumentTypeError("URL '" + file + "' is invalid!")
+            # then test the type
+            return eval(ctype)(ifile)
+        inner_function.__name__ = ctype+AbstractInputFile.SIZE_LIMIT_SPLITER+csizel
+        return inner_function
+    except:
+        return type
+
 class MultipleParameters(object):
     def __init__(self, types, required, choices, excludes, default, actions):
         self.types = types
@@ -272,17 +300,19 @@ class AbstractParameter(object):
             return {"help": self.help, "required": self.required, "dest": self.name, 
                     "default": True, "action": "store_false"}
         elif self.nargs > 1:
-            return {"type": self.type, "help": self.help, "required": self.required,
+            return {"type": self.get_test_function(), "help": self.help, "required": self.required,
                     "dest": self.name, "default": self.default,
                     "action": self.action, "choices": self.choices, "nargs": "+"}
         else:
-            return {"type": self.type, "help": self.help, "required": self.required,
+            return {"type": self.get_test_function(), "help": self.help, "required": self.required,
                     "dest": self.name, "default": self.default,
                     "action": self.action, "choices": self.choices}
 
     def get_type(self):
         return self.type.__name__
 
+    def get_test_function(self):
+        return create_test_function(self.type)
 
 class AbstractIOFile(object):
 
@@ -600,6 +630,13 @@ class AbstractInputFile(AbstractIOFile):
     """
      @summary : Parent of all InputFile(s) parameters.
     """
+    
+    SIZE_LIMIT_SPLITER = "__sl"
+    
+    def __init__(self, file_format="any", size_limit="0"):
+        AbstractIOFile.__init__(self, file_format)
+        self.size_limit = size_limit
+    
     def _download_urlfile(self, input):
         try:
             uri_object = urlparse(input)
@@ -650,7 +687,7 @@ class AbstractOutputFile(AbstractIOFile):
 class InputFile(StrParameter, AbstractInputFile):
 
     def __new__(self, name, help, file_format="any", default="", type="localfile", choices=None, 
-                required=False, flag=None, group="default", display_name=None):
+                required=False, flag=None, group="default", display_name=None, size_limit="0"):
 
         if hasattr(type, '__call__'):
             type2test = type.__name__
@@ -664,11 +701,16 @@ class InputFile(StrParameter, AbstractInputFile):
                            required=required, group=group, display_name=display_name)
 
     def __init__(self, name, help, file_format="any", default="", type="localfile", choices=None, 
-                required=False, flag=None, group="default", display_name=None):
-        AbstractIOFile.__init__(self, file_format)
+                required=False, flag=None, group="default", display_name=None, size_limit="0"):
+        AbstractInputFile.__init__(self, file_format, size_limit)
         StrParameter.__init__(self, name, help, flag=flag, default=default, type=type, choices=choices, 
                            required=required, group=group, display_name=display_name)
 
+    def get_test_function(self):
+        if (self.size_limit == "0"): ctype = self.type
+        else: ctype = self.get_type()+AbstractInputFile.SIZE_LIMIT_SPLITER+self.size_limit
+        return create_test_function(ctype)
+
     def prepare_input_file(self, input):
         # handle url inputs
         new_path, is_uri = self._download_urlfile(input)
@@ -713,7 +755,7 @@ class ParameterList(list, AbstractParameter):
 class InputFileList(ParameterList, AbstractInputFile):
 
     def __init__(self, name, help, file_format="any", default=None, type="localfile", choices=None, 
-                 required=False, flag=None, group="default", display_name=None):
+                 required=False, flag=None, group="default", display_name=None, size_limit="0"):
 
         if default == None: default = []   
         if hasattr(type, '__call__'):
@@ -724,7 +766,7 @@ class InputFileList(ParameterList, AbstractInputFile):
             raise ValueError("InputFile.__new__: wrong type provided: '"+type2test+"', this should be choosen between '" 
                              + "', '".join(INPUTFILE_TYPES)+"'")
 
-        AbstractIOFile.__init__(self, file_format)
+        AbstractInputFile.__init__(self, file_format, size_limit)
         ParameterList.__init__(self, name, help, flag=flag, default=default, type=type, choices=choices, 
                                required=required, group=group, display_name=display_name)
 
@@ -741,6 +783,11 @@ class InputFileList(ParameterList, AbstractInputFile):
         elif issubclass( default.__class__, AbstractOutputFile ):
             return list.__init__(self, default)
 
+    def get_test_function(self):
+        if (self.size_limit == "0"): ctype = self.type
+        else: ctype = self.get_type()+AbstractInputFile.SIZE_LIMIT_SPLITER+self.size_limit
+        return create_test_function(ctype)
+
     def prepare_input_files(self, inputs):
         path2test = inputs
         if not path2test.__class__.__name__ == "list":

src/jflow/utils.py

@@ -18,6 +18,7 @@
 import re
 import smtplib
 import socket
+import math
 
 try:
     import DNS
@@ -71,6 +72,34 @@ def get_nb_string(value, length=6):
     s_value = zeros + s_value
     return s_value
 
+def get_nb_octet(size):
+    """
+    Return the number of bytes: value has to be formated like this: 5Mb, 20Gb ...
+    """
+    octets_link = ["bytes", "Kb", "Mb", "Gb", "Tb", "Pb", "Eb", "Zb"]
+    unit = size[len(size)-2:len(size)]
+    pow_val = int(octets_link.index(unit)) * 10
+    val = pow(2, pow_val)
+    nb_octet = int(size[:len(size)-2]) * val
+    return nb_octet
+
+def get_octet_string_representation(size):
+    """
+    Return the string representation of a byte
+    """
+    octets_link = ["bytes", "Kb", "Mb", "Gb", "Tb", "Pb", "Eb", "Zb"]
+    p = int(math.ceil(float(len(str(size)))/float(3) - float(1)))
+    pow_needed = p * 10
+    pow_needed = pow(2, pow_needed)
+    value = str(float(size)/float(pow_needed))
+    tmp = value.split(".")
+    value = tmp[0] + "." + tmp[1][:2]
+    try:
+        value = value + " " + octets_link[p]
+    except:
+        raise TypeError("In core.common:project_id unexpected input value for size: " + str(size))
+    return str(value)
+
 def get_argument_pattern( list, start_number=1 ):
     """
     Return the argument pattern for arguments files. Ex : with 3 element in list this function returns ["${1} ${2} ${3}", 4].

src/jflow/workflow.py

@@ -37,6 +37,7 @@ import jflow.utils as utils
 from jflow.utils import validate_email
 from jflow.graph import Graph
 from jflow.workflows_manager import WorkflowsManager, JFlowConfigReader
+from jflow.utils import get_octet_string_representation, get_nb_octet
 from jflow.parameter import *
 from cctools.util import time_format
 
@@ -122,9 +123,12 @@ class Workflow(threading.Thread):
             self._serialize()
     
     def add_input_file(self, name, help, file_format="any", default=None, type="inputfile", 
-                       required=False, flag=None, group="default", display_name=None, add_to=None):
+                       required=False, flag=None, group="default", display_name=None, size_limit="0", add_to=None):
+        # check if the size provided is correct
+        try: int(get_nb_octet(size_limit))
+        except: size_limit="0"
         new_param = InputFile(name, help, flag=flag, file_format=file_format, default=default, 
-                              type=type, required=required, group=group, display_name=display_name)
+                              type=type, required=required, group=group, display_name=display_name, size_limit=size_limit)
         # if this input should be added to a particular parameter
         if add_to:
             try:
@@ -136,10 +140,13 @@ class Workflow(threading.Thread):
             self.__setattr__(name, new_param)
             
     def add_input_file_list(self, name, help, file_format="any", default=None, type="inputfile", 
-                            required=False, flag=None, group="default", display_name=None, add_to=None):
+                            required=False, flag=None, group="default", display_name=None, size_limit="0", add_to=None):
+        # check if the size provided is correct
         if default == None: default = []
+        try: int(get_nb_octet(size_limit))
+        except: size_limit="0"
         new_param = InputFileList(name, help, flag=flag, file_format=file_format, default=default, 
-                                  type=type, required=required, group=group, display_name=display_name)
+                                  type=type, required=required, group=group, display_name=display_name, size_limit=size_limit)
         # if this input should be added to a particular parameter
         if add_to:
             try: