Commit 674c1bbd authored by Jerome Mariette's avatar Jerome Mariette
Browse files

No commit message

No commit message
parent 1eea52b5
......@@ -17,12 +17,84 @@
import types
import datetime
import logging
import argparse
import os
import tempfile
from argparse import _ensure_value
import urllib2
import copy as _copy
from urlparse import urlparse
from jflow.config_reader import JFlowConfigReader
# import custom types
from workflows.types import *
# function in charge to download an url file
def download_urlfile(input):
try:
uri_object = urlparse(input)
opener = urllib2.urlopen(input)
block_size = 8000
jflowconf = JFlowConfigReader()
tmp_directory = os.path.join(jflowconf.get_tmp_directory(), os.path.basename(tempfile.NamedTemporaryFile().name))
os.mkdir(tmp_directory)
file_path = os.path.join(tmp_directory, os.path.basename(uri_object.path))
local_file = open(file_path, 'wb')
if os.path.basename(uri_object.path) is not None and os.path.basename(uri_object.path) != "":
metadata = opener.info()
file_size = int(metadata.getheaders("Content-Length")[0])
while True:
buffer = opener.read(block_size)
# End of download
if not buffer: break
# Parts of download
local_file.write(buffer)
local_file.close()
logging.getLogger("types.urlfile").debug("URL file '{0}' successfully downloaded as: {1}".format(input, file_path))
return [file_path, True]
except:
return [input, False]
# define all input type available
INPUTFILE_TYPES = ["inputfile", "localfile", "urlfile", "browsefile"]
def browsefile(file):
# browsefile are not available from command line, considere it as a localfile
# from the gui, this will not been tested this way
return localfile(file)
def localfile(file):
if os.path.isfile(file):
return os.path.abspath(file)
else:
raise argparse.ArgumentTypeError("Local file '" + file + "' does not exists! Please provide a valid file path!")
def urlfile(file):
uri_object = urlparse(file)
try:
opener = urllib2.urlopen(file)
except:
raise argparse.ArgumentTypeError("URL '" + file + "' is invalid!")
file_name = os.path.basename(uri_object.path)
if file_name is not None and file_name != "":
metadata = opener.info()
file_size = int(metadata.getheaders("Content-Length")[0])
if file_size == 0:
raise argparse.ArgumentTypeError("The URL file '" + file + "' is empty!")
return file
else:
raise argparse.ArgumentTypeError("URL '" + file + "' does not contain any file name!")
def inputfile(file):
# test the format
uri_object = urlparse(file)
# check the file
if uri_object.scheme == '':
return localfile(file)
else:
return urlfile(file)
class MultipleParameters(object):
def __init__(self, types, required, choices, excludes, default, actions):
......@@ -328,34 +400,75 @@ class Parameter(str, AbstractParameter):
class InputFile(Parameter, AbstractFile):
def __new__(self, name, help, file_format="any", default="", type=types.StringType, choices=None,
def __new__(self, name, help, file_format="any", default="", type="localfile", choices=None,
required=False, flag=None, group="default", display_name=None):
if hasattr(type, '__call__'):
type2test = type.__name__
else: type2test = type
if type2test not in INPUTFILE_TYPES:
raise ValueError("InputFile.__new__: wrong type provided: '"+type2test+"', this should be choosen between '"
+ "', '".join(INPUTFILE_TYPES)+"'")
return str.__new__(self, default)
def __init__(self, name, help, file_format="any", default="", type=types.StringType, choices=None,
def __init__(self, name, help, file_format="any", default="", type="localfile", choices=None,
required=False, flag=None, group="default", display_name=None):
AbstractFile.__init__(self, file_format)
Parameter.__init__(self, name, help, flag=flag, type=type, choices=choices,
required=required, group=group, display_name=display_name)
"""
def _prepare_input_files(self):
# handle url inputs
if self.type.__name__ == "inputfile" or self.type.__name__ == "urlfile":
new_args[param], is_uri = self._download_urlfile(args[param])
kparams = {}
for param in parameters:
kparams[param.name] = param
new_args = args
for param in args.keys():
is_uri, is_local = False, False
# handle url inputs
if kparams[param].type.__name__ == "inputfile" or kparams[param].type.__name__ == "urlfile":
if args[param].__class__.__name__ == "list":
for i, val in enumerate(args[param]):
new_args[param][i], is_uri = self._download_urlfile(args[param][i])
else:
new_args[param], is_uri = self._download_urlfile(args[param])
# handle upload inputs
try: is_local = os.path.isfile(args[param])
except: is_local = False
if not is_uri and not is_local and (kparams[param].type.__name__ == "inputfile" or kparams[param].type.__name__ == "browsefile"):
if args[param].__class__.__name__ == "list":
for i, val in enumerate(args[param]):
if args[param][i]:
new_args[param][i] = os.path.join(self.jflow_config_reader.get_tmp_directory(), (args[param][i]))
else:
if args[param]:
new_args[param] = os.path.join(self.jflow_config_reader.get_tmp_directory(), (args[param]))
return new_args
"""
class OutputFile(Parameter, AbstractFile):
def __new__(self, name, help, file_format="any", default="", type=types.StringType, choices=None,
def __new__(self, name, help, file_format="any", default="", choices=None,
required=False, flag=None, group="default", display_name=None):
return str.__new__(self, default)
def __init__(self, name, help, file_format="any", default="", type=types.StringType, choices=None,
def __init__(self, name, help, file_format="any", default="", choices=None,
required=False, flag=None, group="default", display_name=None):
AbstractFile.__init__(self, file_format)
Parameter.__init__(self, name, help, flag=flag, type=type, choices=choices,
Parameter.__init__(self, name, help, flag=flag, type="localfile", choices=choices,
required=required, group=group, display_name=display_name)
class ParameterList(list, AbstractParameter):
def __init__(self, name, help, default=[], type=types.StringType, choices=None, required=False,
def __init__(self, name, help, default=None, type=types.StringType, choices=None, required=False,
flag=None, sub_parameters=None, group="default", display_name=None):
if default == None: default = []
AbstractParameter.__init__(self, name, help, flag=flag, type=type, choices=choices, required=required,
action="append", sub_parameters=sub_parameters, group=group, display_name=display_name)
if default.__class__.__name__ == "str":
......@@ -366,8 +479,19 @@ class ParameterList(list, AbstractParameter):
class InputFileList(ParameterList, AbstractFile):
def __init__(self, name, help, file_format="any", default=[], type=types.StringType, choices=None,
def __init__(self, name, help, file_format="any", default=None, type="localfile", choices=None,
required=False, flag=None, group="default", display_name=None):
if default == None: default = []
if hasattr(type, '__call__'):
type2test = type.__name__
else: type2test = type
if type2test not in INPUTFILE_TYPES:
raise ValueError("InputFile.__new__: wrong type provided: '"+type2test+"', this should be choosen between '"
+ "', '".join(INPUTFILE_TYPES)+"'")
AbstractFile.__init__(self, file_format)
ParameterList.__init__(self, name, help, flag=flag, type=type, choices=choices,
required=required, group=group, display_name=display_name)
......@@ -379,10 +503,11 @@ class InputFileList(ParameterList, AbstractFile):
class OutputFileList(ParameterList, AbstractFile):
def __init__(self, name, help, file_format="any", default=[], type=types.StringType,
choices=None, required=False, flag=None, group="default", display_name=None):
def __init__(self, name, help, file_format="any", default=None, choices=None,
required=False, flag=None, group="default", display_name=None):
if default == None: default = []
AbstractFile.__init__(self, file_format)
ParameterList.__init__(self, name, help, flag=flag, type=type, choices=choices,
ParameterList.__init__(self, name, help, flag=flag, type="localfile", choices=choices,
required=required, group=group, display_name=display_name)
if default.__class__.__name__ == "str":
return list.__init__(self, [default])
......
......@@ -25,14 +25,12 @@ import tempfile
import pickle
import time
import threading
import urllib2
import types
import logging
from logging import FileHandler
from ConfigParser import ConfigParser, NoOptionError
from inspect import getcallargs
from urlparse import urlparse
import jflow
import jflow.utils as utils
......@@ -63,7 +61,6 @@ class Workflow(threading.Thread):
STATUS_COMPLETED = "completed"
STATUS_FAILED = "failed"
STATUS_ABORTED = "aborted"
INPUTFILE_TYPES = ["inputfile", "localfile", "urlfile", "browsefile"]
def __init__(self, args={}, id=None, function= "process", parameters_section=None):
......@@ -121,9 +118,6 @@ class Workflow(threading.Thread):
def add_input_file(self, name, help, file_format="any", default=None, type="inputfile", choices=None,
required=False, flag=None, group="default", display_name=None, is_list=False, add_to=None):
if type not in self.INPUTFILE_TYPES:
raise ValueError("Workflow.add_input_file: wrong type provided: '"+type+"', this should be choosen between '"
+ "', '".join(self.INPUTFILE_TYPES)+"'")
if is_list:
if default == None: default = []
new_param = InputFileList(name, help, flag=flag, file_format=file_format, default=default, type=type,
......@@ -170,9 +164,6 @@ class Workflow(threading.Thread):
def add_parameter(self, name, help, default=None, type=types.StringType, choices=None, required=False,
flag=None, group="default", display_name=None, is_list=False, add_to=None):
if type in self.INPUTFILE_TYPES:
raise ValueError("Workflow.add_parameter: wrong type provided: '"+type+"', this should not be choosen between '"
+ "', '".join(self.INPUTFILE_TYPES)+"'")
if is_list:
if default == None: default = []
new_param = ParameterList(name, help, flag=flag, default=default, type=type, choices=choices,
......@@ -261,58 +252,6 @@ class Workflow(threading.Thread):
else:
final_sub_args[sub_param.name] = sub_param.default
self.__getattribute__(param.name).append(sub_args)
def _download_urlfile(self, input):
try:
uri_object = urlparse(input)
opener = urllib2.urlopen(input)
block_size = 8000
jflowconf = JFlowConfigReader()
tmp_directory = os.path.join(jflowconf.get_tmp_directory(), os.path.basename(tempfile.NamedTemporaryFile().name))
os.mkdir(tmp_directory)
file_path = os.path.join(tmp_directory, os.path.basename(uri_object.path))
local_file = open(file_path, 'wb')
if os.path.basename(uri_object.path) is not None and os.path.basename(uri_object.path) != "":
metadata = opener.info()
file_size = int(metadata.getheaders("Content-Length")[0])
while True:
buffer = opener.read(block_size)
# End of download
if not buffer: break
# Parts of download
local_file.write(buffer)
local_file.close()
logging.getLogger("types.urlfile").debug("URL file '{0}' successfully downloaded as: {1}".format(input, file_path))
return [file_path, True]
except:
return [input, False]
def _prepare_input_files(self, parameters, args):
kparams = {}
for param in parameters:
kparams[param.name] = param
new_args = args
for param in args.keys():
is_uri, is_local = False, False
# handle url inputs
if kparams[param].type.__name__ == "inputfile" or kparams[param].type.__name__ == "urlfile":
if args[param].__class__.__name__ == "list":
for i, val in enumerate(args[param]):
new_args[param][i], is_uri = self._download_urlfile(args[param][i])
else:
new_args[param], is_uri = self._download_urlfile(args[param])
# handle upload inputs
try: is_local = os.path.isfile(args[param])
except: is_local = False
if not is_uri and not is_local and (kparams[param].type.__name__ == "inputfile" or kparams[param].type.__name__ == "browsefile"):
if args[param].__class__.__name__ == "list":
for i, val in enumerate(args[param]):
if args[param][i]:
new_args[param][i] = os.path.join(self.jflow_config_reader.get_tmp_directory(), (args[param][i]))
else:
if args[param]:
new_args[param] = os.path.join(self.jflow_config_reader.get_tmp_directory(), (args[param]))
return new_args
def set_stderr(self):
if hasattr(self, "stderr") and self.stderr is not None:
......
......@@ -15,53 +15,10 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
import os
import argparse
import datetime
import urllib2
import tempfile
import logging
from urlparse import urlparse
from jflow.config_reader import JFlowConfigReader
def date(datestr):
return datetime.datetime.strptime(datestr, '%d/%m/%Y')
def browsefile(file):
# browsefile are not available from command line, considere it as a localfile
# from the gui, this will not been tested this way
return localfile(file)
def localfile(file):
if os.path.isfile(file):
return os.path.abspath(file)
else:
raise argparse.ArgumentTypeError("Local file '" + file + "' does not exists! Please provide a valid file path!")
def urlfile(file):
uri_object = urlparse(file)
try:
opener = urllib2.urlopen(file)
except:
raise argparse.ArgumentTypeError("URL '" + file + "' is invalid!")
file_name = os.path.basename(uri_object.path)
if file_name is not None and file_name != "":
metadata = opener.info()
file_size = int(metadata.getheaders("Content-Length")[0])
if file_size == 0:
raise argparse.ArgumentTypeError("The URL file '" + file + "' is empty!")
return file
else:
raise argparse.ArgumentTypeError("URL '" + file + "' does not contain any file name!")
def inputfile(file):
# test the format
uri_object = urlparse(file)
# check the file
if uri_object.scheme == '':
return localfile(file)
else:
return urlfile(file)
\ No newline at end of file
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment