Commit ced62122 authored by Jerome Mariette's avatar Jerome Mariette
Browse files

url, browse and co OK

parent 39ee8c0f
......@@ -30,31 +30,6 @@ from jflow.config_reader import JFlowConfigReader
# import custom types
from workflows.types import *
# function in charge to download an url file
def download_urlfile(input):
try:
uri_object = urlparse(input)
opener = urllib2.urlopen(input)
block_size = 8000
jflowconf = JFlowConfigReader()
tmp_directory = os.path.join(jflowconf.get_tmp_directory(), os.path.basename(tempfile.NamedTemporaryFile().name))
os.mkdir(tmp_directory)
file_path = os.path.join(tmp_directory, os.path.basename(uri_object.path))
local_file = open(file_path, 'wb')
if os.path.basename(uri_object.path) is not None and os.path.basename(uri_object.path) != "":
metadata = opener.info()
file_size = int(metadata.getheaders("Content-Length")[0])
while True:
buffer = opener.read(block_size)
# End of download
if not buffer: break
# Parts of download
local_file.write(buffer)
local_file.close()
logging.getLogger("types.urlfile").debug("URL file '{0}' successfully downloaded as: {1}".format(input, file_path))
return [file_path, True]
except:
return [input, False]
# define all input type available
INPUTFILE_TYPES = ["inputfile", "localfile", "urlfile", "browsefile"]
......@@ -418,38 +393,7 @@ class InputFile(Parameter, AbstractFile):
AbstractFile.__init__(self, file_format)
Parameter.__init__(self, name, help, flag=flag, type=type, choices=choices,
required=required, group=group, display_name=display_name)
"""
def _prepare_input_files(self):
# handle url inputs
if self.type.__name__ == "inputfile" or self.type.__name__ == "urlfile":
new_args[param], is_uri = self._download_urlfile(args[param])
kparams = {}
for param in parameters:
kparams[param.name] = param
new_args = args
for param in args.keys():
is_uri, is_local = False, False
# handle url inputs
if kparams[param].type.__name__ == "inputfile" or kparams[param].type.__name__ == "urlfile":
if args[param].__class__.__name__ == "list":
for i, val in enumerate(args[param]):
new_args[param][i], is_uri = self._download_urlfile(args[param][i])
else:
new_args[param], is_uri = self._download_urlfile(args[param])
# handle upload inputs
try: is_local = os.path.isfile(args[param])
except: is_local = False
if not is_uri and not is_local and (kparams[param].type.__name__ == "inputfile" or kparams[param].type.__name__ == "browsefile"):
if args[param].__class__.__name__ == "list":
for i, val in enumerate(args[param]):
if args[param][i]:
new_args[param][i] = os.path.join(self.jflow_config_reader.get_tmp_directory(), (args[param][i]))
else:
if args[param]:
new_args[param] = os.path.join(self.jflow_config_reader.get_tmp_directory(), (args[param]))
return new_args
"""
class OutputFile(Parameter, AbstractFile):
......
......@@ -210,6 +210,32 @@ class Workflow(threading.Thread):
attribute_value.type.excludes[new_group] = flags2exclude
break
# function in charge to download an url file
def _download_urlfile(self, input):
try:
uri_object = urlparse(input)
opener = urllib2.urlopen(input)
block_size = 8000
jflowconf = JFlowConfigReader()
tmp_directory = os.path.join(jflowconf.get_tmp_directory(), os.path.basename(tempfile.NamedTemporaryFile().name))
os.mkdir(tmp_directory)
file_path = os.path.join(tmp_directory, os.path.basename(uri_object.path))
local_file = open(file_path, 'wb')
if os.path.basename(uri_object.path) is not None and os.path.basename(uri_object.path) != "":
metadata = opener.info()
file_size = int(metadata.getheaders("Content-Length")[0])
while True:
buffer = opener.read(block_size)
# End of download
if not buffer: break
# Parts of download
local_file.write(buffer)
local_file.close()
logging.getLogger("types.urlfile").debug("URL file '{0}' successfully downloaded as: {1}".format(input, file_path))
return [file_path, True]
except:
return [input, False]
def _set_parameters(self, args):
parameters = self.get_parameters()
for param in parameters:
......@@ -221,7 +247,14 @@ class Workflow(threading.Thread):
required=param.required, flag=param.flag, group=param.group, display_name=param.display_name)
self.__setattr__(param.name, new_param)
elif param.__class__ == InputFile:
new_param = InputFile(param.name, param.help, file_format=param.file_format, default=args[param.name], type=param.type,
# handle url inputs
default, is_uri = self._download_urlfile(args[param.name])
# handle upload inputs
try: is_local = os.path.isfile(args[param])
except: is_local = False
if not is_uri and not is_local and param.type.__name__ == "inputfile" or param.type.__name__ == "browsefile":
default = os.path.join(self.jflow_config_reader.get_tmp_directory(), (args[param.name]))
new_param = InputFile(param.name, param.help, file_format=param.file_format, default=default, type=param.type,
choices=param.choices, required=param.required, flag=param.flag,
group=param.group, display_name=param.display_name)
self.__setattr__(param.name, new_param)
......@@ -230,7 +263,27 @@ class Workflow(threading.Thread):
choices=param.choices, required=param.required, flag=param.flag,
group=param.group, display_name=param.display_name)
self.__setattr__(param.name, new_param)
elif param.__class__ == ParameterList or param.__class__ == InputFileList or param.__class__ == OutputFileList:
elif param.__class__ == InputFileList:
# handle url inputs
default_vals, is_uri = [], False
if args[param.name].__class__.__name__ == "list":
for path in args[param.name]:
new_url, is_uri = self._download_urlfile(path)
if is_uri: default_vals.append(new_url)
else:
new_url, is_uri = self._download_urlfile(args[param.name])
if is_uri: default_vals.append(new_url)
# handle upload inputs
try: is_local = os.path.isfile(args[param])
except: is_local = False
if not is_uri and not is_local and param.type.__name__ == "inputfile" or param.type.__name__ == "browsefile":
if args[param.name].__class__.__name__ == "list":
for path in args[param.name]:
default_vals.append(os.path.join(self.jflow_config_reader.get_tmp_directory(), (path)))
else:
default_vals.append(os.path.join(self.jflow_config_reader.get_tmp_directory(), (args[param.name])))
self.__getattribute__(param.name).extend(default_vals)
elif param.__class__ == ParameterList or param.__class__ == OutputFileList:
self.__getattribute__(param.name).extend(args[param.name])
elif param.__class__ == MultiParameter:
sub_args = {}
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment