Commit ebee8531 authored by Jerome Mariette's avatar Jerome Mariette
Browse files

improve code

parent 053677a5
......@@ -602,8 +602,31 @@ class AbstractInputFile(AbstractIOFile):
"""
@summary : Parent of all InputFile(s) parameters.
"""
pass
def _download_urlfile(self, input):
try:
uri_object = urlparse(input)
opener = urllib2.urlopen(input)
block_size = 8000
jflowconf = JFlowConfigReader()
tmp_directory = os.path.join(jflowconf.get_tmp_directory(), os.path.basename(tempfile.NamedTemporaryFile().name))
os.mkdir(tmp_directory)
file_path = os.path.join(tmp_directory, os.path.basename(uri_object.path))
local_file = open(file_path, 'wb')
if os.path.basename(uri_object.path) is not None and os.path.basename(uri_object.path) != "":
metadata = opener.info()
file_size = int(metadata.getheaders("Content-Length")[0])
while True:
buffer = opener.read(block_size)
# End of download
if not buffer: break
# Parts of download
local_file.write(buffer)
local_file.close()
logging.getLogger("AbstractInputFile.download_urlfile").debug("URL file '{0}' successfully downloaded as: {1}".format(input, file_path))
return [file_path, True]
except:
return [input, False]
class AbstractOutputFile(AbstractIOFile):
"""
......@@ -634,6 +657,19 @@ class InputFile(StrParameter, AbstractInputFile):
StrParameter.__init__(self, name, help, flag=flag, default=default, type=type, choices=choices,
required=required, group=group, display_name=display_name)
def prepare_input_file(self, input):
# handle url inputs
new_path, is_uri = self._download_urlfile(input)
# handle upload inputs
try: is_local = os.path.isfile(input)
except: is_local = False
if not is_uri and not is_local and self.type.__name__ == "inputfile" or self.type.__name__ == "browsefile":
jflow_config_reader = JFlowConfigReader()
new_path = os.path.join(jflow_config_reader.get_tmp_directory(), input)
if is_local: new_path = input
return new_path
class OutputFile(StrParameter, AbstractOutputFile):
def __new__(self, name, help, file_format="any", default="", choices=None,
......@@ -692,6 +728,31 @@ class InputFileList(ParameterList, AbstractInputFile):
elif issubclass( default.__class__, AbstractOutputFile ):
return list.__init__(self, default)
def prepare_input_files(self, inputs):
path2test = inputs
if not path2test.__class__.__name__ == "list":
path2test = [path2test]
new_vals, done = [], []
# handle url inputs
for path in path2test:
new_url, is_uri = self._download_urlfile(path)
if is_uri:
new_vals.append(new_url)
done.append(path)
for d in done: path2test.remove(d)
done = []
# handle localfile
for path in path2test:
if os.path.isfile(path):
new_vals.append(path)
done.append(path)
for d in done: path2test.remove(d)
# handle upload inputs
if self.type.__name__ == "inputfile" or self.type.__name__ == "browsefile":
for path in path2test:
jflow_config_reader = JFlowConfigReader()
new_vals.append(os.path.join(jflow_config_reader.get_tmp_directory(), path))
return new_vals
class OutputFileList(ParameterList, AbstractOutputFile):
......
......@@ -212,32 +212,6 @@ class Workflow(threading.Thread):
flags2exclude.append(paramsexclude.flag)
attribute_value.type.excludes[new_group] = flags2exclude
break
# function in charge to download an url file
def _download_urlfile(self, input):
try:
uri_object = urlparse(input)
opener = urllib2.urlopen(input)
block_size = 8000
jflowconf = JFlowConfigReader()
tmp_directory = os.path.join(jflowconf.get_tmp_directory(), os.path.basename(tempfile.NamedTemporaryFile().name))
os.mkdir(tmp_directory)
file_path = os.path.join(tmp_directory, os.path.basename(uri_object.path))
local_file = open(file_path, 'wb')
if os.path.basename(uri_object.path) is not None and os.path.basename(uri_object.path) != "":
metadata = opener.info()
file_size = int(metadata.getheaders("Content-Length")[0])
while True:
buffer = opener.read(block_size)
# End of download
if not buffer: break
# Parts of download
local_file.write(buffer)
local_file.close()
logging.getLogger("types.urlfile").debug("URL file '{0}' successfully downloaded as: {1}".format(input, file_path))
return [file_path, True]
except:
return [input, False]
def _set_parameters(self, args):
parameters = self.get_parameters()
......@@ -252,13 +226,7 @@ class Workflow(threading.Thread):
self.__setattr__(param.name, new_param)
elif param.__class__ == InputFile:
if args[param.name]:
# handle url inputs
default, is_uri = self._download_urlfile(args[param.name])
# handle upload inputs
try: is_local = os.path.isfile(args[param])
except: is_local = False
if not is_uri and not is_local and param.type.__name__ == "inputfile" or param.type.__name__ == "browsefile":
default = os.path.join(self.jflow_config_reader.get_tmp_directory(), (args[param.name]))
default = param.prepare_input_file(args[param.name])
else:
default = param.default
new_param = InputFile(param.name, param.help, file_format=param.file_format, default=default, type=param.type,
......@@ -267,26 +235,7 @@ class Workflow(threading.Thread):
self.__setattr__(param.name, new_param)
elif param.__class__ == InputFileList:
if args[param.name]:
path2test = args[param.name]
if not path2test.__class__.__name__ == "list":
path2test = [path2test]
default_vals = []
# handle url inputs
for path in path2test:
new_url, is_uri = self._download_urlfile(path)
if is_uri:
default_vals.append(new_url)
path2test.remove(path)
# handle localfile
for path in path2test:
if os.path.isfile(path):
default_vals.append(path)
path2test.remove(path)
# handle upload inputs
if param.type.__name__ == "inputfile" or param.type.__name__ == "browsefile":
for path in path2test:
default_vals.append(os.path.join(self.jflow_config_reader.get_tmp_directory(), (path)))
self.__setattr__(param.name, default_vals)
self.__setattr__(param.name, param.prepare_input_files(args[param.name]))
else:
self.__setattr__(param.name, [])
elif param.__class__ == ParameterList:
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment