Commit 633a7c2f authored by Jerome Mariette's avatar Jerome Mariette
Browse files

clean some code

parent 2b447a82
......@@ -210,161 +210,57 @@ class Workflow(threading.Thread):
else:
self.__getattribute__(param.name)[sub_param.name] = self.__getattribute__(param.name).default[sub_param.name]
"""
def _get_from_config(self, parameters_section):
reader = ConfigParser()
reader.read(self._get_property_path())
parameters, param_order = [], []
def _download_urlfile(self, input):
try:
params, sub_order, excludes, hash_params = {}, {}, [], {}
# first grab all parameters name
for param in reader.items(parameters_section):
cvalue = reader.get(parameters_section, param[0])
cparam = param[0].split(".")[0]
ckey = ".".join(param[0].split(".")[1:])
if len(ckey.split(".")) > 1:
if sub_order.has_key(cparam):
if ckey.split(".")[0] not in sub_order[cparam]: sub_order[cparam].append(ckey.split(".")[0])
else:
sub_order[cparam] =[ckey.split(".")[0]]
if not params.has_key(cparam):
params[cparam] = {ckey: cvalue}
else:
params[cparam][ckey] = cvalue
if cparam not in param_order: param_order.append(cparam)
for param_name in param_order:
sub_params, current_params, sub_objs, sub_excludes, hash_sub_params = {}, {}, [], [], {}
for param in params[param_name]:
sub_params_values = param.split(".")
if len(sub_params_values) == 2:
# TODO: handle exclusive for sub parameter
key = None
if sub_params_values[1] == "name":
key = "display_name"
else:
key = sub_params_values[1]
if key == "exclude":
found = False
for exclude_group in sub_excludes:
if sub_params_values[0] in exclude_group:
exclude_group.append(params[param_name][param])
found = True
elif params[param_name][param] in exclude_group:
exclude_group.append(sub_params_values[0])
found = True
if not found: sub_excludes.append([sub_params_values[0], params[param_name][param]])
else:
if not sub_params.has_key(sub_params_values[0]):
sub_params[sub_params_values[0]] = {key: params[param_name][param]}
sub_params[sub_params_values[0]]["name"] = sub_params_values[0]
else:
sub_params[sub_params_values[0]][key] = params[param_name][param]
else:
if param == "name": current_params["display_name"] = params[param_name][param]
elif param == "exclude":
found = False
for exclude_group in excludes:
if param_name in exclude_group:
exclude_group.append(params[param_name][param])
found = True
elif params[param_name][param] in exclude_group:
exclude_group.append(param_name)
found = True
if not found: excludes.append([param_name, params[param_name][param]])
else: current_params[param] = params[param_name][param]
current_params["name"] = param_name
if sub_order.has_key(param_name):
for sub_param in sub_order[param_name]:
sub = Parameter(**sub_params[sub_param])
hash_sub_params[sub.name] = sub
sub_objs.append(sub)
# overwrite the group to exclude some parameters
for i, exclude_group in enumerate(sub_excludes):
group_name = "exclude-"+str(i)
for sparam in set(exclude_group):
hash_sub_params[sparam].group = group_name
current_params["sub_parameters"] = sub_objs
oparam = Parameter(**current_params)
parameters.append(oparam)
hash_params[oparam.name] = oparam
# overwrite the group to exclude some parameters
for i, exclude_group in enumerate(excludes):
group_name = "exclude-"+str(i)
for param in set(exclude_group):
hash_params[param].group = group_name
uri_object = urlparse(input)
opener = urllib2.urlopen(input)
block_size = 8000
jflowconf = JFlowConfigReader()
tmp_directory = os.path.join(jflowconf.get_tmp_directory(), os.path.basename(tempfile.NamedTemporaryFile().name))
os.mkdir(tmp_directory)
file_path = os.path.join(tmp_directory, os.path.basename(uri_object.path))
local_file = open(file_path, 'wb')
if os.path.basename(uri_object.path) is not None and os.path.basename(uri_object.path) != "":
metadata = opener.info()
file_size = int(metadata.getheaders("Content-Length")[0])
while True:
buffer = opener.read(block_size)
# End of download
if not buffer: break
# Parts of download
local_file.write(buffer)
local_file.close()
logging.getLogger("types.urlfile").debug("URL file '{0}' successfully downloaded as: {1}".format(input, file_path))
return [file_path, True]
except:
pass
# finally add the metadata parameter
metadata = Parameter(name="metadata", flag="--metadata", help="Which metadata should be linked to this workflow",
action="append", display_name="Metadata")
parameters.append(metadata)
return [reader.get("global", "name"),
reader.get("global", "description"),
parameters]
return [input, False]
def _extend_and_format_args(self, parameters, args):
extended_args = {}
def _prepare_input_files(self, parameters, args):
kparams = {}
for param in parameters:
try: args[param.name] = args[param.name].encode('ascii','ignore')
except: pass
# if this parameter has been modified by the user
if args.has_key(param.name):
if param.action == MiltipleAppendAction:
extended_args[param.name] = []
for arg in args[param.name]:
sub_hash = {}
for sub_param in param.sub_parameters:
# find the value
value = None
for values in arg:
if values[0] == sub_param.flag:
try: value = values[1].encode('ascii','ignore')
except: value = values[1]
if value == None: value = sub_param.default
sub_hash[sub_param.name] = value
extended_args[param.name].append(sub_hash)
# if it's an append parameter, the result should be a list
elif param.action == "append":
if args[param.name]:
try: value = args[param.name].split(",")
except: value = args[param.name]
else: value = []
extended_args[param.name] = value
# if it's a date
elif param.type == date:
if isinstance(args[param.name], str):
extended_args[param.name] = date(args[param.name])
else:
extended_args[param.name] = args[param.name]
# if it's a boolean
elif param.type == types.BooleanType:
extended_args[param.name] = args[param.name] in [True, 1, 'true', '1', 't', 'y', 'yes']
elif param.type.__name__ == "MultipleParameters":
extended_args[param.name] = {}
for sub_param in param.sub_parameters:
# find the value
value = None
for values in args[param.name]:
if values[0] == sub_param.flag:
try: value = values[1].encode('ascii','ignore')
except: value = values[1]
if value == None: value = sub_param.default
extended_args[param.name][sub_param.name] = value
kparams[param.name] = param
new_args = args
for param in args.keys():
is_uri, is_local = False, False
# handle url inputs
if kparams[param].type.__name__ == "inputfile" or kparams[param].type.__name__ == "urlfile":
if args[param].__class__.__name__ == "list":
for i, val in enumerate(args[param]):
new_args[param][i], is_uri = self._download_urlfile(args[param][i])
else:
extended_args[param.name] = args[param.name]
# otherwise use the default
else:
if param.type.__name__ == "MultipleParameters":
extended_args[param.name] = {}
for sub_param in param.sub_parameters:
extended_args[param.name][sub_param.name] = sub_param.default
new_args[param], is_uri = self._download_urlfile(args[param])
# handle upload inputs
try: is_local = os.path.isfile(args[param])
except: is_local = False
if not is_uri and not is_local and (kparams[param].type.__name__ == "inputfile" or kparams[param].type.__name__ == "browsefile"):
if args[param].__class__.__name__ == "list":
for i, val in enumerate(args[param]):
if args[param][i]:
new_args[param][i] = os.path.join(self.jflow_config_reader.get_tmp_directory(), (args[param][i]))
else:
extended_args[param.name] = param.default
return extended_args
"""
if args[param]:
new_args[param] = os.path.join(self.jflow_config_reader.get_tmp_directory(), (args[param]))
return new_args
def set_stderr(self):
if hasattr(self, "stderr") and self.stderr is not None:
......@@ -628,61 +524,6 @@ class Workflow(threading.Thread):
except:
logging.getLogger("wf." + str(self.id)).exception("RunWorflowException")
raise
def _download_urlfile(self, input):
try:
uri_object = urlparse(input)
opener = urllib2.urlopen(input)
block_size = 8000
jflowconf = JFlowConfigReader()
tmp_directory = os.path.join(jflowconf.get_tmp_directory(), os.path.basename(tempfile.NamedTemporaryFile().name))
os.mkdir(tmp_directory)
file_path = os.path.join(tmp_directory, os.path.basename(uri_object.path))
local_file = open(file_path, 'wb')
if os.path.basename(uri_object.path) is not None and os.path.basename(uri_object.path) != "":
metadata = opener.info()
file_size = int(metadata.getheaders("Content-Length")[0])
while True:
buffer = opener.read(block_size)
# End of download
if not buffer: break
# Parts of download
local_file.write(buffer)
local_file.close()
logging.getLogger("types.urlfile").debug("URL file '{0}' successfully downloaded as: {1}".format(input, file_path))
return [file_path, True]
except:
return [input, False]
def _prepare_input_files(self, parameters, args):
kparams = {}
for param in parameters:
kparams[param.name] = param
new_args = args
for param in args.keys():
is_uri, is_local = False, False
# handle url inputs
if kparams[param].type.__name__ == "inputfile" or kparams[param].type.__name__ == "urlfile":
if args[param].__class__.__name__ == "list":
for i, val in enumerate(args[param]):
new_args[param][i], is_uri = self._download_urlfile(args[param][i])
else:
new_args[param], is_uri = self._download_urlfile(args[param])
# handle upload inputs
try: is_local = os.path.isfile(args[param])
except: is_local = False
if not is_uri and not is_local and (kparams[param].type.__name__ == "inputfile" or kparams[param].type.__name__ == "browsefile"):
if args[param].__class__.__name__ == "list":
for i, val in enumerate(args[param]):
if args[param][i]:
new_args[param][i] = os.path.join(self.jflow_config_reader.get_tmp_directory(), (args[param][i]))
else:
if args[param]:
new_args[param] = os.path.join(self.jflow_config_reader.get_tmp_directory(), (args[param]))
return new_args
def _update_status_from_log(self):
# first update the status from weaver folders
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment