Commit 009b6a0a authored by Jerome Mariette's avatar Jerome Mariette
Browse files

No commit message

No commit message
parent 53f35b35
...@@ -17,6 +17,7 @@ ...@@ -17,6 +17,7 @@
import types import types
import datetime import datetime
import argparse
# import custom types # import custom types
from workflows.types import * from workflows.types import *
...@@ -24,41 +25,47 @@ from workflows.types import * ...@@ -24,41 +25,47 @@ from workflows.types import *
class MiltipleParameters(object): class MiltipleParameters(object):
def __init__(self, types): def __init__(self, types):
self.types = types self.types = types
self.index = 0 self.index = None
def __repr__(self): self.__name__ = "MiltipleParameters"
return self.types[self.index].__name__
def __call__(self, arg): def __call__(self, arg):
value = self.types[self.index](arg) parts = arg.split("=")
self.index = (self.index + 1) % len(self.types) if not self.types.has_key(parts[0]):
return value raise argparse.ArgumentTypeError(parts[0] + " is an invalid flag! Available ones are: "+", ".join(self.types.keys()))
try:
value = self.types[parts[0]](parts[1])
except:
raise argparse.ArgumentTypeError("invalid " + self.types[parts[0]].__name__ + " value: '" + parts[1] + "' for sub parameter '" + parts[0] + "'")
self.index = parts[0]
return (parts[0], value)
class Parameter(object): class Parameter(object):
""" """
""" """
def __init__(self, name, help, flag=None, default=None, type=types.StringType, choices=None, def __init__(self, name, help, flag, default=None, type=types.StringType, choices=None,
required=False, action="store", sub_parameters=[]): required=False, action="store", sub_parameters=[]):
self.name = name self.name = name
self.flag = flag self.flag = flag
self.help = help self.help = help
self.action = action self.action = action
self.nargs = 1 self.nargs = None
self.sub_parameters = sub_parameters self.sub_parameters = sub_parameters
if len(self.sub_parameters) > 0: if len(self.sub_parameters) > 0:
self.nargs = len(self.sub_parameters) self.nargs = "+"
if type == "date": if type == "date":
self.type = date self.type = date
elif type == "multiple": elif type == "multiple":
sub_param_types, sub_param_names = [], [] sub_param_hash, sub_param_types, sub_param_names = {}, [], []
for sub_param in self.sub_parameters: for sub_param in self.sub_parameters:
sub_param_names.append(sub_param.name) try: sub_type = sub_param.type
try: sub_param_types.append(sub_param.type) except: sub_type = types.StringType
except: sub_param_types.append(types.StringType) sub_param_hash[sub_param.flag] = sub_type
self.type = MiltipleParameters(sub_param_types) sub_param_names.append(sub_param.flag)
self.help += " | Format: " + " ".join([cname + "(" + ctype.__name__.upper() + ")" for ctype, cname in zip(sub_param_types, sub_param_names)]) + ")" sub_param_types.append(sub_type)
self.type = MiltipleParameters(sub_param_hash)
self.help += " | Format: " + " ".join([cname + "=<" + ctype.__name__.upper() + ">" for ctype, cname in zip(sub_param_types, sub_param_names)]) + ")"
elif isinstance(type, types.FunctionType): elif isinstance(type, types.FunctionType):
self.type = type self.type = type
else: else:
...@@ -77,13 +84,13 @@ class Parameter(object): ...@@ -77,13 +84,13 @@ class Parameter(object):
elif self.type == types.BooleanType : elif self.type == types.BooleanType :
if self.default: self.default = str(self.default).lower() in ("true", "t", "1") if self.default: self.default = str(self.default).lower() in ("true", "t", "1")
else: self.default = True else: self.default = True
elif type == "multiple":
self.default = []
for param in self.sub_parameters:
self.default.append(param.default)
elif self.action == "append": elif self.action == "append":
self.default = [] self.default = []
elif type == "multiple":
self.default = {}
for param in self.sub_parameters:
self.default[param.name] = param.default
def export_to_argparse(self): def export_to_argparse(self):
if self.type == types.BooleanType and str(self.default).lower() in ("false", "f", "0"): if self.type == types.BooleanType and str(self.default).lower() in ("false", "f", "0"):
return {"help": self.help, "required": self.required, "dest": self.name, return {"help": self.help, "required": self.required, "dest": self.name,
...@@ -94,7 +101,7 @@ class Parameter(object): ...@@ -94,7 +101,7 @@ class Parameter(object):
elif self.nargs > 1: elif self.nargs > 1:
return {"type": self.type, "help": self.help, "required": self.required, return {"type": self.type, "help": self.help, "required": self.required,
"dest": self.name, "default": self.default, "dest": self.name, "default": self.default,
"action": self.action, "choices": self.choices, "nargs": self.nargs} "action": self.action, "choices": self.choices, "nargs": "+"}
else: else:
return {"type": self.type, "help": self.help, "required": self.required, return {"type": self.type, "help": self.help, "required": self.required,
"dest": self.name, "default": self.default, "dest": self.name, "default": self.default,
...@@ -105,4 +112,4 @@ class Parameter(object): ...@@ -105,4 +112,4 @@ class Parameter(object):
def __str__(self): def __str__(self):
return self.name + ": " + str(self.flag) + " | " + self.help + " (default=" + \ return self.name + ": " + str(self.flag) + " | " + self.help + " (default=" + \
str(self.default) + ", required=" + str(self.required) + ")" str(self.default) + ", required=" + str(self.required) + ", flag=" + self.flag + ")"
...@@ -28,14 +28,13 @@ import threading ...@@ -28,14 +28,13 @@ import threading
from ConfigParser import ConfigParser, NoOptionError from ConfigParser import ConfigParser, NoOptionError
import types import types
import logging import logging
from inspect import getcallargs
import jflow import jflow
import jflow.utils as utils import jflow.utils as utils
from jflow.workflows_manager import WorkflowsManager, JFlowConfigReader from jflow.workflows_manager import WorkflowsManager, JFlowConfigReader
from jflow.parameter import Parameter, date from jflow.parameter import Parameter, date
from inspect import getcallargs
from weaver.script import ABSTRACTIONS from weaver.script import ABSTRACTIONS
from weaver.script import DATASETS from weaver.script import DATASETS
from weaver.script import FUNCTIONS from weaver.script import FUNCTIONS
...@@ -60,7 +59,7 @@ class Workflow(threading.Thread): ...@@ -60,7 +59,7 @@ class Workflow(threading.Thread):
STATUS_COMPLETED = "completed" STATUS_COMPLETED = "completed"
STATUS_FAILED = "failed" STATUS_FAILED = "failed"
STATUS_ABORTED = "aborted" STATUS_ABORTED = "aborted"
def __init__(self, args={}, id=None, function= "process", parameters_section="parameters"): def __init__(self, args={}, id=None, function= "process", parameters_section="parameters"):
""" """
""" """
...@@ -243,15 +242,26 @@ class Workflow(threading.Thread): ...@@ -243,15 +242,26 @@ class Workflow(threading.Thread):
self._serialize() self._serialize()
def _extend_and_format_args(self, parameters, args): def _extend_and_format_args(self, parameters, args):
extended_args = {} extended_args = {}
for param in parameters: for param in parameters:
try: args[param.name] = args[param.name].encode('ascii','ignore') try: args[param.name] = args[param.name].encode('ascii','ignore')
except: pass except: pass
# if this parameter has been modified by the user # if this parameter has been modified by the user
if args.has_key(param.name): if args.has_key(param.name):
if param.action == "append" and param.type.__name__ == "MiltipleParameters":
extended_args[param.name] = []
for arg in args[param.name]:
sub_hash = {}
for sub_param in param.sub_parameters:
# find the value
value = None
for values in arg:
if values[0] == sub_param.flag: value = values[1]
if value == None: value = sub_param.default
sub_hash[sub_param.name] = value
extended_args[param.name].append(sub_hash)
# if it's an append parameter, the result should be a list # if it's an append parameter, the result should be a list
if param.action == "append": elif param.action == "append":
if args[param.name]: if args[param.name]:
try: value = args[param.name].split(",") try: value = args[param.name].split(",")
except: value = args[param.name] except: value = args[param.name]
...@@ -266,11 +276,25 @@ class Workflow(threading.Thread): ...@@ -266,11 +276,25 @@ class Workflow(threading.Thread):
# if it's a boolean # if it's a boolean
elif param.type == types.BooleanType: elif param.type == types.BooleanType:
extended_args[param.name] = args[param.name] in [True, 1, 'true', '1', 't', 'y', 'yes'] extended_args[param.name] = args[param.name] in [True, 1, 'true', '1', 't', 'y', 'yes']
elif param.type.__name__ == "MiltipleParameters":
extended_args[param.name] = {}
for sub_param in param.sub_parameters:
# find the value
value = None
for values in args[param.name]:
if values[0] == sub_param.flag: value = values[1]
if value == None: value = sub_param.default
extended_args[param.name][sub_param.name] = value
else: else:
extended_args[param.name] = args[param.name] extended_args[param.name] = args[param.name]
# otherwise use the default # otherwise use the default
else: else:
extended_args[param.name] = param.default if param.type.__name__ == "MiltipleParameters":
extended_args[param.name] = {}
for sub_param in param.sub_parameters:
extended_args[param.name][sub_param.name] = sub_param.default
else:
extended_args[param.name] = param.default
return extended_args return extended_args
def _get_from_config(self): def _get_from_config(self):
...@@ -303,7 +327,8 @@ class Workflow(threading.Thread): ...@@ -303,7 +327,8 @@ class Workflow(threading.Thread):
else: else:
current_params[param] = params[param_name][param] current_params[param] = params[param_name][param]
for sub_param in sorted(sub_params.keys()): for sub_param in sorted(sub_params.keys()):
sub_objs.append(Parameter(**sub_params[sub_param])) sub = Parameter(**sub_params[sub_param])
sub_objs.append(sub)
current_params["sub_parameters"] = sub_objs current_params["sub_parameters"] = sub_objs
oparam = Parameter(**current_params) oparam = Parameter(**current_params)
parameters.append(oparam) parameters.append(oparam)
......
...@@ -64,7 +64,6 @@ class WorkflowsManager(object): ...@@ -64,7 +64,6 @@ class WorkflowsManager(object):
workflow.start() workflow.start()
def run_workflow(self, workflow_class, args, function="process", parameters_section="parameters"): def run_workflow(self, workflow_class, args, function="process", parameters_section="parameters"):
print parameters_section
# Load all modules within the workflow module # Load all modules within the workflow module
for importer, modname, ispkg in pkgutil.iter_modules(workflows.__path__, workflows.__name__ + "."): for importer, modname, ispkg in pkgutil.iter_modules(workflows.__path__, workflows.__name__ + "."):
__import__(modname) __import__(modname)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment