workflow.py

#
# Copyright (C) 2012 INRA
# 
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
# 
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
# 
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
#

import inspect
import os
import re
import sys
import uuid
import pkgutil
import tempfile
import pickle
import time
import threading
import urllib2 
import types
import logging

from ConfigParser import ConfigParser, NoOptionError
from inspect import getcallargs
from urlparse import urlparse

import jflow
import jflow.utils as utils
from jflow.workflows_manager import WorkflowsManager, JFlowConfigReader
from jflow.parameter import Parameter, date, MiltipleAppendAction

from weaver.script import ABSTRACTIONS
from weaver.script import DATASETS
from weaver.script import FUNCTIONS
from weaver.script import NESTS
from weaver.script import OPTIONS
from weaver.script import STACKS
from weaver.nest import Nest
from weaver.options import Options
from cctools.makeflow import MakeflowLog
from cctools.makeflow.log import Node


class Workflow(threading.Thread):
    """
    """

    PROPERTIES_FILE_NAME = "workflow.properties"
    MAKEFLOW_LOG_FILE_NAME = "Makeflow.makeflowlog"
    DUMP_FILE_NAME = ".workflow.dump"
    WORKING = ".working"
    STATUS_STARTED = "started"
    STATUS_COMPLETED = "completed"
    STATUS_FAILED = "failed"
    STATUS_ABORTED = "aborted"

    def __init__(self, args={}, id=None, function= "process", parameters_section="parameters"):
        """ 
        """
        # define as a thread
        threading.Thread.__init__(self)
        self.jflow_config_reader = JFlowConfigReader()
        self.manager = WorkflowsManager()
        self.components_to_exec = []
        self.makes = {}
        self.globals = {}
        self.options = Options()
        self.status = self.STATUS_STARTED
        self.start_time = None
        self.end_time = None
        self.step = None
        self.dynamic_component_present = False
        self.function = function
        self.parameters_section = parameters_section
        self.comp_pckg = self._import_components()
        self.engine_arguments = ''
        self.component_nameids_is_init = False
        self.component_nameids = {}
        # try to parse engine arguments
        try:
            type, options, limit_submission = self.jflow_config_reader.get_batch()
            if limit_submission : self.engine_arguments += ' -J ' + str(limit_submission)
            if type: self.engine_arguments += ' -T ' + type
            if options : self.engine_arguments += ' -B "' + options + '"'
        except: self.engine_arguments = None
        try:
            # Add config information
            self.name, self.description, self.parameters = self._get_from_config(self.parameters_section)
        except:
            raise IOError(self.__class__.__name__ + " workflow property file not found or invalid.")
        
        self.id = id
        self.args = self._extend_and_format_args(self.parameters, args)
        self.args = self._prepare_input_files(self.args)
        self.metadata = self.args["metadata"]
        if self.id is not None:
            self.directory = self.manager.get_workflow_directory(self.name, self.id)
            if not os.path.isdir(self.directory):
                os.makedirs(self.directory, 0751)
            self._serialize()
    
    @staticmethod
    def config_parser(arg_lines):
        for arg in arg_lines:
            yield arg
    
    def __setstate__(self, state):
        self.__dict__ = state.copy()
        threading.Thread.__init__(self, name=self.name)
        
    def __getstate__(self):
        """
        Threading uses Lock Object, do not consider these objects when serializing a workflow
        """
        odict = self.__dict__.copy()
        del odict['_Thread__started']
        del odict['_Thread__block']
        del odict['_Thread__stderr']
        return odict
    
    def get_parameters_per_groups(self, parameters_section="parameters"):
        name, description, parameters = self._get_from_config(parameters_section)
        pgparameters, parameters_order = {}, []
        for param in parameters:
            if param.group not in parameters_order: parameters_order.append(param.group)
            if pgparameters.has_key(param.group):
                pgparameters[param.group].append(param)
            else:
                pgparameters[param.group] = [param]
        return [pgparameters, parameters_order]
    
    def get_exec_path(self, software):
        exec_path = self.jflow_config_reader.get_exec(software)
        if exec_path is None and os.path.isfile(os.path.join(os.path.dirname(inspect.getfile(self.__class__)), "../bin", software)):
            exec_path = os.path.join(os.path.dirname(inspect.getfile(self.__class__)), "../bin", software)
        elif exec_path is None and os.path.isfile(os.path.join(os.path.dirname(inspect.getfile(self.__class__)), "bin", software)):
            exec_path = os.path.join(os.path.dirname(inspect.getfile(self.__class__)), "bin", software)
        elif exec_path is None and utils.which(software) == None:
            sys.stderr.write("Error: '" + software + "' path connot be retrieved either in the PATH and in the application.properties file!\n")
            sys.exit(1)
        elif exec_path is None and utils.which(software) != None: 
            exec_path = software
        elif exec_path != None and not os.path.isfile(exec_path):
            sys.stderr.write("Error: '" + exec_path + "' set for '" + software + "' does not exists, please provide a valid path!\n")
            sys.exit(1)
        return exec_path
    
    def add_component(self, component_name, args=[], kwargs={}, component_prefix="default"):
        # first build and check if this component is OK
        if self.comp_pckg.has_key(component_name):
            my_pckge = __import__(self.comp_pckg[component_name], globals(), locals(), [component_name], -1)
            # build the object and define required field
            cmpt_object = getattr(my_pckge, component_name)()
            cmpt_object.output_directory = self.get_component_output_directory(component_name, component_prefix)
            cmpt_object.prefix = component_prefix
            if kwargs: cmpt_object.define_parameters(**kwargs)
            else: cmpt_object.define_parameters(*args)
            
            # there is a dynamic component
            if cmpt_object.is_dynamic():
                self.dynamic_component_present = True
                # if already init, add the component to the list and check if weaver should be executed
                if self.component_nameids_is_init:
                    # add the component
                    self.components_to_exec.append(cmpt_object)
                    self._execute_weaver()
                    # update outputs
                    for output in cmpt_object.get_dynamic_outputs():
                        output.update()
                else:
                    if self._component_is_duplicated(cmpt_object):
                        raise ValueError("Component " + cmpt_object.__class__.__name__ + " with prefix " + 
                                            cmpt_object.prefix + " already exist in this pipeline!")
                    self.component_nameids[cmpt_object.get_nameid()] = None
                    self.components_to_exec = []
            else:
                if self.component_nameids_is_init:
                    # add the component
                    self.components_to_exec.append(cmpt_object)
                elif not self.component_nameids_is_init and not self.dynamic_component_present:
                    if self._component_is_duplicated(cmpt_object):
                        raise ValueError("Component " + cmpt_object.__class__.__name__ + " with prefix " + 
                                            cmpt_object.prefix + " already exist in this pipeline!")
                    self.components_to_exec.append(cmpt_object)
                else:
                    if self._component_is_duplicated(cmpt_object):
                        raise ValueError("Component " + cmpt_object.__class__.__name__ + " with prefix " + 
                                            cmpt_object.prefix + " already exist in this pipeline!")
                    self.component_nameids[cmpt_object.get_nameid()] = None

            return cmpt_object
        else:
            raise ImportError(component_name + " component cannot be loaded, available components are: {0}".format(", ".join(self.comp_pckg.keys())))
    
    def pre_process(self):
        pass
    
    def process(self):
        """ 
        Run the workflow, has to be implemented by subclasses
        """
        raise NotImplementedError
    
    def post_process(self):
        pass
    
    def get_temporary_file(self, suffix=".txt"):
        # first check if tmp directory exists
        if not os.path.isdir(self.jflow_config_reader.get_tmp_directory()):
            os.makedirs(self.jflow_config_reader.get_tmp_directory(), 0751)
        tempfile_name = os.path.basename(tempfile.NamedTemporaryFile(suffix=suffix).name)
        return os.path.join(self.jflow_config_reader.get_tmp_directory(), tempfile_name)

    def get_component_output_directory(self, component_name, component_prefix):
        return os.path.join(self.directory, component_name + "_" + component_prefix)
    
    def get_components_nameid(self):
        return self.component_nameids.keys()
    
    def wf_execution_wrapper(self):
        getattr(self, self.function)()
    
    def run(self):
        """
        Only require for Threading
        """
        # if this is the first time the workflow run
        if self.step == None :
            self.start_time = time.time()
            self.step = 0
            self.status = self.STATUS_STARTED
            self.end_time = None
            self._serialize()
        # if pre_processing has not been done yet    
        if self.step == 0:
            self.pre_process()
            self.step = 1
            self._serialize()
        # if collecting components and running workflow has not been done yet
        if self.step == 1:
            try: 
                self.wf_execution_wrapper()
            except SystemExit, e: 
                self.status = self.STATUS_FAILED
                self.end_time = time.time()
                self._serialize()
                raise
            self.component_nameids_is_init = True
            if self.dynamic_component_present:
                self.step = 2
            else:
                self._execute_weaver()
                self.step = 3
            self._serialize() 
        # if the workflow was a dynamic one
        if self.step == 2:
            try: 
                self.wf_execution_wrapper()
            except SystemExit, e: 
                self.status = self.STATUS_FAILED
                self.end_time = time.time()
                self._serialize()
                raise
            if len(self.components_to_exec) > 0:
                self._execute_weaver()
            self.step = 3
            self._serialize()    
        # if post processing has ne been done yet
        if self.step == 3:
            self.post_process()
            if self.status == self.STATUS_STARTED: self.status = self.STATUS_COMPLETED        
            self.end_time = time.time()
            self._serialize()
    
    def _download_urlfile(self, input):
        try:
            uri_object = urlparse(input)
            opener = urllib2.urlopen(input)
            block_size = 8000
            jflowconf = JFlowConfigReader()
            tmp_directory = os.path.join(jflowconf.get_tmp_directory(), os.path.basename(tempfile.NamedTemporaryFile().name))
            os.mkdir(tmp_directory) 
            file_path = os.path.join(tmp_directory, os.path.basename(uri_object.path))
            local_file = open(file_path, 'wb')
            if os.path.basename(uri_object.path) is not None and os.path.basename(uri_object.path) != "":
                metadata = opener.info()
                file_size = int(metadata.getheaders("Content-Length")[0])
                while True:
                    buffer = opener.read(block_size)
                    # End of download
                    if not buffer: break
                    # Parts of download
                    local_file.write(buffer)
                local_file.close()
                logging.getLogger("types.urlfile").debug("URL file '{0}' successfully downloaded as: {1}".format(input, file_path))
            return file_path
        except:
            return input
    
    def _prepare_input_files(self, args):
        new_args = args
        for param in args.keys():
            if args[param].__class__.__name__ == "list":
                for i, val in enumerate(args[param]):
                    new_args[param][i] = self._download_urlfile(args[param][i])
            else:
                new_args[param] = self._download_urlfile(args[param])
        return new_args
    
    def _extend_and_format_args(self, parameters, args):
        extended_args = {}
        for param in parameters:
            try: args[param.name] = args[param.name].encode('ascii','ignore')
            except: pass
            # if this parameter has been modified by the user
            if args.has_key(param.name):
                if param.action == MiltipleAppendAction:
                    extended_args[param.name] = []
                    for arg in args[param.name]:
                        sub_hash = {}
                        for sub_param in param.sub_parameters:
                            # find the value
                            value = None
                            for values in arg:
                                if values[0] == sub_param.flag: 
                                    try: value = values[1].encode('ascii','ignore')
                                    except: value = values[1]
                            if value == None: value = sub_param.default
                            sub_hash[sub_param.name] = value
                        extended_args[param.name].append(sub_hash)
                # if it's an append parameter, the result should be a list
                elif param.action == "append":
                    if args[param.name]:
                        try: value = args[param.name].split(",") 
                        except: value = args[param.name]
                    else: value = []
                    extended_args[param.name] = value
                # if it's a date
                elif param.type == date:
                    if isinstance(args[param.name], str):
                        extended_args[param.name] = date(args[param.name])
                    else:
                        extended_args[param.name] = args[param.name]
                # if it's a boolean
                elif param.type == types.BooleanType:
                    extended_args[param.name] = args[param.name] in [True, 1, 'true', '1', 't', 'y', 'yes']
                elif param.type.__name__ == "MultipleParameters":
                    extended_args[param.name] = {}
                    for sub_param in param.sub_parameters:
                        # find the value
                        value = None
                        for values in args[param.name]:
                            if values[0] == sub_param.flag:
                                try: value = values[1].encode('ascii','ignore')
                                except: value = values[1]
                        if value == None: value = sub_param.default
                        extended_args[param.name][sub_param.name] = value
                else:
                    extended_args[param.name] = args[param.name]
            # otherwise use the default
            else:
                if param.type.__name__ == "MultipleParameters":
                    extended_args[param.name] = {}
                    for sub_param in param.sub_parameters:
                        extended_args[param.name][sub_param.name] = sub_param.default
                else:
                    extended_args[param.name] = param.default
        return extended_args
    
    def _get_from_config(self, parameters_section):
        reader = ConfigParser()
        reader.read(self._get_property_path())
        parameters, param_order = [], []
        try:
            params, sub_order, excludes, hash_params = {}, {}, [], {}
            # first grab all parameters name
            for param in reader.items(parameters_section):
                cvalue = reader.get(parameters_section, param[0])
                cparam = param[0].split(".")[0]
                ckey = ".".join(param[0].split(".")[1:])
                if len(ckey.split(".")) > 1:
                    if sub_order.has_key(cparam):
                        if ckey.split(".")[0] not in sub_order[cparam]: sub_order[cparam].append(ckey.split(".")[0])
                    else:
                        sub_order[cparam] =[ckey.split(".")[0]]
                if not params.has_key(cparam):
                    params[cparam] = {ckey: cvalue}
                else:
                    params[cparam][ckey] = cvalue
                if cparam not in param_order: param_order.append(cparam)
            
            for param_name in param_order:
                sub_params, current_params, sub_objs, sub_excludes, hash_sub_params = {}, {}, [], [], {}
                for param in params[param_name]:
                    sub_params_values = param.split(".")
                    if len(sub_params_values) == 2:
                        # TODO: handle exclusive for sub parameter
                        key = None
                        if sub_params_values[1] == "name": 
                            key = "display_name"
                        else:
                            key = sub_params_values[1]
                        if key == "exclude":
                            found = False
                            for exclude_group in sub_excludes:
                                if sub_params_values[0] in exclude_group:
                                    exclude_group.append(params[param_name][param])
                                    found = True
                                elif params[param_name][param] in exclude_group:
                                    exclude_group.append(sub_params_values[0])
                                    found = True
                            if not found: sub_excludes.append([sub_params_values[0], params[param_name][param]])
                        else:
                            if not sub_params.has_key(sub_params_values[0]):
                                sub_params[sub_params_values[0]] = {key: params[param_name][param]}
                                sub_params[sub_params_values[0]]["name"] = sub_params_values[0]
                            else:
                                sub_params[sub_params_values[0]][key] = params[param_name][param]
                    else:
                        if param == "name": current_params["display_name"] = params[param_name][param]
                        elif param == "exclude":
                            found = False
                            for exclude_group in excludes:
                                if param_name in exclude_group:
                                    exclude_group.append(params[param_name][param])
                                    found = True
                                elif params[param_name][param] in exclude_group:
                                    exclude_group.append(param_name)
                                    found = True
                            if not found: excludes.append([param_name, params[param_name][param]])
                        else: current_params[param] = params[param_name][param]
                current_params["name"] = param_name
                if sub_order.has_key(param_name):
                    for sub_param in sub_order[param_name]:
                        sub = Parameter(**sub_params[sub_param])
                        hash_sub_params[sub.name] = sub
                        sub_objs.append(sub)
                    # overwrite the group to exclude some parameters
                    for i, exclude_group in enumerate(sub_excludes):
                        group_name = "exclude-"+str(i)
                        for sparam in set(exclude_group):
                            hash_sub_params[sparam].group = group_name
                current_params["sub_parameters"] = sub_objs
                oparam = Parameter(**current_params)
                parameters.append(oparam)
                hash_params[oparam.name] = oparam
            # overwrite the group to exclude some parameters
            for i, exclude_group in enumerate(excludes):
                group_name = "exclude-"+str(i)
                for param in set(exclude_group):
                    hash_params[param].group = group_name
        except:
            pass
        # finally add the metadata parameter
        metadata = Parameter(name="metadata", flag="--metadata", help="Which metadata should be linked to this workflow", 
                             action="append", display_name="Metadata")
        parameters.append(metadata)
        return [reader.get("global", "name"),
                reader.get("global", "description"),
                parameters]
    
    def _update_status_from_log(self):
        # first update the status from weaver folders
        # TODO update self.end_time
        try:
            working_directory = os.path.join(self.directory, self.WORKING)
            make_states = []
            for wdir in os.listdir(working_directory):
                log_path = os.path.join(working_directory, wdir, self.MAKEFLOW_LOG_FILE_NAME)
                log = MakeflowLog(log_path)
                log.parse()
                make_states.append(log.state)
            if self.STATUS_ABORTED in make_states:
                self.status = self.STATUS_ABORTED
            elif self.STATUS_FAILED in make_states:
                self.status = self.STATUS_FAILED
            elif self.status != self.STATUS_COMPLETED:
                self.status = self.STATUS_STARTED
        except: pass
        self._serialize()

    def get_status(self):
        self._update_status_from_log()
        return self.status
    
    def get_resource(self, resource):
        return self.jflow_config_reader.get_resource(resource)
    
    def get_component_status(self, component_nameid):
        status = {"time": 0.0,
                  "tasks": 0,
                  "waiting": 0,
                  "running": 0,
                  "failed":0,
                  "aborted": 0,
                  "completed": 0}
        try:
            log = MakeflowLog(self.component_nameids[component_nameid])
            log.parse()
            symbols = set(n.symbol for n in log.nodes if n.symbol)
            if not symbols: return None
            for n in log.nodes:
                if not n.symbol: continue
                if n.symbol == component_nameid:
                    status["tasks"] += 1
                    status["time"]  += n.elapsed_time
                    if n.state == Node.WAITING:
                        status["waiting"] += 1
                    elif n.state == Node.RUNNING:
                        status["running"] += 1
                    elif n.state == Node.FAILED:
                        status["failed"] += 1
                    elif n.state == Node.ABORTED:
                        status["aborted"] += 1
                    elif n.state == Node.COMPLETED:
                        status["completed"] += 1
        except: pass
        return status        
    
    def makeflow_pretty_print_node(self, dag, node):
        sys.stdout.write('{0:>10} {1} {2}\n'.format('NODE', node.id, node.symbol))
        
        for output_file in sorted(node.output_files):
            sys.stdout.write('{0:>10} {1:>10} {2}\n'.format('', 'OUTPUT', output_file))
    
        for input_file in sorted(node.input_files):
            sys.stdout.write('{0:>10} {1:>10} {2}\n'.format('', 'INPUT', input_file))
            
        sys.stdout.write('{0:>10} {1:>10} {2}\n'.format('', 'COMMAND', node.command))

    def _execute_weaver(self, engine_wrapper=None):
        # Add nest path and path to script to Python module path to allow
        # for importing modules outside of $PYTHONPATH
        make_directory, new_make = self._get_current_make()
        current_working_directory = os.path.join(os.path.join(self.directory, self.WORKING), make_directory)
        sys.path.insert(0, os.path.abspath(os.path.dirname(current_working_directory)))

        # Load built-ins
        self._import('abstraction', ABSTRACTIONS)
        self._import('dataset', DATASETS)
        self._import('function', FUNCTIONS)
        self._import('nest', NESTS)
        self._import('options', OPTIONS)
        self._import('stack', STACKS)
                
        # Execute nest
        with Nest(current_working_directory, wrapper=engine_wrapper, path=self.jflow_config_reader.get_makeflow_path()) as nest:
            with self.options:
                if new_make:
                    try:
                        for component in self.components_to_exec:
                            nest.symbol = component.get_nameid()
                            nest.batch = component.batch_options
                            self.component_nameids[component.get_nameid()] = os.path.join(current_working_directory, self.MAKEFLOW_LOG_FILE_NAME)
                            component.execute()
                        # create the DAG
                        nest.compile()
                    except Exception as e:
                        self.status = self.STATUS_FAILED
                        self.end_time = time.time()
                        self._serialize()
                        raise
                self.components_to_exec = []
                # Once a weaver script is compiled, serialize the workflow
                self._serialize()
                try:
                    nest.execute(self.engine_arguments, exit_on_failure=True)
                except:
                    self.status = self.STATUS_FAILED
                    self.end_time = time.time()
                    self._serialize()
                    raise

    def _get_current_make(self):
        current_component, make_directory, new_make = [], None, False
        for component in self.components_to_exec:
            current_component.append(component.get_nameid())
        for make in self.makes:
            if set(current_component) == set(self.makes[make]):
                make_directory = make
        # If the components in the queue have already been compiled
        if make_directory is None:
            make_directory = uuid.uuid4().hex[:10]
            self.makes[make_directory] = current_component
            new_make = True
        return [make_directory, new_make]
    
    def _serialize(self):
        self.dump_path = os.path.join(self.directory, self.DUMP_FILE_NAME)
        workflow_dump = open(self.dump_path, "wb")
        pickle.dump(self, workflow_dump)
        workflow_dump.close()

    def _component_is_duplicated(self, component):
        if component.get_nameid() in self.component_nameids.keys():
            return True
        return False

    def _get_property_path(self):
        """ 
        Return(type:string):  the path to the workflow properties file, None if does not exist
        """
        property_file = os.path.join(os.path.dirname(inspect.getfile(self.__class__)), self.PROPERTIES_FILE_NAME)
        return property_file if os.path.isfile(property_file) else None

    def _import_components(self):
        pckge = {}
        # then import pipeline packages
        pipeline_dir = os.path.dirname(inspect.getfile(self.__class__))
        for importer, modname, ispkg in pkgutil.iter_modules([os.path.join(pipeline_dir, "components")], "workflows." + 
                                                             os.path.basename(pipeline_dir) + ".components."):
            try:
                m = __import__(modname)
                for class_name, obj in inspect.getmembers(sys.modules[modname], inspect.isclass):
                    if issubclass(obj, jflow.component.Component) and obj.__name__ != jflow.component.Component.__name__:
                        pckge[class_name] = modname
            except Exception as e:
                logging.getLogger("Workflow._import_components").debug("Component <{0}> cannot be loaded: {1}".format(modname, e))
        # finally import workflows shared packages
        workflows_dir = os.path.dirname(os.path.dirname(inspect.getfile(self.__class__)))
        for importer, modname, ispkg in pkgutil.iter_modules([os.path.join(workflows_dir, "components")], "workflows.components."):
            try:
                m = __import__(modname)
                for class_name, obj in inspect.getmembers(sys.modules[modname], inspect.isclass):
                    if issubclass(obj, jflow.component.Component) and obj.__name__ != jflow.component.Component.__name__:
                        pckge[class_name] = modname
            except Exception as e:
                logging.getLogger("Workflow._import_components").debug("Component <{0}> cannot be loaded: {1}".format(modname, e))
        return pckge

    def _import(self, module, symbols):
        """ Import ``symbols`` from ``module`` into global namespace. """
        # Import module
        m = 'weaver.{0}'.format(module)
        m = __import__(m, self.globals, self.globals, symbols, -1)

        # Import symbols from module into global namespace, which we store as
        # an attribute for later use (i.e. during compile)
        for symbol in symbols:
            self.globals[symbol] = getattr(m, symbol)