jflow_cli.py 10.4 KB
Newer Older
Jerome Mariette's avatar
Jerome Mariette committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
#
# Copyright (C) 2012 INRA
# 
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
# 
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
# 
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
#

import sys
import argparse
Jerome Mariette's avatar
Jerome Mariette committed
20
import time
Jerome Mariette's avatar
Jerome Mariette committed
21
22
23
24
25
26
27
28

try:
    import _preamble
except ImportError:
    sys.exc_clear()

from jflow.workflows_manager import WorkflowsManager
from jflow.workflow import Workflow
Jerome Mariette's avatar
Jerome Mariette committed
29
30
import jflow.utils as utils
from cctools.util import time_format
Jerome Mariette's avatar
Jerome Mariette committed
31

32
33
34
35
36
class JflowArgumentParser (argparse.ArgumentParser):
    def _read_args_from_files(self, arg_strings):
        # expand arguments referencing files
        new_arg_strings = []
        for arg_string in arg_strings:
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
            # if it's not a comment or an empty line
            if not arg_string.startswith("#") and arg_string:
                # for regular arguments, just add them back into the list
                if not arg_string or arg_string[0] not in self.fromfile_prefix_chars:
                    new_arg_strings.append(arg_string)
                # replace arguments referencing files with the file content
                else:
                    try:
                        with open(arg_string[1:]) as args_file:
                            arg_strings = []
                            # give to the convert_arg_line_to_args a table of lines instead of line per line
                            for arg in self.convert_arg_line_to_args(args_file.read().splitlines()):
                                arg_strings.append(arg)
                            arg_strings = self._read_args_from_files(arg_strings)
                            new_arg_strings.extend(arg_strings)
                    except OSError:
                        err = _sys.exc_info()[1]
                        self.error(str(err))
55
56
        # return the modified argument list
        return new_arg_strings
Jerome Mariette's avatar
Jerome Mariette committed
57

58
def display_workflow_status(workflow, detailed=False, display_errors=False):
Jerome Mariette's avatar
Jerome Mariette committed
59
60
61
62
63
64
65
66
    if workflow.start_time: start_time = time.asctime(time.localtime(workflow.start_time))
    else: start_time = "-"
    if workflow.start_time and workflow.end_time: elapsed_time = str(workflow.end_time-workflow.start_time)
    elif workflow.start_time: elapsed_time = str(time.time()-workflow.start_time)
    else: elapsed_time = "-"
    if workflow.end_time: end_time = time.asctime(time.localtime(workflow.end_time))
    else: end_time = "-"
    if detailed:
Frédéric Escudié's avatar
Frédéric Escudié committed
67
68
        # Global
        title = "Workflow #" + utils.get_nb_string(workflow.id) + " (" + workflow.name + ") is " + \
Jerome Mariette's avatar
Jerome Mariette committed
69
                workflow.get_status() + ", time elapsed: " + str(elapsed_time) + " (from " + start_time + \
Frédéric Escudié's avatar
Frédéric Escudié committed
70
71
72
73
74
75
76
77
                " to " + end_time + ")"
        worflow_errors = ""
        error = workflow.get_errors()
        if error is not None:
            worflow_errors = "Workflow Error :\n  \033[91m" + error["location"] + "\n    " + "\n    ".join(error["msg"]) + "\033[0m"
        # By components
        components_errors = ""
        status = "Components Status :\n"
Jerome Mariette's avatar
Jerome Mariette committed
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
        for i, component in enumerate(workflow.get_components_nameid()):
            status_info = workflow.get_component_status(component)
            try: perc_waiting = (status_info["waiting"]*100.0)/status_info["tasks"]
            except: perc_waiting = 0
            try: perc_running = (status_info["running"]*100.0)/status_info["tasks"]
            except: perc_running = 0
            try: perc_failed = (status_info["failed"]*100.0)/status_info["tasks"]
            except: perc_failed = 0
            try: perc_aborted = (status_info["aborted"]*100.0)/status_info["tasks"]
            except: perc_aborted = 0
            try: perc_completed = (status_info["completed"]*100.0)/status_info["tasks"]
            except: perc_completed = 0
            
            if status_info["running"] > 0: running = "\033[94mrunning:" + str(status_info["running"]) + "\033[0m"
            else: running = "running:" + str(status_info["running"])
            if status_info["waiting"] > 0: waiting = "\033[93mwaiting:" + str(status_info["waiting"]) + "\033[0m"
            else: waiting = "waiting:" + str(status_info["waiting"])            
            if status_info["failed"] > 0: failed = "\033[91mfailed:" + str(status_info["failed"]) + "\033[0m"
            else: failed = "failed:" + str(status_info["failed"])
            if status_info["aborted"] > 0: aborted = "\033[95maborted:" + str(status_info["aborted"]) + "\033[0m"
            else: aborted = "aborted:" + str(status_info["aborted"])
            if status_info["completed"] == status_info["tasks"] and status_info["completed"] > 0: completed = "\033[92mcompleted:" + str(status_info["completed"]) + "\033[0m"
            else: completed = "completed:" + str(status_info["completed"])
101
102
            
            if display_errors and len(status_info["failed_commands"]) > 0:
Frédéric Escudié's avatar
Frédéric Escudié committed
103
104
105
                if components_errors == "" :
                    components_errors = "Failed Commands :\n"  
                components_errors += "  - " + component + " :\n    " + "\n    ".join(status_info["failed_commands"])
Jerome Mariette's avatar
Jerome Mariette committed
106
107
108
109
            status += "  - " + component + ", time elapsed " + time_format(status_info["time"]) + \
                " (total:" + str(status_info["tasks"]) + ", " + waiting + ", " + running + ", " + failed + \
                ", " + aborted + ", " + completed + ")"
            if i<len(workflow.get_components_nameid())-1: status += "\n"
Frédéric Escudié's avatar
Frédéric Escudié committed
110
111
112
113
114
115
        # Format str
        pretty_str = title
        pretty_str += ("\n" + worflow_errors) if worflow_errors != "" else ""
        pretty_str += ("\n" + status) if status != "" else ""
        pretty_str += ("\n" + components_errors) if components_errors != "" else ""
        return pretty_str
Jerome Mariette's avatar
Jerome Mariette committed
116
117
118
119
    else:
        return utils.get_nb_string(workflow.id) + "\t" + workflow.name + "\t" + workflow.get_status() + \
            "\t" + elapsed_time + "\t" + start_time + "\t" + end_time

Jerome Mariette's avatar
Jerome Mariette committed
120
121
if __name__ == '__main__':

Jerome Mariette's avatar
Jerome Mariette committed
122
123
124
    # Create a workflow manager to get access to our workflows
    wfmanager = WorkflowsManager()
    
Jerome Mariette's avatar
Jerome Mariette committed
125
    # Create the top-level parser
Jerome Mariette's avatar
Jerome Mariette committed
126
    parser = JflowArgumentParser()
Jerome Mariette's avatar
Jerome Mariette committed
127
128
    subparsers = parser.add_subparsers(title='Available sub commands')
    
Jerome Mariette's avatar
Jerome Mariette committed
129
    # Add rerun workflow availability
Jerome Mariette's avatar
Jerome Mariette committed
130
    sub_parser = subparsers.add_parser("rerun", help="Rerun a specific workflow")
Jerome Mariette's avatar
Jerome Mariette committed
131
132
133
134
    sub_parser.add_argument("--workflow-id", type=str, help="Which workflow should be rerun",
                            required=True, dest="workflow_id")
    sub_parser.set_defaults(cmd_object="rerun")

Jerome Mariette's avatar
Jerome Mariette committed
135
136
137
138
139
140
    # Add rerun workflow availability
    sub_parser = subparsers.add_parser("execution-graph", help="Display the workflow execution graph")
    sub_parser.add_argument("--workflow-id", type=str, help="Which workflow should be considered",
                            required=True, dest="workflow_id")
    sub_parser.set_defaults(cmd_object="execution_graph")

Jerome Mariette's avatar
Jerome Mariette committed
141
    # Add status workflow availability
Jerome Mariette's avatar
Jerome Mariette committed
142
    sub_parser = subparsers.add_parser("status", help="Monitor a specific workflow")
Jerome Mariette's avatar
Jerome Mariette committed
143
144
145
146
    sub_parser.add_argument("--workflow-id", type=str, help="Which workflow status should be displayed",
                            default=None, dest="workflow_id")
    sub_parser.add_argument("--all", action="store_true", help="Display all workflows status",
                            default=False, dest="all")
147
148
    sub_parser.add_argument("--errors", action="store_true", help="Display failed commands",
                            default=False, dest="display_errors")
Jerome Mariette's avatar
Jerome Mariette committed
149
150
    sub_parser.set_defaults(cmd_object="status")
    
Jerome Mariette's avatar
Jerome Mariette committed
151
    # Add available pipelines
152
    wf_instances, wf_methodes = wfmanager.get_available_workflows()
Jerome Mariette's avatar
Jerome Mariette committed
153
    wf_classes = []
Jerome Mariette's avatar
Jerome Mariette committed
154
    for instance in wf_instances:
Jerome Mariette's avatar
Jerome Mariette committed
155
        wf_classes.append(instance.__class__.__name__)
Jerome Mariette's avatar
Jerome Mariette committed
156
        # create the subparser for each applications
Jerome Mariette's avatar
Jerome Mariette committed
157
158
        sub_parser = subparsers.add_parser(instance.name, help=instance.description, fromfile_prefix_chars='@')
        sub_parser.convert_arg_line_to_args = instance.__class__.config_parser
Jerome Mariette's avatar
Jerome Mariette committed
159
        [parameters_groups, parameters_order] = instance.get_parameters_per_groups()
160
        for group in parameters_order:
Jerome Mariette's avatar
Jerome Mariette committed
161
162
163
            if group == "default":
                for param in parameters_groups[group]:
                    sub_parser.add_argument(param.flag, **param.export_to_argparse())
Jerome Mariette's avatar
Jerome Mariette committed
164
165
166
167
168
169
170
171
172
173
            elif group.startswith("exclude-"):
                is_required = False
                for param in parameters_groups[group]:
                    if param.required:
                        is_required = True
                        # an exlcusive parameter cannot be required, the require is at the group level
                        param.required = False
                pgroup = sub_parser.add_mutually_exclusive_group(required=is_required)
                for param in parameters_groups[group]:
                    pgroup.add_argument(param.flag, **param.export_to_argparse())
Jerome Mariette's avatar
Jerome Mariette committed
174
175
176
177
            else:
                pgroup = sub_parser.add_argument_group(group)
                for param in parameters_groups[group]:
                    pgroup.add_argument(param.flag, **param.export_to_argparse())
Jerome Mariette's avatar
Jerome Mariette committed
178
        sub_parser.set_defaults(cmd_object=instance.__class__.__name__)
179
    args = vars(parser.parse_args())
180

Jerome Mariette's avatar
Jerome Mariette committed
181
182
183
184
    if args["cmd_object"] in wf_classes:
        wfmanager.run_workflow(args["cmd_object"], args)
    elif args["cmd_object"] == "rerun":
        wfmanager.rerun_workflow(args["workflow_id"])
Jerome Mariette's avatar
Jerome Mariette committed
185
    elif args["cmd_object"] == "execution_graph":
Jerome Mariette's avatar
Jerome Mariette committed
186
187
        workflow = wfmanager.get_workflow(args["workflow_id"])
        print workflow.get_execution_graph()
Jerome Mariette's avatar
Jerome Mariette committed
188
189
190
    elif args["cmd_object"] == "status":
        if args["workflow_id"]:
            workflow = wfmanager.get_workflow(args["workflow_id"])
191
            print display_workflow_status(workflow, True, args["display_errors"])
Jerome Mariette's avatar
Jerome Mariette committed
192
193
194
195
196
197
198
199
200
        else:
            workflows = wfmanager.get_workflows()
            if len(workflows) > 0:
                status = "ID\tNAME\tSTATUS\tELAPSED_TIME\tSTART_TIME\tEND_TIME\n"
                for i, workflow in enumerate(workflows):
                    status += display_workflow_status(workflow)
                    if i<len(workflows)-1: status += "\n"
            else: status = "no workflow available"
            print status
Jerome Mariette's avatar
Jerome Mariette committed
201