jflow_cli.py 8.99 KB
Newer Older
Jerome Mariette's avatar
Jerome Mariette committed
1
#
Jerome Mariette's avatar
Jerome Mariette committed
2
# Copyright (C) 2015 INRA
Jerome Mariette's avatar
Jerome Mariette committed
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
# 
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
# 
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
# 
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
#

import sys
Ibouniyamine Nabihoudine's avatar
Ibouniyamine Nabihoudine committed
19
import argparse
Jerome Mariette's avatar
Jerome Mariette committed
20
import time
Jerome Mariette's avatar
Jerome Mariette committed
21
22
23
24
25
26
27
28

try:
    import _preamble
except ImportError:
    sys.exc_clear()

from jflow.workflows_manager import WorkflowsManager
from jflow.workflow import Workflow
Jerome Mariette's avatar
Jerome Mariette committed
29
import jflow.utils as utils
Jerome Mariette's avatar
Jerome Mariette committed
30

Ibouniyamine Nabihoudine's avatar
Ibouniyamine Nabihoudine committed
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
class JflowArgumentParser (argparse.ArgumentParser):
    def _read_args_from_files(self, arg_strings):
        # expand arguments referencing files
        new_arg_strings = []
        for arg_string in arg_strings:
            # if it's not a comment or an empty line
            if not arg_string.startswith("#") and arg_string:
                # for regular arguments, just add them back into the list
                if not arg_string or arg_string[0] not in self.fromfile_prefix_chars:
                    new_arg_strings.append(arg_string)
                # replace arguments referencing files with the file content
                else:
                    try:
                        with open(arg_string[1:]) as args_file:
                            arg_strings = []
                            # give to the convert_arg_line_to_args a table of lines instead of line per line
                            for arg in self.convert_arg_line_to_args(args_file.read().splitlines()):
                                arg_strings.append(arg)
                            arg_strings = self._read_args_from_files(arg_strings)
                            new_arg_strings.extend(arg_strings)
                    except OSError:
                        err = _sys.exc_info()[1]
                        self.error(str(err))
        # return the modified argument list
        return new_arg_strings


Jerome Mariette's avatar
Jerome Mariette committed
58
59
if __name__ == '__main__':

Jerome Mariette's avatar
Jerome Mariette committed
60
61
62
    # Create a workflow manager to get access to our workflows
    wfmanager = WorkflowsManager()
    
Jerome Mariette's avatar
Jerome Mariette committed
63
    # Create the top-level parser
Jerome Mariette's avatar
Jerome Mariette committed
64
    parser = JflowArgumentParser()
Jerome Mariette's avatar
Jerome Mariette committed
65
66
    subparsers = parser.add_subparsers(title='Available sub commands')
    
Jerome Mariette's avatar
Jerome Mariette committed
67
    # Add rerun workflow availability
Jerome Mariette's avatar
Jerome Mariette committed
68
    sub_parser = subparsers.add_parser("rerun", help="Rerun a specific workflow")
Jerome Mariette's avatar
Jerome Mariette committed
69
70
71
72
    sub_parser.add_argument("--workflow-id", type=str, help="Which workflow should be rerun",
                            required=True, dest="workflow_id")
    sub_parser.set_defaults(cmd_object="rerun")

73
74
75
76
77
78
79
80
    # Add rerun workflow availability
    sub_parser = subparsers.add_parser("reset", help="Reset a workflow component")
    sub_parser.add_argument("--workflow-id", type=str, help="Which workflow should be used",
                            required=True, dest="workflow_id")
    sub_parser.add_argument("--component-name", type=str, help="Which component should be reseted",
                            required=True, dest="component_name")
    sub_parser.set_defaults(cmd_object="reset")

Jerome Mariette's avatar
Jerome Mariette committed
81
82
83
84
85
86
    # Add delete workflow availability
    sub_parser = subparsers.add_parser("delete", help="Delete a workflow")
    sub_parser.add_argument("--workflow-id", type=str, help="Which workflow should be deleted",
                            required=True, dest="workflow_id")
    sub_parser.set_defaults(cmd_object="delete")

Jerome Mariette's avatar
Jerome Mariette committed
87
88
89
90
91
92
    # Add rerun workflow availability
    sub_parser = subparsers.add_parser("execution-graph", help="Display the workflow execution graph")
    sub_parser.add_argument("--workflow-id", type=str, help="Which workflow should be considered",
                            required=True, dest="workflow_id")
    sub_parser.set_defaults(cmd_object="execution_graph")

Jerome Mariette's avatar
Jerome Mariette committed
93
    # Add status workflow availability
Jerome Mariette's avatar
Jerome Mariette committed
94
    sub_parser = subparsers.add_parser("status", help="Monitor a specific workflow")
Jerome Mariette's avatar
Jerome Mariette committed
95
96
97
98
    sub_parser.add_argument("--workflow-id", type=str, help="Which workflow status should be displayed",
                            default=None, dest="workflow_id")
    sub_parser.add_argument("--all", action="store_true", help="Display all workflows status",
                            default=False, dest="all")
99
100
    sub_parser.add_argument("--errors", action="store_true", help="Display failed commands",
                            default=False, dest="display_errors")
Jerome Mariette's avatar
Jerome Mariette committed
101
102
    sub_parser.set_defaults(cmd_object="status")
    
Jerome Mariette's avatar
Jerome Mariette committed
103
    # Add available pipelines
104
    wf_instances, wf_methodes = wfmanager.get_available_workflows()
Jerome Mariette's avatar
Jerome Mariette committed
105
    wf_classes = []
Jerome Mariette's avatar
Jerome Mariette committed
106
    for instance in wf_instances:
Ibouniyamine Nabihoudine's avatar
Ibouniyamine Nabihoudine committed
107
        wf_classes.append(instance.__class__.__name__)
Jerome Mariette's avatar
Jerome Mariette committed
108
        # create the subparser for each applications
Jerome Mariette's avatar
Jerome Mariette committed
109
110
        sub_parser = subparsers.add_parser(instance.name, help=instance.description, fromfile_prefix_chars='@')
        sub_parser.convert_arg_line_to_args = instance.__class__.config_parser
Jerome Mariette's avatar
Jerome Mariette committed
111
        [parameters_groups, parameters_order] = instance.get_parameters_per_groups()
112
        for group in parameters_order:
Jerome Mariette's avatar
Jerome Mariette committed
113
114
115
            if group == "default":
                for param in parameters_groups[group]:
                    sub_parser.add_argument(param.flag, **param.export_to_argparse())
Jerome Mariette's avatar
Jerome Mariette committed
116
117
118
119
120
121
122
123
124
125
            elif group.startswith("exclude-"):
                is_required = False
                for param in parameters_groups[group]:
                    if param.required:
                        is_required = True
                        # an exlcusive parameter cannot be required, the require is at the group level
                        param.required = False
                pgroup = sub_parser.add_mutually_exclusive_group(required=is_required)
                for param in parameters_groups[group]:
                    pgroup.add_argument(param.flag, **param.export_to_argparse())
Jerome Mariette's avatar
Jerome Mariette committed
126
127
128
129
            else:
                pgroup = sub_parser.add_argument_group(group)
                for param in parameters_groups[group]:
                    pgroup.add_argument(param.flag, **param.export_to_argparse())
Ibouniyamine Nabihoudine's avatar
Ibouniyamine Nabihoudine committed
130
        sub_parser.set_defaults(cmd_object=instance.__class__.__name__)
131
    args = vars(parser.parse_args())
132

Jerome Mariette's avatar
Jerome Mariette committed
133
134
135
136
    if args["cmd_object"] in wf_classes:
        wfmanager.run_workflow(args["cmd_object"], args)
    elif args["cmd_object"] == "rerun":
        wfmanager.rerun_workflow(args["workflow_id"])
137
    elif args["cmd_object"] == "reset":
138
139
140
141
        try:
            wfmanager.reset_workflow_component(args["workflow_id"], args["component_name"])
        except Exception as e:
            utils.display_error_message(str(e))
Jerome Mariette's avatar
Jerome Mariette committed
142
    elif args["cmd_object"] == "delete":
143
144
145
146
        try:
            wfmanager.delete_workflow(args["workflow_id"])
        except Exception as e:
            utils.display_error_message(str(e))
Jerome Mariette's avatar
Jerome Mariette committed
147
    elif args["cmd_object"] == "execution_graph":
148
149
150
151
        try:
            workflow = wfmanager.get_workflow(args["workflow_id"])
        except Exception as e:
            utils.display_error_message(str(e))
152
        gr = workflow.get_execution_graph()
153
154
        inputs, components = [], []
        for node in gr.nodes():
Jerome Mariette's avatar
Jerome Mariette committed
155
156
157
158
159
            if Workflow.INPUTFILE_GRAPH_LABEL in gr.node_attributes(node):
                inputs.append(gr.node_attributes(node)[1])
            elif Workflow.INPUTFILES_GRAPH_LABEL in gr.node_attributes(node):
                inputs.append(gr.node_attributes(node)[1])
            elif Workflow.INPUTDIRECTORY_GRAPH_LABEL in gr.node_attributes(node):
160
                inputs.append(gr.node_attributes(node)[1])
161
            elif Workflow.COMPONENT_GRAPH_LABEL in gr.node_attributes(node):
162
                components.append(gr.node_attributes(node)[1])
Jerome Mariette's avatar
Jerome Mariette committed
163
164
165
        print(("inputs: ", inputs))
        print(("components: ", components))
        print(("edges: ", gr.edges()))
166
        
Jerome Mariette's avatar
Jerome Mariette committed
167
168
    elif args["cmd_object"] == "status":
        if args["workflow_id"]:
169
170
171
172
            try:
                workflow = wfmanager.get_workflow(args["workflow_id"])
            except Exception as e:
                utils.display_error_message(str(e))
Jerome Mariette's avatar
Jerome Mariette committed
173
            print((Workflow.get_status_under_text_format(workflow, True, args["display_errors"])))
Jerome Mariette's avatar
Jerome Mariette committed
174
        else:
175
176
177
178
            try:
                workflows = wfmanager.get_workflows(use_cache=True)
            except Exception as e:
                utils.display_error_message(str(e))
Jerome Mariette's avatar
Jerome Mariette committed
179
            if len(workflows) > 0:
Jerome Mariette's avatar
Jerome Mariette committed
180
181
182
183
184
                workflows_by_id, wfids = {}, []
                # first sort workflow by ID
                for workflow in workflows:
                    wfids.append(workflow.id)
                    workflows_by_id[workflow.id] = workflow
Jerome Mariette's avatar
Jerome Mariette committed
185
                status = "ID\tNAME\tSTATUS\tELAPSED_TIME\tSTART_TIME\tEND_TIME\n"
Jerome Mariette's avatar
Jerome Mariette committed
186
                for i, wfid in enumerate(sorted(wfids, reverse=True)):
187
                    status += Workflow.get_status_under_text_format(workflows_by_id[wfid])
Jerome Mariette's avatar
Jerome Mariette committed
188
189
                    if i<len(workflows)-1: status += "\n"
            else: status = "no workflow available"
Jerome Mariette's avatar
Jerome Mariette committed
190
            print(status)
Jerome Mariette's avatar
Jerome Mariette committed
191