Maintenance - Mise à jour mensuelle Lundi 6 Avril 2020 entre 7h00 et 9h00

ng6_cli.py 9.08 KB
Newer Older
1
#
Penom Nom's avatar
Penom Nom committed
2
# Copyright (C) 2015 INRA
3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19
# 
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
# 
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
# 
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
#

import sys
import argparse
20
import time
21 22 23 24 25 26 27 28

try:
    import _preamble
except ImportError:
    sys.exc_clear()

from jflow.workflows_manager import WorkflowsManager
from jflow.workflow import Workflow
29
import jflow.utils as utils
30

31 32 33 34 35
class JflowArgumentParser (argparse.ArgumentParser):
    def _read_args_from_files(self, arg_strings):
        # expand arguments referencing files
        new_arg_strings = []
        for arg_string in arg_strings:
36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53
            # if it's not a comment or an empty line
            if not arg_string.startswith("#") and arg_string:
                # for regular arguments, just add them back into the list
                if not arg_string or arg_string[0] not in self.fromfile_prefix_chars:
                    new_arg_strings.append(arg_string)
                # replace arguments referencing files with the file content
                else:
                    try:
                        with open(arg_string[1:]) as args_file:
                            arg_strings = []
                            # give to the convert_arg_line_to_args a table of lines instead of line per line
                            for arg in self.convert_arg_line_to_args(args_file.read().splitlines()):
                                arg_strings.append(arg)
                            arg_strings = self._read_args_from_files(arg_strings)
                            new_arg_strings.extend(arg_strings)
                    except OSError:
                        err = _sys.exc_info()[1]
                        self.error(str(err))
54 55
        # return the modified argument list
        return new_arg_strings
56

57

58 59
if __name__ == '__main__':

60 61 62
    # Create a workflow manager to get access to our workflows
    wfmanager = WorkflowsManager()
    
63
    # Create the top-level parser
64
    parser = JflowArgumentParser()
65
    subparsers = parser.add_subparsers(title='Available sub commands')
66
    
67
    # Add rerun workflow availability
Penom Nom's avatar
Penom Nom committed
68
    sub_parser = subparsers.add_parser("rerun", help="Rerun a specific workflow")
69 70 71
    sub_parser.add_argument("--workflow-id", type=str, help="Which workflow should be rerun",
                            required=True, dest="workflow_id")
    sub_parser.set_defaults(cmd_object="rerun")
72

Penom Nom's avatar
Penom Nom committed
73 74 75 76 77 78 79 80
    # Add rerun workflow availability
    sub_parser = subparsers.add_parser("reset", help="Reset a workflow component")
    sub_parser.add_argument("--workflow-id", type=str, help="Which workflow should be used",
                            required=True, dest="workflow_id")
    sub_parser.add_argument("--component-name", type=str, help="Which component should be reseted",
                            required=True, dest="component_name")
    sub_parser.set_defaults(cmd_object="reset")

81 82 83 84 85 86
    # Add delete workflow availability
    sub_parser = subparsers.add_parser("delete", help="Delete a workflow")
    sub_parser.add_argument("--workflow-id", type=str, help="Which workflow should be deleted",
                            required=True, dest="workflow_id")
    sub_parser.set_defaults(cmd_object="delete")

Penom Nom's avatar
Penom Nom committed
87 88 89 90 91 92
    # Add rerun workflow availability
    sub_parser = subparsers.add_parser("execution-graph", help="Display the workflow execution graph")
    sub_parser.add_argument("--workflow-id", type=str, help="Which workflow should be considered",
                            required=True, dest="workflow_id")
    sub_parser.set_defaults(cmd_object="execution_graph")

93
    # Add status workflow availability
Penom Nom's avatar
Penom Nom committed
94
    sub_parser = subparsers.add_parser("status", help="Monitor a specific workflow")
95 96
    sub_parser.add_argument("--workflow-id", type=str, help="Which workflow status should be displayed",
                            default=None, dest="workflow_id")
97 98
    sub_parser.add_argument("--all", action="store_true", help="Display all workflows status",
                            default=False, dest="all")
Penom Nom's avatar
Penom Nom committed
99 100
    sub_parser.add_argument("--errors", action="store_true", help="Display failed commands",
                            default=False, dest="display_errors")
101
    sub_parser.set_defaults(cmd_object="status")
102
    
103
    # Add available pipelines
Penom Nom's avatar
Penom Nom committed
104
    wf_instances, wf_methodes = wfmanager.get_available_workflows()
105
    wf_classes = []
106
    for instance in wf_instances:
107
        wf_classes.append(instance.__class__.__name__)
108
        # create the subparser for each applications
109 110
        sub_parser = subparsers.add_parser(instance.name, help=instance.description, fromfile_prefix_chars='@')
        sub_parser.convert_arg_line_to_args = instance.__class__.config_parser
Jerome Mariette's avatar
Jerome Mariette committed
111
        [parameters_groups, parameters_order] = instance.get_parameters_per_groups()
Penom Nom's avatar
Penom Nom committed
112
        for group in parameters_order:
Jerome Mariette's avatar
Jerome Mariette committed
113 114 115
            if group == "default":
                for param in parameters_groups[group]:
                    sub_parser.add_argument(param.flag, **param.export_to_argparse())
Jerome Mariette's avatar
Jerome Mariette committed
116 117 118 119 120 121 122 123 124 125
            elif group.startswith("exclude-"):
                is_required = False
                for param in parameters_groups[group]:
                    if param.required:
                        is_required = True
                        # an exlcusive parameter cannot be required, the require is at the group level
                        param.required = False
                pgroup = sub_parser.add_mutually_exclusive_group(required=is_required)
                for param in parameters_groups[group]:
                    pgroup.add_argument(param.flag, **param.export_to_argparse())
Jerome Mariette's avatar
Jerome Mariette committed
126 127 128 129
            else:
                pgroup = sub_parser.add_argument_group(group)
                for param in parameters_groups[group]:
                    pgroup.add_argument(param.flag, **param.export_to_argparse())
Jerome Mariette's avatar
Jerome Mariette committed
130
        sub_parser.set_defaults(cmd_object=instance.__class__.__name__)
131
    args = vars(parser.parse_args())
Penom Nom's avatar
Penom Nom committed
132 133 134 135 136
    
    if not "cmd_object" in args:
        print(parser.format_help())
        parser.exit(0, "")
    
137
    if args["cmd_object"] in wf_classes:
Jerome Mariette's avatar
Jerome Mariette committed
138
        wfmanager.run_workflow(args["cmd_object"], args)
139
    elif args["cmd_object"] == "rerun":
Jerome Mariette's avatar
Jerome Mariette committed
140
        wfmanager.rerun_workflow(args["workflow_id"])
Penom Nom's avatar
Penom Nom committed
141
    elif args["cmd_object"] == "reset":
142 143 144 145 146 147 148 149 150
        try:
            wfmanager.reset_workflow_component(args["workflow_id"], args["component_name"])
        except Exception as e:
            utils.display_error_message(str(e))
    elif args["cmd_object"] == "delete":
        try:
            wfmanager.delete_workflow(args["workflow_id"])
        except Exception as e:
            utils.display_error_message(str(e))
Penom Nom's avatar
Penom Nom committed
151
    elif args["cmd_object"] == "execution_graph":
152 153 154 155
        try:
            workflow = wfmanager.get_workflow(args["workflow_id"])
        except Exception as e:
            utils.display_error_message(str(e))
156
        gr = workflow.get_execution_graph()
157 158
        inputs, components = [], []
        for node in gr.nodes():
Penom Nom's avatar
Penom Nom committed
159 160 161 162 163
            if Workflow.INPUTFILE_GRAPH_LABEL in gr.node_attributes(node):
                inputs.append(gr.node_attributes(node)[1])
            elif Workflow.INPUTFILES_GRAPH_LABEL in gr.node_attributes(node):
                inputs.append(gr.node_attributes(node)[1])
            elif Workflow.INPUTDIRECTORY_GRAPH_LABEL in gr.node_attributes(node):
Penom Nom's avatar
Penom Nom committed
164
                inputs.append(gr.node_attributes(node)[1])
165
            elif Workflow.COMPONENT_GRAPH_LABEL in gr.node_attributes(node):
Penom Nom's avatar
Penom Nom committed
166
                components.append(gr.node_attributes(node)[1])
Penom Nom's avatar
Penom Nom committed
167 168 169
        print("inputs: ", inputs)
        print("components: ", components)
        print("edges: ", gr.edges())
170
        
171
    elif args["cmd_object"] == "status":
172
        if args["workflow_id"]:
173 174 175 176
            try:
                workflow = wfmanager.get_workflow(args["workflow_id"])
            except Exception as e:
                utils.display_error_message(str(e))
Penom Nom's avatar
Penom Nom committed
177
            print(Workflow.get_status_under_text_format(workflow, True, args["display_errors"]))
178
        else:
179 180 181 182
            try:
                workflows = wfmanager.get_workflows(use_cache=True)
            except Exception as e:
                utils.display_error_message(str(e))
183
            if len(workflows) > 0:
184 185 186 187 188
                workflows_by_id, wfids = {}, []
                # first sort workflow by ID
                for workflow in workflows:
                    wfids.append(workflow.id)
                    workflows_by_id[workflow.id] = workflow
189
                status = "ID\tNAME\tSTATUS\tELAPSED_TIME\tSTART_TIME\tEND_TIME\n"
190 191
                for i, wfid in enumerate(sorted(wfids, reverse=True)):
                    status += Workflow.get_status_under_text_format(workflows_by_id[wfid])
192 193
                    if i<len(workflows)-1: status += "\n"
            else: status = "no workflow available"
Penom Nom's avatar
Penom Nom committed
194
            print(status)
195