jflow_server.py 13.7 KB
Newer Older
Jerome Mariette's avatar
Jerome Mariette committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
#
# Copyright (C) 2012 INRA
# 
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
# 
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
# 
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
#

import cherrypy
Jerome Mariette's avatar
Jerome Mariette committed
19
20
import cgi
import tempfile
Jerome Mariette's avatar
Jerome Mariette committed
21
22
23
24
25
import json
import sys
from functools import wraps
import time
import os
26
import argparse
Jerome Mariette's avatar
Jerome Mariette committed
27
import logging
Jerome Mariette's avatar
Jerome Mariette committed
28

Jerome Mariette's avatar
Jerome Mariette committed
29
30
from argparse import ArgumentTypeError

Jerome Mariette's avatar
Jerome Mariette committed
31
32
33
34
35
try:
    import _preamble
except ImportError:
    sys.exc_clear()

36
import jflow
Jerome Mariette's avatar
Jerome Mariette committed
37
from jflow.workflows_manager import WorkflowsManager
38
from jflow.config_reader import JFlowConfigReader
Jerome Mariette's avatar
Jerome Mariette committed
39
from jflow.workflow import Workflow
Jerome Mariette's avatar
Jerome Mariette committed
40
from workflows.types import *
Jerome Mariette's avatar
Jerome Mariette committed
41
42
43
import jflow.utils as utils
from cctools.util import time_format

44
# function in charge to upload large files
Jerome Mariette's avatar
Jerome Mariette committed
45
46
47
48
49
50
51
52
53
54
55
56
57
class myFieldStorage(cgi.FieldStorage):
    """Our version uses a named temporary file instead of the default
    non-named file; keeping it visibile (named), allows us to create a
    2nd link after the upload is done, thus avoiding the overhead of
    making a copy to the destination filename."""
    
    def make_file(self, binary=None):
        return tempfile.NamedTemporaryFile()
def noBodyProcess():
    """Sets cherrypy.request.process_request_body = False, giving
    us direct control of the file upload destination. By default
    cherrypy loads it to memory, we are directing it to disk."""
    cherrypy.request.process_request_body = False
58
59
60
61
62
63
64
65
cherrypy.tools.noBodyProcess = cherrypy.Tool('before_finalize', noBodyProcess)

# define functions in charge to handle cross domain calls
def CORS():
    cherrypy.response.headers['Access-Control-Allow-Origin'] = '*'
    cherrypy.response.headers['Access-Control-Allow-Methods'] = 'OPTIONS, GET, POST'
    cherrypy.response.headers['Access-Control-Allow-Headers'] = 'Content-Type, Content-Range, Content-Disposition'
cherrypy.tools.CORS = cherrypy.Tool('before_finalize', CORS)
Jerome Mariette's avatar
Jerome Mariette committed
66
67


Jerome Mariette's avatar
Jerome Mariette committed
68
69
class JFlowServer (object):

70
71
    MULTIPLE_TYPE_SPLITER = "._."

Jerome Mariette's avatar
Jerome Mariette committed
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
    def __init__(self):
        # Create a workflow manager to get access to our workflows
        self.wfmanager = WorkflowsManager()

    def jsonify(func):
        '''JSON and JSONP decorator for CherryPy'''
        @wraps(func)
        def wrapper(*args, **kw):
            value = func(*args, **kw)
            cherrypy.response.headers["Content-Type"] = "application/json"
            # if JSONP request
            if kw.has_key("callback"): return kw["callback"] + "(" + json.dumps(value) + ")"
            # else return the JSON
            else: return json.dumps(value)
        return wrapper

    def jsonify_workflow_status(self, workflow, init_to_zero=False):
        
        if workflow.start_time: start_time = time.asctime(time.localtime(workflow.start_time))
        else: start_time = "-"
        if workflow.start_time and workflow.end_time: elapsed_time = str(workflow.end_time-workflow.start_time)
        elif workflow.start_time: elapsed_time = str(time.time()-workflow.start_time)
        else: elapsed_time = "-"
        if workflow.end_time: end_time = time.asctime(time.localtime(workflow.end_time))
        else: end_time = "-"
        if init_to_zero:
            return {"id":utils.get_nb_string(workflow.id),
                    "name": workflow.name,
                    "status": Workflow.STATUS_STARTED,
                    "elapsed_time": str(elapsed_time),
                    "start_time": start_time,
                    "end_time": end_time,
                    "components": []}
        else :
            components = []
            for i, component in enumerate(workflow.get_components_nameid()):
                status_info = workflow.get_component_status(component)
                try: perc_waiting = (status_info["waiting"]*100.0)/status_info["tasks"]
                except: perc_waiting = 0
                try: perc_running = (status_info["running"]*100.0)/status_info["tasks"]
                except: perc_running = 0
                try: perc_failed = (status_info["failed"]*100.0)/status_info["tasks"]
                except: perc_failed = 0
                try: perc_aborted = (status_info["aborted"]*100.0)/status_info["tasks"]
                except: perc_aborted = 0
                try: perc_completed = (status_info["completed"]*100.0)/status_info["tasks"]
                except: perc_completed = 0
                components.append({"name": component,
                                  "elapsed_time": time_format(status_info["time"]),
                                  "total": status_info["tasks"],
                                  "waiting": status_info["waiting"],
                                  "failed": status_info["failed"],
                                  "running": status_info["running"],
                                  "aborted": status_info["aborted"],
                                  "completed": status_info["completed"]})
            status = {"id":utils.get_nb_string(workflow.id),
                      "name": workflow.name,
Jerome Mariette's avatar
Jerome Mariette committed
129
                      "metadata": workflow.metadata,
Jerome Mariette's avatar
Jerome Mariette committed
130
131
132
133
134
135
136
                      "status": workflow.get_status(),
                      "elapsed_time": str(elapsed_time),
                      "start_time": start_time,
                      "end_time": end_time,
                      "components": components}
                
            return status
137

Jerome Mariette's avatar
Jerome Mariette committed
138
139
140
141
142
143
    @cherrypy.expose
    @jsonify
    def get_available_workflows(self, **kwargs):
        workflows = []
        wf_instances = self.wfmanager.get_available_workflows()
        for instance in wf_instances:
Jerome Mariette's avatar
Jerome Mariette committed
144
            parameters, parameters_per_groups, groups = [], {}, ["default"]
Jerome Mariette's avatar
Jerome Mariette committed
145
            for param in instance.parameters:
146
147
148
149
150
151
152
                # if it's a multiple action change the action by the name
                if param.action == jflow.parameter.MiltipleAction:
                    action = "MiltipleAction"
                elif param.action == jflow.parameter.MiltipleAppendAction:
                    action = "MiltipleAppendAction"
                else:
                    action = param.action
Jerome Mariette's avatar
Jerome Mariette committed
153
154
155
156
                hash_param = {"help": param.help,
                              "required": param.required,
                              "default": param.default,
                              "choices": param.choices,
157
                              "action": action,
Jerome Mariette's avatar
Jerome Mariette committed
158
159
                              "type": param.get_type(),
                              "name": param.name,
Jerome Mariette's avatar
Jerome Mariette committed
160
161
                              "display_name": param.display_name,
                              "group": param.group}
162
163
164
165
166
167
168
169
170
171
172
                # if it's a multiple type add sub parameters
                if type(param.type) == jflow.parameter.MultipleParameters:
                    hash_param["sub_parameters"] = [] 
                    for sub_param in param.sub_parameters:
                        hash_param["sub_parameters"].append({"help": sub_param.help,
                              "required": sub_param.required,
                              "default": sub_param.default,
                              "choices": sub_param.choices,
                              "action": sub_param.action,
                              "type": sub_param.get_type(),
                              "name": param.name + JFlowServer.MULTIPLE_TYPE_SPLITER + sub_param.flag,
173
174
                              "display_name": sub_param.display_name,
                              "group": param.group})
Jerome Mariette's avatar
Jerome Mariette committed
175
176
177
178
179
                parameters.append(hash_param)
                if parameters_per_groups.has_key(param.group):
                    parameters_per_groups[param.group].append(hash_param)
                else: parameters_per_groups[param.group] = [hash_param]
                groups.append(param.group)
Jerome Mariette's avatar
Jerome Mariette committed
180
181
182
            workflows.append({"name": instance.name,
                              "help": instance.description,
                              "class": instance.__class__.__name__,
Jerome Mariette's avatar
Jerome Mariette committed
183
184
185
                              "parameters": parameters,
                              "parameters_per_groups": parameters_per_groups,
                              "groups": list(set(groups))})
Jerome Mariette's avatar
Jerome Mariette committed
186
187
188
189
190
        return workflows

    @cherrypy.expose
    @jsonify
    def run_workflow(self, **kwargs):
191
192
193
194
195
196
197
198
199
200
201
        kwargs_modified = {}
        for key in kwargs.keys():
            parts = key.split(JFlowServer.MULTIPLE_TYPE_SPLITER)
            if len(parts) == 1:
                kwargs_modified[key] = kwargs[key]
            elif len(parts) == 2:
                if kwargs_modified.has_key(parts[0]):
                    kwargs_modified[parts[0]].append((parts[1], kwargs[key]))
                else:
                    kwargs_modified[parts[0]] = [(parts[1], kwargs[key])]
        workflow = self.wfmanager.run_workflow(kwargs_modified["workflow_class"], kwargs_modified)
Jerome Mariette's avatar
Jerome Mariette committed
202
203
204
205
206
207
208
209
        return self.jsonify_workflow_status(workflow, True)

    @cherrypy.expose
    @jsonify
    def rerun_workflow(self, **kwargs):
        self.wfmanager.rerun_workflow(kwargs["workflow_id"])
        workflow = self.wfmanager.get_workflow(kwargs["workflow_id"])
        return self.jsonify_workflow_status(workflow)
210
211
212
213
214
215
216
    
    
    @cherrypy.expose
    @cherrypy.tools.noBodyProcess()
    @cherrypy.tools.CORS()
    def upload(self, **kwargs):
        file = kwargs.keys()[0]
Jerome Mariette's avatar
Jerome Mariette committed
217
        
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
        # the file transfer can take a long time; by default cherrypy
        # limits responses to 300s; we increase it to 1h
        cherrypy.response.timeout = 3600
         
        # convert the header keys to lower case
        lcHDRS = {}
        for key, val in cherrypy.request.headers.iteritems():
            lcHDRS[key.lower()] = val
         
        # at this point we could limit the upload on content-length...
        # incomingBytes = int(lcHDRS['content-length'])
         
        # create our version of cgi.FieldStorage to parse the MIME encoded
        # form data where the file is contained
        print "**********************************************************"
        print cherrypy.request.rfile
        print "**********************************************************"
        formFields = myFieldStorage(fp=cherrypy.request.rfile,
                                    headers=lcHDRS,
                                    environ={'REQUEST_METHOD':'POST'},
                                    keep_blank_values=True)

        # we now create a 2nd link to the file, using the submitted
        # filename; if we renamed, there would be a failure because
        # the NamedTemporaryFile, used by our version of cgi.FieldStorage,
        # explicitly deletes the original filename
        print "..............................................................."
        print formFields
        print "..............................................................."
        theFile = formFields[file]
        print theFile
        os.link(theFile.file.name, '/tmp/'+theFile.filename)
         
        return "ok, got it filename='%s'" % theFile.filename
        
    
Jerome Mariette's avatar
Jerome Mariette committed
254
255
256
257
258
259
260
261
262
263
    @cherrypy.expose
    @jsonify
    def get_workflows_status(self, **kwargs):
        if kwargs.has_key("workflow_id"):
            workflow = self.wfmanager.get_workflow(kwargs["workflow_id"])
            return self.jsonify_workflow_status(workflow)
        else:
            status = []
            workflows = self.wfmanager.get_workflows()
            for workflow in workflows:
Jerome Mariette's avatar
Jerome Mariette committed
264
265
266
267
268
269
270
271
272
273
274
                if kwargs.has_key("metadata_filter"):
                    is_ok = False
                    for wf_meta in workflow.metadata:
                        for metadata in kwargs["metadata_filter"].split(","):
                            if wf_meta == metadata:
                                is_ok = True
                                break
                        if is_ok: break
                    if is_ok: status.append(self.jsonify_workflow_status(workflow))
                else:
                    status.append(self.jsonify_workflow_status(workflow))
Jerome Mariette's avatar
Jerome Mariette committed
275
276
            return status

Philippe Bardou's avatar
Philippe Bardou committed
277
278
279
280
281
    @cherrypy.expose
    @jsonify
    def get_workflow_outputs(self, **kwargs):
        return self.wfmanager.get_workflow_ouputs(kwargs["workflow_id"])
    
Jerome Mariette's avatar
Jerome Mariette committed
282
283
284
285
286
287
288
289
290
291
292
293
294
295
    @cherrypy.expose
    @jsonify
    def validate_field(self, **kwargs):
        try:
            value_key = None
            for key in kwargs.keys():
                if key != "type" and key != "callback" and key != "_": 
                    value_key = key
                    break
            eval(kwargs["type"])(kwargs[value_key])
            return True
        except Exception, e:
            return str(e)

296

Jerome Mariette's avatar
Jerome Mariette committed
297
if __name__ == '__main__':
298
299
300
301
302
303
304
305
306
    
    parser = argparse.ArgumentParser()
    parser.add_argument("--daemon", action="store_true", dest="daemon", default=False, help="Run the server as daemon")
    args = vars(parser.parse_args())
    
    # daemonize the server if asked to
    if args["daemon"]:
        from cherrypy.process.plugins import Daemonizer
        Daemonizer(cherrypy.engine).subscribe()
307
308
309
    # define the socket host and port
    jflowconf = JFlowConfigReader()
    socket_opts = jflowconf.get_socket_options()
Jerome Mariette's avatar
Jerome Mariette committed
310
311
312
313
314
315
316
317

    # remove any limit on the request body size; cherrypy's default is 100MB
    # (maybe we should just increase it ?)
    cherrypy.server.max_request_body_size = 0
    
    # increase server socket timeout to 60s; we are more tolerant of bad
    # quality client-server connections (cherrypy's defult is 10s)
    cherrypy.server.socket_timeout = 60
318

319
320
    cherrypy.config.update({'server.socket_host': socket_opts[0],
                            'server.socket_port': socket_opts[1]})
Jerome Mariette's avatar
Jerome Mariette committed
321
    # start the server
322
    cherrypy.quickstart(JFlowServer())