Commit e3aee9e8 authored by Frédéric Escudié's avatar Frédéric Escudié
Browse files

Use specific field storage to upload file.

parent dd772dbb
......@@ -42,20 +42,32 @@ import jflow.utils as utils
from cctools.util import time_format
# function in charge to upload large files
class myFieldStorage(cgi.FieldStorage):
class UploadFieldStorage(cgi.FieldStorage):
"""Our version uses a named temporary file instead of the default
non-named file; keeping it visibile (named), allows us to create a
2nd link after the upload is done, thus avoiding the overhead of
making a copy to the destination filename."""
def get_tmp_directory(self):
jflowconf = JFlowConfigReader()
return jflowconf.get_tmp_directory()
def make_file(self, binary=None):
return tempfile.NamedTemporaryFile()
jflowconf = JFlowConfigReader()
tmp_folder = jflowconf.get_tmp_directory()
if not os.path.exists( tmp_folder ):
try : os.mkdir(tmp_folder)
except : pass
return tempfile.NamedTemporaryFile(dir=tmp_folder)
def noBodyProcess():
"""Sets cherrypy.request.process_request_body = False, giving
us direct control of the file upload destination. By default
cherrypy loads it to memory, we are directing it to disk."""
cherrypy.request.process_request_body = False
cherrypy.tools.noBodyProcess = cherrypy.Tool('before_finalize', noBodyProcess)
cherrypy.tools.noBodyProcess = cherrypy.Tool('before_request_body', noBodyProcess)
# define functions in charge to handle cross domain calls
def CORS():
......@@ -69,7 +81,7 @@ class JFlowServer (object):
MULTIPLE_TYPE_SPLITER = "._."
JFLOW_WDATA = "data"
def __init__(self):
# Create a workflow manager to get access to our workflows
self.wfmanager = WorkflowsManager()
......@@ -133,9 +145,9 @@ class JFlowServer (object):
"start_time": start_time,
"end_time": end_time,
"components": components}
return status
@cherrypy.expose
@jsonify
def get_available_workflows(self, **kwargs):
......@@ -208,71 +220,42 @@ class JFlowServer (object):
self.wfmanager.rerun_workflow(kwargs["workflow_id"])
workflow = self.wfmanager.get_workflow(kwargs["workflow_id"])
return self.jsonify_workflow_status(workflow)
@cherrypy.expose
def upload(self, **kwargs):
prefix = ""
for key in kwargs.keys():
if key == "prefix":
prefix = kwargs[key]
else:
file_param = key
# the file transfer can take a long time; by default cherrypy
# limits responses to 300s; we increase it to 1h
cherrypy.response.timeout = 3600
# upload file by chunks
FH_sever_file = open(os.path.join(self.jflow_config_reader.get_tmp_directory(), prefix+"_"+kwargs[file_param].filename.encode('ascii','ignore')), "w")
while True:
data = kwargs[file_param].file.read(8192)
if not data:
break
FH_sever_file.write(data)
FH_sever_file.close()
@cherrypy.expose
@cherrypy.tools.noBodyProcess()
@cherrypy.tools.CORS()
def upload2(self, **kwargs):
file = kwargs.keys()[0]
def upload(self):
# the file transfer can take a long time; by default cherrypy
# limits responses to 300s; we increase it to 1h
cherrypy.response.timeout = 3600
# convert the header keys to lower case
lcHDRS = {}
for key, val in cherrypy.request.headers.iteritems():
lcHDRS[key.lower()] = val
# at this point we could limit the upload on content-length...
# incomingBytes = int(lcHDRS['content-length'])
# create our version of cgi.FieldStorage to parse the MIME encoded
# form data where the file is contained
print "**********************************************************"
print cherrypy.request.rfile
print "**********************************************************"
formFields = myFieldStorage(fp=cherrypy.request.rfile,
headers=lcHDRS,
environ={'REQUEST_METHOD':'POST'},
keep_blank_values=True)
# we now create a 2nd link to the file, using the submitted
formFields = UploadFieldStorage(fp=cherrypy.request.rfile,
headers=lcHDRS,
environ={'REQUEST_METHOD':'POST'},
keep_blank_values=True)
# we now create a link to the file, using the submitted
# filename; if we renamed, there would be a failure because
# the NamedTemporaryFile, used by our version of cgi.FieldStorage,
# explicitly deletes the original filename
print "..............................................................."
print formFields
print "..............................................................."
theFile = formFields[file]
print theFile
os.link(theFile.file.name, '/tmp/'+theFile.filename)
return "ok, got it filename='%s'" % theFile.filename
for current in formFields.keys():
if current != 'prefix':
currentFile = formFields[current]
os.link(
currentFile.file.name,
os.path.join(formFields.get_tmp_directory(), formFields.getvalue("prefix") + "_" + currentFile.filename.encode('ascii','ignore'))
)
@cherrypy.expose
@jsonify
def get_workflows_status(self, **kwargs):
......@@ -300,7 +283,7 @@ class JFlowServer (object):
@jsonify
def get_workflow_outputs(self, **kwargs):
return self.wfmanager.get_workflow_ouputs(JFlowServer.JFLOW_WDATA, kwargs["workflow_id"])
@cherrypy.expose
@jsonify
def validate_field(self, **kwargs):
......@@ -315,13 +298,12 @@ class JFlowServer (object):
except Exception, e:
return str(e)
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument("--daemon", action="store_true", dest="daemon", default=False, help="Run the server as daemon")
args = vars(parser.parse_args())
# daemonize the server if asked to
if args["daemon"]:
from cherrypy.process.plugins import Daemonizer
......@@ -333,7 +315,7 @@ if __name__ == '__main__':
# remove any limit on the request body size; cherrypy's default is 100MB
# (maybe we should just increase it ?)
cherrypy.server.max_request_body_size = 0
# increase server socket timeout to 60s; we are more tolerant of bad
# quality client-server connections (cherrypy's defult is 10s)
cherrypy.server.socket_timeout = 60
......@@ -341,8 +323,8 @@ if __name__ == '__main__':
app_conf = {'/': {'tools.staticdir.root': jflowconf.get_work_directory()},
os.path.join('/', JFlowServer.JFLOW_WDATA): {'tools.staticdir.on' : True,
'tools.staticdir.dir' : jflowconf.get_work_directory()}}
cherrypy.config.update({'server.socket_host': socket_opts[0],
'server.socket_port': socket_opts[1]})
# start the server
cherrypy.quickstart(JFlowServer(), config=app_conf)
cherrypy.quickstart(JFlowServer(), config=app_conf)
\ No newline at end of file
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment