url downoad ok

bin/jflow_server.py

@@ -22,9 +22,10 @@ from functools import wraps
 import time
 import os
 import argparse
-from argparse import ArgumentTypeError
 import logging
 
+from argparse import ArgumentTypeError
+
 try:
     import _preamble
 except ImportError:
@@ -109,40 +110,6 @@ class JFlowServer (object):
                 
             return status
     
-    """
-    @cherrypy.expose
-    @jsonify
-    def upload_urlfile(self, **kwargs):
-        url = kwargs['url']
-        try:
-            uri_object = urlparse(url)
-            opener = urllib2.urlopen(url)
-        except:
-            return {"errorMsg": "URL '" + url + "' is invalid!", "uploaded": False}
-        block_size = 8000
-        jflowconf = JFlowConfigReader()
-        tmp_directory = os.path.join(jflowconf.get_tmp_directory(), os.path.basename(tempfile.NamedTemporaryFile().name))
-        os.mkdir(tmp_directory) 
-        file_path = os.path.join(tmp_directory, os.path.basename(uri_object.path))
-        local_file = open(file_path, 'wb')
-        if os.path.basename(uri_object.path) is not None and os.path.basename(uri_object.path) != "":
-            metadata = opener.info()
-            file_size = int(metadata.getheaders("Content-Length")[0])
-            if file_size == 0:
-                return {"errorMsg": "The URL file '" + url + "' is empty!", "uploaded": False}
-            while True:
-                buffer = opener.read(block_size)
-                # End of download
-                if not buffer: break
-                # Parts of download
-                local_file.write(buffer)
-            local_file.close()
-            logging.getLogger("types.urlfile").debug("URL file '{0}' successfully downloaded as: {1}".format(url, file_path))
-            return {"file_name": os.path.basename(file_path), "file_path": file_path, "uploaded": True}
-        else:
-            return {"errorMsg": "URL '" + url + "' does not contain any file name!", "uploaded": False}
-    """
-    
     @cherrypy.expose
     @jsonify
     def get_available_workflows(self, **kwargs):

src/jflow/workflow.py

@@ -25,10 +25,13 @@ import tempfile
 import pickle
 import time
 import threading
-from ConfigParser import ConfigParser, NoOptionError
+import urllib2 
 import types
 import logging
+
+from ConfigParser import ConfigParser, NoOptionError
 from inspect import getcallargs
+from urlparse import urlparse
 
 import jflow
 import jflow.utils as utils
@@ -97,6 +100,7 @@ class Workflow(threading.Thread):
         
         self.id = id
         self.args = self._extend_and_format_args(self.parameters, args)
+        self.args = self._prepare_input_files(self.args)
         self.metadata = self.args["metadata"]
         if self.id is not None:
             self.directory = self.manager.get_workflow_directory(self.name, self.id)
@@ -277,6 +281,41 @@ class Workflow(threading.Thread):
             self.end_time = time.time()
             self._serialize()
     
+    def _download_urlfile(self, input):
+        try:
+            uri_object = urlparse(input)
+            opener = urllib2.urlopen(input)
+            block_size = 8000
+            jflowconf = JFlowConfigReader()
+            tmp_directory = os.path.join(jflowconf.get_tmp_directory(), os.path.basename(tempfile.NamedTemporaryFile().name))
+            os.mkdir(tmp_directory) 
+            file_path = os.path.join(tmp_directory, os.path.basename(uri_object.path))
+            local_file = open(file_path, 'wb')
+            if os.path.basename(uri_object.path) is not None and os.path.basename(uri_object.path) != "":
+                metadata = opener.info()
+                file_size = int(metadata.getheaders("Content-Length")[0])
+                while True:
+                    buffer = opener.read(block_size)
+                    # End of download
+                    if not buffer: break
+                    # Parts of download
+                    local_file.write(buffer)
+                local_file.close()
+                logging.getLogger("types.urlfile").debug("URL file '{0}' successfully downloaded as: {1}".format(input, file_path))
+            return file_path
+        except:
+            return input
+    
+    def _prepare_input_files(self, args):
+        new_args = args
+        for param in args.keys():
+            if args[param].__class__.__name__ == "list":
+                for i, val in enumerate(args[param]):
+                    new_args[param][i] = self._download_urlfile(args[param][i])
+            else:
+                new_args[param] = self._download_urlfile(args[param])
+        return new_args
+    
     def _extend_and_format_args(self, parameters, args):
         extended_args = {}
         for param in parameters: