url ok from cli

src/jflow/workflow.py

@@ -21,6 +21,7 @@ import re
 import sys
 import uuid
 import pkgutil
+import urllib2 
 import tempfile
 import pickle
 import time
@@ -29,6 +30,7 @@ from ConfigParser import ConfigParser, NoOptionError
 import types
 import logging
 from inspect import getcallargs
+from urlparse import urlparse
 
 import jflow
 import jflow.utils as utils
@@ -222,6 +224,44 @@ class Workflow(threading.Thread):
     def get_components_nameid(self):
         return self.component_nameids.keys()
     
+    def _download_url(self, url):
+        print "Downloading " + url + " ..."
+        uri_object = urlparse(url)
+        opener = urllib2.urlopen(url)
+        block_size = 8000
+        jflowconf = JFlowConfigReader()
+        tmp_directory = os.path.join(jflowconf.get_tmp_directory(), os.path.basename(tempfile.NamedTemporaryFile().name))
+        os.mkdir(tmp_directory) 
+        downloaded_file_path = os.path.join(tmp_directory, os.path.basename(uri_object.path))
+        downloaded_file = open(downloaded_file_path, 'wb')
+        metadata = opener.info()
+        while True:
+            buffer = opener.read(block_size)
+            # End of download
+            if not buffer: break
+            # Parts of download
+            downloaded_file.write(buffer)
+        downloaded_file.close()
+        logging.getLogger("types.ifile").debug("URL file '{0}' successfully downloaded as: {1}".format(url, downloaded_file_path))
+        return downloaded_file_path
+    
+    def _is_url(self, file):
+        uri_object = urlparse(file)
+        return uri_object.scheme != ''
+    
+    def _download_input_files(self):
+        for param in self.parameters:
+            if param.type.__name__ == "inputfile":
+                if param.action == "append":
+                    for i, file in enumerate(self.args[param.name]):
+                        if self._is_url(file):
+                            # download file
+                            self.args[param.name][i] = self._download_url(file)
+                else:
+                    if self._is_url(file):
+                        # download file
+                        self.args[param.name] = self._download_url(file)
+    
     def run(self):
         """
         Only require for Threading
@@ -232,6 +272,7 @@ class Workflow(threading.Thread):
             self.step = 0
             self.status = self.STATUS_STARTED
             self.end_time = None
+            self._download_input_files()
             self._serialize()
         # if pre_processing has not been done yet    
         if self.step == 0:

workflows/types.py

@@ -35,32 +35,27 @@ def localfile(file):
     else:
         raise argparse.ArgumentTypeError("Local file '" + file + "' does not exists! Please provide a valid file path!")
     
-def urlfile(url):
-    try:
-        uri_object = urlparse(url)
-        opener = urllib2.urlopen(url)
-    except:
-        raise argparse.ArgumentTypeError("URL '" + url + "' is invalid!")
-    block_size = 8000
-    jflowconf = JFlowConfigReader()
-    tmp_directory = os.path.join(jflowconf.get_tmp_directory(), os.path.basename(tempfile.NamedTemporaryFile().name))
-    os.mkdir(tmp_directory) 
-    file_name = os.path.join(tmp_directory, os.path.basename(uri_object.path))
-    local_file = open(file_name, 'wb')
-    if file_name is not None and file_name != "":
-        metadata = opener.info()
-        file_size = int(metadata.getheaders("Content-Length")[0])
-        if file_size == 0:
-            raise argparse.ArgumentTypeError("The URL file '" + url + "' is empty!")
-        while True:
-            buffer = opener.read(block_size)
-            # End of download
-            if not buffer: break
-            # Parts of download
-            local_file.write(buffer)
-        local_file.close()
-        logging.getLogger("types.urlfile").debug("URL file '{0}' successfully downloaded as: {1}".format(url, file_name))
-        return file_name
+def inputfile(ifile):
+    # test the format
+    uri_object = urlparse(ifile)
+    # check the file
+    if uri_object.scheme == '':
+        if os.path.isfile(ifile):
+            return os.path.abspath(ifile)
+        else:
+            raise argparse.ArgumentTypeError("Local file '" + ifile + "' does not exists! Please provide a valid file path!")
     else:
-        raise argparse.ArgumentTypeError("URL '" + url + "' does not contain any file name!")
+        try:
+            opener = urllib2.urlopen(ifile)
+        except:
+            raise argparse.ArgumentTypeError("URL '" + ifile + "' is invalid!")
+        file_name = os.path.basename(uri_object.path)
+        if file_name is not None and file_name != "":
+            metadata = opener.info()
+            file_size = int(metadata.getheaders("Content-Length")[0])
+            if file_size == 0:
+                raise argparse.ArgumentTypeError("The URL file '" + ifile + "' is empty!")
+            return ifile
+        else:
+            raise argparse.ArgumentTypeError("URL '" + ifile + "' does not contain any file name!")
     
\ No newline at end of file