Commit 762a481e authored by Jerome Mariette's avatar Jerome Mariette
Browse files

url ok from cli

parent bc7b2073
...@@ -21,6 +21,7 @@ import re ...@@ -21,6 +21,7 @@ import re
import sys import sys
import uuid import uuid
import pkgutil import pkgutil
import urllib2
import tempfile import tempfile
import pickle import pickle
import time import time
...@@ -29,6 +30,7 @@ from ConfigParser import ConfigParser, NoOptionError ...@@ -29,6 +30,7 @@ from ConfigParser import ConfigParser, NoOptionError
import types import types
import logging import logging
from inspect import getcallargs from inspect import getcallargs
from urlparse import urlparse
import jflow import jflow
import jflow.utils as utils import jflow.utils as utils
...@@ -222,6 +224,44 @@ class Workflow(threading.Thread): ...@@ -222,6 +224,44 @@ class Workflow(threading.Thread):
def get_components_nameid(self): def get_components_nameid(self):
return self.component_nameids.keys() return self.component_nameids.keys()
def _download_url(self, url):
print "Downloading " + url + " ..."
uri_object = urlparse(url)
opener = urllib2.urlopen(url)
block_size = 8000
jflowconf = JFlowConfigReader()
tmp_directory = os.path.join(jflowconf.get_tmp_directory(), os.path.basename(tempfile.NamedTemporaryFile().name))
os.mkdir(tmp_directory)
downloaded_file_path = os.path.join(tmp_directory, os.path.basename(uri_object.path))
downloaded_file = open(downloaded_file_path, 'wb')
metadata = opener.info()
while True:
buffer = opener.read(block_size)
# End of download
if not buffer: break
# Parts of download
downloaded_file.write(buffer)
downloaded_file.close()
logging.getLogger("types.ifile").debug("URL file '{0}' successfully downloaded as: {1}".format(url, downloaded_file_path))
return downloaded_file_path
def _is_url(self, file):
uri_object = urlparse(file)
return uri_object.scheme != ''
def _download_input_files(self):
for param in self.parameters:
if param.type.__name__ == "inputfile":
if param.action == "append":
for i, file in enumerate(self.args[param.name]):
if self._is_url(file):
# download file
self.args[param.name][i] = self._download_url(file)
else:
if self._is_url(file):
# download file
self.args[param.name] = self._download_url(file)
def run(self): def run(self):
""" """
Only require for Threading Only require for Threading
...@@ -232,6 +272,7 @@ class Workflow(threading.Thread): ...@@ -232,6 +272,7 @@ class Workflow(threading.Thread):
self.step = 0 self.step = 0
self.status = self.STATUS_STARTED self.status = self.STATUS_STARTED
self.end_time = None self.end_time = None
self._download_input_files()
self._serialize() self._serialize()
# if pre_processing has not been done yet # if pre_processing has not been done yet
if self.step == 0: if self.step == 0:
......
...@@ -35,32 +35,27 @@ def localfile(file): ...@@ -35,32 +35,27 @@ def localfile(file):
else: else:
raise argparse.ArgumentTypeError("Local file '" + file + "' does not exists! Please provide a valid file path!") raise argparse.ArgumentTypeError("Local file '" + file + "' does not exists! Please provide a valid file path!")
def urlfile(url): def inputfile(ifile):
try: # test the format
uri_object = urlparse(url) uri_object = urlparse(ifile)
opener = urllib2.urlopen(url) # check the file
except: if uri_object.scheme == '':
raise argparse.ArgumentTypeError("URL '" + url + "' is invalid!") if os.path.isfile(ifile):
block_size = 8000 return os.path.abspath(ifile)
jflowconf = JFlowConfigReader() else:
tmp_directory = os.path.join(jflowconf.get_tmp_directory(), os.path.basename(tempfile.NamedTemporaryFile().name)) raise argparse.ArgumentTypeError("Local file '" + ifile + "' does not exists! Please provide a valid file path!")
os.mkdir(tmp_directory)
file_name = os.path.join(tmp_directory, os.path.basename(uri_object.path))
local_file = open(file_name, 'wb')
if file_name is not None and file_name != "":
metadata = opener.info()
file_size = int(metadata.getheaders("Content-Length")[0])
if file_size == 0:
raise argparse.ArgumentTypeError("The URL file '" + url + "' is empty!")
while True:
buffer = opener.read(block_size)
# End of download
if not buffer: break
# Parts of download
local_file.write(buffer)
local_file.close()
logging.getLogger("types.urlfile").debug("URL file '{0}' successfully downloaded as: {1}".format(url, file_name))
return file_name
else: else:
raise argparse.ArgumentTypeError("URL '" + url + "' does not contain any file name!") try:
opener = urllib2.urlopen(ifile)
except:
raise argparse.ArgumentTypeError("URL '" + ifile + "' is invalid!")
file_name = os.path.basename(uri_object.path)
if file_name is not None and file_name != "":
metadata = opener.info()
file_size = int(metadata.getheaders("Content-Length")[0])
if file_size == 0:
raise argparse.ArgumentTypeError("The URL file '" + ifile + "' is empty!")
return ifile
else:
raise argparse.ArgumentTypeError("URL '" + ifile + "' does not contain any file name!")
\ No newline at end of file
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment