Commit 762a481e authored by Jerome Mariette's avatar Jerome Mariette
Browse files

url ok from cli

parent bc7b2073
......@@ -21,6 +21,7 @@ import re
import sys
import uuid
import pkgutil
import urllib2
import tempfile
import pickle
import time
......@@ -29,6 +30,7 @@ from ConfigParser import ConfigParser, NoOptionError
import types
import logging
from inspect import getcallargs
from urlparse import urlparse
import jflow
import jflow.utils as utils
......@@ -222,6 +224,44 @@ class Workflow(threading.Thread):
def get_components_nameid(self):
return self.component_nameids.keys()
def _download_url(self, url):
print "Downloading " + url + " ..."
uri_object = urlparse(url)
opener = urllib2.urlopen(url)
block_size = 8000
jflowconf = JFlowConfigReader()
tmp_directory = os.path.join(jflowconf.get_tmp_directory(), os.path.basename(tempfile.NamedTemporaryFile().name))
os.mkdir(tmp_directory)
downloaded_file_path = os.path.join(tmp_directory, os.path.basename(uri_object.path))
downloaded_file = open(downloaded_file_path, 'wb')
metadata = opener.info()
while True:
buffer = opener.read(block_size)
# End of download
if not buffer: break
# Parts of download
downloaded_file.write(buffer)
downloaded_file.close()
logging.getLogger("types.ifile").debug("URL file '{0}' successfully downloaded as: {1}".format(url, downloaded_file_path))
return downloaded_file_path
def _is_url(self, file):
uri_object = urlparse(file)
return uri_object.scheme != ''
def _download_input_files(self):
for param in self.parameters:
if param.type.__name__ == "inputfile":
if param.action == "append":
for i, file in enumerate(self.args[param.name]):
if self._is_url(file):
# download file
self.args[param.name][i] = self._download_url(file)
else:
if self._is_url(file):
# download file
self.args[param.name] = self._download_url(file)
def run(self):
"""
Only require for Threading
......@@ -232,6 +272,7 @@ class Workflow(threading.Thread):
self.step = 0
self.status = self.STATUS_STARTED
self.end_time = None
self._download_input_files()
self._serialize()
# if pre_processing has not been done yet
if self.step == 0:
......
......@@ -35,32 +35,27 @@ def localfile(file):
else:
raise argparse.ArgumentTypeError("Local file '" + file + "' does not exists! Please provide a valid file path!")
def urlfile(url):
def inputfile(ifile):
# test the format
uri_object = urlparse(ifile)
# check the file
if uri_object.scheme == '':
if os.path.isfile(ifile):
return os.path.abspath(ifile)
else:
raise argparse.ArgumentTypeError("Local file '" + ifile + "' does not exists! Please provide a valid file path!")
else:
try:
uri_object = urlparse(url)
opener = urllib2.urlopen(url)
opener = urllib2.urlopen(ifile)
except:
raise argparse.ArgumentTypeError("URL '" + url + "' is invalid!")
block_size = 8000
jflowconf = JFlowConfigReader()
tmp_directory = os.path.join(jflowconf.get_tmp_directory(), os.path.basename(tempfile.NamedTemporaryFile().name))
os.mkdir(tmp_directory)
file_name = os.path.join(tmp_directory, os.path.basename(uri_object.path))
local_file = open(file_name, 'wb')
raise argparse.ArgumentTypeError("URL '" + ifile + "' is invalid!")
file_name = os.path.basename(uri_object.path)
if file_name is not None and file_name != "":
metadata = opener.info()
file_size = int(metadata.getheaders("Content-Length")[0])
if file_size == 0:
raise argparse.ArgumentTypeError("The URL file '" + url + "' is empty!")
while True:
buffer = opener.read(block_size)
# End of download
if not buffer: break
# Parts of download
local_file.write(buffer)
local_file.close()
logging.getLogger("types.urlfile").debug("URL file '{0}' successfully downloaded as: {1}".format(url, file_name))
return file_name
raise argparse.ArgumentTypeError("The URL file '" + ifile + "' is empty!")
return ifile
else:
raise argparse.ArgumentTypeError("URL '" + url + "' does not contain any file name!")
raise argparse.ArgumentTypeError("URL '" + ifile + "' does not contain any file name!")
\ No newline at end of file
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment