Skip to content
GitLab
Commit aad58bf9 authored by Penom Nom's avatar Penom Nom
Browse files

fix download

parent 79ab67bb
Expand all Hide whitespace changes
Inline Side-by-side
src/ng6/ng6workflow.py
View file @ aad58bf9
......@@ -117,12 +117,13 @@ class BasicNG6Workflow (Workflow):
self.set_to_address(infos['email'])
class DownloadWorkflow(BasicNG6Workflow):
class DownloadWorkflow(Workflow):
"""
Base class for download workflows classes
"""
def __init__(self, args={}, id=None, function= "process"):
BasicNG6Workflow.__init__(self, args, id, function)
Workflow.__init__(self, args, id, function)
self.add_parameter("admin_login", "Who is the project administrator", type = 'ng6userlogin', display_name="Admin login")
self.add_parameter_list('data_id', 'Ids of a run from which rawdata will be retrieved', type = 'existingrun')
self.add_parameter_list('run_id', 'Ids of run from which all data will be retrieved', type = 'existingrun')
self.add_parameter_list('analysis_id', 'Ids of analysis to retrieve', type = 'existinganalysis')
......
src/ng6/t3MySQLdb.py
View file @ aad58bf9
......@@ -389,7 +389,7 @@ class t3MySQLdb(object):
'tx_nG6_run.date DESC'
]
sql = " ".join(parts)
qresult = self.execute(sql)
qresult = self.execute(sql, dictc = True)
row = qresult.rows[0]
return {
'id' : row['run_id'],
......@@ -816,7 +816,7 @@ class t3MySQLdb(object):
]
sql = " ".join(parts)
qresult = self.execute(sql)
qresult = self.execute(sql, dictc = True)
# project analysis, add project informations
if qresult.rowcount > 0:
......@@ -871,7 +871,7 @@ class t3MySQLdb(object):
sql = " ".join(parts)
qresult = self.execute(sql)
qresult = self.execute(sql, dictc = True)
row = qresult.rows[0]
return {
......
src/ng6/utils.py
View file @ aad58bf9
......@@ -32,13 +32,6 @@ import base64
import pty
import sys
def b_decode(func):
def b_decode_wrapper(name):
if isinstance(name, bytes) :
# defualt decodeto utf-8
return func(name).decode()
return func(name)
return b_decode_wrapper
class Utils(object):
"""
......@@ -335,15 +328,20 @@ class Utils(object):
@staticmethod
def encode_passwd(clear):
return base64.urlsafe_b64encode(clear.encode()).decode()
"""
enc = []
for i in range(len(clear)):
key_c = Utils.PASS_KEY[i % len(Utils.PASS_KEY)]
enc_c = chr((ord(clear[i]) + ord(key_c)) % 256)
enc.append(enc_c)
return base64.urlsafe_b64encode("".join(enc))
"""
@staticmethod
def decode_passwd(enc):
return base64.urlsafe_b64decode(enc.encode()).decode()
"""
dec = []
enc = base64.urlsafe_b64decode(enc)
for i in range(len(enc)):
......@@ -351,6 +349,7 @@ class Utils(object):
dec_c = chr((256 + ord(enc[i]) - ord(key_c)) % 256)
dec.append(dec_c)
return "".join(dec)
"""
@staticmethod
def get_directories_structure_and_content(ng6_username, data_folder, output_folder, prefixed_ids):
......@@ -442,13 +441,13 @@ class SSH(object):
return (pid, f)
def _read(self, f):
x = ''
x = b''
try:
x = os.read(f, 1024)
except Exception as e:
# this always fails with io error
pass
return x
return x.decode()
def ssh_results(self, pid, f):
output = ''
......@@ -456,7 +455,7 @@ class SSH(object):
m = re.search( 'assword:', got)
if m:
# send passwd
os.write(f, self.password + '\n')
os.write(f, (self.password + '\n').encode() )
# read two lines
tmp = self._read(f)
tmp += self._read(f)
......
ui/nG6/res/js/tx_nG6_pi5.js
View file @ aad58bf9
This diff is collapsed.
workflows/download_archive/__init__.py
View file @ aad58bf9
......@@ -26,5 +26,5 @@ class DownloadArchive (DownloadWorkflow):
self.add_parameter('archive_mail', 'Provide a valid email address to be warned when the archive is created', display_name = "User mail", type="email", required = True)
def process(self):
self.add_component('CreateArchive', [self.admin_login, self.archive_mail, self.data_id, self.run_id, self.analysis_id])
self.add_component('CreateArchive', [self.archive_mail, self.data_id, self.run_id, self.analysis_id, self.admin_login])
workflows/download_archive/components/createarchive.py
View file @ aad58bf9
......@@ -22,7 +22,7 @@ from jflow.component import Component
from weaver.function import PythonFunction
def create_archive( ng6_username, email, output_folder, ids_dump_path, web_archive, archive_file ):
def create_archive(archive_file, email, output_folder, ids_dump_path, web_archive, ng6_username = None ):
import os
import pickle
import shutil
......@@ -42,6 +42,8 @@ def create_archive( ng6_username, email, output_folder, ids_dump_path, web_archi
fh.close()
prefixed_ids = ";".join(prefixed_ids)
if ng6_username in ['', 'None', 'none', 'null', None] :
ng6_username = None
src_directories, dest_directories = Utils.get_directories_structure_and_content(ng6_username, data_folder, output_folder, prefixed_ids)
for i, source_dir in enumerate(src_directories):
......@@ -73,7 +75,7 @@ def create_archive( ng6_username, email, output_folder, ids_dump_path, web_archi
class CreateArchive (Component):
def define_parameters(self, login, email, data_ids = [], run_ids = [], analysis_ids = []):
def define_parameters(self, email, data_ids = [], run_ids = [], analysis_ids = [], login = None):
self.add_parameter('login', 'The login of the user in ng6', type = 'ng6userlogin', default = login)
self.add_parameter('email', 'The email where the url to the archive will be sent', default = email, required = True)
self.add_parameter_list('data_ids', 'Ids of a run from which rawdata will be retrieved', default = data_ids)
......@@ -94,8 +96,8 @@ class CreateArchive (Component):
fh.close()
web_archive = self._webify_outputs(self.archive)
archive = PythonFunction(create_archive, cmd_format = "{EXE} {ARG} {IN} {OUT} ")
archive(arguments = [ self.login, self.email, output_folder, ids_dump_path, web_archive], outputs = self.archive)
archive = PythonFunction(create_archive, cmd_format = "{EXE} {IN} {OUT} {ARG}" )
archive(arguments = [ self.email, output_folder, ids_dump_path, web_archive, self.login], outputs = self.archive)
def _webify_outputs(self, path):
......
workflows/download_symlink/__init__.py
View file @ aad58bf9
......@@ -30,9 +30,9 @@ class DownloadSymlink (DownloadWorkflow):
self.add_parameter('username', 'A valid user username on the server', type = "localuser" ,required = True)
self.add_parameter('password', 'The connection password for the username on the server', type="password", required = True)
self.add_parameter('user_directory', """The directory where the links will be created. This directory must exists on the server and the
provided user must have write permission on this directory""", default = config.get_server_directory(),
provided user must have write permission on this directory""", default = config.get_user_base_directory(),
required = True, type="localdirectory")
def process(self):
self.add_component('CreateSymlink', [self.admin_login, self.username, self.password, self.user_directory, self.data_id, self.run_id, self.analysis_id])
self.add_component('CreateSymlink', [self.username, self.password, self.user_directory, self.data_id, self.run_id, self.analysis_id, self.admin_login])
workflows/download_symlink/components/createsymlink.py
View file @ aad58bf9
......@@ -23,7 +23,7 @@ from ng6.utils import Utils
from weaver.function import PythonFunction
def create_symbolik_link (username, password, ng6_username, output_folder, ids_dump_path, output_list, user_script):
def create_symbolik_link (ids_dump_path, output_list, user_script, username, password, output_folder, ng6_username = None):
import os
import pickle
import subprocess
......@@ -31,11 +31,12 @@ def create_symbolik_link (username, password, ng6_username, output_folder, ids_d
from ng6.config_reader import NG6ConfigReader
from ng6.utils import Utils, SSH
config = NG6ConfigReader()
data_folder = config.get_save_directory()
password = Utils.decode_passwd(password)
if ng6_username in ['', 'None', 'none', 'null', None] :
ng6_username = None
try:
fh = open(ids_dump_path, "rb")
prefixed_ids = pickle.load(fh)
......@@ -72,7 +73,7 @@ def create_symbolik_link (username, password, ng6_username, output_folder, ids_d
class CreateSymlink (Component):
def define_parameters(self, login, username, password, user_directory, data_ids = [], run_ids = [], analysis_ids = []):
def define_parameters(self, username, password, user_directory, data_ids = [], run_ids = [], analysis_ids = [], login = None):
self.add_parameter('login', 'The login of the user in ng6', type = 'ng6userlogin', default = login)
self.add_parameter('username', 'Valid username on the server', required = True, default = username)
self.add_parameter('password', 'A password for the usernmae on the server', required = True, default = Utils.encode_passwd(password))
......@@ -93,8 +94,8 @@ class CreateSymlink (Component):
pickle.dump( run_ids + data_ids + analysis_ids, fh)
fh.close()
createlinks = PythonFunction(create_symbolik_link, cmd_format = "{EXE} {ARG} {IN} {OUT} ")
createlinks(arguments = [ self.username, "'"+self.password+"'", self.login, self.user_directory ],
createlinks = PythonFunction(create_symbolik_link, cmd_format = "{EXE} {IN} {OUT} {ARG} ")
createlinks(arguments = [ self.username, "'"+self.password+"'", self.user_directory, self.login ],
inputs = ids_dump_path , outputs = [self.output_list, self.user_script], local = True)
\ No newline at end of file
workflows/download_url/__init__.py
View file @ aad58bf9
......@@ -26,5 +26,5 @@ class DownloadURL (DownloadWorkflow):
pass
def process(self):
self.add_component('GetURLS', [self.admin_login, self.data_id, self.run_id, self.analysis_id])
self.add_component('GetURLS', [self.data_id, self.run_id, self.analysis_id, self.admin_login])
workflows/download_url/components/geturls.py
View file @ aad58bf9
......@@ -22,7 +22,7 @@ from jflow.component import Component
from weaver.function import PythonFunction
def get_urls( ng6_username, ids_dump_path, output_url):
def get_urls( output_url, ids_dump_path, ng6_username = None):
import os
import pickle
import subprocess
......@@ -34,7 +34,9 @@ def get_urls( ng6_username, ids_dump_path, output_url):
ng6config = NG6ConfigReader()
data_directory = ng6config.get_save_directory()
if ng6_username in ['', 'None', 'none', 'null', None] :
ng6_username = None
fh = open(ids_dump_path, "rb")
prefixed_ids = pickle.load(fh)
fh.close()
......@@ -56,7 +58,7 @@ def get_urls( ng6_username, ids_dump_path, output_url):
class GetURLS (Component):
def define_parameters(self, login, data_ids = [], run_ids = [], analysis_ids = []):
def define_parameters(self, data_ids = [], run_ids = [], analysis_ids = [], login = None):
self.add_parameter('login', 'The login of the user in ng6', type = 'ng6userlogin', default = login)
self.add_parameter_list('data_ids', 'Ids of a run from which rawdata will be retrieved', default = data_ids)
self.add_parameter_list('run_ids', 'Ids of run from which all data will be retrieved', default = run_ids)
......@@ -74,7 +76,7 @@ class GetURLS (Component):
pickle.dump( run_ids + data_ids + analysis_ids, fh)
fh.close()
fn = PythonFunction(get_urls, cmd_format = "{EXE} {ARG} {IN} {OUT} ")
fn(arguments = [ self.login, ids_dump_path], outputs = self.output_url, local = True)
fn = PythonFunction(get_urls, cmd_format = "{EXE} {IN} {OUT} {ARG} ")
fn(arguments = [ ids_dump_path, self.login], outputs = self.output_url, local = True)
\ No newline at end of file
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment