fix download

src/ng6/ng6workflow.py

@@ -117,12 +117,13 @@ class BasicNG6Workflow (Workflow):
             self.set_to_address(infos['email'])
 
 
-class DownloadWorkflow(BasicNG6Workflow):
+class DownloadWorkflow(Workflow):
     """
         Base class for download workflows classes
     """
     def __init__(self, args={}, id=None, function= "process"):
-        BasicNG6Workflow.__init__(self, args, id, function)
+        Workflow.__init__(self, args, id, function)
+        self.add_parameter("admin_login", "Who is the project administrator", type = 'ng6userlogin', display_name="Admin login")
         self.add_parameter_list('data_id', 'Ids of a run from which rawdata will be retrieved', type = 'existingrun')
         self.add_parameter_list('run_id', 'Ids of run from which all data will be retrieved', type = 'existingrun')
         self.add_parameter_list('analysis_id', 'Ids of analysis to retrieve', type = 'existinganalysis')

src/ng6/t3MySQLdb.py

@@ -389,7 +389,7 @@ class t3MySQLdb(object):
                 'tx_nG6_run.date DESC'
         ]
         sql = " ".join(parts)
-        qresult = self.execute(sql)
+        qresult = self.execute(sql, dictc = True)
         row = qresult.rows[0]
         return { 
                 'id'            : row['run_id'],  
@@ -816,7 +816,7 @@ class t3MySQLdb(object):
         ]
         
         sql = " ".join(parts)
-        qresult = self.execute(sql)
+        qresult = self.execute(sql, dictc = True)
         
         # project analysis, add project informations
         if qresult.rowcount > 0:
@@ -871,7 +871,7 @@ class t3MySQLdb(object):
             
             
             sql = " ".join(parts)
-            qresult = self.execute(sql)
+            qresult = self.execute(sql, dictc = True)
             row = qresult.rows[0]
 
             return {

src/ng6/utils.py

@@ -32,13 +32,6 @@ import base64
 import pty
 import sys
 
-def b_decode(func):
-    def b_decode_wrapper(name):
-        if isinstance(name, bytes) :
-            # defualt decodeto utf-8
-            return func(name).decode()
-        return func(name)
-    return b_decode_wrapper
 
 class Utils(object):
     """
@@ -335,15 +328,20 @@ class Utils(object):
     
     @staticmethod
     def encode_passwd(clear):
+        return base64.urlsafe_b64encode(clear.encode()).decode()
+        """
         enc = []
         for i in range(len(clear)):
             key_c = Utils.PASS_KEY[i % len(Utils.PASS_KEY)]
             enc_c = chr((ord(clear[i]) + ord(key_c)) % 256)
             enc.append(enc_c)
         return base64.urlsafe_b64encode("".join(enc))
+        """
     
     @staticmethod
     def decode_passwd(enc):
+        return base64.urlsafe_b64decode(enc.encode()).decode()
+        """
         dec = []
         enc = base64.urlsafe_b64decode(enc)
         for i in range(len(enc)):
@@ -351,6 +349,7 @@ class Utils(object):
             dec_c = chr((256 + ord(enc[i]) - ord(key_c)) % 256)
             dec.append(dec_c)
         return "".join(dec)
+        """
     
     @staticmethod
     def get_directories_structure_and_content(ng6_username, data_folder, output_folder, prefixed_ids):
@@ -442,13 +441,13 @@ class SSH(object):
             return (pid, f)     
     
     def _read(self, f):
-        x = ''
+        x = b''
         try:
             x = os.read(f, 1024)
         except Exception as e:
             # this always fails with io error
             pass
-        return x
+        return x.decode()
 
     def ssh_results(self, pid, f):
         output = ''
@@ -456,7 +455,7 @@ class SSH(object):
         m = re.search( 'assword:', got)
         if m:
             # send passwd
-            os.write(f, self.password + '\n')
+            os.write(f, (self.password + '\n').encode() )
             # read two lines
             tmp = self._read(f)
             tmp += self._read(f)

workflows/download_archive/__init__.py

@@ -26,5 +26,5 @@ class DownloadArchive (DownloadWorkflow):
         self.add_parameter('archive_mail', 'Provide a valid email address to be warned when the archive is created', display_name = "User mail", type="email", required = True)
         
     def process(self):
-        self.add_component('CreateArchive', [self.admin_login, self.archive_mail, self.data_id, self.run_id, self.analysis_id])
+        self.add_component('CreateArchive', [self.archive_mail, self.data_id, self.run_id, self.analysis_id, self.admin_login])
         

workflows/download_archive/components/createarchive.py

@@ -22,7 +22,7 @@ from jflow.component import Component
 
 from weaver.function import PythonFunction
 
-def create_archive( ng6_username, email, output_folder, ids_dump_path, web_archive, archive_file ):
+def create_archive(archive_file,  email, output_folder, ids_dump_path, web_archive, ng6_username  = None ):
     import os
     import pickle
     import shutil
@@ -42,6 +42,8 @@ def create_archive( ng6_username, email, output_folder, ids_dump_path, web_archi
     fh.close()
     
     prefixed_ids = ";".join(prefixed_ids)
+    if ng6_username in ['', 'None', 'none', 'null', None] :
+        ng6_username = None
     src_directories, dest_directories = Utils.get_directories_structure_and_content(ng6_username, data_folder, output_folder, prefixed_ids)
     
     for i, source_dir in enumerate(src_directories):
@@ -73,7 +75,7 @@ def create_archive( ng6_username, email, output_folder, ids_dump_path, web_archi
     
 class CreateArchive (Component):
     
-    def define_parameters(self, login,  email, data_ids = [], run_ids = [], analysis_ids = []):
+    def define_parameters(self, email, data_ids = [], run_ids = [], analysis_ids = [], login = None):
         self.add_parameter('login', 'The login of the user in ng6', type = 'ng6userlogin', default = login)
         self.add_parameter('email', 'The email where the url to the archive will be sent', default = email, required = True)
         self.add_parameter_list('data_ids', 'Ids of a run from which rawdata will be retrieved', default = data_ids)
@@ -94,8 +96,8 @@ class CreateArchive (Component):
         fh.close()
         web_archive = self._webify_outputs(self.archive)
         
-        archive = PythonFunction(create_archive, cmd_format = "{EXE} {ARG} {IN} {OUT} ")
-        archive(arguments = [ self.login, self.email, output_folder, ids_dump_path, web_archive], outputs = self.archive)
+        archive = PythonFunction(create_archive, cmd_format = "{EXE} {IN} {OUT} {ARG}" )
+        archive(arguments = [ self.email, output_folder, ids_dump_path, web_archive, self.login], outputs = self.archive)
     
     
     def _webify_outputs(self, path):

workflows/download_symlink/__init__.py

@@ -30,9 +30,9 @@ class DownloadSymlink (DownloadWorkflow):
         self.add_parameter('username', 'A valid user username on the server', type = "localuser" ,required = True)
         self.add_parameter('password', 'The connection password for the username on the server', type="password", required = True)
         self.add_parameter('user_directory', """The directory where the links will be created. This directory must exists on the server and the 
-        provided user must have write permission on this directory""", default = config.get_server_directory(), 
+        provided user must have write permission on this directory""", default = config.get_user_base_directory(), 
                            required = True, type="localdirectory")
         
     def process(self):
-        self.add_component('CreateSymlink', [self.admin_login, self.username, self.password, self.user_directory, self.data_id, self.run_id, self.analysis_id])
+        self.add_component('CreateSymlink', [self.username, self.password, self.user_directory, self.data_id, self.run_id, self.analysis_id, self.admin_login])
         

workflows/download_symlink/components/createsymlink.py

@@ -23,7 +23,7 @@ from ng6.utils import Utils
 
 from weaver.function import PythonFunction
 
-def create_symbolik_link (username, password, ng6_username, output_folder, ids_dump_path, output_list, user_script):
+def create_symbolik_link (ids_dump_path, output_list, user_script, username, password, output_folder, ng6_username = None):
     import os
     import pickle
     import subprocess
@@ -31,11 +31,12 @@ def create_symbolik_link (username, password, ng6_username, output_folder, ids_d
     from ng6.config_reader import NG6ConfigReader 
     from ng6.utils import Utils, SSH
     
-    
     config = NG6ConfigReader()
     data_folder = config.get_save_directory()
     password = Utils.decode_passwd(password)
-    
+    if ng6_username in ['', 'None', 'none', 'null', None] :
+        ng6_username = None
+        
     try:
         fh = open(ids_dump_path, "rb")
         prefixed_ids = pickle.load(fh)
@@ -72,7 +73,7 @@ def create_symbolik_link (username, password, ng6_username, output_folder, ids_d
 
 class CreateSymlink (Component):
     
-    def define_parameters(self, login, username, password, user_directory, data_ids = [], run_ids = [], analysis_ids = []):
+    def define_parameters(self, username, password, user_directory, data_ids = [], run_ids = [], analysis_ids = [], login = None):
         self.add_parameter('login', 'The login of the user in ng6', type = 'ng6userlogin', default = login)
         self.add_parameter('username', 'Valid username on the server', required = True, default = username)
         self.add_parameter('password', 'A password for the usernmae on the server', required = True, default = Utils.encode_passwd(password))
@@ -93,8 +94,8 @@ class CreateSymlink (Component):
         pickle.dump( run_ids + data_ids + analysis_ids, fh)
         fh.close()
         
-        createlinks = PythonFunction(create_symbolik_link, cmd_format = "{EXE} {ARG} {IN} {OUT} ")
-        createlinks(arguments = [ self.username, "'"+self.password+"'", self.login, self.user_directory ], 
+        createlinks = PythonFunction(create_symbolik_link, cmd_format = "{EXE} {IN} {OUT} {ARG} ")
+        createlinks(arguments = [ self.username, "'"+self.password+"'", self.user_directory, self.login ], 
                     inputs = ids_dump_path , outputs = [self.output_list, self.user_script], local = True)
         
         
\ No newline at end of file

workflows/download_url/__init__.py

@@ -26,5 +26,5 @@ class DownloadURL (DownloadWorkflow):
         pass
     
     def process(self):
-        self.add_component('GetURLS', [self.admin_login, self.data_id, self.run_id, self.analysis_id])
+        self.add_component('GetURLS', [self.data_id, self.run_id, self.analysis_id, self.admin_login])
         

workflows/download_url/components/geturls.py

@@ -22,7 +22,7 @@ from jflow.component import Component
 
 from weaver.function import PythonFunction
 
-def get_urls( ng6_username, ids_dump_path, output_url):
+def get_urls( output_url, ids_dump_path, ng6_username = None):
     import os
     import pickle
     import subprocess
@@ -34,7 +34,9 @@ def get_urls( ng6_username, ids_dump_path, output_url):
     
     ng6config = NG6ConfigReader()
     data_directory = ng6config.get_save_directory()
-    
+    if ng6_username in ['', 'None', 'none', 'null', None] :
+        ng6_username = None
+        
     fh = open(ids_dump_path, "rb")
     prefixed_ids = pickle.load(fh)
     fh.close()
@@ -56,7 +58,7 @@ def get_urls( ng6_username, ids_dump_path, output_url):
 
 class GetURLS (Component):
     
-    def define_parameters(self, login, data_ids = [], run_ids = [], analysis_ids = []):
+    def define_parameters(self, data_ids = [], run_ids = [], analysis_ids = [], login = None):
         self.add_parameter('login', 'The login of the user in ng6', type = 'ng6userlogin', default = login)
         self.add_parameter_list('data_ids', 'Ids of a run from which rawdata will be retrieved', default = data_ids)
         self.add_parameter_list('run_ids', 'Ids of run from which all data will be retrieved', default = run_ids)
@@ -74,7 +76,7 @@ class GetURLS (Component):
         pickle.dump( run_ids + data_ids + analysis_ids, fh)
         fh.close()
         
-        fn = PythonFunction(get_urls, cmd_format = "{EXE} {ARG} {IN} {OUT} ")
-        fn(arguments = [ self.login, ids_dump_path], outputs = self.output_url, local = True)
+        fn = PythonFunction(get_urls, cmd_format = "{EXE} {IN} {OUT} {ARG} ")
+        fn(arguments = [ ids_dump_path, self.login], outputs = self.output_url, local = True)
     
     
\ No newline at end of file