createarchive.py 4.37 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
#
# Copyright (C) 2015 INRA
# 
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
# 
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
# 
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
#

import os
import pickle

from jflow.component import Component

from weaver.function import PythonFunction

Penom Nom's avatar
Penom Nom committed
25
def create_archive( ng6_username, email, output_folder, ids_dump_path, web_archive, archive_file ):
26
27
28
29
30
31
32
    import os
    import pickle
    import shutil
    import subprocess
    import tarfile
    
    from jflow.utils import robust_rmtree
Penom Nom's avatar
Penom Nom committed
33
    
Penom Nom's avatar
Penom Nom committed
34
    from ng6.utils import Utils
Penom Nom's avatar
Penom Nom committed
35
36
37
38
    from ng6.config_reader import NG6ConfigReader
    
    config = NG6ConfigReader()
    data_folder = config.get_save_directory()
39
40
41
42
43
44
    
    fh = open(ids_dump_path, "rb")
    prefixed_ids = pickle.load(fh)
    fh.close()
    
    prefixed_ids = ";".join(prefixed_ids)
Penom Nom's avatar
Penom Nom committed
45
    src_directories, dest_directories = Utils.get_directories_structure_and_content(ng6_username, data_folder, output_folder, prefixed_ids)
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
    
    for i, source_dir in enumerate(src_directories):
        dest_dir = dest_directories[i]
        if not os.path.exists(dest_dir):
            os.makedirs(dest_dir, 0755)
        
        for filename in os.listdir(source_dir):
            source_file_path = os.path.join(source_dir, filename)
            dest_file_path = os.path.join(dest_dir, filename)
            if os.path.isfile(source_file_path) and filename not in [ "analyse.xml", "index.html", "run.cfg"] :
                shutil.copyfile(source_file_path, dest_file_path)
    
    # create archive
    with tarfile.open(archive_file, "w:gz") as tar :
        tar.add(output_folder, os.path.basename(output_folder))

    # send mail
    try :
        message = 'Dear user, your archive is ready for download at the following link : %s'%web_archive
        subject = '[NG6] Archive ready to be downloaded'
        subprocess.check_call( "echo '%s'  | mail -s '%s' %s"%(message, subject, email)   , shell = True)
    except :
        raise Exception('Unable to send mail')
    
    # delete output folder
    robust_rmtree(output_folder)
    
    
class CreateArchive (Component):
    
Penom Nom's avatar
Penom Nom committed
76
    def define_parameters(self, login,  email, data_ids = [], run_ids = [], analysis_ids = []):
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
        self.add_parameter('login', 'The login of the user in ng6', type = 'ng6userlogin', default = login)
        self.add_parameter('email', 'The email where the url to the archive will be sent', default = email, required = True)
        self.add_parameter_list('data_ids', 'Ids of a run from which rawdata will be retrieved', default = data_ids)
        self.add_parameter_list('run_ids', 'Ids of run from which all data will be retrieved', default = run_ids)
        self.add_parameter_list('analysis_ids', 'Ids of analysis to retrieve', default = analysis_ids)
        
        self.add_output_file("archive", "created archive name", filename = "ng6_data_archive.tar.gz")
        
    def process(self):
        run_ids = [ "run_%s"%i for i in self.run_ids ]
        data_ids = [ "data_%s"%i for i in self.data_ids ]
        analysis_ids = [ "analyse_%s"%i for i in self.analysis_ids ]
        
        ids_dump_path = self.get_temporary_file(".dump")
        fh =  open(ids_dump_path, "wb")
        pickle.dump( run_ids + data_ids + analysis_ids, fh)
        output_folder = os.path.join(self.output_directory, 'ng6_data')
        fh.close()
        web_archive = self._webify_outputs(self.archive)
        
        archive = PythonFunction(create_archive, cmd_format = "{EXE} {ARG} {IN} {OUT} ")
Penom Nom's avatar
Penom Nom committed
98
        archive(arguments = [ self.login, self.email, output_folder, ids_dump_path, web_archive], outputs = self.archive)
99
100
101
102
103
104
    
    
    def _webify_outputs(self, path):
        work_dir  = self.config_reader.get_work_directory()
        if work_dir.endswith("/"): work_dir = work_dir[:-1]
        socket_opt = self.config_reader.get_socket_options()
105
        return 'http://' + socket_opt[0] + ':' + str(socket_opt[1]) + '/' + path.replace(work_dir, 'data')
106