Commit 69411079 authored by Penom Nom's avatar Penom Nom
Browse files

No commit message

No commit message
parent 92a9f756
#
# Copyright (C) 2012 INRA
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
\ No newline at end of file
#
# Copyright (C) 2012 INRA
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
import os
import pickle
from jflow.component import Component
from weaver.function import PythonFunction
def create_archive( ng6_username, email, data_folder, output_folder, ids_dump_path, web_archive, archive_file ):
import os
import pickle
import shutil
import subprocess
import tarfile
from jflow.utils import robust_rmtree
from workflows.download.lib.utils import get_directories_structure_and_content
fh = open(ids_dump_path, "rb")
prefixed_ids = pickle.load(fh)
fh.close()
prefixed_ids = ";".join(prefixed_ids)
src_directories, dest_directories = get_directories_structure_and_content(ng6_username, data_folder, output_folder, prefixed_ids)
for i, source_dir in enumerate(src_directories):
dest_dir = dest_directories[i]
if not os.path.exists(dest_dir):
os.makedirs(dest_dir, 0755)
for filename in os.listdir(source_dir):
source_file_path = os.path.join(source_dir, filename)
dest_file_path = os.path.join(dest_dir, filename)
if os.path.isfile(source_file_path) and filename not in [ "analyse.xml", "index.html", "run.cfg"] :
shutil.copyfile(source_file_path, dest_file_path)
# create archive
with tarfile.open(archive_file, "w:gz") as tar :
tar.add(output_folder, os.path.basename(output_folder))
# send mail
try :
message = 'Dear user, your archive is ready for download at the following link : %s'%web_archive
subject = '[NG6] Archive ready to be downloaded'
subprocess.check_call( "echo '%s' | mail -s '%s' %s"%(message, subject, email) , shell = True)
except :
raise Exception('Unable to send mail')
# delete output folder
robust_rmtree(output_folder)
class CreateArchive (Component):
def define_parameters(self, login, data_directory, email, data_ids = [], run_ids = [], analysis_ids = []):
self.add_parameter('login', 'The login of the user in ng6', type = 'ng6userlogin', default = login)
self.add_parameter('data_directory', 'Where are stored all the datas', default = data_directory, required = True)
self.add_parameter('email', 'The email where the url to the archive will be sent', default = email, required = True)
self.add_parameter_list('data_ids', 'Ids of a run from which rawdata will be retrieved', default = data_ids)
self.add_parameter_list('run_ids', 'Ids of run from which all data will be retrieved', default = run_ids)
self.add_parameter_list('analysis_ids', 'Ids of analysis to retrieve', default = analysis_ids)
self.add_output_file("archive", "created archive name", filename = "ng6_data_archive.tar.gz")
def process(self):
run_ids = [ "run_%s"%i for i in self.run_ids ]
data_ids = [ "data_%s"%i for i in self.data_ids ]
analysis_ids = [ "analyse_%s"%i for i in self.analysis_ids ]
ids_dump_path = self.get_temporary_file(".dump")
fh = open(ids_dump_path, "wb")
pickle.dump( run_ids + data_ids + analysis_ids, fh)
output_folder = os.path.join(self.output_directory, 'ng6_data')
fh.close()
web_archive = self._webify_outputs(self.archive)
archive = PythonFunction(create_archive, cmd_format = "{EXE} {ARG} {IN} {OUT} ")
archive(arguments = [ self.login, self.email, self.data_directory, output_folder, ids_dump_path, web_archive], outputs = self.archive)
def _webify_outputs(self, path):
work_dir = self.config_reader.get_work_directory()
if work_dir.endswith("/"): work_dir = work_dir[:-1]
socket_opt = self.config_reader.get_socket_options()
return 'http://' + socket_opt[0] + ':' + str(socket_opt[1]) + '/' + path.replace(work_dir, 'data'),
\ No newline at end of file
#
# Copyright (C) 2012 INRA
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
import os
import pickle
from jflow.component import Component
from ng6.utils import Utils
from weaver.function import PythonFunction
def create_symbolik_link (username, password, ng6_username, data_folder, output_folder, email, ids_dump_path, output_list, user_script):
import os
import pickle
import subprocess
from ng6.config_reader import NG6ConfigReader
from ng6.utils import Utils
from workflows.download.lib.utils import get_directories_structure_and_content, SSH
password = Utils.decode_passwd(password)
try:
fh = open(ids_dump_path, "rb")
prefixed_ids = pickle.load(fh)
fh.close()
prefixed_ids = ";".join(prefixed_ids)
src_directories, dest_directories = get_directories_structure_and_content(ng6_username, data_folder, output_folder, prefixed_ids)
destinations = []
with open(user_script, "w") as us:
with open(output_list, "w") as ol :
for i, source_dir in enumerate(src_directories):
dest_dir = dest_directories[i]
if not os.path.isdir(dest_dir) :
us.write("mkdir -p -m 755 %s ; \n"%dest_dir)
for filename in os.listdir(source_dir):
source_file_path = os.path.join(source_dir, filename)
dest_file_path = os.path.join(dest_dir, filename)
destinations.append(dest_file_path)
if os.path.isfile(source_file_path) and filename not in [ "analyse.xml", "index.html", "run.cfg"] :
if not os.path.exists(dest_file_path):
us.write("ln -s %s %s ;\n"%(source_file_path,dest_file_path))
ol.write(dest_file_path+"\n")
# execution of the script as the user
config = NG6ConfigReader()
server_params = config.get_server_parameters()
s = SSH(username, password, server_params[0])
s.cmd('bash %s'%user_script)
if email != 'none':
try :
message = 'Dear user, your links have been created on the server. Bellow are the path of all output files :\n\n'
message = message + '\n'.join(destinations)
subject = '[NG6] Your links have been created'
subprocess.check_call( "echo '%s' | mail -s '%s' %s"%(message, subject, email) , shell = True)
except : pass
except Exception as e:
raise Exception('Unable to create link : %s'%e)
class CreateSymlink (Component):
def define_parameters(self, login, username, password, data_directory, user_directory, email = None, data_ids = [], run_ids = [], analysis_ids = []):
self.add_parameter('data_directory', 'data_directory', default = data_directory, required = True)
self.add_parameter('login', 'The login of the user in ng6', type = 'ng6userlogin', default = login)
self.add_parameter('username', 'Valid username on the server', required = True, default = username)
self.add_parameter('password', 'A password for the usernmae on the server', required = True, default = Utils.encode_passwd(password))
self.add_parameter('user_directory', 'Destination', required = True, default = user_directory)
self.add_parameter('email', 'user mail', default = ( 'none', email)[ email is not None])
self.add_parameter_list('data_ids', 'Ids or a run from which rawdata will be retrieved', default = data_ids)
self.add_parameter_list('run_ids', 'Ids of run from which all data will be retrieved', default = run_ids)
self.add_parameter_list('analysis_ids', 'Ids of analysis to retrieve', default = analysis_ids)
self.add_output_file( 'user_script', 'The bash script that will be executed by the user', filename = "userscript.sh")
self.add_output_file( 'output_list', 'output_list', filename = "output_list.txt")
def process(self):
run_ids = [ "run_%s"%i for i in self.run_ids ]
data_ids = [ "data_%s"%i for i in self.data_ids ]
analysis_ids = [ "analyse_%s"%i for i in self.analysis_ids ]
ids_dump_path = self.get_temporary_file(".dump")
fh = open(ids_dump_path, "wb")
pickle.dump( run_ids + data_ids + analysis_ids, fh)
fh.close()
createlinks = PythonFunction(create_symbolik_link, cmd_format = "{EXE} {ARG} {IN} {OUT} ")
createlinks(arguments = [ self.username, "'"+self.password+"'", self.login, self.data_directory, self.user_directory, "'"+self.email+"'" ],
inputs = ids_dump_path , outputs = [self.output_list, self.user_script])
\ No newline at end of file
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment