Skip to content

ng6
ng6
Commit a58aa0cb authored by Penom Nom's avatar Penom Nom
Browse files
Options

add dowload workflow

parent 2474ad4c
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 795 additions and 378 deletions
src/ng6/t3MySQLdb.py
View file @ a58aa0cb
This diff is collapsed.
ui/nG6/class.tx_nG6_eid.php
View file @ a58aa0cb
......@@ -82,32 +82,6 @@ class tx_nG6_eid {
// Return the html_tree
print $html_tree;
// If downloading data is requested
} else if ($type == 'download') {
$format = trim(t3lib_div::_GP('format'));
// If link in home directory
if ($format == 'home_link') {
$ids = trim(t3lib_div::_GP('ids'));
$data_folder = trim(t3lib_div::_GP('data_folder'));
$user_login = trim(t3lib_div::_GP('user_login'));
$user_pwd = trim(t3lib_div::_GP('user_pwd'));
$user_directory = trim(t3lib_div::_GP('user_directory'));
$retcode = $this->create_symbolic_links($user_id, $user_login, $user_pwd, $ids, $data_folder, $user_directory);
print $retcode;
// If a .tar.gz file
} else {
$ids = trim(t3lib_div::_GP('ids'));
$data_folder = trim(t3lib_div::_GP('data_folder'));
$tmp_folder = trim(t3lib_div::_GP('tmp_folder'));
$user_email = trim(t3lib_div::_GP('user_email'));
$title_email = trim(t3lib_div::_GP('title_email'));
$msg_email = trim(t3lib_div::_GP('msg_email'));
$archive_info = $this->create_archive_script($user_id, $ids, $data_folder, $tmp_folder, "tar.gz", $user_email, $title_email, $msg_email);
$process = new Process("tcsh ".$archive_info[0]);
print "pid=".$process->getPid()."&archive_path=".$archive_info[1];
}
// If asked to check a job status
} else if($type == 'check_status') {
$pid = trim(t3lib_div::_GP('pid'));
......@@ -837,162 +811,6 @@ class tx_nG6_eid {
}
}
/**
* Creates an archiving script to archive data from ids
*
* @param string $user_id the user id
* @param array $ids list of ids to download
* @param string $data_folder the data folder where are stored the data
* @param string $tmp_folder the output directory
* @param string $format the archiving format (tar.gz)
* @param string $user_email the user email
* @param string $title_email the email title to send
* @param string $msg_email the email message to send
* @return array (the path to the script, the url path to the final archive)
*/
private function create_archive_script($user_id, $ids, $data_folder, $tmp_folder, $format, $user_email, $title_email, $msg_email) {
// First get the archive name where will be stored the data
$random_key = tx_nG6_utils::create_random_key(10);
$archive_path = $tmp_folder."/ng6_".$random_key;
while( file_exists($archive_path) )
$archive_path = $tmp_folder."/ng6_".$random_key;
// Then get the directories structure
$directories = $this->get_directories_structure_and_content($user_id, $ids, $data_folder, $archive_path);
$archive_script_path = $tmp_folder."/ng6_".$random_key.".sh";
// And write down the script in charge to create the archive
$fp = fopen($archive_script_path, 'w');
foreach($directories[0] as $index => $src_directory) {
if (!file_exists( $directories[1][$index])) {
fwrite($fp, "mkdir -p ".$directories[1][$index]."\n");
}
// For each files in the source directory
foreach(scandir($src_directory) as $file) {
if (is_file($src_directory."/".$file) and $file != "analyse.xml" and $file != "index.html" and $file != "run.cfg") {
fwrite($fp, "cp ".$src_directory."/".$file." ".$directories[1][$index]."\n");
}
}
}
fwrite($fp, "cd ".$tmp_folder."\n");
fwrite($fp, "tar -czf ng6_".$random_key.".tar.gz ng6_".$random_key."\n");
fwrite($fp, "rm -rf ".$archive_path."\n");
fwrite($fp, 'echo "'.str_replace(array('SPAN_ARCHIVE_NAME'), array("ng6_".$random_key.".tar.gz"), $msg_email).'" | mail -s "'.$title_email.'" '.$user_email."\n");
fclose($fp);
return array($archive_script_path, "ng6_".$random_key.".tar.gz");
}
/**
* Creates symbolic links between ids from data_folder into user_directory
*
* @param string $user_id the user id
* @param string $user_login the user login to make ssh
* @param string $user_pwd the user pwd to make ssh
* @param array $ids list of ids to download
* @param string $data_folder the data folder where are stored the data
* @param string $user_directory the output directory
* @return 0=>everything ok, 1=>user rigth problem, 2=>wrong authentification, 3=>connection error
*/
private function create_symbolic_links($user_id, $user_login, $user_pwd, $ids, $data_folder, $user_directory) {
// First try to connect the specified user using ssh
$connection = ssh2_connect('127.0.0.1', 22);
if (!$connection) return 3;
if (!ssh2_auth_password($connection, $user_login, $user_pwd)) return 2;
// Then get the directories structure
$directories = $this->get_directories_structure_and_content($user_id, $ids, $data_folder, $user_directory);
// And process the directories structure
$creation = true;
foreach($directories[0] as $index => $src_directory) {
if (!file_exists("ssh2.sftp://".$connection.$directories[1][$index])) {
if (!mkdir("ssh2.sftp://".$connection.$directories[1][$index], 0755, true)) {
$creation = false;
break;
}
// For each files in the source directory
foreach(scandir($src_directory) as $file) {
if (is_file($src_directory."/".$file) and $file != "analyse.xml" and $file != "index.html" and $file != "run.cfg") {
if (!ssh2_exec($connection, 'ln -s '.$src_directory.'/'.$file.' '.$directories[1][$index].'/'.$file)) {
$creation = false;
break;
}
}
}
}
}
if (!$creation) return 1;
return 0;
}
/**
* Returns a table with the right directories structure considering ids
*
* @param string $user_id The user id
* @param array $ids List of ids to download
* @param string $data_folder The data folder where are stored the data
* @param string $output_folder The output directory
* @return array [0] => source_directories, [1] => dest_directories
*/
private function get_directories_structure_and_content($user_id, $ids, $data_folder, $output_folder) {
$src_directories = array();
$dest_directories = array();
$vals = preg_split("/;/", $ids);
// Characters to be replaced in the names of the run, the analyses ...
$char_to_replace = array(' ','/');
// For each project/run/analysis
foreach($vals as $val) {
if ($val != "undefined" && $val != "") {
$id = preg_split("/_/", $val);
if ($id[0] == "data") {
$run = tx_nG6_db::select_run($id[1]);
if (!in_array($data_folder.$run["directory"], $src_directories)) {
$src_directories[] = $data_folder.$run["directory"];
$ddest = $output_folder."/Project_".str_replace($char_to_replace, '_', $run["project_name"]);
$ddest .= '.'.$run["project_id"]."/Run_".str_replace($char_to_replace, '_', $run["name"]).".".$id[1]."/RawData";
$dest_directories[] = $ddest;
}
} else if ($id[0] == "run") {
$run = tx_nG6_db::select_run($id[1]);
if (!in_array($data_folder.$run["directory"], $src_directories)) {
$src_directories[] = $data_folder.$run["directory"];
$ddest = $output_folder."/Project_".str_replace($char_to_replace, '_', $run["project_name"]);
$ddest .= '.'.$run["project_id"]."/Run_".str_replace($char_to_replace, '_', $run["name"]).".".$id[1]."/RawData";
$dest_directories[] = $ddest;
}
foreach(tx_nG6_db::get_user_run_analysis($user_id, $id[1], 'tx_nG6_analyze.name') as $analyse_id => $analyse_values) {
$aid = preg_split("/_/", $analyse_id);
if (!in_array($data_folder.$analyse_values["directory"], $src_directories)) {
$src_directories[] = $data_folder.$analyse_values["directory"];
$ddest = $output_folder."/Project_".str_replace($char_to_replace, '_', $run["project_name"]);
$ddest .= '.'.$run["project_id"]."/Run_".str_replace($char_to_replace, '_', $run["name"]);
$ddest .= ".".$id[1]."/Analyse_".str_replace($char_to_replace, '_', $analyse_values["name"]).".".$aid[1];
$dest_directories[] = $ddest;
}
}
} else if ($id[0] == "analyse") {
$analyse = tx_nG6_db::select_analyse($id[1]);
if (!in_array($data_folder.$analyse["directory"], $src_directories)) {
$src_directories[] = $data_folder.$analyse["directory"];
// If it's a project analyse
if ($analyse["run_id"] == 'None') {
$ddest = $output_folder."/Project_".str_replace($char_to_replace, '_', $analyse["project_name"]);
$ddest .= ".".$analyse["project_id"]."/Project_analyses/".str_replace($char_to_replace, '_', $analyse["name"]).".".$id[1];
$dest_directories[] = $ddest;
} else {
$ddest = $output_folder."/Project_".str_replace($char_to_replace, '_', $analyse["project_name"]);
$ddest .= ".".$analyse["project_id"]."/Run_".str_replace($char_to_replace, '_', $analyse["run_name"]);
$ddest .= ".".$analyse["run_id"]."/Analyse_".str_replace($char_to_replace, '_', $analyse["name"]).".".$id[1];
$dest_directories[] = $ddest;
}
}
}
}
}
return array($src_directories, $dest_directories);
}
}
if (defined('TYPO3_MODE') && $TYPO3_CONF_VARS[TYPO3_MODE]['XCLASS']['ext/nG6/class.tx_nG6_eid.php']) {
......
ui/nG6/pi1/modals.tpl
View file @ a58aa0cb
......@@ -40,18 +40,6 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
</div>
<div id="setAndRunModalBody" class="modal-body"></div>
<div id="setAndRunModalFooter" class="modal-footer">
<!--
<div class="btn-group">
<button id="close" class="btn btn-default" data-dismiss="modal"><i class="glyphicon glyphicon-remove"></i> Close</button>
</div>
<div class="btn-group">
<button id="refresh_workflow_status" class="btn btn-default"><i class="glyphicon glyphicon-refresh"></i> Refresh</button>
</div>
<div class="btn-group">
<button id="reset_workflow" class="btn btn-default"><i class="glyphicon glyphicon-refresh"></i> Reset</button>
<button id="run_workflow" class="btn btn-primary"><i class="glyphicon glyphicon-cog"></i> Run</button>
</div>
-->
</div>
</div>
</div>
......
ui/nG6/pi5/class.tx_nG6_pi5.php
View file @ a58aa0cb
......@@ -82,9 +82,12 @@ class tx_nG6_pi5 extends tslib_pibase {
$smarty->assign('llang', $this->LOCAL_LANG['default']);
}
// get server url from pi1
$smarty->assign('server_url', $GLOBALS['TSFE']->tmpl->setup['plugin.']['tx_nG6_pi1.']['server_url']);
$smarty->assign('server_name', $this->conf["server_name"]);
$smarty->assign('data_folder', $this->conf["data"]);
$smarty->assign('temp_folder', $this->conf["temp"]);
$smarty->assign('user_login', $GLOBALS['TSFE']->fe_user->user['username']);
if ($this->conf["directory_prefix"]) {
$smarty->assign('directory_prefix', $this->conf["directory_prefix"]);
} else {
......
ui/nG6/pi5/download_view.tpl
View file @ a58aa0cb
......@@ -22,6 +22,8 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
<input type="hidden" id="user_id" value="{$user_id}" />
<input type="hidden" id="tmp_url" value="{$tmp_url}" />
<input type="hidden" id="server_url" value="{$server_url}" />
<input type="hidden" id="user_login" value="{$user_login}" />
<div class="sub-content sc-top">
<div class="ng6-content-header-left download">
......@@ -38,25 +40,6 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
Please select below the data you want to download...
</p>
</div>
<h5>Download format :</h5>
<div style="float:left">
<div class="radio">
<label class="radio" style="font-size:13px">
<input type="radio" name="format" value="tar.gz" >
.tar.gz
</label>
</div>
{* If the server is configured, provide the create link option *}
{if $server_name}
<div class="radio">
<label class="radio" style="font-size:13px">
<input type="radio" name="format" value="home_link" checked>
links on {$server_name}
</label>
</div>
{/if}
</div>
<div style="float:right;margin-top:10px">
<button id="download_btn" class="btn btn-default disabled" type="button">Download</button>
</div>
......@@ -85,10 +68,7 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
<h3 id="myModalLabel"></h3>
</div>
<div id="myModalBody" class="modal-body"></div>
<div class="modal-footer">
<button data-dismiss="modal" class="btn btn-default"><i class="glyphicon glyphicon-remove"></i> Close</button>
<button id="action" class="btn btn-primary"><i class="glyphicon glyphicon-arrow-down"></i> Download</button>
</div>
<div id="myModalFooter" class="modal-footer"></div>
</div>
</div>
</div>
ui/nG6/res/js/tx_nG6_pi5.js
View file @ a58aa0cb
This diff is collapsed.
workflows/download/__init__.py 0 → 100644
View file @ a58aa0cb
#
# Copyright (C) 2012 INRA
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from jflow.workflow import Workflow
from ng6.config_reader import NG6ConfigReader
class Download (Workflow):
def get_description(self):
return "Download selected data"
def define_parameters(self, function="process"):
self.add_parameter('login', 'The login of the user in ng6', required = True, type = 'ng6userlogin')
self.add_parameter_list('data_id', 'Ids of a run from which rawdata will be retrieved', type = 'existingrun')
self.add_parameter_list('run_id', 'Ids of run from which all data will be retrieved', type = 'existingrun')
self.add_parameter_list('analysis_id', 'Ids of analysis to retrieve', type = 'existinganalysis')
self.add_multiple_parameter("home_link", "Create a symbolik link in a home directory")
self.add_parameter('username', 'Valid username on the server', type = "localuser" ,required = True, add_to ="home_link")
self.add_parameter('password', 'A password for the usernmae on the server', type="password", required = True, add_to ="home_link")
self.add_parameter('user_directory', 'Destination', default = "/save/scratch/", required = True, add_to ="home_link")
self.add_multiple_parameter("local_archive", "Download an archive on your computer")
self.add_parameter('user_email', 'Provide an email to be warned when the process is over', type="email", required = True, add_to ="local_archive")
self.add_exclusion_rule("home_link", "local_archive")
def process(self):
config = NG6ConfigReader()
save_directory = config.get_save_directory()
if self.home_link['username'] and self.home_link['password'] :
self.add_component('CreateSymlink', [self.login, self.home_link['username'], self.home_link['password'],
save_directory, self.home_link['user_directory'], self.data_id, self.run_id, self.analysis_id])
elif self.local_archive['user_email'] :
self.add_component('CreateArchive', [self.login, save_directory, self.local_archive['user_email'], self.data_id, self.run_id, self.analysis_id])
workflows/download/components/__init__.py 0 → 100644
View file @ a58aa0cb
#
# Copyright (C) 2012 INRA
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
\ No newline at end of file
workflows/download/components/createarchive.py 0 → 100644
View file @ a58aa0cb
#
# Copyright (C) 2012 INRA
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
import os
import pickle
from jflow.component import Component
from weaver.function import PythonFunction
def create_archive( ng6_username, email, data_folder, output_folder, ids_dump_path, web_archive, archive_file ):
import os
import pickle
import shutil
import subprocess
import tarfile
import smtplib
from email.mime.text import MIMEText
from ng6.t3MySQLdb import t3MySQLdb
from workflows.download.lib.utils import get_directories_structure_and_content
t3mysql = t3MySQLdb()
user_id = t3mysql.get_user_id(ng6_username)
prefixed_ids = ";".join(pickle.load(open(ids_dump_path, "rb")))
src_directories, dest_directories = get_directories_structure_and_content(user_id, data_folder, output_folder, prefixed_ids)
for i, source_dir in enumerate(src_directories):
dest_dir = dest_directories[i]
if not os.path.exists(dest_dir):
os.makedirs(dest_dir, 0755)
for filename in os.listdir(source_dir):
source_file_path = os.path.join(source_dir, filename)
dest_file_path = os.path.join(dest_dir, filename)
if os.path.isfile(source_file_path) and filename not in [ "analyse.xml", "index.html", "run.cfg"] :
shutil.copyfile(source_file_path, dest_file_path)
# create archive
with tarfile.open(archive_file, "w:gz") as tar :
tar.add(output_folder, os.path.basename(output_folder))
# send mail
try :
msg = MIMEText('Dear user, your archive is ready for download at the following link : %s'%web_archive)
msg['Subject'] = '[NG6] Archive ready to be downloaded'
msg['From'] = 'noreply@ng6-support.fr'
msg['To'] = email
s = smtplib.SMTP('localhost')
s.sendmail( 'noreply@ng6-support.fr' , [email], msg.as_string())
s.quit()
except :
raise Exception('Unable to send mail')
class CreateArchive (Component):
def define_parameters(self, login, data_directory, email, data_ids = [], run_ids = [], analysis_ids = []):
self.add_parameter('login', 'The login of the user in ng6', required = True, type = 'ng6userlogin', default = login)
self.add_parameter('data_directory', 'Where are stored all the datas', default = data_directory, required = True)
self.add_parameter('email', 'The email where the url to the archive will be sent', default = email, required = True)
self.add_parameter_list('data_ids', 'Ids of a run from which rawdata will be retrieved', default = data_ids)
self.add_parameter_list('run_ids', 'Ids of run from which all data will be retrieved', default = run_ids)
self.add_parameter_list('analysis_ids', 'Ids of analysis to retrieve', default = analysis_ids)
self.add_output_file("archive", "created archive name", filename = "ng6_data_archive.tar.gz")
def process(self):
run_ids = [ "run_%s"%i for i in self.run_ids ]
data_ids = [ "data_%s"%i for i in self.data_ids ]
analysis_ids = [ "analyse_%s"%i for i in self.analysis_ids ]
ids_dump_path = self.get_temporary_file(".dump")
pickle.dump( run_ids + data_ids + analysis_ids, open(ids_dump_path, "wb"))
output_folder = os.path.join(self.output_directory, 'ng6_data')
web_archive = self._webify_outputs(self.archive)
archive = PythonFunction(create_archive, cmd_format = "{EXE} {ARG} {IN} {OUT} ")
archive(arguments = [ self.login, self.email, self.data_directory, output_folder, ids_dump_path, web_archive], outputs = self.archive)
def _webify_outputs(self, path):
work_dir = self.config_reader.get_work_directory()
if work_dir.endswith("/"): work_dir = work_dir[:-1]
socket_opt = self.config_reader.get_socket_options()
return 'http://' + socket_opt[0] + ':' + str(socket_opt[1]) + '/' + path.replace(work_dir, 'data'),
\ No newline at end of file
workflows/download/components/createsymlink.py 0 → 100644
View file @ a58aa0cb
#
# Copyright (C) 2012 INRA
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
import os
import pickle
from jflow.component import Component
from weaver.function import PythonFunction
def create_symbolik_link (unixname, password, ng6_username, data_folder, output_folder, ids_dump_path, output_list):
import os
import pwd
import subprocess
import pickle
from ng6.t3MySQLdb import t3MySQLdb
from workflows.download.lib.utils import get_directories_structure_and_content, test_local_user
test_local_user(unixname, password)
user_data = pwd.getpwnam(unixname)
def preexec_fn():
os.setuid(user_data.pw_uid)
os.setgid(user_data.pw_gid)
t3mysql = t3MySQLdb()
user_id = t3mysql.get_user_id(ng6_username)
prefixed_ids = ";".join(pickle.load(open(ids_dump_path, "rb")))
src_directories, dest_directories = get_directories_structure_and_content(user_id, data_folder, output_folder, prefixed_ids)
with open(output_list, "w") as ol :
for i, source_dir in enumerate(src_directories):
dest_dir = dest_directories[i]
if not os.path.isdir(dest_dir) :
os.makedirs(dest_dir, 0755)
for filename in os.listdir(source_dir):
source_file_path = os.path.join(source_dir, filename)
dest_file_path = os.path.join(dest_dir, filename)
if os.path.isfile(source_file_path) and filename not in [ "analyse.xml", "index.html", "run.cfg"] :
if not os.path.exists(dest_file_path):
os.symlink(source_file_path, dest_file_path)
ol.write(dest_file_path+"\n")
class CreateSymlink (Component):
def define_parameters(self, login, username, password, data_directory, user_directory, data_ids = [], run_ids = [], analysis_ids = []):
self.add_parameter('data_directory', 'data_directory', default = data_directory, required = True)
self.add_parameter('login', 'The login of the user in ng6', required = True, type = 'ng6userlogin', default = login)
self.add_parameter('username', 'Valid username on the server', required = True, default = username)
self.add_parameter('password', 'A password for the usernmae on the server', required = True, default = password)
self.add_parameter('user_directory', 'Destination', required = True, default = user_directory)
self.add_parameter_list('data_ids', 'Ids or a run from which rawdata will be retrieved', default = data_ids)
self.add_parameter_list('run_ids', 'Ids of run from which all data will be retrieved', default = run_ids)
self.add_parameter_list('analysis_ids', 'Ids of analysis to retrieve', default = analysis_ids)
self.add_output_file( 'output_list', 'output_list', filename = "output_list.txt")
def process(self):
run_ids = [ "run_%s"%i for i in self.run_ids ]
data_ids = [ "data_%s"%i for i in self.data_ids ]
analysis_ids = [ "analyse_%s"%i for i in self.analysis_ids ]
ids_dump_path = self.get_temporary_file(".dump")
pickle.dump( run_ids + data_ids + analysis_ids, open(ids_dump_path, "wb"))
createlinks = PythonFunction(create_symbolik_link, cmd_format = "{EXE} {ARG} {IN} {OUT} ")
createlinks(arguments = [ self.username, "'"+self.password+"'", self.login, self.data_directory, self.user_directory], inputs = ids_dump_path , outputs = self.output_list)
\ No newline at end of file
workflows/download/lib/__init__.py 0 → 100644
View file @ a58aa0cb
#
# Copyright (C) 2012 INRA
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
\ No newline at end of file
workflows/download/lib/utils.py 0 → 100644
View file @ a58aa0cb
#
# Copyright (C) 2015 INRA
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
import os
import re
import pexpect
from ng6.t3MySQLdb import t3MySQLdb
def test_local_user(username, password):
try:
# no display display prompt from ssh-add
os.unsetenv('DISPLAY')
os.unsetenv('SSH_ASKPASS')
connection = pexpect.spawn ('ssh %s@127.0.0.1'%username)
connection.expect ('(?i).*password:.*')
connection.sendline (password)
connection.sendline ('whoami')
i = connection.expect(["(?i).*Permission\s+denied.*", username])
# expect whoami to give the username back
if i != 1 :
raise Exception('Connection denied for %s'%username)
connection.close()
except Exception as e:
raise e
def get_directories_structure_and_content(user_id, data_folder, output_folder, prefixed_ids):
src_directories = []
dest_directories = []
t3mysql = t3MySQLdb()
for prefixed_id in prefixed_ids.split(';') :
vals = prefixed_id.split('_')
prefix = vals[0]
id = vals[1]
if prefix == "data" or prefix == "run" :
run = t3mysql.select_run_informations(id)
source_dir = data_folder +'/'+ run['directory']
project_name = "Project_%s.%s" % ( re.sub( r"[\s\/]", "_", run['project_name']), run['project_id'] )
run_name = "Run_%s.%s" %( run['name'].replace(' ', '_').replace('/', '_'), id )
if source_dir not in src_directories :
src_directories.append(source_dir)
dest_directories.append(os.path.join( output_folder, project_name, run_name, "RawData" ))
if prefix == "run" :
for analyse_id, analyse_values in t3mysql.get_user_run_analysis(user_id, id, 'tx_nG6_analyze.name').items() :
analysis_dir = data_folder + '/' + analyse_values["directory"]
analysis_name = "Analyse_%s.%s"%( re.sub( r"[\s\/]", "_", analyse_values['name']), analyse_id )
if analysis_dir not in src_directories :
src_directories.append(analysis_dir)
dest_directories.append(os.path.join( output_folder, project_name, run_name, analysis_name ))
elif prefix == "analyse" :
analyse = t3mysql.select_analysis_informations(id)
source_dir = data_folder +'/'+ analyse["directory"]
project_name = "Project_%s.%s" % ( re.sub( r"[\s\/]", "_", analyse["project_name"]), analyse["project_id"])
# it's a run analysis
if analyse["run_id"]:
run_name = "Run_%s.%s" % ( re.sub( r"[\s\/]", "_", analyse["run_name"]), analyse["run_id"])
analysis_name = "Analyse_%s.%s" % ( re.sub( r"[\s\/]", "_", analyse["name"]), id)
dest_directories.append(os.path.join( output_folder, project_name, run_name, analysis_name ))
# it's a project analysis
else :
analysis_name = "Project_analyses/%s.%s" % ( re.sub( r"[\s\/]", "_", analyse["name"]), id)
dest_directories.append(os.path.join( output_folder, project_name, analysis_name ))
if len(src_directories) != len(dest_directories) :
raise Exception("Error while retrieveing directory structure and content")
sources, destinations = [], []
for i, e in enumerate(src_directories):
if os.path.isdir(e) :
sources.append(e)
destinations.append(dest_directories[i])
return sources, destinations
workflows/types.py
View file @ a58aa0cb
......@@ -79,8 +79,30 @@ def existingproject(name):
return name
raise Exception
except :
raise argparse.ArgumentTypeError("project name '%s' does not exists" % name)
raise argparse.ArgumentTypeError("The project name '%s' does not exists" % name)
def existingrun(id):
try:
t3mysql = t3MySQLdb()
t3mysql.select_run(id)
return id
except :
raise argparse.ArgumentTypeError("The run associated with id '%s' does not exists" % id)
def existinganalysis(id):
try:
t3mysql = t3MySQLdb()
t3mysql.select_analysis(id)
return id
except :
raise argparse.ArgumentTypeError("The analysis associated with id '%s' does not exists" % id)