Skip to content

GitLab

Commit 1c51484c authored by Celine Noirot's avatar Celine Noirot
Browse files

create purgedemand workflow to encode password in ajax request

parent 352dbed2
Hide whitespace changes
Inline Side-by-side
src/ng6/t3MySQLdb.py
View file @ 1c51484c
......@@ -891,4 +891,133 @@ class t3MySQLdb(object):
'version' : row['analyze_version'],
'is_editable' : row['analyze_is_editable']
}
\ No newline at end of file
def select_purge_demand_directories(self, demand_id):
"""
Select all directories to purge
@param purge_demand_id:
"""
dirs=[]
if ',' in demand_id :
parts = [
'SELECT',
'tx_nG6_purge_demand.uid AS demand_id,',
'tx_nG6_purge_demand.analyze_ids AS analyze_ids,',
'tx_nG6_purge_demand.run_ids AS run_ids',
'FROM',
'tx_nG6_purge_demand ',
'WHERE',
'tx_nG6_purge_demand.uid in ('+str(demand_id)+')',
]
else:
parts = [
'SELECT',
'tx_nG6_purge_demand.uid AS demand_id,',
'tx_nG6_purge_demand.analyze_ids AS analyze_ids,',
'tx_nG6_purge_demand.run_ids AS run_ids',
'FROM',
'tx_nG6_purge_demand ',
'WHERE',
'tx_nG6_purge_demand.uid = '+str(demand_id),
]
sql = " ".join(parts)
print ("select_purge_demand_directories",sql)
qresult = self.execute(sql, dictc = True)
# analysis directories
if qresult.rowcount >= 1:
analysis_ids=[]
run_ids=[]
for res in qresult.rows :
analysis_ids.append(res['analyze_ids'] )
run_ids.append(res['run_ids'] )
parts = [
'SELECT',
'tx_nG6_analyze.directory AS analyze_directory',
'FROM',
'tx_nG6_analyze',
'WHERE',
'tx_nG6_analyze.uid IN (' + ','.join(analysis_ids) +')'
]
sql = " ".join(parts)
print ("select_purge_demand_directories",sql)
qresult = self.execute(sql, dictc = True)
for result in qresult.rows:
dirs.append(self.cfg_reader.get_save_directory() + result['analyze_directory'])
# run directories
parts = [
'SELECT',
'tx_nG6_run.directory AS run_directory',
'FROM',
'tx_nG6_run',
'WHERE',
'tx_nG6_run.uid IN (' + ','.join(run_ids) +')'
]
print ("select_purge_demand_directories",sql)
sql = " ".join(parts)
qresult = self.execute(sql, dictc = True)
for result in qresult.rows:
dirs.append(self.cfg_reader.get_save_directory() + result['run_directory'])
return dirs
def update_fields(self, table, uid, field, value, no_quote=[]):
if 'uid' in field :
return "Unable to execute 'update_fields' on field uid"
parts = [
'UPDATE',
table,
'SET']
update_part=[]
for f,v in zip(field,value):
if f not in no_quote :
update_part.append (f +' = "' +str(v)+'"')
else:
update_part.append (f +' = ' +str(v))
uids=uid
if isinstance(uid, list) :
uids=','.join(uid)
parts.extend([','.join(update_part),
'WHERE',
table+'.uid IN (' + uids +')' ] )
sql = " ".join(parts)
print ('### ',sql)
qresult = self.execute(sql, dictc = True)
return(qresult)
def set_purge_demand_deleted(self, demand_ids):
date = str(time.mktime(datetime.date.today().timetuple())).split(".")[0]
parts = [
'SELECT',
'tx_nG6_purge_demand.uid AS demand_id,',
'tx_nG6_purge_demand.analyze_ids AS analyze_ids,',
'tx_nG6_purge_demand.run_ids AS run_ids',
'FROM',
'tx_nG6_purge_demand ',
'WHERE',
'tx_nG6_purge_demand.uid IN ('+str(demand_ids) +')' ]
sql = " ".join(parts)
print ("set_purge_demand_deleted",sql)
qresult = self.execute(sql, dictc = True)
if qresult.rowcount >= 1:
analysis_ids=[]
run_ids=[]
for res in qresult.rows :
analysis_ids.append(res['analyze_ids'] )
run_ids.append(res['run_ids'] )
self.update_fields('tx_nG6_run', run_ids, ["purged_size","data_state", "purged_date", "storage_size"],
["storage_size","purged",date, 0],["purged_size"])
self.update_fields('tx_nG6_analyze',analysis_ids, ["purged_size","data_state", "purged_date", "storage_size"],
["storage_size","purged",date, 0],
["purged_size"])
self.update_fields('tx_nG6_purge_demand',demand_ids,
["processed_date","demand_state"],
[date,"deleted"])
src/ng6/utils.py
View file @ 1c51484c
......@@ -424,7 +424,19 @@ class Utils(object):
destinations.append(dest_directories[i])
return sources, destinations
def get_purge_demand_directories(demand_ids):
from ng6.t3MySQLdb import t3MySQLdb
t3mysql = t3MySQLdb()
return t3mysql.select_purge_demand_directories(demand_ids)
def set_purge_demand_deleted(demand_ids):
from ng6.t3MySQLdb import t3MySQLdb
t3mysql = t3MySQLdb()
return t3mysql.set_purge_demand_deleted(demand_ids)
class SSH(object):
def __init__(self, user, passwd, hostname, port=22):
self.username = user
......@@ -476,4 +488,4 @@ class SSH(object):
return self.ssh_results(pid, f)
\ No newline at end of file
ui/nG6/lib/class.tx_nG6_utils.php
View file @ 1c51484c
......@@ -149,13 +149,13 @@ class tx_nG6_utils {
//TODO : BUG
// si unix user plantage de la suppression
// si ng6test erreur du ssh2_auth_password, voir avec MS
error_log("connection: ".$connection."\n");
foreach ($files as $file){
if (substr($file,-4) != ".png") {
if (substr($file,-4) != ".png" AND substr($file,-5) != ".html") {
// And process the directories structure
$res=ssh2_exec($connection, 'rm '.$file);
error_log("purge_files: rm -rf >".$file."< return : ".$res."\n");
if (!$res) {
$res=ssh2_exec($connection, 'rm '.$directory."/".$file);
error_log("purge_files: rm -rf >".$directory."/".$file."< return : ".$res."\n");
if (file_exists($directory."/".$file)) {
return 1;
}
}
......
ui/nG6/pi6/administration_view.tpl
View file @ 1c51484c
......@@ -19,6 +19,7 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
<input type="hidden" id="user_is_login" value="{$login_user}" />
<input type="hidden" id="user_id" value="{$user_id}" />
<input type="hidden" id="user_name" value="{$user_name}" />
<input type="hidden" id="data_folder" value="{$data_folder}" />
<input type="hidden" id="usergroup" value="{$group_info.uid}" />
<input type="hidden" id="server_url" value="{$server_url}" />
......
ui/nG6/pi6/class.tx_nG6_pi6.php
View file @ 1c51484c
......@@ -88,6 +88,7 @@ class tx_nG6_pi6 extends tslib_pibase {
$group_list = tx_nG6_db::get_group_list();
$smarty->assign('login_user', $GLOBALS['TSFE']->loginUser);
$smarty->assign('user_id', $GLOBALS['TSFE']->fe_user->user['uid']);
$smarty->assign('user_name', $GLOBALS['TSFE']->fe_user->user['username']);
$smarty->assign('group_list', $group_list);
$smarty->assign('data_folder', $this->conf["data"]);
$smarty->assign('server_url', $this->conf['server_url']);
......@@ -188,10 +189,10 @@ nG6 team';
$headers[] = 'Reply-To: '.$GLOBALS['TYPO3_CONF_VARS']['EXT']['extConf']['ng6']['email_from'];
$headers[] = 'X-Mailer: PHP/' . phpversion();
#TODO Uncomment
#mail($to, $subject, $mail, implode("\r\n", $headers));
mail($to, $subject, $mail, implode("\r\n", $headers));
//return $headers ;
}
return $mail;
return implode("\r\n", $headers).$mail;
}
function set_purge_demand_processed($demand_ids,$process, $value,$user,$pwd){
......@@ -225,32 +226,8 @@ nG6 team';
$date= time();
$all_runs_purged=array();
$all_analyses_purged=array();
$connexion=tx_nG6_utils::get_ssh_connection($user,$pwd);
if ( $connexion )
{
foreach ($all_runs as $run_id){
$r = tx_nG6_db::select_run($run_id);
if ($r["directory"] != "" ) {
$res = tx_nG6_utils::purge_files($connexion, $data_folder.$r["directory"]);
if ($res == 0){
$all_runs_purged[]=$run_id;
}else{
error_log("Error ".$res." while purging run id >".$run_id."< directory : >".$data_folder.$r["directory"]."< \n");
}
}
}
foreach ($all_analyses as $analyse_id){
$a = tx_nG6_db::select_analyse($analyse_id);
if ($a["directory"] != "" ) {
$res = tx_nG6_utils::purge_files($connexion, $data_folder.$a["directory"]);
if ($res == 0){
$all_analyses_purged[]=$analyse_id;
}else{
error_log("Error ".$res." while purging analyse id >".$analyse_id."< directory : >".$data_folder.$a["directory"]."< \n");
}
}
}
//purged_size = storage_size !!!
tx_nG6_db::update_field('tx_nG6_run', $all_runs_purged, array("purged_size","data_state", "purged_date", "storage_size"),
array("storage_size","purged",$date, 0),
......
ui/nG6/res/js/tx_nG6_pi6.js
View file @ 1c51484c
......@@ -397,6 +397,7 @@ $(function () {
// Delete data after selecting list of ids demand
$("#delete_data_from_list").click(function(){
console.log("delete_data_from_list");
demands="";
$(':checked[id^=chk_demand').each(function(){
demand_id=$(this).val();
......@@ -434,8 +435,212 @@ $(function () {
function delete_data(list_ids,purge_demand_datatable){
var data_folder = $('#data_folder').val();
console.log("delete_data");
console.log(list_ids);
var _get_workflow_status = function(workflow_id, callback, error_callback){
$.ajax({
url : $("#server_url").val() + '/get_workflow_status?display=list&workflow_id=' + workflow_id ,
dataType : 'jsonp',
timeout: 20000 ,
success : function(data){
if (callback){
callback(data);
}
},
error : function(jqXHR, textStatus, errorThrown){
if (error_callback){
error_callback(jqXHR, textStatus, errorThrown);
}
}
});
};
var _retrieve_url_list = function(modal, running_wf, ufilename , process_data){
modal.$body.html([
'<div>',
' <div class="alert alert-info">',
' <p>',
' <strong>Data</strong> are being deleted. Use the refresh button to get the list of files.',
' </p>',
' </div>',
' <div id="wfstatus">',
' </div>',
'</div>'
].join(''));
modal.$footer.html([
' <div class="btn-group">',
' <button id="refresh_workflow" class="btn btn-default"><i class="glyphicon glyphicon-refresh"></i> Get data</button>',
' </div>',
' <div class="btn-group">',
' <button id="close" class="btn btn-default" data-dismiss="modal"><i class="glyphicon glyphicon-remove"></i> Close</button>',
' </div>',
].join(''));
$('#wfstatus').wfstatus({
workflowID : running_wf.id,
display : 'list',
serverURL: $("#server_url").val(),
})
$("#refresh_workflow").click(function(){
modal.$body.html("<div class='tx-nG6-wait'> <strong>Please</strong> wait...</div>");
button = this;
button.disabled = true;
_get_workflow_status(running_wf.id ,
// successcb
function(statusdata){
if (statusdata.status == "completed"){
_retrieve_data(running_wf.id, function(data){
$(button).hide();
$.each(data, function(i, component) {
$.each(component, function(filename, file) {
if (filename == ufilename){
_display_file_content(file.url, modal.$body, process_data);
}
});
});
},
//errorcb
function(jqXHR, textStatus, errorThrown){
modal.$body.html([
'<div class="alert alert-danger">',
' <p><strong>Failed !</strong> to get outputs from ' + $("#server_url").val() + ' </p>',
'</div>',
].join(''));
}
);
}
else {
$('#wfstatus').wfstatus('reload');
button.disabled = false;
}
},
// errorcb
function(){
$('#wfstatus').wfstatus('reload');
button.disabled = false;
}
);
});
};
var _retrieve_data = function(workflow_id, callback, error_callback){
$.ajax({
url : $("#server_url").val() + '/get_workflow_outputs?workflow_id=' + workflow_id,
dataType : 'jsonp',
success : function(data){
if (callback) {
callback(data);
}
},
error : function(jqXHR, textStatus, errorThrown){
if (error_callback){
error_callback(jqXHR, textStatus, errorThrown);
}
}
});
};
var _display_file_content = function(filepath, $elmt, process_data){
$.get(filepath, function(data){
if (process_data) {
data = process_data(data);
}
$elmt.html([
'<div class="alert">',
' <p>',
' <strong>Success !</strong> You will find bellow the list of deleted file.',
' Use the <i class="glyphicon glyphicon-floppy-save"></i> button to copy all paths to the clipboard.<br/>',
' </p>',
'</div>',
'<div>',
' <button id="copy_urls" class="btn btn-default btn-sm pull-right" data-clipboard-target="url_filelist">',
' <i class="glyphicon glyphicon-floppy-save"></i>&nbsp;',
' </button><br/>',
' <textarea readonly id="url_filelist" rows="25" style="resize:none;white-space: pre; word-wrap: normal; overflow-x: scroll; width: 100%">',
data.trim(),
' </textarea>',
'</div>'
].join(''));
var client = new ZeroClipboard($("#copy_urls"));
});
};
$('#setAndRunModalLabel').html("Loading");
$('#setAndRunModalFooter').html([
' <div class="btn-group">',
' <button id="close" class="btn btn-default" data-dismiss="modal"><i class="glyphicon glyphicon-remove"></i> Close</button>',
' </div>',
' <div class="btn-group">',
' <button id="reset_workflow" class="btn btn-default"><i class="glyphicon glyphicon-refresh"></i> Reset</button>',
' <button id="run_workflow" class="btn btn-primary"><i class="glyphicon glyphicon-floppy-save"></i> Start</button>',
' </div>'
].join(''));
$("#reset_workflow, #run_workflow").hide();
$("#close").show();
$('#setAndRunModalBody').html([
'<div >',
' <p id="wf-help" class="text-justify"></p><br/><br/>',
' <div id="wfform"><div class="tx-nG6-wait">Please wait</div></div>',
'</div>',
].join(''));
var modal = {
'$label' : $('#setAndRunModalLabel'),
'$body' : $('#setAndRunModalBody'),
'$footer' : $('#setAndRunModalFooter'),
'$modal' : $('#setAndRunModal')
};
console.log(list_ids);
var wfform_options = {
workflowClass: "PurgeDemand",
serverURL: $("#server_url").val(),
displayRunButton: false,
displayResetButton: false,
parameters : {
"demand" : list_ids,
"admin_login" : $("#user_name").val()
}
};
$('#wfform').on("loaded.wfform", function(event, workflow) {
$("#reset_workflow, #run_workflow").show();
$("#close").hide();
if(workflow){
modal.$label.html(workflow["name"]) ;
$('#wf-help').html(workflow["help"]);
}
else{
modal.$body.html('<div class="alert alert-warning"> <strong>Warning !</strong> The workflow "PurgeDemand" is not available</div>');
}
});
$("#reset_workflow").click(function(){
$('#wfform').wfform('reset');
});
$("#run_workflow").click(function(){
$('#wfform').wfform('run');
});
// remove plugin data
$('#wfform').removeData();
modal.$modal.modal();
modal.$modal.find('#wfform').wfform(wfform_options);
modal.$modal.find('#wfform').on('run.wfform', function(event, running_wf) {
console.log(running_wf)
//_retrieve_url_list(modal, running_wf, 'output_list.txt');
});
/*
$("#modal-label-tmpl").text("Purge login");
$("#modal-body-tmpl").html('<p id="wf-help" class="text-justify">For purging files, please enter a unix login and password.</p>'
+'<div> <div class="form-group param-field">'
......@@ -476,7 +681,7 @@ function delete_data(list_ids,purge_demand_datatable){
$('#purge_demand_list').show();
}
});
});
});*/
}
function extend_date(list_ids, purge_demand_datatable, obsolete_project_datatable){
......
workflows/purge_demand/__init__.py
View file @ 1c51484c
......@@ -15,11 +15,11 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from ng6.ng6workflow import DownloadWorkflow
from ng6.ng6workflow import BasicNG6Workflow
from ng6.config_reader import NG6ConfigReader
class PurgeDemand (DownloadWorkflow):
class PurgeDemand (BasicNG6Workflow):
def get_description(self):
return """This workflow allows you to purge files on the server.
......
workflows/purge_demand/components/purgedemand.py
View file @ 1c51484c
......@@ -34,38 +34,46 @@ def remove_files (output_list, user_script, username, password , ids):
password = PasswordParameter.decrypt(password)
try:
#src_directories, dest_directories = Utils.get_directories_structure_and_content(ng6_username, data_folder, output_folder, prefixed_ids)
destinations = []
with open(user_script, "w") as us:
with open(output_list, "w") as ol :
for i, purge_id in enumerate(ids.split(",")):
ol.write("purge_id "+purge_id +"\n")
#ol.write(dest_file_path+"\n")
directories = Utils.get_purge_demand_directories(ids)
with open(output_list, "w") as log:
with open(user_script,"w") as us:
log.write("purge_id "+ids +"\n")
for d in directories:
files=[]
log.write(" - "+d +"\n")
if os.path.exists(d):
for f in os.listdir(d) :
if not f.endswith(".png") and not f.endswith(".html") :
files.append(f)
us.write("rm -f " + os.path.join(d,f) +";\n")
if len(files)>0 :
log.write("\tfiles: "+", ".join(files) +"\n")
# execution of the script as the user
server_params = config.get_server_parameters()
s = SSH(username, password, server_params[0])
#s.cmd('bash %s'%user_script)
s.cmd('bash %s'%user_script)
Utils.set_purge_demand_deleted(ids)
except Exception as e:
raise Exception('Unable to remove files : %s'%e)
class PurgeDemand (Component):
def define_parameters(self, username, password, demand_ids, data_ids = [], run_ids = [], analysis_ids = [], login = None):
def define_parameters(self, username, password, demand_ids = []):
self.add_parameter('username', 'Valid username on the server', required = True, default = username)
self.add_parameter('password', 'A password for the usernmae on the server', required = True, default = password, type = "password")
self.add_parameter_list('demand_ids', 'Ids of a demand from which data will be remove', default = demand_ids)
self.add_output_file( 'output_list', 'output_list', filename = "remove_list.txt")
def process(self):
userscript=self.get_temporary_file(".sh")
print (self.output_list, ','.join(self.demand_ids))
self.add_python_execution(remove_files, cmd_format = "{EXE} {OUT} {ARG} ",
arguments = [ self.username, self.password, ','.join(self.demand_ids)],
outputs = [self.output_list,userscript], local = True)
\ No newline at end of file
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment