Commit 1c51484c authored by Celine Noirot's avatar Celine Noirot
Browse files

create purgedemand workflow to encode password in ajax request

parent 352dbed2
......@@ -891,4 +891,133 @@ class t3MySQLdb(object):
'version' : row['analyze_version'],
'is_editable' : row['analyze_is_editable']
}
\ No newline at end of file
def select_purge_demand_directories(self, demand_id):
"""
Select all directories to purge
@param purge_demand_id:
"""
dirs=[]
if ',' in demand_id :
parts = [
'SELECT',
'tx_nG6_purge_demand.uid AS demand_id,',
'tx_nG6_purge_demand.analyze_ids AS analyze_ids,',
'tx_nG6_purge_demand.run_ids AS run_ids',
'FROM',
'tx_nG6_purge_demand ',
'WHERE',
'tx_nG6_purge_demand.uid in ('+str(demand_id)+')',
]
else:
parts = [
'SELECT',
'tx_nG6_purge_demand.uid AS demand_id,',
'tx_nG6_purge_demand.analyze_ids AS analyze_ids,',
'tx_nG6_purge_demand.run_ids AS run_ids',
'FROM',
'tx_nG6_purge_demand ',
'WHERE',
'tx_nG6_purge_demand.uid = '+str(demand_id),
]
sql = " ".join(parts)
print ("select_purge_demand_directories",sql)
qresult = self.execute(sql, dictc = True)
# analysis directories
if qresult.rowcount >= 1:
analysis_ids=[]
run_ids=[]
for res in qresult.rows :
analysis_ids.append(res['analyze_ids'] )
run_ids.append(res['run_ids'] )
parts = [
'SELECT',
'tx_nG6_analyze.directory AS analyze_directory',
'FROM',
'tx_nG6_analyze',
'WHERE',
'tx_nG6_analyze.uid IN (' + ','.join(analysis_ids) +')'
]
sql = " ".join(parts)
print ("select_purge_demand_directories",sql)
qresult = self.execute(sql, dictc = True)
for result in qresult.rows:
dirs.append(self.cfg_reader.get_save_directory() + result['analyze_directory'])
# run directories
parts = [
'SELECT',
'tx_nG6_run.directory AS run_directory',
'FROM',
'tx_nG6_run',
'WHERE',
'tx_nG6_run.uid IN (' + ','.join(run_ids) +')'
]
print ("select_purge_demand_directories",sql)
sql = " ".join(parts)
qresult = self.execute(sql, dictc = True)
for result in qresult.rows:
dirs.append(self.cfg_reader.get_save_directory() + result['run_directory'])
return dirs
def update_fields(self, table, uid, field, value, no_quote=[]):
if 'uid' in field :
return "Unable to execute 'update_fields' on field uid"
parts = [
'UPDATE',
table,
'SET']
update_part=[]
for f,v in zip(field,value):
if f not in no_quote :
update_part.append (f +' = "' +str(v)+'"')
else:
update_part.append (f +' = ' +str(v))
uids=uid
if isinstance(uid, list) :
uids=','.join(uid)
parts.extend([','.join(update_part),
'WHERE',
table+'.uid IN (' + uids +')' ] )
sql = " ".join(parts)
print ('### ',sql)
qresult = self.execute(sql, dictc = True)
return(qresult)
def set_purge_demand_deleted(self, demand_ids):
date = str(time.mktime(datetime.date.today().timetuple())).split(".")[0]
parts = [
'SELECT',
'tx_nG6_purge_demand.uid AS demand_id,',
'tx_nG6_purge_demand.analyze_ids AS analyze_ids,',
'tx_nG6_purge_demand.run_ids AS run_ids',
'FROM',
'tx_nG6_purge_demand ',
'WHERE',
'tx_nG6_purge_demand.uid IN ('+str(demand_ids) +')' ]
sql = " ".join(parts)
print ("set_purge_demand_deleted",sql)
qresult = self.execute(sql, dictc = True)
if qresult.rowcount >= 1:
analysis_ids=[]
run_ids=[]
for res in qresult.rows :
analysis_ids.append(res['analyze_ids'] )
run_ids.append(res['run_ids'] )
self.update_fields('tx_nG6_run', run_ids, ["purged_size","data_state", "purged_date", "storage_size"],
["storage_size","purged",date, 0],["purged_size"])
self.update_fields('tx_nG6_analyze',analysis_ids, ["purged_size","data_state", "purged_date", "storage_size"],
["storage_size","purged",date, 0],
["purged_size"])
self.update_fields('tx_nG6_purge_demand',demand_ids,
["processed_date","demand_state"],
[date,"deleted"])
......@@ -424,7 +424,19 @@ class Utils(object):
destinations.append(dest_directories[i])
return sources, destinations
def get_purge_demand_directories(demand_ids):
from ng6.t3MySQLdb import t3MySQLdb
t3mysql = t3MySQLdb()
return t3mysql.select_purge_demand_directories(demand_ids)
def set_purge_demand_deleted(demand_ids):
from ng6.t3MySQLdb import t3MySQLdb
t3mysql = t3MySQLdb()
return t3mysql.set_purge_demand_deleted(demand_ids)
class SSH(object):
def __init__(self, user, passwd, hostname, port=22):
self.username = user
......@@ -476,4 +488,4 @@ class SSH(object):
return self.ssh_results(pid, f)
\ No newline at end of file
......@@ -149,13 +149,13 @@ class tx_nG6_utils {
//TODO : BUG
// si unix user plantage de la suppression
// si ng6test erreur du ssh2_auth_password, voir avec MS
error_log("connection: ".$connection."\n");
foreach ($files as $file){
if (substr($file,-4) != ".png") {
if (substr($file,-4) != ".png" AND substr($file,-5) != ".html") {
// And process the directories structure
$res=ssh2_exec($connection, 'rm '.$file);
error_log("purge_files: rm -rf >".$file."< return : ".$res."\n");
if (!$res) {
$res=ssh2_exec($connection, 'rm '.$directory."/".$file);
error_log("purge_files: rm -rf >".$directory."/".$file."< return : ".$res."\n");
if (file_exists($directory."/".$file)) {
return 1;
}
}
......
......@@ -19,6 +19,7 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
<input type="hidden" id="user_is_login" value="{$login_user}" />
<input type="hidden" id="user_id" value="{$user_id}" />
<input type="hidden" id="user_name" value="{$user_name}" />
<input type="hidden" id="data_folder" value="{$data_folder}" />
<input type="hidden" id="usergroup" value="{$group_info.uid}" />
<input type="hidden" id="server_url" value="{$server_url}" />
......
......@@ -88,6 +88,7 @@ class tx_nG6_pi6 extends tslib_pibase {
$group_list = tx_nG6_db::get_group_list();
$smarty->assign('login_user', $GLOBALS['TSFE']->loginUser);
$smarty->assign('user_id', $GLOBALS['TSFE']->fe_user->user['uid']);
$smarty->assign('user_name', $GLOBALS['TSFE']->fe_user->user['username']);
$smarty->assign('group_list', $group_list);
$smarty->assign('data_folder', $this->conf["data"]);
$smarty->assign('server_url', $this->conf['server_url']);
......@@ -188,10 +189,10 @@ nG6 team';
$headers[] = 'Reply-To: '.$GLOBALS['TYPO3_CONF_VARS']['EXT']['extConf']['ng6']['email_from'];
$headers[] = 'X-Mailer: PHP/' . phpversion();
#TODO Uncomment
#mail($to, $subject, $mail, implode("\r\n", $headers));
mail($to, $subject, $mail, implode("\r\n", $headers));
//return $headers ;
}
return $mail;
return implode("\r\n", $headers).$mail;
}
function set_purge_demand_processed($demand_ids,$process, $value,$user,$pwd){
......@@ -225,32 +226,8 @@ nG6 team';
$date= time();
$all_runs_purged=array();
$all_analyses_purged=array();
$connexion=tx_nG6_utils::get_ssh_connection($user,$pwd);
if ( $connexion )
{
foreach ($all_runs as $run_id){
$r = tx_nG6_db::select_run($run_id);
if ($r["directory"] != "" ) {
$res = tx_nG6_utils::purge_files($connexion, $data_folder.$r["directory"]);
if ($res == 0){
$all_runs_purged[]=$run_id;
}else{
error_log("Error ".$res." while purging run id >".$run_id."< directory : >".$data_folder.$r["directory"]."< \n");
}
}
}
foreach ($all_analyses as $analyse_id){
$a = tx_nG6_db::select_analyse($analyse_id);
if ($a["directory"] != "" ) {
$res = tx_nG6_utils::purge_files($connexion, $data_folder.$a["directory"]);
if ($res == 0){
$all_analyses_purged[]=$analyse_id;
}else{
error_log("Error ".$res." while purging analyse id >".$analyse_id."< directory : >".$data_folder.$a["directory"]."< \n");
}
}
}
//purged_size = storage_size !!!
tx_nG6_db::update_field('tx_nG6_run', $all_runs_purged, array("purged_size","data_state", "purged_date", "storage_size"),
array("storage_size","purged",$date, 0),
......
......@@ -397,6 +397,7 @@ $(function () {
// Delete data after selecting list of ids demand
$("#delete_data_from_list").click(function(){
console.log("delete_data_from_list");
demands="";
$(':checked[id^=chk_demand').each(function(){
demand_id=$(this).val();
......@@ -434,8 +435,212 @@ $(function () {
function delete_data(list_ids,purge_demand_datatable){
var data_folder = $('#data_folder').val();
console.log("delete_data");
console.log(list_ids);
var _get_workflow_status = function(workflow_id, callback, error_callback){
$.ajax({
url : $("#server_url").val() + '/get_workflow_status?display=list&workflow_id=' + workflow_id ,
dataType : 'jsonp',
timeout: 20000 ,
success : function(data){
if (callback){
callback(data);
}
},
error : function(jqXHR, textStatus, errorThrown){
if (error_callback){
error_callback(jqXHR, textStatus, errorThrown);
}
}
});
};
var _retrieve_url_list = function(modal, running_wf, ufilename , process_data){
modal.$body.html([
'<div>',
' <div class="alert alert-info">',
' <p>',
' <strong>Data</strong> are being deleted. Use the refresh button to get the list of files.',
' </p>',
' </div>',
' <div id="wfstatus">',
' </div>',
'</div>'
].join(''));
modal.$footer.html([
' <div class="btn-group">',
' <button id="refresh_workflow" class="btn btn-default"><i class="glyphicon glyphicon-refresh"></i> Get data</button>',
' </div>',
' <div class="btn-group">',
' <button id="close" class="btn btn-default" data-dismiss="modal"><i class="glyphicon glyphicon-remove"></i> Close</button>',
' </div>',
].join(''));
$('#wfstatus').wfstatus({
workflowID : running_wf.id,
display : 'list',
serverURL: $("#server_url").val(),
})
$("#refresh_workflow").click(function(){
modal.$body.html("<div class='tx-nG6-wait'> <strong>Please</strong> wait...</div>");
button = this;
button.disabled = true;
_get_workflow_status(running_wf.id ,
// successcb
function(statusdata){
if (statusdata.status == "completed"){
_retrieve_data(running_wf.id, function(data){
$(button).hide();
$.each(data, function(i, component) {
$.each(component, function(filename, file) {
if (filename == ufilename){
_display_file_content(file.url, modal.$body, process_data);
}
});
});
},
//errorcb
function(jqXHR, textStatus, errorThrown){
modal.$body.html([
'<div class="alert alert-danger">',
' <p><strong>Failed !</strong> to get outputs from ' + $("#server_url").val() + ' </p>',
'</div>',
].join(''));
}
);
}
else {
$('#wfstatus').wfstatus('reload');
button.disabled = false;
}
},
// errorcb
function(){
$('#wfstatus').wfstatus('reload');
button.disabled = false;
}
);
});
};
var _retrieve_data = function(workflow_id, callback, error_callback){
$.ajax({
url : $("#server_url").val() + '/get_workflow_outputs?workflow_id=' + workflow_id,
dataType : 'jsonp',
success : function(data){
if (callback) {
callback(data);
}
},
error : function(jqXHR, textStatus, errorThrown){
if (error_callback){
error_callback(jqXHR, textStatus, errorThrown);
}
}
});
};
var _display_file_content = function(filepath, $elmt, process_data){
$.get(filepath, function(data){
if (process_data) {
data = process_data(data);
}
$elmt.html([
'<div class="alert">',
' <p>',
' <strong>Success !</strong> You will find bellow the list of deleted file.',
' Use the <i class="glyphicon glyphicon-floppy-save"></i> button to copy all paths to the clipboard.<br/>',
' </p>',
'</div>',
'<div>',
' <button id="copy_urls" class="btn btn-default btn-sm pull-right" data-clipboard-target="url_filelist">',
' <i class="glyphicon glyphicon-floppy-save"></i>&nbsp;',
' </button><br/>',
' <textarea readonly id="url_filelist" rows="25" style="resize:none;white-space: pre; word-wrap: normal; overflow-x: scroll; width: 100%">',
data.trim(),
' </textarea>',
'</div>'
].join(''));
var client = new ZeroClipboard($("#copy_urls"));
});
};
$('#setAndRunModalLabel').html("Loading");
$('#setAndRunModalFooter').html([
' <div class="btn-group">',
' <button id="close" class="btn btn-default" data-dismiss="modal"><i class="glyphicon glyphicon-remove"></i> Close</button>',
' </div>',
' <div class="btn-group">',
' <button id="reset_workflow" class="btn btn-default"><i class="glyphicon glyphicon-refresh"></i> Reset</button>',
' <button id="run_workflow" class="btn btn-primary"><i class="glyphicon glyphicon-floppy-save"></i> Start</button>',
' </div>'
].join(''));
$("#reset_workflow, #run_workflow").hide();
$("#close").show();
$('#setAndRunModalBody').html([
'<div >',
' <p id="wf-help" class="text-justify"></p><br/><br/>',
' <div id="wfform"><div class="tx-nG6-wait">Please wait</div></div>',
'</div>',
].join(''));
var modal = {
'$label' : $('#setAndRunModalLabel'),
'$body' : $('#setAndRunModalBody'),
'$footer' : $('#setAndRunModalFooter'),
'$modal' : $('#setAndRunModal')
};
console.log(list_ids);
var wfform_options = {
workflowClass: "PurgeDemand",
serverURL: $("#server_url").val(),
displayRunButton: false,
displayResetButton: false,
parameters : {
"demand" : list_ids,
"admin_login" : $("#user_name").val()
}
};
$('#wfform').on("loaded.wfform", function(event, workflow) {
$("#reset_workflow, #run_workflow").show();
$("#close").hide();
if(workflow){
modal.$label.html(workflow["name"]) ;
$('#wf-help').html(workflow["help"]);
}
else{
modal.$body.html('<div class="alert alert-warning"> <strong>Warning !</strong> The workflow "PurgeDemand" is not available</div>');
}
});
$("#reset_workflow").click(function(){
$('#wfform').wfform('reset');
});
$("#run_workflow").click(function(){
$('#wfform').wfform('run');
});
// remove plugin data
$('#wfform').removeData();
modal.$modal.modal();
modal.$modal.find('#wfform').wfform(wfform_options);
modal.$modal.find('#wfform').on('run.wfform', function(event, running_wf) {
console.log(running_wf)
//_retrieve_url_list(modal, running_wf, 'output_list.txt');
});
/*
$("#modal-label-tmpl").text("Purge login");
$("#modal-body-tmpl").html('<p id="wf-help" class="text-justify">For purging files, please enter a unix login and password.</p>'
+'<div> <div class="form-group param-field">'
......@@ -476,7 +681,7 @@ function delete_data(list_ids,purge_demand_datatable){
$('#purge_demand_list').show();
}
});
});
});*/
}
function extend_date(list_ids, purge_demand_datatable, obsolete_project_datatable){
......
......@@ -15,11 +15,11 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from ng6.ng6workflow import DownloadWorkflow
from ng6.ng6workflow import BasicNG6Workflow
from ng6.config_reader import NG6ConfigReader
class PurgeDemand (DownloadWorkflow):
class PurgeDemand (BasicNG6Workflow):
def get_description(self):
return """This workflow allows you to purge files on the server.
......
......@@ -34,38 +34,46 @@ def remove_files (output_list, user_script, username, password , ids):
password = PasswordParameter.decrypt(password)
try:
#src_directories, dest_directories = Utils.get_directories_structure_and_content(ng6_username, data_folder, output_folder, prefixed_ids)
destinations = []
with open(user_script, "w") as us:
with open(output_list, "w") as ol :
for i, purge_id in enumerate(ids.split(",")):
ol.write("purge_id "+purge_id +"\n")
#ol.write(dest_file_path+"\n")
directories = Utils.get_purge_demand_directories(ids)
with open(output_list, "w") as log:
with open(user_script,"w") as us:
log.write("purge_id "+ids +"\n")
for d in directories:
files=[]
log.write(" - "+d +"\n")
if os.path.exists(d):
for f in os.listdir(d) :
if not f.endswith(".png") and not f.endswith(".html") :
files.append(f)
us.write("rm -f " + os.path.join(d,f) +";\n")
if len(files)>0 :
log.write("\tfiles: "+", ".join(files) +"\n")
# execution of the script as the user
server_params = config.get_server_parameters()
s = SSH(username, password, server_params[0])
#s.cmd('bash %s'%user_script)
s.cmd('bash %s'%user_script)
Utils.set_purge_demand_deleted(ids)
except Exception as e:
raise Exception('Unable to remove files : %s'%e)
class PurgeDemand (Component):
def define_parameters(self, username, password, demand_ids, data_ids = [], run_ids = [], analysis_ids = [], login = None):
def define_parameters(self, username, password, demand_ids = []):
self.add_parameter('username', 'Valid username on the server', required = True, default = username)
self.add_parameter('password', 'A password for the usernmae on the server', required = True, default = password, type = "password")
self.add_parameter_list('demand_ids', 'Ids of a demand from which data will be remove', default = demand_ids)
self.add_output_file( 'output_list', 'output_list', filename = "remove_list.txt")
def process(self):
userscript=self.get_temporary_file(".sh")
print (self.output_list, ','.join(self.demand_ids))
self.add_python_execution(remove_files, cmd_format = "{EXE} {OUT} {ARG} ",
arguments = [ self.username, self.password, ','.join(self.demand_ids)],
outputs = [self.output_list,userscript], local = True)
\ No newline at end of file
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment