first condor support

src/core/tandem_run/tandemcondorprocess.cpp

@@ -50,8 +50,9 @@ TandemCondorProcess::TandemCondorProcess(WorkMonitorInterface * p_monitor, const
     */
 
     QSettings settings;
-    QString condor_tmp_dir = QString("%1/xtpcpp").arg(settings.value("condor/tmp_dir", "/gorgone/pappso/temp").toString());
+    QString condor_tmp_dir = QString("%1/xtpcpp").arg(settings.value("condor/tmp_dir", "/tmp").toString());
     _p_tmp_dir = new QTemporaryDir(condor_tmp_dir);
+    _p_tmp_dir->setAutoRemove(true);
     _condor_submit_command = settings.value("condor/submit", "/usr/bin/condor_submit").toString();
 
 
@@ -87,7 +88,7 @@ void TandemCondorProcess::prepareXmlDatabaseFile() {
 }
 
 void TandemCondorProcess::run() {
-    qDebug() << "TandeCondorProcess::run begin ";
+    qDebug() << "TandemCondorProcess::run begin ";
     QFileInfo preset_info(_tandem_run_batch._preset_file);
     _preset_file = QString("%1/%2").arg(_p_tmp_dir->path()).arg(preset_info.fileName());
     QFile::copy(_tandem_run_batch._preset_file, _preset_file);
@@ -111,9 +112,9 @@ void TandemCondorProcess::run() {
         *p_out <<     "request_memory= 50000" << endl;
         *p_out <<     "request_cpus = 1" << endl;
         *p_out <<     "Executable = " << _tandem_run_batch._tandem_bin_path << endl;
-        *p_out <<     "Log        = \"" << _p_tmp_dir->path() << "/condor.log\"" << endl;
-        *p_out <<     "Output        = \"" << _p_tmp_dir->path() << "/tandem.$(Process).out\"" << endl;
-        *p_out <<     "Error        = \"" << _p_tmp_dir->path() << "/tandem.$(Process).error\"" << endl;
+        *p_out <<     "Log        = " << _p_tmp_dir->path() << "/condor.log" << endl;
+        *p_out <<     "Output        = " << _p_tmp_dir->path() << "/tandem.$(Process).out" << endl;
+        *p_out <<     "Error        = " << _p_tmp_dir->path() << "/tandem.$(Process).error" << endl;
         /*
         Log        = /gorgone/pappso/tmp/temp_condor_job93294001891239208719639434471283743/submit_condor.log
         Output        = /gorgone/pappso/tmp/temp_condor_job93294001891239208719639434471283743/tandem.$(Process).out
@@ -136,7 +137,7 @@ void TandemCondorProcess::run() {
     for (QString mz_file : _tandem_run_batch._mz_file_list) {
 
 
-        QTemporaryFile *  p_xml_input_file = new QTemporaryFile(_p_tmp_dir->path());
+        QTemporaryFile *  p_xml_input_file = new QTemporaryFile(QString("%1/tandem").arg(_p_tmp_dir->path()));
 
         input_file_list.push_back(p_xml_input_file);
         p_xml_input_file->setAutoRemove(false);
@@ -151,7 +152,7 @@ void TandemCondorProcess::run() {
             delete p_xml_out;
 
 
-            *p_out <<     "Arguments        = \"" << QFileInfo( p_xml_input_file->fileName()).absolutePath() << "\"" << endl;
+            *p_out <<     "Arguments        = " << QFileInfo( p_xml_input_file->fileName()).absoluteFilePath() << endl;
             *p_out <<     "Queue" << endl;
         } else
         {
@@ -169,9 +170,9 @@ void TandemCondorProcess::run() {
         delete p_out;
     }
 
-    
+
     //now run condor job on submit_file
-    
+
 
     QStringList arguments;
 
@@ -179,6 +180,7 @@ void TandemCondorProcess::run() {
 
     QProcess * condor_process = new QProcess();
     //hk_process->setWorkingDirectory(QFileInfo(_hardklor_exe).absolutePath());
+    qDebug() << "TandemCondorProcess::run command " << _condor_submit_command << " " << arguments.join(" ");
     condor_process->start(_condor_submit_command, arguments);
 
 
@@ -186,10 +188,36 @@ void TandemCondorProcess::run() {
         throw pappso::PappsoException(QObject::tr("HTCondor X!Tandem process failed to start"));
     }
 
-    while(condor_process->waitForReadyRead(_max_xt_time_ms)) {
-        _p_monitor->appendText( condor_process->readAll().data());
-        //data.append(xt_process->readAll());
+    if (!condor_process->waitForFinished(_max_xt_time_ms)) {
+        throw pappso::PappsoException(QObject::tr("HTCondor X!Tandem process failed to finish"));
+    }
+
+    QString perr = condor_process->readAllStandardError();
+    if (perr.length()) {
+
+        qDebug() << "TandemCondorProcess::run readAllStandardError " << perr;
+        throw pappso::PappsoException(QObject::tr("HTCondor X!Tandem process failed :\n%1").arg(perr));
+    }
+    else {
+        qDebug() << "TandemCondorProcess::run readAllStandardError OK " << perr;
+    }
+
+    QString pjob = condor_process->readAllStandardOutput();
+    if (pjob.length()) {
+        qDebug() << "TandemCondorProcess::run readAllStandardOutput OK " << pjob;
+    }
+    else {
+        qDebug() << "TandemCondorProcess::run readAllStandardOutput " << pjob;
+        throw pappso::PappsoException(QObject::tr("HTCondor X!Tandem process failed :\n%1").arg(pjob));
     }
+
+    //Submitting job(s).\n1 job(s) submitted to cluster 29.\n
+    parseCondorJobNumber(pjob);
+    
+    _p_monitor->setProgressMaximumValue(_condor_job_size);
+    qDebug() << "TandemCondorProcess::run job=" << _condor_cluster_number << " size=" << _condor_job_size;
+
+
     /*
     if (!xt_process->waitForFinished(_max_xt_time_ms)) {
         throw pappso::PappsoException(QObject::tr("can't wait for X!Tandem process to finish : timeout at %1").arg(_max_xt_time_ms));
@@ -199,6 +227,7 @@ void TandemCondorProcess::run() {
 
     QProcess::ExitStatus Status = condor_process->exitStatus();
 
+    qDebug() << "TandemCondorProcess::run ExitStatus " << Status << result.data();
     if (Status != 0)
     {
         // != QProcess::NormalExit
@@ -206,6 +235,177 @@ void TandemCondorProcess::run() {
     }
 
     delete condor_process;
-    
-    qDebug() << "TandeCondorProcess::run end" ;
+
+
+    //condor is running job : we have to survey condor job using "condor_q -xml _condor_cluster_number"
+
+    surveyCondorJob();
+
+    qDebug() << "TandemCondorProcess::run end" ;
 }
+
+void TandemCondorProcess::surveyCondorJob() {
+    /*
+    package fr.inra.pappso.xtandempipeline.sax_parse;
+
+    import org.apache.log4j.Logger;
+    import org.xml.sax.Attributes;
+    import org.xml.sax.SAXException;
+
+    import fr.inra.pappso.xtandempipeline.MsException.MSMSException;
+    import fr.inra.pappso.xtandempipeline.thread.XtandemCondorAnalysis;
+
+    public class HandlerHTCondorQueue extends SaxBaseHandler {
+
+    private static final Logger logger = Logger
+    		.getLogger(HandlerHTCondorQueue.class);
+    private Object inName;
+    private Integer currentProcId;
+    private Integer currentJobStatus;
+    private String currentRemoteHost;
+    private String currentLastRemoteHost;
+    private int junexpanded = 0;
+    private int jidle = 0;
+    private int jrunning = 0;
+    private int jremoved = 0;
+    private int jcompleted = 0;
+    private int jheld = 0;
+    private int jsubmission_error = 0;
+    private XtandemCondorAnalysis xtandemCondorAnalysis;
+    private boolean empty = true;
+
+    public HandlerHTCondorQueue(XtandemCondorAnalysis xtandemCondorAnalysis)
+    		throws MSMSException {
+    	super();
+    	this.xtandemCondorAnalysis = xtandemCondorAnalysis;
+    }
+
+    @Override
+    public void startDocument() throws SAXException {
+    	this.empty = true;
+    }
+
+    @Override
+    public void endDocument() throws SAXException {
+    	int reallyCompletedJobs = 0;
+    	if (this.empty) {
+    		reallyCompletedJobs = xtandemCondorAnalysis.getCondorJobNumber();
+    	} else {
+    		int totalJobs = junexpanded + jidle + jrunning + jremoved
+    				+ jcompleted + jheld + jsubmission_error;
+    		int diffJobs = xtandemCondorAnalysis.getCondorJobNumber()
+    				- totalJobs;
+    		reallyCompletedJobs = jcompleted + diffJobs;
+    	}
+
+    	xtandemCondorAnalysis.setJobCompleted(junexpanded, jidle, jrunning,
+    			jremoved, reallyCompletedJobs, jheld, jsubmission_error);
+
+    	String statusMessage = "" + junexpanded + " unexpanded jobs\n";
+    	statusMessage += jidle + " idle jobs\n";
+    	statusMessage += jrunning + " running jobs\n";
+    	statusMessage += jremoved + " removed jobs\n";
+    	statusMessage += reallyCompletedJobs + " completed jobs\n";
+    	statusMessage += jheld + " held jobs\n";
+    	statusMessage += jsubmission_error + " submission_errors\n";
+
+    	logger.debug(statusMessage);
+    	xtandemCondorAnalysis.setStatusMessage(statusMessage);
+
+    }
+
+    // <c>
+    // <a n="ProcId"><i>0</i></a>
+    public void startElementc(String nameSpaceUri, Attributes attrs) {
+    	this.empty = false;
+    	currentProcId = null;
+    	currentJobStatus = null;
+    	currentRemoteHost = null;
+    	currentLastRemoteHost = null;
+
+    }
+
+    public void startElementa(String nameSpaceUri, Attributes attrs) {
+    	// logger.debug("startElementgroup begin");
+    	// <group label="performance parameters" type="parameters">
+    	this.inName = attrs.getValue("n");
+
+    }
+
+    public void endElementi(String nameSpaceUri, String data)
+    		throws MSMSException {
+    	if (inName.equals("ProcId")) {
+    		this.currentProcId = new Integer(data);
+    		// logger.debug(currentProcId);
+    	} else if (inName.equals("JobStatus")) {
+    		// <a n="JobStatus"><i>2</i></a>
+    		this.currentJobStatus = new Integer(data);
+    		// logger.debug(currentProcId);
+    	}
+
+    }
+
+    public void endElements(String nameSpaceUri, String data)
+    		throws MSMSException {
+    	if (inName.equals("RemoteHost")) {
+    		// <a n="RemoteHost"><s>slot1@proteus3</s></a>
+    		this.currentRemoteHost = data;
+    		// logger.debug(currentProcId);
+    	} else if (inName.equals("LastRemoteHost")) {
+    		// <a n="LastRemoteHost"><s>slot1@proteus4</s></a>
+    		this.currentLastRemoteHost = data;
+    		// logger.debug(currentProcId);
+    	}
+
+    }
+
+    public void endElementa(String nameSpaceUri, String data) {
+    	this.inName = "";
+    }
+
+    public void endElementc(String nameSpaceUri, String data)
+    		throws MSMSException {
+    	/*
+    	 * 0 Unexpanded U 1 Idle I 2 Running R 3 Removed X 4 Completed C 5 Held
+    	 * H 6 Submission_err E
+    	 *
+    	if (currentJobStatus == 0) {
+    		this.junexpanded += 1;
+    	} else if (currentJobStatus == 1) {
+    		this.jidle += 1;
+    	} else if (currentJobStatus == 2) {
+    		this.jrunning += 1;
+    	} else if (currentJobStatus == 3) {
+    		this.jremoved += 1;
+    	} else if (currentJobStatus == 4) {
+    		this.jcompleted += 1;
+    	} else if (currentJobStatus == 5) {
+    		this.jheld += 1;
+    	} else if (currentJobStatus == 6) {
+    		this.jsubmission_error += 1;
+    	}
+
+    	logger.debug("currentRemoteHost:" + this.currentRemoteHost
+    			+ " currentLastRemoteHost" + this.currentLastRemoteHost + " "
+    			+ currentJobStatus);
+    }
+
+    }
+    */
+}
+
+
+void TandemCondorProcess::parseCondorJobNumber(QString condor_job) {
+    // Submitting job(s)...
+    // 3 job(s) submitted to cluster 3.
+    QRegExp txt_submit("([0-9]*) job\\(s\\) submitted to cluster ([0-9]*).");
+
+    if (txt_submit.indexIn(condor_job, 0) != -1) {
+        _condor_cluster_number = txt_submit.cap(2).toUInt();
+        _condor_job_size = txt_submit.cap(1).toUInt();
+    }
+    else {
+        throw pappso::PappsoException(QObject::tr("unable to find HTCondor job numbers in %1").arg(condor_job));
+    }
+}
+

src/core/tandem_run/tandemcondorprocess.h

@@ -41,9 +41,15 @@ public:
 
     virtual void prepareXmlDatabaseFile();
     virtual void run();
+private :
+    void parseCondorJobNumber(QString condor_job);
+    void surveyCondorJob();
 private:
     QTemporaryDir * _p_tmp_dir;
     QString _condor_submit_command;
+    
+    unsigned int _condor_cluster_number;
+    unsigned int _condor_job_size;
 
 };
 

src/gui/workerthread.cpp

@@ -38,6 +38,8 @@
 #include "mainwindow.h"
 #include "project_view/projectwindow.h"
 #include "../core/tandem_run/tandembatchprocess.h"
+#include "../core/tandem_run/tandemcondorprocess.h"
+#include <QSettings>
 
 WorkerThread::WorkerThread(MainWindow * p_main_window)
 {
@@ -51,10 +53,10 @@ WorkerThread::WorkerThread(MainWindow * p_main_window)
     connect(_p_work_monitor, &WorkMonitor::workerMessage, p_main_window,&MainWindow::doDisplayLoadingMessage);
     connect(_p_work_monitor, &WorkMonitor::workerMessagePercent, p_main_window,&MainWindow::doDisplayLoadingMessagePercent);
     connect(_p_work_monitor, &WorkMonitor::workerAppendText, p_main_window,&MainWindow::doWorkerAppendText);
-    
+
     connect(p_main_window, &MainWindow::operateXpipFile, this,&WorkerThread::doXpipFileLoad);
     connect(this, &WorkerThread::projectReady, p_main_window,&MainWindow::doProjectReady);
-    
+
     connect(p_main_window, &MainWindow::operateLoadingResults, this,&WorkerThread::doLoadingResults);
 
     connect(p_main_window, &MainWindow::operateRunningXtandem, this,&WorkerThread::doRunningXtandem);
@@ -72,7 +74,7 @@ WorkerThread::WorkerThread(MainWindow * p_main_window)
     connect(p_main_window, &MainWindow::operateWritingOdsFile, this,&WorkerThread::doWritingOdsFile);
     connect(this, &WorkerThread::operationFailed, p_main_window,&MainWindow::doOperationFailed);
     connect(this, &WorkerThread::operationFinished, p_main_window,&MainWindow::doOperationFinished);
-    
+
 #else
 // Qt4 code
     //worker message
@@ -112,7 +114,7 @@ WorkerThread::WorkerThread(ProjectWindow * p_project_window)
 
 #if QT_VERSION >= 0x050000
     // Qt5 code
-    
+
     //worker message
     connect(_p_work_monitor, &WorkMonitor::workerMessage, p_project_window,&ProjectWindow::doDisplayLoadingMessage);
     connect(_p_work_monitor, &WorkMonitor::workerMessagePercent, p_project_window,&ProjectWindow::doDisplayLoadingMessagePercent);
@@ -172,7 +174,7 @@ void WorkerThread::doXpipFileLoad(QString filename) {
     try {
         QFileInfo new_xpip_file(filename);
         emit loadingMessage(tr("loading XPIP file"));
- qDebug() << "WorkerThread::doXpipFileLoad new_xpip_file "<< new_xpip_file.absoluteFilePath();
+        qDebug() << "WorkerThread::doXpipFileLoad new_xpip_file "<< new_xpip_file.absoluteFilePath();
         XpipFile xpip_file(new_xpip_file);
         ProjectSp project_sp = xpip_file.getProjectSp(_p_work_monitor);
 
@@ -333,8 +335,17 @@ void WorkerThread::doRunningXtandem(TandemRunBatch tandem_run_batch) {
         emit loadingMessage(tr("Running X!Tandem, please wait"));
         qDebug() <<  "WorkerThread::doRunningXtandem tandem_run_batch " << tandem_run_batch._tandem_bin_path;
 
-        TandemBatchProcess xt_process (_p_work_monitor, tandem_run_batch);
-        xt_process.run();
+        QSettings settings;
+        QString default_location = settings.value("condor/tmp_dir", "").toString();
+
+        if (default_location.isEmpty()) {
+            TandemBatchProcess xt_process (_p_work_monitor, tandem_run_batch);
+            xt_process.run();
+        }
+        else {
+            TandemCondorProcess xt_process (_p_work_monitor, tandem_run_batch);
+            xt_process.run();
+        }
 
         emit operationFinished();
     }