condor_q sax handler

src/CMakeLists.txt

@@ -78,6 +78,7 @@ SET(CPP_FILES
   grouping/ptm/ptmislandsubgroup.cpp
   grouping/ptm/ptmsamplescan.cpp
   gui/peptide_detail_view/spectrum_widget/spectrumpainter.cpp
+  input/condorqxmlsaxhandler.cpp
   input/identificationpwizreader.cpp
   input/xpipsaxhandler.cpp
   input/xtandemsaxhandler.cpp

src/core/tandem_run/tandemcondorprocess.cpp

@@ -32,6 +32,9 @@
 #include <pappsomspp/pappsoexception.h>
 #include <QSettings>
 #include <QProcess>
+#include <QXmlSimpleReader>
+#include <QThread>
+#include "../../input/condorqxmlsaxhandler.h"
 
 TandemCondorProcess::TandemCondorProcess(WorkMonitorInterface * p_monitor, const TandemRunBatch & tandem_run_batch) : TandemBatchProcess(p_monitor, tandem_run_batch) {
     /*
@@ -54,6 +57,8 @@ TandemCondorProcess::TandemCondorProcess(WorkMonitorInterface * p_monitor, const
     _p_tmp_dir = new QTemporaryDir(condor_tmp_dir);
     _p_tmp_dir->setAutoRemove(true);
     _condor_submit_command = settings.value("condor/submit", "/usr/bin/condor_submit").toString();
+    _condor_q_command = settings.value("condor/condor_q", "/usr/bin/condor_q").toString();
+    _condor_request_memory = settings.value("condor/request_memory", "10000").toUInt();
 
 
     if (!_p_tmp_dir->isValid()) {
@@ -109,7 +114,7 @@ void TandemCondorProcess::run() {
         *p_out <<     "Universe   = vanilla" << endl;
         *p_out <<     "notification   = Error" << endl;
         *p_out <<     "Rank       = Mips" << endl;
-        *p_out <<     "request_memory= 50000" << endl;
+        *p_out <<     "request_memory= " << _condor_request_memory << endl;
         *p_out <<     "request_cpus = 1" << endl;
         *p_out <<     "Executable = " << _tandem_run_batch._tandem_bin_path << endl;
         *p_out <<     "Log        = " << _p_tmp_dir->path() << "/condor.log" << endl;
@@ -213,7 +218,7 @@ void TandemCondorProcess::run() {
 
     //Submitting job(s).\n1 job(s) submitted to cluster 29.\n
     parseCondorJobNumber(pjob);
-    
+
     _p_monitor->setProgressMaximumValue(_condor_job_size);
     qDebug() << "TandemCondorProcess::run job=" << _condor_cluster_number << " size=" << _condor_job_size;
 
@@ -237,161 +242,87 @@ void TandemCondorProcess::run() {
     delete condor_process;
 
 
-    //condor is running job : we have to survey condor job using "condor_q -xml _condor_cluster_number"
-
     surveyCondorJob();
 
     qDebug() << "TandemCondorProcess::run end" ;
 }
 
 void TandemCondorProcess::surveyCondorJob() {
-    /*
-    package fr.inra.pappso.xtandempipeline.sax_parse;
-
-    import org.apache.log4j.Logger;
-    import org.xml.sax.Attributes;
-    import org.xml.sax.SAXException;
-
-    import fr.inra.pappso.xtandempipeline.MsException.MSMSException;
-    import fr.inra.pappso.xtandempipeline.thread.XtandemCondorAnalysis;
-
-    public class HandlerHTCondorQueue extends SaxBaseHandler {
-
-    private static final Logger logger = Logger
-    		.getLogger(HandlerHTCondorQueue.class);
-    private Object inName;
-    private Integer currentProcId;
-    private Integer currentJobStatus;
-    private String currentRemoteHost;
-    private String currentLastRemoteHost;
-    private int junexpanded = 0;
-    private int jidle = 0;
-    private int jrunning = 0;
-    private int jremoved = 0;
-    private int jcompleted = 0;
-    private int jheld = 0;
-    private int jsubmission_error = 0;
-    private XtandemCondorAnalysis xtandemCondorAnalysis;
-    private boolean empty = true;
-
-    public HandlerHTCondorQueue(XtandemCondorAnalysis xtandemCondorAnalysis)
-    		throws MSMSException {
-    	super();
-    	this.xtandemCondorAnalysis = xtandemCondorAnalysis;
-    }
 
-    @Override
-    public void startDocument() throws SAXException {
-    	this.empty = true;
+    //condor is running job : we have to survey condor job using "condor_q -xml _condor_cluster_number"
+    while(true) {
+
+        QThread::msleep(_condor_status_timer_millisecond);
+        getCondorJobState();
     }
+}
+void TandemCondorProcess::getCondorJobState() {
 
-    @Override
-    public void endDocument() throws SAXException {
-    	int reallyCompletedJobs = 0;
-    	if (this.empty) {
-    		reallyCompletedJobs = xtandemCondorAnalysis.getCondorJobNumber();
-    	} else {
-    		int totalJobs = junexpanded + jidle + jrunning + jremoved
-    				+ jcompleted + jheld + jsubmission_error;
-    		int diffJobs = xtandemCondorAnalysis.getCondorJobNumber()
-    				- totalJobs;
-    		reallyCompletedJobs = jcompleted + diffJobs;
-    	}
-
-    	xtandemCondorAnalysis.setJobCompleted(junexpanded, jidle, jrunning,
-    			jremoved, reallyCompletedJobs, jheld, jsubmission_error);
-
-    	String statusMessage = "" + junexpanded + " unexpanded jobs\n";
-    	statusMessage += jidle + " idle jobs\n";
-    	statusMessage += jrunning + " running jobs\n";
-    	statusMessage += jremoved + " removed jobs\n";
-    	statusMessage += reallyCompletedJobs + " completed jobs\n";
-    	statusMessage += jheld + " held jobs\n";
-    	statusMessage += jsubmission_error + " submission_errors\n";
-
-    	logger.debug(statusMessage);
-    	xtandemCondorAnalysis.setStatusMessage(statusMessage);
 
-    }
+    QStringList arguments;
 
-    // <c>
-    // <a n="ProcId"><i>0</i></a>
-    public void startElementc(String nameSpaceUri, Attributes attrs) {
-    	this.empty = false;
-    	currentProcId = null;
-    	currentJobStatus = null;
-    	currentRemoteHost = null;
-    	currentLastRemoteHost = null;
+    arguments << "-xml" << QString("%1").arg(_condor_cluster_number);
 
-    }
+    QProcess condor_q_process;
+    //hk_process->setWorkingDirectory(QFileInfo(_hardklor_exe).absolutePath());
+    qDebug() << "TandemCondorProcess::getCondorJobState command " << _condor_q_command << " " << arguments.join(" ");
+    condor_q_process.start(_condor_q_command, arguments);
 
-    public void startElementa(String nameSpaceUri, Attributes attrs) {
-    	// logger.debug("startElementgroup begin");
-    	// <group label="performance parameters" type="parameters">
-    	this.inName = attrs.getValue("n");
 
+    if (!condor_q_process.waitForStarted()) {
+        throw pappso::PappsoException(QObject::tr("HTCondor condor_q process failed to start"));
     }
 
-    public void endElementi(String nameSpaceUri, String data)
-    		throws MSMSException {
-    	if (inName.equals("ProcId")) {
-    		this.currentProcId = new Integer(data);
-    		// logger.debug(currentProcId);
-    	} else if (inName.equals("JobStatus")) {
-    		// <a n="JobStatus"><i>2</i></a>
-    		this.currentJobStatus = new Integer(data);
-    		// logger.debug(currentProcId);
-    	}
-
+    if (!condor_q_process.waitForFinished(_max_xt_time_ms)) {
+        throw pappso::PappsoException(QObject::tr("HTCondor condor_q process failed to finish"));
     }
 
-    public void endElements(String nameSpaceUri, String data)
-    		throws MSMSException {
-    	if (inName.equals("RemoteHost")) {
-    		// <a n="RemoteHost"><s>slot1@proteus3</s></a>
-    		this.currentRemoteHost = data;
-    		// logger.debug(currentProcId);
-    	} else if (inName.equals("LastRemoteHost")) {
-    		// <a n="LastRemoteHost"><s>slot1@proteus4</s></a>
-    		this.currentLastRemoteHost = data;
-    		// logger.debug(currentProcId);
-    	}
+    QString perr = condor_q_process.readAllStandardError();
+    if (perr.length()) {
 
+        qDebug() << "TandemCondorProcess::getCondorJobState readAllStandardError " << perr;
+        throw pappso::PappsoException(QObject::tr("HTCondor condor_q process failed :\n%1").arg(perr));
     }
-
-    public void endElementa(String nameSpaceUri, String data) {
-    	this.inName = "";
+    else {
+        qDebug() << "TandemCondorProcess::getCondorJobState readAllStandardError OK " << perr;
     }
 
-    public void endElementc(String nameSpaceUri, String data)
-    		throws MSMSException {
-    	/*
-    	 * 0 Unexpanded U 1 Idle I 2 Running R 3 Removed X 4 Completed C 5 Held
-    	 * H 6 Submission_err E
-    	 *
-    	if (currentJobStatus == 0) {
-    		this.junexpanded += 1;
-    	} else if (currentJobStatus == 1) {
-    		this.jidle += 1;
-    	} else if (currentJobStatus == 2) {
-    		this.jrunning += 1;
-    	} else if (currentJobStatus == 3) {
-    		this.jremoved += 1;
-    	} else if (currentJobStatus == 4) {
-    		this.jcompleted += 1;
-    	} else if (currentJobStatus == 5) {
-    		this.jheld += 1;
-    	} else if (currentJobStatus == 6) {
-    		this.jsubmission_error += 1;
-    	}
-
-    	logger.debug("currentRemoteHost:" + this.currentRemoteHost
-    			+ " currentLastRemoteHost" + this.currentLastRemoteHost + " "
-    			+ currentJobStatus);
+    QString pjob = condor_q_process.readAllStandardOutput();
+    if (pjob.length()) {
+        qDebug() << "TandemCondorProcess::getCondorJobState readAllStandardOutput OK " << pjob;
+    }
+    else {
+        qDebug() << "TandemCondorProcess::getCondorJobState readAllStandardOutput " << pjob;
+        throw pappso::PappsoException(QObject::tr("HTCondor condor_q process failed :\n%1").arg(pjob));
     }
 
+    //Submitting job(s).\n1 job(s) submitted to cluster 29.\n
+    parseCondorQueue(pjob);
+}
+
+void TandemCondorProcess::parseCondorQueue(QString & condor_q_xml) {
+
+
+    CondorQxmlSaxHandler * parser = new CondorQxmlSaxHandler(this);
+
+    QXmlSimpleReader simplereader;
+    simplereader.setContentHandler(parser);
+    simplereader.setErrorHandler(parser);
+
+    qDebug() << "TandemCondorProcess::parseCondorQueue Read condor_q_xml";
+
+    QXmlInputSource xml_input_source;
+    xml_input_source.setData(condor_q_xml);
+
+    if (simplereader.parse(xml_input_source)) {
+    } else {
+        qDebug() << parser->errorString();
+
+        throw pappso::PappsoException(QObject::tr("Error reading condor_q xml string :\n %1").arg(condor_q_xml));
     }
-    */
+    delete parser;
+
+
 }
 
 

src/core/tandem_run/tandemcondorprocess.h

@@ -44,12 +44,17 @@ public:
 private :
     void parseCondorJobNumber(QString condor_job);
     void surveyCondorJob();
+    void getCondorJobState();
+    void parseCondorQueue(QString & condor_q_xml);
 private:
     QTemporaryDir * _p_tmp_dir;
     QString _condor_submit_command;
+    QString _condor_q_command;
     
     unsigned int _condor_cluster_number;
     unsigned int _condor_job_size;
+    unsigned int _condor_request_memory;
+    unsigned int _condor_status_timer_millisecond= 1000;
 
 };
 

src/input/condorqxmlsaxhandler.cpp

+/**
+ * \file input/condorqxmlsaxhandler.cpp
+ * \date 15/9/2017
+ * \author Olivier Langella
+ * \brief parse condor_q XML
+ */
+
+
+/*******************************************************************************
+* Copyright (c) 2017 Olivier Langella <Olivier.Langella@u-psud.fr>.
+*
+* This file is part of XTPcpp.
+*
+*     XTPcpp is free software: you can redistribute it and/or modify
+*     it under the terms of the GNU General Public License as published by
+*     the Free Software Foundation, either version 3 of the License, or
+*     (at your option) any later version.
+*
+*     XTPcpp is distributed in the hope that it will be useful,
+*     but WITHOUT ANY WARRANTY; without even the implied warranty of
+*     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*     GNU General Public License for more details.
+*
+*     You should have received a copy of the GNU General Public License
+*     along with XTPcpp.  If not, see <http://www.gnu.org/licenses/>.
+*
+* Contributors:
+*     Olivier Langella <Olivier.Langella@u-psud.fr> - initial API and implementation
+******************************************************************************/
+
+#include "condorqxmlsaxhandler.h"
+#include "../core/tandem_run/tandemcondorprocess.h"
+#include <pappsomspp/pappsoexception.h>
+
+CondorQxmlSaxHandler::CondorQxmlSaxHandler(TandemCondorProcess * tandem_condor_process) {
+}
+CondorQxmlSaxHandler::~CondorQxmlSaxHandler() {
+}
+
+
+
+bool CondorQxmlSaxHandler::startElement(const QString & namespaceURI, const QString & localName,
+                                  const QString & qName, const QXmlAttributes & attributes) {
+    qDebug()<< "CondorQxmlSaxHandler::startElement begin" << namespaceURI << " " << localName << " " << qName ;
+    _tag_stack.push_back(qName);
+    bool is_ok = true;
+
+    try {
+        //startElement_group
+        /*
+        if (qName == "match") {
+            is_ok = startElement_match(attributes);
+        } else if (qName == "protein") {
+            is_ok = startElement_protein(attributes);
+        }
+*/
+        _current_text.clear();
+    }
+    catch (pappso::PappsoException exception_pappso) {
+        _errorStr = QObject::tr("ERROR in CondorQxmlSaxHandler::startElement tag %1, PAPPSO exception:\n%2").arg(qName).arg(exception_pappso.qwhat());
+        return false;
+    }
+    catch (std::exception exception_std) {
+        _errorStr = QObject::tr("ERROR in CondorQxmlSaxHandler::startElement tag %1, std exception:\n%2").arg(qName).arg(exception_std.what());
+        return false;
+    }
+    return is_ok;
+}
+
+bool CondorQxmlSaxHandler::endElement(const QString & namespaceURI, const QString & localName,
+                                const QString & qName) {
+
+    bool is_ok = true;
+    // endElement_peptide_list
+    try {
+        /*
+        if (qName == "protein")
+        {
+            is_ok = endElement_protein();
+        }
+        else if (qName == "identification") {
+            is_ok = endElement_identification();
+        }
+        */
+    }
+    catch (pappso::PappsoException exception_pappso) {
+        _errorStr = QObject::tr("ERROR in CondorQxmlSaxHandler::endElement tag %1, PAPPSO exception:\n%2").arg(qName).arg(exception_pappso.qwhat());
+        return false;
+    }
+    catch (std::exception exception_std) {
+        _errorStr = QObject::tr("ERROR in CondorQxmlSaxHandler::endElement tag %1, std exception:\n%2").arg(qName).arg(exception_std.what());
+        return false;
+    }
+
+    _current_text.clear();
+    _tag_stack.pop_back();
+
+    return is_ok;
+}
+
+
+bool CondorQxmlSaxHandler::error(const QXmlParseException &exception) {
+    _errorStr = QObject::tr("Parse error at line %1, column %2 :\n"
+                            "%3").arg(exception.lineNumber()).arg(exception.columnNumber()).arg(
+                    exception.message());
+
+    return false;
+}
+
+
+bool CondorQxmlSaxHandler::fatalError(const QXmlParseException &exception) {
+    _errorStr = QObject::tr("Parse error at line %1, column %2 :\n"
+                            "%3").arg(exception.lineNumber()).arg(exception.columnNumber()).arg(
+                    exception.message());
+    return false;
+}
+
+QString CondorQxmlSaxHandler::errorString() const {
+    return _errorStr;
+}
+
+
+bool CondorQxmlSaxHandler::endDocument() {
+    //_p_project->getProteinStore().setRegexpDecoyPattern(_p_project->getProteinStore().getRegexpDecoy().pattern());
+    return true;
+}
+
+bool CondorQxmlSaxHandler::startDocument() {
+    return true;
+}
+
+bool CondorQxmlSaxHandler::characters(const QString &str) {
+    _current_text += str;
+    return true;
+}
+
+/*
+package fr.inra.pappso.xtandempipeline.sax_parse;
+
+import org.apache.log4j.Logger;
+import org.xml.sax.Attributes;
+import org.xml.sax.SAXException;
+
+import fr.inra.pappso.xtandempipeline.MsException.MSMSException;
+import fr.inra.pappso.xtandempipeline.thread.XtandemCondorAnalysis;
+
+public class HandlerHTCondorQueue extends SaxBaseHandler {
+
+private static final Logger logger = Logger
+		.getLogger(HandlerHTCondorQueue.class);
+private Object inName;
+private Integer currentProcId;
+private Integer currentJobStatus;
+private String currentRemoteHost;
+private String currentLastRemoteHost;
+private int junexpanded = 0;
+private int jidle = 0;
+private int jrunning = 0;
+private int jremoved = 0;
+private int jcompleted = 0;
+private int jheld = 0;
+private int jsubmission_error = 0;
+private XtandemCondorAnalysis xtandemCondorAnalysis;
+private boolean empty = true;
+
+public HandlerHTCondorQueue(XtandemCondorAnalysis xtandemCondorAnalysis)
+		throws MSMSException {
+	super();
+	this.xtandemCondorAnalysis = xtandemCondorAnalysis;
+}
+
+@Override
+public void startDocument() throws SAXException {
+	this.empty = true;
+}
+
+@Override
+public void endDocument() throws SAXException {
+	int reallyCompletedJobs = 0;
+	if (this.empty) {
+		reallyCompletedJobs = xtandemCondorAnalysis.getCondorJobNumber();
+	} else {
+		int totalJobs = junexpanded + jidle + jrunning + jremoved
+				+ jcompleted + jheld + jsubmission_error;
+		int diffJobs = xtandemCondorAnalysis.getCondorJobNumber()
+				- totalJobs;
+		reallyCompletedJobs = jcompleted + diffJobs;
+	}
+
+	xtandemCondorAnalysis.setJobCompleted(junexpanded, jidle, jrunning,
+			jremoved, reallyCompletedJobs, jheld, jsubmission_error);
+
+	String statusMessage = "" + junexpanded + " unexpanded jobs\n";
+	statusMessage += jidle + " idle jobs\n";
+	statusMessage += jrunning + " running jobs\n";
+	statusMessage += jremoved + " removed jobs\n";
+	statusMessage += reallyCompletedJobs + " completed jobs\n";
+	statusMessage += jheld + " held jobs\n";
+	statusMessage += jsubmission_error + " submission_errors\n";
+
+	logger.debug(statusMessage);
+	xtandemCondorAnalysis.setStatusMessage(statusMessage);
+
+}
+
+// <c>
+// <a n="ProcId"><i>0</i></a>
+public void startElementc(String nameSpaceUri, Attributes attrs) {
+	this.empty = false;
+	currentProcId = null;
+	currentJobStatus = null;
+	currentRemoteHost = null;
+	currentLastRemoteHost = null;
+
+}
+
+public void startElementa(String nameSpaceUri, Attributes attrs) {
+	// logger.debug("startElementgroup begin");
+	// <group label="performance parameters" type="parameters">
+	this.inName = attrs.getValue("n");
+
+}
+
+public void endElementi(String nameSpaceUri, String data)
+		throws MSMSException {
+	if (inName.equals("ProcId")) {
+		this.currentProcId = new Integer(data);
+		// logger.debug(currentProcId);
+	} else if (inName.equals("JobStatus")) {
+		// <a n="JobStatus"><i>2</i></a>
+		this.currentJobStatus = new Integer(data);
+		// logger.debug(currentProcId);
+	}
+
+}
+
+public void endElements(String nameSpaceUri, String data)
+		throws MSMSException {
+	if (inName.equals("RemoteHost")) {
+		// <a n="RemoteHost"><s>slot1@proteus3</s></a>
+		this.currentRemoteHost = data;
+		// logger.debug(currentProcId);
+	} else if (inName.equals("LastRemoteHost")) {
+		// <a n="LastRemoteHost"><s>slot1@proteus4</s></a>
+		this.currentLastRemoteHost = data;
+		// logger.debug(currentProcId);
+	}
+
+}
+
+public void endElementa(String nameSpaceUri, String data) {
+	this.inName = "";
+}
+
+public void endElementc(String nameSpaceUri, String data)
+		throws MSMSException {
+	/*
+	 * 0 Unexpanded U 1 Idle I 2 Running R 3 Removed X 4 Completed C 5 Held
+	 * H 6 Submission_err E
+	 *
+	if (currentJobStatus == 0) {
+		this.junexpanded += 1;
+	} else if (currentJobStatus == 1) {
+		this.jidle += 1;
+	} else if (currentJobStatus == 2) {
+		this.jrunning += 1;
+	} else if (currentJobStatus == 3) {
+		this.jremoved += 1;
+	} else if (currentJobStatus == 4) {
+		this.jcompleted += 1;
+	} else if (currentJobStatus == 5) {
+		this.jheld += 1;
+	} else if (currentJobStatus == 6) {
+		this.jsubmission_error += 1;
+	}
+
+	logger.debug("currentRemoteHost:" + this.currentRemoteHost
+			+ " currentLastRemoteHost" + this.currentLastRemoteHost + " "
+			+ currentJobStatus);
+}
+
+}
+*/

src/input/condorqxmlsaxhandler.h

+/**
+ * \file input/condorqxmlsaxhandler.h
+ * \date 15/9/2017
+ * \author Olivier Langella
+ * \brief parse condor_q XML
+ */
+
+
+/*******************************************************************************
+* Copyright (c) 2017 Olivier Langella <Olivier.Langella@u-psud.fr>.
+*
+* This file is part of XTPcpp.
+*
+*     XTPcpp is free software: you can redistribute it and/or modify
+*     it under the terms of the GNU General Public License as published by
+*     the Free Software Foundation, either version 3 of the License, or
+*     (at your option) any later version.
+*
+*     XTPcpp is distributed in the hope that it will be useful,
+*     but WITHOUT ANY WARRANTY; without even the implied warranty of
+*     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*     GNU General Public License for more details.
+*
+*     You should have received a copy of the GNU General Public License
+*     along with XTPcpp.  If not, see <http://www.gnu.org/licenses/>.
+*
+* Contributors:
+*     Olivier Langella <Olivier.Langella@u-psud.fr> - initial API and implementation
+******************************************************************************/
+
+#ifndef CONDORQXMLSAXHANDLER_H
+#define CONDORQXMLSAXHANDLER_H
+
+#include <QXmlDefaultHandler>
+
+class TandemCondorProcess;
+
+class CondorQxmlSaxHandler: public QXmlDefaultHandler
+{
+public:
+    CondorQxmlSaxHandler(TandemCondorProcess * tandem_condor_process);
+    ~CondorQxmlSaxHandler();
+    
+    
+    bool startElement(const QString & namespaceURI, const QString & localName,
+                      const QString & qName, const QXmlAttributes & attributes);
+
+    bool endElement(const QString & namespaceURI, const QString & localName,
+                    const QString & qName);
+
+    bool startDocument();
+
+    bool endDocument();
+
+    bool characters(const QString &str);
+
+    bool fatalError(const QXmlParseException &exception);
+    bool error(const QXmlParseException &exception);
+
+    QString errorString() const;
+
+private:
+    TandemCondorProcess * _tandem_condor_process;
+    
+    std::vector<QString> _tag_stack;
+    QString _errorStr;
+    QString _current_text;
+
+};
+
+#endif // CONDORQXMLSAXHANDLER_H