From 16e7d124553d4985f146a200618a674a879132d0 Mon Sep 17 00:00:00 2001
From: Olivier Langella <olivier.langella@u-psud.fr>
Date: Wed, 13 Sep 2017 22:32:10 +0200
Subject: [PATCH] execute condor batch process
---
src/core/tandem_run/tandembatchprocess.cpp | 15 ++-
src/core/tandem_run/tandembatchprocess.h | 10 +-
src/core/tandem_run/tandemcondorprocess.cpp | 136 +++++++++++++++++++-
src/core/tandem_run/tandemcondorprocess.h | 6 +
4 files changed, 152 insertions(+), 15 deletions(-)
diff --git a/src/core/tandem_run/tandembatchprocess.cpp b/src/core/tandem_run/tandembatchprocess.cpp
index b6ecc7ed..29e15729 100644
--- a/src/core/tandem_run/tandembatchprocess.cpp
+++ b/src/core/tandem_run/tandembatchprocess.cpp
@@ -38,6 +38,8 @@ TandemBatchProcess::TandemBatchProcess(WorkMonitorInterface * p_monitor, const T
_p_monitor = p_monitor;
}
+TandemBatchProcess::~TandemBatchProcess() {
+}
void TandemBatchProcess::writeXmlDatabaseFile (QXmlStreamWriter * p_out) {
p_out->setAutoFormatting(true);
p_out->writeStartDocument("1.0");
@@ -65,18 +67,19 @@ void TandemBatchProcess::writeXmlDatabaseFile (QXmlStreamWriter * p_out) {
}
void TandemBatchProcess::prepareXmlDatabaseFile() {
- _xml_database_file.setAutoRemove(false);
+ _tmp_database_file.setAutoRemove(true);
- if (_xml_database_file.open())
+ if (_tmp_database_file.open())
{
+ _xml_database_file = QFileInfo( _tmp_database_file.fileName()).absoluteFilePath();
QXmlStreamWriter * p_out = new QXmlStreamWriter();
- p_out->setDevice(&_xml_database_file);
+ p_out->setDevice(&_tmp_database_file);
writeXmlDatabaseFile(p_out);
- _xml_database_file.close();
+ _tmp_database_file.close();
delete p_out;
} else
{
- throw pappso::PappsoException(QObject::tr("error : cannot open the XML database file : %1\n").arg(_xml_database_file.fileName()));
+ throw pappso::PappsoException(QObject::tr("error : cannot open the XML database file : %1\n").arg(_tmp_database_file.fileName()));
}
}
@@ -127,7 +130,7 @@ void TandemBatchProcess::writeXmlInputFile (QXmlStreamWriter * p_out, const QStr
p_out->writeStartElement("note");
p_out->writeAttribute("type","input");
p_out->writeAttribute("label","list path, taxonomy information");
- p_out->writeCharacters(_xml_database_file.fileName());
+ p_out->writeCharacters(_xml_database_file);
p_out->writeEndElement();
diff --git a/src/core/tandem_run/tandembatchprocess.h b/src/core/tandem_run/tandembatchprocess.h
index cda9bdc9..1ce5bb78 100644
--- a/src/core/tandem_run/tandembatchprocess.h
+++ b/src/core/tandem_run/tandembatchprocess.h
@@ -38,12 +38,13 @@ class TandemBatchProcess
{
public:
TandemBatchProcess(WorkMonitorInterface * p_monitor, const TandemRunBatch & tandem_run_batch);
+ virtual ~TandemBatchProcess();
virtual void run();
protected:
void writeXmlDatabaseFile (QXmlStreamWriter * p_out);
void writeXmlInputFile (QXmlStreamWriter * p_out, const QString & mz_file);
- void prepareXmlDatabaseFile();
+ virtual void prepareXmlDatabaseFile();
private:
void runOne(const QString & mz_file);
@@ -51,10 +52,13 @@ protected:
QString _preset_file;
TandemRunBatch _tandem_run_batch;
WorkMonitorInterface * _p_monitor;
- QTemporaryFile _xml_database_file;
+ QString _xml_database_file;
+ int _max_xt_time_ms = (60000 * 60 * 24); //1 day
private:
- int _max_xt_time_ms = (60000 * 60 * 24); //1 day
+
+
+ QTemporaryFile _tmp_database_file;
};
#endif // TANDEMBATCHPROCESS_H
diff --git a/src/core/tandem_run/tandemcondorprocess.cpp b/src/core/tandem_run/tandemcondorprocess.cpp
index b2252ab3..3b812692 100644
--- a/src/core/tandem_run/tandemcondorprocess.cpp
+++ b/src/core/tandem_run/tandemcondorprocess.cpp
@@ -30,6 +30,8 @@
#include "tandemcondorprocess.h"
#include <QDebug>
#include <pappsomspp/pappsoexception.h>
+#include <QSettings>
+#include <QProcess>
TandemCondorProcess::TandemCondorProcess(WorkMonitorInterface * p_monitor, const TandemRunBatch & tandem_run_batch) : TandemBatchProcess(p_monitor, tandem_run_batch) {
/*
@@ -46,14 +48,87 @@ TandemCondorProcess::TandemCondorProcess(WorkMonitorInterface * p_monitor, const
Arguments = /gorgone/pappso/tmp/temp_condor_job93294001891239208719639434471283743/xtandem_param2054956555872858570.xml
Queue
*/
+
+ QSettings settings;
+ QString condor_tmp_dir = QString("%1/xtpcpp").arg(settings.value("condor/tmp_dir", "/gorgone/pappso/temp").toString());
+ _p_tmp_dir = new QTemporaryDir(condor_tmp_dir);
+ _condor_submit_command = settings.value("condor/submit", "/usr/bin/condor_submit").toString();
+
+
+ if (!_p_tmp_dir->isValid()) {
+ // dir.path() returns the unique directory path
+ throw pappso::PappsoException(QObject::tr("problem creating condor temporary directory in %1\n").arg(condor_tmp_dir));
+ }
+
+
+}
+
+TandemCondorProcess::~TandemCondorProcess () {
+ delete _p_tmp_dir;
+}
+
+void TandemCondorProcess::prepareXmlDatabaseFile() {
+
+ QFile xml_database_file(QString("%1/database.xml").arg(_p_tmp_dir->path()));
+
+ if (xml_database_file.open(QIODevice::WriteOnly))
+ {
+ _xml_database_file = QFileInfo( xml_database_file.fileName()).absoluteFilePath();
+ QXmlStreamWriter * p_out = new QXmlStreamWriter();
+ p_out->setDevice(&xml_database_file);
+ writeXmlDatabaseFile(p_out);
+ xml_database_file.close();
+ delete p_out;
+ } else
+ {
+ throw pappso::PappsoException(QObject::tr("error : cannot open the XML database file : %1\n").arg(xml_database_file.fileName()));
+ }
+
}
void TandemCondorProcess::run() {
- qDebug() << "TandeCondorProcess::run begin " << _tandem_run_batch._preset_file;
+ qDebug() << "TandeCondorProcess::run begin ";
+ QFileInfo preset_info(_tandem_run_batch._preset_file);
+ _preset_file = QString("%1/%2").arg(_p_tmp_dir->path()).arg(preset_info.fileName());
+ QFile::copy(_tandem_run_batch._preset_file, _preset_file);
_preset_file = _tandem_run_batch._preset_file;
prepareXmlDatabaseFile();
+
+ //condor submit file :
+ QFile submit_file(QString("%1/submit.txt").arg(_p_tmp_dir->path()));
+ QTextStream * p_out = nullptr;
+
+ if (submit_file.open(QIODevice::WriteOnly))
+ {
+ p_out = new QTextStream();
+ p_out->setDevice(&submit_file);
+
+ *p_out << "Universe = vanilla" << endl;
+ *p_out << "notification = Error" << endl;
+ *p_out << "Rank = Mips" << endl;
+ *p_out << "request_memory= 50000" << endl;
+ *p_out << "request_cpus = 1" << endl;
+ *p_out << "Executable = " << _tandem_run_batch._tandem_bin_path << endl;
+ *p_out << "Log = \"" << _p_tmp_dir->path() << "/condor.log\"" << endl;
+ *p_out << "Output = \"" << _p_tmp_dir->path() << "/tandem.$(Process).out\"" << endl;
+ *p_out << "Error = \"" << _p_tmp_dir->path() << "/tandem.$(Process).error\"" << endl;
+ /*
+ Log = /gorgone/pappso/tmp/temp_condor_job93294001891239208719639434471283743/submit_condor.log
+ Output = /gorgone/pappso/tmp/temp_condor_job93294001891239208719639434471283743/tandem.$(Process).out
+ Error = /gorgone/pappso/tmp/temp_condor_job93294001891239208719639434471283743/tandem.$(Process).error
+
+ Arguments = /gorgone/pappso/tmp/temp_condor_job93294001891239208719639434471283743/xtandem_param2054956555872858570.xml
+ Queue
+ */
+ } else
+ {
+ throw pappso::PappsoException(QObject::tr("error : cannot open condor submit file : %1\n").arg(submit_file.fileName()));
+ }
+
+
+
std::vector<QTemporaryFile *> input_file_list;
int i=0;
@@ -61,19 +136,23 @@ void TandemCondorProcess::run() {
for (QString mz_file : _tandem_run_batch._mz_file_list) {
- QTemporaryFile * p_xml_input_file = new QTemporaryFile();
+ QTemporaryFile * p_xml_input_file = new QTemporaryFile(_p_tmp_dir->path());
input_file_list.push_back(p_xml_input_file);
p_xml_input_file->setAutoRemove(false);
if (p_xml_input_file->open())
{
- QXmlStreamWriter * p_out = new QXmlStreamWriter();
- p_out->setDevice(p_xml_input_file);
+ QXmlStreamWriter * p_xml_out = new QXmlStreamWriter();
+ p_xml_out->setDevice(p_xml_input_file);
- writeXmlInputFile(p_out, mz_file);
+ writeXmlInputFile(p_xml_out, mz_file);
p_xml_input_file->close();
- delete p_out;
+ delete p_xml_out;
+
+
+ *p_out << "Arguments = \"" << QFileInfo( p_xml_input_file->fileName()).absolutePath() << "\"" << endl;
+ *p_out << "Queue" << endl;
} else
{
throw pappso::PappsoException(QObject::tr("error : cannot open the XML X!Tandem input file : %1\n").arg(p_xml_input_file->fileName()));
@@ -83,5 +162,50 @@ void TandemCondorProcess::run() {
for (QTemporaryFile * p_xml_input_file: input_file_list) {
delete p_xml_input_file;
}
+
+
+ if (p_out != nullptr) {
+ submit_file.close();
+ delete p_out;
+ }
+
+
+ //now run condor job on submit_file
+
+
+ QStringList arguments;
+
+ arguments << QFileInfo( submit_file.fileName()).absoluteFilePath();
+
+ QProcess * condor_process = new QProcess();
+ //hk_process->setWorkingDirectory(QFileInfo(_hardklor_exe).absolutePath());
+ condor_process->start(_condor_submit_command, arguments);
+
+
+ if (!condor_process->waitForStarted()) {
+ throw pappso::PappsoException(QObject::tr("HTCondor X!Tandem process failed to start"));
+ }
+
+ while(condor_process->waitForReadyRead(_max_xt_time_ms)) {
+ _p_monitor->appendText( condor_process->readAll().data());
+ //data.append(xt_process->readAll());
+ }
+ /*
+ if (!xt_process->waitForFinished(_max_xt_time_ms)) {
+ throw pappso::PappsoException(QObject::tr("can't wait for X!Tandem process to finish : timeout at %1").arg(_max_xt_time_ms));
+ }
+ */
+ QByteArray result = condor_process->readAll();
+
+ QProcess::ExitStatus Status = condor_process->exitStatus();
+
+ if (Status != 0)
+ {
+ // != QProcess::NormalExit
+ throw pappso::PappsoException(QObject::tr("error executing HTCondor Status != 0 : %1 %2\n%3").arg(_tandem_run_batch._tandem_bin_path).arg(arguments.join(" ").arg(result.data())));
+ }
+
+ delete condor_process;
+
qDebug() << "TandeCondorProcess::run end" ;
}
diff --git a/src/core/tandem_run/tandemcondorprocess.h b/src/core/tandem_run/tandemcondorprocess.h
index a6bcc26a..2747c2b5 100644
--- a/src/core/tandem_run/tandemcondorprocess.h
+++ b/src/core/tandem_run/tandemcondorprocess.h
@@ -31,13 +31,19 @@
#define TANDEMCONDORPROCESS_H
#include "tandembatchprocess.h"
+#include <QTemporaryDir>
class TandemCondorProcess: public TandemBatchProcess
{
public:
TandemCondorProcess(WorkMonitorInterface * p_monitor, const TandemRunBatch & tandem_run_batch);
+ virtual ~TandemCondorProcess();
+ virtual void prepareXmlDatabaseFile();
virtual void run();
+private:
+ QTemporaryDir * _p_tmp_dir;
+ QString _condor_submit_command;
};
--
GitLab