From 9d218452e787c9b46c126aa27d1249fae2c2ef04 Mon Sep 17 00:00:00 2001
From: Floreal Cabanettes <floreal.cabanettes@inra.fr>
Date: Fri, 2 Feb 2018 17:16:22 +0100
Subject: [PATCH] Allow memory for cluster to be different in all-vs-all mode,
and start make it adaptive to genome length in this mode
---
application.properties | 3 +++
src/dgenies/config_reader.py | 9 +++++++++
src/dgenies/lib/job_manager.py | 14 ++++++++++----
3 files changed, 22 insertions(+), 4 deletions(-)
diff --git a/application.properties b/application.properties
index b6957bd..2f03f88 100644
--- a/application.properties
+++ b/application.properties
@@ -48,7 +48,10 @@ min_target_size = 700M
prepare_script = ###PROGRAM###/bin/all_prepare.py
python3_script = python3
+# Max memory:
memory = 32
+# Max memory for all-vs-all mode:
+memory_ava = 32
threads = 4
diff --git a/src/dgenies/config_reader.py b/src/dgenies/config_reader.py
index 9b0c9ef..a5c54a8 100644
--- a/src/dgenies/config_reader.py
+++ b/src/dgenies/config_reader.py
@@ -340,6 +340,15 @@ class AppConfigReader:
except (NoOptionError, NoSectionError):
return 32
+ def _get_cluster_memory_ava(self):
+ try:
+ memory = int(self.reader.get("cluster", "memory_ava"))
+ if memory % self._get_cluster_threads() != 0:
+ raise ValueError("ERROR in config: cluster memory must be divisible by the number of cluster threads!")
+ return memory
+ except (NoOptionError, NoSectionError):
+ return self._get_cluster_memory()
+
def _get_cluster_threads(self):
try:
return int(self.reader.get("cluster", "threads"))
diff --git a/src/dgenies/lib/job_manager.py b/src/dgenies/lib/job_manager.py
index 31edf0f..bcf6624 100644
--- a/src/dgenies/lib/job_manager.py
+++ b/src/dgenies/lib/job_manager.py
@@ -18,7 +18,7 @@ import traceback
from pathlib import Path
from urllib import request, parse
from dgenies.bin.split_fa import Splitter
-from dgenies.bin.index import index_file
+from dgenies.bin.index import index_file, Index
from dgenies.bin.filter_contigs import Filter
from dgenies.bin.merge_splitted_chrms import Merger
from dgenies.bin.sort_paf import Sorter
@@ -291,9 +291,15 @@ class JobManager:
if step == "prepare":
jt.nativeSpecification = native_specs.format(8000, 1, "02:00:00")
elif step == "start":
- jt.nativeSpecification = native_specs.format(
- self.config.cluster_memory // self.config.cluster_threads * 1000, self.config.cluster_threads,
- "02:00:00")
+ memory = self.config.cluster_memory
+ if self.query is None:
+ memory = self.config.cluster_memory_ava
+ if memory > 32:
+ name, order, contigs, reversed_c, abs_start, c_len = Index.load(self.idx_t, False)
+ if c_len <= 500000000:
+ memory = 32
+ jt.nativeSpecification = native_specs.format(memory // self.config.cluster_threads * 1000,
+ self.config.cluster_threads, "02:00:00")
elif batch_system_type == "sge":
if native_specs == "###DEFAULT###":
native_specs = "-l mem={0},h_vmem={0} -pe parallel_smp {1}"
--
GitLab