Commit 9d218452 authored by Floreal Cabanettes's avatar Floreal Cabanettes
Browse files

Allow memory for cluster to be different in all-vs-all mode, and start make it...

Allow memory for cluster to be different in all-vs-all mode, and start make it adaptive to genome length in this mode
parent 50fcc605
...@@ -48,7 +48,10 @@ min_target_size = 700M ...@@ -48,7 +48,10 @@ min_target_size = 700M
prepare_script = ###PROGRAM###/bin/all_prepare.py prepare_script = ###PROGRAM###/bin/all_prepare.py
python3_script = python3 python3_script = python3
# Max memory:
memory = 32 memory = 32
# Max memory for all-vs-all mode:
memory_ava = 32
threads = 4 threads = 4
......
...@@ -340,6 +340,15 @@ class AppConfigReader: ...@@ -340,6 +340,15 @@ class AppConfigReader:
except (NoOptionError, NoSectionError): except (NoOptionError, NoSectionError):
return 32 return 32
def _get_cluster_memory_ava(self):
try:
memory = int(self.reader.get("cluster", "memory_ava"))
if memory % self._get_cluster_threads() != 0:
raise ValueError("ERROR in config: cluster memory must be divisible by the number of cluster threads!")
return memory
except (NoOptionError, NoSectionError):
return self._get_cluster_memory()
def _get_cluster_threads(self): def _get_cluster_threads(self):
try: try:
return int(self.reader.get("cluster", "threads")) return int(self.reader.get("cluster", "threads"))
......
...@@ -18,7 +18,7 @@ import traceback ...@@ -18,7 +18,7 @@ import traceback
from pathlib import Path from pathlib import Path
from urllib import request, parse from urllib import request, parse
from dgenies.bin.split_fa import Splitter from dgenies.bin.split_fa import Splitter
from dgenies.bin.index import index_file from dgenies.bin.index import index_file, Index
from dgenies.bin.filter_contigs import Filter from dgenies.bin.filter_contigs import Filter
from dgenies.bin.merge_splitted_chrms import Merger from dgenies.bin.merge_splitted_chrms import Merger
from dgenies.bin.sort_paf import Sorter from dgenies.bin.sort_paf import Sorter
...@@ -291,9 +291,15 @@ class JobManager: ...@@ -291,9 +291,15 @@ class JobManager:
if step == "prepare": if step == "prepare":
jt.nativeSpecification = native_specs.format(8000, 1, "02:00:00") jt.nativeSpecification = native_specs.format(8000, 1, "02:00:00")
elif step == "start": elif step == "start":
jt.nativeSpecification = native_specs.format( memory = self.config.cluster_memory
self.config.cluster_memory // self.config.cluster_threads * 1000, self.config.cluster_threads, if self.query is None:
"02:00:00") memory = self.config.cluster_memory_ava
if memory > 32:
name, order, contigs, reversed_c, abs_start, c_len = Index.load(self.idx_t, False)
if c_len <= 500000000:
memory = 32
jt.nativeSpecification = native_specs.format(memory // self.config.cluster_threads * 1000,
self.config.cluster_threads, "02:00:00")
elif batch_system_type == "sge": elif batch_system_type == "sge":
if native_specs == "###DEFAULT###": if native_specs == "###DEFAULT###":
native_specs = "-l mem={0},h_vmem={0} -pe parallel_smp {1}" native_specs = "-l mem={0},h_vmem={0} -pe parallel_smp {1}"
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment