Commit 9d218452 authored by Floreal Cabanettes's avatar Floreal Cabanettes
Browse files

Allow memory for cluster to be different in all-vs-all mode, and start make it...

Allow memory for cluster to be different in all-vs-all mode, and start make it adaptive to genome length in this mode
parent 50fcc605
......@@ -48,7 +48,10 @@ min_target_size = 700M
prepare_script = ###PROGRAM###/bin/all_prepare.py
python3_script = python3
# Max memory:
memory = 32
# Max memory for all-vs-all mode:
memory_ava = 32
threads = 4
......
......@@ -340,6 +340,15 @@ class AppConfigReader:
except (NoOptionError, NoSectionError):
return 32
def _get_cluster_memory_ava(self):
try:
memory = int(self.reader.get("cluster", "memory_ava"))
if memory % self._get_cluster_threads() != 0:
raise ValueError("ERROR in config: cluster memory must be divisible by the number of cluster threads!")
return memory
except (NoOptionError, NoSectionError):
return self._get_cluster_memory()
def _get_cluster_threads(self):
try:
return int(self.reader.get("cluster", "threads"))
......
......@@ -18,7 +18,7 @@ import traceback
from pathlib import Path
from urllib import request, parse
from dgenies.bin.split_fa import Splitter
from dgenies.bin.index import index_file
from dgenies.bin.index import index_file, Index
from dgenies.bin.filter_contigs import Filter
from dgenies.bin.merge_splitted_chrms import Merger
from dgenies.bin.sort_paf import Sorter
......@@ -291,9 +291,15 @@ class JobManager:
if step == "prepare":
jt.nativeSpecification = native_specs.format(8000, 1, "02:00:00")
elif step == "start":
jt.nativeSpecification = native_specs.format(
self.config.cluster_memory // self.config.cluster_threads * 1000, self.config.cluster_threads,
"02:00:00")
memory = self.config.cluster_memory
if self.query is None:
memory = self.config.cluster_memory_ava
if memory > 32:
name, order, contigs, reversed_c, abs_start, c_len = Index.load(self.idx_t, False)
if c_len <= 500000000:
memory = 32
jt.nativeSpecification = native_specs.format(memory // self.config.cluster_threads * 1000,
self.config.cluster_threads, "02:00:00")
elif batch_system_type == "sge":
if native_specs == "###DEFAULT###":
native_specs = "-l mem={0},h_vmem={0} -pe parallel_smp {1}"
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment