Allow memory for cluster to be different in all-vs-all mode, and start make it...

Allow memory for cluster to be different in all-vs-all mode, and start make it adaptive to genome length in this mode

application.properties

@@ -48,7 +48,10 @@ min_target_size = 700M
 
 prepare_script = ###PROGRAM###/bin/all_prepare.py
 python3_script = python3
+# Max memory:
 memory = 32
+# Max memory for all-vs-all mode:
+memory_ava = 32
 threads = 4
 
 

src/dgenies/config_reader.py

@@ -340,6 +340,15 @@ class AppConfigReader:
         except (NoOptionError, NoSectionError):
             return 32
 
+    def _get_cluster_memory_ava(self):
+        try:
+            memory = int(self.reader.get("cluster", "memory_ava"))
+            if memory % self._get_cluster_threads() != 0:
+                raise ValueError("ERROR in config: cluster memory must be divisible by the number of cluster threads!")
+            return memory
+        except (NoOptionError, NoSectionError):
+            return self._get_cluster_memory()
+
     def _get_cluster_threads(self):
         try:
             return int(self.reader.get("cluster", "threads"))

src/dgenies/lib/job_manager.py

@@ -18,7 +18,7 @@ import traceback
 from pathlib import Path
 from urllib import request, parse
 from dgenies.bin.split_fa import Splitter
-from dgenies.bin.index import index_file
+from dgenies.bin.index import index_file, Index
 from dgenies.bin.filter_contigs import Filter
 from dgenies.bin.merge_splitted_chrms import Merger
 from dgenies.bin.sort_paf import Sorter
@@ -291,9 +291,15 @@ class JobManager:
             if step == "prepare":
                 jt.nativeSpecification = native_specs.format(8000, 1, "02:00:00")
             elif step == "start":
-                jt.nativeSpecification = native_specs.format(
-                    self.config.cluster_memory // self.config.cluster_threads * 1000, self.config.cluster_threads,
-                    "02:00:00")
+                memory = self.config.cluster_memory
+                if self.query is None:
+                    memory = self.config.cluster_memory_ava
+                    if memory > 32:
+                        name, order, contigs, reversed_c, abs_start, c_len = Index.load(self.idx_t, False)
+                        if c_len <= 500000000:
+                            memory = 32
+                jt.nativeSpecification = native_specs.format(memory // self.config.cluster_threads * 1000,
+                                                             self.config.cluster_threads, "02:00:00")
         elif batch_system_type == "sge":
             if native_specs == "###DEFAULT###":
                 native_specs = "-l mem={0},h_vmem={0} -pe parallel_smp {1}"