Commit 3b65b7ff authored by Floreal Cabanettes's avatar Floreal Cabanettes
Browse files

Make amount of memory and number of threads customizable for mapping cluster submissions

parent 2faefb0d
......@@ -6,7 +6,7 @@ data_folder = ###CONFIG###/data
# batch system type: local, sge, slurm
batch_system_type = local
threads = 4
threads_local = 4
web_url = http://localhost:5000
# Max size of uploaded files (also for files from URL, size uncompressed):
......@@ -48,6 +48,8 @@ min_target_size = 700M
prepare_script = ###PROGRAM###/bin/all_prepare.py
python3_script = python3
memory = 32
threads = 4
[softwares]
......
......@@ -76,7 +76,7 @@ class AppConfigReader:
def _get_nb_threads(self):
try:
return self.reader.get("global", "threads")
return self.reader.get("global", "threads_local")
except NoOptionError:
return "4"
......@@ -331,6 +331,21 @@ class AppConfigReader:
except (NoOptionError, NoSectionError):
return "python3"
def _get_cluster_memory(self):
try:
memory = int(self.reader.get("cluster", "memory"))
if memory % self._get_cluster_threads() != 0:
raise ValueError("ERROR in config: cluster memory must be divisible by the number of cluster threads!")
return memory
except (NoOptionError, NoSectionError):
return 32
def _get_cluster_threads(self):
try:
return int(self.reader.get("cluster", "threads"))
except (NoOptionError, NoSectionError):
return 4
def _get_debug(self):
try:
return self.reader.get("debug", "enable").lower() == "true"
......
......@@ -290,14 +290,17 @@ class JobManager:
if step == "prepare":
jt.nativeSpecification = native_specs.format(8000, 1, "02:00:00")
elif step == "start":
jt.nativeSpecification = native_specs.format(8000, 4, "02:00:00")
jt.nativeSpecification = native_specs.format(
self.config.cluster_memory / self.config.cluster_threads * 1000, self.config.cluster_threads,
"02:00:00")
elif batch_system_type == "sge":
if native_specs == "###DEFAULT###":
native_specs = "-l mem={0},h_vmem={0} -pe parallel_smp {1}"
if step == "prepare":
jt.nativeSpecification = native_specs.format(8000, 1)
elif step == "start":
jt.nativeSpecification = native_specs.format(8000, 4)
jt.nativeSpecification = native_specs.format(
self.config.cluster_memory / self.config.cluster_threads * 1000, self.config.cluster_threads)
jt.workingDirectory = self.output_dir
jobid = s.runJob(jt)
self.id_process = jobid
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment