config_reader.py 14.3 KB
Newer Older
1
import os
2
import re
3
import inspect
4
from pathlib import Path
5
from configparser import RawConfigParser, NoOptionError, NoSectionError
6
from dgenies.lib.decorators import Singleton
7
8


Floreal Cabanettes's avatar
Floreal Cabanettes committed
9
10
@Singleton
class AppConfigReader:
11
    """
Floreal Cabanettes's avatar
Floreal Cabanettes committed
12
    Store all configs
13
14
15
16
    """

    def __init__(self):
        """
Floreal Cabanettes's avatar
Floreal Cabanettes committed
17
18
        All "get_*" functions results are stored in the "self.*" corresponding attribute
        Example: results of the get_upload_folder function is stored in self.upload_folder
19
        """
20
        self.app_dir = os.path.dirname(inspect.getfile(self.__class__))
21
        config_file = []
22
23
24
25
26
27
28
29
30
        config_file_search = [os.path.join(os.path.abspath(os.sep), "dgenies", "application.properties"),
                              "/etc/dgenies/application.properties",
                              "/etc/dgenies/application.properties.local",
                              os.path.join(str(Path.home()), ".dgenies", "application.properties")]

        for my_config_file in config_file_search:
            if os.path.exists(my_config_file):
                config_file.append(my_config_file)
        if len(config_file) == 0:
31
32
            raise FileNotFoundError("ERROR: application.properties not found. Please copy the example file and "
                                    "check properties are correct for you!")
33
        self.reader = RawConfigParser()
34
        self.reader.read(config_file)
Floreal Cabanettes's avatar
Floreal Cabanettes committed
35
36
        for attr in dir(self):
            attr_o = getattr(self, attr)
37
            if attr.startswith("_get_") and callable(attr_o):
Floreal Cabanettes's avatar
Floreal Cabanettes committed
38
                try:
39
                    setattr(self, attr[5:], attr_o())
40
41
                except Exception as e:
                    print(e)
42

Floreal Cabanettes's avatar
Floreal Cabanettes committed
43
44
    def _replace_vars(self, path, config=False):
        new_path = path.replace("###USER###", os.path.expanduser("~"))\
45
            .replace("###PROGRAM###", os.path.dirname(os.path.dirname(os.path.realpath(__file__))))
Floreal Cabanettes's avatar
Floreal Cabanettes committed
46
47
48
49
50
51
52
53
54
55
56
57
        if "###CONFIG###" in new_path:
            if config:
                raise Exception("###CONFIG### tag not allowed for config dir")
            else:
                return new_path.replace("###CONFIG###", self._get_config_dir())
        return new_path

    def _get_config_dir(self):
        try:
            return self._replace_vars(self.reader.get("global", "config_dir"), True)
        except NoOptionError:
            return self._replace_vars("###USER###/.dgenies")
58

59
    def _get_upload_folder(self):
60
        try:
61
            return self._replace_vars(self.reader.get("global", "upload_folder"))
62
63
64
        except NoOptionError:
            raise Exception("No upload folder found in application.properties (global section)")

65
    def _get_app_data(self):
66
        try:
67
            return self._replace_vars(self.reader.get("global", "data_folder"))
68
        except NoOptionError:
69
70
            raise Exception("No data folder found in application.properties (global section)")

71
    def _get_batch_system_type(self):
72
73
74
75
76
        try:
            return self.reader.get("global", "batch_system_type")
        except NoOptionError:
            return "local"

77
    def _get_nb_threads(self):
78
        try:
79
            return self.reader.get("global", "threads_local")
80
81
82
        except NoOptionError:
            return "4"

83
    def _get_web_url(self):
84
        try:
85
            return self._replace_vars(self.reader.get("global", "web_url"))
86
87
88
        except NoOptionError:
            return "http://localhost:5000"

89
    def _get_max_upload_size(self):
90
        try:
91
            max_size_b = self._replace_vars(self.reader.get("global", "max_upload_size"))
92
93
            if max_size_b == "-1":
                return -1
94
            size_v = float(max_size_b[:-1])
95
96
97
            size_unit = max_size_b[-1].upper()
            if size_unit not in ["M", "G"]:
                raise ValueError("Max size unit must be M or G")
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
            max_size = int(size_v * 1024 * 1024)
            if size_unit == "G":
                max_size *= 1024
            return max_size
        except NoOptionError:
            return -1

    def _get_max_upload_size_ava(self):
        try:
            max_size_b = self._replace_vars(self.reader.get("global", "max_upload_size_ava"))
            if max_size_b == "-1":
                return -1
            size_v = float(max_size_b[:-1])
            size_unit = max_size_b[-1].upper()
            if size_unit not in ["M", "G"]:
                raise ValueError("Max size unit must be M or G")
114
            max_size = int(size_v * 1024 * 1024)
115
116
117
118
119
120
            if size_unit == "G":
                max_size *= 1024
            return max_size
        except NoOptionError:
            return -1

121
    def _get_max_upload_file_size(self):
122
        try:
123
            max_size_b = self._replace_vars(self.reader.get("global", "max_upload_file_size"))
124
125
126
127
128
129
130
131
132
133
134
135
136
            if max_size_b == "-1":
                return -1
            size_v = float(max_size_b[:-1])
            size_unit = max_size_b[-1].upper()
            if size_unit not in ["M", "G"]:
                raise ValueError("Max size unit must be M or G")
            max_size = int(size_v * 1024 * 1024)
            if size_unit == "G":
                max_size *= 1024
            return max_size
        except NoOptionError:
            return 1024 * 1024 * 1024

137
    def _get_minimap2_exec(self):
138
        try:
139
            entry = self.reader.get("softwares", "minimap2")
140
            return entry if entry != "###DEFAULT###" else os.path.join(self.app_dir, "bin", "minimap2")
141
        except NoOptionError:
142
            return os.path.join(self.app_dir, "bin", "minimap2")
143

144
    def _get_minimap2_cluster_exec(self):
145
146
147
148
        try:
            entry = self.reader.get("softwares", "minimap2_cluster")
            return entry if entry != "###DEFAULT###" else "minimap2"
        except NoOptionError:
149
            return self._get_minimap2_exec()
150

151
    def _get_database_type(self):
152
        try:
Floreal Cabanettes's avatar
Floreal Cabanettes committed
153
154
155
156
            return self.reader.get("database", "type")
        except NoOptionError:
            return "sqlite"

157
    def _get_database_url(self):
Floreal Cabanettes's avatar
Floreal Cabanettes committed
158
        try:
159
160
161
162
163
164
165
166
167
            url = self._replace_vars(self.reader.get("database", "url"))
            if self._get_database_type() == "sqlite" and url != ":memory:":
                parent_dir = os.path.dirname(url)
                if not os.path.exists(parent_dir):
                    try:
                        os.makedirs(parent_dir)
                    except FileNotFoundError:
                        pass
            return url
168
169
        except NoOptionError:
            return ":memory:"
170

171
    def _get_database_port(self):
Floreal Cabanettes's avatar
Floreal Cabanettes committed
172
173
174
        try:
            return int(self.reader.get("database", "port"))
        except (NoOptionError, ValueError):
175
            db_type = self._get_database_type()
Floreal Cabanettes's avatar
Floreal Cabanettes committed
176
177
178
179
180
181
            if db_type == "mysql":
                return 3306
            elif db_type == "sqlite":
                return -1
            raise Exception("Missing parameter: database port")

182
    def _get_database_db(self):
Floreal Cabanettes's avatar
Floreal Cabanettes committed
183
184
185
186
187
188
        try:
            db = self.reader.get("database", "db")
            if db == "":
                raise ValueError()
            return db
        except (NoOptionError, ValueError):
189
            if self._get_database_type() == "sqlite":
Floreal Cabanettes's avatar
Floreal Cabanettes committed
190
191
192
                return ""
            raise Exception("Missing parameter: database db name")

193
    def _get_database_user(self):
Floreal Cabanettes's avatar
Floreal Cabanettes committed
194
195
196
197
198
199
        try:
            user = self.reader.get("database", "user")
            if user == "":
                raise ValueError()
            return user
        except (NoOptionError, ValueError):
200
            if self._get_database_type() == "sqlite":
Floreal Cabanettes's avatar
Floreal Cabanettes committed
201
202
203
                return ""
            raise Exception("Missing parameter: database user")

204
    def _get_database_password(self):
Floreal Cabanettes's avatar
Floreal Cabanettes committed
205
206
207
208
209
210
        try:
            passwd = self.reader.get("database", "password")
            if passwd == "":
                raise ValueError()
            return passwd
        except (NoOptionError, ValueError):
211
            if self._get_database_type() == "sqlite":
Floreal Cabanettes's avatar
Floreal Cabanettes committed
212
213
214
                return ""
            raise Exception("Missing parameter: database password")

215
    def _get_mail_status_sender(self):
216
        try:
217
            return self._replace_vars(self.reader.get("mail", "status"))
218
219
220
        except NoOptionError:
            return "status@dgenies"

221
    def _get_mail_reply(self):
222
        try:
223
            return self._replace_vars(self.reader.get("mail", "reply"))
224
225
226
        except NoOptionError:
            return "status@dgenies"

227
    def _get_mail_org(self):
228
        try:
229
            return self._replace_vars(self.reader.get("mail", "org"))
230
231
232
        except NoOptionError:
            return None

233
    def _get_send_mail_status(self):
234
        try:
235
236
237
238
            return self.reader.get("mail", "send_mail_status").lower() == "true"
        except NoOptionError:
            return True

239
    def _get_disable_mail(self):
240
241
        try:
            return self.reader.get("mail", "disable").lower() == "true"
242
243
        except NoOptionError:
            return False
244

245
    def _get_cron_clean_time(self):
246
        try:
247
            value = self.reader.get("cron", "clean_time").lower()
248
            match = re.match(r"(([0-9])|([0-1][0-9])|(2[0-3]))[hH]([0-5][0-9])", value)
249
            if match is not None:
250
251
                return [int(match.group(1)), int(match.group(5))]
            else:
252
                print("Incorrect clean hour format!")
253
                return [1, 0]
254
255
256
        except (NoOptionError, NoSectionError):
            return [1, 0]

257
    def _get_cron_clean_freq(self):
258
        try:
259
            return int(self.reader.get("cron", "clean_freq"))
260
261
        except (NoOptionError, NoSectionError):
            return 1
262

263
    def _get_local_nb_runs(self):
264
265
266
267
        try:
            return int(self.reader.get("jobs", "run_local"))
        except (NoOptionError, NoSectionError):
            return 1
268

269
    def _get_nb_data_prepare(self):
270
271
272
273
        try:
            return int(self.reader.get("jobs", "data_prepare"))
        except (NoOptionError, NoSectionError):
            return 2
274

275
    def _get_max_concurrent_dl(self):
276
277
278
279
280
        try:
            return int(self.reader.get("jobs", "max_concurrent_dl"))
        except (NoOptionError, NoSectionError):
            return 5

281
    def _get_drmaa_lib_path(self):
282
        try:
283
284
285
286
            path = self.reader.get("cluster", "drmaa_lib_path")
            if path != "###SET_IT###":
                return path
            return None
287
        except (NoOptionError, NoSectionError):
288
            if self._get_batch_system_type() != "local":
289
290
291
                raise Exception("No drmaa library set. It is required if the batch system type is not 'local'")
            return None

292
    def _get_drmaa_native_specs(self):
293
294
295
296
        try:
            return self.reader.get("cluster", "native_specs")
        except (NoOptionError, NoSectionError):
            return "###DEFAULT###"
297

298
    def _get_max_run_local(self):
299
        try:
Floreal Cabanettes's avatar
Floreal Cabanettes committed
300
            return int(self.reader.get("cluster", "max_run_local"))
301
        except (NoOptionError, NoSectionError):
302
303
304
305
306
307
308
            return 10

    def _get_max_wait_local(self):
        try:
            return int(self.reader.get("cluster", "max_wait_local"))
        except (NoOptionError, NoSectionError):
            return 5
309

310
    def _get_min_query_size(self):
311
312
313
314
315
316
317
318
319
320
321
322
323
        try:
            size_b = self.reader.get("cluster", "min_query_size")
            size_v = int(size_b[:-1])
            size_unit = size_b[-1].upper()
            if size_unit not in ["M", "G"]:
                raise ValueError("Min query size unit must be M or G")
            min_size = size_v * 1024 * 1024
            if size_unit == "G":
                min_size *= 1024
            return min_size
        except (NoOptionError, NoSectionError):
            return 0

324
    def _get_min_target_size(self):
325
326
327
328
329
330
331
332
333
334
335
336
        try:
            size_b = self.reader.get("cluster", "min_target_size")
            size_v = int(size_b[:-1])
            size_unit = size_b[-1].upper()
            if size_unit not in ["M", "G"]:
                raise ValueError("Min query size unit must be M or G")
            min_size = size_v * 1024 * 1024
            if size_unit == "G":
                min_size *= 1024
            return min_size
        except (NoOptionError, NoSectionError):
            return 0
Floreal Cabanettes's avatar
Floreal Cabanettes committed
337

338
339
340
341
342
343
    def _get_cluster_prepare_script(self):
        try:
            return self._replace_vars(self.reader.get("cluster", "prepare_script"))
        except (NoOptionError, NoSectionError):
            return self._replace_vars("###PROGRAM###/bin/prepare_data.sh")

344
    def _get_cluster_python_exec(self):
345
        try:
346
            return self._replace_vars(self.reader.get("cluster", "python3_exec"))
347
348
349
        except (NoOptionError, NoSectionError):
            return "python3"

350
351
352
353
354
355
356
357
358
    def _get_cluster_memory(self):
        try:
            memory = int(self.reader.get("cluster", "memory"))
            if memory % self._get_cluster_threads() != 0:
                raise ValueError("ERROR in config: cluster memory must be divisible by the number of cluster threads!")
            return memory
        except (NoOptionError, NoSectionError):
            return 32

359
360
361
362
363
364
365
366
367
    def _get_cluster_memory_ava(self):
        try:
            memory = int(self.reader.get("cluster", "memory_ava"))
            if memory % self._get_cluster_threads() != 0:
                raise ValueError("ERROR in config: cluster memory must be divisible by the number of cluster threads!")
            return memory
        except (NoOptionError, NoSectionError):
            return self._get_cluster_memory()

368
369
370
371
372
373
    def _get_cluster_threads(self):
        try:
            return int(self.reader.get("cluster", "threads"))
        except (NoOptionError, NoSectionError):
            return 4

374
    def _get_debug(self):
Floreal Cabanettes's avatar
Floreal Cabanettes committed
375
376
377
378
379
        try:
            return self.reader.get("debug", "enable").lower() == "true"
        except (NoOptionError, NoSectionError):
            return False

380
    def _get_log_dir(self):
Floreal Cabanettes's avatar
Floreal Cabanettes committed
381
        try:
382
            log_dir = self._replace_vars(self.reader.get("debug", "log_dir"))
Floreal Cabanettes's avatar
Floreal Cabanettes committed
383
        except (NoOptionError, NoSectionError):
Floreal Cabanettes's avatar
Floreal Cabanettes committed
384
            log_dir = self._replace_vars("###CONFIG###/logs")
385
386
387
388
389
        if not os.path.exists(log_dir):
            os.makedirs(log_dir)
        elif not os.path.isdir(log_dir):
            raise TypeError("Log dir must be a directory")
        return log_dir
Floreal Cabanettes's avatar
Floreal Cabanettes committed
390

391
    def _get_allowed_ip_tests(self):
Floreal Cabanettes's avatar
Floreal Cabanettes committed
392
393
394
395
396
397
398
399
        allowed_ip = {"127.0.0.1"}
        try:
            allowed_ip_txt = self.reader.get("debug", "allowed_ip_tests")
            for ip in re.split(r",(\s+)?", allowed_ip_txt):
                allowed_ip.add(ip)
        except (NoOptionError, NoSectionError):
            pass
        return allowed_ip