external_source.hpp 16.7 KB
Newer Older
1
// Copyright (c) 2020-2021 INRA Distributed under the Boost Software License,
2
3
4
5
6
7
8
9
// Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
// http://www.boost.org/LICENSE_1_0.txt)

#ifndef ORG_VLEPROJECT_IRRITATOR_EXTERNAL_SOURCE_2021
#define ORG_VLEPROJECT_IRRITATOR_EXTERNAL_SOURCE_2021

#include <irritator/core.hpp>

Gauthier Quesnel's avatar
Gauthier Quesnel committed
10
#include <filesystem>
11
#include <fstream>
12
13
14
15
#include <iomanip>
#include <map>
#include <random>
#include <vector>
16

17
namespace irt {
18

19
enum class external_source_type
Gauthier Quesnel's avatar
Gauthier Quesnel committed
20
{
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
    binary_file,
    constant,
    random,
    text_file
};

enum class block_vector_policy
{
    not_reuse_free_list,
    reuse_free_list
};

template<block_vector_policy P>
struct block_vector_base
{
    double* buffer = nullptr;
    sz size = 0;
    sz max_size = 0;
    sz capacity = 0;
    sz block_size = 0;
    sz free_head = static_cast<sz>(-1);

    block_vector_base() = default;
Gauthier Quesnel's avatar
Gauthier Quesnel committed
44

45
46
47
48
    block_vector_base(const block_vector_base&) = delete;
    block_vector_base& operator=(const block_vector_base&) = delete;

    ~block_vector_base() noexcept
49
    {
50
51
52
        if (buffer)
            g_free_fn(buffer);
    }
53

54
55
56
    bool empty() const noexcept
    {
        return size == 0;
57
58
    }

59
    status init(sz block_size_, sz capacity_) noexcept
Gauthier Quesnel's avatar
Gauthier Quesnel committed
60
    {
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
        if (capacity_ == 0)
            return status::block_allocator_bad_capacity;

        if (capacity_ != capacity) {
            if (buffer)
                g_free_fn(buffer);

            buffer = static_cast<double*>(g_alloc_fn(capacity_ * block_size_));
            if (buffer == nullptr)
                return status::block_allocator_not_enough_memory;
        }

        size = 0;
        max_size = 0;
        capacity = capacity_;
        block_size = block_size_;
        free_head = static_cast<sz>(-1);

        return status::success;
    }

    bool can_alloc() const noexcept
    {
        if constexpr (P == block_vector_policy::reuse_free_list)
            return free_head != static_cast<sz>(-1) || max_size < capacity;
        else
            return max_size < capacity;
    }

    double* alloc() noexcept
    {
        sz new_block;

        if constexpr (P == block_vector_policy::reuse_free_list) {
            if (free_head != static_cast<sz>(-1)) {
                new_block = free_head;
                free_head = static_cast<sz>(buffer[free_head * block_size]);
            } else {
                new_block = max_size * block_size;
                ++max_size;
            }
        } else {
            new_block = max_size * block_size;
            ++max_size;
        }

        ++size;
        return &buffer[new_block];
    }

    void free([[maybe_unused]] double* block) noexcept
    {
        if constexpr (P == block_vector_policy::reuse_free_list) {
            auto ptr_diff = buffer - block;
            auto block_index = ptr_diff / block_size;

            block[0] = static_cast<double>(free_head);
            free_head = static_cast<sz>(block_index);
            --size;

            if (size == 0) {
                max_size = 0;
                free_head = static_cast<sz>(-1);
            }
        }
    }
};

using limited_block_vector =
  block_vector_base<block_vector_policy::not_reuse_free_list>;
using block_vector = block_vector_base<block_vector_policy::reuse_free_list>;

inline bool
external_source_type_cast(int value, external_source_type* type) noexcept
{
    if (value < 0 || value > 3)
        return false;

    *type = enum_cast<external_source_type>(value);
    return true;
}

143
144
145
146
147
148
149
150
151
152
static inline const char* external_source_type_string[] = { "binary_file",
                                                            "constant",
                                                            "random",
                                                            "text_file" };

inline const char*
external_source_str(const external_source_type type) noexcept
{
    return external_source_type_string[static_cast<int>(type)];
}
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174

enum class distribution_type
{
    bernouilli,
    binomial,
    cauchy,
    chi_squared,
    exponential,
    exterme_value,
    fisher_f,
    gamma,
    geometric,
    lognormal,
    negative_binomial,
    normal,
    poisson,
    student_t,
    uniform_int,
    uniform_real,
    weibull,
};

175
static const char* distribution_type_string[] = {
176
177
178
179
180
181
    "bernouilli",        "binomial", "cauchy",  "chi_squared", "exponential",
    "exterme_value",     "fisher_f", "gamma",   "geometric",   "lognormal",
    "negative_binomial", "normal",   "poisson", "student_t",   "uniform_int",
    "uniform_real",      "weibull"
};

182
183
184
185
186
187
188
inline const char*
distribution_str(const distribution_type type) noexcept
{

    return distribution_type_string[static_cast<int>(type)];
}

189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
struct constant_source
{
    small_string<23> name;
    std::vector<double> buffer;

    status init(sz block_size) noexcept
    {
        try {
            buffer.reserve(block_size);
            return status::success;
        } catch (const std::bad_alloc& /*e*/) {
            return status::gui_not_enough_memory;
        }
    }

    status start_or_restart() noexcept
    {}

    status update(source& src) noexcept
    {
        src.buffer = buffer.data();
        src.size = static_cast<int>(buffer.size());
Gauthier Quesnel's avatar
Gauthier Quesnel committed
211
        src.index = 0;
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234

        return status::success;
    }

    status finalize(source& src) noexcept
    {
        src.clear();

        return status::success;
    }

    status operator()(source& src, source::operation_type op) noexcept
    {
        switch (op) {
        case source::operation_type::initialize:
            return update(src);
        case source::operation_type::update:
            return update(src);
        case source::operation_type::finalize:
            return finalize(src);
        }

        irt_unreachable();
Gauthier Quesnel's avatar
Gauthier Quesnel committed
235
236
237
    }
};

238
struct binary_file_source
239
{
240
241
    small_string<23> name;
    limited_block_vector buffer;
Gauthier Quesnel's avatar
Gauthier Quesnel committed
242
    std::filesystem::path file_path;
243
    std::ifstream ifs;
244
    sz size = 0; // Number of double read
245

246
    status init(sz block_size, sz capacity) noexcept
247
    {
248
249
250
251
252
253
254
        file_path.clear();

        if (ifs.is_open())
            ifs.close();

        return buffer.init(block_size, capacity);
    }
255

256
257
    status start_or_restart() noexcept
    {
Gauthier Quesnel's avatar
Gauthier Quesnel committed
258
259
260
261
        if (!ifs) {
            ifs.open(file_path);

            if (!ifs)
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
                return status::source_empty;
        } else {
            ifs.seekg(0);
        }

        return fill_buffer();
    }

    status update(source& src)
    {
        if (!buffer.can_alloc()) {
            if (src.buffer)
                buffer.free(src.buffer);

            irt_return_if_bad(fill_buffer());
Gauthier Quesnel's avatar
Gauthier Quesnel committed
277
278
        }

279
280
281
        src.buffer = buffer.alloc();
        src.size = static_cast<int>(buffer.block_size);
        src.index = 0;
282

283
        return status::success;
284
285
    }

286
    status finalize(source& src)
287
    {
288
289
        if (src.buffer)
            buffer.free(src.buffer);
290

291
        src.clear();
292

293
294
        return status::success;
    }
295

296
297
298
299
300
301
302
303
304
305
306
307
    status operator()(source& src, source::operation_type op)
    {
        switch (op) {
        case source::operation_type::initialize:
            return update(src);
        case source::operation_type::update:
            return update(src);
        case source::operation_type::finalize:
            return finalize(src);
        }

        irt_unreachable();
308
309
310
    }

private:
311
    status fill_buffer() noexcept
312
    {
313
314
315
        ifs.read(reinterpret_cast<char*>(buffer.buffer),
                 buffer.capacity * buffer.block_size);
        const auto read = ifs.gcount();
316

317
318
319
        buffer.capacity = static_cast<sz>(read) / buffer.block_size;
        if (buffer.capacity == 0)
            return status::source_empty;
320

321
        return status::success;
322
323
324
    }
};

325
struct text_file_source
326
{
327
328
    small_string<23> name;
    limited_block_vector buffer;
Gauthier Quesnel's avatar
Gauthier Quesnel committed
329
    std::filesystem::path file_path;
330
331
    std::ifstream ifs;

332
    status init(sz block_size, sz capacity) noexcept
333
    {
334
335
        return buffer.init(block_size, capacity);
        file_path.clear();
336

337
338
339
340
341
342
        if (ifs.is_open())
            ifs.close();
    }

    status start_or_restart() noexcept
    {
Gauthier Quesnel's avatar
Gauthier Quesnel committed
343
344
345
346
        if (!ifs) {
            ifs.open(file_path);

            if (!ifs)
347
348
349
                return status::source_empty;
        } else {
            ifs.seekg(0);
Gauthier Quesnel's avatar
Gauthier Quesnel committed
350
351
        }

352
353
354
355
356
357
358
359
360
361
362
        return fill_buffer();
    }

    status update(source& src)
    {
        if (!buffer.can_alloc()) {
            if (src.buffer)
                buffer.free(src.buffer);

            irt_return_if_bad(fill_buffer());
        }
363

364
365
366
367
368
        src.buffer = buffer.alloc();
        src.size = static_cast<int>(buffer.block_size);
        src.index = 0;

        return status::success;
369
370
    }

371
    status finalize(source& src)
372
    {
373
374
        if (src.buffer)
            buffer.free(src.buffer);
375

376
377
378
379
        src.clear();

        return status::success;
    }
380

381
382
383
384
385
386
387
388
389
390
    status operator()(source& src, source::operation_type op)
    {
        switch (op) {
        case source::operation_type::initialize:
            return update(src);
        case source::operation_type::update:
            return update(src);
        case source::operation_type::finalize:
            return finalize(src);
        }
391

392
        irt_unreachable();
393
394
395
    }

private:
396
    status fill_buffer() noexcept
397
398
    {
        size_t i = 0;
399
400
401
        const size_t e = buffer.capacity * buffer.block_size;
        for (; i < e && ifs.good(); ++i) {
            if (!(ifs >> buffer.buffer[i]))
402
403
404
                break;
        }

405
406
407
        buffer.capacity = i / buffer.block_size;
        if (buffer.capacity == 0)
            return status::source_empty;
408

409
        return status::success;
410
411
412
413
414
    }
};

struct random_source
{
415
416
417
418
419
    small_string<23> name;
    block_vector buffer;
    distribution_type distribution = distribution_type::uniform_int;
    double a, b, p, mean, lambda, alpha, beta, stddev, m, s, n;
    int a32, b32, t32, k32;
420

421
422
423
424
425
426
427
428
429
    template<typename RandomGenerator, typename Distribution>
    void generate(RandomGenerator& gen,
                  Distribution dist,
                  double* ptr,
                  sz size) noexcept
    {
        for (sz i = 0; i < size; ++i)
            ptr[i] = dist(gen);
    }
430

431
432
    template<typename RandomGenerator>
    void generate(RandomGenerator& gen, double* ptr, sz size) noexcept
433
    {
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
        switch (distribution) {
        case distribution_type::uniform_int:
            generate(gen, std::uniform_int_distribution(a32, b32), ptr, size);
            break;

        case distribution_type::uniform_real:
            generate(gen, std::uniform_real_distribution(a, b), ptr, size);
            break;

        case distribution_type::bernouilli:
            generate(gen, std::bernoulli_distribution(p), ptr, size);
            break;

        case distribution_type::binomial:
            generate(gen, std::binomial_distribution(t32, p), ptr, size);
            break;

        case distribution_type::negative_binomial:
            generate(
              gen, std::negative_binomial_distribution(t32, p), ptr, size);
            break;

        case distribution_type::geometric:
            generate(gen, std::geometric_distribution(p), ptr, size);
            break;

        case distribution_type::poisson:
            generate(gen, std::poisson_distribution(mean), ptr, size);
            break;

        case distribution_type::exponential:
            generate(gen, std::exponential_distribution(lambda), ptr, size);
            break;

        case distribution_type::gamma:
            generate(gen, std::gamma_distribution(alpha, beta), ptr, size);
            break;

        case distribution_type::weibull:
            generate(gen, std::weibull_distribution(a, b), ptr, size);
            break;

        case distribution_type::exterme_value:
            generate(gen, std::extreme_value_distribution(a, b), ptr, size);
            break;

        case distribution_type::normal:
            generate(gen, std::normal_distribution(mean, stddev), ptr, size);
            break;

        case distribution_type::lognormal:
            generate(gen, std::lognormal_distribution(m, s), ptr, size);
            break;

        case distribution_type::chi_squared:
            generate(gen, std::chi_squared_distribution(n), ptr, size);
            break;

        case distribution_type::cauchy:
            generate(gen, std::cauchy_distribution(a, b), ptr, size);
            break;

        case distribution_type::fisher_f:
            generate(gen, std::fisher_f_distribution(m, n), ptr, size);
            break;

        case distribution_type::student_t:
            generate(gen, std::student_t_distribution(n), ptr, size);
            break;
        }
504
505
    }

506
    status init(sz block_size, sz capacity) noexcept
507
    {
508
509
510
        distribution = distribution_type::uniform_int;
        a = 0;
        b = 100;
511

512
513
        return buffer.init(block_size, capacity);
    }
514

515
516
517
518
519
520
521
522
523
524
    status update(source& src) noexcept
    {
        if (src.buffer == nullptr) {
            if (!buffer.can_alloc())
                return status::source_empty;

            src.buffer = buffer.alloc();
            src.size = static_cast<int>(buffer.block_size);
            src.index = 0;
        }
525

526
527
        std::mt19937_64 gen;
        generate(gen, src.buffer, src.size);
528

529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
        return status::success;
    }

    status finalize(source& src)
    {
        if (src.buffer)
            buffer.free(src.buffer);

        src.clear();

        return status::success;
    }

    status operator()(source& src, source::operation_type op)
    {
        switch (op) {
        case source::operation_type::initialize:
            return update(src);
        case source::operation_type::update:
            return update(src);
        case source::operation_type::finalize:
            return finalize(src);
551
552
        }

553
        irt_unreachable();
554
    }
555
556
557
558
559
560
};

enum class constant_source_id : u64;
enum class binary_file_source_id : u64;
enum class text_file_source_id : u64;
enum class random_source_id : u64;
561

562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
struct external_source
{
    data_array<constant_source, constant_source_id> constant_sources;
    data_array<binary_file_source, binary_file_source_id> binary_file_sources;
    data_array<text_file_source, text_file_source_id> text_file_sources;
    data_array<random_source, random_source_id> random_sources;
    std::mt19937_64 generator;
    
    // Chunk of memory used by a model.
    sz block_size = 512;

    // Number of models can be attached to an external source.
    sz block_number = 1024; 

    status init(const sz size) noexcept
577
    {
578
579
580
581
        irt_return_if_bad(constant_sources.init(size));
        irt_return_if_bad(binary_file_sources.init(size));
        irt_return_if_bad(text_file_sources.init(size));
        irt_return_if_bad(random_sources.init(size));
582

583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
        return status::success;
    }

    status operator()(source& src, const source::operation_type op) noexcept
    {
        external_source_type type;
        if (!external_source_type_cast(src.type, &type))
            return status::source_unknown;

        switch (type) {
        case external_source_type::binary_file: {
            const auto src_id = enum_cast<binary_file_source_id>(src.id);
            if (auto* bin_src = binary_file_sources.try_to_get(src_id);
                bin_src) {
                return (*bin_src)(src, op);
            }
        } break;
        case external_source_type::constant: {
            const auto src_id = enum_cast<constant_source_id>(src.id);
            if (auto* cst_src = constant_sources.try_to_get(src_id); cst_src) {
                return (*cst_src)(src, op);
            }
        } break;

        case external_source_type::random: {
            const auto src_id = enum_cast<random_source_id>(src.id);
            if (auto* rnd_src = random_sources.try_to_get(src_id); rnd_src) {
                return (*rnd_src)(src, op);
            }
        } break;

        case external_source_type::text_file: {
            const auto src_id = enum_cast<text_file_source_id>(src.id);
            if (auto* txt_src = text_file_sources.try_to_get(src_id); txt_src) {
                return (*txt_src)(src, op);
            }
        } break;
        }
621

622
        irt_unreachable();
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
    }
};

enum class random_file_type
{
    binary,
    text,
};

template<typename RandomGenerator, typename Distribution>
inline int
generate_random_file(std::ostream& os,
                     RandomGenerator& gen,
                     Distribution& dist,
                     const std::size_t size,
                     const random_file_type type) noexcept
{
    switch (type) {
    case random_file_type::text: {
        if (!os)
            return -1;

        for (std::size_t sz = 0; sz < size; ++sz)
            if (!(os << dist(gen) << '\n'))
                return -2;
    } break;

    case random_file_type::binary: {
        if (!os)
            return -1;

        for (std::size_t sz = 0; sz < size; ++sz) {
            const double value = dist(gen);
            os.write(reinterpret_cast<const char*>(&value), sizeof(value));
        }
    } break;
    }

    return 0;
}

664
} // namespace irt
665
666

#endif