external_source.hpp 16.7 KB
Newer Older
1
// Copyright (c) 2020-2021 INRA Distributed under the Boost Software License,
2
3
4
5
6
7
8
9
// Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
// http://www.boost.org/LICENSE_1_0.txt)

#ifndef ORG_VLEPROJECT_IRRITATOR_EXTERNAL_SOURCE_2021
#define ORG_VLEPROJECT_IRRITATOR_EXTERNAL_SOURCE_2021

#include <irritator/core.hpp>

Gauthier Quesnel's avatar
Gauthier Quesnel committed
10
#include <filesystem>
11
#include <fstream>
12
13
14
15
#include <iomanip>
#include <map>
#include <random>
#include <vector>
16

17
namespace irt {
18

19
enum class external_source_type
Gauthier Quesnel's avatar
Gauthier Quesnel committed
20
{
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
    binary_file,
    constant,
    random,
    text_file
};

enum class block_vector_policy
{
    not_reuse_free_list,
    reuse_free_list
};

template<block_vector_policy P>
struct block_vector_base
{
    double* buffer = nullptr;
    sz size = 0;
    sz max_size = 0;
    sz capacity = 0;
    sz block_size = 0;
    sz free_head = static_cast<sz>(-1);

    block_vector_base() = default;
Gauthier Quesnel's avatar
Gauthier Quesnel committed
44

45
46
47
48
    block_vector_base(const block_vector_base&) = delete;
    block_vector_base& operator=(const block_vector_base&) = delete;

    ~block_vector_base() noexcept
49
    {
50
51
52
        if (buffer)
            g_free_fn(buffer);
    }
53

54
55
56
    bool empty() const noexcept
    {
        return size == 0;
57
58
    }

59
    status init(sz block_size_, sz capacity_) noexcept
Gauthier Quesnel's avatar
Gauthier Quesnel committed
60
    {
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
        if (capacity_ == 0)
            return status::block_allocator_bad_capacity;

        if (capacity_ != capacity) {
            if (buffer)
                g_free_fn(buffer);

            buffer = static_cast<double*>(g_alloc_fn(capacity_ * block_size_));
            if (buffer == nullptr)
                return status::block_allocator_not_enough_memory;
        }

        size = 0;
        max_size = 0;
        capacity = capacity_;
        block_size = block_size_;
        free_head = static_cast<sz>(-1);

        return status::success;
    }

    bool can_alloc() const noexcept
    {
        if constexpr (P == block_vector_policy::reuse_free_list)
            return free_head != static_cast<sz>(-1) || max_size < capacity;
        else
            return max_size < capacity;
    }

    double* alloc() noexcept
    {
        sz new_block;

        if constexpr (P == block_vector_policy::reuse_free_list) {
            if (free_head != static_cast<sz>(-1)) {
                new_block = free_head;
                free_head = static_cast<sz>(buffer[free_head * block_size]);
            } else {
                new_block = max_size * block_size;
                ++max_size;
            }
        } else {
            new_block = max_size * block_size;
            ++max_size;
        }

        ++size;
        return &buffer[new_block];
    }

    void free([[maybe_unused]] double* block) noexcept
    {
        if constexpr (P == block_vector_policy::reuse_free_list) {
            auto ptr_diff = buffer - block;
            auto block_index = ptr_diff / block_size;

            block[0] = static_cast<double>(free_head);
            free_head = static_cast<sz>(block_index);
            --size;

            if (size == 0) {
                max_size = 0;
                free_head = static_cast<sz>(-1);
            }
        }
    }
};

using limited_block_vector =
  block_vector_base<block_vector_policy::not_reuse_free_list>;
using block_vector = block_vector_base<block_vector_policy::reuse_free_list>;

inline bool
external_source_type_cast(int value, external_source_type* type) noexcept
{
    if (value < 0 || value > 3)
        return false;

    *type = enum_cast<external_source_type>(value);
    return true;
}

143
144
145
146
147
148
149
150
151
152
static inline const char* external_source_type_string[] = { "binary_file",
                                                            "constant",
                                                            "random",
                                                            "text_file" };

inline const char*
external_source_str(const external_source_type type) noexcept
{
    return external_source_type_string[static_cast<int>(type)];
}
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174

enum class distribution_type
{
    bernouilli,
    binomial,
    cauchy,
    chi_squared,
    exponential,
    exterme_value,
    fisher_f,
    gamma,
    geometric,
    lognormal,
    negative_binomial,
    normal,
    poisson,
    student_t,
    uniform_int,
    uniform_real,
    weibull,
};

175
static const char* distribution_type_string[] = {
176
177
178
179
180
181
    "bernouilli",        "binomial", "cauchy",  "chi_squared", "exponential",
    "exterme_value",     "fisher_f", "gamma",   "geometric",   "lognormal",
    "negative_binomial", "normal",   "poisson", "student_t",   "uniform_int",
    "uniform_real",      "weibull"
};

182
183
184
185
186
187
188
inline const char*
distribution_str(const distribution_type type) noexcept
{

    return distribution_type_string[static_cast<int>(type)];
}

189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
struct constant_source
{
    small_string<23> name;
    std::vector<double> buffer;

    status init(sz block_size) noexcept
    {
        try {
            buffer.reserve(block_size);
            return status::success;
        } catch (const std::bad_alloc& /*e*/) {
            return status::gui_not_enough_memory;
        }
    }

    status start_or_restart() noexcept
Gauthier Quesnel's avatar
Gauthier Quesnel committed
205
206
207
    {
        return status::success;
    }
208
209
210
211
212

    status update(source& src) noexcept
    {
        src.buffer = buffer.data();
        src.size = static_cast<int>(buffer.size());
Gauthier Quesnel's avatar
Gauthier Quesnel committed
213
        src.index = 0;
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236

        return status::success;
    }

    status finalize(source& src) noexcept
    {
        src.clear();

        return status::success;
    }

    status operator()(source& src, source::operation_type op) noexcept
    {
        switch (op) {
        case source::operation_type::initialize:
            return update(src);
        case source::operation_type::update:
            return update(src);
        case source::operation_type::finalize:
            return finalize(src);
        }

        irt_unreachable();
Gauthier Quesnel's avatar
Gauthier Quesnel committed
237
238
239
    }
};

240
struct binary_file_source
241
{
242
243
    small_string<23> name;
    limited_block_vector buffer;
Gauthier Quesnel's avatar
Gauthier Quesnel committed
244
    std::filesystem::path file_path;
245
    std::ifstream ifs;
246
    sz size = 0; // Number of double read
247

248
    status init(sz block_size, sz capacity) noexcept
249
    {
250
251
252
253
254
255
256
        file_path.clear();

        if (ifs.is_open())
            ifs.close();

        return buffer.init(block_size, capacity);
    }
257

258
259
    status start_or_restart() noexcept
    {
Gauthier Quesnel's avatar
Gauthier Quesnel committed
260
261
262
263
        if (!ifs) {
            ifs.open(file_path);

            if (!ifs)
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
                return status::source_empty;
        } else {
            ifs.seekg(0);
        }

        return fill_buffer();
    }

    status update(source& src)
    {
        if (!buffer.can_alloc()) {
            if (src.buffer)
                buffer.free(src.buffer);

            irt_return_if_bad(fill_buffer());
Gauthier Quesnel's avatar
Gauthier Quesnel committed
279
280
        }

281
282
283
        src.buffer = buffer.alloc();
        src.size = static_cast<int>(buffer.block_size);
        src.index = 0;
284

285
        return status::success;
286
287
    }

288
    status finalize(source& src)
289
    {
290
291
        if (src.buffer)
            buffer.free(src.buffer);
292

293
        src.clear();
294

295
296
        return status::success;
    }
297

298
299
300
301
302
303
304
305
306
307
308
309
    status operator()(source& src, source::operation_type op)
    {
        switch (op) {
        case source::operation_type::initialize:
            return update(src);
        case source::operation_type::update:
            return update(src);
        case source::operation_type::finalize:
            return finalize(src);
        }

        irt_unreachable();
310
311
312
    }

private:
313
    status fill_buffer() noexcept
314
    {
315
316
317
        ifs.read(reinterpret_cast<char*>(buffer.buffer),
                 buffer.capacity * buffer.block_size);
        const auto read = ifs.gcount();
318

319
320
321
        buffer.capacity = static_cast<sz>(read) / buffer.block_size;
        if (buffer.capacity == 0)
            return status::source_empty;
322

323
        return status::success;
324
325
326
    }
};

327
struct text_file_source
328
{
329
330
    small_string<23> name;
    limited_block_vector buffer;
Gauthier Quesnel's avatar
Gauthier Quesnel committed
331
    std::filesystem::path file_path;
332
333
    std::ifstream ifs;

334
    status init(sz block_size, sz capacity) noexcept
335
    {
336
337
        return buffer.init(block_size, capacity);
        file_path.clear();
338

339
340
341
342
343
344
        if (ifs.is_open())
            ifs.close();
    }

    status start_or_restart() noexcept
    {
Gauthier Quesnel's avatar
Gauthier Quesnel committed
345
346
347
348
        if (!ifs) {
            ifs.open(file_path);

            if (!ifs)
349
350
351
                return status::source_empty;
        } else {
            ifs.seekg(0);
Gauthier Quesnel's avatar
Gauthier Quesnel committed
352
353
        }

354
355
356
357
358
359
360
361
362
363
364
        return fill_buffer();
    }

    status update(source& src)
    {
        if (!buffer.can_alloc()) {
            if (src.buffer)
                buffer.free(src.buffer);

            irt_return_if_bad(fill_buffer());
        }
365

366
367
368
369
370
        src.buffer = buffer.alloc();
        src.size = static_cast<int>(buffer.block_size);
        src.index = 0;

        return status::success;
371
372
    }

373
    status finalize(source& src)
374
    {
375
376
        if (src.buffer)
            buffer.free(src.buffer);
377

378
379
380
381
        src.clear();

        return status::success;
    }
382

383
384
385
386
387
388
389
390
391
392
    status operator()(source& src, source::operation_type op)
    {
        switch (op) {
        case source::operation_type::initialize:
            return update(src);
        case source::operation_type::update:
            return update(src);
        case source::operation_type::finalize:
            return finalize(src);
        }
393

394
        irt_unreachable();
395
396
397
    }

private:
398
    status fill_buffer() noexcept
399
400
    {
        size_t i = 0;
401
402
403
        const size_t e = buffer.capacity * buffer.block_size;
        for (; i < e && ifs.good(); ++i) {
            if (!(ifs >> buffer.buffer[i]))
404
405
406
                break;
        }

407
408
409
        buffer.capacity = i / buffer.block_size;
        if (buffer.capacity == 0)
            return status::source_empty;
410

411
        return status::success;
412
413
414
415
416
    }
};

struct random_source
{
417
418
419
420
421
    small_string<23> name;
    block_vector buffer;
    distribution_type distribution = distribution_type::uniform_int;
    double a, b, p, mean, lambda, alpha, beta, stddev, m, s, n;
    int a32, b32, t32, k32;
422

423
424
425
426
427
428
429
430
431
    template<typename RandomGenerator, typename Distribution>
    void generate(RandomGenerator& gen,
                  Distribution dist,
                  double* ptr,
                  sz size) noexcept
    {
        for (sz i = 0; i < size; ++i)
            ptr[i] = dist(gen);
    }
432

433
434
    template<typename RandomGenerator>
    void generate(RandomGenerator& gen, double* ptr, sz size) noexcept
435
    {
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
        switch (distribution) {
        case distribution_type::uniform_int:
            generate(gen, std::uniform_int_distribution(a32, b32), ptr, size);
            break;

        case distribution_type::uniform_real:
            generate(gen, std::uniform_real_distribution(a, b), ptr, size);
            break;

        case distribution_type::bernouilli:
            generate(gen, std::bernoulli_distribution(p), ptr, size);
            break;

        case distribution_type::binomial:
            generate(gen, std::binomial_distribution(t32, p), ptr, size);
            break;

        case distribution_type::negative_binomial:
            generate(
              gen, std::negative_binomial_distribution(t32, p), ptr, size);
            break;

        case distribution_type::geometric:
            generate(gen, std::geometric_distribution(p), ptr, size);
            break;

        case distribution_type::poisson:
            generate(gen, std::poisson_distribution(mean), ptr, size);
            break;

        case distribution_type::exponential:
            generate(gen, std::exponential_distribution(lambda), ptr, size);
            break;

        case distribution_type::gamma:
            generate(gen, std::gamma_distribution(alpha, beta), ptr, size);
            break;

        case distribution_type::weibull:
            generate(gen, std::weibull_distribution(a, b), ptr, size);
            break;

        case distribution_type::exterme_value:
            generate(gen, std::extreme_value_distribution(a, b), ptr, size);
            break;

        case distribution_type::normal:
            generate(gen, std::normal_distribution(mean, stddev), ptr, size);
            break;

        case distribution_type::lognormal:
            generate(gen, std::lognormal_distribution(m, s), ptr, size);
            break;

        case distribution_type::chi_squared:
            generate(gen, std::chi_squared_distribution(n), ptr, size);
            break;

        case distribution_type::cauchy:
            generate(gen, std::cauchy_distribution(a, b), ptr, size);
            break;

        case distribution_type::fisher_f:
            generate(gen, std::fisher_f_distribution(m, n), ptr, size);
            break;

        case distribution_type::student_t:
            generate(gen, std::student_t_distribution(n), ptr, size);
            break;
        }
506
507
    }

508
    status init(sz block_size, sz capacity) noexcept
509
    {
510
511
512
        distribution = distribution_type::uniform_int;
        a = 0;
        b = 100;
513

514
515
        return buffer.init(block_size, capacity);
    }
516

517
518
519
520
521
522
523
524
525
526
    status update(source& src) noexcept
    {
        if (src.buffer == nullptr) {
            if (!buffer.can_alloc())
                return status::source_empty;

            src.buffer = buffer.alloc();
            src.size = static_cast<int>(buffer.block_size);
            src.index = 0;
        }
527

528
529
        std::mt19937_64 gen;
        generate(gen, src.buffer, src.size);
530

531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
        return status::success;
    }

    status finalize(source& src)
    {
        if (src.buffer)
            buffer.free(src.buffer);

        src.clear();

        return status::success;
    }

    status operator()(source& src, source::operation_type op)
    {
        switch (op) {
        case source::operation_type::initialize:
            return update(src);
        case source::operation_type::update:
            return update(src);
        case source::operation_type::finalize:
            return finalize(src);
553
554
        }

555
        irt_unreachable();
556
    }
557
558
559
560
561
562
};

enum class constant_source_id : u64;
enum class binary_file_source_id : u64;
enum class text_file_source_id : u64;
enum class random_source_id : u64;
563

564
565
566
567
568
569
570
struct external_source
{
    data_array<constant_source, constant_source_id> constant_sources;
    data_array<binary_file_source, binary_file_source_id> binary_file_sources;
    data_array<text_file_source, text_file_source_id> text_file_sources;
    data_array<random_source, random_source_id> random_sources;
    std::mt19937_64 generator;
Gauthier Quesnel's avatar
Gauthier Quesnel committed
571

572
573
574
575
    // Chunk of memory used by a model.
    sz block_size = 512;

    // Number of models can be attached to an external source.
Gauthier Quesnel's avatar
Gauthier Quesnel committed
576
    sz block_number = 1024;
577
578

    status init(const sz size) noexcept
579
    {
580
581
582
583
        irt_return_if_bad(constant_sources.init(size));
        irt_return_if_bad(binary_file_sources.init(size));
        irt_return_if_bad(text_file_sources.init(size));
        irt_return_if_bad(random_sources.init(size));
584

585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
        return status::success;
    }

    status operator()(source& src, const source::operation_type op) noexcept
    {
        external_source_type type;
        if (!external_source_type_cast(src.type, &type))
            return status::source_unknown;

        switch (type) {
        case external_source_type::binary_file: {
            const auto src_id = enum_cast<binary_file_source_id>(src.id);
            if (auto* bin_src = binary_file_sources.try_to_get(src_id);
                bin_src) {
                return (*bin_src)(src, op);
            }
        } break;
        case external_source_type::constant: {
            const auto src_id = enum_cast<constant_source_id>(src.id);
            if (auto* cst_src = constant_sources.try_to_get(src_id); cst_src) {
                return (*cst_src)(src, op);
            }
        } break;

        case external_source_type::random: {
            const auto src_id = enum_cast<random_source_id>(src.id);
            if (auto* rnd_src = random_sources.try_to_get(src_id); rnd_src) {
                return (*rnd_src)(src, op);
            }
        } break;

        case external_source_type::text_file: {
            const auto src_id = enum_cast<text_file_source_id>(src.id);
            if (auto* txt_src = text_file_sources.try_to_get(src_id); txt_src) {
                return (*txt_src)(src, op);
            }
        } break;
        }
623

624
        irt_unreachable();
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
    }
};

enum class random_file_type
{
    binary,
    text,
};

template<typename RandomGenerator, typename Distribution>
inline int
generate_random_file(std::ostream& os,
                     RandomGenerator& gen,
                     Distribution& dist,
                     const std::size_t size,
                     const random_file_type type) noexcept
{
    switch (type) {
    case random_file_type::text: {
        if (!os)
            return -1;

        for (std::size_t sz = 0; sz < size; ++sz)
            if (!(os << dist(gen) << '\n'))
                return -2;
    } break;

    case random_file_type::binary: {
        if (!os)
            return -1;

        for (std::size_t sz = 0; sz < size; ++sz) {
            const double value = dist(gen);
            os.write(reinterpret_cast<const char*>(&value), sizeof(value));
        }
    } break;
    }

    return 0;
}

666
} // namespace irt
667
668

#endif