external_source.hpp 16.4 KB
Newer Older
1
// Copyright (c) 2020-2021 INRA Distributed under the Boost Software License,
2
3
4
5
6
7
8
9
// Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
// http://www.boost.org/LICENSE_1_0.txt)

#ifndef ORG_VLEPROJECT_IRRITATOR_EXTERNAL_SOURCE_2021
#define ORG_VLEPROJECT_IRRITATOR_EXTERNAL_SOURCE_2021

#include <irritator/core.hpp>

Gauthier Quesnel's avatar
Gauthier Quesnel committed
10
#include <filesystem>
11
#include <fstream>
12
13
14
15
#include <iomanip>
#include <map>
#include <random>
#include <vector>
16

17
namespace irt {
18

19
enum class external_source_type
Gauthier Quesnel's avatar
Gauthier Quesnel committed
20
{
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
    binary_file,
    constant,
    random,
    text_file
};

enum class block_vector_policy
{
    not_reuse_free_list,
    reuse_free_list
};

template<block_vector_policy P>
struct block_vector_base
{
    double* buffer = nullptr;
    sz size = 0;
    sz max_size = 0;
    sz capacity = 0;
    sz block_size = 0;
    sz free_head = static_cast<sz>(-1);

    block_vector_base() = default;
Gauthier Quesnel's avatar
Gauthier Quesnel committed
44

45
46
47
48
    block_vector_base(const block_vector_base&) = delete;
    block_vector_base& operator=(const block_vector_base&) = delete;

    ~block_vector_base() noexcept
49
    {
50
51
52
        if (buffer)
            g_free_fn(buffer);
    }
53

54
55
56
    bool empty() const noexcept
    {
        return size == 0;
57
58
    }

59
    status init(sz block_size_, sz capacity_) noexcept
Gauthier Quesnel's avatar
Gauthier Quesnel committed
60
    {
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
        if (capacity_ == 0)
            return status::block_allocator_bad_capacity;

        if (capacity_ != capacity) {
            if (buffer)
                g_free_fn(buffer);

            buffer = static_cast<double*>(g_alloc_fn(capacity_ * block_size_));
            if (buffer == nullptr)
                return status::block_allocator_not_enough_memory;
        }

        size = 0;
        max_size = 0;
        capacity = capacity_;
        block_size = block_size_;
        free_head = static_cast<sz>(-1);

        return status::success;
    }

    bool can_alloc() const noexcept
    {
        if constexpr (P == block_vector_policy::reuse_free_list)
            return free_head != static_cast<sz>(-1) || max_size < capacity;
        else
            return max_size < capacity;
    }

    double* alloc() noexcept
    {
        sz new_block;

        if constexpr (P == block_vector_policy::reuse_free_list) {
            if (free_head != static_cast<sz>(-1)) {
                new_block = free_head;
                free_head = static_cast<sz>(buffer[free_head * block_size]);
            } else {
                new_block = max_size * block_size;
                ++max_size;
            }
        } else {
            new_block = max_size * block_size;
            ++max_size;
        }

        ++size;
        return &buffer[new_block];
    }

    void free([[maybe_unused]] double* block) noexcept
    {
        if constexpr (P == block_vector_policy::reuse_free_list) {
            auto ptr_diff = buffer - block;
            auto block_index = ptr_diff / block_size;

            block[0] = static_cast<double>(free_head);
            free_head = static_cast<sz>(block_index);
            --size;

            if (size == 0) {
                max_size = 0;
                free_head = static_cast<sz>(-1);
            }
        }
    }
};

using limited_block_vector =
  block_vector_base<block_vector_policy::not_reuse_free_list>;
using block_vector = block_vector_base<block_vector_policy::reuse_free_list>;

inline bool
external_source_type_cast(int value, external_source_type* type) noexcept
{
    if (value < 0 || value > 3)
        return false;

    *type = enum_cast<external_source_type>(value);
    return true;
}

static inline const char* external_source_str[] = { "binary_file",
                                                    "constant",
                                                    "random",
                                                    "text_file" };

enum class distribution_type
{
    bernouilli,
    binomial,
    cauchy,
    chi_squared,
    exponential,
    exterme_value,
    fisher_f,
    gamma,
    geometric,
    lognormal,
    negative_binomial,
    normal,
    poisson,
    student_t,
    uniform_int,
    uniform_real,
    weibull,
};

static inline const char* distribution_type_str[] = {
    "bernouilli",        "binomial", "cauchy",  "chi_squared", "exponential",
    "exterme_value",     "fisher_f", "gamma",   "geometric",   "lognormal",
    "negative_binomial", "normal",   "poisson", "student_t",   "uniform_int",
    "uniform_real",      "weibull"
};

struct constant_source
{
    small_string<23> name;
    std::vector<double> buffer;

    status init(sz block_size) noexcept
    {
        try {
            buffer.reserve(block_size);
            return status::success;
        } catch (const std::bad_alloc& /*e*/) {
            return status::gui_not_enough_memory;
        }
    }

    status start_or_restart() noexcept
    {}

    status update(source& src) noexcept
    {
        src.buffer = buffer.data();
        src.size = static_cast<int>(buffer.size());
Gauthier Quesnel's avatar
Gauthier Quesnel committed
198
        src.index = 0;
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221

        return status::success;
    }

    status finalize(source& src) noexcept
    {
        src.clear();

        return status::success;
    }

    status operator()(source& src, source::operation_type op) noexcept
    {
        switch (op) {
        case source::operation_type::initialize:
            return update(src);
        case source::operation_type::update:
            return update(src);
        case source::operation_type::finalize:
            return finalize(src);
        }

        irt_unreachable();
Gauthier Quesnel's avatar
Gauthier Quesnel committed
222
223
224
    }
};

225
struct binary_file_source
226
{
227
228
    small_string<23> name;
    limited_block_vector buffer;
Gauthier Quesnel's avatar
Gauthier Quesnel committed
229
    std::filesystem::path file_path;
230
    std::ifstream ifs;
231
    sz size = 0; // Number of double read
232

233
    status init(sz block_size, sz capacity) noexcept
234
    {
235
236
237
238
239
240
241
        file_path.clear();

        if (ifs.is_open())
            ifs.close();

        return buffer.init(block_size, capacity);
    }
242

243
244
    status start_or_restart() noexcept
    {
Gauthier Quesnel's avatar
Gauthier Quesnel committed
245
246
247
248
        if (!ifs) {
            ifs.open(file_path);

            if (!ifs)
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
                return status::source_empty;
        } else {
            ifs.seekg(0);
        }

        return fill_buffer();
    }

    status update(source& src)
    {
        if (!buffer.can_alloc()) {
            if (src.buffer)
                buffer.free(src.buffer);

            irt_return_if_bad(fill_buffer());
Gauthier Quesnel's avatar
Gauthier Quesnel committed
264
265
        }

266
267
268
        src.buffer = buffer.alloc();
        src.size = static_cast<int>(buffer.block_size);
        src.index = 0;
269

270
        return status::success;
271
272
    }

273
    status finalize(source& src)
274
    {
275
276
        if (src.buffer)
            buffer.free(src.buffer);
277

278
        src.clear();
279

280
281
        return status::success;
    }
282

283
284
285
286
287
288
289
290
291
292
293
294
    status operator()(source& src, source::operation_type op)
    {
        switch (op) {
        case source::operation_type::initialize:
            return update(src);
        case source::operation_type::update:
            return update(src);
        case source::operation_type::finalize:
            return finalize(src);
        }

        irt_unreachable();
295
296
297
    }

private:
298
    status fill_buffer() noexcept
299
    {
300
301
302
        ifs.read(reinterpret_cast<char*>(buffer.buffer),
                 buffer.capacity * buffer.block_size);
        const auto read = ifs.gcount();
303

304
305
306
        buffer.capacity = static_cast<sz>(read) / buffer.block_size;
        if (buffer.capacity == 0)
            return status::source_empty;
307

308
        return status::success;
309
310
311
    }
};

312
struct text_file_source
313
{
314
315
    small_string<23> name;
    limited_block_vector buffer;
Gauthier Quesnel's avatar
Gauthier Quesnel committed
316
    std::filesystem::path file_path;
317
318
    std::ifstream ifs;

319
    status init(sz block_size, sz capacity) noexcept
320
    {
321
322
        return buffer.init(block_size, capacity);
        file_path.clear();
323

324
325
326
327
328
329
        if (ifs.is_open())
            ifs.close();
    }

    status start_or_restart() noexcept
    {
Gauthier Quesnel's avatar
Gauthier Quesnel committed
330
331
332
333
        if (!ifs) {
            ifs.open(file_path);

            if (!ifs)
334
335
336
                return status::source_empty;
        } else {
            ifs.seekg(0);
Gauthier Quesnel's avatar
Gauthier Quesnel committed
337
338
        }

339
340
341
342
343
344
345
346
347
348
349
        return fill_buffer();
    }

    status update(source& src)
    {
        if (!buffer.can_alloc()) {
            if (src.buffer)
                buffer.free(src.buffer);

            irt_return_if_bad(fill_buffer());
        }
350

351
352
353
354
355
        src.buffer = buffer.alloc();
        src.size = static_cast<int>(buffer.block_size);
        src.index = 0;

        return status::success;
356
357
    }

358
    status finalize(source& src)
359
    {
360
361
        if (src.buffer)
            buffer.free(src.buffer);
362

363
364
365
366
        src.clear();

        return status::success;
    }
367

368
369
370
371
372
373
374
375
376
377
    status operator()(source& src, source::operation_type op)
    {
        switch (op) {
        case source::operation_type::initialize:
            return update(src);
        case source::operation_type::update:
            return update(src);
        case source::operation_type::finalize:
            return finalize(src);
        }
378

379
        irt_unreachable();
380
381
382
    }

private:
383
    status fill_buffer() noexcept
384
385
    {
        size_t i = 0;
386
387
388
        const size_t e = buffer.capacity * buffer.block_size;
        for (; i < e && ifs.good(); ++i) {
            if (!(ifs >> buffer.buffer[i]))
389
390
391
                break;
        }

392
393
394
        buffer.capacity = i / buffer.block_size;
        if (buffer.capacity == 0)
            return status::source_empty;
395

396
        return status::success;
397
398
399
400
401
    }
};

struct random_source
{
402
403
404
405
406
    small_string<23> name;
    block_vector buffer;
    distribution_type distribution = distribution_type::uniform_int;
    double a, b, p, mean, lambda, alpha, beta, stddev, m, s, n;
    int a32, b32, t32, k32;
407

408
409
410
411
412
413
414
415
416
    template<typename RandomGenerator, typename Distribution>
    void generate(RandomGenerator& gen,
                  Distribution dist,
                  double* ptr,
                  sz size) noexcept
    {
        for (sz i = 0; i < size; ++i)
            ptr[i] = dist(gen);
    }
417

418
419
    template<typename RandomGenerator>
    void generate(RandomGenerator& gen, double* ptr, sz size) noexcept
420
    {
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
        switch (distribution) {
        case distribution_type::uniform_int:
            generate(gen, std::uniform_int_distribution(a32, b32), ptr, size);
            break;

        case distribution_type::uniform_real:
            generate(gen, std::uniform_real_distribution(a, b), ptr, size);
            break;

        case distribution_type::bernouilli:
            generate(gen, std::bernoulli_distribution(p), ptr, size);
            break;

        case distribution_type::binomial:
            generate(gen, std::binomial_distribution(t32, p), ptr, size);
            break;

        case distribution_type::negative_binomial:
            generate(
              gen, std::negative_binomial_distribution(t32, p), ptr, size);
            break;

        case distribution_type::geometric:
            generate(gen, std::geometric_distribution(p), ptr, size);
            break;

        case distribution_type::poisson:
            generate(gen, std::poisson_distribution(mean), ptr, size);
            break;

        case distribution_type::exponential:
            generate(gen, std::exponential_distribution(lambda), ptr, size);
            break;

        case distribution_type::gamma:
            generate(gen, std::gamma_distribution(alpha, beta), ptr, size);
            break;

        case distribution_type::weibull:
            generate(gen, std::weibull_distribution(a, b), ptr, size);
            break;

        case distribution_type::exterme_value:
            generate(gen, std::extreme_value_distribution(a, b), ptr, size);
            break;

        case distribution_type::normal:
            generate(gen, std::normal_distribution(mean, stddev), ptr, size);
            break;

        case distribution_type::lognormal:
            generate(gen, std::lognormal_distribution(m, s), ptr, size);
            break;

        case distribution_type::chi_squared:
            generate(gen, std::chi_squared_distribution(n), ptr, size);
            break;

        case distribution_type::cauchy:
            generate(gen, std::cauchy_distribution(a, b), ptr, size);
            break;

        case distribution_type::fisher_f:
            generate(gen, std::fisher_f_distribution(m, n), ptr, size);
            break;

        case distribution_type::student_t:
            generate(gen, std::student_t_distribution(n), ptr, size);
            break;
        }
491
492
    }

493
    status init(sz block_size, sz capacity) noexcept
494
    {
495
496
497
        distribution = distribution_type::uniform_int;
        a = 0;
        b = 100;
498

499
500
        return buffer.init(block_size, capacity);
    }
501

502
503
504
505
506
507
508
509
510
511
    status update(source& src) noexcept
    {
        if (src.buffer == nullptr) {
            if (!buffer.can_alloc())
                return status::source_empty;

            src.buffer = buffer.alloc();
            src.size = static_cast<int>(buffer.block_size);
            src.index = 0;
        }
512

513
514
        std::mt19937_64 gen;
        generate(gen, src.buffer, src.size);
515

516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
        return status::success;
    }

    status finalize(source& src)
    {
        if (src.buffer)
            buffer.free(src.buffer);

        src.clear();

        return status::success;
    }

    status operator()(source& src, source::operation_type op)
    {
        switch (op) {
        case source::operation_type::initialize:
            return update(src);
        case source::operation_type::update:
            return update(src);
        case source::operation_type::finalize:
            return finalize(src);
538
539
        }

540
        irt_unreachable();
541
    }
542
543
544
545
546
547
};

enum class constant_source_id : u64;
enum class binary_file_source_id : u64;
enum class text_file_source_id : u64;
enum class random_source_id : u64;
548

549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
struct external_source
{
    data_array<constant_source, constant_source_id> constant_sources;
    data_array<binary_file_source, binary_file_source_id> binary_file_sources;
    data_array<text_file_source, text_file_source_id> text_file_sources;
    data_array<random_source, random_source_id> random_sources;
    std::mt19937_64 generator;
    
    // Chunk of memory used by a model.
    sz block_size = 512;

    // Number of models can be attached to an external source.
    sz block_number = 1024; 

    status init(const sz size) noexcept
564
    {
565
566
567
568
        irt_return_if_bad(constant_sources.init(size));
        irt_return_if_bad(binary_file_sources.init(size));
        irt_return_if_bad(text_file_sources.init(size));
        irt_return_if_bad(random_sources.init(size));
569

570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
        return status::success;
    }

    status operator()(source& src, const source::operation_type op) noexcept
    {
        external_source_type type;
        if (!external_source_type_cast(src.type, &type))
            return status::source_unknown;

        switch (type) {
        case external_source_type::binary_file: {
            const auto src_id = enum_cast<binary_file_source_id>(src.id);
            if (auto* bin_src = binary_file_sources.try_to_get(src_id);
                bin_src) {
                return (*bin_src)(src, op);
            }
        } break;
        case external_source_type::constant: {
            const auto src_id = enum_cast<constant_source_id>(src.id);
            if (auto* cst_src = constant_sources.try_to_get(src_id); cst_src) {
                return (*cst_src)(src, op);
            }
        } break;

        case external_source_type::random: {
            const auto src_id = enum_cast<random_source_id>(src.id);
            if (auto* rnd_src = random_sources.try_to_get(src_id); rnd_src) {
                return (*rnd_src)(src, op);
            }
        } break;

        case external_source_type::text_file: {
            const auto src_id = enum_cast<text_file_source_id>(src.id);
            if (auto* txt_src = text_file_sources.try_to_get(src_id); txt_src) {
                return (*txt_src)(src, op);
            }
        } break;
        }
608

609
        irt_unreachable();
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
    }
};

enum class random_file_type
{
    binary,
    text,
};

template<typename RandomGenerator, typename Distribution>
inline int
generate_random_file(std::ostream& os,
                     RandomGenerator& gen,
                     Distribution& dist,
                     const std::size_t size,
                     const random_file_type type) noexcept
{
    switch (type) {
    case random_file_type::text: {
        if (!os)
            return -1;

        for (std::size_t sz = 0; sz < size; ++sz)
            if (!(os << dist(gen) << '\n'))
                return -2;
    } break;

    case random_file_type::binary: {
        if (!os)
            return -1;

        for (std::size_t sz = 0; sz < size; ++sz) {
            const double value = dist(gen);
            os.write(reinterpret_cast<const char*>(&value), sizeof(value));
        }
    } break;
    }

    return 0;
}

651
} // namespace irt
652
653

#endif