New disk cache implementation fixes awful design flaw in previous version.

include/basic_file_checks.h

+#ifndef _SPELL_BASIC_FILE_CHECKS_H_
+#define _SPELL_BASIC_FILE_CHECKS_H_
+
+
+struct file_stat {
+    bool exists;
+    int err;
+    bool is_file;
+    bool is_dir;
+    bool writable;
+    bool readable;
+    file_stat(const std::string& path)
+    {
+        struct stat st;
+        if (stat(path.c_str(), &st)) {
+            exists = false;
+            err = errno;
+        } else {
+            exists = true;
+            err = 0;
+        }
+        if (exists) {
+            is_file = !!S_ISREG(st.st_mode);
+            is_dir = !!S_ISDIR(st.st_mode);
+            writable = !!(st.st_mode & (S_IWUSR | S_IWGRP | S_IWOTH));
+            readable = !!(st.st_mode & (S_IRUSR | S_IRGRP | S_IROTH));
+        } else {
+            is_file = is_dir = writable = readable = false;
+        }
+    }
+};
+
+
+static inline
+bool check_file(const std::string& path, bool req_directory, bool req_writable, bool display=true)
+{
+    file_stat fs(path);
+    if (fs.err != 0) {
+        if (display) {
+            MSG_ERROR("Path " << path << " is invalid: " << strerror(errno), "Check whether " << path << " exists and is accessible");
+        }
+        return false;
+    }
+
+    if (req_directory) {
+        if (!fs.is_dir) {
+            if (display) {
+                MSG_ERROR(path << " is not a directory", "Specify the path to directory, not a file [" << path << ']');
+            }
+            return false;
+        }
+    } else if (!fs.is_file) {
+        if (display) {
+            MSG_ERROR(path << " is not a regular file or a symbolic link to a regular file", "Specify the path to a file, not a directory [" << path << ']');
+        }
+        return false;
+    }
+
+    if (!fs.readable) {
+        if (display) {
+             MSG_ERROR(path << " is not readable", "Verify the permissions of " << path);
+        }
+        return false;
+    }
+    if (req_writable && !fs.writable) {
+        if (display) {
+             MSG_ERROR(path << " is not writable", "Verify the permissions of " << path);
+        }
+        return false;
+    }
+    return true;
+}
+
+static inline
+bool ensure_directory_exists(const std::string& path)
+{
+    return check_file(path, true, true, false)
+        || mkdir(path.c_str(), 0770) != -1;
+}
+
+#endif
+

include/bayes/output.h

@@ -977,7 +977,7 @@ struct pop_data_type {
             for (size_t f = 0; f < n_fam; ++f) {
                 std::string k = read_str(ifs);
                 size_t n_ind = read_size(ifs);
-                auto& fam = ret.families[k];
+                /*auto& fam = ret.families[k];*/
                 for (size_t i = 0; i < n_ind; ++i) {
                     ret.families[k].push_back(read_size(ifs));
                 }
@@ -1081,7 +1081,7 @@ struct pop_data_type {
                     os << "| Chromosome " << kv.first << std::endl;
                     for (const auto& gen_lv: kv.second) {
                         os << "|   Generation " << gen_lv.first << std::endl;
-                        const auto& family = pop_data.families.find(gen_lv.first)->second;
+                        /*const auto& family = pop_data.families.find(gen_lv.first)->second;*/
                         for (size_t i = 0; i < gen_lv.second.size(); ++i) {
                             os << "|     #" << i << "  " << pop_data.get_geno_matrix(gen_lv.first, i)->name << std::endl;
                             prepend(os, "|     ", gen_lv.second[i]);
@@ -1091,7 +1091,7 @@ struct pop_data_type {
             } else {
                 for (const auto& kv: pop_data.LV.data_by_marker) {
                     os << "|   Generation " << kv.first << std::endl;
-                    const auto& family = pop_data.families.find(kv.first)->second;
+                    /*const auto& family = pop_data.families.find(kv.first)->second;*/
                     for (const auto& mark_lv: kv.second) {
                         os << "| Marker " << kv.first << std::endl;
                         for (size_t i = 0; i < mark_lv.second.size(); ++i) {

include/cache/base.h

+#ifndef _SPELL_CACHE_BASE_H_
+#define _SPELL_CACHE_BASE_H_
+
+
+#include <future>
+#include <tuple>
+#include <unordered_map>
+#include <iostream>
+#include <string>
+#include <sstream>
+#include "function_wrapper.h"
+#include "chrono.h"
+
+
+
+
+struct chrono_trace {
+    const std::string& name;
+    chrono_trace(const std::string& n) : name(n) { chrono::increment(name); chrono::start(name); }
+    ~chrono_trace() { chrono::stop(name); }
+};
+
+#include "error.h"
+#include "input.h"
+#include "cache/md5.h"
+#include "cache/file.h"
+#include "cache/registry.h"
+
+extern "C" {
+/*#include <dlfcn.h>*/
+#include <malloc.h>
+/*#include <string.h>*/
+}
+/*#include <cxxabi.h>*/
+
+/*
+char*
+__cxa_demangle(const char* __mangled_name, char* __output_buffer, size_t* __length, int* __status);
+*/
+
+/*using demangle = abi::__cxa_demangle;*/
+
+#if 0
+static inline
+std::unordered_map<void*, std::string>&
+demangled_names_registry()
+{
+	static std::unordered_map<void*, std::string> _;
+	return _;
+}
+
+
+template <typename Ret, typename... Args>
+std::string&
+get_func_name(Ret (*f) (Args...))
+{
+	union {
+		Ret (*fptr) (Args...);
+		void* vptr;
+	} tmp = {f};
+	std::string& ret = demangled_names_registry()[tmp.vptr];
+	if (!ret.size()) {
+		Dl_info info;
+		dladdr(tmp.vptr, &info);
+		int status = 0;
+		char* buf = abi::__cxa_demangle(info.dli_sname, NULL, 0, &status);
+		ret.assign(buf, strchr(buf, '('));
+		free(buf);
+		/*std::cout << tmp.vptr << " => " << info.dli_sname << std::endl;*/
+	}
+	return ret;
+}
+
+
+template <typename Ret, typename... Args>
+std::string&
+get_func_name(Ret (&f) (Args...))
+{
+	return get_func_name(&f);
+}
+#endif
+
+
+enum CachingPolicy : int { Oneshot = 0, Mem = 1, Disk = 2, Sync = 4 };
+/*enum CachingPolicy : int { Oneshot = 0, Mem = Oneshot, Disk = Oneshot, Sync = 4 };*/
+constexpr CachingPolicy operator | (CachingPolicy c1, CachingPolicy c2) { return CachingPolicy(int(c1) | int(c2)); }
+constexpr CachingPolicy operator & (CachingPolicy c1, CachingPolicy c2) { return CachingPolicy(int(c1) & int(c2)); }
+
+template <typename X> struct clean_type { typedef typename std::remove_reference<X>::type type; };
+template <typename X> struct clean_type<const X&> { typedef X type; };
+template <typename X> struct clean_type<X&> { typedef X type; };
+template <typename X> struct clean_type<const X> { typedef X type; };
+
+
+/* forward */ struct md5_digest;
+
+
+static inline std::string& cache_directory() { return active_settings->work_directory; }
+
+template <typename T> struct generic_value_interface;
+template <typename T> struct value;
+template <typename T> struct range;
+template <typename T> struct collection;
+
+template <typename T> struct fail : std::integral_constant<bool, false> {};
+
+template <typename T> struct value<collection<T>> { static_assert(fail<T>::value, "Can't instantiate this"); };
+
+
+template <typename T> struct immediate_value;
+
+template <typename FuncType> struct async_computation;
+template <typename FuncType> struct cached_computation;
+template <typename FuncType> struct computed_value_factory;
+template <typename FuncType> struct computed_value;
+template <typename FuncType> struct cached_computed_value;
+
+/* a value<T> behaves as a pointer to T */
+
+template <typename T>
+    struct generic_value_interface {
+        typedef T value_type;
+
+        virtual ~generic_value_interface() {}
+
+        virtual value_type& operator * () = 0;
+        virtual value_type* operator -> () = 0;
+
+        virtual const value_type& operator * () const = 0;
+        virtual const value_type* operator -> () const = 0;
+
+        virtual size_t hash() const = 0;
+        virtual md5_digest& md5(md5_digest&) const = 0;
+
+        bool operator == (const generic_value_interface<T>& gvi) const
+        {
+            return **this == *gvi;
+        }
+#if 0
+        /*virtual bool equal(const generic_value_interface<T>&) const { return false; }*/
+        virtual
+            bool equal(const generic_value_interface<T>& gvi) const
+            /*override*/
+            {
+                return **this == *gvi;
+                /*return gvi.__equal__(m_storage);*/
+            }
+
+        /*virtual bool __equal__(const T&) const = 0;  // { return false; }*/
+#endif
+    };
+
+/* Lightweight */
+template <typename T>
+    struct value {
+        typedef T value_type;
+
+        value() : m_impl() {}
+        value(generic_value_interface<T>* v) : m_impl(v) {}
+        value(const T& immed) : m_impl(new immediate_value<T>(immed)) {}
+        value(T&& immed) : m_impl(new immediate_value<T>(std::forward<T>(immed))) {}
+
+        value_type& operator * () { return m_impl->operator * (); }
+        value_type* operator -> () { return m_impl->operator -> (); }
+        const value_type& operator * () const { return m_impl->operator * (); }
+        const value_type* operator -> () const { return m_impl->operator -> (); }
+        size_t hash() const { return m_impl->hash(); }
+        md5_digest& md5(md5_digest& md) const { return m_impl->md5(md); }
+
+        value<T>& operator = (const T& immed)
+        {
+            /*m_impl = new immediate_value<T>(immed);*/
+            m_impl = std::make_shared<immediate_value<T>>(immed);
+            return *this;
+        }
+
+        value<T>& operator = (std::shared_ptr<generic_value_interface<T>>& new_impl)
+        {
+            m_impl = new_impl;
+            return *this;
+        }
+
+        value<T>& operator = (const value<T>& new_impl)
+        {
+            m_impl = new_impl.m_impl;
+            return *this;
+        }
+
+        bool valid() const { return (bool) m_impl; }
+
+		operator bool () const { return valid(); }
+
+        /*bool equal(const value<T>& v) const { return m_impl == v.m_impl || m_impl->equal(*v.m_impl); }*/
+        bool equal(const value<T>& v) const { return m_impl == v.m_impl || *m_impl == *v.m_impl; }
+
+    protected:
+        std::shared_ptr<generic_value_interface<T>> m_impl;
+    };
+
+template <typename T>
+bool operator == (const value<T>& v1, const value<T>& v2) { return v1.equal(v2); }
+
+template <typename T>
+bool operator != (const value<T>& v1, const value<T>& v2) { return !v1.equal(v2); }
+
+
+template <typename VT>
+using clean_value_type = value<typename clean_type<VT>::type>;
+
+
+namespace std {
+    template <typename T>
+        struct hash<value<T>> {
+            size_t operator () (const value<T>& v) const
+            {
+                return v.hash();
+            }
+        };
+
+    template <typename T>
+        struct hash<collection<T>> {
+            size_t operator () (const collection<T>& c) const
+            {
+                md5_digest h;
+                h.update(c.begin(), c.end());
+                return h.context;
+            }
+        };
+}
+
+template <typename VT>
+std::ostream& operator << (std::ostream& os, const value<VT>& v)
+{
+    if (v.valid()) {
+        return os << (*v);
+    } else {
+        return os << "<nil>";
+    }
+}
+
+template <typename VT>
+md5_digest& operator << (md5_digest& md5, const value<VT>& v)
+{
+    return v.md5(md5);
+}
+
+
+struct md5_hash_type {
+    md5_digest md5;
+    std::string accum;
+    std::string append;
+};
+
+
+
+#define do_with_arg_pack(_expr) do { using _ = int[]; (void)_{0, ((_expr), void(), 0)...}; } while(0)
+
+namespace new_redux {
+    /* pattern found on http://stackoverflow.com/a/19098481 */
+
+    template <typename... Args>
+        size_t hash(Args... args)
+        {
+            size_t accum = 0;
+            do_with_arg_pack(accum ^= std::hash<Args>()(args));
+            /*using apply_to_pack = int[];*/
+            /*(void)apply_to_pack{0, (accum ^= std::hash<Args>()(args), void(), 0)...};*/
+            return accum;
+        }
+
+    template <typename... Args>
+        md5_digest feed_md5(Args... args)
+        {
+            md5_digest m;
+            do_with_arg_pack(m << args);
+            return m;
+        }
+
+    template <typename... Args>
+        std::string md5(Args... args)
+        {
+            /*do_with_arg_pack(std::cout << args << std::endl);*/
+            return feed_md5(args...);
+        }
+
+    template <typename... Args>
+        std::string md5_append(Args... args)
+        {
+            std::stringstream s;
+            do_with_arg_pack(s << md5(args));
+            /*std::cout << "* Long MD5 " << s.str() << std::endl;*/
+            return s.str();
+        }
+
+}
+
+template <typename... Elems>
+size_t compute_hash(const Elems&... e)
+{
+    /*redux::hash h; return redux::reduce()(0, h, e...);*/
+    return new_redux::hash(e...);
+}
+
+template <typename... Elems>
+std::string compute_md5(const Elems&... e)
+{
+    /*md5_digest md;*/
+    /*redux::md5 m;*/
+    /*redux::reduce()(md, m, e...);*/
+    /*return md;*/
+    return new_redux::md5(e...);
+}
+
+template <typename... Elems>
+std::string append_md5(const Elems&... e)
+{
+    /*std::stringstream ss; redux::md5_append ma; return redux::reduce()(ss, ma, e...).str();*/
+    return new_redux::md5_append(e...);
+}
+
+
+template <typename ValueType>
+    struct immediate_value : generic_value_interface<ValueType> {
+        typedef ValueType value_type;
+
+        value_type m_storage;
+
+        immediate_value(const ValueType& v) : m_storage(v) {}
+        immediate_value(ValueType&& v) : m_storage(std::forward<ValueType>(v)) {}
+        template <typename... Args>
+            immediate_value(Args... x) : m_storage(x...) {}
+
+        virtual
+            value_type& operator * ()
+            override { return m_storage; }
+        virtual
+            value_type* operator -> ()
+            override { return &m_storage; }
+        virtual
+            const value_type& operator * () const
+            override { return m_storage; }
+        virtual
+            const value_type* operator -> () const
+            override { return &m_storage; }
+
+        virtual
+            size_t hash() const
+            override { return std::hash<ValueType>()(m_storage); }
+        virtual
+            md5_digest& md5(md5_digest& md) const
+            override { return md << m_storage; }
+#if 0
+        virtual
+            bool __equal__(const ValueType& v) const
+            override
+            {
+                return m_storage == v;
+            }
+#endif
+    };
+
+
+template <typename ValueType>
+    struct unique_value : generic_value_interface<ValueType> {
+        typedef ValueType value_type;
+
+        value_type m_storage;
+
+        unique_value(const ValueType& v) : m_storage(v) {}
+        unique_value(ValueType&& v) : m_storage(std::forward<ValueType>(v)) {}
+        unique_value() : m_storage() {}
+
+        virtual
+            value_type& operator * ()
+            override { return m_storage; }
+        virtual
+            value_type* operator -> ()
+            override { return &m_storage; }
+        virtual
+            const value_type& operator * () const
+            override { return m_storage; }
+        virtual
+            const value_type* operator -> () const
+            override { return &m_storage; }
+
+        virtual
+            size_t hash() const
+            override { return std::hash<const void*>()(this); }
+        virtual
+            md5_digest& md5(md5_digest& md) const
+            override { return md << m_storage; }
+    };
+
+/*template <typename M, typename R, typename C>*/
+    /*struct immediate_value<labelled_matrix<M, R, C>>*/
+        /*: unique_value<labelled_matrix<M, R, C>> {};*/
+
+
+/* ranges and collections */
+
+template <typename T> struct collection : std::vector<value<T>> {
+    using std::vector<value<T>>::vector;
+    using std::vector<value<T>>::operator [];
+    template <typename INTEGRAL_TYPE>
+        value<T>&
+        operator [] (const value<INTEGRAL_TYPE>& i)
+        { return (*this)[*i]; }
+    template <typename INTEGRAL_TYPE>
+        const value<T>&
+        operator [] (const value<INTEGRAL_TYPE>& i) const
+        { return (*this)[*i]; }
+};
+
+/* T must have operators + and < */
+template <typename T>
+    struct range {
+        T m_min, m_max, m_step;
+        range(T min, T max, T step)
+            : m_min(min), m_max(max), m_step(step)
+        {}
+        struct iterator {
+            T m_data;
+            T m_step;
+            T m_max;
+            iterator(T value, T max, T step)
+                : m_data(value), m_step(step), m_max(max)
+            {}
+            iterator& operator ++ ()
+            {
+                m_data += m_step;
+                if (m_data > m_max) {
+                    m_data = m_max;
+                }
+                return *this;
+            }
+            bool operator == (const iterator& i) const
+            {
+                return m_data == i.m_data;
+            }
+            bool operator != (const iterator& i) const
+            {
+                return m_data != i.m_data;
+            }
+            value<T> operator * () const { return {m_data}; }
+        };
+        iterator begin() const { return {m_min, m_max, m_step}; }
+        iterator end() const { return {m_max, m_max, 0}; }
+    };
+
+
+template <typename _Coll>
+    struct as_collection {
+        typedef typename _Coll::value_type T;
+        typedef typename _Coll::const_iterator ci_type;
+
+        ci_type m_begin, m_end;
+
+        as_collection(const _Coll& c)
+            : m_begin(c.begin()), m_end(c.end())
+        {}
+
+        struct iterator {
+            ci_type ci;
+            iterator(const ci_type& it)
+                : ci(it)
+            {}
+            iterator& operator ++ ()
+            {
+                ++ci;
+                return *this;
+            }
+            bool operator == (const iterator& i) const
+            {
+                return ci == i.ci;
+            }
+            bool operator != (const iterator& i) const
+            {
+                return ci != i.ci;
+            }
+            value<T> operator * () const { return value<T>{*ci}; }
+        };
+
+        iterator begin() const { return {m_begin}; }
+        iterator end() const { return {m_end}; }
+    };
+
+template <typename _Coll>
+as_collection<_Coll> values_of(const _Coll& c) { return {c}; }
+
+template <typename X>
+value<X> as_value(const X& x) { return {x}; }
+
+
+#endif
+

include/cache/md5.h

@@ -239,6 +239,7 @@ struct md5_digest {
     template <typename Whatever>
         md5_digest& update(Whatever x)
         {
+            /*MSG_WARNING("Using hash fallback in MD5 digest of type " << typeid(x).name());*/
             return blend(std::hash<Whatever>()(x));
         }
 
@@ -282,6 +283,7 @@ struct md5_digest {
         md5_digest&
         update(const std::future<T>&)