Commit e6cf0408 authored by Damien Leroux's avatar Damien Leroux
Browse files

New disk cache implementation fixes awful design flaw in previous version.

parent 36f2c24c
#ifndef _SPELL_BASIC_FILE_CHECKS_H_
#define _SPELL_BASIC_FILE_CHECKS_H_
struct file_stat {
bool exists;
int err;
bool is_file;
bool is_dir;
bool writable;
bool readable;
file_stat(const std::string& path)
{
struct stat st;
if (stat(path.c_str(), &st)) {
exists = false;
err = errno;
} else {
exists = true;
err = 0;
}
if (exists) {
is_file = !!S_ISREG(st.st_mode);
is_dir = !!S_ISDIR(st.st_mode);
writable = !!(st.st_mode & (S_IWUSR | S_IWGRP | S_IWOTH));
readable = !!(st.st_mode & (S_IRUSR | S_IRGRP | S_IROTH));
} else {
is_file = is_dir = writable = readable = false;
}
}
};
static inline
bool check_file(const std::string& path, bool req_directory, bool req_writable, bool display=true)
{
file_stat fs(path);
if (fs.err != 0) {
if (display) {
MSG_ERROR("Path " << path << " is invalid: " << strerror(errno), "Check whether " << path << " exists and is accessible");
}
return false;
}
if (req_directory) {
if (!fs.is_dir) {
if (display) {
MSG_ERROR(path << " is not a directory", "Specify the path to directory, not a file [" << path << ']');
}
return false;
}
} else if (!fs.is_file) {
if (display) {
MSG_ERROR(path << " is not a regular file or a symbolic link to a regular file", "Specify the path to a file, not a directory [" << path << ']');
}
return false;
}
if (!fs.readable) {
if (display) {
MSG_ERROR(path << " is not readable", "Verify the permissions of " << path);
}
return false;
}
if (req_writable && !fs.writable) {
if (display) {
MSG_ERROR(path << " is not writable", "Verify the permissions of " << path);
}
return false;
}
return true;
}
static inline
bool ensure_directory_exists(const std::string& path)
{
return check_file(path, true, true, false)
|| mkdir(path.c_str(), 0770) != -1;
}
#endif
......@@ -977,7 +977,7 @@ struct pop_data_type {
for (size_t f = 0; f < n_fam; ++f) {
std::string k = read_str(ifs);
size_t n_ind = read_size(ifs);
auto& fam = ret.families[k];
/*auto& fam = ret.families[k];*/
for (size_t i = 0; i < n_ind; ++i) {
ret.families[k].push_back(read_size(ifs));
}
......@@ -1081,7 +1081,7 @@ struct pop_data_type {
os << "| Chromosome " << kv.first << std::endl;
for (const auto& gen_lv: kv.second) {
os << "| Generation " << gen_lv.first << std::endl;
const auto& family = pop_data.families.find(gen_lv.first)->second;
/*const auto& family = pop_data.families.find(gen_lv.first)->second;*/
for (size_t i = 0; i < gen_lv.second.size(); ++i) {
os << "| #" << i << " " << pop_data.get_geno_matrix(gen_lv.first, i)->name << std::endl;
prepend(os, "| ", gen_lv.second[i]);
......@@ -1091,7 +1091,7 @@ struct pop_data_type {
} else {
for (const auto& kv: pop_data.LV.data_by_marker) {
os << "| Generation " << kv.first << std::endl;
const auto& family = pop_data.families.find(kv.first)->second;
/*const auto& family = pop_data.families.find(kv.first)->second;*/
for (const auto& mark_lv: kv.second) {
os << "| Marker " << kv.first << std::endl;
for (size_t i = 0; i < mark_lv.second.size(); ++i) {
......
#ifndef _SPELL_CACHE_BASE_H_
#define _SPELL_CACHE_BASE_H_
#include <future>
#include <tuple>
#include <unordered_map>
#include <iostream>
#include <string>
#include <sstream>
#include "function_wrapper.h"
#include "chrono.h"
struct chrono_trace {
const std::string& name;
chrono_trace(const std::string& n) : name(n) { chrono::increment(name); chrono::start(name); }
~chrono_trace() { chrono::stop(name); }
};
#include "error.h"
#include "input.h"
#include "cache/md5.h"
#include "cache/file.h"
#include "cache/registry.h"
extern "C" {
/*#include <dlfcn.h>*/
#include <malloc.h>
/*#include <string.h>*/
}
/*#include <cxxabi.h>*/
/*
char*
__cxa_demangle(const char* __mangled_name, char* __output_buffer, size_t* __length, int* __status);
*/
/*using demangle = abi::__cxa_demangle;*/
#if 0
static inline
std::unordered_map<void*, std::string>&
demangled_names_registry()
{
static std::unordered_map<void*, std::string> _;
return _;
}
template <typename Ret, typename... Args>
std::string&
get_func_name(Ret (*f) (Args...))
{
union {
Ret (*fptr) (Args...);
void* vptr;
} tmp = {f};
std::string& ret = demangled_names_registry()[tmp.vptr];
if (!ret.size()) {
Dl_info info;
dladdr(tmp.vptr, &info);
int status = 0;
char* buf = abi::__cxa_demangle(info.dli_sname, NULL, 0, &status);
ret.assign(buf, strchr(buf, '('));
free(buf);
/*std::cout << tmp.vptr << " => " << info.dli_sname << std::endl;*/
}
return ret;
}
template <typename Ret, typename... Args>
std::string&
get_func_name(Ret (&f) (Args...))
{
return get_func_name(&f);
}
#endif
enum CachingPolicy : int { Oneshot = 0, Mem = 1, Disk = 2, Sync = 4 };
/*enum CachingPolicy : int { Oneshot = 0, Mem = Oneshot, Disk = Oneshot, Sync = 4 };*/
constexpr CachingPolicy operator | (CachingPolicy c1, CachingPolicy c2) { return CachingPolicy(int(c1) | int(c2)); }
constexpr CachingPolicy operator & (CachingPolicy c1, CachingPolicy c2) { return CachingPolicy(int(c1) & int(c2)); }
template <typename X> struct clean_type { typedef typename std::remove_reference<X>::type type; };
template <typename X> struct clean_type<const X&> { typedef X type; };
template <typename X> struct clean_type<X&> { typedef X type; };
template <typename X> struct clean_type<const X> { typedef X type; };
/* forward */ struct md5_digest;
static inline std::string& cache_directory() { return active_settings->work_directory; }
template <typename T> struct generic_value_interface;
template <typename T> struct value;
template <typename T> struct range;
template <typename T> struct collection;
template <typename T> struct fail : std::integral_constant<bool, false> {};
template <typename T> struct value<collection<T>> { static_assert(fail<T>::value, "Can't instantiate this"); };
template <typename T> struct immediate_value;
template <typename FuncType> struct async_computation;
template <typename FuncType> struct cached_computation;
template <typename FuncType> struct computed_value_factory;
template <typename FuncType> struct computed_value;
template <typename FuncType> struct cached_computed_value;
/* a value<T> behaves as a pointer to T */
template <typename T>
struct generic_value_interface {
typedef T value_type;
virtual ~generic_value_interface() {}
virtual value_type& operator * () = 0;
virtual value_type* operator -> () = 0;
virtual const value_type& operator * () const = 0;
virtual const value_type* operator -> () const = 0;
virtual size_t hash() const = 0;
virtual md5_digest& md5(md5_digest&) const = 0;
bool operator == (const generic_value_interface<T>& gvi) const
{
return **this == *gvi;
}
#if 0
/*virtual bool equal(const generic_value_interface<T>&) const { return false; }*/
virtual
bool equal(const generic_value_interface<T>& gvi) const
/*override*/
{
return **this == *gvi;
/*return gvi.__equal__(m_storage);*/
}
/*virtual bool __equal__(const T&) const = 0; // { return false; }*/
#endif
};
/* Lightweight */
template <typename T>
struct value {
typedef T value_type;
value() : m_impl() {}
value(generic_value_interface<T>* v) : m_impl(v) {}
value(const T& immed) : m_impl(new immediate_value<T>(immed)) {}
value(T&& immed) : m_impl(new immediate_value<T>(std::forward<T>(immed))) {}
value_type& operator * () { return m_impl->operator * (); }
value_type* operator -> () { return m_impl->operator -> (); }
const value_type& operator * () const { return m_impl->operator * (); }
const value_type* operator -> () const { return m_impl->operator -> (); }
size_t hash() const { return m_impl->hash(); }
md5_digest& md5(md5_digest& md) const { return m_impl->md5(md); }
value<T>& operator = (const T& immed)
{
/*m_impl = new immediate_value<T>(immed);*/
m_impl = std::make_shared<immediate_value<T>>(immed);
return *this;
}
value<T>& operator = (std::shared_ptr<generic_value_interface<T>>& new_impl)
{
m_impl = new_impl;
return *this;
}
value<T>& operator = (const value<T>& new_impl)
{
m_impl = new_impl.m_impl;
return *this;
}
bool valid() const { return (bool) m_impl; }
operator bool () const { return valid(); }
/*bool equal(const value<T>& v) const { return m_impl == v.m_impl || m_impl->equal(*v.m_impl); }*/
bool equal(const value<T>& v) const { return m_impl == v.m_impl || *m_impl == *v.m_impl; }
protected:
std::shared_ptr<generic_value_interface<T>> m_impl;
};
template <typename T>
bool operator == (const value<T>& v1, const value<T>& v2) { return v1.equal(v2); }
template <typename T>
bool operator != (const value<T>& v1, const value<T>& v2) { return !v1.equal(v2); }
template <typename VT>
using clean_value_type = value<typename clean_type<VT>::type>;
namespace std {
template <typename T>
struct hash<value<T>> {
size_t operator () (const value<T>& v) const
{
return v.hash();
}
};
template <typename T>
struct hash<collection<T>> {
size_t operator () (const collection<T>& c) const
{
md5_digest h;
h.update(c.begin(), c.end());
return h.context;
}
};
}
template <typename VT>
std::ostream& operator << (std::ostream& os, const value<VT>& v)
{
if (v.valid()) {
return os << (*v);
} else {
return os << "<nil>";
}
}
template <typename VT>
md5_digest& operator << (md5_digest& md5, const value<VT>& v)
{
return v.md5(md5);
}
struct md5_hash_type {
md5_digest md5;
std::string accum;
std::string append;
};
#define do_with_arg_pack(_expr) do { using _ = int[]; (void)_{0, ((_expr), void(), 0)...}; } while(0)
namespace new_redux {
/* pattern found on http://stackoverflow.com/a/19098481 */
template <typename... Args>
size_t hash(Args... args)
{
size_t accum = 0;
do_with_arg_pack(accum ^= std::hash<Args>()(args));
/*using apply_to_pack = int[];*/
/*(void)apply_to_pack{0, (accum ^= std::hash<Args>()(args), void(), 0)...};*/
return accum;
}
template <typename... Args>
md5_digest feed_md5(Args... args)
{
md5_digest m;
do_with_arg_pack(m << args);
return m;
}
template <typename... Args>
std::string md5(Args... args)
{
/*do_with_arg_pack(std::cout << args << std::endl);*/
return feed_md5(args...);
}
template <typename... Args>
std::string md5_append(Args... args)
{
std::stringstream s;
do_with_arg_pack(s << md5(args));
/*std::cout << "* Long MD5 " << s.str() << std::endl;*/
return s.str();
}
}
template <typename... Elems>
size_t compute_hash(const Elems&... e)
{
/*redux::hash h; return redux::reduce()(0, h, e...);*/
return new_redux::hash(e...);
}
template <typename... Elems>
std::string compute_md5(const Elems&... e)
{
/*md5_digest md;*/
/*redux::md5 m;*/
/*redux::reduce()(md, m, e...);*/
/*return md;*/
return new_redux::md5(e...);
}
template <typename... Elems>
std::string append_md5(const Elems&... e)
{
/*std::stringstream ss; redux::md5_append ma; return redux::reduce()(ss, ma, e...).str();*/
return new_redux::md5_append(e...);
}
template <typename ValueType>
struct immediate_value : generic_value_interface<ValueType> {
typedef ValueType value_type;
value_type m_storage;
immediate_value(const ValueType& v) : m_storage(v) {}
immediate_value(ValueType&& v) : m_storage(std::forward<ValueType>(v)) {}
template <typename... Args>
immediate_value(Args... x) : m_storage(x...) {}
virtual
value_type& operator * ()
override { return m_storage; }
virtual
value_type* operator -> ()
override { return &m_storage; }
virtual
const value_type& operator * () const
override { return m_storage; }
virtual
const value_type* operator -> () const
override { return &m_storage; }
virtual
size_t hash() const
override { return std::hash<ValueType>()(m_storage); }
virtual
md5_digest& md5(md5_digest& md) const
override { return md << m_storage; }
#if 0
virtual
bool __equal__(const ValueType& v) const
override
{
return m_storage == v;
}
#endif
};
template <typename ValueType>
struct unique_value : generic_value_interface<ValueType> {
typedef ValueType value_type;
value_type m_storage;
unique_value(const ValueType& v) : m_storage(v) {}
unique_value(ValueType&& v) : m_storage(std::forward<ValueType>(v)) {}
unique_value() : m_storage() {}
virtual
value_type& operator * ()
override { return m_storage; }
virtual
value_type* operator -> ()
override { return &m_storage; }
virtual
const value_type& operator * () const
override { return m_storage; }
virtual
const value_type* operator -> () const
override { return &m_storage; }
virtual
size_t hash() const
override { return std::hash<const void*>()(this); }
virtual
md5_digest& md5(md5_digest& md) const
override { return md << m_storage; }
};
/*template <typename M, typename R, typename C>*/
/*struct immediate_value<labelled_matrix<M, R, C>>*/
/*: unique_value<labelled_matrix<M, R, C>> {};*/
/* ranges and collections */
template <typename T> struct collection : std::vector<value<T>> {
using std::vector<value<T>>::vector;
using std::vector<value<T>>::operator [];
template <typename INTEGRAL_TYPE>
value<T>&
operator [] (const value<INTEGRAL_TYPE>& i)
{ return (*this)[*i]; }
template <typename INTEGRAL_TYPE>
const value<T>&
operator [] (const value<INTEGRAL_TYPE>& i) const
{ return (*this)[*i]; }
};
/* T must have operators + and < */
template <typename T>
struct range {
T m_min, m_max, m_step;
range(T min, T max, T step)
: m_min(min), m_max(max), m_step(step)
{}
struct iterator {
T m_data;
T m_step;
T m_max;
iterator(T value, T max, T step)
: m_data(value), m_step(step), m_max(max)
{}
iterator& operator ++ ()
{
m_data += m_step;
if (m_data > m_max) {
m_data = m_max;
}
return *this;
}
bool operator == (const iterator& i) const
{
return m_data == i.m_data;
}
bool operator != (const iterator& i) const
{
return m_data != i.m_data;
}
value<T> operator * () const { return {m_data}; }
};
iterator begin() const { return {m_min, m_max, m_step}; }
iterator end() const { return {m_max, m_max, 0}; }
};
template <typename _Coll>
struct as_collection {
typedef typename _Coll::value_type T;
typedef typename _Coll::const_iterator ci_type;
ci_type m_begin, m_end;
as_collection(const _Coll& c)
: m_begin(c.begin()), m_end(c.end())
{}
struct iterator {
ci_type ci;
iterator(const ci_type& it)
: ci(it)
{}
iterator& operator ++ ()
{
++ci;
return *this;
}
bool operator == (const iterator& i) const
{
return ci == i.ci;
}
bool operator != (const iterator& i) const
{
return ci != i.ci;
}
value<T> operator * () const { return value<T>{*ci}; }
};
iterator begin() const { return {m_begin}; }
iterator end() const { return {m_end}; }
};
template <typename _Coll>
as_collection<_Coll> values_of(const _Coll& c) { return {c}; }
template <typename X>
value<X> as_value(const X& x) { return {x}; }
#endif
......@@ -239,6 +239,7 @@ struct md5_digest {
template <typename Whatever>
md5_digest& update(Whatever x)
{
/*MSG_WARNING("Using hash fallback in MD5 digest of type " << typeid(x).name());*/
return blend(std::hash<Whatever>()(x));
}
......@@ -282,6 +283,7 @@ struct md5_digest {
md5_digest&
update(const std::future<T>&)
{