Commit 94ecaebd authored by Damien Leroux's avatar Damien Leroux
Browse files

Improved interface to define tasks, fixed nasty bugs and forced task lists to...

Improved interface to define tasks, fixed nasty bugs and forced task lists to contain pointers instead of instances to prevent copies.
parent 8a296073
......@@ -80,10 +80,22 @@ template <typename... ITEMS>
return !at_end;
}
template <typename I> struct _get;
template <int... Indices>
struct _get<sequence<Indices...>> {
static
std::tuple<typename item_iterator<ITEMS>::value_type...>
get(const std::tuple<iterator_context<ITEMS>...>& iter)
{
return std::make_tuple(*std::get<Indices>(iter)...);
}
};
std::tuple<typename item_iterator<ITEMS>::value_type...> operator * () const
{
if (!at_end) {
return getter::process(iter);
/*return getter::process(iter);*/
return _get<indices>::get(iter);
}
throw 0;
}
......@@ -94,8 +106,8 @@ template <typename... ITEMS>
value_type operator * () const { return *cp; }
iterator& operator ++ () { cp.next(); }
iterator& operator ++ (int) { cp.next(); }
iterator& operator ++ () { cp.next(); return *this; }
iterator& operator ++ (int) { cp.next(); return *this; }
bool operator == (const iterator& i) { return i.cp.at_end && cp.at_end; }
bool operator != (const iterator& i) { return !(*this == i); }
};
......@@ -112,6 +124,18 @@ template <typename... ITEMS>
iterator end() { std::cout << "END" << std::endl; return {{}}; }
};
/*template <typename... E>*/
/*cartesian_product<E...> make_cartesian_product(E&&... x)*/
/*{*/
/*return {x...};*/
/*}*/
template <typename... E>
cartesian_product<E...> make_cartesian_product(const std::tuple<E...>& x)
{
return {x};
}
}
#endif
......
......@@ -2,6 +2,7 @@
#define _MCQTL_CACHE_FACTORY_H_
#include <vector>
#include <memory>
#include <iostream>
#include <future>
#include <typeinfo>
......@@ -52,30 +53,41 @@ struct launch {
}
};
template <typename Type>
using computed_type = std::future<Type>;
/*using computed_type = Type;*/
template <typename Type, typename... Dependencies>
struct computed_value : value<std::future<Type>> {
virtual Type compute(const Dependencies&...) = 0;
template <typename Class, typename Type, typename... Dependencies>
struct computed_value : value<computed_type<Type>> {
/*virtual Type do_compute(const Dependencies&...) = 0;*/
typedef typename make_sequence<sizeof...(Dependencies)>::type indices;
computed_value(const std::tuple<Dependencies...>& dep)
: dependencies(dep)
{ val = async_compute_cache(); }
computed_value(std::tuple<Dependencies...>&& dep)
: dependencies(std::forward(dep))
: dependencies(std::move(dep))
{ val = async_compute_cache(); }
computed_value(const Dependencies&... dep)
: dependencies(dep...)
{ val = async_compute_cache(); }
computed_value(Dependencies&&... dep)
: dependencies(std::forward(dep)...)
: dependencies(std::move(dep)...)
{ val = async_compute_cache(); }
virtual ~computed_value()
{
std::cout << "destroying task ";
_print_tuple(std::cout, dependencies);
std::cout << std::endl;
}
std::string cache_filename() const
{
std::stringstream f;
md5_digest filename_digest;
compute_digest_(filename_digest, dependencies);
compute_digest_(filename_digest);
f << ((std::string)filename_digest) << '_' << pretty_name();
return f.str();
}
......@@ -83,58 +95,44 @@ struct computed_value : value<std::future<Type>> {
std::string dependencies_digest() const
{
std::stringstream f;
append_digest_(f, dependencies);
append_digest_(f);
return f.str();
}
operator Type () { return val.get(); }
/*operator Type () { return val; }*/
static std::string& cache_directory() { static std::string _ = DEFAULT_CACHE_DIR; return _; }
std::tuple<Dependencies...> dependencies;
private:
using value<std::future<Type>>::val;
using value<computed_type<Type>>::val;
Type compute_cache()
{
Type ret;
std::string path = cache_directory() + cache_filename();
std::cout << __FILE__ << ':' << __LINE__ << '@' << __func__ << std::endl;
file_stat fs(path);
std::cout << __FILE__ << ':' << __LINE__ << '@' << __func__ << std::endl;
if (fs.exists) {
std::cout << __FILE__ << ':' << __LINE__ << '@' << __func__ << std::endl;
if (!fs.readable) {
std::cout << __FILE__ << ':' << __LINE__ << '@' << __func__ << std::endl;
ERROR("File " << path << " is not readable.", "Check permissions for file " << path);
std::cout << __FILE__ << ':' << __LINE__ << '@' << __func__ << std::endl;
unlink(path.c_str());
std::cout << __FILE__ << ':' << __LINE__ << '@' << __func__ << std::endl;
} else if (load(path, ret)) {
std::cout << __FILE__ << ':' << __LINE__ << '@' << __func__ << std::endl;
INFO("Reloading data from path " << path);
std::cout << __FILE__ << ':' << __LINE__ << '@' << __func__ << std::endl;
return ret;
std::cout << __FILE__ << ':' << __LINE__ << '@' << __func__ << std::endl;
} else {
std::cout << __FILE__ << ':' << __LINE__ << '@' << __func__ << std::endl;
INFO("Cache invalid. Computing data and caching in " << path);
std::cout << __FILE__ << ':' << __LINE__ << '@' << __func__ << std::endl;
compute_and_save(path, ret);
std::cout << __FILE__ << ':' << __LINE__ << '@' << __func__ << std::endl;
}
std::cout << __FILE__ << ':' << __LINE__ << '@' << __func__ << std::endl;
} else {
std::cout << __FILE__ << ':' << __LINE__ << '@' << __func__ << std::endl;
INFO("Computing data and caching in " << path);
std::cout << __FILE__ << ':' << __LINE__ << '@' << __func__ << std::endl;
compute_and_save(path, ret);
std::cout << __FILE__ << ':' << __LINE__ << '@' << __func__ << std::endl;
}
std::cout << __FILE__ << ':' << __LINE__ << '@' << __func__ << std::endl;
return ret;
}
std::future<Type> async_compute_cache()
computed_type<Type> async_compute_cache()
{
return std::async(std::launch::async,
[this] ()
......@@ -145,39 +143,30 @@ private:
bool load(const std::string& filename, Type& v)
{
std::cout << __FILE__ << ':' << __LINE__ << '@' << __func__ << std::endl;
std::ifstream ifs(filename);
std::cout << __FILE__ << ':' << __LINE__ << '@' << __func__ << std::endl;
cache_input ar(ifs);
std::cout << __FILE__ << ':' << __LINE__ << '@' << __func__ << std::endl;
#if 1
std::string check;
std::cout << __FILE__ << ':' << __LINE__ << '@' << __func__ << std::endl;
ar & check;
std::cout << __FILE__ << ':' << __LINE__ << '@' << __func__ << std::endl;
if (check != dependencies_digest()) {
std::cout << __FILE__ << ':' << __LINE__ << '@' << __func__ << std::endl;
#else
std::tuple<Dependencies...> check;
ar & check;
if (check != dependencies) {
#endif
return false;
std::cout << __FILE__ << ':' << __LINE__ << '@' << __func__ << std::endl;
}
std::cout << __FILE__ << ':' << __LINE__ << '@' << __func__ << std::endl;
ar & v;
std::cout << __FILE__ << ':' << __LINE__ << '@' << __func__ << std::endl;
return true;
}
void compute_and_save(const std::string& filename, Type& v)
{
std::cout << __FILE__ << ':' << __LINE__ << '@' << __func__ << std::endl;
std::ofstream ofs(filename);
std::cout << __FILE__ << ':' << __LINE__ << '@' << __func__ << std::endl;
cache_output ar(ofs);
std::cout << __FILE__ << ':' << __LINE__ << '@' << __func__ << std::endl;
ar & dependencies_digest();
std::cout << __FILE__ << ':' << __LINE__ << '@' << __func__ << std::endl;
v = compute_(indices(), dependencies);
std::cout << __FILE__ << ':' << __LINE__ << '@' << __func__ << std::endl;
ar & v;
std::cout << __FILE__ << ':' << __LINE__ << '@' << __func__ << std::endl;
}
std::string pretty_name() const
......@@ -190,44 +179,22 @@ private:
return typeid(simple_type).name();
}
std::tuple<Dependencies...> dependencies;
Type compute_(const Dependencies&... t)
{
/*return std::async(std::launch::async,*/
/*[this](const Dependencies&... d) { return compute(d...); },*/
/*t...);*/
return compute(t...);
return dynamic_cast<Class*>(this)->do_compute(t...);
}
template <int... Indices>
Type compute_(sequence<Indices...>, const std::tuple<Dependencies...>& t)
{
/*return std::async(std::launch::async,*/
/*[this](const Dependencies&... d) { return compute(d...); },*/
/*std::get<Indices>(t)...);*/
return compute(std::get<Indices>(t)...);
return dynamic_cast<Class*>(this)->do_compute(std::get<Indices>(t)...);
}
/*std::future<Type> compute_(Dependencies&&... t)*/
/*{*/
/*return async(launch::policy,*/
/*[this](Dependencies&&... d) { return compute(d...); },*/
/*std::forward(t)...);*/
/*}*/
/**/
/*template <int... Indices>*/
/*std::future<Type> compute_(sequence<Indices...>, std::tuple<Dependencies...>&& t)*/
/*{*/
/*return async(launch::policy,*/
/*[this](Dependencies&&... d) { return compute(d...); },*/
/*std::get<Indices>(std::forward(t))...);*/
/*}*/
/**/
template <int I, int N> struct tuple_digest_ {
static void compute(md5_digest& md5, const std::tuple<Dependencies...>& t)
{
md5 << std::get<I>(t);
/*std::cout << "[compute] digesting #" << I << ' ' << std::get<I>(t) << std::endl;*/
tuple_digest_<I + 1, N>::compute(md5, t);
}
......@@ -236,7 +203,7 @@ private:
md5_digest md5;
md5 << std::get<I>(t);
std::string dig = md5;
std::cout << "digesting " << std::get<I>(t) << ": " << dig << std::endl;
/*std::cout << "[append] digesting #" << I << ' ' << std::get<I>(t) << ": " << dig << std::endl;*/
s << dig;
tuple_digest_<I + 1, N>::append(s, t);
}
......@@ -247,73 +214,43 @@ private:
static void append(std::stringstream&, const std::tuple<Dependencies...>&) {}
};
void compute_digest_(md5_digest& md5, const std::tuple<Dependencies...>& t) const
void compute_digest_(md5_digest& md5) const
{
tuple_digest_<0, sizeof...(Dependencies)>::compute(md5, t);
tuple_digest_<0, sizeof...(Dependencies)>::compute(md5, dependencies);
}
void append_digest_(std::stringstream& s, const std::tuple<Dependencies...>& t) const
void append_digest_(std::stringstream& s) const
{
tuple_digest_<0, sizeof...(Dependencies)>::append(s, t);
tuple_digest_<0, sizeof...(Dependencies)>::append(s, dependencies);
}
/*template <typename Dcar, typename... Dcdr>*/
/*void compute_digest_(md5_digest& md5, Dcar car, Dcdr... cdr)*/
/*{*/
/*md5 << car;*/
/*compute_digest_(md5, cdr...);*/
/*}*/
/**/
/*template <typename Dcar>*/
/*void compute_digest_(md5_digest& md5, Dcar car)*/
/*{*/
/*md5 << car;*/
/*}*/
/**/
/*void compute_digest_(md5_digest& md5) {}*/
};
template <typename TYPE, typename... DEPENDENCY>
struct item {
typedef TYPE value_type;
typedef std::unordered_map<std::tuple<Dependencies...>, std::shared_ptr<Class>> registry_t;
std::string digest;
public:
#if 0
static registry_t& registry() { static registry_t _; return _; }
std::tuple<DEPENDENCY...> dependencies;
static const bool is_collection = if_any_type<predicate::is_composite, DEPENDENCY...>::value;
item(DEPENDENCY... x)
: digest(compute_digest_(x...)), dependencies(x...)
{}
friend
std::ostream& operator << (std::ostream& os, item<DEPENDENCY...>& x)
{
}
private:
template <typename Dcar, typename... Dcdr>
md5_digest& compute_digest_(md5_digest& md5, Dcar car, Dcdr... cdr)
{
md5 << car;
compute_digest_(md5, cdr...);
return md5;
static std::shared_ptr<Class> get(const std::tuple<Dependencies...>& t)
{
std::shared_ptr<Class>& ret = registry()[t];
if (!ret) {
ret = new Class(t);
}
return ret;
}
template <typename Dcar>
md5_digest& compute_digest_(md5_digest& md5, Dcar car)
{
md5 << car;
return md5;
}
static void clear_registry() { registry().clear(); }
#else
static Class* get(std::tuple<Dependencies...>&& t)
{
return new Class(t);
}
md5_digest& compute_digest_(md5_digest& md5)
static Class* get(const std::tuple<Dependencies...>& t)
{
return md5;
return new Class(t);
}
#endif
};
......@@ -321,8 +258,9 @@ template <bool val, typename TASK_CLASS, typename... Elems> struct detect_compos
template <typename TASK_CLASS, typename... Elems>
struct detect_composite_impl<true, TASK_CLASS, Elems...> {
static const bool composite = true;
typedef std::vector<TASK_CLASS> type;
typedef std::vector<std::shared_ptr<TASK_CLASS>> type;
};
template <typename TASK_CLASS, typename... Elems>
struct detect_composite_impl<false, TASK_CLASS, Elems...> {
static const bool composite = false;
......@@ -344,17 +282,56 @@ struct make_task_helper;
template <typename TASK_CLASS, typename... Elems>
struct make_task_helper<true, TASK_CLASS, Elems...> {
static std::vector<TASK_CLASS> make(const std::tuple<Elems...>& t)
static std::vector<std::shared_ptr<TASK_CLASS>> make(const std::tuple<Elems...>& t)
{
std::vector<TASK_CLASS> ret;
std::vector<std::shared_ptr<TASK_CLASS>> ret;
cartesian_product<Elems...> cp(t);
do {
ret.emplace_back(TASK_CLASS(*cp));
/*ret.emplace_back(TASK_CLASS::get(*cp));*/
ret.emplace_back(new TASK_CLASS(*cp));
} while (cp.next());
return ret;
}
};
template <typename Func>
struct func_descr;
template <typename Ret, typename... Args>
struct func_descr<Ret(Args...)> {
typedef Ret return_type;
typedef Ret (func_type)(Args...);
typedef std::tuple<Args...> arguments_tuple_type;
typedef std::tuple<value<Args>...> dependencies_tuple_type;
template <typename Class>
using computed_value_type = computed_value<Class, Ret, value<Args>...>;
template <Ret (*Comp) (Args...)>
struct impl : computed_value_type<impl<Comp>> {
using computed_value_type<impl<Comp>>::computed_value_type;
Ret do_compute(const value<Args>&... args)
{
return Comp(args...);
}
};
};
template <typename Ret, typename... Args>
struct func_descr<Ret (*) (Args...)> : func_descr<Ret(Args...)> {};
template <typename Cls, typename Ret, typename... Args>
struct func_descr<Ret (Cls::*) (Args...)> : func_descr<Ret(Args...)> {
typedef Cls class_type;
};
#define cvalue(_x_) func_descr<decltype(&_x_)>::template impl<_x_>
#define cvalue_ctor(_x_) using typename func_descr<decltype(&_x_)>::template impl<_x_>::impl
template <typename TASK_CLASS, typename... Elems>
struct make_task_helper<false, TASK_CLASS, Elems...> {
static TASK_CLASS make(const std::tuple<Elems...>& t) { return TASK_CLASS(t); }
......@@ -364,14 +341,11 @@ template <typename TASK_CLASS, typename... Elems>
auto make_task(std::tuple<Elems...>& t)
-> typename detect_composite<TASK_CLASS, Elems...>::type
{
return make_task_helper<detect_composite<Elems...>::composite, TASK_CLASS, Elems...>::make(t);
std::cout << "make_task with " << t << std::endl;
return make_task_helper<detect_composite<TASK_CLASS, Elems...>::composite, TASK_CLASS, Elems...>::make(t);
}
template <typename V>
std::ostream& operator << (std::ostream& os, const value<V>& v) { return os << v.val; }
}
#endif
......
......@@ -66,17 +66,35 @@ cache_output& operator & (cache_output& c, const std::string& s)
cache_input& operator & (cache_input& c, std::string& s)
{
std::cout << __FILE__ << ':' << __LINE__ << '@' << __func__ << std::endl;
size_t sz = 0;
std::cout << __FILE__ << ':' << __LINE__ << '@' << __func__ << std::endl;
c & sz;
std::cout << __FILE__ << ':' << __LINE__ << '@' << __func__ << std::endl;
std::cout << " sz=" << sz << std::endl;
s.resize(sz, ' ');
std::cout << __FILE__ << ':' << __LINE__ << '@' << __func__ << std::endl;
return c.op(const_cast<char*>(s.c_str()), sz);
}
template <int I, int N, typename CACHE_DIRECTION, typename... Elems>
typename std::enable_if<I != N, cache_file<CACHE_DIRECTION>>::type&
operator & (cache_file<CACHE_DIRECTION>& c, std::tuple<Elems...>& t)
{
c & std::get<I>(t);
return operator &<I + 1, N, CACHE_DIRECTION, Elems...> (c, t);
}
template <int I, int N, typename CACHE_DIRECTION, typename... Elems>
typename std::enable_if<I == N, cache_file<CACHE_DIRECTION>>::type&
operator & (cache_file<CACHE_DIRECTION>& c, std::tuple<Elems...>& t)
{
return c;
}
template <typename CACHE_DIRECTION, typename... Elems>
cache_file<CACHE_DIRECTION>&
operator & (cache_file<CACHE_DIRECTION>& c, std::tuple<Elems...>& t)
{
return operator &<0, sizeof...(Elems), CACHE_DIRECTION, Elems...> (c, t);
}
#endif
......@@ -2,6 +2,7 @@
#define _MCQTL_CACHE_TUPLE_UTILS_H_
#include <tuple>
#include <iostream>
namespace cache {
......@@ -82,6 +83,7 @@ template <int... X>
if (PROCESSOR<typename std::tuple_element<I, std::tuple<TupleElems...>>::type>()(std::get<I>(x))) {
return backward_tuple<reverse<BackwardIndices...>>::process(x);
}
return false;
}
};
......@@ -131,5 +133,43 @@ template <int... X>
}
template <int I, typename... TElems>
typename std::enable_if<I == sizeof...(TElems), std::ostream>::type&
_print_tuple(std::ostream& os, const std::tuple<TElems...>& t)
{
return os << ')';
}
template <int I, typename... TElems>
typename std::enable_if<(I > 0 && I < sizeof...(TElems)), std::ostream>::type&
_print_tuple(std::ostream& os, const std::tuple<TElems...>& t)
{
os << ", " << std::get<I>(t);
return _print_tuple<I + 1, TElems...>(os, t);
}
template <int I, typename... TElems>
typename std::enable_if<I == 0, std::ostream>::type&
_print_tuple(std::ostream& os, const std::tuple<TElems...>& t)
{
os << std::get<I>(t);
return _print_tuple<I + 1, TElems...>(os, t);
}
template <typename... TElems>
std::ostream& _print_tuple(std::ostream& os, const std::tuple<TElems...>& t)
{
os << '(';
return _print_tuple<0, TElems...>(os, t);
}
template <typename... TElems>
std::ostream&
operator << (std::ostream& os, const std::tuple<TElems...>& t)
{
return _print_tuple(os, t);
}
#endif
......@@ -19,7 +19,7 @@ struct value {
iterator& operator ++ (int) { end = true; return *this; }
bool operator != (const iterator& i) const { return end != i.end; }
bool operator == (const iterator& i) const { return end == i.end; }
iterator& operator = (const iterator& i) { val = i.val; end = i.end; }
iterator& operator = (const iterator& i) { val = i.val; end = i.end; return *this; }
};
iterator begin() const { return {*this}; }
iterator end() const { return {}; }
......@@ -39,11 +39,11 @@ struct range {
iterator(INT_TYPE i) : cursor(i) {}
iterator(const iterator& i) : cursor(i.cursor) {}
value_type operator * () const { return {cursor}; }
iterator& operator ++ () { ++cursor; return *this; }
iterator& operator ++ (int) { ++cursor; return *this; }
iterator& operator ++ () { cursor += 1; return *this; }
iterator& operator ++ (int) { cursor += 1; return *this; }
bool operator != (const iterator& i) const { return cursor != i.cursor; }
bool operator == (const iterator& i) const { return cursor == i.cursor; }
iterator& operator = (const iterator& i) { cursor = i.cursor; }
iterator& operator = (const iterator& i) { cursor = i.cursor; return *this; }
};
typedef iterator const_iterator;
iterator begin() const { return {min}; }
......@@ -66,7 +66,7 @@ struct collection {
iterator& operator ++ (int) { ++cursor; return *this; }
bool operator != (const iterator& i) const { return cursor != i.cursor; }
bool operator == (const iterator& i) const { return cursor == i.cursor; }
iterator& operator = (const iterator& i) { cursor = i.cursor; }
iterator& operator = (const iterator& i) { cursor = i.cursor; return *this; }
};
typedef iterator const_iterator;
iterator begin() const { return {coll.begin()}; }
......@@ -75,5 +75,29 @@ struct collection {
}
template <typename V>