packages feed

souffle-haskell 2.0.1 → 2.1.0

raw patch · 73 files changed

+12259/−13710 lines, 73 filesdep −extradep −megaparsecPVP ok

version bump matches the API change (PVP)

Dependencies removed: extra, megaparsec

API changes (from Hackage documentation)

+ Language.Souffle.Compiled: instance GHC.Base.Monoid a => GHC.Base.Monoid (Language.Souffle.Compiled.SouffleM a)
+ Language.Souffle.Compiled: instance GHC.Base.Semigroup a => GHC.Base.Semigroup (Language.Souffle.Compiled.SouffleM a)
+ Language.Souffle.Interpreted: instance GHC.Base.Monoid a => GHC.Base.Monoid (Language.Souffle.Interpreted.SouffleM a)
+ Language.Souffle.Interpreted: instance GHC.Base.Semigroup a => GHC.Base.Semigroup (Language.Souffle.Interpreted.SouffleM a)

Files

CHANGELOG.md view
@@ -3,6 +3,13 @@ All notable changes to this project (as seen by library users) will be documented in this file. The CHANGELOG is available on [Github](https://github.com/luc-tielen/souffle-haskell.git/CHANGELOG.md). +## [2.1.0] - 2021-01-03++- souffle-haskell now supports Souffle version 2.0.2.+- Fix GHC 8.10 specific warnings and compile error.+- Support Semigroup and Monoid instances for composing Souffle actions in+  other ways.+ ## [2.0.1] - 2020-09-05  ## Fixed
LICENSE view
@@ -1,6 +1,6 @@ The MIT License -Copyright (c) 2019 Luc Tielen+Copyright (c) 2020 Luc Tielen  Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal
README.md view
@@ -149,7 +149,7 @@ it can be linked with other languages (including Haskell!).  For an example, take a look at the configuration for the-[test suite](https://github.com/luc-tielen/souffle-haskell/blob/master/package.yaml#L68-L80) of this project.+[test suite](https://github.com/luc-tielen/souffle-haskell/blob/master/package.yaml#L118-L133) of this project.  If you run into C++ compilation issues when using stack, this might be because the `-std=c++17` flag is not being used correctly when compiling souffle-haskell.@@ -229,6 +229,7 @@ Setup your environment by entering the following command:  ```bash+$ cachix use luctielen  # Optional (improves setup time *significantly*) $ nix-shell ```
− cbits/souffle/BTree.h 
@@ -1,2334 +0,0 @@-/*- * Souffle - A Datalog Compiler- * Copyright (c) 2013, 2015, Oracle and/or its affiliates. All rights reserved- * Licensed under the Universal Permissive License v 1.0 as shown at:- * - https://opensource.org/licenses/UPL- * - <souffle root>/licenses/SOUFFLE-UPL.txt- */--/************************************************************************- *- * @file BTree.h- *- * An implementation of a generic B-tree data structure including- * interfaces for utilizing instances as set or multiset containers.- *- ***********************************************************************/--#pragma once--#include "utility/CacheUtil.h"-#include "utility/ContainerUtil.h"-#include "utility/ParallelUtil.h"-#include <algorithm>-#include <cassert>-#include <cstddef>-#include <cstdint>-#include <iostream>-#include <iterator>-#include <string>-#include <tuple>-#include <type_traits>-#include <typeinfo>-#include <vector>--namespace souffle {--namespace detail {--// ---------- comparators ----------------/**- * A generic comparator implementation as it is used by- * a b-tree based on types that can be less-than and- * equality comparable.- */-template <typename T>-struct comparator {-    /**-     * Compares the values of a and b and returns-     * -1 if a<b, 1 if a>b and 0 otherwise-     */-    int operator()(const T& a, const T& b) const {-        return (a > b) - (a < b);-    }-    bool less(const T& a, const T& b) const {-        return a < b;-    }-    bool equal(const T& a, const T& b) const {-        return a == b;-    }-};--// ---------- search strategies ----------------/**- * A common base class for search strategies in b-trees.- */-struct search_strategy {};--/**- * A linear search strategy for looking up keys in b-tree nodes.- */-struct linear_search : public search_strategy {-    /**-     * Required user-defined default constructor.-     */-    linear_search() = default;--    /**-     * Obtains an iterator referencing an element equivalent to the-     * given key in the given range. If no such element is present,-     * a reference to the first element not less than the given key-     * is returned.-     */-    template <typename Key, typename Iter, typename Comp>-    inline Iter operator()(const Key& k, Iter a, Iter b, Comp& comp) const {-        return lower_bound(k, a, b, comp);-    }--    /**-     * Obtains a reference to the first element in the given range that-     * is not less than the given key.-     */-    template <typename Key, typename Iter, typename Comp>-    inline Iter lower_bound(const Key& k, Iter a, Iter b, Comp& comp) const {-        auto c = a;-        while (c < b) {-            auto r = comp(*c, k);-            if (r >= 0) {-                return c;-            }-            ++c;-        }-        return b;-    }--    /**-     * Obtains a reference to the first element in the given range that-     * such that the given key is less than the referenced element.-     */-    template <typename Key, typename Iter, typename Comp>-    inline Iter upper_bound(const Key& k, Iter a, Iter b, Comp& comp) const {-        auto c = a;-        while (c < b) {-            if (comp(*c, k) > 0) {-                return c;-            }-            ++c;-        }-        return b;-    }-};--/**- * A binary search strategy for looking up keys in b-tree nodes.- */-struct binary_search : public search_strategy {-    /**-     * Required user-defined default constructor.-     */-    binary_search() = default;--    /**-     * Obtains an iterator pointing to some element within the given-     * range that is equal to the given key, if available. If multiple-     * elements are equal to the given key, an undefined instance will-     * be obtained (no guaranteed lower or upper boundary).  If no such-     * element is present, a reference to the first element not less than-     * the given key will be returned.-     */-    template <typename Key, typename Iter, typename Comp>-    Iter operator()(const Key& k, Iter a, Iter b, Comp& comp) const {-        Iter c;-        auto count = b - a;-        while (count > 0) {-            auto step = count >> 1;-            c = a + step;-            auto r = comp(*c, k);-            if (r == 0) {-                return c;-            }-            if (r < 0) {-                a = ++c;-                count -= step + 1;-            } else {-                count = step;-            }-        }-        return a;-    }--    /**-     * Obtains a reference to the first element in the given range that-     * is not less than the given key.-     */-    template <typename Key, typename Iter, typename Comp>-    Iter lower_bound(const Key& k, Iter a, Iter b, Comp& comp) const {-        Iter c;-        auto count = b - a;-        while (count > 0) {-            auto step = count >> 1;-            c = a + step;-            if (comp(*c, k) < 0) {-                a = ++c;-                count -= step + 1;-            } else {-                count = step;-            }-        }-        return a;-    }--    /**-     * Obtains a reference to the first element in the given range that-     * such that the given key is less than the referenced element.-     */-    template <typename Key, typename Iter, typename Comp>-    Iter upper_bound(const Key& k, Iter a, Iter b, Comp& comp) const {-        Iter c;-        auto count = b - a;-        while (count > 0) {-            auto step = count >> 1;-            c = a + step;-            if (comp(k, *c) >= 0) {-                a = ++c;-                count -= step + 1;-            } else {-                count = step;-            }-        }-        return a;-    }-};--// ---------- search strategies selection ----------------/**- * A template-meta class to select search strategies for b-trees- * depending on the key type.- */-template <typename S>-struct strategy_selection {-    using type = S;-};--struct linear : public strategy_selection<linear_search> {};-struct binary : public strategy_selection<binary_search> {};--// by default every key utilizes binary search-template <typename Key>-struct default_strategy : public binary {};--template <>-struct default_strategy<int> : public linear {};--template <typename... Ts>-struct default_strategy<std::tuple<Ts...>> : public linear {};--/**- * The default non-updater- */-template <typename T>-struct updater {-    void update(T& /* old_t */, const T& /* new_t */) {}-};--/**- * The actual implementation of a b-tree data structure.- *- * @tparam Key             .. the element type to be stored in this tree- * @tparam Comparator     .. a class defining an order on the stored elements- * @tparam Allocator     .. utilized for allocating memory for required nodes- * @tparam blockSize    .. determines the number of bytes/block utilized by leaf nodes- * @tparam SearchStrategy .. enables switching between linear, binary or any other search strategy- * @tparam isSet        .. true = set, false = multiset- */-template <typename Key, typename Comparator,-        typename Allocator,  // is ignored so far - TODO: add support-        unsigned blockSize, typename SearchStrategy, bool isSet, typename WeakComparator = Comparator,-        typename Updater = detail::updater<Key>>-class btree {-public:-    class iterator;-    using const_iterator = iterator;--    using key_type = Key;-    using element_type = Key;-    using chunk = range<iterator>;--protected:-    /* ------------- static utilities ----------------- */--    const static SearchStrategy search;--    /* ---------- comparison utilities ---------------- */--    mutable Comparator comp;--    bool less(const Key& a, const Key& b) const {-        return comp.less(a, b);-    }--    bool equal(const Key& a, const Key& b) const {-        return comp.equal(a, b);-    }--    mutable WeakComparator weak_comp;--    bool weak_less(const Key& a, const Key& b) const {-        return weak_comp.less(a, b);-    }--    bool weak_equal(const Key& a, const Key& b) const {-        return weak_comp.equal(a, b);-    }--    /* -------------- updater utilities ------------- */--    mutable Updater upd;-    void update(Key& old_k, const Key& new_k) {-        upd.update(old_k, new_k);-    }--    /* -------------- the node type ----------------- */--    using size_type = std::size_t;-    using field_index_type = uint8_t;-    using lock_type = OptimisticReadWriteLock;--    struct node;--    /**-     * The base type of all node types containing essential-     * book-keeping information.-     */-    struct base {-#ifdef IS_PARALLEL--        // the parent node-        node* volatile parent;--        // a lock for synchronizing parallel operations on this node-        lock_type lock;--        // the number of keys in this node-        volatile size_type numElements;--        // the position in the parent node-        volatile field_index_type position;-#else-        // the parent node-        node* parent;--        // the number of keys in this node-        size_type numElements;--        // the position in the parent node-        field_index_type position;-#endif--        // a flag indicating whether this is a inner node or not-        const bool inner;--        /**-         * A simple constructor for nodes-         */-        base(bool inner) : parent(nullptr), numElements(0), position(0), inner(inner) {}--        bool isLeaf() const {-            return !inner;-        }--        bool isInner() const {-            return inner;-        }--        node* getParent() const {-            return parent;-        }--        field_index_type getPositionInParent() const {-            return position;-        }--        size_type getNumElements() const {-            return numElements;-        }-    };--    struct inner_node;--    /**-     * The actual, generic node implementation covering the operations-     * for both, inner and leaf nodes.-     */-    struct node : public base {-        /**-         * The number of keys/node desired by the user.-         */-        static constexpr size_t desiredNumKeys =-                ((blockSize > sizeof(base)) ? blockSize - sizeof(base) : 0) / sizeof(Key);--        /**-         * The actual number of keys/node corrected by functional requirements.-         */-        static constexpr size_t maxKeys = (desiredNumKeys > 3) ? desiredNumKeys : 3;--        // the keys stored in this node-        Key keys[maxKeys];--        // a simple constructor-        node(bool inner) : base(inner) {}--        /**-         * A deep-copy operation creating a clone of this node.-         */-        node* clone() const {-            // create a clone of this node-            node* res = (this->isInner()) ? static_cast<node*>(new inner_node())-                                          : static_cast<node*>(new leaf_node());--            // copy basic fields-            res->position = this->position;-            res->numElements = this->numElements;--            for (size_type i = 0; i < this->numElements; ++i) {-                res->keys[i] = this->keys[i];-            }--            // if this is a leaf we are done-            if (this->isLeaf()) {-                return res;-            }--            // copy child nodes recursively-            auto* ires = (inner_node*)res;-            for (size_type i = 0; i <= this->numElements; ++i) {-                ires->children[i] = this->getChild(i)->clone();-                ires->children[i]->parent = res;-            }--            // that's it-            return res;-        }--        /**-         * A utility function providing a reference to this node as-         * an inner node.-         */-        inner_node& asInnerNode() {-            assert(this->inner && "Invalid cast!");-            return *static_cast<inner_node*>(this);-        }--        /**-         * A utility function providing a reference to this node as-         * a const inner node.-         */-        const inner_node& asInnerNode() const {-            assert(this->inner && "Invalid cast!");-            return *static_cast<const inner_node*>(this);-        }--        /**-         * Computes the number of nested levels of the tree rooted-         * by this node.-         */-        size_type getDepth() const {-            if (this->isLeaf()) {-                return 1;-            }-            return getChild(0)->getDepth() + 1;-        }--        /**-         * Counts the number of nodes contained in the sub-tree rooted-         * by this node.-         */-        size_type countNodes() const {-            if (this->isLeaf()) {-                return 1;-            }-            size_type sum = 1;-            for (unsigned i = 0; i <= this->numElements; ++i) {-                sum += getChild(i)->countNodes();-            }-            return sum;-        }--        /**-         * Counts the number of entries contained in the sub-tree rooted-         * by this node.-         */-        size_type countEntries() const {-            if (this->isLeaf()) {-                return this->numElements;-            }-            size_type sum = this->numElements;-            for (unsigned i = 0; i <= this->numElements; ++i) {-                sum += getChild(i)->countEntries();-            }-            return sum;-        }--        /**-         * Determines the amount of memory used by the sub-tree rooted-         * by this node.-         */-        size_type getMemoryUsage() const {-            if (this->isLeaf()) {-                return sizeof(leaf_node);-            }-            size_type res = sizeof(inner_node);-            for (unsigned i = 0; i <= this->numElements; ++i) {-                res += getChild(i)->getMemoryUsage();-            }-            return res;-        }--        /**-         * Obtains a pointer to the array of child-pointers-         * of this node -- if it is an inner node.-         */-        node** getChildren() {-            return asInnerNode().children;-        }--        /**-         * Obtains a pointer to the array of const child-pointers-         * of this node -- if it is an inner node.-         */-        node* const* getChildren() const {-            return asInnerNode().children;-        }--        /**-         * Obtains a reference to the child of the given index.-         */-        node* getChild(size_type s) const {-            return asInnerNode().children[s];-        }--        /**-         * Checks whether this node is empty -- can happen due to biased insertion.-         */-        bool isEmpty() const {-            return this->numElements == 0;-        }--        /**-         * Checks whether this node is full.-         */-        bool isFull() const {-            return this->numElements == maxKeys;-        }--        /**-         * Obtains the point at which full nodes should be split.-         * Conventional b-trees always split in half. However, in cases-         * where in-order insertions are frequent, a split assigning-         * larger portions to the right fragment provide higher performance-         * and a better node-filling rate.-         */-        int getSplitPoint(int /*unused*/) {-            return std::min(3 * maxKeys / 4, maxKeys - 2);-        }--        /**-         * Splits this node.-         *-         * @param root .. a pointer to the root-pointer of the enclosing b-tree-         *                 (might have to be updated if the root-node needs to be split)-         * @param idx  .. the position of the insert causing the split-         */-#ifdef IS_PARALLEL-        void split(node** root, lock_type& root_lock, int idx, std::vector<node*>& locked_nodes) {-            assert(this->lock.is_write_locked());-            assert(!this->parent || this->parent->lock.is_write_locked());-            assert((this->parent != nullptr) || root_lock.is_write_locked());-            assert(this->isLeaf() || souffle::contains(locked_nodes, this));-            assert(!this->parent || souffle::contains(locked_nodes, const_cast<node*>(this->parent)));-#else-        void split(node** root, lock_type& root_lock, int idx) {-#endif-            assert(this->numElements == maxKeys);--            // get middle element-            int split_point = getSplitPoint(idx);--            // create a new sibling node-            node* sibling = (this->inner) ? static_cast<node*>(new inner_node())-                                          : static_cast<node*>(new leaf_node());--#ifdef IS_PARALLEL-            // lock sibling-            sibling->lock.start_write();-            locked_nodes.push_back(sibling);-#endif--            // move data over to the new node-            for (unsigned i = split_point + 1, j = 0; i < maxKeys; ++i, ++j) {-                sibling->keys[j] = keys[i];-            }--            // move child pointers-            if (this->inner) {-                // move pointers to sibling-                auto* other = static_cast<inner_node*>(sibling);-                for (unsigned i = split_point + 1, j = 0; i <= maxKeys; ++i, ++j) {-                    other->children[j] = getChildren()[i];-                    other->children[j]->parent = other;-                    other->children[j]->position = j;-                }-            }--            // update number of elements-            this->numElements = split_point;-            sibling->numElements = maxKeys - split_point - 1;--            // update parent-#ifdef IS_PARALLEL-            grow_parent(root, root_lock, sibling, locked_nodes);-#else-            grow_parent(root, root_lock, sibling);-#endif-        }--        /**-         * Moves keys from this node to one of its siblings or splits-         * this node to make some space for the insertion of an element at-         * position idx.-         *-         * Returns the number of elements moved to the left side, 0 in case-         * of a split. The number of moved elements will be <= the given idx.-         *-         * @param root .. the root node of the b-tree being part of-         * @param idx  .. the position of the insert triggering this operation-         */-        // TODO: remove root_lock ... no longer needed-#ifdef IS_PARALLEL-        int rebalance_or_split(node** root, lock_type& root_lock, int idx, std::vector<node*>& locked_nodes) {-            assert(this->lock.is_write_locked());-            assert(!this->parent || this->parent->lock.is_write_locked());-            assert((this->parent != nullptr) || root_lock.is_write_locked());-            assert(this->isLeaf() || souffle::contains(locked_nodes, this));-            assert(!this->parent || souffle::contains(locked_nodes, const_cast<node*>(this->parent)));-#else-        int rebalance_or_split(node** root, lock_type& root_lock, int idx) {-#endif--            // this node is full ... and needs some space-            assert(this->numElements == maxKeys);--            // get snap-shot of parent-            auto parent = this->parent;-            auto pos = this->position;--            // Option A) re-balance data-            if (parent && pos > 0) {-                node* left = parent->getChild(pos - 1);--#ifdef IS_PARALLEL-                // lock access to left sibling-                if (!left->lock.try_start_write()) {-                    // left node is currently updated => skip balancing and split-                    split(root, root_lock, idx, locked_nodes);-                    return 0;-                }-#endif--                // compute number of elements to be movable to left-                //    space available in left vs. insertion index-                size_type num = std::min<int>(maxKeys - left->numElements, idx);--                // if there are elements to move ..-                if (num > 0) {-                    Key* splitter = &(parent->keys[this->position - 1]);--                    // .. move keys to left node-                    left->keys[left->numElements] = *splitter;-                    for (size_type i = 0; i < num - 1; ++i) {-                        left->keys[left->numElements + 1 + i] = keys[i];-                    }-                    *splitter = keys[num - 1];--                    // shift keys in this node to the left-                    for (size_type i = 0; i < this->numElements - num; ++i) {-                        keys[i] = keys[i + num];-                    }--                    // .. and children if necessary-                    if (this->isInner()) {-                        auto* ileft = static_cast<inner_node*>(left);-                        auto* iright = static_cast<inner_node*>(this);--                        // move children-                        for (size_type i = 0; i < num; ++i) {-                            ileft->children[left->numElements + i + 1] = iright->children[i];-                        }--                        // update moved children-                        for (size_type i = 0; i < num; ++i) {-                            iright->children[i]->parent = ileft;-                            iright->children[i]->position = left->numElements + i + 1;-                        }--                        // shift child-pointer to the left-                        for (size_type i = 0; i < this->numElements - num + 1; ++i) {-                            iright->children[i] = iright->children[i + num];-                        }--                        // update position of children-                        for (size_type i = 0; i < this->numElements - num + 1; ++i) {-                            iright->children[i]->position = i;-                        }-                    }--                    // update node sizes-                    left->numElements += num;-                    this->numElements -= num;--#ifdef IS_PARALLEL-                    left->lock.end_write();-#endif--                    // done-                    return num;-                }--#ifdef IS_PARALLEL-                left->lock.abort_write();-#endif-            }--            // Option B) split node-#ifdef IS_PARALLEL-            split(root, root_lock, idx, locked_nodes);-#else-            split(root, root_lock, idx);-#endif-            return 0;  // = no re-balancing-        }--    private:-        /**-         * Inserts a new sibling into the parent of this node utilizing-         * the last key of this node as a separation key. (for internal-         * use only)-         *-         * @param root .. a pointer to the root-pointer of the containing tree-         * @param sibling .. the new right-sibling to be add to the parent node-         */-#ifdef IS_PARALLEL-        void grow_parent(node** root, lock_type& root_lock, node* sibling, std::vector<node*>& locked_nodes) {-            assert(this->lock.is_write_locked());-            assert(!this->parent || this->parent->lock.is_write_locked());-            assert((this->parent != nullptr) || root_lock.is_write_locked());-            assert(this->isLeaf() || souffle::contains(locked_nodes, this));-            assert(!this->parent || souffle::contains(locked_nodes, const_cast<node*>(this->parent)));-#else-        void grow_parent(node** root, lock_type& root_lock, node* sibling) {-#endif--            if (this->parent == nullptr) {-                assert(*root == this);--                // create a new root node-                auto* new_root = new inner_node();-                new_root->numElements = 1;-                new_root->keys[0] = keys[this->numElements];--                new_root->children[0] = this;-                new_root->children[1] = sibling;--                // link this and the sibling node to new root-                this->parent = new_root;-                sibling->parent = new_root;-                sibling->position = 1;--                // switch root node-                *root = new_root;--            } else {-                // insert new element in parent element-                auto parent = this->parent;-                auto pos = this->position;--#ifdef IS_PARALLEL-                parent->insert_inner(-                        root, root_lock, pos, this, keys[this->numElements], sibling, locked_nodes);-#else-                parent->insert_inner(root, root_lock, pos, this, keys[this->numElements], sibling);-#endif-            }-        }--        /**-         * Inserts a new element into an inner node (for internal use only).-         *-         * @param root .. a pointer to the root-pointer of the containing tree-         * @param pos  .. the position to insert the new key-         * @param key  .. the key to insert-         * @param newNode .. the new right-child of the inserted key-         */-#ifdef IS_PARALLEL-        void insert_inner(node** root, lock_type& root_lock, unsigned pos, node* predecessor, const Key& key,-                node* newNode, std::vector<node*>& locked_nodes) {-            assert(this->lock.is_write_locked());-            assert(souffle::contains(locked_nodes, this));-#else-        void insert_inner(node** root, lock_type& root_lock, unsigned pos, node* predecessor, const Key& key,-                node* newNode) {-#endif--            // check capacity-            if (this->numElements >= maxKeys) {-#ifdef IS_PARALLEL-                assert(!this->parent || this->parent->lock.is_write_locked());-                assert((this->parent) || root_lock.is_write_locked());-                assert(!this->parent || souffle::contains(locked_nodes, const_cast<node*>(this->parent)));-#endif--                // split this node-#ifdef IS_PARALLEL-                pos -= rebalance_or_split(root, root_lock, pos, locked_nodes);-#else-                pos -= rebalance_or_split(root, root_lock, pos);-#endif--                // complete insertion within new sibling if necessary-                if (pos > this->numElements) {-                    // correct position-                    pos = pos - this->numElements - 1;--                    // get new sibling-                    auto other = this->parent->getChild(this->position + 1);--#ifdef IS_PARALLEL-                    // make sure other side is write locked-                    assert(other->lock.is_write_locked());-                    assert(souffle::contains(locked_nodes, other));--                    // search for new position (since other may have been altered in the meanwhile)-                    size_type i = 0;-                    for (; i <= other->numElements; ++i) {-                        if (other->getChild(i) == predecessor) {-                            break;-                        }-                    }--                    pos = (i > other->numElements) ? 0 : i;-                    other->insert_inner(root, root_lock, pos, predecessor, key, newNode, locked_nodes);-#else-                    other->insert_inner(root, root_lock, pos, predecessor, key, newNode);-#endif-                    return;-                }-            }--            // move bigger keys one forward-            for (int i = this->numElements - 1; i >= (int)pos; --i) {-                keys[i + 1] = keys[i];-                getChildren()[i + 2] = getChildren()[i + 1];-                ++getChildren()[i + 2]->position;-            }--            // ensure proper position-            assert(getChild(pos) == predecessor);--            // insert new element-            keys[pos] = key;-            getChildren()[pos + 1] = newNode;-            newNode->parent = this;-            newNode->position = pos + 1;-            ++this->numElements;-        }--    public:-        /**-         * Prints a textual representation of this tree to the given output stream.-         * This feature is mainly intended for debugging and tuning purposes.-         *-         * @see btree::printTree-         */-        void printTree(std::ostream& out, const std::string& prefix) const {-            // print the header-            out << prefix << "@" << this << "[" << ((int)(this->position)) << "] - "-                << (this->inner ? "i" : "") << "node : " << this->numElements << "/" << maxKeys << " [";--            // print the keys-            for (unsigned i = 0; i < this->numElements; i++) {-                out << keys[i];-                if (i != this->numElements - 1) {-                    out << ",";-                }-            }-            out << "]";--            // print references to children-            if (this->inner) {-                out << " - [";-                for (unsigned i = 0; i <= this->numElements; i++) {-                    out << getChildren()[i];-                    if (i != this->numElements) {-                        out << ",";-                    }-                }-                out << "]";-            }--#ifdef IS_PARALLEL-            // print the lock state-            if (this->lock.is_write_locked()) {-                std::cout << " locked";-            }-#endif--            out << "\n";--            // print the children recursively-            if (this->inner) {-                for (unsigned i = 0; i < this->numElements + 1; ++i) {-                    static_cast<const inner_node*>(this)->children[i]->printTree(out, prefix + "    ");-                }-            }-        }--        /**-         * A function decomposing the sub-tree rooted by this node into approximately equally-         * sized chunks. To minimize computational overhead, no strict load balance nor limit-         * on the number of actual chunks is given.-         *-         * @see btree::getChunks()-         *-         * @param res   .. the list of chunks to be extended-         * @param num   .. the number of chunks to be produced-         * @param begin .. the iterator to start the first chunk with-         * @param end   .. the iterator to end the last chunk with-         * @return the handed in list of chunks extended by generated chunks-         */-        std::vector<chunk>& collectChunks(-                std::vector<chunk>& res, size_type num, const iterator& begin, const iterator& end) const {-            assert(num > 0);--            // special case: this node is empty-            if (isEmpty()) {-                if (begin != end) {-                    res.push_back(chunk(begin, end));-                }-                return res;-            }--            // special case: a single chunk is requested-            if (num == 1) {-                res.push_back(chunk(begin, end));-                return res;-            }--            // cut-off-            if (this->isLeaf() || num < (this->numElements + 1)) {-                auto step = this->numElements / num;-                if (step == 0) {-                    step = 1;-                }--                size_type i = 0;--                // the first chunk starts at the begin-                res.push_back(chunk(begin, iterator(this, step - 1)));--                // split up the main part-                for (i = step - 1; i < this->numElements - step; i += step) {-                    res.push_back(chunk(iterator(this, i), iterator(this, i + step)));-                }--                // the last chunk runs to the end-                res.push_back(chunk(iterator(this, i), end));--                // done-                return res;-            }--            // else: collect chunks of sub-set elements--            auto part = num / (this->numElements + 1);-            assert(part > 0);-            getChild(0)->collectChunks(res, part, begin, iterator(this, 0));-            for (size_type i = 1; i < this->numElements; i++) {-                getChild(i)->collectChunks(res, part, iterator(this, i - 1), iterator(this, i));-            }-            getChild(this->numElements)-                    ->collectChunks(res, num - (part * this->numElements),-                            iterator(this, this->numElements - 1), end);--            // done-            return res;-        }--        /**-         * A function to verify the consistency of this node.-         *-         * @param root ... a reference to the root of the enclosing tree.-         * @return true if valid, false otherwise-         */-        template <typename Comp>-        bool check(Comp& comp, const node* root) const {-            bool valid = true;--            // check fill-state-            if (this->numElements > maxKeys) {-                std::cout << "Node with " << this->numElements << "/" << maxKeys << " encountered!\n";-                valid = false;-            }--            // check root state-            if (root == this) {-                if (this->parent != nullptr) {-                    std::cout << "Root not properly linked!\n";-                    valid = false;-                }-            } else {-                // check parent relation-                if (!this->parent) {-                    std::cout << "Invalid null-parent!\n";-                    valid = false;-                } else {-                    if (this->parent->getChildren()[this->position] != this) {-                        std::cout << "Parent reference invalid!\n";-                        std::cout << "   Node:     " << this << "\n";-                        std::cout << "   Parent:   " << this->parent << "\n";-                        std::cout << "   Position: " << ((int)this->position) << "\n";-                        valid = false;-                    }--                    // check parent key-                    if (valid && this->position != 0 &&-                            !(comp(this->parent->keys[this->position - 1], keys[0]) < ((isSet) ? 0 : 1))) {-                        std::cout << "Left parent key not lower bound!\n";-                        std::cout << "   Node:     " << this << "\n";-                        std::cout << "   Parent:   " << this->parent << "\n";-                        std::cout << "   Position: " << ((int)this->position) << "\n";-                        std::cout << "   Key:   " << (this->parent->keys[this->position]) << "\n";-                        std::cout << "   Lower: " << (keys[0]) << "\n";-                        valid = false;-                    }--                    // check parent key-                    if (valid && this->position != this->parent->numElements &&-                            !(comp(keys[this->numElements - 1], this->parent->keys[this->position]) <-                                    ((isSet) ? 0 : 1))) {-                        std::cout << "Right parent key not lower bound!\n";-                        std::cout << "   Node:     " << this << "\n";-                        std::cout << "   Parent:   " << this->parent << "\n";-                        std::cout << "   Position: " << ((int)this->position) << "\n";-                        std::cout << "   Key:   " << (this->parent->keys[this->position]) << "\n";-                        std::cout << "   Upper: " << (keys[0]) << "\n";-                        valid = false;-                    }-                }-            }--            // check element order-            if (this->numElements > 0) {-                for (unsigned i = 0; i < this->numElements - 1; i++) {-                    if (valid && !(comp(keys[i], keys[i + 1]) < ((isSet) ? 0 : 1))) {-                        std::cout << "Element order invalid!\n";-                        std::cout << " @" << this << " key " << i << " is " << keys[i] << " vs "-                                  << keys[i + 1] << "\n";-                        valid = false;-                    }-                }-            }--            // check state of sub-nodes-            if (this->inner) {-                for (unsigned i = 0; i <= this->numElements; i++) {-                    valid &= getChildren()[i]->check(comp, root);-                }-            }--            return valid;-        }-    };  // namespace detail--    /**-     * The data type representing inner nodes of the b-tree. It extends-     * the generic implementation of a node by the storage locations-     * of child pointers.-     */-    struct inner_node : public node {-        // references to child nodes owned by this node-        node* children[node::maxKeys + 1];--        // a simple default constructor initializing member fields-        inner_node() : node(true) {}--        // clear up child nodes recursively-        ~inner_node() {-            for (unsigned i = 0; i <= this->numElements; ++i) {-                if (children[i] != nullptr) {-                    if (children[i]->isLeaf()) {-                        delete static_cast<leaf_node*>(children[i]);-                    } else {-                        delete static_cast<inner_node*>(children[i]);-                    }-                }-            }-        }-    };--    /**-     * The data type representing leaf nodes of the b-tree. It does not-     * add any capabilities to the generic node type.-     */-    struct leaf_node : public node {-        // a simple default constructor initializing member fields-        leaf_node() : node(false) {}-    };--    // ------------------- iterators --------------------------public:-    /**-     * The iterator type to be utilized for scanning through btree instances.-     */-    class iterator : public std::iterator<std::forward_iterator_tag, Key> {-        // a pointer to the node currently referred to-        node const* cur;--        // the index of the element currently addressed within the referenced node-        field_index_type pos = 0;--    public:-        // default constructor -- creating an end-iterator-        iterator() : cur(nullptr) {}--        // creates an iterator referencing a specific element within a given node-        iterator(node const* cur, field_index_type pos) : cur(cur), pos(pos) {}--        // a copy constructor-        iterator(const iterator& other) : cur(other.cur), pos(other.pos) {}--        // an assignment operator-        iterator& operator=(const iterator& other) {-            cur = other.cur;-            pos = other.pos;-            return *this;-        }--        // the equality operator as required by the iterator concept-        bool operator==(const iterator& other) const {-            return cur == other.cur && pos == other.pos;-        }--        // the not-equality operator as required by the iterator concept-        bool operator!=(const iterator& other) const {-            return !(*this == other);-        }--        // the deref operator as required by the iterator concept-        const Key& operator*() const {-            return cur->keys[pos];-        }--        // the increment operator as required by the iterator concept-        iterator& operator++() {-            // the quick mode -- if in a leaf and there are elements left-            if (cur->isLeaf() && ++pos < cur->getNumElements()) {-                return *this;-            }--            // otherwise it is a bit more tricky--            // A) currently in an inner node => go to the left-most child-            if (cur->isInner()) {-                cur = cur->getChildren()[pos + 1];-                while (!cur->isLeaf()) {-                    cur = cur->getChildren()[0];-                }-                pos = 0;--                // nodes may be empty due to biased insertion-                if (!cur->isEmpty()) {-                    return *this;-                }-            }--            // B) we are at the right-most element of a leaf => go to next inner node-            assert(cur->isLeaf());-            assert(pos == cur->getNumElements());--            while (cur != nullptr && pos == cur->getNumElements()) {-                pos = cur->getPositionInParent();-                cur = cur->getParent();-            }-            return *this;-        }--        // prints a textual representation of this iterator to the given stream (mainly for debugging)-        void print(std::ostream& out = std::cout) const {-            out << cur << "[" << (int)pos << "]";-        }-    };--    /**-     * A collection of operation hints speeding up some of the involved operations-     * by exploiting temporal locality.-     */-    template <unsigned size = 1>-    struct btree_operation_hints {-        using node_cache = LRUCache<node*, size>;--        // the node where the last insertion terminated-        node_cache last_insert;--        // the node where the last find-operation terminated-        node_cache last_find_end;--        // the node where the last lower-bound operation terminated-        node_cache last_lower_bound_end;--        // the node where the last upper-bound operation terminated-        node_cache last_upper_bound_end;--        // default constructor-        btree_operation_hints() = default;--        // resets all hints (to be triggered e.g. when deleting nodes)-        void clear() {-            last_insert.clear(nullptr);-            last_find_end.clear(nullptr);-            last_lower_bound_end.clear(nullptr);-            last_upper_bound_end.clear(nullptr);-        }-    };--    using operation_hints = btree_operation_hints<1>;--protected:-#ifdef IS_PARALLEL-    // a pointer to the root node of this tree-    node* volatile root;--    // a lock to synchronize update operations on the root pointer-    lock_type root_lock;-#else-    // a pointer to the root node of this tree-    node* root;--    // required to not duplicate too much code-    lock_type root_lock;-#endif--    // a pointer to the left-most node of this tree (initial note for iteration)-    leaf_node* leftmost;--    /* -------------- operator hint statistics ----------------- */--    // an aggregation of statistical values of the hint utilization-    struct hint_statistics {-        // the counter for insertion operations-        CacheAccessCounter inserts;--        // the counter for contains operations-        CacheAccessCounter contains;--        // the counter for lower_bound operations-        CacheAccessCounter lower_bound;--        // the counter for upper_bound operations-        CacheAccessCounter upper_bound;-    };--    // the hint statistic of this b-tree instance-    mutable hint_statistics hint_stats;--public:-    // the maximum number of keys stored per node-    static constexpr size_t max_keys_per_node = node::maxKeys;--    // -- ctors / dtors ----    // the default constructor creating an empty tree-    btree(Comparator comp = Comparator(), WeakComparator weak_comp = WeakComparator())-            : comp(std::move(comp)), weak_comp(std::move(weak_comp)), root(nullptr), leftmost(nullptr) {}--    // a constructor creating a tree from the given iterator range-    template <typename Iter>-    btree(const Iter& a, const Iter& b) : root(nullptr), leftmost(nullptr) {-        insert(a, b);-    }--    // a move constructor-    btree(btree&& other)-            : comp(other.comp), weak_comp(other.weak_comp), root(other.root), leftmost(other.leftmost) {-        other.root = nullptr;-        other.leftmost = nullptr;-    }--    // a copy constructor-    btree(const btree& set) : comp(set.comp), weak_comp(set.weak_comp), root(nullptr), leftmost(nullptr) {-        // use assignment operator for a deep copy-        *this = set;-    }--protected:-    /**-     * An internal constructor enabling the specific creation of a tree-     * based on internal parameters.-     */-    btree(size_type /* size */, node* root, leaf_node* leftmost) : root(root), leftmost(leftmost) {}--public:-    // the destructor freeing all contained nodes-    ~btree() {-        clear();-    }--    // -- mutators and observers ----    // emptiness check-    bool empty() const {-        return root == nullptr;-    }--    // determines the number of elements in this tree-    size_type size() const {-        return (root) ? root->countEntries() : 0;-    }--    /**-     * Inserts the given key into this tree.-     */-    bool insert(const Key& k) {-        operation_hints hints;-        return insert(k, hints);-    }--    /**-     * Inserts the given key into this tree.-     */-    bool insert(const Key& k, operation_hints& hints) {-#ifdef IS_PARALLEL--        // special handling for inserting first element-        while (root == nullptr) {-            // try obtaining root-lock-            if (!root_lock.try_start_write()) {-                // somebody else was faster => re-check-                continue;-            }--            // check loop condition again-            if (root != nullptr) {-                // somebody else was faster => normal insert-                root_lock.end_write();-                break;-            }--            // create new node-            leftmost = new leaf_node();-            leftmost->numElements = 1;-            leftmost->keys[0] = k;-            root = leftmost;--            // operation complete => we can release the root lock-            root_lock.end_write();--            hints.last_insert.access(leftmost);--            return true;-        }--        // insert using iterative implementation--        node* cur = nullptr;--        // test last insert hints-        lock_type::Lease cur_lease;--        auto checkHint = [&](node* last_insert) {-            // ignore null pointer-            if (!last_insert) return false;-            // get a read lease on indicated node-            auto hint_lease = last_insert->lock.start_read();-            // check whether it covers the key-            if (!weak_covers(last_insert, k)) return false;-            // and if there was no concurrent modification-            if (!last_insert->lock.validate(hint_lease)) return false;-            // use hinted location-            cur = last_insert;-            // and keep lease-            cur_lease = hint_lease;-            // we found a hit-            return true;-        };--        if (hints.last_insert.any(checkHint)) {-            // register this as a hit-            hint_stats.inserts.addHit();-        } else {-            // register this as a miss-            hint_stats.inserts.addMiss();-        }--        // if there is no valid hint ..-        if (!cur) {-            do {-                // get root - access lock-                auto root_lease = root_lock.start_read();--                // start with root-                cur = root;--                // get lease of the next node to be accessed-                cur_lease = cur->lock.start_read();--                // check validity of root pointer-                if (root_lock.end_read(root_lease)) {-                    break;-                }--            } while (true);-        }--        while (true) {-            // handle inner nodes-            if (cur->inner) {-                auto a = &(cur->keys[0]);-                auto b = &(cur->keys[cur->numElements]);--                auto pos = search.lower_bound(k, a, b, weak_comp);-                auto idx = pos - a;--                // early exit for sets-                if (isSet && pos != b && weak_equal(*pos, k)) {-                    // validate results-                    if (!cur->lock.validate(cur_lease)) {-                        // start over again-                        return insert(k, hints);-                    }--                    // update provenance information-                    if (typeid(Comparator) != typeid(WeakComparator) && less(k, *pos)) {-                        if (!cur->lock.try_upgrade_to_write(cur_lease)) {-                            // start again-                            return insert(k, hints);-                        }-                        update(*pos, k);-                        cur->lock.end_write();-                        return true;-                    }--                    // we found the element => no check of lock necessary-                    return false;-                }--                // get next pointer-                auto next = cur->getChild(idx);--                // get lease on next level-                auto next_lease = next->lock.start_read();--                // check whether there was a write-                if (!cur->lock.end_read(cur_lease)) {-                    // start over-                    return insert(k, hints);-                }--                // go to next-                cur = next;--                // move on lease-                cur_lease = next_lease;--                continue;-            }--            // the rest is for leaf nodes-            assert(!cur->inner);--            // -- insert node in leaf node ----            auto a = &(cur->keys[0]);-            auto b = &(cur->keys[cur->numElements]);--            auto pos = search.upper_bound(k, a, b, weak_comp);-            auto idx = pos - a;--            // early exit for sets-            if (isSet && pos != a && weak_equal(*(pos - 1), k)) {-                // validate result-                if (!cur->lock.validate(cur_lease)) {-                    // start over again-                    return insert(k, hints);-                }--                // update provenance information-                if (typeid(Comparator) != typeid(WeakComparator) && less(k, *(pos - 1))) {-                    if (!cur->lock.try_upgrade_to_write(cur_lease)) {-                        // start again-                        return insert(k, hints);-                    }-                    update(*(pos - 1), k);-                    cur->lock.end_write();-                    return true;-                }--                // we found the element => done-                return false;-            }--            // upgrade to write-permission-            if (!cur->lock.try_upgrade_to_write(cur_lease)) {-                // something has changed => restart-                hints.last_insert.access(cur);-                return insert(k, hints);-            }--            if (cur->numElements >= node::maxKeys) {-                // -- lock parents ---                auto priv = cur;-                auto parent = priv->parent;-                std::vector<node*> parents;-                do {-                    if (parent) {-                        parent->lock.start_write();-                        while (true) {-                            // check whether parent is correct-                            if (parent == priv->parent) {-                                break;-                            }-                            // switch parent-                            parent->lock.abort_write();-                            parent = priv->parent;-                            parent->lock.start_write();-                        }-                    } else {-                        // lock root lock => since cur is root-                        root_lock.start_write();-                    }--                    // record locked node-                    parents.push_back(parent);--                    // stop at "sphere of influence"-                    if (!parent || !parent->isFull()) {-                        break;-                    }--                    // go one step higher-                    priv = parent;-                    parent = parent->parent;--                } while (true);--                // split this node-                auto old_root = root;-                idx -= cur->rebalance_or_split(const_cast<node**>(&root), root_lock, idx, parents);--                // release parent lock-                for (auto it = parents.rbegin(); it != parents.rend(); ++it) {-                    auto parent = *it;--                    // release this lock-                    if (parent) {-                        parent->lock.end_write();-                    } else {-                        if (old_root != root) {-                            root_lock.end_write();-                        } else {-                            root_lock.abort_write();-                        }-                    }-                }--                // insert element in right fragment-                if (((size_type)idx) > cur->numElements) {-                    // release current lock-                    cur->lock.end_write();--                    // insert in sibling-                    return insert(k, hints);-                }-            }--            // ok - no split necessary-            assert(cur->numElements < node::maxKeys && "Split required!");--            // move keys-            for (int j = cur->numElements; j > idx; --j) {-                cur->keys[j] = cur->keys[j - 1];-            }--            // insert new element-            cur->keys[idx] = k;-            cur->numElements++;--            // release lock on current node-            cur->lock.end_write();--            // remember last insertion position-            hints.last_insert.access(cur);-            return true;-        }--#else-        // special handling for inserting first element-        if (empty()) {-            // create new node-            leftmost = new leaf_node();-            leftmost->numElements = 1;-            leftmost->keys[0] = k;-            root = leftmost;--            hints.last_insert.access(leftmost);--            return true;-        }--        // insert using iterative implementation-        node* cur = root;--        auto checkHints = [&](node* last_insert) {-            if (!last_insert) return false;-            if (!weak_covers(last_insert, k)) return false;-            cur = last_insert;-            return true;-        };--        // test last insert-        if (hints.last_insert.any(checkHints)) {-            hint_stats.inserts.addHit();-        } else {-            hint_stats.inserts.addMiss();-        }--        while (true) {-            // handle inner nodes-            if (cur->inner) {-                auto a = &(cur->keys[0]);-                auto b = &(cur->keys[cur->numElements]);--                auto pos = search.lower_bound(k, a, b, weak_comp);-                auto idx = pos - a;--                // early exit for sets-                if (isSet && pos != b && weak_equal(*pos, k)) {-                    // update provenance information-                    if (typeid(Comparator) != typeid(WeakComparator) && less(k, *pos)) {-                        update(*pos, k);-                        return true;-                    }--                    return false;-                }--                cur = cur->getChild(idx);-                continue;-            }--            // the rest is for leaf nodes-            assert(!cur->inner);--            // -- insert node in leaf node ----            auto a = &(cur->keys[0]);-            auto b = &(cur->keys[cur->numElements]);--            auto pos = search.upper_bound(k, a, b, weak_comp);-            auto idx = pos - a;--            // early exit for sets-            if (isSet && pos != a && weak_equal(*(pos - 1), k)) {-                // update provenance information-                if (typeid(Comparator) != typeid(WeakComparator) && less(k, *(pos - 1))) {-                    update(*(pos - 1), k);-                    return true;-                }--                return false;-            }--            if (cur->numElements >= node::maxKeys) {-                // split this node-                idx -= cur->rebalance_or_split(&root, root_lock, idx);--                // insert element in right fragment-                if (((size_type)idx) > cur->numElements) {-                    idx -= cur->numElements + 1;-                    cur = cur->parent->getChild(cur->position + 1);-                }-            }--            // ok - no split necessary-            assert(cur->numElements < node::maxKeys && "Split required!");--            // move keys-            for (int j = cur->numElements; j > idx; --j) {-                cur->keys[j] = cur->keys[j - 1];-            }--            // insert new element-            cur->keys[idx] = k;-            cur->numElements++;--            // remember last insertion position-            hints.last_insert.access(cur);--            return true;-        }-#endif-    }--    /**-     * Inserts the given range of elements into this tree.-     */-    template <typename Iter>-    void insert(const Iter& a, const Iter& b) {-        // TODO: improve this beyond a naive insert-        operation_hints hints;-        // a naive insert so far .. seems to work fine-        for (auto it = a; it != b; ++it) {-            // use insert with hint-            insert(*it, hints);-        }-    }--    // Obtains an iterator referencing the first element of the tree.-    iterator begin() const {-        return iterator(leftmost, 0);-    }--    // Obtains an iterator referencing the position after the last element of the tree.-    iterator end() const {-        return iterator();-    }--    /**-     * Partitions the full range of this set into up to a given number of chunks.-     * The chunks will cover approximately the same number of elements. Also, the-     * number of chunks will only approximate the desired number of chunks.-     *-     * @param num .. the number of chunks requested-     * @return a list of chunks partitioning this tree-     */-    std::vector<chunk> partition(size_type num) const {-        return getChunks(num);-    }--    std::vector<chunk> getChunks(size_type num) const {-        std::vector<chunk> res;-        if (empty()) {-            return res;-        }-        return root->collectChunks(res, num, begin(), end());-    }--    /**-     * Determines whether the given element is a member of this tree.-     */-    bool contains(const Key& k) const {-        operation_hints hints;-        return contains(k, hints);-    }--    /**-     * Determines whether the given element is a member of this tree.-     */-    bool contains(const Key& k, operation_hints& hints) const {-        return find(k, hints) != end();-    }--    /**-     * Locates the given key within this tree and returns an iterator-     * referencing its position. If not found, an end-iterator will be returned.-     */-    iterator find(const Key& k) const {-        operation_hints hints;-        return find(k, hints);-    }--    /**-     * Locates the given key within this tree and returns an iterator-     * referencing its position. If not found, an end-iterator will be returned.-     */-    iterator find(const Key& k, operation_hints& hints) const {-        if (empty()) {-            return end();-        }--        node* cur = root;--        auto checkHints = [&](node* last_find_end) {-            if (!last_find_end) return false;-            if (!covers(last_find_end, k)) return false;-            cur = last_find_end;-            return true;-        };--        // test last location searched (temporal locality)-        if (hints.last_find_end.any(checkHints)) {-            // register it as a hit-            hint_stats.contains.addHit();-        } else {-            // register it as a miss-            hint_stats.contains.addMiss();-        }--        // an iterative implementation (since 2/7 faster than recursive)--        while (true) {-            auto a = &(cur->keys[0]);-            auto b = &(cur->keys[cur->numElements]);--            auto pos = search(k, a, b, comp);--            if (pos < b && equal(*pos, k)) {-                hints.last_find_end.access(cur);-                return iterator(cur, pos - a);-            }--            if (!cur->inner) {-                hints.last_find_end.access(cur);-                return end();-            }--            // continue search in child node-            cur = cur->getChild(pos - a);-        }-    }--    /**-     * Obtains a lower boundary for the given key -- hence an iterator referencing-     * the smallest value that is not less the given key. If there is no such element,-     * an end-iterator will be returned.-     */-    iterator lower_bound(const Key& k) const {-        operation_hints hints;-        return lower_bound(k, hints);-    }--    /**-     * Obtains a lower boundary for the given key -- hence an iterator referencing-     * the smallest value that is not less the given key. If there is no such element,-     * an end-iterator will be returned.-     */-    iterator lower_bound(const Key& k, operation_hints& hints) const {-        if (empty()) {-            return end();-        }--        node* cur = root;--        auto checkHints = [&](node* last_lower_bound_end) {-            if (!last_lower_bound_end) return false;-            if (!covers(last_lower_bound_end, k)) return false;-            cur = last_lower_bound_end;-            return true;-        };--        // test last searched node-        if (hints.last_lower_bound_end.any(checkHints)) {-            hint_stats.lower_bound.addHit();-        } else {-            hint_stats.lower_bound.addMiss();-        }--        iterator res = end();-        while (true) {-            auto a = &(cur->keys[0]);-            auto b = &(cur->keys[cur->numElements]);--            auto pos = search.lower_bound(k, a, b, comp);-            auto idx = pos - a;--            if (!cur->inner) {-                hints.last_lower_bound_end.access(cur);-                return (pos != b) ? iterator(cur, idx) : res;-            }--            if (isSet && pos != b && equal(*pos, k)) {-                return iterator(cur, idx);-            }--            if (pos != b) {-                res = iterator(cur, idx);-            }--            cur = cur->getChild(idx);-        }-    }--    /**-     * Obtains an upper boundary for the given key -- hence an iterator referencing-     * the first element that the given key is less than the referenced value. If-     * there is no such element, an end-iterator will be returned.-     */-    iterator upper_bound(const Key& k) const {-        operation_hints hints;-        return upper_bound(k, hints);-    }--    /**-     * Obtains an upper boundary for the given key -- hence an iterator referencing-     * the first element that the given key is less than the referenced value. If-     * there is no such element, an end-iterator will be returned.-     */-    iterator upper_bound(const Key& k, operation_hints& hints) const {-        if (empty()) {-            return end();-        }--        node* cur = root;--        auto checkHints = [&](node* last_upper_bound_end) {-            if (!last_upper_bound_end) return false;-            if (!coversUpperBound(last_upper_bound_end, k)) return false;-            cur = last_upper_bound_end;-            return true;-        };--        // test last search node-        if (hints.last_upper_bound_end.any(checkHints)) {-            hint_stats.upper_bound.addHit();-        } else {-            hint_stats.upper_bound.addMiss();-        }--        iterator res = end();-        while (true) {-            auto a = &(cur->keys[0]);-            auto b = &(cur->keys[cur->numElements]);--            auto pos = search.upper_bound(k, a, b, comp);-            auto idx = pos - a;--            if (!cur->inner) {-                hints.last_upper_bound_end.access(cur);-                return (pos != b) ? iterator(cur, idx) : res;-            }--            if (pos != b) {-                res = iterator(cur, idx);-            }--            cur = cur->getChild(idx);-        }-    }--    /**-     * Clears this tree.-     */-    void clear() {-        if (root != nullptr) {-            if (root->isLeaf()) {-                delete static_cast<leaf_node*>(root);-            } else {-                delete static_cast<inner_node*>(root);-            }-        }-        root = nullptr;-        leftmost = nullptr;-    }--    /**-     * Swaps the content of this tree with the given tree. This-     * is a much more efficient operation than creating a copy and-     * realizing the swap utilizing assignment operations.-     */-    void swap(btree& other) {-        // swap the content-        std::swap(root, other.root);-        std::swap(leftmost, other.leftmost);-    }--    // Implementation of the assignment operation for trees.-    btree& operator=(const btree& other) {-        // check identity-        if (this == &other) {-            return *this;-        }--        // create a deep-copy of the content of the other tree-        // shortcut for empty sets-        if (other.empty()) {-            return *this;-        }--        // clone content (deep copy)-        root = other.root->clone();--        // update leftmost reference-        auto tmp = root;-        while (!tmp->isLeaf()) {-            tmp = tmp->getChild(0);-        }-        leftmost = static_cast<leaf_node*>(tmp);--        // done-        return *this;-    }--    // Implementation of an equality operation for trees.-    bool operator==(const btree& other) const {-        // check identity-        if (this == &other) {-            return true;-        }--        // check size-        if (size() != other.size()) {-            return false;-        }-        if (size() < other.size()) {-            return other == *this;-        }--        // check content-        for (const auto& key : other) {-            if (!contains(key)) {-                return false;-            }-        }-        return true;-    }--    // Implementation of an inequality operation for trees.-    bool operator!=(const btree& other) const {-        return !(*this == other);-    }--    // -- for debugging ----    // Determines the number of levels contained in this tree.-    size_type getDepth() const {-        return (empty()) ? 0 : root->getDepth();-    }--    // Determines the number of nodes contained in this tree.-    size_type getNumNodes() const {-        return (empty()) ? 0 : root->countNodes();-    }--    // Determines the amount of memory used by this data structure-    size_type getMemoryUsage() const {-        return sizeof(*this) + (empty() ? 0 : root->getMemoryUsage());-    }--    /*-     * Prints a textual representation of this tree to the given-     * output stream (mostly for debugging and tuning).-     */-    void printTree(std::ostream& out = std::cout) const {-        out << "B-Tree with " << size() << " elements:\n";-        if (empty()) {-            out << " - empty - \n";-        } else {-            root->printTree(out, "");-        }-    }--    /**-     * Prints a textual summary of statistical properties of this-     * tree to the given output stream (for debugging and tuning).-     */-    void printStats(std::ostream& out = std::cout) const {-        auto nodes = getNumNodes();-        out << " ---------------------------------\n";-        out << "  Elements: " << size() << "\n";-        out << "  Depth:    " << (empty() ? 0 : root->getDepth()) << "\n";-        out << "  Nodes:    " << nodes << "\n";-        out << " ---------------------------------\n";-        out << "  Size of inner node: " << sizeof(inner_node) << "\n";-        out << "  Size of leaf node:  " << sizeof(leaf_node) << "\n";-        out << "  Size of Key:        " << sizeof(Key) << "\n";-        out << "  max keys / node:  " << node::maxKeys << "\n";-        out << "  avg keys / node:  " << (size() / (double)nodes) << "\n";-        out << "  avg filling rate: " << ((size() / (double)nodes) / node::maxKeys) << "\n";-        out << " ---------------------------------\n";-        out << "  insert-hint (hits/misses/total): " << hint_stats.inserts.getHits() << "/"-            << hint_stats.inserts.getMisses() << "/" << hint_stats.inserts.getAccesses() << "\n";-        out << "  contains-hint(hits/misses/total):" << hint_stats.contains.getHits() << "/"-            << hint_stats.contains.getMisses() << "/" << hint_stats.contains.getAccesses() << "\n";-        out << "  lower-bound-hint (hits/misses/total):" << hint_stats.lower_bound.getHits() << "/"-            << hint_stats.lower_bound.getMisses() << "/" << hint_stats.lower_bound.getAccesses() << "\n";-        out << "  upper-bound-hint (hits/misses/total):" << hint_stats.upper_bound.getHits() << "/"-            << hint_stats.upper_bound.getMisses() << "/" << hint_stats.upper_bound.getAccesses() << "\n";-        out << " ---------------------------------\n";-    }--    /**-     * Checks the consistency of this tree.-     */-    bool check() {-        auto ok = empty() || root->check(comp, root);-        if (!ok) {-            printTree();-        }-        return ok;-    }--    /**-     * A static member enabling the bulk-load of ordered data into an empty-     * tree. This function is much more efficient in creating a index over-     * an ordered set of elements than an iterative insertion of values.-     *-     * @tparam Iter .. the type of iterator specifying the range-     *                     it must be a random-access iterator-     */-    template <typename R, typename Iter>-    static typename std::enable_if<std::is_same<typename std::iterator_traits<Iter>::iterator_category,-                                           std::random_access_iterator_tag>::value,-            R>::type-    load(const Iter& a, const Iter& b) {-        // quick exit - empty range-        if (a == b) {-            return R();-        }--        // resolve tree recursively-        auto root = buildSubTree(a, b - 1);--        // find leftmost node-        node* leftmost = root;-        while (!leftmost->isLeaf()) {-            leftmost = leftmost->getChild(0);-        }--        // build result-        return R(b - a, root, static_cast<leaf_node*>(leftmost));-    }--protected:-    /**-     * Determines whether the range covered by the given node is also-     * covering the given key value.-     */-    bool covers(const node* node, const Key& k) const {-        if (isSet) {-            // in sets we can include the ends as covered elements-            return !node->isEmpty() && !less(k, node->keys[0]) && !less(node->keys[node->numElements - 1], k);-        }-        // in multi-sets the ends may not be completely covered-        return !node->isEmpty() && less(node->keys[0], k) && less(k, node->keys[node->numElements - 1]);-    }--    /**-     * Determines whether the range covered by the given node is also-     * covering the given key value.-     */-    bool weak_covers(const node* node, const Key& k) const {-        if (isSet) {-            // in sets we can include the ends as covered elements-            return !node->isEmpty() && !weak_less(k, node->keys[0]) &&-                   !weak_less(node->keys[node->numElements - 1], k);-        }-        // in multi-sets the ends may not be completely covered-        return !node->isEmpty() && weak_less(node->keys[0], k) &&-               weak_less(k, node->keys[node->numElements - 1]);-    }--private:-    /**-     * Determines whether the range covered by this node covers-     * the upper bound of the given key.-     */-    bool coversUpperBound(const node* node, const Key& k) const {-        // ignore edges-        return !node->isEmpty() && !less(k, node->keys[0]) && less(k, node->keys[node->numElements - 1]);-    }--    // Utility function for the load operation above.-    template <typename Iter>-    static node* buildSubTree(const Iter& a, const Iter& b) {-        const int N = node::maxKeys;--        // divide range in N+1 sub-ranges-        int length = (b - a) + 1;--        // terminal case: length is less then maxKeys-        if (length <= N) {-            // create a leaf node-            node* res = new leaf_node();-            res->numElements = length;--            for (int i = 0; i < length; ++i) {-                res->keys[i] = a[i];-            }--            return res;-        }--        // recursive case - compute step size-        int numKeys = N;-        int step = ((length - numKeys) / (numKeys + 1));--        while (numKeys > 1 && (step < N / 2)) {-            numKeys--;-            step = ((length - numKeys) / (numKeys + 1));-        }--        // create inner node-        node* res = new inner_node();-        res->numElements = numKeys;--        Iter c = a;-        for (int i = 0; i < numKeys; i++) {-            // get dividing key-            res->keys[i] = c[step];--            // get sub-tree-            auto child = buildSubTree(c, c + (step - 1));-            child->parent = res;-            child->position = i;-            res->getChildren()[i] = child;--            c = c + (step + 1);-        }--        // and the remaining part-        auto child = buildSubTree(c, b);-        child->parent = res;-        child->position = numKeys;-        res->getChildren()[numKeys] = child;--        // done-        return res;-    }-};  // namespace souffle--// Instantiation of static member search.-template <typename Key, typename Comparator, typename Allocator, unsigned blockSize, typename SearchStrategy,-        bool isSet, typename WeakComparator, typename Updater>-const SearchStrategy-        btree<Key, Comparator, Allocator, blockSize, SearchStrategy, isSet, WeakComparator, Updater>::search;--}  // end namespace detail--/**- * A b-tree based set implementation.- *- * @tparam Key             .. the element type to be stored in this set- * @tparam Comparator     .. a class defining an order on the stored elements- * @tparam Allocator     .. utilized for allocating memory for required nodes- * @tparam blockSize    .. determines the number of bytes/block utilized by leaf nodes- * @tparam SearchStrategy .. enables switching between linear, binary or any other search strategy- */-template <typename Key, typename Comparator = detail::comparator<Key>,-        typename Allocator = std::allocator<Key>,  // is ignored so far-        unsigned blockSize = 256,-        typename SearchStrategy = typename souffle::detail::default_strategy<Key>::type,-        typename WeakComparator = Comparator, typename Updater = souffle::detail::updater<Key>>-class btree_set : public souffle::detail::btree<Key, Comparator, Allocator, blockSize, SearchStrategy, true,-                          WeakComparator, Updater> {-    using super = souffle::detail::btree<Key, Comparator, Allocator, blockSize, SearchStrategy, true,-            WeakComparator, Updater>;--    friend class souffle::detail::btree<Key, Comparator, Allocator, blockSize, SearchStrategy, true,-            WeakComparator, Updater>;--public:-    /**-     * A default constructor creating an empty set.-     */-    btree_set(const Comparator& comp = Comparator(), const WeakComparator& weak_comp = WeakComparator())-            : super(comp, weak_comp) {}--    /**-     * A constructor creating a set based on the given range.-     */-    template <typename Iter>-    btree_set(const Iter& a, const Iter& b) {-        this->insert(a, b);-    }--    // A copy constructor.-    btree_set(const btree_set& other) : super(other) {}--    // A move constructor.-    btree_set(btree_set&& other) : super(std::move(other)) {}--private:-    // A constructor required by the bulk-load facility.-    template <typename s, typename n, typename l>-    btree_set(s size, n* root, l* leftmost) : super(size, root, leftmost) {}--public:-    // Support for the assignment operator.-    btree_set& operator=(const btree_set& other) {-        super::operator=(other);-        return *this;-    }--    // Support for the bulk-load operator.-    template <typename Iter>-    static btree_set load(const Iter& a, const Iter& b) {-        return super::template load<btree_set>(a, b);-    }-};--/**- * A b-tree based multi-set implementation.- *- * @tparam Key             .. the element type to be stored in this set- * @tparam Comparator     .. a class defining an order on the stored elements- * @tparam Allocator     .. utilized for allocating memory for required nodes- * @tparam blockSize    .. determines the number of bytes/block utilized by leaf nodes- * @tparam SearchStrategy .. enables switching between linear, binary or any other search strategy- */-template <typename Key, typename Comparator = detail::comparator<Key>,-        typename Allocator = std::allocator<Key>,  // is ignored so far-        unsigned blockSize = 256,-        typename SearchStrategy = typename souffle::detail::default_strategy<Key>::type,-        typename WeakComparator = Comparator, typename Updater = souffle::detail::updater<Key>>-class btree_multiset : public souffle::detail::btree<Key, Comparator, Allocator, blockSize, SearchStrategy,-                               false, WeakComparator, Updater> {-    using super = souffle::detail::btree<Key, Comparator, Allocator, blockSize, SearchStrategy, false,-            WeakComparator, Updater>;--    friend class souffle::detail::btree<Key, Comparator, Allocator, blockSize, SearchStrategy, false,-            WeakComparator, Updater>;--public:-    /**-     * A default constructor creating an empty set.-     */-    btree_multiset(const Comparator& comp = Comparator(), const WeakComparator& weak_comp = WeakComparator())-            : super(comp, weak_comp) {}--    /**-     * A constructor creating a set based on the given range.-     */-    template <typename Iter>-    btree_multiset(const Iter& a, const Iter& b) {-        this->insert(a, b);-    }--    // A copy constructor.-    btree_multiset(const btree_multiset& other) : super(other) {}--    // A move constructor.-    btree_multiset(btree_multiset&& other) : super(std::move(other)) {}--private:-    // A constructor required by the bulk-load facility.-    template <typename s, typename n, typename l>-    btree_multiset(s size, n* root, l* leftmost) : super(size, root, leftmost) {}--public:-    // Support for the assignment operator.-    btree_multiset& operator=(const btree_multiset& other) {-        super::operator=(other);-        return *this;-    }--    // Support for the bulk-load operator.-    template <typename Iter>-    static btree_multiset load(const Iter& a, const Iter& b) {-        return super::template load<btree_multiset>(a, b);-    }-};--}  // end of namespace souffle
− cbits/souffle/Brie.h 
@@ -1,3166 +0,0 @@-/*- * Souffle - A Datalog Compiler- * Copyright (c) 2013, 2015, Oracle and/or its affiliates. All rights reserved- * Licensed under the Universal Permissive License v 1.0 as shown at:- * - https://opensource.org/licenses/UPL- * - <souffle root>/licenses/SOUFFLE-UPL.txt- */--/************************************************************************- *- * @file Brie.h- *- * This header file contains the implementation for a generic, fixed- * length integer trie.- *- * Tries trie is utilized to store n-ary tuples of integers. Each level- * is implemented via a sparse array (also covered by this header file),- * referencing the following nested level. The leaf level is realized- * by a sparse bit-map to minimize the memory footprint.- *- * Multiple insert operations can be be conducted concurrently on trie- * structures. So can read-only operations. However, inserts and read- * operations may not be conducted at the same time.- *- ***********************************************************************/--#pragma once--#include "CompiledTuple.h"-#include "RamTypes.h"-#include "utility/CacheUtil.h"-#include "utility/ContainerUtil.h"-#include "utility/StreamUtil.h"-#include <algorithm>-#include <atomic>-#include <bitset>-#include <cassert>-#include <cstdint>-#include <cstring>-#include <iostream>-#include <iterator>-#include <limits>-#include <utility>-#include <vector>--#ifdef _WIN32-/**- * When compiling for windows, redefine the gcc builtins which are used to- * their equivalents on the windows platform.- */-#define __sync_synchronize MemoryBarrier-#define __sync_bool_compare_and_swap(ptr, oldval, newval) \-    (InterlockedCompareExchangePointer((void* volatile*)ptr, (void*)newval, (void*)oldval) == (void*)oldval)-#endif  // _WIN32--namespace souffle {--namespace detail {--/**- * A templated functor to obtain default values for- * unspecified elements of sparse array instances.- */-template <typename T>-struct default_factory {-    T operator()() const {-        return T();  // just use the default constructor-    }-};--/**- * A functor representing the identity function.- */-template <typename T>-struct identity {-    T operator()(T v) const {-        return v;-    }-};--/**- * A operation to be utilized by the sparse map when merging- * elements associated to different values.- */-template <typename T>-struct default_merge {-    /**-     * Merges two values a and b when merging spase maps.-     */-    T operator()(T a, T b) const {-        default_factory<T> def;-        // if a is the default => us b, else stick to a-        return (a != def()) ? a : b;-    }-};--}  // end namespace detail--/**- * A sparse array simulates an array associating to every element- * of uint32_t an element of a generic type T. Any non-defined element- * will be default-initialized utilizing the detail::default_factory- * functor.- *- * Internally the array is organized as a balanced tree. The leaf- * level of the tree corresponds to the elements of the represented- * array. Inner nodes utilize individual bits of the indices to reference- * sub-trees. For efficiency reasons, only the minimal sub-tree required- * to cover all non-null / non-default values stored in the array is- * maintained. Furthermore, several levels of nodes are aggreated in a- * B-tree like fashion to inprove cache utilization and reduce the number- * of steps required for lookup and insert operations.- *- * @tparam T the type of the stored elements- * @tparam BITS the number of bits consumed per node-level- *              e.g. if it is set to 3, the resulting tree will be of a degree of- *              2^3=8, and thus 8 child-pointers will be stored in each inner node- *              and as many values will be stored in each leaf node.- * @tparam merge_op the functor to be utilized when merging the content of two- *              instances of this type.- * @tparam copy_op a functor to be applied to each stored value when copying an- *              instance of this array. For instance, this is utilized by the- *              trie implementation to create a clone of each sub-tree instead- *              of preserving the original pointer.- */-template <typename T, unsigned BITS = 6, typename merge_op = detail::default_merge<T>,-        typename copy_op = detail::identity<T>>-class SparseArray {-    using key_type = uint64_t;--    // some internal constants-    static constexpr int BIT_PER_STEP = BITS;-    static constexpr int NUM_CELLS = 1 << BIT_PER_STEP;-    static constexpr key_type INDEX_MASK = NUM_CELLS - 1;--public:-    // the type utilized for indexing contained elements-    using index_type = key_type;--    // the type of value stored in this array-    using value_type = T;--    // the atomic view on stored values-    using atomic_value_type = std::atomic<value_type>;--private:-    struct Node;--    /**-     * The value stored in a single cell of a inner-     * or leaf node.-     */-    union Cell {-        // an atomic view on the pointer referencing a nested level-        std::atomic<Node*> aptr;--        // a pointer to the nested level (unsynchronized operations)-        Node* ptr;--        // an atomic view on the value stored in this cell (leaf node)-        atomic_value_type avalue;--        // the value stored in this cell (unsynchronized access, leaf node)-        value_type value;-    };--    /**-     * The node type of the internally maintained tree.-     */-    struct Node {-        // a pointer to the parent node (for efficient iteration)-        const Node* parent;-        // the pointers to the child nodes (inner nodes) or the stored values (leaf nodes)-        Cell cell[NUM_CELLS];-    };--    /**-     * A struct describing all the information required by the container-     * class to manage the wrapped up tree.-     */-    struct RootInfo {-        // the root node of the tree-        Node* root;-        // the number of levels of the tree-        uint32_t levels;-        // the absolute offset of the theoretical first element in the tree-        index_type offset;--        // the first leaf node in the tree-        Node* first;-        // the absolute offset of the first element in the first leaf node-        index_type firstOffset;-    };--    union {-        RootInfo unsynced;         // for sequential operations-        volatile RootInfo synced;  // for synchronized operations-    };--public:-    /**-     * A default constructor creating an empty sparse array.-     */-    SparseArray() : unsynced(RootInfo{nullptr, 0, 0, nullptr, std::numeric_limits<index_type>::max()}) {}--    /**-     * A copy constructor for sparse arrays. It creates a deep-     * copy of the data structure maintained by the handed in-     * array instance.-     */-    SparseArray(const SparseArray& other)-            : unsynced(RootInfo{clone(other.unsynced.root, other.unsynced.levels), other.unsynced.levels,-                      other.unsynced.offset, nullptr, other.unsynced.firstOffset}) {-        if (unsynced.root) {-            unsynced.root->parent = nullptr;-            unsynced.first = findFirst(unsynced.root, unsynced.levels);-        }-    }--    /**-     * A r-value based copy constructor for sparse arrays. It-     * takes over ownership of the structure maintained by the-     * handed in array.-     */-    SparseArray(SparseArray&& other)-            : unsynced(RootInfo{other.unsynced.root, other.unsynced.levels, other.unsynced.offset,-                      other.unsynced.first, other.unsynced.firstOffset}) {-        other.unsynced.root = nullptr;-        other.unsynced.levels = 0;-        other.unsynced.first = nullptr;-    }--    /**-     * A destructor for sparse arrays clearing up the internally-     * maintained data structure.-     */-    ~SparseArray() {-        clean();-    }--    /**-     * An assignment creating a deep copy of the handed in-     * array structure (utilizing the copy functor provided-     * as a template parameter).-     */-    SparseArray& operator=(const SparseArray& other) {-        if (this == &other) return *this;--        // clean this one-        clean();--        // copy content-        unsynced.levels = other.unsynced.levels;-        unsynced.root = clone(other.unsynced.root, unsynced.levels);-        if (unsynced.root) {-            unsynced.root->parent = nullptr;-        }-        unsynced.offset = other.unsynced.offset;-        unsynced.first = (unsynced.root) ? findFirst(unsynced.root, unsynced.levels) : nullptr;-        unsynced.firstOffset = other.unsynced.firstOffset;--        // done-        return *this;-    }--    /**-     * An assignment operation taking over ownership-     * from a r-value reference to a sparse array.-     */-    SparseArray& operator=(SparseArray&& other) {-        // clean this one-        clean();--        // harvest content-        unsynced.root = other.unsynced.root;-        unsynced.levels = other.unsynced.levels;-        unsynced.offset = other.unsynced.offset;-        unsynced.first = other.unsynced.first;-        unsynced.firstOffset = other.unsynced.firstOffset;--        // reset other-        other.unsynced.root = nullptr;-        other.unsynced.levels = 0;-        other.unsynced.first = nullptr;--        // done-        return *this;-    }--    /**-     * Tests whether this sparse array is empty, thus it only-     * contains default-values, or not.-     */-    bool empty() const {-        return unsynced.root == nullptr;-    }--    /**-     * Computes the number of non-empty elements within this-     * sparse array.-     */-    std::size_t size() const {-        // quick one for the empty map-        if (empty()) return 0;--        // count elements -- since maintaining is making inserts more expensive-        std::size_t res = 0;-        for (auto it = begin(); it != end(); ++it) {-            ++res;-        }-        return res;-    }--private:-    /**-     * Computes the memory usage of the given sub-tree.-     */-    static std::size_t getMemoryUsage(const Node* node, int level) {-        // support null-nodes-        if (!node) return 0;--        // add size of current node-        std::size_t res = sizeof(Node);--        // sum up memory usage of child nodes-        if (level > 0) {-            for (int i = 0; i < NUM_CELLS; i++) {-                res += getMemoryUsage(node->cell[i].ptr, level - 1);-            }-        }--        // done-        return res;-    }--public:-    /**-     * Computes the total memory usage of this data structure.-     */-    std::size_t getMemoryUsage() const {-        // the memory of the wrapper class-        std::size_t res = sizeof(*this);--        // add nodes-        if (unsynced.root) {-            res += getMemoryUsage(unsynced.root, unsynced.levels);-        }--        // done-        return res;-    }--    /**-     * Resets the content of this array to default values for each contained-     * element.-     */-    void clear() {-        clean();-        unsynced.root = nullptr;-        unsynced.levels = 0;-        unsynced.first = nullptr;-        unsynced.firstOffset = std::numeric_limits<index_type>::max();-    }--    /**-     * A struct to be utilized as a local, temporal context by client code-     * to speed up the execution of various operations (optional parameter).-     */-    struct op_context {-        index_type lastIndex{0};-        Node* lastNode{nullptr};-        op_context() = default;-    };--private:-    // ----------------------------------------------------------------------    //              Optimistic Locking of Root-Level Infos-    // -----------------------------------------------------------------------    /**-     * A struct to cover a snapshot of the root node state.-     */-    struct RootInfoSnapshot {-        // the current pointer to a root node-        Node* root;-        // the current number of levels-        uint32_t levels;-        // the current offset of the first theoretical element-        index_type offset;-        // a version number for the optimistic locking-        uintptr_t version;-    };--    /**-     * Obtains the current version of the root.-     */-    uint64_t getRootVersion() const {-        // here it is assumed that the load of a 64-bit word is atomic-        return (uint64_t)synced.root;-    }--    /**-     * Obtains a snapshot of the current root information.-     */-    RootInfoSnapshot getRootInfo() const {-        RootInfoSnapshot res{};-        do {-            // first take the mod counter-            do {-                // if res.mod % 2 == 1 .. there is an update in progress-                res.version = getRootVersion();-            } while (res.version % 2);--            // then the rest-            res.root = synced.root;-            res.levels = synced.levels;-            res.offset = synced.offset;--            // check consistency of obtained data (optimistic locking)-        } while (res.version != getRootVersion());--        // got a consistent snapshot-        return res;-    }--    /**-     * Updates the current root information based on the handed in modified-     * snapshot instance if the version number of the snapshot still corresponds-     * to the current version. Otherwise a concurrent update took place and the-     * operation is aborted.-     *-     * @param info the updated information to be assigned to the active root-info data-     * @return true if successfully updated, false if aborted-     */-    bool tryUpdateRootInfo(const RootInfoSnapshot& info) {-        // check mod counter-        uintptr_t version = info.version;--        // update root to invalid pointer (ending with 1)-        if (!__sync_bool_compare_and_swap(&synced.root, (Node*)version, (Node*)(version + 1))) {-            return false;-        }--        // conduct update-        synced.levels = info.levels;-        synced.offset = info.offset;--        // update root (and thus the version to enable future retrievals)-        __sync_synchronize();-        synced.root = info.root;--        // done-        return true;-    }--    /**-     * A struct summarizing the state of the first node reference.-     */-    struct FirstInfoSnapshot {-        // the pointer to the first node-        Node* node;-        // the offset of the first node-        index_type offset;-        // the version number of the first node (for the optimistic locking)-        uintptr_t version;-    };--    /**-     * Obtains the current version number of the first node information.-     */-    uint64_t getFirstVersion() const {-        // here it is assumed that the load of a 64-bit word is atomic-        return (uint64_t)synced.first;-    }--    /**-     * Obtains a snapshot of the current first-node information.-     */-    FirstInfoSnapshot getFirstInfo() const {-        FirstInfoSnapshot res{};-        do {-            // first take the version-            do {-                res.version = getFirstVersion();-            } while (res.version % 2);--            // collect the values-            res.node = synced.first;-            res.offset = synced.firstOffset;--        } while (res.version != getFirstVersion());--        // we got a consistent snapshot-        return res;-    }--    /**-     * Updates the information stored regarding the first node in a-     * concurrent setting utilizing a optimistic locking approach.-     * This is identical to the approach utilized for the root info.-     */-    bool tryUpdateFirstInfo(const FirstInfoSnapshot& info) {-        // check mod counter-        uintptr_t version = info.version;--        // temporary update first pointer to point to uneven value (lock-out)-        if (!__sync_bool_compare_and_swap(&synced.first, (Node*)version, (Node*)(version + 1))) {-            return false;-        }--        // conduct update-        synced.firstOffset = info.offset;--        // update node pointer (and thus the version number)-        __sync_synchronize();-        synced.first = info.node;  // must be last (and atomic)--        // done-        return true;-    }--public:-    /**-     * Obtains a mutable reference to the value addressed by the given index.-     *-     * @param i the index of the element to be addressed-     * @return a mutable reference to the corresponding element-     */-    value_type& get(index_type i) {-        op_context ctxt;-        return get(i, ctxt);-    }--    /**-     * Obtains a mutable reference to the value addressed by the given index.-     *-     * @param i the index of the element to be addressed-     * @param ctxt a operation context to exploit state-less temporal locality-     * @return a mutable reference to the corresponding element-     */-    value_type& get(index_type i, op_context& ctxt) {-        return getLeaf(i, ctxt).value;-    }--    /**-     * Obtains a mutable reference to the atomic value addressed by the given index.-     *-     * @param i the index of the element to be addressed-     * @return a mutable reference to the corresponding element-     */-    atomic_value_type& getAtomic(index_type i) {-        op_context ctxt;-        return getAtomic(i, ctxt);-    }--    /**-     * Obtains a mutable reference to the atomic value addressed by the given index.-     *-     * @param i the index of the element to be addressed-     * @param ctxt a operation context to exploit state-less temporal locality-     * @return a mutable reference to the corresponding element-     */-    atomic_value_type& getAtomic(index_type i, op_context& ctxt) {-        return getLeaf(i, ctxt).avalue;-    }--private:-    /**-     * An internal function capable of navigating to a given leaf node entry.-     * If the cell does not exist yet it will be created as a side-effect.-     *-     * @param i the index of the requested cell-     * @param ctxt a operation context to exploit state-less temporal locality-     * @return a reference to the requested cell-     */-    inline Cell& getLeaf(index_type i, op_context& ctxt) {-        // check context-        if (ctxt.lastNode && (ctxt.lastIndex == (i & ~INDEX_MASK))) {-            // return reference to referenced-            return ctxt.lastNode->cell[i & INDEX_MASK];-        }--        // get snapshot of root-        auto info = getRootInfo();--        // check for emptiness-        if (info.root == nullptr) {-            // build new root node-            info.root = newNode();--            // initialize the new node-            info.root->parent = nullptr;-            info.offset = i & ~(INDEX_MASK);--            // try updating root information atomically-            if (tryUpdateRootInfo(info)) {-                // success -- finish get call--                // update first-                auto firstInfo = getFirstInfo();-                while (info.offset < firstInfo.offset) {-                    firstInfo.node = info.root;-                    firstInfo.offset = info.offset;-                    if (!tryUpdateFirstInfo(firstInfo)) {-                        // there was some concurrent update => check again-                        firstInfo = getFirstInfo();-                    }-                }--                // return reference to proper cell-                return info.root->cell[i & INDEX_MASK];-            }--            // somebody else was faster => use standard insertion procedure-            delete info.root;--            // retrieve new root info-            info = getRootInfo();--            // make sure there is a root-            assert(info.root);-        }--        // for all other inserts-        //   - check boundary-        //   - navigate to node-        //   - insert value--        // check boundaries-        while (!inBoundaries(i, info.levels, info.offset)) {-            // boundaries need to be expanded by growing upwards-            raiseLevel(info);  // try raising level unless someone else did already-            // update root info-            info = getRootInfo();-        }--        // navigate to node-        Node* node = info.root;-        unsigned level = info.levels;-        while (level != 0) {-            // get X coordinate-            auto x = getIndex(i, level);--            // decrease level counter-            --level;--            // check next node-            std::atomic<Node*>& aNext = node->cell[x].aptr;-            Node* next = aNext;-            if (!next) {-                // create new sub-tree-                Node* newNext = newNode();-                newNext->parent = node;--                // try to update next-                if (!aNext.compare_exchange_strong(next, newNext)) {-                    // some other thread was faster => use updated next-                    delete newNext;-                } else {-                    // the locally created next is the new next-                    next = newNext;--                    // update first-                    if (level == 0) {-                        // compute offset of this node-                        auto off = i & ~INDEX_MASK;--                        // fast over-approximation of whether a update is necessary-                        if (off < unsynced.firstOffset) {-                            // update first reference if this one is the smallest-                            auto first_info = getFirstInfo();-                            while (off < first_info.offset) {-                                first_info.node = next;-                                first_info.offset = off;-                                if (!tryUpdateFirstInfo(first_info)) {-                                    // there was some concurrent update => check again-                                    first_info = getFirstInfo();-                                }-                            }-                        }-                    }-                }--                // now next should be defined-                assert(next);-            }--            // continue one level below-            node = next;-        }--        // update context-        ctxt.lastIndex = (i & ~INDEX_MASK);-        ctxt.lastNode = node;--        // return reference to cell-        return node->cell[i & INDEX_MASK];-    }--public:-    /**-     * Updates the value stored in cell i by the given value.-     */-    void update(index_type i, const value_type& val) {-        op_context ctxt;-        update(i, val, ctxt);-    }--    /**-     * Updates the value stored in cell i by the given value. A operation-     * context can be provided for exploiting temporal locality.-     */-    void update(index_type i, const value_type& val, op_context& ctxt) {-        get(i, ctxt) = val;-    }--    /**-     * Obtains the value associated to index i -- which might be-     * the default value of the covered type if the value hasn't been-     * defined previously.-     */-    value_type operator[](index_type i) const {-        return lookup(i);-    }--    /**-     * Obtains the value associated to index i -- which might be-     * the default value of the covered type if the value hasn't been-     * defined previously.-     */-    value_type lookup(index_type i) const {-        op_context ctxt;-        return lookup(i, ctxt);-    }--    /**-     * Obtains the value associated to index i -- which might be-     * the default value of the covered type if the value hasn't been-     * defined previously. A operation context can be provided for-     * exploiting temporal locality.-     */-    value_type lookup(index_type i, op_context& ctxt) const {-        // check whether it is empty-        if (!unsynced.root) return souffle::detail::default_factory<value_type>()();--        // check boundaries-        if (!inBoundaries(i)) return souffle::detail::default_factory<value_type>()();--        // check context-        if (ctxt.lastNode && ctxt.lastIndex == (i & ~INDEX_MASK)) {-            return ctxt.lastNode->cell[i & INDEX_MASK].value;-        }--        // navigate to value-        Node* node = unsynced.root;-        unsigned level = unsynced.levels;-        while (level != 0) {-            // get X coordinate-            auto x = getIndex(i, level);--            // decrease level counter-            --level;--            // check next node-            Node* next = node->cell[x].ptr;--            // check next step-            if (!next) return souffle::detail::default_factory<value_type>()();--            // continue one level below-            node = next;-        }--        // remember context-        ctxt.lastIndex = (i & ~INDEX_MASK);-        ctxt.lastNode = node;--        // return reference to cell-        return node->cell[i & INDEX_MASK].value;-    }--private:-    /**-     * A static operation utilized internally for merging sub-trees recursively.-     *-     * @param parent the parent node of the current merge operation-     * @param trg a reference to the pointer the cloned node should be stored to-     * @param src the node to be cloned-     * @param levels the height of the cloned node-     */-    static void merge(const Node* parent, Node*& trg, const Node* src, int levels) {-        // if other side is null => done-        if (src == nullptr) {-            return;-        }--        // if the trg sub-tree is empty, clone the corresponding branch-        if (trg == nullptr) {-            trg = clone(src, levels);-            if (trg != nullptr) {-                trg->parent = parent;-            }-            return;  // done-        }--        // otherwise merge recursively--        // the leaf-node step-        if (levels == 0) {-            merge_op merg;-            for (int i = 0; i < NUM_CELLS; ++i) {-                trg->cell[i].value = merg(trg->cell[i].value, src->cell[i].value);-            }-            return;-        }--        // the recursive step-        for (int i = 0; i < NUM_CELLS; ++i) {-            merge(trg, trg->cell[i].ptr, src->cell[i].ptr, levels - 1);-        }-    }--public:-    /**-     * Adds all the values stored in the given array to this array.-     */-    void addAll(const SparseArray& other) {-        // skip if other is empty-        if (other.empty()) {-            return;-        }--        // special case: emptiness-        if (empty()) {-            // use assignment operator-            *this = other;-            return;-        }--        // adjust levels-        while (unsynced.levels < other.unsynced.levels || !inBoundaries(other.unsynced.offset)) {-            raiseLevel();-        }--        // navigate to root node equivalent of the other node in this tree-        auto level = unsynced.levels;-        Node** node = &unsynced.root;-        while (level > other.unsynced.levels) {-            // get X coordinate-            auto x = getIndex(other.unsynced.offset, level);--            // decrease level counter-            --level;--            // check next node-            Node*& next = (*node)->cell[x].ptr;-            if (!next) {-                // create new sub-tree-                next = newNode();-                next->parent = *node;-            }--            // continue one level below-            node = &next;-        }--        // merge sub-branches from here-        merge((*node)->parent, *node, other.unsynced.root, level);--        // update first-        if (unsynced.firstOffset > other.unsynced.firstOffset) {-            unsynced.first = findFirst(*node, level);-            unsynced.firstOffset = other.unsynced.firstOffset;-        }-    }--    // ----------------------------------------------------------------------    //                           Iterator-    // -----------------------------------------------------------------------    /**-     * The iterator type to be utilized to iterate over the non-default elements of this array.-     */-    class iterator {-        using pair_type = std::pair<index_type, value_type>;--        // a pointer to the leaf node currently processed or null (end)-        const Node* node;--        // the value currently pointed to-        pair_type value;--    public:-        // default constructor -- creating an end-iterator-        iterator() : node(nullptr) {}--        iterator(const Node* node, pair_type value) : node(node), value(std::move(value)) {}--        iterator(const Node* first, index_type firstOffset) : node(first), value(firstOffset, 0) {-            // if the start is the end => we are done-            if (!first) return;--            // load the value-            if (first->cell[0].value == value_type()) {-                ++(*this);  // walk to first element-            } else {-                value.second = first->cell[0].value;-            }-        }--        // a copy constructor-        iterator(const iterator& other) = default;--        // an assignment operator-        iterator& operator=(const iterator& other) = default;--        // the equality operator as required by the iterator concept-        bool operator==(const iterator& other) const {-            // only equivalent if pointing to the end-            return (node == nullptr && other.node == nullptr) ||-                   (node == other.node && value.first == other.value.first);-        }--        // the not-equality operator as required by the iterator concept-        bool operator!=(const iterator& other) const {-            return !(*this == other);-        }--        // the deref operator as required by the iterator concept-        const pair_type& operator*() const {-            return value;-        }--        // support for the pointer operator-        const pair_type* operator->() const {-            return &value;-        }--        // the increment operator as required by the iterator concept-        iterator& operator++() {-            // get current offset-            index_type x = value.first & INDEX_MASK;--            // go to next non-empty value in current node-            do {-                x++;-            } while (x < NUM_CELLS && node->cell[x].value == value_type());--            // check whether one has been found-            if (x < NUM_CELLS) {-                // update value and be done-                value.first = (value.first & ~INDEX_MASK) | x;-                value.second = node->cell[x].value;-                return *this;  // done-            }--            // go to parent-            node = node->parent;-            int level = 1;--            // get current index on this level-            x = getIndex(value.first, level);-            x++;--            while (level > 0 && node) {-                // search for next child-                while (x < NUM_CELLS) {-                    if (node->cell[x].ptr != nullptr) {-                        break;-                    }-                    x++;-                }--                // pick next step-                if (x < NUM_CELLS) {-                    // going down-                    node = node->cell[x].ptr;-                    value.first &= getLevelMask(level + 1);-                    value.first |= x << (BIT_PER_STEP * level);-                    level--;-                    x = 0;-                } else {-                    // going up-                    node = node->parent;-                    level++;--                    // get current index on this level-                    x = getIndex(value.first, level);-                    x++;  // go one step further-                }-            }--            // check whether it is the end of range-            if (node == nullptr) {-                return *this;-            }--            // search the first value in this node-            x = 0;-            while (node->cell[x].value == value_type()) {-                x++;-            }--            // update value-            value.first |= x;-            value.second = node->cell[x].value;--            // done-            return *this;-        }--        // True if this iterator is passed the last element.-        bool isEnd() const {-            return node == nullptr;-        }--        // enables this iterator core to be printed (for debugging)-        void print(std::ostream& out) const {-            out << "SparseArrayIter(" << node << " @ " << value << ")";-        }--        friend std::ostream& operator<<(std::ostream& out, const iterator& iter) {-            iter.print(out);-            return out;-        }-    };--    /**-     * Obtains an iterator referencing the first non-default element or end in-     * case there are no such elements.-     */-    iterator begin() const {-        return iterator(unsynced.first, unsynced.firstOffset);-    }--    /**-     * An iterator referencing the position after the last non-default element.-     */-    iterator end() const {-        return iterator();-    }--    /**-     * An operation to obtain an iterator referencing an element addressed by the-     * given index. If the corresponding element is a non-default value, a corresponding-     * iterator will be returned. Otherwise end() will be returned.-     */-    iterator find(index_type i) const {-        op_context ctxt;-        return find(i, ctxt);-    }--    /**-     * An operation to obtain an iterator referencing an element addressed by the-     * given index. If the corresponding element is a non-default value, a corresponding-     * iterator will be returned. Otherwise end() will be returned. A operation context-     * can be provided for exploiting temporal locality.-     */-    iterator find(index_type i, op_context& ctxt) const {-        // check whether it is empty-        if (!unsynced.root) return end();--        // check boundaries-        if (!inBoundaries(i)) return end();--        // check context-        if (ctxt.lastNode && ctxt.lastIndex == (i & ~INDEX_MASK)) {-            Node* node = ctxt.lastNode;--            // check whether there is a proper entry-            value_type value = node->cell[i & INDEX_MASK].value;-            if (value == value_type{}) {-                return end();-            }-            // return iterator pointing to value-            return iterator(node, std::make_pair(i, value));-        }--        // navigate to value-        Node* node = unsynced.root;-        unsigned level = unsynced.levels;-        while (level != 0) {-            // get X coordinate-            auto x = getIndex(i, level);--            // decrease level counter-            --level;--            // check next node-            Node* next = node->cell[x].ptr;--            // check next step-            if (!next) return end();--            // continue one level below-            node = next;-        }--        // register in context-        ctxt.lastNode = node;-        ctxt.lastIndex = (i & ~INDEX_MASK);--        // check whether there is a proper entry-        value_type value = node->cell[i & INDEX_MASK].value;-        if (value == value_type{}) {-            return end();-        }--        // return iterator pointing to cell-        return iterator(node, std::make_pair(i, value));-    }--    /**-     * An operation obtaining the smallest non-default element such that it's index is >=-     * the given index.-     */-    iterator lowerBound(index_type i) const {-        op_context ctxt;-        return lowerBound(i, ctxt);-    }--    /**-     * An operation obtaining the smallest non-default element such that it's index is >=-     * the given index. A operation context can be provided for exploiting temporal locality.-     */-    iterator lowerBound(index_type i, op_context&) const {-        // check whether it is empty-        if (!unsynced.root) return end();--        // check boundaries-        if (!inBoundaries(i)) {-            // if it is on the lower end, return minimum result-            if (i < unsynced.offset) {-                const auto& value = unsynced.first->cell[0].value;-                auto res = iterator(unsynced.first, std::make_pair(unsynced.offset, value));-                if (value == value_type()) {-                    ++res;-                }-                return res;-            }-            // otherwise it is on the high end, return end iterator-            return end();-        }--        // navigate to value-        Node* node = unsynced.root;-        unsigned level = unsynced.levels;-        while (true) {-            // get X coordinate-            auto x = getIndex(i, level);--            // check next node-            Node* next = node->cell[x].ptr;--            // check next step-            if (!next) {-                if (x == NUM_CELLS - 1) {-                    ++level;-                    node = const_cast<Node*>(node->parent);-                    if (!node) return end();-                }--                // continue search-                i = i & getLevelMask(level);--                // find next higher value-                i += 1ull << (BITS * level);--            } else {-                if (level == 0) {-                    // found boundary-                    return iterator(node, std::make_pair(i, node->cell[x].value));-                }--                // decrease level counter-                --level;--                // continue one level below-                node = next;-            }-        }-    }--    /**-     * An operation obtaining the smallest non-default element such that it's index is greater-     * the given index.-     */-    iterator upperBound(index_type i) const {-        op_context ctxt;-        return upperBound(i, ctxt);-    }--    /**-     * An operation obtaining the smallest non-default element such that it's index is greater-     * the given index. A operation context can be provided for exploiting temporal locality.-     */-    iterator upperBound(index_type i, op_context& ctxt) const {-        if (i == std::numeric_limits<index_type>::max()) {-            return end();-        }-        return lowerBound(i + 1, ctxt);-    }--private:-    /**-     * An internal debug utility printing the internal structure of this sparse array to the given output-     * stream.-     */-    void dump(bool detailed, std::ostream& out, const Node& node, int level, index_type offset,-            int indent = 0) const {-        auto x = getIndex(offset, level + 1);-        out << times("\t", indent) << x << ": Node " << &node << " on level " << level-            << " parent: " << node.parent << " -- range: " << offset << " - "-            << (offset + ~getLevelMask(level + 1)) << "\n";--        if (level == 0) {-            for (int i = 0; i < NUM_CELLS; i++) {-                if (detailed || node.cell[i].value != value_type()) {-                    out << times("\t", indent + 1) << i << ": [" << (offset + i) << "] " << node.cell[i].value-                        << "\n";-                }-            }-        } else {-            for (int i = 0; i < NUM_CELLS; i++) {-                if (node.cell[i].ptr) {-                    dump(detailed, out, *node.cell[i].ptr, level - 1,-                            offset + (i * (index_type(1) << (level * BIT_PER_STEP))), indent + 1);-                } else if (detailed) {-                    auto low = offset + (i * (1 << (level * BIT_PER_STEP)));-                    auto hig = low + ~getLevelMask(level);-                    out << times("\t", indent + 1) << i << ": empty range " << low << " - " << hig << "\n";-                }-            }-        }-        out << "\n";-    }--public:-    /**-     * A debug utility printing the internal structure of this sparse array to the given output stream.-     */-    void dump(bool detail = false, std::ostream& out = std::cout) const {-        if (!unsynced.root) {-            out << " - empty - \n";-            return;-        }-        out << "root:  " << unsynced.root << "\n";-        out << "offset: " << unsynced.offset << "\n";-        out << "first: " << unsynced.first << "\n";-        out << "fist offset: " << unsynced.firstOffset << "\n";-        dump(detail, out, *unsynced.root, unsynced.levels, unsynced.offset);-    }--private:-    // ---------------------------------------------------------------------------    //                                 Utilities-    // ----------------------------------------------------------------------------    /**-     * Creates new nodes and initializes them with 0.-     */-    static Node* newNode() {-        auto* res = new Node();-        std::memset(res->cell, 0, sizeof(Cell) * NUM_CELLS);-        return res;-    }--    /**-     * Destroys a node and all its sub-nodes recursively.-     */-    static void freeNodes(Node* node, int level) {-        if (!node) return;-        if (level != 0) {-            for (int i = 0; i < NUM_CELLS; i++) {-                freeNodes(node->cell[i].ptr, level - 1);-            }-        }-        delete node;-    }--    /**-     * Conducts a cleanup of the internal tree structure.-     */-    void clean() {-        freeNodes(unsynced.root, unsynced.levels);-        unsynced.root = nullptr;-        unsynced.levels = 0;-    }--    /**-     * Clones the given node and all its sub-nodes.-     */-    static Node* clone(const Node* node, int level) {-        // support null-pointers-        if (node == nullptr) {-            return nullptr;-        }--        // create a clone-        auto* res = new Node();--        // handle leaf level-        if (level == 0) {-            copy_op copy;-            for (int i = 0; i < NUM_CELLS; i++) {-                res->cell[i].value = copy(node->cell[i].value);-            }-            return res;-        }--        // for inner nodes clone each child-        for (int i = 0; i < NUM_CELLS; i++) {-            auto cur = clone(node->cell[i].ptr, level - 1);-            if (cur != nullptr) {-                cur->parent = res;-            }-            res->cell[i].ptr = cur;-        }--        // done-        return res;-    }--    /**-     * Obtains the left-most leaf-node of the tree rooted by the given node-     * with the given level.-     */-    static Node* findFirst(Node* node, int level) {-        while (level > 0) {-            bool found = false;-            for (int i = 0; i < NUM_CELLS; i++) {-                Node* cur = node->cell[i].ptr;-                if (cur) {-                    node = cur;-                    --level;-                    found = true;-                    break;-                }-            }-            assert(found && "No first node!");-        }--        return node;-    }--    /**-     * Raises the level of this tree by one level. It does so by introducing-     * a new root node and inserting the current root node as a child node.-     */-    void raiseLevel() {-        // something went wrong when we pass that line-        assert(unsynced.levels < (sizeof(index_type) * 8 / BITS) + 1);--        // create new root-        Node* node = newNode();-        node->parent = nullptr;--        // insert existing root as child-        auto x = getIndex(unsynced.offset, unsynced.levels + 1);-        node->cell[x].ptr = unsynced.root;--        // swap the root-        unsynced.root->parent = node;--        // update root-        unsynced.root = node;-        ++unsynced.levels;--        // update offset be removing additional bits-        unsynced.offset &= getLevelMask(unsynced.levels + 1);-    }--    /**-     * Attempts to raise the height of this tree based on the given root node-     * information and updates the root-info snapshot correspondingly.-     */-    void raiseLevel(RootInfoSnapshot& info) {-        // something went wrong when we pass that line-        assert(info.levels < (sizeof(index_type) * 8 / BITS) + 1);--        // create new root-        Node* newRoot = newNode();-        newRoot->parent = nullptr;--        // insert existing root as child-        auto x = getIndex(info.offset, info.levels + 1);-        newRoot->cell[x].ptr = info.root;--        // exchange the root in the info struct-        auto oldRoot = info.root;-        info.root = newRoot;--        // update level counter-        ++info.levels;--        // update offset-        info.offset &= getLevelMask(info.levels + 1);--        // try exchanging root info-        if (tryUpdateRootInfo(info)) {-            // success => final step, update parent of old root-            oldRoot->parent = info.root;-        } else {-            // throw away temporary new node-            delete newRoot;-        }-    }--    /**-     * Tests whether the given index is covered by the boundaries defined-     * by the hight and offset of the internally maintained tree.-     */-    bool inBoundaries(index_type a) const {-        return inBoundaries(a, unsynced.levels, unsynced.offset);-    }--    /**-     * Tests whether the given index is within the boundaries defined by the-     * given tree hight and offset.-     */-    static bool inBoundaries(index_type a, uint32_t levels, index_type offset) {-        auto mask = getLevelMask(levels + 1);-        return (a & mask) == offset;-    }--    /**-     * Obtains the index within the arrays of cells of a given index on a given-     * level of the internally maintained tree.-     */-    static index_type getIndex(RamDomain a, unsigned level) {-        return (a & (INDEX_MASK << (level * BIT_PER_STEP))) >> (level * BIT_PER_STEP);-    }--    /**-     * Computes the bit-mask to be applicable to obtain the offset of a node on a-     * given tree level.-     */-    static index_type getLevelMask(unsigned level) {-        if (level > (sizeof(index_type) * 8 / BITS)) return 0;-        return (~(index_type(0)) << (level * BIT_PER_STEP));-    }-};--/**- * A sparse bit-map is a bit map virtually assigning a bit value to every value if the- * uint32_t domain. However, only 1-bits are stored utilizing a nested sparse array- * structure.- *- * @tparam BITS similar to the BITS parameter of the sparse array type- */-template <unsigned BITS = 4>-class SparseBitMap {-    // the element type stored in the nested sparse array-    using value_t = uint64_t;--    // define the bit-level merge operation-    struct merge_op {-        value_t operator()(value_t a, value_t b) const {-            return a | b;  // merging bit masks => bitwise or operation-        }-    };--    // the type of the internal data store-    using data_store_t = SparseArray<value_t, BITS, merge_op>;-    using atomic_value_t = typename data_store_t::atomic_value_type;--    // some constants for manipulating stored values-    static constexpr short BITS_PER_ENTRY = sizeof(value_t) * 8;-    static constexpr short LEAF_INDEX_WIDTH = static_cast<short>(__builtin_ctz(BITS_PER_ENTRY));-    static constexpr uint64_t LEAF_INDEX_MASK = BITS_PER_ENTRY - 1;--public:-    // the type to address individual entries-    using index_type = typename data_store_t::index_type;--private:-    // it utilizes a sparse map to store its data-    data_store_t store;--public:-    // a simple default constructor-    SparseBitMap() = default;--    // a default copy constructor-    SparseBitMap(const SparseBitMap&) = default;--    // a default r-value copy constructor-    SparseBitMap(SparseBitMap&&) = default;--    // a default assignment operator-    SparseBitMap& operator=(const SparseBitMap&) = default;--    // a default r-value assignment operator-    SparseBitMap& operator=(SparseBitMap&&) = default;--    // checks whether this bit-map is empty -- thus it does not have any 1-entries-    bool empty() const {-        return store.empty();-    }--    // the type utilized for recording context information for exploiting temporal locality-    using op_context = typename data_store_t::op_context;--    /**-     * Sets the bit addressed by i to 1.-     */-    bool set(index_type i) {-        op_context ctxt;-        return set(i, ctxt);-    }--    /**-     * Sets the bit addressed by i to 1. A context for exploiting temporal locality-     * can be provided.-     */-    bool set(index_type i, op_context& ctxt) {-        atomic_value_t& val = store.getAtomic(i >> LEAF_INDEX_WIDTH, ctxt);-        value_t bit = (1ull << (i & LEAF_INDEX_MASK));--#ifdef __GNUC__-#if __GNUC__ >= 7-        // In GCC >= 7 the usage of fetch_or causes a bug that needs further investigation-        // For now, this two-instruction based implementation provides a fix that does-        // not sacrifice too much performance.--        while (true) {-            auto order = std::memory_order::memory_order_relaxed;--            // load current value-            value_t old = val.load(order);--            // if bit is already set => we are done-            if (old & bit) return false;--            // set the bit, if failed, repeat-            if (!val.compare_exchange_strong(old, old | bit, order, order)) continue;--            // it worked, new bit added-            return true;-        }--#endif-#endif--        value_t old = val.fetch_or(bit, std::memory_order::memory_order_relaxed);-        return (old & bit) == 0u;-    }--    /**-     * Determines the whether the bit addressed by i is set or not.-     */-    bool test(index_type i) const {-        op_context ctxt;-        return test(i, ctxt);-    }--    /**-     * Determines the whether the bit addressed by i is set or not. A context for-     * exploiting temporal locality can be provided.-     */-    bool test(index_type i, op_context& ctxt) const {-        value_t bit = (1ull << (i & LEAF_INDEX_MASK));-        return store.lookup(i >> LEAF_INDEX_WIDTH, ctxt) & bit;-    }--    /**-     * Determines the whether the bit addressed by i is set or not.-     */-    bool operator[](index_type i) const {-        return test(i);-    }--    /**-     * Resets all contained bits to 0.-     */-    void clear() {-        store.clear();-    }--    /**-     * Determines the number of bits set.-     */-    std::size_t size() const {-        // this is computed on demand to keep the set operation simple.-        std::size_t res = 0;-        for (const auto& cur : store) {-            res += __builtin_popcountll(cur.second);-        }-        return res;-    }--    /**-     * Computes the total memory usage of this data structure.-     */-    std::size_t getMemoryUsage() const {-        // compute the total memory usage-        return sizeof(*this) - sizeof(data_store_t) + store.getMemoryUsage();-    }--    /**-     * Sets all bits set in other to 1 within this bit map.-     */-    void addAll(const SparseBitMap& other) {-        // nothing to do if it is a self-assignment-        if (this == &other) return;--        // merge the sparse store-        store.addAll(other.store);-    }--    // ----------------------------------------------------------------------    //                           Iterator-    // -----------------------------------------------------------------------    /**-     * An iterator iterating over all indices set to 1.-     */-    class iterator : public std::iterator<std::forward_iterator_tag, index_type> {-        using nested_iterator = typename data_store_t::iterator;--        // the iterator through the underlying sparse data structure-        nested_iterator iter;--        // the currently consumed mask-        uint64_t mask = 0;--        // the value currently pointed to-        index_type value{};--    public:-        // default constructor -- creating an end-iterator-        iterator() = default;--        iterator(const nested_iterator& iter)-                : iter(iter), mask(toMask(iter->second)), value(iter->first << LEAF_INDEX_WIDTH) {-            moveToNextInMask();-        }--        iterator(const nested_iterator& iter, uint64_t m, index_type value)-                : iter(iter), mask(m), value(value) {}--        // a copy constructor-        iterator(const iterator& other) = default;--        // an assignment operator-        iterator& operator=(const iterator& other) = default;--        // the equality operator as required by the iterator concept-        bool operator==(const iterator& other) const {-            // only equivalent if pointing to the end-            return iter == other.iter && mask == other.mask;-        }--        // the not-equality operator as required by the iterator concept-        bool operator!=(const iterator& other) const {-            return !(*this == other);-        }--        // the deref operator as required by the iterator concept-        const index_type& operator*() const {-            return value;-        }--        // support for the pointer operator-        const index_type* operator->() const {-            return &value;-        }--        // the increment operator as required by the iterator concept-        iterator& operator++() {-            // progress in current mask-            if (moveToNextInMask()) return *this;--            // go to next entry-            ++iter;--            // update value-            if (!iter.isEnd()) {-                value = iter->first << LEAF_INDEX_WIDTH;-                mask = toMask(iter->second);-                moveToNextInMask();-            }--            // done-            return *this;-        }--        bool isEnd() const {-            return iter.isEnd();-        }--        void print(std::ostream& out) const {-            out << "SparseBitMapIter(" << iter << " -> " << std::bitset<64>(mask) << " @ " << value << ")";-        }--        // enables this iterator core to be printed (for debugging)-        friend std::ostream& operator<<(std::ostream& out, const iterator& iter) {-            iter.print(out);-            return out;-        }--        static uint64_t toMask(const value_t& value) {-            static_assert(sizeof(value_t) == sizeof(uint64_t), "Fixed for 64-bit compiler.");-            return reinterpret_cast<const uint64_t&>(value);-        }--    private:-        bool moveToNextInMask() {-            // check if there is something left-            if (mask == 0) return false;--            // get position of leading 1-            auto pos = __builtin_ctzll(mask);--            // consume this bit-            mask &= ~(1llu << pos);--            // update value-            value &= ~LEAF_INDEX_MASK;-            value |= pos;--            // done-            return true;-        }-    };--    /**-     * Obtains an iterator pointing to the first index set to 1. If there-     * is no such bit, end() will be returned.-     */-    iterator begin() const {-        auto it = store.begin();-        if (it.isEnd()) return end();-        return iterator(it);-    }--    /**-     * Returns an iterator referencing the position after the last set bit.-     */-    iterator end() const {-        return iterator();-    }--    /**-     * Obtains an iterator referencing the position i if the corresponding-     * bit is set, end() otherwise.-     */-    iterator find(index_type i) const {-        op_context ctxt;-        return find(i, ctxt);-    }--    /**-     * Obtains an iterator referencing the position i if the corresponding-     * bit is set, end() otherwise. An operation context can be provided-     * to exploit temporal locality.-     */-    iterator find(index_type i, op_context& ctxt) const {-        // check prefix part-        auto it = store.find(i >> LEAF_INDEX_WIDTH, ctxt);-        if (it.isEnd()) return end();--        // check bit-set part-        uint64_t mask = iterator::toMask(it->second);-        if (!(mask & (1llu << (i & LEAF_INDEX_MASK)))) return end();--        // OK, it is there => create iterator-        mask &= ((1ull << (i & LEAF_INDEX_MASK)) - 1);  // remove all bits before pos i-        return iterator(it, mask, i);-    }--    /**-     * Locates an iterator to the first element in this sparse bit map not less-     * than the given index.-     */-    iterator lower_bound(index_type i) const {-        auto it = store.lowerBound(i >> LEAF_INDEX_WIDTH);-        if (it.isEnd()) return end();--        // check bit-set part-        uint64_t mask = iterator::toMask(it->second);--        // if there is no bit remaining in this mask, check next mask.-        if (!(mask & ((~uint64_t(0)) << (i & LEAF_INDEX_MASK)))) {-            index_type next = ((i >> LEAF_INDEX_WIDTH) + 1) << LEAF_INDEX_WIDTH;-            if (next < i) return end();-            return lower_bound(next);-        }--        // there are bits left, use least significant bit of those-        if (it->first == i >> LEAF_INDEX_WIDTH) {-            mask &= ((~uint64_t(0)) << (i & LEAF_INDEX_MASK));  // remove all bits before pos i-        }--        // compute value represented by least significant bit-        index_type pos = __builtin_ctzll(mask);--        // remove this bit as well-        mask = mask & ~(1ull << pos);--        // construct value of this located bit-        index_type val = (it->first << LEAF_INDEX_WIDTH) | pos;-        return iterator(it, mask, val);-    }--    /**-     * Locates an iterator to the first element in this sparse bit map than is greater-     * than the given index.-     */-    iterator upper_bound(index_type i) const {-        if (i == std::numeric_limits<index_type>::max()) {-            return end();-        }-        return lower_bound(i + 1);-    }--    /**-     * A debugging utility printing the internal structure of this map to the-     * given output stream.-     */-    void dump(bool detail = false, std::ostream& out = std::cout) const {-        store.dump(detail, out);-    }--    /**-     * Provides write-protected access to the internal store for running-     * analysis on the data structure.-     */-    const data_store_t& getStore() const {-        return store;-    }-};--// ----------------------------------------------------------------------//                              TRIE-// -----------------------------------------------------------------------namespace detail {--/**- * A base class for the Trie implementation allowing various- * specializations of the Trie template to inherit common functionality.- *- * @tparam Dim the number of dimensions / arity of the stored tuples- * @tparam Derived the type derived from this base class- */-template <unsigned Dim, typename Derived>-class TrieBase {-public:-    /**-     * The type of the stored entries / tuples.-     */-    using entry_type = typename souffle::Tuple<RamDomain, Dim>;--    // -- operation wrappers ----    /**-     * A generic function enabling the insertion of tuple values in a user-friendly way.-     */-    template <typename... Values>-    bool insert(Values... values) {-        return static_cast<Derived&>(*this).insert(entry_type{{RamDomain(values)...}});-    }--    /**-     * A generic function enabling the convenient conduction of a membership check.-     */-    template <typename... Values>-    bool contains(Values... values) const {-        return static_cast<const Derived&>(*this).contains(entry_type{{RamDomain(values)...}});-    }--    // ----------------------------------------------------------------------    //                           Iterator-    // -----------------------------------------------------------------------    /**-     * An iterator over the stored entries.-     *-     * Iterators for tries consist of a top-level iterator maintaining the-     * master copy of a materialized tuple and a recursively nested iterator-     * core -- one for each nested trie level.-     */-    template <template <unsigned D> class IterCore>-    class iterator : public std::iterator<std::forward_iterator_tag, entry_type> {-        template <unsigned Len, unsigned Pos, unsigned Dimensions>-        friend struct fix_binding;--        template <unsigned Pos, unsigned Dimensions>-        friend struct fix_lower_bound;--        template <unsigned Pos, unsigned Dimensions>-        friend struct fix_upper_bound;--        template <unsigned Pos, unsigned Dimensions>-        friend struct fix_first;--        // the iterator core of this level-        using iter_core_t = IterCore<0>;--        // the wrapped iterator-        iter_core_t iter_core;--        // the value currently pointed to-        entry_type value;--    public:-        // default constructor -- creating an end-iterator-        iterator() = default;--        // a copy constructor-        iterator(const iterator& other) = default;--        iterator(iterator&& other) = default;--        template <typename Param>-        explicit iterator(const Param& param) : iter_core(param, value) {}--        // an assignment operator-        iterator& operator=(const iterator& other) = default;--        // the equality operator as required by the iterator concept-        bool operator==(const iterator& other) const {-            // equivalent if pointing to the same value-            return iter_core == other.iter_core;-        }--        // the not-equality operator as required by the iterator concept-        bool operator!=(const iterator& other) const {-            return !(*this == other);-        }--        // the deref operator as required by the iterator concept-        const entry_type& operator*() const {-            return value;-        }--        // support for the pointer operator-        const entry_type* operator->() const {-            return &value;-        }--        // the increment operator as required by the iterator concept-        iterator& operator++() {-            iter_core.inc(value);-            return *this;-        }--        // enables this iterator to be printed (for debugging)-        void print(std::ostream& out) const {-            out << "iter(" << iter_core << " -> " << value << ")";-        }--        friend std::ostream& operator<<(std::ostream& out, const iterator& iter) {-            iter.print(out);-            return out;-        }-    };--    /* -------------- operator hint statistics ----------------- */--    // an aggregation of statistical values of the hint utilization-    struct hint_statistics {-        // the counter for insertion operations-        CacheAccessCounter inserts;--        // the counter for contains operations-        CacheAccessCounter contains;--        // the counter for get_boundaries operations-        CacheAccessCounter get_boundaries;-    };--protected:-    // the hint statistic of this b-tree instance-    mutable hint_statistics hint_stats;--public:-    void printStats(std::ostream& out) const {-        out << "---------------------------------\n";-        out << "  insert-hint (hits/misses/total): " << hint_stats.inserts.getHits() << "/"-            << hint_stats.inserts.getMisses() << "/" << hint_stats.inserts.getAccesses() << "\n";-        out << "  contains-hint (hits/misses/total):" << hint_stats.contains.getHits() << "/"-            << hint_stats.contains.getMisses() << "/" << hint_stats.contains.getAccesses() << "\n";-        out << "  get-boundaries-hint (hits/misses/total):" << hint_stats.get_boundaries.getHits() << "/"-            << hint_stats.get_boundaries.getMisses() << "/" << hint_stats.get_boundaries.getAccesses()-            << "\n";-        out << "---------------------------------\n";-    }-};--/**- * A functor extracting a reference to a nested iterator core from an enclosing- * iterator core.- */-template <unsigned Level>-struct get_nested_iter_core {-    template <typename IterCore>-    auto operator()(IterCore& core) -> decltype(get_nested_iter_core<Level - 1>()(core.getNested())) {-        return get_nested_iter_core<Level - 1>()(core.getNested());-    }-};--template <>-struct get_nested_iter_core<0> {-    template <typename IterCore>-    IterCore& operator()(IterCore& core) {-        return core;-    }-};--/**- * A functor initializing an iterator upon creation to reference the first- * element in the associated Trie.- */-template <unsigned Pos, unsigned Dim>-struct fix_first {-    template <unsigned bits, typename iterator>-    void operator()(const SparseBitMap<bits>& store, iterator& iter) const {-        // set iterator to first in store-        auto first = store.begin();-        get_nested_iter_core<Pos>()(iter.iter_core).setIterator(first);-        iter.value[Pos] = *first;-    }--    template <typename Store, typename iterator>-    void operator()(const Store& store, iterator& iter) const {-        // set iterator to first in store-        auto first = store.begin();-        get_nested_iter_core<Pos>()(iter.iter_core).setIterator(first);-        iter.value[Pos] = first->first;-        // and continue recursively-        fix_first<Pos + 1, Dim>()(first->second->getStore(), iter);-    }-};--template <unsigned Dim>-struct fix_first<Dim, Dim> {-    template <typename Store, typename iterator>-    void operator()(const Store&, iterator&) const {-        // terminal case => nothing to do-    }-};--/**- * A functor initializing an iterator upon creation to reference the first element- * exhibiting a given prefix within a given Trie.- */-template <unsigned Len, unsigned Pos, unsigned Dim>-struct fix_binding {-    template <unsigned bits, typename iterator, typename entry_type>-    bool operator()(-            const SparseBitMap<bits>& store, iterator& begin, iterator& end, const entry_type& entry) const {-        // search in current level-        auto cur = store.find(entry[Pos]);--        // if not present => fail-        if (cur == store.end()) return false;--        // take current value-        get_nested_iter_core<Pos>()(begin.iter_core).setIterator(cur);-        ++cur;-        get_nested_iter_core<Pos>()(end.iter_core).setIterator(cur);--        // update iterator value-        begin.value[Pos] = entry[Pos];--        // no more remaining levels to fix-        return true;-    }--    template <typename Store, typename iterator, typename entry_type>-    bool operator()(const Store& store, iterator& begin, iterator& end, const entry_type& entry) const {-        // search in current level-        auto cur = store.find(entry[Pos]);--        // if not present => fail-        if (cur == store.end()) return false;--        // take current value as start-        get_nested_iter_core<Pos>()(begin.iter_core).setIterator(cur);--        // update iterator value-        begin.value[Pos] = entry[Pos];--        // fix remaining nested iterators-        auto res = fix_binding<Len - 1, Pos + 1, Dim>()(cur->second->getStore(), begin, end, entry);--        // update end of iterator-        if (get_nested_iter_core<Pos + 1>()(end.iter_core).getIterator() == cur->second->getStore().end()) {-            ++cur;-            if (cur != store.end()) {-                fix_first<Pos + 1, Dim>()(cur->second->getStore(), end);-            }-        }-        get_nested_iter_core<Pos>()(end.iter_core).setIterator(cur);--        // done-        return res;-    }-};--template <unsigned Pos, unsigned Dim>-struct fix_binding<0, Pos, Dim> {-    template <unsigned bits, typename iterator, typename entry_type>-    bool operator()(const SparseBitMap<bits>& store, iterator& begin, iterator& /* end */,-            const entry_type& /* entry */) const {-        // move begin to begin of store-        auto a = store.begin();-        get_nested_iter_core<Pos>()(begin.iter_core).setIterator(a);-        begin.value[Pos] = *a;--        return true;-    }--    template <typename Store, typename iterator, typename entry_type>-    bool operator()(const Store& store, iterator& begin, iterator& end, const entry_type& entry) const {-        // move begin to begin of store-        auto a = store.begin();-        get_nested_iter_core<Pos>()(begin.iter_core).setIterator(a);-        begin.value[Pos] = a->first;--        // continue recursively-        fix_binding<0, Pos + 1, Dim>()(a->second->getStore(), begin, end, entry);-        return true;-    }-};--template <unsigned Dim>-struct fix_binding<0, Dim, Dim> {-    template <typename Store, typename iterator, typename entry_type>-    bool operator()(const Store& /* store */, iterator& /* begin */, iterator& /* end */,-            const entry_type& /* entry */) const {-        // nothing more to do-        return true;-    }-};--/**- * A functor initializing an iterator upon creation to reference the first element- * within a given Trie being not less than a given value .- */-template <unsigned Pos, unsigned Dim>-struct fix_lower_bound {-    template <unsigned bits, typename iterator, typename entry_type>-    bool operator()(const SparseBitMap<bits>& store, iterator& iter, const entry_type& entry) const {-        // search in current level-        auto cur = store.lower_bound(entry[Pos]);--        if (cur == store.end()) return false;--        get_nested_iter_core<Pos>()(iter.iter_core).setIterator(cur);--        assert(entry[Pos] <= RamDomain(*cur));-        iter.value[Pos] = *cur;--        // no more remaining levels to fix-        return true;-    }--    template <typename Store, typename iterator, typename entry_type>-    bool operator()(const Store& store, iterator& iter, const entry_type& entry) const {-        // search in current level-        auto cur = store.lowerBound(entry[Pos]);--        // if no lower boundary is found, be done-        if (cur == store.end()) return false;-        assert(RamDomain(cur->first) >= entry[Pos]);--        // if the lower bound is higher than the requested value, go to first in subtree-        if (RamDomain(cur->first) > entry[Pos]) {-            get_nested_iter_core<Pos>()(iter.iter_core).setIterator(cur);-            iter.value[Pos] = cur->first;-            fix_first<Pos + 1, Dim>()(cur->second->getStore(), iter);-            return true;-        }--        // attempt to fix the rest-        if (!fix_lower_bound<Pos + 1, Dim>()(cur->second->getStore(), iter, entry)) {-            // if it does not work, since there are no matching elements in this branch, go to next-            entry_type sub = entry;-            sub[Pos] += 1;-            for (size_t i = Pos + 1; i < Dim; ++i) {-                sub[i] = 0;-            }-            return (*this)(store, iter, sub);-        }--        // remember result-        get_nested_iter_core<Pos>()(iter.iter_core).setIterator(cur);--        // update iterator value-        iter.value[Pos] = cur->first;--        // done!-        return true;-    }-};--/**- * A functor initializing an iterator upon creation to reference the first element- * within a given Trie being greater than a given value .- */-template <unsigned Pos, unsigned Dim>-struct fix_upper_bound {-    template <unsigned bits, typename iterator, typename entry_type>-    bool operator()(const SparseBitMap<bits>& store, iterator& iter, const entry_type& entry) const {-        // search in current level-        auto cur = store.upper_bound(entry[Pos]);--        if (cur == store.end()) {-            return false;-        }--        get_nested_iter_core<Pos>()(iter.iter_core).setIterator(cur);--        assert(entry[Pos] <= RamDomain(*cur));-        iter.value[Pos] = *cur;--        // no more remaining levels to fix-        return true;-    }--    template <typename Store, typename iterator, typename entry_type>-    bool operator()(const Store& store, iterator& iter, const entry_type& entry) const {-        // search in current level (if it is not the last level, we need a lower bound)-        auto cur = store.lowerBound(entry[Pos]);--        // if no lower boundary is found, be done-        if (cur == store.end()) {-            return false;-        }-        assert(RamDomain(cur->first) >= entry[Pos]);--        // if the lower bound is higher than the requested value, go to first in subtree-        if (RamDomain(cur->first) > entry[Pos]) {-            get_nested_iter_core<Pos>()(iter.iter_core).setIterator(cur);-            iter.value[Pos] = cur->first;-            fix_first<Pos + 1, Dim>()(cur->second->getStore(), iter);-            return true;-        }--        // attempt to fix the rest-        if (!fix_upper_bound<Pos + 1, Dim>()(cur->second->getStore(), iter, entry)) {-            // if it does not work, since there are no matching elements in this branch, go to next-            entry_type sub = entry;-            sub[Pos] += 1;-            for (size_t i = Pos + 1; i < Dim; ++i) {-                sub[i] = 0;-            }-            return (*this)(store, iter, sub);-        }--        // remember result-        get_nested_iter_core<Pos>()(iter.iter_core).setIterator(cur);--        // update iterator value-        iter.value[Pos] = cur->first;--        // done!-        return true;-    }-};--}  // namespace detail--/**- * The most generic implementation of a Trie forming the top-level of any- * Trie storing tuples of arity > 1.- */-template <unsigned Dim>-class Trie : public detail::TrieBase<Dim, Trie<Dim>> {-    template <unsigned D>-    friend class Trie;--    template <unsigned D, typename Derived>-    friend class TrieBase;--    // a shortcut for the common base class type-    using base = typename detail::TrieBase<Dim, Trie<Dim>>;--    // the type of the nested tries (1 dimension less)-    using nested_trie_type = Trie<Dim - 1>;--    // the merge operation capable of merging two nested tries-    struct nested_trie_merger {-        nested_trie_type* operator()(nested_trie_type* a, const nested_trie_type* b) const {-            if (!b) return a;-            if (!a) return new nested_trie_type(*b);-            a->insertAll(*b);-            return a;-        }-    };--    // the operation capable of cloning a nested trie-    struct nested_trie_cloner {-        nested_trie_type* operator()(nested_trie_type* a) const {-            if (!a) return a;-            return new nested_trie_type(*a);-        }-    };--    // the data structure utilized for indexing nested tries-    using store_type = SparseArray<nested_trie_type*,-            6,  // = 2^6 entries per block-            nested_trie_merger, nested_trie_cloner>;--    // the actual data store-    store_type store;--public:-    using entry_type = typename souffle::Tuple<RamDomain, Dim>;-    using element_type = entry_type;--    // ----------------------------------------------------------------------    //                           Iterator-    // -----------------------------------------------------------------------    /**-     * The iterator core for trie iterators involving this level.-     */-    template <unsigned I = 0>-    class iterator_core {-        // the iterator for the current level-        using store_iter_t = typename store_type::iterator;--        // the type of the nested iterator-        using nested_iter_core = typename Trie<Dim - 1>::template iterator_core<I + 1>;--        store_iter_t iter;--        nested_iter_core nested;--    public:-        /** default end-iterator constructor */-        iterator_core() = default;--        template <typename Tuple>-        iterator_core(const store_iter_t& iter, Tuple& entry) : iter(iter) {-            entry[I] = iter->first;-            nested = iter->second->template getBeginCoreIterator<I + 1>(entry);-        }--        void setIterator(const store_iter_t& iter) {-            this->iter = iter;-        }--        store_iter_t& getIterator() {-            return this->iter;-        }--        nested_iter_core& getNested() {-            return nested;-        }--        template <typename Tuple>-        bool inc(Tuple& entry) {-            // increment nested iterator-            if (nested.inc(entry)) return true;--            // increment the iterator on this level-            ++iter;--            // check whether the end has been reached-            if (iter.isEnd()) return false;--            // otherwise update entry value-            entry[I] = iter->first;--            // and restart nested-            nested = iter->second->template getBeginCoreIterator<I + 1>(entry);-            return true;-        }--        bool operator==(const iterator_core& other) const {-            return nested == other.nested && iter == other.iter;-        }--        bool operator!=(const iterator_core& other) const {-            return !(*this == other);-        }--        // enables this iterator core to be printed (for debugging)-        void print(std::ostream& out) const {-            out << iter << " | " << nested;-        }--        friend std::ostream& operator<<(std::ostream& out, const iterator_core& iter) {-            iter.print(out);-            return out;-        }-    };--    // the type of iterator to be utilized when iterating of instances of this trie-    using iterator = typename base::template iterator<iterator_core>;--    // the operation context aggregating all operation contexts of nested structures-    struct op_context {-        using local_ctxt = typename store_type::op_context;-        using nested_ctxt = typename nested_trie_type::op_context;--        // for insert and contain-        local_ctxt local{};-        RamDomain lastQuery{};-        nested_trie_type* lastNested{nullptr};-        nested_ctxt nestedCtxt{};--        // for boundaries-        unsigned lastBoundaryLevels{Dim + 1};-        entry_type lastBoundaryRequest{};-        range<iterator> lastBoundaries{iterator(), iterator()};--        op_context() = default;-    };--    using operation_hints = op_context;--    using base::contains;-    using base::insert;--    /**-     * A simple destructore.-     */-    ~Trie() {-        for (auto& cur : store) {-            delete cur.second;  // clears all nested tries-        }-    }--    /**-     * Determines whether this trie is empty or not.-     */-    bool empty() const {-        return store.empty();-    }--    /**-     * Determines the number of entries in this trie.-     */-    std::size_t size() const {-        // the number of elements is lazy-evaluated-        std::size_t res = 0;-        for (const auto& cur : store) {-            res += cur.second->size();-        }-        return res;-    }--    /**-     * Computes the total memory usage of this data structure.-     */-    std::size_t getMemoryUsage() const {-        // compute the total memory usage of this level-        std::size_t res = sizeof(*this) - sizeof(store) + store.getMemoryUsage();--        // add the memory usage of sub-levels-        for (const auto& cur : store) {-            res += cur.second->getMemoryUsage();-        }--        // done-        return res;-    }--    /**-     * Removes all entries within this trie.-     */-    void clear() {-        // delete lower levels-        for (auto& cur : store) {-            delete cur.second;-        }--        // clear store-        store.clear();-    }--    /**-     * Inserts a new entry.-     *-     * @param tuple the entry to be added-     * @return true if the same tuple hasn't been present before, false otherwise-     */-    bool insert(const entry_type& tuple) {-        op_context ctxt;-        return insert(tuple, ctxt);-    }--    /**-     * Inserts a new entry. A operation context may be provided to exploit temporal-     * locality.-     *-     * @param tuple the entry to be added-     * @param ctxt the operation context to be utilized-     * @return true if the same tuple hasn't been present before, false otherwise-     */-    bool insert(const entry_type& tuple, op_context& ctxt) {-        return insert_internal<0>(tuple, ctxt);-    }--    /**-     * Determines whether a given tuple is present within the set specified-     * by this trie.-     *-     * @param tuple the tuple to be tested-     * @return true if present, false otherwise-     */-    bool contains(const entry_type& tuple) const {-        op_context ctxt;-        return contains(tuple, ctxt);-    }--    /**-     * Determines whether a given tuple is present within the set specified-     * by this trie. A operation context may be provided to exploit temporal-     * locality.-     *-     * @param tuple the entry to be added-     * @param ctxt the operation context to be utilized-     * @return true if the same tuple hasn't been present before, false otherwise-     */-    bool contains(const entry_type& tuple, op_context& ctxt) const {-        return contains_internal<0>(tuple, ctxt);-    }--    /**-     * Inserts all elements stored within the given trie into this trie.-     *-     * @param other the elements to be inserted into this trie-     */-    void insertAll(const Trie& other) {-        store.addAll(other.store);-    }--    /**-     * Obtains an iterator referencing the first element stored within this trie.-     */-    iterator begin() const {-        auto it = store.begin();-        if (it.isEnd()) return end();-        return iterator(it);-    }--    /**-     * Obtains an iterator referencing the position after the last element stored-     * within this trie.-     */-    iterator end() const {-        return iterator();-    }--    iterator find(const entry_type& entry) const {-        op_context ctxt;-        return find(entry, ctxt);-    }--    iterator find(const entry_type& entry, op_context& ctxt) const {-        auto range = getBoundaries<Dim>(entry, ctxt);-        return (!range.empty()) ? range.begin() : end();-    }--    /**-     * Obtains a range of elements matching the prefix of the given entry up to-     * levels elements.-     *-     * @tparam levels the length of the requested matching prefix-     * @param entry the entry to be looking for-     * @return the corresponding range of matching elements-     */-    template <unsigned levels>-    range<iterator> getBoundaries(const entry_type& entry) const {-        op_context ctxt;-        return getBoundaries<levels>(entry, ctxt);-    }--    /**-     * Obtains a range of elements matching the prefix of the given entry up to-     * levels elements. A operation context may be provided to exploit temporal-     * locality.-     *-     * @tparam levels the length of the requested matching prefix-     * @param entry the entry to be looking for-     * @param ctxt the operation context to be utilized-     * @return the corresponding range of matching elements-     */-    template <unsigned levels>-    range<iterator> getBoundaries(const entry_type& entry, op_context& ctxt) const {-        // if nothing is bound => just use begin and end-        if (levels == 0) return make_range(begin(), end());--        // check context-        if (ctxt.lastBoundaryLevels == levels) {-            bool fit = true;-            for (unsigned i = 0; i < levels; ++i) {-                fit = fit && (entry[i] == ctxt.lastBoundaryRequest[i]);-            }--            // if it fits => take it-            if (fit) {-                base::hint_stats.get_boundaries.addHit();-                return ctxt.lastBoundaries;-            }-        }--        // the hint has not been a hit-        base::hint_stats.get_boundaries.addMiss();--        // start with two end iterators-        iterator begin{};-        iterator end{};--        // adapt them level by level-        auto found = detail::fix_binding<levels, 0, Dim>()(store, begin, end, entry);-        if (!found) return make_range(iterator(), iterator());--        // update context-        ctxt.lastBoundaryLevels = levels;-        ctxt.lastBoundaryRequest = entry;-        ctxt.lastBoundaries = make_range(begin, end);--        // use the result-        return ctxt.lastBoundaries;-    }--    /**-     * Obtains an iterator to the first element not less than the given entry value.-     *-     * @param entry the lower bound for this search-     * @param ctxt the operation context to be utilized-     * @return an iterator addressing the first element in this structure not less than the given value-     */-    iterator lower_bound(const entry_type& entry, op_context& /* ctxt */) const {-        // start with a default-initialized iterator-        iterator res;--        // adapt it level by level-        bool found = detail::fix_lower_bound<0, Dim>()(store, res, entry);--        // use the result-        return found ? res : end();-    }--    /**-     * Obtains an iterator to the first element not less than the given entry value.-     *-     * @param entry the lower bound for this search-     * @return an iterator addressing the first element in this structure not less than the given value-     */-    iterator lower_bound(const entry_type& entry) const {-        op_context ctxt;-        return lower_bound(entry, ctxt);-    }--    /**-     * Obtains an iterator to the first element greater than the given entry value, or end if there is no such-     * element.-     *-     * @param entry the upper bound for this search-     * @param ctxt the operation context to be utilized-     * @return an iterator addressing the first element in this structure greater than the given value-     */-    iterator upper_bound(const entry_type& entry, op_context& /* ctxt */) const {-        // start with a default-initialized iterator-        iterator res;--        // adapt it level by level-        bool found = detail::fix_upper_bound<0, Dim>()(store, res, entry);--        // use the result-        return found ? res : end();-    }--    /**-     * Obtains an iterator to the first element greater than the given entry value, or end if there is no such-     * element.-     *-     * @param entry the upper bound for this search-     * @return an iterator addressing the first element in this structure greater than the given value-     */-    iterator upper_bound(const entry_type& entry) const {-        op_context ctxt;-        return upper_bound(entry, ctxt);-    }--    /**-     * Computes a partition of an approximate number of chunks of the content-     * of this trie. Thus, the union of the resulting set of disjoint ranges is-     * equivalent to the content of this trie.-     *-     * @param chunks the number of chunks requested-     * @return a list of sub-ranges forming a partition of the content of this trie-     */-    std::vector<range<iterator>> partition(unsigned chunks = 500) const {-        std::vector<range<iterator>> res;--        // shortcut for empty trie-        if (this->empty()) return res;--        // use top-level elements for partitioning-        int step = std::max(store.size() / chunks, size_t(1));--        int c = 1;-        auto priv = begin();-        for (auto it = store.begin(); it != store.end(); ++it, c++) {-            if (c % step != 0 || c == 1) {-                continue;-            }-            auto cur = iterator(it);-            res.push_back(make_range(priv, cur));-            priv = cur;-        }-        // add final chunk-        res.push_back(make_range(priv, end()));-        return res;-    }--    /**-     * Provides a protected access to the internally maintained store.-     */-    const store_type& getStore() const {-        return store;-    }--private:-    /**-     * Creates a core iterator for this trie level and updates component-     * I of the given entry to exhibit the corresponding first value.-     *-     * @tparam I the index of the tuple to be processed by the resulting iterator core-     * @tparam Tuple the type of the tuple to be processed by the resulting iterator core-     * @param entry a reference to the tuple to be updated to the first value-     * @return the requested iterator core instance-     */-    template <unsigned I, typename Tuple>-    iterator_core<I> getBeginCoreIterator(Tuple& entry) const {-        return iterator_core<I>(store.begin(), entry);-    }--    /**-     * The internally utilized implementation of the insert operation inserting-     * a given tuple into this sub-trie.-     *-     * @tparam I the component index associated to this level-     * @tparam Tuple the tuple type to be inserted-     * @param tuple the tuple to be inserted-     * @param ctxt a operation context to exploit temporal locality-     * @return true if this tuple wasn't contained before, false otherwise-     */-    template <unsigned I, typename Tuple>-    bool insert_internal(const Tuple& tuple, op_context& ctxt) {-        using value_t = typename store_type::value_type;-        using atomic_value_t = typename store_type::atomic_value_type;--        // check context-        if (ctxt.lastNested && ctxt.lastQuery == tuple[I]) {-            base::hint_stats.inserts.addHit();-            return ctxt.lastNested->template insert_internal<I + 1>(tuple, ctxt.nestedCtxt);-        } else {-            base::hint_stats.inserts.addMiss();-        }--        // lookup nested-        atomic_value_t& next = store.getAtomic(tuple[I], ctxt.local);--        // get pure pointer to next level-        value_t nextPtr = next;--        // conduct a lock-free lazy-creation of nested trees-        if (!nextPtr) {-            // create a new sub-tree-            auto newNested = new nested_trie_type();--            // register new sub-tree atomically-            if (next.compare_exchange_weak(nextPtr, newNested)) {-                nextPtr = newNested;  // worked-            } else {-                delete newNested;  // some other thread was faster => use its version-            }-        }--        // make sure a next has been established-        assert(nextPtr);--        // clear context if necessary-        if (nextPtr != ctxt.lastNested) {-            ctxt.lastQuery = tuple[I];-            ctxt.lastNested = nextPtr;-            ctxt.nestedCtxt = typename op_context::nested_ctxt();-        }--        // conduct recursive step-        return nextPtr->template insert_internal<I + 1>(tuple, ctxt.nestedCtxt);-    }--    /**-     * An internal implementation of the contains member function determining-     * whether a given tuple is present within this sub-trie or not.-     *-     * @tparam I the component index associated to this level-     * @tparam Tuple the tuple type to be checked-     * @param tuple the tuple to be checked-     * @param ctxt a operation context to exploit temporal locality-     * @return true if this tuple is present, false otherwise-     */-    template <unsigned I, typename Tuple>-    bool contains_internal(const Tuple& tuple, op_context& ctxt) const {-        // check context-        if (ctxt.lastNested && ctxt.lastQuery == tuple[I]) {-            base::hint_stats.contains.addHit();-            return ctxt.lastNested->template contains_internal<I + 1>(tuple, ctxt.nestedCtxt);-        } else {-            base::hint_stats.contains.addMiss();-        }--        // lookup next step-        auto next = store.lookup(tuple[I], ctxt.local);--        // clear context if necessary-        if (next != ctxt.lastNested) {-            ctxt.lastQuery = tuple[I];-            ctxt.lastNested = next;-            ctxt.nestedCtxt = typename op_context::nested_ctxt();-        }--        // conduct recursive step-        return next && next->template contains_internal<I + 1>(tuple, ctxt.nestedCtxt);-    }-};--/**- * A template specialization for tries representing a set.- * For improved memory efficiency, this level is the leaf-node level- * of all tries exhibiting an arity >= 1. Internally, values are stored utilizing- * sparse bit maps.- */-template <>-class Trie<1u> : public detail::TrieBase<1u, Trie<1u>> {-    template <unsigned Dim>-    friend class Trie;--    template <unsigned Dim, typename Derived>-    friend class detail::TrieBase;--    // a shortcut for the base type-    using base = typename detail::TrieBase<1u, Trie<1u>>;--    // the map type utilized internally-    using map_type = SparseBitMap<>;--    // the internal data store-    map_type map;--public:-    using element_type = entry_type;-    using op_context = typename map_type::op_context;-    using operation_hints = op_context;--    using base::contains;-    using base::insert;--    /**-     * Determines whether this trie is empty or not.-     */-    bool empty() const {-        return map.empty();-    }--    /**-     * Determines the number of elements stored in this trie.-     */-    std::size_t size() const {-        return map.size();-    }--    /**-     * Computes the total memory usage of this data structure.-     */-    std::size_t getMemoryUsage() const {-        // compute the total memory usage-        return sizeof(*this) - sizeof(map_type) + map.getMemoryUsage();-    }--    /**-     * Removes all elements form this trie.-     */-    void clear() {-        map.clear();-    }--    /**-     * Inserts the given tuple into this trie.-     *-     * @param tuple the tuple to be inserted-     * @return true if the tuple has not been present before, false otherwise-     */-    bool insert(const entry_type& tuple) {-        op_context ctxt;-        return insert(tuple, ctxt);-    }--    /**-     * Inserts the given tuple into this trie.-     * An operation context can be provided to exploit temporal locality.-     *-     * @param tuple the tuple to be inserted-     * @param ctxt an operation context for exploiting temporal locality-     * @return true if the tuple has not been present before, false otherwise-     */-    bool insert(const entry_type& tuple, op_context& ctxt) {-        return insert_internal<0>(tuple, ctxt);-    }--    /**-     * Determines whether the given tuple is present in this trie or not.-     *-     * @param tuple the tuple to be tested-     * @return true if present, false otherwise-     */-    bool contains(const entry_type& tuple) const {-        op_context ctxt;-        return contains(tuple, ctxt);-    }--    /**-     * Determines whether the given tuple is present in this trie or not.-     * An operation context can be provided to exploit temporal locality.-     *-     * @param tuple the tuple to be tested-     * @param ctxt an operation context for exploiting temporal locality-     * @return true if present, false otherwise-     */-    bool contains(const entry_type& tuple, op_context& ctxt) const {-        return contains_internal<0>(tuple, ctxt);-    }--    /**-     * Inserts all tuples stored within the given trie into this trie.-     * This operation is considerably more efficient than the consecutive-     * insertion of the elements in other into this trie.-     */-    void insertAll(const Trie& other) {-        map.addAll(other.map);-    }--    // ----------------------------------------------------------------------    //                           Iterator-    // -----------------------------------------------------------------------    /**-     * The iterator core of this level contributing to the construction of-     * a composed trie iterator.-     */-    template <unsigned I = 0>-    class iterator_core {-        // the iterator for this level-        using iter_type = typename map_type::iterator;--        // the referenced bit-map iterator-        iter_type iter;--    public:-        /** default end-iterator constructor */-        iterator_core() = default;--        template <typename Tuple>-        iterator_core(const iter_type& iter, Tuple& entry) : iter(iter) {-            entry[I] = *iter;-        }--        void setIterator(const iter_type& iter) {-            this->iter = iter;-        }--        iter_type& getIterator() {-            return this->iter;-        }--        template <typename Tuple>-        bool inc(Tuple& entry) {-            // increment the iterator on this level-            ++iter;--            // check whether the end has been reached-            if (iter.isEnd()) return false;--            // otherwise update entry value-            entry[I] = *iter;-            return true;-        }--        bool operator==(const iterator_core& other) const {-            return iter == other.iter;-        }--        bool operator!=(const iterator_core& other) const {-            return !(*this == other);-        }--        // enables this iterator core to be printed (for debugging)-        void print(std::ostream& out) const {-            out << iter;-        }--        friend std::ostream& operator<<(std::ostream& out, const iterator_core& iter) {-            iter.print(out);-            return out;-        }-    };--    // the iterator type utilized by this trie type-    using iterator = typename base::template iterator<iterator_core>;--    /**-     * Obtains an iterator referencing the first element stored within this trie-     * or end() if this trie is empty.-     */-    iterator begin() const {-        if (map.empty()) return end();-        return iterator(map.begin());-    }--    /**-     * Obtains an iterator referencing the first position after the last element-     * within this trie.-     */-    iterator end() const {-        return iterator();-    }--    /**-     * Obtains a partition of this tire such that the resulting list of ranges-     * cover disjoint subsets of the elements stored in this trie. Their union-     * is equivalent to the content of this trie.-     */-    std::vector<range<iterator>> partition(unsigned chunks = 500) const {-        std::vector<range<iterator>> res;--        // shortcut for empty trie-        if (this->empty()) return res;--        // use top-level elements for partitioning-        int step = static_cast<int>(std::max(map.size() / chunks, size_t(1)));--        int c = 1;-        auto priv = begin();-        for (auto it = map.begin(); it != map.end(); ++it, c++) {-            if (c % step != 0 || c == 1) {-                continue;-            }-            auto cur = iterator(it);-            res.push_back(make_range(priv, cur));-            priv = cur;-        }-        // add final chunk-        res.push_back(make_range(priv, end()));-        return res;-    }--    /**-     * Obtains a range of elements matching the prefix of the given entry up to-     * levels elements.-     *-     * @tparam levels the length of the requested matching prefix-     * @param entry the entry to be looking for-     * @return the corresponding range of matching elements-     */-    template <unsigned levels>-    range<iterator> getBoundaries(const entry_type& entry) const {-        op_context ctxt;-        return getBoundaries<levels>(entry, ctxt);-    }--    /**-     * Obtains a range of elements matching the prefix of the given entry up to-     * levels elements. A operation context may be provided to exploit temporal-     * locality.-     *-     * @tparam levels the length of the requested matching prefix-     * @param entry the entry to be looking for-     * @param ctxt the operation context to be utilized-     * @return the corresponding range of matching elements-     */-    template <unsigned levels>-    range<iterator> getBoundaries(const entry_type& entry, op_context& ctxt) const {-        // for levels = 0-        if (levels == 0) return make_range(begin(), end());-        // for levels = 1-        auto pos = map.find(entry[0], ctxt);-        if (pos == map.end()) return make_range(end(), end());-        auto next = pos;-        ++next;-        return make_range(iterator(pos), iterator(next));-    }--    iterator lower_bound(const entry_type& entry, op_context&) const {-        return iterator(map.lower_bound(entry[0]));-    }--    iterator lower_bound(const entry_type& entry) const {-        op_context ctxt;-        return lower_bound(entry, ctxt);-    }--    iterator upper_bound(const entry_type& entry, op_context&) const {-        return iterator(map.upper_bound(entry[0]));-    }--    iterator upper_bound(const entry_type& entry) const {-        op_context ctxt;-        return upper_bound(entry, ctxt);-    }--    /**-     * Provides protected access to the internally maintained store.-     */-    const map_type& getStore() const {-        return map;-    }--private:-    /**-     * Creates a core iterator for this trie level and updates component-     * I of the given entry to exhibit the corresponding first value.-     *-     * @tparam I the index of the tuple to be processed by the resulting iterator core-     * @tparam Tuple the type of the tuple to be processed by the resulting iterator core-     * @param entry a reference to the tuple to be updated to the first value-     * @return the requested iterator core instance-     */-    template <unsigned I, typename Tuple>-    iterator_core<I> getBeginCoreIterator(Tuple& entry) const {-        return iterator_core<I>(map.begin(), entry);-    }--    /**-     * The internally utilized implementation of the insert operation inserting-     * a given tuple into this sub-trie.-     *-     * @tparam I the component index associated to this level-     * @tparam Tuple the tuple type to be inserted-     * @param tuple the tuple to be inserted-     * @param ctxt a operation context to exploit temporal locality-     * @return true if this tuple wasn't contained before, false otherwise-     */-    template <unsigned I, typename Tuple>-    bool insert_internal(const Tuple& tuple, op_context& ctxt) {-        return map.set(tuple[I], ctxt);-    }--    /**-     * An internal implementation of the contains member function determining-     * whether a given tuple is present within this sub-trie or not.-     *-     * @tparam I the component index associated to this level-     * @tparam Tuple the tuple type to be checked-     * @param tuple the tuple to be checked-     * @param ctxt a operation context to exploit temporal locality-     * @return true if this tuple is present, false otherwise-     */-    template <unsigned I, typename Tuple>-    bool contains_internal(const Tuple& tuple, op_context& ctxt) const {-        return map.test(tuple[I], ctxt);-    }-};--}  // end namespace souffle
− cbits/souffle/CompiledOptions.h 
@@ -1,257 +0,0 @@-/*- * Souffle - A Datalog Compiler- * Copyright (c) 2013, 2015, Oracle and/or its affiliates. All rights reserved- * Licensed under the Universal Permissive License v 1.0 as shown at:- * - https://opensource.org/licenses/UPL- * - <souffle root>/licenses/SOUFFLE-UPL.txt- */--/************************************************************************- *- * @file CompiledOptions.h- *- * A header file offering command-line option support for compiled- * RAM programs.- *- ***********************************************************************/--#pragma once--#include <cstdio>-#include <iostream>-#include <string>-#include <getopt.h>-#include <stdlib.h>-#include <sys/stat.h>--namespace souffle {--/**- * A utility class for parsing command line arguments within generated- * query programs.- */-class CmdOptions {-protected:-    /**-     * source file-     */-    std::string src;--    /**-     * fact directory-     */-    std::string input_dir;--    /**-     * output directory-     */-    std::string output_dir;--    /**-     * profiling flag-     */-    bool profiling;--    /**-     * profile filename-     */-    std::string profile_name;--    /**-     * number of threads-     */-    size_t num_jobs;--public:-    // all argument constructor-    CmdOptions(const char* s, const char* id, const char* od, bool pe, const char* pfn, size_t nj)-            : src(s), input_dir(id), output_dir(od), profiling(pe), profile_name(pfn), num_jobs(nj) {}--    /**-     * get source code name-     */-    const std::string& getSourceFileName() const {-        return src;-    }--    /**-     * get input directory-     */-    const std::string& getInputFileDir() const {-        return input_dir;-    }--    /**-     * get output directory-     */-    const std::string& getOutputFileDir() const {-        return output_dir;-    }--    /**-     * is profiling switched on-     */-    bool isProfiling() const {-        return profiling;-    }--    /**-     * get filename of profile-     */-    const std::string& getProfileName() const {-        return profile_name;-    }--    /**-     * get number of jobs-     */-    size_t getNumJobs() const {-        return num_jobs;-    }--    /**-     * Parses the given command line parameters, handles -h help requests or errors-     * and returns whether the parsing was successful or not.-     */-    bool parse(int argc, char** argv) {-        // get executable name-        std::string exec_name = "analysis";-        if (argc > 0) {-            exec_name = argv[0];-        }--        // local options-        std::string fact_dir = input_dir;-        std::string out_dir = output_dir;--// avoid warning due to Solaris getopt.h-#pragma GCC diagnostic push-#pragma GCC diagnostic ignored "-Wwrite-strings"-        // long options-        option longOptions[] = {{"facts", true, nullptr, 'F'}, {"output", true, nullptr, 'D'},-                {"profile", true, nullptr, 'p'}, {"jobs", true, nullptr, 'j'}, {"index", true, nullptr, 'i'},-                // the terminal option -- needs to be null-                {nullptr, false, nullptr, 0}};-#pragma GCC diagnostic pop--        // check whether all options are fine-        bool ok = true;--        int c; /* command-line arguments processing */-        while ((c = getopt_long(argc, argv, "D:F:hp:j:i:", longOptions, nullptr)) != EOF) {-            switch (c) {-                /* Fact directories */-                case 'F':-                    if (!existDir(optarg)) {-                        printf("Fact directory %s does not exists!\n", optarg);-                        ok = false;-                    }-                    fact_dir = optarg;-                    break;-                /* Output directory for resulting .csv files */-                case 'D':-                    if (*optarg && !existDir(optarg)) {-                        printf("Output directory %s does not exists!\n", optarg);-                        ok = false;-                    }-                    out_dir = optarg;-                    break;-                case 'p':-                    if (!profiling) {-                        std::cerr << "\nError: profiling was not enabled in compilation\n\n";-                        printHelpPage(exec_name);-                        exit(EXIT_FAILURE);-                    }-                    profile_name = optarg;-                    break;-                case 'j':-#ifdef _OPENMP-                    if (std::string(optarg) == "auto") {-                        num_jobs = 0;-                    } else {-                        int num = atoi(optarg);-                        if (num > 0) {-                            num_jobs = num;-                        } else {-                            std::cerr << "Invalid number of jobs [-j]: " << optarg << "\n";-                            ok = false;-                        }-                    }-#else-                    std::cerr << "\nWarning: OpenMP was not enabled in compilation\n\n";-#endif-                    break;-                default: printHelpPage(exec_name); return false;-            }-        }--        // update member fields-        input_dir = fact_dir;-        output_dir = out_dir;--        // return success state-        return ok;-    }--private:-    /**-     * Prints the help page if it has been requested or there was a typo in the command line arguments.-     */-    void printHelpPage(const std::string& exec_name) const {-        std::cerr << "====================================================================\n";-        std::cerr << " Datalog Program: " << src << "\n";-        std::cerr << " Usage: " << exec_name << " [OPTION]\n\n";-        std::cerr << " Options:\n";-        std::cerr << "    -D <DIR>, --output=<DIR>     -- Specify directory for output relations\n";-        std::cerr << "                                    (default: " << output_dir << ")\n";-        std::cerr << "                                    (suppress output with \"\")\n";-        std::cerr << "    -F <DIR>, --facts=<DIR>      -- Specify directory for fact files\n";-        std::cerr << "                                    (default: " << input_dir << ")\n";-        if (profiling) {-            std::cerr << "    -p <file>, --profile=<file>  -- Specify filename for profiling\n";-            std::cerr << "                                    (default: " << profile_name << ")\n";-        }-#ifdef _OPENMP-        std::cerr << "    -j <NUM>, --jobs=<NUM>       -- Specify number of threads\n";-        if (num_jobs > 0) {-            std::cerr << "                                    (default: " << num_jobs << ")\n";-        } else {-            std::cerr << "                                    (default: auto)\n";-        }-#endif-        std::cerr << "    -h                           -- prints this help page.\n";-        std::cerr << "--------------------------------------------------------------------\n";-        std::cout << " Copyright (c) 2016-20 The Souffle Developers." << std::endl;-        std::cout << " Copyright (c) 2013-16 Oracle and/or its affiliates." << std::endl;-        std::cerr << " All rights reserved.\n";-        std::cerr << "====================================================================\n";-    }--    /**-     *  Check whether a file exists in the file system-     */-    inline bool existFile(const std::string& name) const {-        struct stat buffer;-        if (stat(name.c_str(), &buffer) == 0) {-            if ((buffer.st_mode & S_IFREG) != 0) {-                return true;-            }-        }-        return false;-    }--    /**-     *  Check whether a directory exists in the file system-     */-    bool existDir(const std::string& name) const {-        struct stat buffer;-        if (stat(name.c_str(), &buffer) == 0) {-            if ((buffer.st_mode & S_IFDIR) != 0) {-                return true;-            }-        }-        return false;-    }-};--}  // end of namespace souffle
cbits/souffle/CompiledSouffle.h view
@@ -16,17 +16,17 @@  #pragma once -#include "souffle/Brie.h" #include "souffle/CompiledTuple.h"-#include "souffle/EquivalenceRelation.h"-#include "souffle/IOSystem.h" #include "souffle/RamTypes.h" #include "souffle/RecordTable.h" #include "souffle/SignalHandler.h" #include "souffle/SouffleInterface.h" #include "souffle/SymbolTable.h"-#include "souffle/Table.h"-#include "souffle/WriteStream.h"+#include "souffle/datastructure/Brie.h"+#include "souffle/datastructure/EquivalenceRelation.h"+#include "souffle/datastructure/Table.h"+#include "souffle/io/IOSystem.h"+#include "souffle/io/WriteStream.h" #include "souffle/utility/CacheUtil.h" #include "souffle/utility/ContainerUtil.h" #include "souffle/utility/EvaluatorUtil.h"@@ -38,8 +38,8 @@ #include "souffle/utility/StringUtil.h" #ifndef __EMBEDDED_SOUFFLE__ #include "souffle/CompiledOptions.h"-#include "souffle/Logger.h"-#include "souffle/ProfileEvent.h"+#include "souffle/profile/Logger.h"+#include "souffle/profile/ProfileEvent.h" #endif #include <array> #include <atomic>@@ -178,10 +178,16 @@     context createContext() {         return context();     }-    class iterator : public std::iterator<std::forward_iterator_tag, RamDomain*> {+    class iterator {         bool value;      public:+        typedef std::forward_iterator_tag iterator_category;+        typedef RamDomain* value_type;+        typedef ptrdiff_t difference_type;+        typedef value_type* pointer;+        typedef value_type& reference;+         iterator(bool v = false) : value(v) {}          const RamDomain* operator*() {@@ -238,7 +244,7 @@     void purge() {         data = false;     }-    void printHintStatistics(std::ostream& /* o */, std::string /* prefix */) const {}+    void printStatistics(std::ostream& /* o */) const {} };  /** info relations */@@ -323,7 +329,7 @@     void purge() {         data.clear();     }-    void printHintStatistics(std::ostream& /* o */, std::string /* prefix */) const {}+    void printStatistics(std::ostream& /* o */) const {} };  }  // namespace souffle
− cbits/souffle/EquivalenceRelation.h 
@@ -1,730 +0,0 @@-/*- * Souffle - A Datalog Compiler- * Copyright (c) 2017 The Souffle Developers. All Rights reserved- * Licensed under the Universal Permissive License v 1.0 as shown at:- * - https://opensource.org/licenses/UPL- * - <souffle root>/licenses/SOUFFLE-UPL.txt- */--/************************************************************************- *- * @file EquivalenceRelation.h- *- * Defines a binary relation interface to be used with Souffle as a relational store.- * Pairs inserted into this relation implicitly store a reflexive, symmetric, and transitive relation- * with each other.- *- ***********************************************************************/--#pragma once--#include "LambdaBTree.h"-#include "PiggyList.h"-#include "RamTypes.h"-#include "UnionFind.h"-#include "utility/ContainerUtil.h"-#include "utility/ParallelUtil.h"-#include <atomic>-#include <cassert>-#include <cstddef>-#include <functional>-#include <iostream>-#include <iterator>-#include <set>-#include <shared_mutex>-#include <stdexcept>-#include <tuple>-#include <utility>-#include <vector>--namespace souffle {-template <typename TupleType>-class EquivalenceRelation {-    using value_type = typename TupleType::value_type;--    // mapping from representative to disjoint set-    // just a cache, essentially, used for iteration over-    using StatesList = souffle::PiggyList<value_type>;-    using StatesBucket = StatesList*;-    using StorePair = std::pair<value_type, StatesBucket>;-    using StatesMap = souffle::LambdaBTreeSet<StorePair, std::function<StatesBucket(StorePair&)>,-            souffle::EqrelMapComparator<StorePair>>;--public:-    using element_type = TupleType;--    EquivalenceRelation() : statesMapStale(false){};-    ~EquivalenceRelation() {-        emptyPartition();-    }--    /**-     * A collection of operation hints speeding up some of the involved operations-     * by exploiting temporal locality.-     * Unused in this class, as there is no speedup to be gained.-     * This is just defined as the class expects it.-     */-    struct operation_hints {-        // resets all hints (to be triggered e.g. when deleting nodes)-        void clear() {}-    };--    /**-     * Insert the two values symbolically as a binary relation-     * @param x node to be added/paired-     * @param y node to be added/paired-     * @return true if the pair is new to the data structure-     */-    bool insert(value_type x, value_type y) {-        operation_hints z;-        return insert(x, y, z);-    };--    /**-     * Insert the tuple symbolically.-     * @param tuple The tuple to be inserted-     * @return true if the tuple is new to the data structure-     */-    bool insert(const TupleType& tuple) {-        operation_hints hints;-        return insert(tuple[0], tuple[1], hints);-    };--    /**-     * Insert the two values symbolically as a binary relation-     * @param x node to be added/paired-     * @param y node to be added/paired-     * @param z the hints to where the pair should be inserted (not applicable atm)-     * @return true if the pair is new to the data structure-     */-    bool insert(value_type x, value_type y, operation_hints) {-        // indicate that iterators will have to generate on request-        this->statesMapStale.store(true, std::memory_order_relaxed);-        bool retval = contains(x, y);-        sds.unionNodes(x, y);-        return retval;-    }--    /**-     * inserts all nodes from the other relation into this one-     * @param other the binary relation from which to add elements from-     */-    void insertAll(const EquivalenceRelation<TupleType>& other) {-        other.genAllDisjointSetLists();--        // iterate over partitions at a time-        for (typename StatesMap::chunk it : other.equivalencePartition.getChunks(MAX_THREADS)) {-            for (auto& p : it) {-                value_type rep = p.first;-                StatesList& pl = *p.second;-                const size_t ksize = pl.size();-                for (size_t i = 0; i < ksize; ++i) {-                    this->sds.unionNodes(rep, pl.get(i));-                }-            }-        }-        // invalidate iterators unconditionally-        this->statesMapStale.store(true, std::memory_order_relaxed);-    }--    /**-     * Extend this relation with another relation, expanding this equivalence relation-     * The supplied relation is the old knowledge, whilst this relation only contains-     * explicitly new knowledge. After this operation the "implicitly new tuples" are now-     * explicitly inserted this relation.-     */-    void extend(const EquivalenceRelation<TupleType>& other) {-        // nothing to extend if there's no new/original knowledge-        if (other.size() == 0 || this->size() == 0) return;--        this->genAllDisjointSetLists();-        other.genAllDisjointSetLists();--        std::set<value_type> repsCovered;--        // find all the disjoint sets that need to be added to this relation-        // that exist in other (and exist in this)-        {-            auto it = this->sds.sparseToDenseMap.begin();-            auto end = this->sds.sparseToDenseMap.end();-            value_type el;-            for (; it != end; ++it) {-                std::tie(el, std::ignore) = *it;-                if (other.containsElement(el)) {-                    value_type rep = other.sds.findNode(el);-                    if (repsCovered.count(rep) == 0) {-                        repsCovered.emplace(rep);-                    }-                }-            }-        }--        // add the intersecting dj sets into this one-        {-            value_type el;-            value_type rep;-            auto it = other.sds.sparseToDenseMap.begin();-            auto end = other.sds.sparseToDenseMap.end();-            for (; it != end; ++it) {-                std::tie(el, std::ignore) = *it;-                rep = other.sds.findNode(el);-                if (repsCovered.count(rep) != 0) {-                    this->insert(el, rep);-                }-            }-        }-    }--    /**-     * Returns whether there exists a pair with these two nodes-     * @param x front of pair-     * @param y back of pair-     */-    bool contains(value_type x, value_type y) const {-        return sds.contains(x, y);-    }--    /**-     * Returns whether there exists given tuple.-     * @param tuple The tuple to search for.-     */-    bool contains(const TupleType& tuple, operation_hints&) const {-        return contains(tuple[0], tuple[1]);-    };--    void emptyPartition() const {-        // delete the beautiful values inside (they're raw ptrs, so they need to be.)-        for (auto& pair : equivalencePartition) {-            delete pair.second;-        }-        // invalidate it my dude-        this->statesMapStale.store(true, std::memory_order_relaxed);--        equivalencePartition.clear();-    }--    /**-     * Empty the relation-     */-    void clear() {-        statesLock.lock();--        sds.clear();-        emptyPartition();--        statesLock.unlock();-    }--    /**-     * Size of relation-     * @return the sum of the number of pairs per disjoint set-     */-    size_t size() const {-        genAllDisjointSetLists();--        statesLock.lock_shared();--        size_t retVal = 0;-        for (auto& e : this->equivalencePartition) {-            const size_t s = e.second->size();-            retVal += s * s;-        }--        statesLock.unlock_shared();-        return retVal;-    }--    // an almighty iterator for several types of iteration.-    // Unfortunately, subclassing isn't an option with souffle-    //   - we don't deal with pointers (so no virtual)-    //   - and a single iter type is expected (see Relation::iterator e.g.) (i think)-    class iterator : public std::iterator<std::forward_iterator_tag, TupleType> {-    public:-        // one iterator for signalling the end (simplifies)-        explicit iterator(const EquivalenceRelation* br, bool /* signalIsEndIterator */)-                : br(br), isEndVal(true){};--        explicit iterator(const EquivalenceRelation* br)-                : br(br), ityp(IterType::ALL), djSetMapListIt(br->equivalencePartition.begin()),-                  djSetMapListEnd(br->equivalencePartition.end()) {-            // no need to fast forward if this iterator is empty-            if (djSetMapListIt == djSetMapListEnd) {-                isEndVal = true;-                return;-            }-            // grab the pointer to the list, and make it our current list-            djSetList = (*djSetMapListIt).second;-            assert(djSetList->size() != 0);--            updateAnterior();-            updatePosterior();-        }--        // WITHIN: iterator for everything within the same DJset (used for EquivalenceRelation.partition())-        explicit iterator(const EquivalenceRelation* br, const StatesBucket within)-                : br(br), ityp(IterType::WITHIN), djSetList(within) {-            // empty dj set-            if (djSetList->size() == 0) {-                isEndVal = true;-            }--            updateAnterior();-            updatePosterior();-        }--        // ANTERIOR: iterator that yields all (former, _) \in djset(former) (djset(former) === within)-        explicit iterator(const EquivalenceRelation* br, const value_type former, const StatesBucket within)-                : br(br), ityp(IterType::ANTERIOR), djSetList(within) {-            if (djSetList->size() == 0) {-                isEndVal = true;-            }--            setAnterior(former);-            updatePosterior();-        }--        // ANTPOST: iterator that yields all (former, latter) \in djset(former), (djset(former) ==-        // djset(latter) == within)-        explicit iterator(const EquivalenceRelation* br, const value_type former, value_type latter,-                const StatesBucket within)-                : br(br), ityp(IterType::ANTPOST), djSetList(within) {-            if (djSetList->size() == 0) {-                isEndVal = true;-            }--            setAnterior(former);-            setPosterior(latter);-        }--        /** explicit set first half of cPair */-        inline void setAnterior(const value_type a) {-            this->cPair[0] = a;-        }--        /** quick update to whatever the current index is pointing to */-        inline void updateAnterior() {-            this->cPair[0] = this->djSetList->get(this->cAnteriorIndex);-        }--        /** explicit set second half of cPair */-        inline void setPosterior(const value_type b) {-            this->cPair[1] = b;-        }--        /** quick update to whatever the current index is pointing to */-        inline void updatePosterior() {-            this->cPair[1] = this->djSetList->get(this->cPosteriorIndex);-        }--        // copy ctor-        iterator(const iterator& other) = default;-        // move ctor-        iterator(iterator&& other) = default;-        // assign iter-        iterator& operator=(const iterator& other) = default;--        bool operator==(const iterator& other) const {-            if (isEndVal && other.isEndVal) return br == other.br;-            return isEndVal == other.isEndVal && cPair == other.cPair;-        }--        bool operator!=(const iterator& other) const {-            return !((*this) == other);-        }--        const TupleType& operator*() const {-            return cPair;-        }--        const TupleType* operator->() const {-            return &cPair;-        }--        /* pre-increment */-        iterator& operator++() {-            if (isEndVal) {-                throw std::out_of_range("error: incrementing an out of range iterator");-            }--            switch (ityp) {-                case IterType::ALL:-                    // move posterior along one-                    // see if we can't move the posterior along-                    if (++cPosteriorIndex == djSetList->size()) {-                        // move anterior along one-                        // see if we can't move the anterior along one-                        if (++cAnteriorIndex == djSetList->size()) {-                            // move the djset it along one-                            // see if we can't move it along one (we're at the end)-                            if (++djSetMapListIt == djSetMapListEnd) {-                                isEndVal = true;-                                return *this;-                            }--                            // we can't iterate along this djset if it is empty-                            djSetList = (*djSetMapListIt).second;-                            if (djSetList->size() == 0) {-                                throw std::out_of_range("error: encountered a zero size djset");-                            }--                            // update our cAnterior and cPosterior-                            cAnteriorIndex = 0;-                            cPosteriorIndex = 0;-                            updateAnterior();-                            updatePosterior();-                        }--                        // we moved our anterior along one-                        updateAnterior();--                        cPosteriorIndex = 0;-                        updatePosterior();-                    }-                    // we just moved our posterior along one-                    updatePosterior();--                    break;-                case IterType::ANTERIOR:-                    // step posterior along one, and if we can't, then we're done.-                    if (++cPosteriorIndex == djSetList->size()) {-                        isEndVal = true;-                        return *this;-                    }-                    updatePosterior();--                    break;-                case IterType::ANTPOST:-                    // fixed anterior and posterior literally only points to one, so if we increment, its the-                    // end-                    isEndVal = true;-                    break;-                case IterType::WITHIN:-                    // move posterior along one-                    // see if we can't move the posterior along-                    if (++cPosteriorIndex == djSetList->size()) {-                        // move anterior along one-                        // see if we can't move the anterior along one-                        if (++cAnteriorIndex == djSetList->size()) {-                            isEndVal = true;-                            return *this;-                        }--                        // we moved our anterior along one-                        updateAnterior();--                        cPosteriorIndex = 0;-                        updatePosterior();-                    }-                    // we just moved our posterior along one-                    updatePosterior();-                    break;-            }--            return *this;-        }--    private:-        const EquivalenceRelation* br = nullptr;-        // special tombstone value to notify that this iter represents the end-        bool isEndVal = false;--        // all the different types of iterator this can be-        enum IterType { ALL, ANTERIOR, ANTPOST, WITHIN };-        IterType ityp;--        TupleType cPair;--        // the disjoint set that we're currently iterating through-        StatesBucket djSetList;-        typename StatesMap::iterator djSetMapListIt;-        typename StatesMap::iterator djSetMapListEnd;--        // used for ALL, and POSTERIOR (just a current index in the cList)-        size_t cAnteriorIndex = 0;-        // used for ALL, and ANTERIOR (just a current index in the cList)-        size_t cPosteriorIndex = 0;-    };--public:-    /**-     * iterator pointing to the beginning of the tuples, with no restrictions-     * @return the iterator that corresponds to the beginning of the binary relation-     */-    iterator begin() const {-        genAllDisjointSetLists();-        return iterator(this);-    }--    /**-     * iterator pointing to the end of the tuples-     * @return the iterator which represents the end of the binary rel-     */-    iterator end() const {-        return iterator(this, true);-    }--    /**-     * Obtains a range of elements matching the prefix of the given entry up to-     * levels elements.-     *-     * @tparam levels the length of the requested matching prefix-     * @param entry the entry to be looking for-     * @return the corresponding range of matching elements-     */-    template <unsigned levels>-    range<iterator> getBoundaries(const TupleType& entry) const {-        operation_hints ctxt;-        return getBoundaries<levels>(entry, ctxt);-    }--    /**-     * Obtains a range of elements matching the prefix of the given entry up to-     * levels elements. A operation context may be provided to exploit temporal-     * locality.-     *-     * @tparam levels the length of the requested matching prefix-     * @param entry the entry to be looking for-     * @param ctxt the operation context to be utilized-     * @return the corresponding range of matching elements-     */-    template <unsigned levels>-    range<iterator> getBoundaries(const TupleType& entry, operation_hints&) const {-        // if nothing is bound => just use begin and end-        if (levels == 0) return make_range(begin(), end());--        // as disjoint set is exactly two args (equiv relation)-        // we only need to handle these cases--        if (levels == 1) {-            // need to test if the entry actually exists-            if (!sds.nodeExists(entry[0])) return make_range(end(), end());--            // return an iterator over all (entry[0], _)-            return make_range(anteriorIt(entry[0]), end());-        }--        if (levels == 2) {-            // need to test if the entry actually exists-            if (!sds.contains(entry[0], entry[1])) return make_range(end(), end());--            // if so return an iterator containing exactly that node-            return make_range(antpostit(entry[0], entry[1]), end());-        }--        std::cerr << "invalid state, cannot search for >2 arg start point in getBoundaries, in 2 arg tuple "-                     "store\n";-        throw "invalid state, cannot search for >2 arg start point in getBoundaries, in 2 arg tuple store";--        return make_range(end(), end());-    }--    /**-     * Act similar to getBoundaries. But non-static.-     * This function should be used ONLY by interpreter,-     * and its behavior is tightly coupling with InterpreterIndex.-     * Do Not rely on this interface outside the interpreter.-     *-     * @param entry the entry to be looking for-     * @return the corresponding range of matching elements-     */-    iterator lower_bound(const TupleType& entry, operation_hints&) const {-        if (entry[0] == MIN_RAM_SIGNED && entry[1] == MIN_RAM_SIGNED) {-            // Return an iterator over all tuples.-            return begin();-        }--        if (entry[0] != MIN_RAM_SIGNED && entry[1] == MIN_RAM_SIGNED) {-            // Return an iterator over all (entry[0], _)--            if (!sds.nodeExists(entry[0])) {-                return end();-            }-            return anteriorIt(entry[0]);-        }--        if (entry[0] != MIN_RAM_SIGNED && entry[1] != MIN_RAM_SIGNED) {-            // Return an iterator point to the exact same node.--            if (!sds.contains(entry[0], entry[1])) {-                return end();-            }-            return antpostit(entry[0], entry[1]);-        }--        return end();-    }--    /**-     * This function is only here in order to unify interfaces in InterpreterIndex.-     * Unlike the name suggestes, it omit the arguments and simply return the end-     * iterator of the relation.-     *-     * @param omitted-     * @return the end iterator.-     */-    iterator upper_bound(const TupleType&, operation_hints&) const {-        return end();-    }--    /**-     * Check emptiness.-     */-    bool empty() const {-        return this->size() == 0;-    }--    /**-     * Creates an iterator that generates all pairs (A, X)-     * for a given A, and X are elements within A's disjoint set.-     * @param anteriorVal: The first value of the tuple to be generated for-     * @return the iterator representing this.-     */-    iterator anteriorIt(value_type anteriorVal) const {-        genAllDisjointSetLists();--        // locate the blocklist that the anterior val resides in-        auto found = equivalencePartition.find({sds.findNode(anteriorVal), nullptr});-        assert(found != equivalencePartition.end() && "iterator called on partition that doesn't exist");--        return iterator(this, anteriorVal, (*found).second);-    }--    /**-     * Creates an iterator that generates the pair (A, B)-     * for a given A and B. If A and B don't exist, or aren't in the same set,-     * then the end() iterator is returned.-     * @param anteriorVal: the A value of the tuple-     * @param posteriorVal: the B value of the tuple-     * @return the iterator representing this-     */-    iterator antpostit(value_type anteriorVal, value_type posteriorVal) const {-        // obv if they're in diff sets, then iteration for this pair just ends.-        if (!sds.sameSet(anteriorVal, posteriorVal)) return end();--        genAllDisjointSetLists();--        // locate the blocklist that the val resides in-        auto found = equivalencePartition.find({sds.findNode(posteriorVal), nullptr});-        assert(found != equivalencePartition.end() && "iterator called on partition that doesn't exist");--        return iterator(this, anteriorVal, posteriorVal, (*found).second);-    }--    /**-     * Begin an iterator over all pairs within a single disjoint set - This is used for partition().-     * @param rep the representative of (or element within) a disjoint set of which to generate all pairs-     * @return an iterator that will generate all pairs within the disjoint set-     */-    iterator closure(value_type rep) const {-        genAllDisjointSetLists();--        // locate the blocklist that the val resides in-        auto found = equivalencePartition.find({sds.findNode(rep), nullptr});-        return iterator(this, (*found).second);-    }--    /**-     * Generate an approximate number of iterators for parallel iteration-     * The iterators returned are not necessarily equal in size, but in practise are approximately similarly-     * sized-     * Depending on the structure of the data, there can be more or less partitions returned than requested.-     * @param chunks the number of requested partitions-     * @return a list of the iterators as ranges-     */-    std::vector<souffle::range<iterator>> partition(size_t chunks) const {-        // generate all reps-        genAllDisjointSetLists();--        size_t numPairs = this->size();-        if (numPairs == 0) return {};-        if (numPairs == 1 || chunks <= 1) return {souffle::make_range(begin(), end())};--        // if there's more dj sets than requested chunks, then just return an iter per dj set-        std::vector<souffle::range<iterator>> ret;-        if (chunks <= equivalencePartition.size()) {-            for (auto& p : equivalencePartition) {-                ret.push_back(souffle::make_range(closure(p.first), end()));-            }-            return ret;-        }--        // keep it simple stupid-        // just go through and if the size of the binrel is > numpairs/chunks, then generate an anteriorIt for-        // each-        const size_t perchunk = numPairs / chunks;-        for (const auto& itp : equivalencePartition) {-            const size_t s = itp.second->size();-            if (s * s > perchunk) {-                for (const auto& i : *itp.second) {-                    ret.push_back(souffle::make_range(anteriorIt(i), end()));-                }-            } else {-                ret.push_back(souffle::make_range(closure(itp.first), end()));-            }-        }--        return ret;-    }--    iterator find(const TupleType&, operation_hints&) const {-        throw std::runtime_error("error: find() is not compatible with equivalence relations");-        return begin();-    }--    iterator find(const TupleType& t) const {-        operation_hints context;-        return find(t, context);-    }--protected:-    bool containsElement(value_type e) const {-        return this->sds.nodeExists(e);-    }--private:-    // marked as mutable due to difficulties with the const enforcement via the Relation API-    // const operations *may* safely change internal state (i.e. collapse djset forest)-    mutable souffle::SparseDisjointSet<value_type> sds;--    // read/write lock on equivalencePartition-    mutable std::shared_mutex statesLock;--    mutable StatesMap equivalencePartition;-    // whether the cache is stale-    mutable std::atomic<bool> statesMapStale;--    /**-     * Generate a cache of the sets such that they can be iterated over efficiently.-     * Each set is partitioned into a PiggyList.-     */-    void genAllDisjointSetLists() const {-        statesLock.lock();--        // no need to generate again, already done.-        if (!this->statesMapStale.load(std::memory_order_acquire)) {-            statesLock.unlock();-            return;-        }--        // btree version-        emptyPartition();--        size_t dSetSize = this->sds.ds.a_blocks.size();-        for (size_t i = 0; i < dSetSize; ++i) {-            typename TupleType::value_type sparseVal = this->sds.toSparse(i);-            parent_t rep = this->sds.findNode(sparseVal);--            StorePair p = {rep, nullptr};-            StatesList* mapList = equivalencePartition.insert(p, [&](StorePair& sp) {-                auto* r = new StatesList(1);-                sp.second = r;-                return r;-            });-            mapList->append(sparseVal);-        }--        statesMapStale.store(false, std::memory_order_release);-        statesLock.unlock();-    }-};-}  // namespace souffle
− cbits/souffle/EventProcessor.h 
@@ -1,572 +0,0 @@-/*- * Souffle - A Datalog Compiler- * Copyright (c) 2018, The Souffle Developers. All rights reserved.- * Licensed under the Universal Permissive License v 1.0 as shown at:- * - https://opensource.org/licenses/UPL- * - <souffle root>/licenses/SOUFFLE-UPL.txt- */--/************************************************************************- *- * @file EventProcessor.h- *- * Declares classes for event processor that parse profile events and- * populate the profile database- *- ***********************************************************************/--#pragma once--#include "ProfileDatabase.h"-#include "utility/MiscUtil.h"-#include "utility/StreamUtil.h"-#include <cassert>-#include <chrono>-#include <cstdarg>-#include <cstdint>-#include <cstdlib>-#include <iostream>-#include <map>-#include <string>-#include <vector>--namespace souffle {-namespace profile {-/**- * Abstract Class for EventProcessor- */-class EventProcessor {-public:-    virtual ~EventProcessor() = default;--    /** abstract interface for processing an profile event */-    virtual void process(ProfileDatabase&, const std::vector<std::string>& signature, va_list&) {-        fatal("Unknown profiling processing event: %s", join(signature, " "));-    }-};--/**- * Event Processor Singleton- *- * Singleton that is the connection point for events- */-class EventProcessorSingleton {-public:-    /** get instance */-    static EventProcessorSingleton& instance() {-        static EventProcessorSingleton singleton;-        return singleton;-    }--    /** register an event processor with its keyword */-    void registerEventProcessor(const std::string& keyword, EventProcessor* processor) {-        registry[keyword] = processor;-    }--    /** process a profile event */-    void process(ProfileDatabase& db, const char* txt, ...) {-        va_list args;-        va_start(args, txt);--        // escape signature-        std::string escapedText = escape(txt);-        // obtain event signature by splitting event text-        std::vector<std::string> eventSignature = splitSignature(escapedText);--        // invoke the event processor of the event-        const std::string& keyword = eventSignature[0];-        assert(eventSignature.size() > 0 && "no keyword in event description");-        assert(registry.find(keyword) != registry.end() && "EventProcessor not found!");-        registry[keyword]->process(db, eventSignature, args);--        // terminate access to variadic arguments-        va_end(args);-    }--private:-    /** keyword / event processor mapping */-    std::map<std::string, EventProcessor*> registry;--    EventProcessorSingleton() = default;--    /**-     * Escape escape characters.-     *-     * Remove all escapes, then escape double quotes.-     */-    std::string escape(const std::string& text) {-        std::string str(text);-        size_t start_pos = 0;-        // replace backslashes with double backslash-        while ((start_pos = str.find('\\', start_pos)) != std::string::npos) {-            if (start_pos == str.size()) {-                break;-            }-            ++start_pos;-            if (str[start_pos] == 't' || str[start_pos] == '"' || str[start_pos] == '\\' ||-                    str[start_pos] == 'n' || str[start_pos] == ';') {-                continue;-            }-            str.replace(start_pos - 1, 1, "\\\\");-            ++start_pos;-        }-        return str;-    }--    /** split string */-    static std::vector<std::string> split(std::string str, std::string split_str) {-        // repeat value when splitting so "a   b" -> ["a","b"] not ["a","","","","b"]-        bool repeat = (split_str == " ");--        std::vector<std::string> elems;--        std::string temp;-        std::string hold;-        for (size_t i = 0; i < str.size(); i++) {-            if (repeat) {-                if (str.at(i) == split_str.at(0)) {-                    while (str.at(++i) == split_str.at(0)) {-                        ;  // set i to be at the end of the search string-                    }-                    elems.push_back(temp);-                    temp = "";-                }-                temp += str.at(i);-            } else {-                temp += str.at(i);-                hold += str.at(i);-                for (size_t j = 0; j < hold.size(); j++) {-                    if (hold[j] != split_str[j]) {-                        hold = "";-                    }-                }-                if (hold.size() == split_str.size()) {-                    elems.push_back(temp.substr(0, temp.size() - hold.size()));-                    hold = "";-                    temp = "";-                }-            }-        }-        if (!temp.empty()) {-            elems.push_back(temp);-        }--        return elems;-    }--    /** split string separated by semi-colon */-    static std::vector<std::string> splitSignature(std::string str) {-        for (size_t i = 0; i < str.size(); i++) {-            if (i > 0 && str[i] == ';' && str[i - 1] == '\\') {-                // I'm assuming this isn't a thing that will be naturally found in souffle profiler files-                str[i - 1] = '\b';-                str.erase(i--, 1);-            }-        }-        std::vector<std::string> result = split(str, ";");-        for (auto& i : result) {-            for (char& j : i) {-                if (j == '\b') {-                    j = ';';-                }-            }-        }-        return result;-    }-};--/**- * Non-Recursive Rule Timing Profile Event Processor- */-const class NonRecursiveRuleTimingProcessor : public EventProcessor {-public:-    NonRecursiveRuleTimingProcessor() {-        EventProcessorSingleton::instance().registerEventProcessor("@t-nonrecursive-rule", this);-    }-    void process(ProfileDatabase& db, const std::vector<std::string>& signature, va_list& args) override {-        const std::string& relation = signature[1];-        const std::string& srcLocator = signature[2];-        const std::string& rule = signature[3];-        microseconds start = va_arg(args, microseconds);-        microseconds end = va_arg(args, microseconds);-        size_t startMaxRSS = va_arg(args, size_t);-        size_t endMaxRSS = va_arg(args, size_t);-        size_t size = va_arg(args, size_t);-        db.addSizeEntry(-                {"program", "relation", relation, "non-recursive-rule", rule, "maxRSS", "pre"}, startMaxRSS);-        db.addSizeEntry(-                {"program", "relation", relation, "non-recursive-rule", rule, "maxRSS", "post"}, endMaxRSS);-        db.addTextEntry(-                {"program", "relation", relation, "non-recursive-rule", rule, "source-locator"}, srcLocator);-        db.addDurationEntry(-                {"program", "relation", relation, "non-recursive-rule", rule, "runtime"}, start, end);-        db.addSizeEntry({"program", "relation", relation, "non-recursive-rule", rule, "num-tuples"}, size);-    }-} nonRecursiveRuleTimingProcessor;--/**- * Non-Recursive Rule Number Profile Event Processor- */-const class NonRecursiveRuleNumberProcessor : public EventProcessor {-public:-    NonRecursiveRuleNumberProcessor() {-        EventProcessorSingleton::instance().registerEventProcessor("@n-nonrecursive-rule", this);-    }-    /** process event input */-    void process(ProfileDatabase& db, const std::vector<std::string>& signature, va_list& args) override {-        const std::string& relation = signature[1];-        const std::string& srcLocator = signature[2];-        const std::string& rule = signature[3];-        size_t num = va_arg(args, size_t);-        db.addTextEntry(-                {"program", "relation", relation, "non-recursive-rule", rule, "source-locator"}, srcLocator);-        db.addSizeEntry({"program", "relation", relation, "non-recursive-rule", rule, "num-tuples"}, num);-    }-} nonRecursiveRuleNumberProcessor;--/**- * Recursive Rule Timing Profile Event Processor- */-const class RecursiveRuleTimingProcessor : public EventProcessor {-public:-    RecursiveRuleTimingProcessor() {-        EventProcessorSingleton::instance().registerEventProcessor("@t-recursive-rule", this);-    }-    void process(ProfileDatabase& db, const std::vector<std::string>& signature, va_list& args) override {-        const std::string& relation = signature[1];-        const std::string& version = signature[2];-        const std::string& srcLocator = signature[3];-        const std::string& rule = signature[4];-        microseconds start = va_arg(args, microseconds);-        microseconds end = va_arg(args, microseconds);-        size_t startMaxRSS = va_arg(args, size_t);-        size_t endMaxRSS = va_arg(args, size_t);-        size_t size = va_arg(args, size_t);-        std::string iteration = std::to_string(va_arg(args, size_t));-        db.addSizeEntry({"program", "relation", relation, "iteration", iteration, "recursive-rule", rule,-                                version, "maxRSS", "pre"},-                startMaxRSS);-        db.addSizeEntry({"program", "relation", relation, "iteration", iteration, "recursive-rule", rule,-                                version, "maxRSS", "post"},-                endMaxRSS);-        db.addTextEntry({"program", "relation", relation, "iteration", iteration, "recursive-rule", rule,-                                version, "source-locator"},-                srcLocator);-        db.addDurationEntry({"program", "relation", relation, "iteration", iteration, "recursive-rule", rule,-                                    version, "runtime"},-                start, end);-        db.addSizeEntry({"program", "relation", relation, "iteration", iteration, "recursive-rule", rule,-                                version, "num-tuples"},-                size);-    }-} recursiveRuleTimingProcessor;--/**- * Recursive Rule Number Profile Event Processor- */-const class RecursiveRuleNumberProcessor : public EventProcessor {-public:-    RecursiveRuleNumberProcessor() {-        EventProcessorSingleton::instance().registerEventProcessor("@n-recursive-rule", this);-    }-    void process(ProfileDatabase& db, const std::vector<std::string>& signature, va_list& args) override {-        const std::string& relation = signature[1];-        const std::string& version = signature[2];-        const std::string& srcLocator = signature[3];-        const std::string& rule = signature[4];-        size_t number = va_arg(args, size_t);-        std::string iteration = std::to_string(va_arg(args, size_t));-        db.addTextEntry({"program", "relation", relation, "iteration", iteration, "recursive-rule", rule,-                                version, "source-locator"},-                srcLocator);-        db.addSizeEntry({"program", "relation", relation, "iteration", iteration, "recursive-rule", rule,-                                version, "num-tuples"},-                number);-    }-} recursiveRuleNumberProcessor;--/**- * Non-Recursive Relation Number Profile Event Processor- */-const class NonRecursiveRelationTimingProcessor : public EventProcessor {-public:-    NonRecursiveRelationTimingProcessor() {-        EventProcessorSingleton::instance().registerEventProcessor("@t-nonrecursive-relation", this);-    }-    /** process event input */-    void process(ProfileDatabase& db, const std::vector<std::string>& signature, va_list& args) override {-        const std::string& relation = signature[1];-        const std::string& srcLocator = signature[2];-        microseconds start = va_arg(args, microseconds);-        microseconds end = va_arg(args, microseconds);-        size_t startMaxRSS = va_arg(args, size_t);-        size_t endMaxRSS = va_arg(args, size_t);-        size_t size = va_arg(args, size_t);-        db.addSizeEntry({"program", "relation", relation, "maxRSS", "pre"}, startMaxRSS);-        db.addSizeEntry({"program", "relation", relation, "maxRSS", "post"}, endMaxRSS);-        db.addSizeEntry({"program", "relation", relation, "num-tuples"}, size);-        db.addTextEntry({"program", "relation", relation, "source-locator"}, srcLocator);-        db.addDurationEntry({"program", "relation", relation, "runtime"}, start, end);-    }-} nonRecursiveRelationTimingProcessor;--/**- * Non-Recursive Relation Number Profile Event Processor- */-const class NonRecursiveRelationNumberProcessor : public EventProcessor {-public:-    NonRecursiveRelationNumberProcessor() {-        EventProcessorSingleton::instance().registerEventProcessor("@n-nonrecursive-relation", this);-    }-    /** process event input */-    void process(ProfileDatabase& db, const std::vector<std::string>& signature, va_list& args) override {-        const std::string& relation = signature[1];-        const std::string& srcLocator = signature[2];-        size_t num = va_arg(args, size_t);-        db.addTextEntry({"program", "relation", relation, "source-locator"}, srcLocator);-        db.addSizeEntry({"program", "relation", relation, "num-tuples"}, num);-    }-} nonRecursiveRelationNumberProcessor;--/**- * Recursive Relation Timing Profile Event Processor- */-const class RecursiveRelationTimingProcessor : public EventProcessor {-public:-    RecursiveRelationTimingProcessor() {-        EventProcessorSingleton::instance().registerEventProcessor("@t-recursive-relation", this);-    }-    /** process event input */-    void process(ProfileDatabase& db, const std::vector<std::string>& signature, va_list& args) override {-        const std::string& relation = signature[1];-        const std::string& srcLocator = signature[2];-        microseconds start = va_arg(args, microseconds);-        microseconds end = va_arg(args, microseconds);-        size_t startMaxRSS = va_arg(args, size_t);-        size_t endMaxRSS = va_arg(args, size_t);-        size_t size = va_arg(args, size_t);-        std::string iteration = std::to_string(va_arg(args, size_t));-        db.addTextEntry({"program", "relation", relation, "source-locator"}, srcLocator);-        db.addDurationEntry({"program", "relation", relation, "iteration", iteration, "runtime"}, start, end);-        db.addSizeEntry(-                {"program", "relation", relation, "iteration", iteration, "maxRSS", "pre"}, startMaxRSS);-        db.addSizeEntry(-                {"program", "relation", relation, "iteration", iteration, "maxRSS", "post"}, endMaxRSS);-        db.addSizeEntry({"program", "relation", relation, "iteration", iteration, "num-tuples"}, size);-    }-} recursiveRelationTimingProcessor;--/**- * Recursive Relation Timing Profile Event Processor- */-const class RecursiveRelationNumberProcessor : public EventProcessor {-public:-    RecursiveRelationNumberProcessor() {-        EventProcessorSingleton::instance().registerEventProcessor("@n-recursive-relation", this);-    }-    /** process event input */-    void process(ProfileDatabase& db, const std::vector<std::string>& signature, va_list& args) override {-        const std::string& relation = signature[1];-        const std::string& srcLocator = signature[2];-        size_t number = va_arg(args, size_t);-        std::string iteration = std::to_string(va_arg(args, size_t));-        db.addTextEntry({"program", "relation", relation, "source-locator"}, srcLocator);-        db.addSizeEntry({"program", "relation", relation, "iteration", iteration, "num-tuples"}, number);-    }-} recursiveRelationNumberProcessor;--/**- * Recursive Relation Copy Timing Profile Event Processor- */-const class RecursiveRelationCopyTimingProcessor : public EventProcessor {-public:-    RecursiveRelationCopyTimingProcessor() {-        EventProcessorSingleton::instance().registerEventProcessor("@c-recursive-relation", this);-    }-    /** process event input */-    void process(ProfileDatabase& db, const std::vector<std::string>& signature, va_list& args) override {-        const std::string& relation = signature[1];-        const std::string& srcLocator = signature[2];-        microseconds start = va_arg(args, microseconds);-        microseconds end = va_arg(args, microseconds);-        size_t startMaxRSS = va_arg(args, size_t);-        size_t endMaxRSS = va_arg(args, size_t);-        va_arg(args, size_t);-        std::string iteration = std::to_string(va_arg(args, size_t));-        db.addSizeEntry(-                {"program", "relation", relation, "iteration", iteration, "maxRSS", "pre"}, startMaxRSS);-        db.addSizeEntry(-                {"program", "relation", relation, "iteration", iteration, "maxRSS", "post"}, endMaxRSS);-        db.addTextEntry({"program", "relation", relation, "source-locator"}, srcLocator);-        db.addDurationEntry(-                {"program", "relation", relation, "iteration", iteration, "copytime"}, start, end);-    }-} recursiveRelationCopyTimingProcessor;--/**- * Recursive Relation Copy Timing Profile Event Processor- */-const class RelationIOTimingProcessor : public EventProcessor {-public:-    RelationIOTimingProcessor() {-        EventProcessorSingleton::instance().registerEventProcessor("@t-relation-savetime", this);-        EventProcessorSingleton::instance().registerEventProcessor("@t-relation-loadtime", this);-    }-    /** process event input */-    void process(ProfileDatabase& db, const std::vector<std::string>& signature, va_list& args) override {-        const std::string& relation = signature[1];-        const std::string& srcLocator = signature[2];-        const std::string ioType = signature[3];-        microseconds start = va_arg(args, microseconds);-        microseconds end = va_arg(args, microseconds);-        db.addTextEntry({"program", "relation", relation, "source-locator"}, srcLocator);-        db.addDurationEntry({"program", "relation", relation, ioType}, start, end);-    }-} relationIOTimingProcessor;--/**- * Program Run Event Processor- */-const class ProgramTimepointProcessor : public EventProcessor {-public:-    ProgramTimepointProcessor() {-        EventProcessorSingleton::instance().registerEventProcessor("@time", this);-    }-    /** process event input */-    void process(ProfileDatabase& db, const std::vector<std::string>& signature, va_list& args) override {-        microseconds time = va_arg(args, microseconds);-        auto path = signature;-        path[0] = "program";-        db.addTimeEntry(path, time);-    }-} programTimepointProcessor;--/**- * Program Run Event Processor- */-const class ProgramRuntimeProcessor : public EventProcessor {-public:-    ProgramRuntimeProcessor() {-        EventProcessorSingleton::instance().registerEventProcessor("@runtime", this);-    }-    /** process event input */-    void process(-            ProfileDatabase& db, const std::vector<std::string>& /* signature */, va_list& args) override {-        microseconds start = va_arg(args, microseconds);-        microseconds end = va_arg(args, microseconds);-        db.addDurationEntry({"program", "runtime"}, start, end);-    }-} programRuntimeProcessor;--/**- * Program Resource Utilisation Event Processor- */-const class ProgramResourceUtilisationProcessor : public EventProcessor {-public:-    ProgramResourceUtilisationProcessor() {-        EventProcessorSingleton::instance().registerEventProcessor("@utilisation", this);-    }-    /** process event input */-    void process(-            ProfileDatabase& db, const std::vector<std::string>& /* signature */, va_list& args) override {-        microseconds time = va_arg(args, microseconds);-        uint64_t systemTime = va_arg(args, uint64_t);-        uint64_t userTime = va_arg(args, uint64_t);-        size_t maxRSS = va_arg(args, size_t);-        std::string timeString = std::to_string(time.count());-        db.addSizeEntry({"program", "usage", "timepoint", timeString, "systemtime"}, systemTime);-        db.addSizeEntry({"program", "usage", "timepoint", timeString, "usertime"}, userTime);-        db.addSizeEntry({"program", "usage", "timepoint", timeString, "maxRSS"}, maxRSS);-    }-} programResourceUtilisationProcessor;--/**- * Frequency Atom Processor- */-const class FrequencyAtomProcessor : public EventProcessor {-public:-    FrequencyAtomProcessor() {-        EventProcessorSingleton::instance().registerEventProcessor("@frequency-atom", this);-    }-    /** process event input */-    void process(ProfileDatabase& db, const std::vector<std::string>& signature, va_list& args) override {-        const std::string& relation = signature[1];-        const std::string& version = signature[2];-        const std::string& rule = signature[3];-        const std::string& atom = signature[4];-        const std::string& originalRule = signature[5];-        size_t level = std::stoi(signature[6]);-        size_t number = va_arg(args, size_t);-        size_t iteration = va_arg(args, size_t);-        // non-recursive rule-        if (rule == originalRule) {-            db.addSizeEntry({"program", "relation", relation, "non-recursive-rule", rule, "atom-frequency",-                                    rule, atom, "level"},-                    level);-            db.addSizeEntry({"program", "relation", relation, "non-recursive-rule", rule, "atom-frequency",-                                    rule, atom, "num-tuples"},-                    number);-        } else {-            db.addSizeEntry(-                    {"program", "relation", relation, "iteration", std::to_string(iteration),-                            "recursive-rule", originalRule, version, "atom-frequency", rule, atom, "level"},-                    level);-            db.addSizeEntry({"program", "relation", relation, "iteration", std::to_string(iteration),-                                    "recursive-rule", originalRule, version, "atom-frequency", rule, atom,-                                    "num-tuples"},-                    number);-        }-    }-} frequencyAtomProcessor;--/**- * Reads Processor- */-const class RelationReadsProcessor : public EventProcessor {-public:-    RelationReadsProcessor() {-        EventProcessorSingleton::instance().registerEventProcessor("@relation-reads", this);-    }-    /** process event input */-    void process(ProfileDatabase& db, const std::vector<std::string>& signature, va_list& args) override {-        const std::string& relation = signature[1];-        size_t reads = va_arg(args, size_t);-        db.addSizeEntry({"program", "relation", relation, "reads"}, reads);-    }--} relationReadsProcessor;--/**- * Config entry processor- */-const class ConfigProcessor : public EventProcessor {-public:-    ConfigProcessor() {-        EventProcessorSingleton::instance().registerEventProcessor("@config", this);-    }-    void process(-            ProfileDatabase& db, const std::vector<std::string>& /* signature */, va_list& args) override {-        const std::string key = va_arg(args, char*);-        const std::string& value = va_arg(args, char*);-        db.addTextEntry({"program", "configuration", key}, value);-    }-} configProcessor;--/**- * Text entry processor- */-const class TextProcessor : public EventProcessor {-public:-    TextProcessor() {-        EventProcessorSingleton::instance().registerEventProcessor("@text", this);-    }-    void process(ProfileDatabase& db, const std::vector<std::string>& signature, va_list& args) override {-        const std::string text = va_arg(args, char*);-        auto path = signature;-        path.front() = "program";-        db.addTextEntry(path, text);-    }-} textProcessor;--}  // namespace profile-}  // namespace souffle
− cbits/souffle/IOSystem.h 
@@ -1,98 +0,0 @@-/*- * Souffle - A Datalog Compiler- * Copyright (c) 2013, 2014, Oracle and/or its affiliates. All rights reserved- * Licensed under the Universal Permissive License v 1.0 as shown at:- * - https://opensource.org/licenses/UPL- * - <souffle root>/licenses/SOUFFLE-UPL.txt- */--/************************************************************************- *- * @file IOSystem.h- *- ***********************************************************************/--#pragma once--#include "RamTypes.h"-#include "ReadStream.h"-#include "ReadStreamCSV.h"-#include "ReadStreamJSON.h"-#include "SymbolTable.h"-#include "WriteStream.h"-#include "WriteStreamCSV.h"-#include "WriteStreamJSON.h"--#ifdef USE_SQLITE-#include "ReadStreamSQLite.h"-#include "WriteStreamSQLite.h"-#endif--#include <map>-#include <memory>-#include <stdexcept>-#include <string>--namespace souffle {-class RecordTable;--class IOSystem {-public:-    static IOSystem& getInstance() {-        static IOSystem singleton;-        return singleton;-    }--    void registerWriteStreamFactory(const std::shared_ptr<WriteStreamFactory>& factory) {-        outputFactories[factory->getName()] = factory;-    }--    void registerReadStreamFactory(const std::shared_ptr<ReadStreamFactory>& factory) {-        inputFactories[factory->getName()] = factory;-    }--    /**-     * Return a new WriteStream-     */-    std::unique_ptr<WriteStream> getWriter(const std::map<std::string, std::string>& rwOperation,-            const SymbolTable& symbolTable, const RecordTable& recordTable) const {-        std::string ioType = rwOperation.at("IO");-        if (outputFactories.count(ioType) == 0) {-            throw std::invalid_argument("Requested output type <" + ioType + "> is not supported.");-        }-        return outputFactories.at(ioType)->getWriter(rwOperation, symbolTable, recordTable);-    }-    /**-     * Return a new ReadStream-     */-    std::unique_ptr<ReadStream> getReader(const std::map<std::string, std::string>& rwOperation,-            SymbolTable& symbolTable, RecordTable& recordTable) const {-        std::string ioType = rwOperation.at("IO");-        if (inputFactories.count(ioType) == 0) {-            throw std::invalid_argument("Requested input type <" + ioType + "> is not supported.");-        }-        return inputFactories.at(ioType)->getReader(rwOperation, symbolTable, recordTable);-    }-    ~IOSystem() = default;--private:-    IOSystem() {-        registerReadStreamFactory(std::make_shared<ReadFileCSVFactory>());-        registerReadStreamFactory(std::make_shared<ReadCinCSVFactory>());-        registerReadStreamFactory(std::make_shared<ReadFileJSONFactory>());-        registerReadStreamFactory(std::make_shared<ReadCinJSONFactory>());-        registerWriteStreamFactory(std::make_shared<WriteFileCSVFactory>());-        registerWriteStreamFactory(std::make_shared<WriteCoutCSVFactory>());-        registerWriteStreamFactory(std::make_shared<WriteCoutPrintSizeFactory>());-        registerWriteStreamFactory(std::make_shared<WriteFileJSONFactory>());-        registerWriteStreamFactory(std::make_shared<WriteCoutJSONFactory>());-#ifdef USE_SQLITE-        registerReadStreamFactory(std::make_shared<ReadSQLiteFactory>());-        registerWriteStreamFactory(std::make_shared<WriteSQLiteFactory>());-#endif-    };-    std::map<std::string, std::shared_ptr<WriteStreamFactory>> outputFactories;-    std::map<std::string, std::shared_ptr<ReadStreamFactory>> inputFactories;-};--} /* namespace souffle */
− cbits/souffle/LambdaBTree.h 
@@ -1,620 +0,0 @@-/*- * Souffle - A Datalog Compiler- * Copyright (c) 2018, Souffle Developers- * Licensed under the Universal Permissive License v 1.0 as shown at:- * - https://opensource.org/licenses/UPL- * - <souffle root>/licenses/SOUFFLE-UPL.txt- */--/************************************************************************- *- * @file LambdaBTree.h- *- * An implementation of a generic B-tree data structure including- * interfaces for utilizing instances as set or multiset containers.- * Allows the user to provide a function to execute on successful insert- * Be careful using this, it currently expects a pair as the key.- *- ***********************************************************************/--#pragma once--#include "BTree.h"-#include "utility/ContainerUtil.h"-#include "utility/ParallelUtil.h"-#include <atomic>-#include <cassert>-#include <typeinfo>-#include <vector>--namespace souffle {--namespace detail {-/**- * The actual implementation of a b-tree data structure.- *- * @tparam Key             .. the element type to be stored in this tree- * @tparam Comparator     .. a class defining an order on the stored elements- * @tparam Allocator     .. utilized for allocating memory for required nodes- * @tparam blockSize    .. determines the number of bytes/block utilized by leaf nodes- * @tparam SearchStrategy .. enables switching between linear, binary or any other search strategy- * @tparam isSet        .. true = set, false = multiset- * @tparam Functor      .. a std::function that is called on successful (new) insert- */-template <typename Key, typename Comparator,-        typename Allocator,  // is ignored so far - TODO: add support-        unsigned blockSize, typename SearchStrategy, bool isSet, typename Functor,-        typename WeakComparator = Comparator, typename Updater = detail::updater<Key>>-class LambdaBTree : public btree<Key, Comparator, Allocator, blockSize, SearchStrategy, isSet, WeakComparator,-                            Updater> {-public:-    using parenttype =-            btree<Key, Comparator, Allocator, blockSize, SearchStrategy, isSet, WeakComparator, Updater>;--    LambdaBTree(const Comparator& comp = Comparator(), const WeakComparator& weak_comp = WeakComparator())-            : parenttype(comp, weak_comp) {}--    /**-     * Inserts the given key into this tree.-     */-    typename Functor::result_type insert(Key& k, const Functor& f) {-        typename parenttype::operation_hints hints;-        return insert(k, hints, f);-    }--    // rewriting this because of david's changes-    typename Functor::result_type insert(-            Key& k, typename parenttype::operation_hints& hints, const Functor& f) {-#ifdef IS_PARALLEL--        // special handling for inserting first element-        while (this->root == nullptr) {-            // try obtaining root-lock-            if (!this->root_lock.try_start_write()) {-                // somebody else was faster => re-check-                continue;-            }--            // check loop condition again-            if (this->root != nullptr) {-                // somebody else was faster => normal insert-                this->root_lock.abort_write();-                break;-            }--            // create new node-            this->leftmost = new typename parenttype::leaf_node();-            this->leftmost->numElements = 1;-            // call the functor as we've successfully inserted-            typename Functor::result_type res = f(k);--            this->leftmost->keys[0] = k;-            this->root = this->leftmost;--            // operation complete => we can release the root lock-            this->root_lock.end_write();--            hints.last_insert.access(this->leftmost);--            return res;-        }--        // insert using iterative implementation--        typename parenttype::node* cur = nullptr;--        // test last insert hints-        typename parenttype::lock_type::Lease cur_lease;--        auto checkHint = [&](typename parenttype::node* last_insert) {-            // ignore null pointer-            if (!last_insert) return false;-            // get a read lease on indicated node-            auto hint_lease = last_insert->lock.start_read();-            // check whether it covers the key-            if (!this->weak_covers(last_insert, k)) return false;-            // and if there was no concurrent modification-            if (!last_insert->lock.validate(hint_lease)) return false;-            // use hinted location-            cur = last_insert;-            // and keep lease-            cur_lease = hint_lease;-            // we found a hit-            return true;-        };--        if (hints.last_insert.any(checkHint)) {-            // register this as a hit-            this->hint_stats.inserts.addHit();-        } else {-            // register this as a miss-            this->hint_stats.inserts.addMiss();-        }--        // if there is no valid hint ..-        if (!cur) {-            do {-                // get root - access lock-                auto root_lease = this->root_lock.start_read();--                // start with root-                cur = this->root;--                // get lease of the next node to be accessed-                cur_lease = cur->lock.start_read();--                // check validity of root pointer-                if (this->root_lock.end_read(root_lease)) {-                    break;-                }--            } while (true);-        }--        while (true) {-            // handle inner nodes-            if (cur->inner) {-                auto a = &(cur->keys[0]);-                auto b = &(cur->keys[cur->numElements]);--                auto pos = this->search.lower_bound(k, a, b, this->weak_comp);-                auto idx = pos - a;--                // early exit for sets-                if (isSet && pos != b && this->weak_equal(*pos, k)) {-                    // validate results-                    if (!cur->lock.validate(cur_lease)) {-                        // start over again-                        return insert(k, hints, f);-                    }--                    // update provenance information-                    if (typeid(Comparator) != typeid(WeakComparator) && this->less(k, *pos)) {-                        if (!cur->lock.try_upgrade_to_write(cur_lease)) {-                            // start again-                            return insert(k, hints, f);-                        }-                        this->update(*pos, k);--                        // get result before releasing lock-                        auto res = (*pos).second;--                        cur->lock.end_write();-                        return res;-                    }--                    // get the result before releasing lock-                    auto res = (*pos).second;--                    // check validity-                    if (!cur->lock.validate(cur_lease)) {-                        // start over again-                        return insert(k, hints, f);-                    }--                    // we found the element => return the result-                    return res;-                }--                // get next pointer-                auto next = cur->getChild(idx);--                // get lease on next level-                auto next_lease = next->lock.start_read();--                // check whether there was a write-                if (!cur->lock.end_read(cur_lease)) {-                    // start over-                    return insert(k, hints, f);-                }--                // go to next-                cur = next;--                // move on lease-                cur_lease = next_lease;--                continue;-            }--            // the rest is for leaf nodes-            assert(!cur->inner);--            // -- insert node in leaf node ----            auto a = &(cur->keys[0]);-            auto b = &(cur->keys[cur->numElements]);--            auto pos = this->search.upper_bound(k, a, b, this->weak_comp);-            auto idx = pos - a;--            // early exit for sets-            if (isSet && pos != a && this->weak_equal(*(pos - 1), k)) {-                // validate result-                if (!cur->lock.validate(cur_lease)) {-                    // start over again-                    return insert(k, hints, f);-                }--                // TODO (pnappa): remove provenance from LambdaBTree - no use for it-                // update provenance information-                if (typeid(Comparator) != typeid(WeakComparator) && this->less(k, *(pos - 1))) {-                    if (!cur->lock.try_upgrade_to_write(cur_lease)) {-                        // start again-                        return insert(k, hints, f);-                    }-                    this->update(*(pos - 1), k);--                    // retrieve result before releasing lock-                    auto res = (*(pos - 1)).second;--                    cur->lock.end_write();-                    return res;-                }--                // read result (atomic) -- just as a proof of concept, this is actually not valid!!-                std::atomic<typename Functor::result_type>& loc =-                        *reinterpret_cast<std::atomic<typename Functor::result_type>*>(&(*(pos - 1)).second);-                auto res = loc.load(std::memory_order_relaxed);--                // check validity-                if (!cur->lock.validate(cur_lease)) {-                    // start over again-                    return insert(k, hints, f);-                }--                // we found the element => done-                return res;-            }--            // upgrade to write-permission-            if (!cur->lock.try_upgrade_to_write(cur_lease)) {-                // something has changed => restart-                hints.last_insert.access(cur);-                return insert(k, hints, f);-            }--            if (cur->numElements >= parenttype::node::maxKeys) {-                // -- lock parents ---                auto priv = cur;-                auto parent = priv->parent;-                std::vector<typename parenttype::node*> parents;-                do {-                    if (parent) {-                        parent->lock.start_write();-                        while (true) {-                            // check whether parent is correct-                            if (parent == priv->parent) {-                                break;-                            }-                            // switch parent-                            parent->lock.abort_write();-                            parent = priv->parent;-                            parent->lock.start_write();-                        }-                    } else {-                        // lock root lock => since cur is root-                        this->root_lock.start_write();-                    }--                    // record locked node-                    parents.push_back(parent);--                    // stop at "sphere of influence"-                    if (!parent || !parent->isFull()) {-                        break;-                    }--                    // go one step higher-                    priv = parent;-                    parent = parent->parent;--                } while (true);--                // split this node-                auto old_root = this->root;-                idx -= cur->rebalance_or_split(-                        const_cast<typename parenttype::node**>(&this->root), this->root_lock, idx, parents);--                // release parent lock-                for (auto it = parents.rbegin(); it != parents.rend(); ++it) {-                    auto parent = *it;--                    // release this lock-                    if (parent) {-                        parent->lock.end_write();-                    } else {-                        if (old_root != this->root) {-                            this->root_lock.end_write();-                        } else {-                            this->root_lock.abort_write();-                        }-                    }-                }--                // insert element in right fragment-                if (((typename parenttype::size_type)idx) > cur->numElements) {-                    // release current lock-                    cur->lock.end_write();--                    // insert in sibling-                    return insert(k, hints, f);-                }-            }--            // ok - no split necessary-            assert(cur->numElements < parenttype::node::maxKeys && "Split required!");--            // move keys-            for (int j = cur->numElements; j > idx; --j) {-                cur->keys[j] = cur->keys[j - 1];-            }--            // insert new element-            typename Functor::result_type res = f(k);-            cur->keys[idx] = k;-            cur->numElements++;--            // release lock on current node-            cur->lock.end_write();--            // remember last insertion position-            hints.last_insert.access(cur);-            return res;-        }--#else-        // special handling for inserting first element-        if (this->empty()) {-            // create new node-            this->leftmost = new typename parenttype::leaf_node();-            this->leftmost->numElements = 1;-            // call the functor as we've successfully inserted-            typename Functor::result_type res = f(k);-            this->leftmost->keys[0] = k;-            this->root = this->leftmost;--            hints.last_insert.access(this->leftmost);--            return res;-        }--        // insert using iterative implementation-        typename parenttype::node* cur = this->root;--        auto checkHints = [&](typename parenttype::node* last_insert) {-            if (!last_insert) return false;-            if (!this->weak_covers(last_insert, k)) return false;-            cur = last_insert;-            return true;-        };--        // test last insert-        if (hints.last_insert.any(checkHints)) {-            this->hint_stats.inserts.addHit();-        } else {-            this->hint_stats.inserts.addMiss();-        }--        while (true) {-            // handle inner nodes-            if (cur->inner) {-                auto a = &(cur->keys[0]);-                auto b = &(cur->keys[cur->numElements]);--                auto pos = this->search.lower_bound(k, a, b, this->weak_comp);-                auto idx = pos - a;--                // early exit for sets-                if (isSet && pos != b && this->weak_equal(*pos, k)) {-                    // update provenance information-                    if (typeid(Comparator) != typeid(WeakComparator) && this->less(k, *pos)) {-                        this->update(*pos, k);-                        return (*pos).second;-                    }--                    return (*pos).second;-                }--                cur = cur->getChild(idx);-                continue;-            }--            // the rest is for leaf nodes-            assert(!cur->inner);--            // -- insert node in leaf node ----            auto a = &(cur->keys[0]);-            auto b = &(cur->keys[cur->numElements]);--            auto pos = this->search.upper_bound(k, a, b, this->weak_comp);-            auto idx = pos - a;--            // early exit for sets-            if (isSet && pos != a && this->weak_equal(*(pos - 1), k)) {-                // update provenance information-                if (typeid(Comparator) != typeid(WeakComparator) && this->less(k, *(pos - 1))) {-                    this->update(*(pos - 1), k);-                    return (*(pos - 1)).second;-                }--                return (*(pos - 1)).second;-            }--            if (cur->numElements >= parenttype::node::maxKeys) {-                // split this node-                idx -= cur->rebalance_or_split(-                        const_cast<typename parenttype::node**>(&this->root), this->root_lock, idx);--                // insert element in right fragment-                if (((typename parenttype::size_type)idx) > cur->numElements) {-                    idx -= cur->numElements + 1;-                    cur = cur->parent->getChild(cur->position + 1);-                }-            }--            // ok - no split necessary-            assert(cur->numElements < parenttype::node::maxKeys && "Split required!");--            // move keys-            for (int j = cur->numElements; j > idx; --j) {-                cur->keys[j] = cur->keys[j - 1];-            }--            // call the functor as we've successfully inserted-            typename Functor::result_type res = f(k);-            // insert new element-            cur->keys[idx] = k;-            cur->numElements++;--            // remember last insertion position-            hints.last_insert.access(cur);-            return res;-        }-#endif-    }--    /**-     * Inserts the given range of elements into this tree.-     */-    template <typename Iter>-    void insert(const Iter& a, const Iter& b) {-        // TODO: improve this beyond a naive insert-        typename parenttype::operation_hints hints;-        // a naive insert so far .. seems to work fine-        for (auto it = a; it != b; ++it) {-            // use insert with hint-            insert(*it, hints);-        }-    }--    /**-     * Swaps the content of this tree with the given tree. This-     * is a much more efficient operation than creating a copy and-     * realizing the swap utilizing assignment operations.-     */-    void swap(LambdaBTree& other) {-        // swap the content-        std::swap(this->root, other.root);-        std::swap(this->leftmost, other.leftmost);-    }--    // Implementation of the assignment operation for trees.-    LambdaBTree& operator=(const LambdaBTree& other) {-        // check identity-        if (this == &other) {-            return *this;-        }--        // create a deep-copy of the content of the other tree-        // shortcut for empty sets-        if (other.empty()) {-            return *this;-        }--        // clone content (deep copy)-        this->root = other.root->clone();--        // update leftmost reference-        auto tmp = this->root;-        while (!tmp->isLeaf()) {-            tmp = tmp->getChild(0);-        }-        this->leftmost = static_cast<typename parenttype::leaf_node*>(tmp);--        // done-        return *this;-    }--    // Implementation of an equality operation for trees.-    bool operator==(const LambdaBTree& other) const {-        // check identity-        if (this == &other) {-            return true;-        }--        // check size-        if (this->size() != other.size()) {-            return false;-        }-        if (this->size() < other.size()) {-            return other == *this;-        }--        // check content-        for (const auto& key : other) {-            if (!contains(key)) {-                return false;-            }-        }-        return true;-    }--    // Implementation of an inequality operation for trees.-    bool operator!=(const LambdaBTree& other) const {-        return !(*this == other);-    }-};--}  // end namespace detail--/**- * A b-tree based set implementation.- *- * @tparam Key             .. the element type to be stored in this set- * @tparam Functor         .. a std::function that is invoked on successful insert- * @tparam Comparator     .. a class defining an order on the stored elements- * @tparam Allocator     .. utilized for allocating memory for required nodes- * @tparam blockSize    .. determines the number of bytes/block utilized by leaf nodes- * @tparam SearchStrategy .. enables switching between linear, binary or any other search strategy- */-template <typename Key, typename Functor, typename Comparator = detail::comparator<Key>,-        typename Allocator = std::allocator<Key>,  // is ignored so far-        unsigned blockSize = 256, typename SearchStrategy = typename detail::default_strategy<Key>::type>-class LambdaBTreeSet-        : public detail::LambdaBTree<Key, Comparator, Allocator, blockSize, SearchStrategy, true, Functor> {-    using super = detail::LambdaBTree<Key, Comparator, Allocator, blockSize, SearchStrategy, true, Functor>;--    friend class detail::LambdaBTree<Key, Comparator, Allocator, blockSize, SearchStrategy, true, Functor>;--public:-    /**-     * A default constructor creating an empty set.-     */-    LambdaBTreeSet(const Comparator& comp = Comparator()) : super(comp) {}--    /**-     * A constructor creating a set based on the given range.-     */-    template <typename Iter>-    LambdaBTreeSet(const Iter& a, const Iter& b) {-        this->insert(a, b);-    }--    // A copy constructor.-    LambdaBTreeSet(const LambdaBTreeSet& other) : super(other) {}--    // A move constructor.-    LambdaBTreeSet(LambdaBTreeSet&& other) : super(std::move(other)) {}--private:-    // A constructor required by the bulk-load facility.-    template <typename s, typename n, typename l>-    LambdaBTreeSet(s size, n* root, l* leftmost) : super::parenttype(size, root, leftmost) {}--public:-    // Support for the assignment operator.-    LambdaBTreeSet& operator=(const LambdaBTreeSet& other) {-        super::operator=(other);-        return *this;-    }--    // Support for the bulk-load operator.-    template <typename Iter>-    static LambdaBTreeSet load(const Iter& a, const Iter& b) {-        return super::template load<LambdaBTreeSet>(a, b);-    }-};--}  // end of namespace souffle
− cbits/souffle/Logger.h 
@@ -1,67 +0,0 @@-/*- * Souffle - A Datalog Compiler- * Copyright (c) 2013, 2015, Oracle and/or its affiliates. All rights reserved- * Licensed under the Universal Permissive License v 1.0 as shown at:- * - https://opensource.org/licenses/UPL- * - <souffle root>/licenses/SOUFFLE-UPL.txt- */--/************************************************************************- *- * @file Logger.h- *- * A logger is the utility utilized by RAM programs to create logs and- * traces.- *- ***********************************************************************/--#pragma once--#include "ProfileEvent.h"-#include "utility/MiscUtil.h"-#include <cstddef>-#include <functional>-#include <string>-#include <utility>-#include <sys/resource.h>--namespace souffle {--/**- * The class utilized to times for the souffle profiling tool. This class- * is utilized by both -- the interpreted and compiled version -- to conduct- * the corresponding measurements.- *- * To far, only execution times are logged. More events, e.g. the number of- * processed tuples may be added in the future.- */-class Logger {-public:-    Logger(std::string label, size_t iteration) : Logger(label, iteration, []() { return 0; }) {}--    Logger(std::string label, size_t iteration, std::function<size_t()> size)-            : label(std::move(label)), start(now()), iteration(iteration), size(size), preSize(size()) {-        struct rusage ru {};-        getrusage(RUSAGE_SELF, &ru);-        startMaxRSS = ru.ru_maxrss;-        // Assume that if we are logging the progress of an event then we care about usage during that time.-        ProfileEventSingleton::instance().resetTimerInterval();-    }--    ~Logger() {-        struct rusage ru {};-        getrusage(RUSAGE_SELF, &ru);-        size_t endMaxRSS = ru.ru_maxrss;-        ProfileEventSingleton::instance().makeTimingEvent(-                label, start, now(), startMaxRSS, endMaxRSS, size() - preSize, iteration);-    }--private:-    std::string label;-    time_point start;-    size_t startMaxRSS;-    size_t iteration;-    std::function<size_t()> size;-    size_t preSize;-};-}  // end of namespace souffle
− cbits/souffle/PiggyList.h 
@@ -1,327 +0,0 @@-#pragma once--#include "utility/ParallelUtil.h"-#include <array>-#include <atomic>-#include <cstring>-#include <iostream>-#include <iterator>--#ifdef _WIN32-/**- * MSVC does not provide a builtin for counting leading zeroes like gcc,- * so we have to implement it ourselves.- */-unsigned long __inline __builtin_clzll(unsigned long long value) {-    unsigned long msb = 0;--    if (_BitScanReverse64(&msb, value))-        return 63 - msb;-    else-        return 64;-}-#endif  // _WIN32--using std::size_t;-namespace souffle {--/**- * A PiggyList that allows insertAt functionality.- * This means we can't append, as we don't know the next available element.- * insertAt is dangerous. You must be careful not to call it for the same index twice!- */-template <class T>-class RandomInsertPiggyList {-public:-    RandomInsertPiggyList() = default;-    // an instance where the initial size is not 65k, and instead is user settable (to a power of-    // initialbitsize)-    RandomInsertPiggyList(size_t initialbitsize) : BLOCKBITS(initialbitsize) {}--    /** copy constructor */-    RandomInsertPiggyList(const RandomInsertPiggyList& other) : BLOCKBITS(other.BLOCKBITS) {-        this->numElements.store(other.numElements.load());--        // copy blocks from the old lookup table to this one-        for (size_t i = 0; i < maxContainers; ++i) {-            if (other.blockLookupTable[i].load() != nullptr) {-                // calculate the size of that block-                const size_t blockSize = INITIALBLOCKSIZE << i;--                // allocate that in the new container-                this->blockLookupTable[i].store(new T[blockSize]);--                // then copy the stuff over-                std::memcpy(this->blockLookupTable[i].load(), other.blockLookupTable[i].load(),-                        blockSize * sizeof(T));-            }-        }-    }--    // move ctr-    RandomInsertPiggyList(RandomInsertPiggyList&& other) = delete;-    // copy assign ctor-    RandomInsertPiggyList& operator=(RandomInsertPiggyList& other) = delete;-    // move assign ctor-    RandomInsertPiggyList& operator=(RandomInsertPiggyList&& other) = delete;--    ~RandomInsertPiggyList() {-        freeList();-    }--    inline size_t size() const {-        return numElements.load();-    }--    inline T* getBlock(size_t blockNum) const {-        return blockLookupTable[blockNum];-    }--    inline T& get(size_t index) const {-        size_t nindex = index + INITIALBLOCKSIZE;-        size_t blockNum = (63 - __builtin_clzll(nindex));-        size_t blockInd = (nindex) & ((1 << blockNum) - 1);-        return this->getBlock(blockNum - BLOCKBITS)[blockInd];-    }--    void insertAt(size_t index, T value) {-        // starting with an initial blocksize requires some shifting to transform into a nice powers of two-        // series-        size_t blockNum = (63 - __builtin_clzll(index + INITIALBLOCKSIZE)) - BLOCKBITS;--        // allocate the block if not allocated-        if (blockLookupTable[blockNum].load() == nullptr) {-            slock.lock();-            if (blockLookupTable[blockNum].load() == nullptr) {-                blockLookupTable[blockNum].store(new T[INITIALBLOCKSIZE << blockNum]);-            }-            slock.unlock();-        }--        this->get(index) = value;-        // we ALWAYS increment size, even if there was something there before (its impossible to tell!)-        // the onus is up to the user to not call this for an index twice-        ++numElements;-    }--    void clear() {-        freeList();-        numElements.store(0);-    }-    const size_t BLOCKBITS = 16ul;-    const size_t INITIALBLOCKSIZE = (1ul << BLOCKBITS);--    // number of elements currently stored within-    std::atomic<size_t> numElements{0};--    // 2^64 - 1 elements can be stored (default initialised to nullptrs)-    static constexpr size_t maxContainers = 64;-    std::array<std::atomic<T*>, maxContainers> blockLookupTable = {};--    // for parallel node insertions-    mutable SpinLock slock;--    /**-     * Free the arrays allocated within the linked list nodes-     */-    void freeList() {-        slock.lock();-        // delete all - deleting a nullptr is a no-op-        for (size_t i = 0; i < maxContainers; ++i) {-            delete[] blockLookupTable[i].load();-            // reset the container within to be empty.-            blockLookupTable[i].store(nullptr);-        }-        slock.unlock();-    }-};--template <class T>-class PiggyList {-public:-    PiggyList() : num_containers(0), container_size(0), m_size(0) {}-    PiggyList(size_t initialbitsize)-            : BLOCKBITS(initialbitsize), num_containers(0), container_size(0), m_size(0) {}--    /** copy constructor */-    PiggyList(const PiggyList& other) : BLOCKBITS(other.BLOCKBITS) {-        num_containers.store(other.num_containers.load());-        container_size.store(other.container_size.load());-        m_size.store(other.m_size.load());-        // copy each chunk from other into this-        // the size of the next container to allocate-        size_t cSize = BLOCKSIZE;-        for (size_t i = 0; i < other.num_containers; ++i) {-            this->blockLookupTable[i] = new T[cSize];-            std::memcpy(this->blockLookupTable[i], other.blockLookupTable[i], cSize * sizeof(T));-            cSize <<= 1;-        }-        // if this isn't the case, uhh-        assert((cSize >> 1) == container_size.load());-    }--    /** move constructor */-    PiggyList(PiggyList&& other) = delete;-    /** copy assign ctor **/-    PiggyList& operator=(const PiggyList& other) = delete;--    ~PiggyList() {-        freeList();-    }--    /**-     * Well, returns the number of nodes exist within the list + number of nodes queued to be inserted-     *  The reason for this, is that there may be many nodes queued up-     *  that haven't had time to had containers created and updated-     * @return the number of nodes exist within the list + number of nodes queued to be inserted-     */-    inline size_t size() const {-        return m_size.load();-    };--    inline T* getBlock(size_t blocknum) const {-        return this->blockLookupTable[blocknum];-    }--    size_t append(T element) {-        size_t new_index = m_size.fetch_add(1, std::memory_order_acquire);--        // will this not fit?-        if (container_size < new_index + 1) {-            sl.lock();-            // check and add as many containers as required-            while (container_size < new_index + 1) {-                blockLookupTable[num_containers] = new T[allocsize];-                num_containers += 1;-                container_size += allocsize;-                // double the number elements that will be allocated next time-                allocsize <<= 1;-            }-            sl.unlock();-        }--        this->get(new_index) = element;-        return new_index;-    }--    size_t createNode() {-        size_t new_index = m_size.fetch_add(1, std::memory_order_acquire);--        // will this not fit?-        if (container_size < new_index + 1) {-            sl.lock();-            // check and add as many containers as required-            while (container_size < new_index + 1) {-                blockLookupTable[num_containers] = new T[allocsize];-                num_containers += 1;-                container_size += allocsize;-                // double the number elements that will be allocated next time-                allocsize <<= 1;-            }-            sl.unlock();-        }--        return new_index;-    }--    /**-     * Retrieve a reference to the stored value at index-     * @param index position to search-     * @return the value at index-     */-    inline T& get(size_t index) const {-        // supa fast 2^16 size first block-        size_t nindex = index + BLOCKSIZE;-        size_t blockNum = (63 - __builtin_clzll(nindex));-        size_t blockInd = (nindex) & ((1 << blockNum) - 1);-        return this->getBlock(blockNum - BLOCKBITS)[blockInd];-    }--    /**-     * Clear all elements from the PiggyList-     */-    void clear() {-        freeList();-        m_size = 0;-        num_containers = 0;--        allocsize = BLOCKSIZE;-        container_size = 0;-    }--    class iterator : std::iterator<std::forward_iterator_tag, T> {-        size_t cIndex = 0;-        PiggyList* bl;--    public:-        // default ctor, to silence-        iterator() = default;--        /* begin iterator for iterating over all elements */-        iterator(PiggyList* bl) : bl(bl){};-        /* ender iterator for marking the end of the iteration */-        iterator(PiggyList* bl, size_t beginInd) : cIndex(beginInd), bl(bl){};--        T operator*() {-            return bl->get(cIndex);-        };-        const T operator*() const {-            return bl->get(cIndex);-        };--        iterator& operator++(int) {-            ++cIndex;-            return *this;-        };--        iterator operator++() {-            iterator ret(*this);-            ++cIndex;-            return ret;-        };--        bool operator==(const iterator& x) const {-            return x.cIndex == this->cIndex && x.bl == this->bl;-        };--        bool operator!=(const iterator& x) const {-            return !(x == *this);-        };-    };--    iterator begin() {-        return iterator(this);-    }-    iterator end() {-        return iterator(this, size());-    }-    const size_t BLOCKBITS = 16ul;-    const size_t BLOCKSIZE = (1ul << BLOCKBITS);--    // number of inserted-    std::atomic<size_t> num_containers = 0;-    size_t allocsize = BLOCKSIZE;-    std::atomic<size_t> container_size = 0;-    std::atomic<size_t> m_size = 0;--    // > 2^64 elements can be stored (default initialise to nullptrs)-    static constexpr size_t max_conts = 64;-    std::array<T*, max_conts> blockLookupTable = {};--    // for parallel node insertions-    mutable SpinLock sl;--    /**-     * Free the arrays allocated within the linked list nodes-     */-    void freeList() {-        sl.lock();-        // we don't know which ones are taken up!-        for (size_t i = 0; i < num_containers; ++i) {-            delete[] blockLookupTable[i];-        }-        sl.unlock();-    }-};--}  // namespace souffle
− cbits/souffle/ProfileDatabase.h 
@@ -1,465 +0,0 @@-#pragma once--#include "json11.h"-#include "utility/MiscUtil.h"-#include <cassert>-#include <chrono>-#include <cstddef>-#include <fstream>-#include <iostream>-#include <iterator>-#include <map>-#include <memory>-#include <mutex>-#include <set>-#include <stdexcept>-#include <string>-#include <utility>-#include <vector>--namespace souffle {-namespace profile {--class DirectoryEntry;-class DurationEntry;-class SizeEntry;-class TextEntry;-class TimeEntry;--/**- * Visitor Interface- */-class Visitor {-public:-    virtual ~Visitor() = default;--    // visit entries in a directory-    virtual void visit(DirectoryEntry& e);--    // visit entries-    virtual void visit(DurationEntry&) {}-    virtual void visit(SizeEntry&) {}-    virtual void visit(TextEntry&) {}-    virtual void visit(TimeEntry&) {}-};--/**- * Entry class- *- * abstract class for a key/value entry in a hierarchical database- */-class Entry {-private:-    // entry key-    std::string key;--public:-    Entry(std::string key) : key(std::move(key)) {}-    virtual ~Entry() = default;--    // get key-    const std::string& getKey() const {-        return key;-    };--    // accept visitor-    virtual void accept(Visitor& v) = 0;--    // print-    virtual void print(std::ostream& os, int tabpos) const = 0;-};--/**- * DirectoryEntry entry- */-class DirectoryEntry : public Entry {-private:-    std::map<std::string, std::unique_ptr<Entry>> entries;-    mutable std::mutex lock;--public:-    DirectoryEntry(const std::string& name) : Entry(name) {}--    // get keys-    const std::set<std::string> getKeys() const {-        std::set<std::string> result;-        std::lock_guard<std::mutex> guard(lock);-        for (auto const& cur : entries) {-            result.insert(cur.first);-        }-        return result;-    }--    // write entry-    Entry* writeEntry(std::unique_ptr<Entry> entry) {-        assert(entry != nullptr && "null entry");-        std::lock_guard<std::mutex> guard(lock);-        const std::string& key = entry->getKey();-        // Don't rewrite an existing entry-        if (entries.count(key) == 0) {-            entries[key] = std::move(entry);-        }-        return entries[key].get();-    }--    // read entry-    Entry* readEntry(const std::string& key) const {-        std::lock_guard<std::mutex> guard(lock);-        auto it = entries.find(key);-        if (it != entries.end()) {-            return (*it).second.get();-        } else {-            return nullptr;-        }-    }--    // read directory-    DirectoryEntry* readDirectoryEntry(const std::string& key) const {-        return dynamic_cast<DirectoryEntry*>(readEntry(key));-    }--    // accept visitor-    void accept(Visitor& v) override {-        v.visit(*this);-    }--    // print directory-    void print(std::ostream& os, int tabpos) const override {-        os << std::string(tabpos, ' ') << '"' << getKey() << "\": {" << std::endl;-        bool first{true};-        for (auto const& cur : entries) {-            if (!first) {-                os << ',' << std::endl;-            } else {-                first = false;-            }-            cur.second->print(os, tabpos + 1);-        }-        os << std::endl << std::string(tabpos, ' ') << '}';-    }-};--/**- * SizeEntry- */-class SizeEntry : public Entry {-private:-    size_t size;  // size-public:-    SizeEntry(const std::string& key, size_t size) : Entry(key), size(size) {}--    // get size-    size_t getSize() const {-        return size;-    }--    // accept visitor-    void accept(Visitor& v) override {-        v.visit(*this);-    }--    // print entry-    void print(std::ostream& os, int tabpos) const override {-        os << std::string(tabpos, ' ') << "\"" << getKey() << "\": " << size;-    }-};--/**- * TextEntry- */-class TextEntry : public Entry {-private:-    // entry text-    std::string text;--public:-    TextEntry(const std::string& key, std::string text) : Entry(key), text(std::move(text)) {}--    // get text-    const std::string& getText() const {-        return text;-    }--    // accept visitor-    void accept(Visitor& v) override {-        v.visit(*this);-    }--    // write size entry-    void print(std::ostream& os, int tabpos) const override {-        os << std::string(tabpos, ' ') << "\"" << getKey() << "\": \"" << text << "\"";-    }-};--/**- * Duration Entry- */-class DurationEntry : public Entry {-private:-    // duration start-    microseconds start;--    // duration end-    microseconds end;--public:-    DurationEntry(const std::string& key, microseconds start, microseconds end)-            : Entry(key), start(start), end(end) {}--    // get start-    microseconds getStart() const {-        return start;-    }--    // get end-    microseconds getEnd() const {-        return end;-    }--    // accept visitor-    void accept(Visitor& v) override {-        v.visit(*this);-    }--    // write size entry-    void print(std::ostream& os, int tabpos) const override {-        os << std::string(tabpos, ' ') << '"' << getKey();-        os << R"_(": { "start": )_";-        os << start.count();-        os << ", \"end\": ";-        os << end.count();-        os << '}';-    }-};--/**- * Time Entry- */-class TimeEntry : public Entry {-private:-    // time since start-    microseconds time;--public:-    TimeEntry(const std::string& key, microseconds time) : Entry(key), time(time) {}--    // get start-    microseconds getTime() const {-        return time;-    }--    // accept visitor-    void accept(Visitor& v) override {-        v.visit(*this);-    }--    // write size entry-    void print(std::ostream& os, int tabpos) const override {-        os << std::string(tabpos, ' ') << '"' << getKey();-        os << R"_(": { "time": )_";-        os << time.count();-        os << '}';-    }-};--inline void Visitor::visit(DirectoryEntry& e) {-    std::cout << "Dir " << e.getKey() << "\n";-    for (const auto& cur : e.getKeys()) {-        std::cout << "\t :" << cur << "\n";-        e.readEntry(cur)->accept(*this);-    }-}--class Counter : public Visitor {-private:-    size_t ctr{0};-    std::string key;--public:-    Counter(std::string key) : key(std::move(key)) {}-    void visit(SizeEntry& e) override {-        std::cout << "Size entry : " << e.getKey() << " " << e.getSize() << "\n";-        if (e.getKey() == key) {-            ctr += e.getSize();-        }-    }-    size_t getCounter() const {-        return ctr;-    }-};--/**- * Hierarchical databas- */-class ProfileDatabase {-private:-    std::unique_ptr<DirectoryEntry> root;--protected:-    /**-     * Find path: if directories along the path do not exist, create them.-     */-    DirectoryEntry* lookupPath(const std::vector<std::string>& path) {-        DirectoryEntry* dir = root.get();-        for (const std::string& key : path) {-            assert(!key.empty() && "Key is empty!");-            DirectoryEntry* newDir = dir->readDirectoryEntry(key);-            if (newDir == nullptr) {-                newDir =-                        dynamic_cast<DirectoryEntry*>(dir->writeEntry(std::make_unique<DirectoryEntry>(key)));-            }-            assert(newDir != nullptr && "Attempting to overwrite an existing entry");-            dir = newDir;-        }-        return dir;-    }--    void parseJson(const json11::Json& json, std::unique_ptr<DirectoryEntry>& node) {-        for (auto& cur : json.object_items()) {-            if (cur.second.is_object()) {-                std::string err;-                // Duration entries are also maps-                if (cur.second.has_shape(-                            {{"start", json11::Json::NUMBER}, {"end", json11::Json::NUMBER}}, err)) {-                    auto start = std::chrono::microseconds(cur.second["start"].long_value());-                    auto end = std::chrono::microseconds(cur.second["end"].long_value());-                    node->writeEntry(std::make_unique<DurationEntry>(cur.first, start, end));-                } else if (cur.second.has_shape({{"time", json11::Json::NUMBER}}, err)) {-                    auto time = std::chrono::microseconds(cur.second["time"].long_value());-                    node->writeEntry(std::make_unique<TimeEntry>(cur.first, time));-                } else {-                    auto dir = std::make_unique<DirectoryEntry>(cur.first);-                    parseJson(cur.second, dir);-                    node->writeEntry(std::move(dir));-                }-            } else if (cur.second.is_string()) {-                node->writeEntry(std::make_unique<TextEntry>(cur.first, cur.second.string_value()));-            } else if (cur.second.is_number()) {-                node->writeEntry(std::make_unique<SizeEntry>(cur.first, cur.second.long_value()));-            } else {-                std::string err;-                cur.second.dump(err);-                std::cerr << "Unknown types in profile log: " << cur.first << ": " << err << std::endl;-            }-        }-    }--public:-    ProfileDatabase() : root(std::make_unique<DirectoryEntry>("root")) {}--    ProfileDatabase(const std::string& filename) : root(std::make_unique<DirectoryEntry>("root")) {-        std::ifstream file(filename);-        if (!file.is_open()) {-            throw std::runtime_error("Log file could not be opened.");-        }-        std::string jsonString((std::istreambuf_iterator<char>(file)), (std::istreambuf_iterator<char>()));-        std::string error;-        json11::Json json = json11::Json::parse(jsonString, error);-        if (!error.empty()) {-            throw std::runtime_error("Parse error: " + error);-        }-        parseJson(json["root"], root);-    }--    // add size entry-    void addSizeEntry(std::vector<std::string> qualifier, size_t size) {-        assert(qualifier.size() > 0 && "no qualifier");-        std::vector<std::string> path(qualifier.begin(), qualifier.end() - 1);-        DirectoryEntry* dir = lookupPath(path);--        const std::string& key = qualifier.back();-        std::unique_ptr<SizeEntry> entry = std::make_unique<SizeEntry>(key, size);-        dir->writeEntry(std::move(entry));-    }--    // add text entry-    void addTextEntry(std::vector<std::string> qualifier, const std::string& text) {-        assert(qualifier.size() > 0 && "no qualifier");-        std::vector<std::string> path(qualifier.begin(), qualifier.end() - 1);-        DirectoryEntry* dir = lookupPath(path);--        const std::string& key = qualifier.back();-        std::unique_ptr<TextEntry> entry = std::make_unique<TextEntry>(key, text);-        dir->writeEntry(std::move(entry));-    }--    // add duration entry-    void addDurationEntry(std::vector<std::string> qualifier, microseconds start, microseconds end) {-        assert(qualifier.size() > 0 && "no qualifier");-        std::vector<std::string> path(qualifier.begin(), qualifier.end() - 1);-        DirectoryEntry* dir = lookupPath(path);--        const std::string& key = qualifier.back();-        std::unique_ptr<DurationEntry> entry = std::make_unique<DurationEntry>(key, start, end);-        dir->writeEntry(std::move(entry));-    }--    // add time entry-    void addTimeEntry(std::vector<std::string> qualifier, microseconds time) {-        assert(qualifier.size() > 0 && "no qualifier");-        std::vector<std::string> path(qualifier.begin(), qualifier.end() - 1);-        DirectoryEntry* dir = lookupPath(path);--        const std::string& key = qualifier.back();-        std::unique_ptr<TimeEntry> entry = std::make_unique<TimeEntry>(key, time);-        dir->writeEntry(std::move(entry));-    }--    // compute sum-    size_t computeSum(const std::vector<std::string>& qualifier) {-        assert(qualifier.size() > 0 && "no qualifier");-        std::vector<std::string> path(qualifier.begin(), qualifier.end() - 1);-        DirectoryEntry* dir = lookupPath(path);--        const std::string& key = qualifier.back();-        std::cout << "Key: " << key << std::endl;-        Counter ctr(key);-        dir->accept(ctr);-        return ctr.getCounter();-    }--    /**-     * Return the entry at the given path.-     */-    Entry* lookupEntry(const std::vector<std::string>& path) const {-        DirectoryEntry* dir = root.get();-        auto last = --path.end();-        for (auto it = path.begin(); it != last; ++it) {-            dir = dir->readDirectoryEntry(*it);-            if (dir == nullptr) {-                return nullptr;-            }-        }-        return dir->readEntry(*last);-    }--    /**-     * Return a map of string keys to string values.-     */-    std::map<std::string, std::string> getStringMap(const std::vector<std::string>& path) const {-        std::map<std::string, std::string> kvps;-        auto* parent = dynamic_cast<DirectoryEntry*>(lookupEntry(path));-        if (parent == nullptr) {-            return kvps;-        }--        for (const auto& key : parent->getKeys()) {-            auto* text = dynamic_cast<TextEntry*>(parent->readEntry(key));-            if (text != nullptr) {-                kvps[key] = text->getText();-            }-        }--        return kvps;-    }--    // print database-    void print(std::ostream& os) const {-        os << '{' << std::endl;-        root->print(os, 1);-        os << std::endl << '}' << std::endl;-    };-};--}  // namespace profile-}  // namespace souffle
− cbits/souffle/ProfileEvent.h 
@@ -1,226 +0,0 @@-/*- * Souffle - A Datalog Compiler- * Copyright (c) 2018, The Souffle Developers. All rights reserved.- * Licensed under the Universal Permissive License v 1.0 as shown at:- * - https://opensource.org/licenses/UPL- * - <souffle root>/licenses/SOUFFLE-UPL.txt- */--/************************************************************************- *- * @file ProfileEvent.h- *- * Declares classes for profile events- *- ***********************************************************************/--#pragma once--#include "EventProcessor.h"-#include "ProfileDatabase.h"-#include "utility/MiscUtil.h"-#include <atomic>-#include <chrono>-#include <condition_variable>-#include <cstdint>-#include <ctime>-#include <iostream>-#include <mutex>-#include <sstream>-#include <string>-#include <thread>-#include <sys/resource.h>-#include <sys/time.h>--namespace souffle {--/**- * Profile Event Singleton- */-class ProfileEventSingleton {-    /** profile database */-    profile::ProfileDatabase database;-    std::string filename{""};--    ProfileEventSingleton() = default;--public:-    ~ProfileEventSingleton() {-        stopTimer();-        ProfileEventSingleton::instance().dump();-    }--    /** get instance */-    static ProfileEventSingleton& instance() {-        static ProfileEventSingleton singleton;-        return singleton;-    }--    /** create config record */-    void makeConfigRecord(const std::string& key, const std::string& value) {-        profile::EventProcessorSingleton::instance().process(database, "@config", key.c_str(), value.c_str());-    }--    /** create time event */-    void makeTimeEvent(const std::string& txt) {-        profile::EventProcessorSingleton::instance().process(-                database, txt.c_str(), std::chrono::duration_cast<microseconds>(now().time_since_epoch()));-    }--    /** create an event for recording start and end times */-    void makeTimingEvent(const std::string& txt, time_point start, time_point end, size_t startMaxRSS,-            size_t endMaxRSS, size_t size, size_t iteration) {-        microseconds start_ms = std::chrono::duration_cast<microseconds>(start.time_since_epoch());-        microseconds end_ms = std::chrono::duration_cast<microseconds>(end.time_since_epoch());-        profile::EventProcessorSingleton::instance().process(-                database, txt.c_str(), start_ms, end_ms, startMaxRSS, endMaxRSS, size, iteration);-    }--    /** create quantity event */-    void makeQuantityEvent(const std::string& txt, size_t number, int iteration) {-        profile::EventProcessorSingleton::instance().process(database, txt.c_str(), number, iteration);-    }--    /** create utilisation event */-    void makeUtilisationEvent(const std::string& txt) {-        /* current time */-        microseconds time = std::chrono::duration_cast<microseconds>(now().time_since_epoch());-        /* system CPU time used */-        struct rusage ru {};-        getrusage(RUSAGE_SELF, &ru);-        /* system CPU time used */-        uint64_t systemTime = ru.ru_stime.tv_sec * 1000000 + ru.ru_stime.tv_usec;-        /* user CPU time used */-        uint64_t userTime = ru.ru_utime.tv_sec * 1000000 + ru.ru_utime.tv_usec;-        /* Maximum resident set size (kb) */-        size_t maxRSS = ru.ru_maxrss;--        profile::EventProcessorSingleton::instance().process(-                database, txt.c_str(), time, systemTime, userTime, maxRSS);-    }--    void setOutputFile(std::string filename) {-        this->filename = filename;-    }-    /** Dump all events */-    void dump() {-        if (!filename.empty()) {-            std::ofstream os(filename);-            if (!os.is_open()) {-                std::cerr << "Cannot open profile log file <" + filename + ">";-            } else {-                database.print(os);-            }-        }-    }--    /** Start timer */-    void startTimer() {-        timer.start();-    }--    /** Stop timer */-    void stopTimer() {-        timer.stop();-    }--    void resetTimerInterval(uint32_t interval = 1) {-        timer.resetTimerInterval(interval);-    }-    const profile::ProfileDatabase& getDB() const {-        return database;-    }--    void setDBFromFile(const std::string& filename) {-        database = profile::ProfileDatabase(filename);-    }--private:-    /**  Profile Timer */-    class ProfileTimer {-    private:-        /** time interval between per utilisation read */-        uint32_t t;--        /** timer is running */-        std::atomic<bool> running{false};--        /** thread timer runs on */-        std::thread th;--        std::condition_variable conditionVariable;-        std::mutex timerMutex;--        /** number of utilisation events */-        std::atomic<uint32_t> runCount{0};--        /** run method for thread th */-        void run() {-            ProfileEventSingleton::instance().makeUtilisationEvent("@utilisation");-            ++runCount;-            if (runCount % 128 == 0) {-                increaseInterval();-            }-        }--        uint32_t getInterval() {-            return t;-        }--        /** Increase value of time interval by factor of 2 */-        void increaseInterval() {-            // Don't increase time interval past 60 seconds-            if (t < 60000) {-                t = t * 2;-            }-        }--    public:-        /*-         *  @param interval the size of the timing interval in milliseconds-         */-        ProfileTimer(uint32_t interval = 10) : t(interval) {}--        /** start timer on the thread th */-        void start() {-            if (running) {-                return;-            }-            running = true;--            th = std::thread([this]() {-                while (running) {-                    run();-                    std::unique_lock<std::mutex> lock(timerMutex);-                    conditionVariable.wait_for(lock, std::chrono::milliseconds(getInterval()));-                }-            });-        }--        /** stop timer on the thread th */-        void stop() {-            running = false;-            conditionVariable.notify_all();-            if (th.joinable()) {-                th.join();-            }-        }--        /** Reset timer interval.-         *-         *  The timer interval increases as the program executes. Resetting the interval is useful to-         *  ensure that detailed usage information is gathered even in long running programs, if desired.-         *-         *  @param interval the size of the timing interval in milliseconds-         */-        void resetTimerInterval(uint32_t interval = 10) {-            t = interval;-            runCount = 0;-            conditionVariable.notify_all();-        }-    };--    ProfileTimer timer;-};--}  // namespace souffle
cbits/souffle/RamTypes.h view
@@ -24,24 +24,7 @@  namespace souffle { -enum class TypeAttribute {-    Symbol,-    Signed,    // Signed number-    Unsigned,  // Unsigned number-    Float,     // Floating point number.-    Record,-};--// Printing of the TypeAttribute Enum.-// To be utilised in synthesizer.-std::ostream& operator<<(std::ostream& os, TypeAttribute T);- /**- * Check if type is numeric.- */-bool isNumericType(TypeAttribute ramType);--/**  * Types of elements in a tuple.  *  * Default domain has size of 32 bits; may be overridden by user@@ -105,6 +88,6 @@ constexpr RamUnsigned MIN_RAM_UNSIGNED = std::numeric_limits<RamUnsigned>::min(); constexpr RamUnsigned MAX_RAM_UNSIGNED = std::numeric_limits<RamUnsigned>::max(); -constexpr RamFloat MIN_RAM_FLOAT = std::numeric_limits<RamFloat>::min();+constexpr RamFloat MIN_RAM_FLOAT = std::numeric_limits<RamFloat>::lowest(); constexpr RamFloat MAX_RAM_FLOAT = std::numeric_limits<RamFloat>::max(); }  // end of namespace souffle
− cbits/souffle/ReadStream.h 
@@ -1,179 +0,0 @@-/*- * Souffle - A Datalog Compiler- * Copyright (c) 2013, 2014, Oracle and/or its affiliates. All rights reserved- * Licensed under the Universal Permissive License v 1.0 as shown at:- * - https://opensource.org/licenses/UPL- * - <souffle root>/licenses/SOUFFLE-UPL.txt- */--/************************************************************************- *- * @file ReadStream.h- *- ***********************************************************************/--#pragma once--#include "RamTypes.h"-#include "RecordTable.h"-#include "SerialisationStream.h"-#include "SymbolTable.h"-#include "json11.h"-#include "utility/MiscUtil.h"-#include "utility/StringUtil.h"-#include <cctype>-#include <cstddef>-#include <map>-#include <memory>-#include <ostream>-#include <stdexcept>-#include <string>-#include <vector>--namespace souffle {--class ReadStream : public SerialisationStream<false> {-protected:-    ReadStream(-            const std::map<std::string, std::string>& rwOperation, SymbolTable& symTab, RecordTable& recTab)-            : SerialisationStream(symTab, recTab, rwOperation) {}--public:-    template <typename T>-    void readAll(T& relation) {-        auto lease = symbolTable.acquireLock();-        (void)lease;-        while (const auto next = readNextTuple()) {-            const RamDomain* ramDomain = next.get();-            relation.insert(ramDomain);-        }-    }--protected:-    /**-     * Read a record from a string.-     *-     * @param source - string containing a record-     * @param recordTypeName - record type.-     * @parem pos - start parsing from this position.-     * @param consumed - if not nullptr: number of characters read.-     *-     */-    RamDomain readRecord(const std::string& source, const std::string& recordTypeName, size_t pos = 0,-            size_t* charactersRead = nullptr) {-        const size_t initial_position = pos;--        // Check if record type information are present-        auto&& recordInfo = types["records"][recordTypeName];-        if (recordInfo.is_null()) {-            throw std::invalid_argument("Missing record type information: " + recordTypeName);-        }--        // Handle nil case-        consumeWhiteSpace(source, pos);-        if (source.substr(pos, 3) == "nil") {-            if (charactersRead != nullptr) {-                *charactersRead = 3;-            }-            return 0;-        }--        auto&& recordTypes = recordInfo["types"];-        const size_t recordArity = recordInfo["arity"].long_value();--        std::vector<RamDomain> recordValues(recordArity);--        consumeChar(source, '[', pos);--        for (size_t i = 0; i < recordArity; ++i) {-            const std::string& recordType = recordTypes[i].string_value();-            size_t consumed = 0;--            if (i > 0) {-                consumeChar(source, ',', pos);-            }-            consumeWhiteSpace(source, pos);-            switch (recordType[0]) {-                case 's': {-                    recordValues[i] = readStringInRecord(source, pos, &consumed);-                    break;-                }-                case 'i': {-                    recordValues[i] = RamSignedFromString(source.substr(pos), &consumed);-                    break;-                }-                case 'u': {-                    recordValues[i] = ramBitCast(RamUnsignedFromString(source.substr(pos), &consumed));-                    break;-                }-                case 'f': {-                    recordValues[i] = ramBitCast(RamFloatFromString(source.substr(pos), &consumed));-                    break;-                }-                case 'r': {-                    recordValues[i] = readRecord(source, recordType, pos, &consumed);-                    break;-                }-                default: fatal("Invalid type attribute");-            }-            pos += consumed;-        }-        consumeChar(source, ']', pos);--        if (charactersRead != nullptr) {-            *charactersRead = pos - initial_position;-        }--        return recordTable.pack(recordValues.data(), recordValues.size());-    }--    RamDomain readStringInRecord(const std::string& source, const size_t pos, size_t* charactersRead) {-        size_t endOfSymbol = source.find_first_of(",]", pos);--        if (endOfSymbol == std::string::npos) {-            throw std::invalid_argument("Unexpected end of input in record");-        }--        *charactersRead = endOfSymbol - pos;-        std::string str = source.substr(pos, *charactersRead);--        return symbolTable.unsafeLookup(str);-    }--    /**-     * Read past given character, consuming any preceding whitespace.-     */-    void consumeChar(const std::string& str, char c, size_t& pos) {-        consumeWhiteSpace(str, pos);-        if (pos >= str.length()) {-            throw std::invalid_argument("Unexpected end of input in record");-        }-        if (str[pos] != c) {-            std::stringstream error;-            error << "Expected: \'" << c << "\', got: " << str[pos];-            throw std::invalid_argument(error.str());-        }-        ++pos;-    }--    /**-     * Advance position in the string until first non-whitespace character.-     */-    void consumeWhiteSpace(const std::string& str, size_t& pos) {-        while (pos < str.length() && std::isspace(static_cast<unsigned char>(str[pos]))) {-            ++pos;-        }-    }--    virtual std::unique_ptr<RamDomain[]> readNextTuple() = 0;-};--class ReadStreamFactory {-public:-    virtual std::unique_ptr<ReadStream> getReader(-            const std::map<std::string, std::string>&, SymbolTable&, RecordTable&) = 0;-    virtual const std::string& getName() const = 0;-    virtual ~ReadStreamFactory() = default;-};--} /* namespace souffle */
− cbits/souffle/ReadStreamCSV.h 
@@ -1,328 +0,0 @@-/*- * Souffle - A Datalog Compiler- * Copyright (c) 2013, 2014, Oracle and/or its affiliates. All rights reserved- * Licensed under the Universal Permissive License v 1.0 as shown at:- * - https://opensource.org/licenses/UPL- * - <souffle root>/licenses/SOUFFLE-UPL.txt- */--/************************************************************************- *- * @file ReadStreamCSV.h- *- ***********************************************************************/--#pragma once--#include "RamTypes.h"-#include "ReadStream.h"-#include "SymbolTable.h"-#include "utility/ContainerUtil.h"-#include "utility/FileUtil.h"-#include "utility/StringUtil.h"--#ifdef USE_LIBZ-#include "gzfstream.h"-#else-#include <fstream>-#endif--#include <algorithm>-#include <cassert>-#include <cstddef>-#include <cstdint>-#include <iostream>-#include <map>-#include <memory>-#include <sstream>-#include <stdexcept>-#include <string>-#include <vector>--namespace souffle {-class RecordTable;--class ReadStreamCSV : public ReadStream {-public:-    ReadStreamCSV(std::istream& file, const std::map<std::string, std::string>& rwOperation,-            SymbolTable& symbolTable, RecordTable& recordTable)-            : ReadStream(rwOperation, symbolTable, recordTable),-              delimiter(getOr(rwOperation, "delimiter", "\t")), file(file), lineNumber(0),-              inputMap(getInputColumnMap(rwOperation, arity)) {-        while (inputMap.size() < arity) {-            int size = static_cast<int>(inputMap.size());-            inputMap[size] = size;-        }-    }--protected:-    /**-     * Read and return the next tuple.-     *-     * Returns nullptr if no tuple was readable.-     * @return-     */-    std::unique_ptr<RamDomain[]> readNextTuple() override {-        if (file.eof()) {-            return nullptr;-        }-        std::string line;-        std::unique_ptr<RamDomain[]> tuple = std::make_unique<RamDomain[]>(typeAttributes.size());--        if (!getline(file, line)) {-            return nullptr;-        }-        // Handle Windows line endings on non-Windows systems-        if (!line.empty() && line.back() == '\r') {-            line = line.substr(0, line.length() - 1);-        }-        ++lineNumber;--        size_t start = 0;-        size_t end = 0;-        size_t columnsFilled = 0;-        for (uint32_t column = 0; columnsFilled < arity; column++) {-            size_t charactersRead = 0;-            std::string element = nextElement(line, start, end);-            if (inputMap.count(column) == 0) {-                continue;-            }-            ++columnsFilled;--            try {-                auto&& ty = typeAttributes.at(inputMap[column]);-                switch (ty[0]) {-                    case 's': {-                        tuple[inputMap[column]] = symbolTable.unsafeLookup(element);-                        charactersRead = element.size();-                        break;-                    }-                    case 'r': {-                        tuple[inputMap[column]] = readRecord(element, ty, 0, &charactersRead);-                        break;-                    }-                    case 'i': {-                        tuple[inputMap[column]] = RamSignedFromString(element, &charactersRead);-                        break;-                    }-                    case 'u': {-                        tuple[inputMap[column]] = ramBitCast(readRamUnsigned(element, charactersRead));-                        break;-                    }-                    case 'f': {-                        tuple[inputMap[column]] = ramBitCast(RamFloatFromString(element, &charactersRead));-                        break;-                    }-                    default: fatal("invalid type attribute: `%c`", ty[0]);-                }-                // Check if everything was read.-                if (charactersRead != element.size()) {-                    throw std::invalid_argument(-                            "Expected: " + delimiter + " or \\n. Got: " + element[charactersRead]);-                }-            } catch (...) {-                std::stringstream errorMessage;-                errorMessage << "Error converting <" + element + "> in column " << column + 1 << " in line "-                             << lineNumber << "; ";-                throw std::invalid_argument(errorMessage.str());-            }-        }--        return tuple;-    }--    /**-     * Read an unsigned element. Possible bases are 2, 10, 16-     * Base is indicated by the first two chars.-     */-    RamUnsigned readRamUnsigned(const std::string& element, size_t& charactersRead) {-        // Sanity check-        assert(element.size() > 0);--        RamSigned value = 0;--        // Check prefix and parse the input.-        if (isPrefix("0b", element)) {-            value = RamUnsignedFromString(element, &charactersRead, 2);-        } else if (isPrefix("0x", element)) {-            value = RamUnsignedFromString(element, &charactersRead, 16);-        } else {-            value = RamUnsignedFromString(element, &charactersRead);-        }-        return value;-    }--    std::string nextElement(const std::string& line, size_t& start, size_t& end) {-        std::string element;--        // Handle record/tuple delimiter coincidence.-        if (delimiter.find(',') != std::string::npos) {-            int record_parens = 0;-            size_t next_delimiter = line.find(delimiter, start);--            // Find first delimiter after the record.-            while (end < std::min(next_delimiter, line.length()) || record_parens != 0) {-                // Track the number of parenthesis.-                if (line[end] == '[') {-                    ++record_parens;-                } else if (line[end] == ']') {-                    --record_parens;-                }--                // Check for unbalanced parenthesis.-                if (record_parens < 0) {-                    break;-                };--                ++end;--                // Find a next delimiter if the old one is invalid.-                // But only if inside the unbalance parenthesis.-                if (end == next_delimiter && record_parens != 0) {-                    next_delimiter = line.find(delimiter, end);-                }-            }--            // Handle the end-of-the-line case where parenthesis are unbalanced.-            if (record_parens != 0) {-                std::stringstream errorMessage;-                errorMessage << "Unbalanced record parenthesis " << lineNumber << "; ";-                throw std::invalid_argument(errorMessage.str());-            }-        } else {-            end = std::min(line.find(delimiter, start), line.length());-        }--        // Check for missing value.-        if (start > end) {-            std::stringstream errorMessage;-            errorMessage << "Values missing in line " << lineNumber << "; ";-            throw std::invalid_argument(errorMessage.str());-        }--        element = line.substr(start, end - start);-        start = end + delimiter.size();--        return element;-    }--    std::map<int, int> getInputColumnMap(-            const std::map<std::string, std::string>& rwOperation, const unsigned arity_) const {-        std::string columnString = getOr(rwOperation, "columns", "");-        std::map<int, int> inputColumnMap;--        if (!columnString.empty()) {-            std::istringstream iss(columnString);-            std::string mapping;-            int index = 0;-            while (std::getline(iss, mapping, ':')) {-                inputColumnMap[stoi(mapping)] = index++;-            }-            if (inputColumnMap.size() < arity_) {-                throw std::invalid_argument("Invalid column set was given: <" + columnString + ">");-            }-        } else {-            while (inputColumnMap.size() < arity_) {-                int size = static_cast<int>(inputColumnMap.size());-                inputColumnMap[size] = size;-            }-        }-        return inputColumnMap;-    }--    const std::string delimiter;-    std::istream& file;-    size_t lineNumber;-    std::map<int, int> inputMap;-};--class ReadFileCSV : public ReadStreamCSV {-public:-    ReadFileCSV(const std::map<std::string, std::string>& rwOperation, SymbolTable& symbolTable,-            RecordTable& recordTable)-            : ReadStreamCSV(fileHandle, rwOperation, symbolTable, recordTable),-              baseName(souffle::baseName(getFileName(rwOperation))),-              fileHandle(getFileName(rwOperation), std::ios::in | std::ios::binary) {-        if (!fileHandle.is_open()) {-            throw std::invalid_argument("Cannot open fact file " + baseName + "\n");-        }-        // Strip headers if we're using them-        if (getOr(rwOperation, "headers", "false") == "true") {-            std::string line;-            getline(file, line);-        }-    }--    /**-     * Read and return the next tuple.-     *-     * Returns nullptr if no tuple was readable.-     * @return-     */-    std::unique_ptr<RamDomain[]> readNextTuple() override {-        try {-            return ReadStreamCSV::readNextTuple();-        } catch (std::exception& e) {-            std::stringstream errorMessage;-            errorMessage << e.what();-            errorMessage << "cannot parse fact file " << baseName << "!\n";-            throw std::invalid_argument(errorMessage.str());-        }-    }--    ~ReadFileCSV() override = default;--protected:-    /**-     * Return given filename or construct from relation name.-     * Default name is [configured path]/[relation name].facts-     *-     * @param rwOperation map of IO configuration options-     * @return input filename-     */-    static std::string getFileName(const std::map<std::string, std::string>& rwOperation) {-        auto name = getOr(rwOperation, "filename", rwOperation.at("name") + ".facts");-        if (name.front() != '/') {-            name = getOr(rwOperation, "fact-dir", ".") + "/" + name;-        }-        return name;-    }--    std::string baseName;-#ifdef USE_LIBZ-    gzfstream::igzfstream fileHandle;-#else-    std::ifstream fileHandle;-#endif-};--class ReadCinCSVFactory : public ReadStreamFactory {-public:-    std::unique_ptr<ReadStream> getReader(const std::map<std::string, std::string>& rwOperation,-            SymbolTable& symbolTable, RecordTable& recordTable) override {-        return std::make_unique<ReadStreamCSV>(std::cin, rwOperation, symbolTable, recordTable);-    }--    const std::string& getName() const override {-        static const std::string name = "stdin";-        return name;-    }-    ~ReadCinCSVFactory() override = default;-};--class ReadFileCSVFactory : public ReadStreamFactory {-public:-    std::unique_ptr<ReadStream> getReader(const std::map<std::string, std::string>& rwOperation,-            SymbolTable& symbolTable, RecordTable& recordTable) override {-        return std::make_unique<ReadFileCSV>(rwOperation, symbolTable, recordTable);-    }--    const std::string& getName() const override {-        static const std::string name = "file";-        return name;-    }--    ~ReadFileCSVFactory() override = default;-};--} /* namespace souffle */
− cbits/souffle/ReadStreamJSON.h 
@@ -1,368 +0,0 @@-/*- * Souffle - A Datalog Compiler- * Copyright (c) 2020, The Souffle Developers. All rights reserved- * Licensed under the Universal Permissive License v 1.0 as shown at:- * - https://opensource.org/licenses/UPL- * - <souffle root>/licenses/SOUFFLE-UPL.txt- */--/************************************************************************- *- * @file ReadStreamJSON.h- *- ***********************************************************************/--#pragma once--#include "RamTypes.h"-#include "ReadStream.h"-#include "SymbolTable.h"-#include "utility/ContainerUtil.h"-#include "utility/FileUtil.h"-#include "utility/StringUtil.h"--#include <algorithm>-#include <cassert>-#include <cstddef>-#include <cstdint>-#include <fstream>-#include <iostream>-#include <map>-#include <memory>-#include <queue>-#include <sstream>-#include <stdexcept>-#include <string>-#include <tuple>-#include <vector>--namespace souffle {-class RecordTable;--class ReadStreamJSON : public ReadStream {-public:-    ReadStreamJSON(std::istream& file, const std::map<std::string, std::string>& rwOperation,-            SymbolTable& symbolTable, RecordTable& recordTable)-            : ReadStream(rwOperation, symbolTable, recordTable), file(file), pos(0), isInitialized(false) {-        std::string err;-        params = Json::parse(rwOperation.at("params"), err);-        if (err.length() > 0) {-            fatal("cannot get internal params: %s", err);-        }-    }--protected:-    std::istream& file;-    size_t pos;-    Json jsonSource;-    Json params;-    bool isInitialized;-    bool useObjects;-    std::map<const std::string, const size_t> paramIndex;--    std::unique_ptr<RamDomain[]> readNextTuple() override {-        // for some reasons we cannot initalized our json objects in constructor-        // otherwise it will segfault, so we initialize in the first call-        if (!isInitialized) {-            isInitialized = true;-            std::string error = "";-            std::string source(std::istreambuf_iterator<char>(file), {});--            jsonSource = Json::parse(source, error);-            // it should be wrapped by an extra array-            if (error.length() > 0 || !jsonSource.is_array()) {-                fatal("cannot deserialize json because %s:\n%s", error, source);-            }--            // we only check the first one, since there are extra checks-            // in readNextTupleObject/readNextTupleList-            if (jsonSource[0].is_array()) {-                useObjects = false;-            } else if (jsonSource[0].is_object()) {-                useObjects = true;-                size_t index_pos = 0;-                for (auto param : params["relation"]["params"].array_items()) {-                    paramIndex.insert(std::make_pair(param.string_value(), index_pos));-                    index_pos++;-                }-            } else {-                fatal("the input is neither list nor object format");-            }-        }--        if (useObjects) {-            return readNextTupleObject();-        } else {-            return readNextTupleList();-        }-    }--    std::unique_ptr<RamDomain[]> readNextTupleList() {-        if (pos >= jsonSource.array_items().size()) {-            return nullptr;-        }--        std::unique_ptr<RamDomain[]> tuple = std::make_unique<RamDomain[]>(typeAttributes.size());-        const Json& jsonObj = jsonSource[pos];-        assert(jsonObj.is_array() && "the input is not json array");-        pos++;-        for (size_t i = 0; i < typeAttributes.size(); ++i) {-            try {-                auto&& ty = typeAttributes.at(i);-                switch (ty[0]) {-                    case 's': {-                        tuple[i] = symbolTable.unsafeLookup(jsonObj[i].string_value());-                        break;-                    }-                    case 'r': {-                        tuple[i] = readNextElementList(jsonObj[i], ty);-                        break;-                    }-                    case 'i': {-                        tuple[i] = jsonObj[i].int_value();-                        break;-                    }-                    case 'u': {-                        tuple[i] = jsonObj[i].int_value();-                        break;-                    }-                    case 'f': {-                        tuple[i] = jsonObj[i].number_value();-                        break;-                    }-                    default: fatal("invalid type attribute: `%c`", ty[0]);-                }-            } catch (...) {-                std::stringstream errorMessage;-                if (jsonObj.is_array() && i < jsonObj.array_items().size()) {-                    errorMessage << "Error converting: " << jsonObj[i].dump();-                } else {-                    errorMessage << "Invalid index: " << i;-                }-                throw std::invalid_argument(errorMessage.str());-            }-        }--        return tuple;-    }--    RamDomain readNextElementList(const Json& source, const std::string& recordTypeName) {-        auto&& recordInfo = types["records"][recordTypeName];--        if (recordInfo.is_null()) {-            throw std::invalid_argument("Missing record type information: " + recordTypeName);-        }--        // Handle null case-        if (source.is_null()) {-            return 0;-        }--        assert(source.is_array() && "the input is not json array");-        auto&& recordTypes = recordInfo["types"];-        const size_t recordArity = recordInfo["arity"].long_value();-        std::vector<RamDomain> recordValues(recordArity);-        for (size_t i = 0; i < recordArity; ++i) {-            const std::string& recordType = recordTypes[i].string_value();-            switch (recordType[0]) {-                case 's': {-                    recordValues[i] = symbolTable.unsafeLookup(source[i].string_value());-                    break;-                }-                case 'r': {-                    recordValues[i] = readNextElementList(source[i], recordType);-                    break;-                }-                case 'i': {-                    recordValues[i] = source[i].int_value();-                    break;-                }-                case 'u': {-                    recordValues[i] = source[i].int_value();-                    break;-                }-                case 'f': {-                    recordValues[i] = source[i].number_value();-                    break;-                }-                default: fatal("invalid type attribute");-            }-        }--        return recordTable.pack(recordValues.data(), recordValues.size());-    }--    std::unique_ptr<RamDomain[]> readNextTupleObject() {-        if (pos >= jsonSource.array_items().size()) {-            return nullptr;-        }--        std::unique_ptr<RamDomain[]> tuple = std::make_unique<RamDomain[]>(typeAttributes.size());-        const Json& jsonObj = jsonSource[pos];-        assert(jsonObj.is_object() && "the input is not json object");-        pos++;-        for (auto p : jsonObj.object_items()) {-            try {-                // get the corresponding position by parameter name-                if (paramIndex.find(p.first) == paramIndex.end()) {-                    fatal("invalid parameter: %s", p.first);-                }-                size_t i = paramIndex.at(p.first);-                auto&& ty = typeAttributes.at(i);-                switch (ty[0]) {-                    case 's': {-                        tuple[i] = symbolTable.unsafeLookup(p.second.string_value());-                        break;-                    }-                    case 'r': {-                        tuple[i] = readNextElementObject(p.second, ty);-                        break;-                    }-                    case 'i': {-                        tuple[i] = p.second.int_value();-                        break;-                    }-                    case 'u': {-                        tuple[i] = p.second.int_value();-                        break;-                    }-                    case 'f': {-                        tuple[i] = p.second.number_value();-                        break;-                    }-                    default: fatal("invalid type attribute: `%c`", ty[0]);-                }-            } catch (...) {-                std::stringstream errorMessage;-                errorMessage << "Error converting: " << p.second.dump();-                throw std::invalid_argument(errorMessage.str());-            }-        }--        return tuple;-    }--    RamDomain readNextElementObject(const Json& source, const std::string& recordTypeName) {-        auto&& recordInfo = types["records"][recordTypeName];-        const std::string recordName = recordTypeName.substr(2);-        std::map<const std::string, const size_t> recordIndex;--        size_t index_pos = 0;-        for (auto param : params["records"][recordName]["params"].array_items()) {-            recordIndex.insert(std::make_pair(param.string_value(), index_pos));-            index_pos++;-        }--        if (recordInfo.is_null()) {-            throw std::invalid_argument("Missing record type information: " + recordTypeName);-        }--        // Handle null case-        if (source.is_null()) {-            return 0;-        }--        assert(source.is_object() && "the input is not json object");-        auto&& recordTypes = recordInfo["types"];-        const size_t recordArity = recordInfo["arity"].long_value();-        std::vector<RamDomain> recordValues(recordArity);-        recordValues.reserve(recordIndex.size());-        for (auto readParam : source.object_items()) {-            // get the corresponding position by parameter name-            if (recordIndex.find(readParam.first) == recordIndex.end()) {-                fatal("invalid parameter: %s", readParam.first);-            }-            size_t i = recordIndex.at(readParam.first);-            auto&& type = recordTypes[i].string_value();-            switch (type[0]) {-                case 's': {-                    recordValues[i] = symbolTable.unsafeLookup(readParam.second.string_value());-                    break;-                }-                case 'r': {-                    recordValues[i] = readNextElementObject(readParam.second, type);-                    break;-                }-                case 'i': {-                    recordValues[i] = readParam.second.int_value();-                    break;-                }-                case 'u': {-                    recordValues[i] = readParam.second.int_value();-                    break;-                }-                case 'f': {-                    recordValues[i] = readParam.second.number_value();-                    break;-                }-                default: fatal("invalid type attribute: `%c`", type[0]);-            }-        }--        return recordTable.pack(recordValues.data(), recordValues.size());-    }-};--class ReadFileJSON : public ReadStreamJSON {-public:-    ReadFileJSON(const std::map<std::string, std::string>& rwOperation, SymbolTable& symbolTable,-            RecordTable& recordTable)-            : ReadStreamJSON(fileHandle, rwOperation, symbolTable, recordTable),-              baseName(souffle::baseName(getFileName(rwOperation))),-              fileHandle(getFileName(rwOperation), std::ios::in | std::ios::binary) {-        if (!fileHandle.is_open()) {-            throw std::invalid_argument("Cannot open json file " + baseName + "\n");-        }-    }--    ~ReadFileJSON() override = default;--protected:-    /**-     * Return given filename or construct from relation name.-     * Default name is [configured path]/[relation name].json-     *-     * @param rwOperation map of IO configuration options-     * @return input filename-     */-    static std::string getFileName(const std::map<std::string, std::string>& rwOperation) {-        auto name = getOr(rwOperation, "filename", rwOperation.at("name") + ".json");-        if (name.front() != '/') {-            name = getOr(rwOperation, "fact-dir", ".") + "/" + name;-        }-        return name;-    }--    std::string baseName;-    std::ifstream fileHandle;-};--class ReadCinJSONFactory : public ReadStreamFactory {-public:-    std::unique_ptr<ReadStream> getReader(const std::map<std::string, std::string>& rwOperation,-            SymbolTable& symbolTable, RecordTable& recordTable) override {-        return std::make_unique<ReadStreamJSON>(std::cin, rwOperation, symbolTable, recordTable);-    }--    const std::string& getName() const override {-        static const std::string name = "json";-        return name;-    }-    ~ReadCinJSONFactory() override = default;-};--class ReadFileJSONFactory : public ReadStreamFactory {-public:-    std::unique_ptr<ReadStream> getReader(const std::map<std::string, std::string>& rwOperation,-            SymbolTable& symbolTable, RecordTable& recordTable) override {-        return std::make_unique<ReadFileJSON>(rwOperation, symbolTable, recordTable);-    }--    const std::string& getName() const override {-        static const std::string name = "jsonfile";-        return name;-    }--    ~ReadFileJSONFactory() override = default;-};-}  // namespace souffle
− cbits/souffle/ReadStreamSQLite.h 
@@ -1,195 +0,0 @@-/*- * Souffle - A Datalog Compiler- * Copyright (c) 2013, 2014, Oracle and/or its affiliates. All rights reserved- * Licensed under the Universal Permissive License v 1.0 as shown at:- * - https://opensource.org/licenses/UPL- * - <souffle root>/licenses/SOUFFLE-UPL.txt- */--/************************************************************************- *- * @file ReadStreamSQLite.h- *- ***********************************************************************/--#pragma once--#include "RamTypes.h"-#include "ReadStream.h"-#include "SymbolTable.h"-#include "utility/MiscUtil.h"-#include "utility/StringUtil.h"-#include <cassert>-#include <cstdint>-#include <fstream>-#include <map>-#include <memory>-#include <stdexcept>-#include <string>-#include <vector>-#include <sqlite3.h>--namespace souffle {-class RecordTable;--class ReadStreamSQLite : public ReadStream {-public:-    ReadStreamSQLite(const std::map<std::string, std::string>& rwOperation, SymbolTable& symbolTable,-            RecordTable& recordTable)-            : ReadStream(rwOperation, symbolTable, recordTable), dbFilename(getFileName(rwOperation)),-              relationName(rwOperation.at("name")) {-        openDB();-        checkTableExists();-        prepareSelectStatement();-    }--    ~ReadStreamSQLite() override {-        sqlite3_finalize(selectStatement);-        sqlite3_close(db);-    }--protected:-    /**-     * Read and return the next tuple.-     *-     * Returns nullptr if no tuple was readable.-     * @return-     */-    std::unique_ptr<RamDomain[]> readNextTuple() override {-        if (sqlite3_step(selectStatement) != SQLITE_ROW) {-            return nullptr;-        }--        std::unique_ptr<RamDomain[]> tuple = std::make_unique<RamDomain[]>(arity + auxiliaryArity);--        uint32_t column;-        for (column = 0; column < arity; column++) {-            std::string element(reinterpret_cast<const char*>(sqlite3_column_text(selectStatement, column)));--            if (element.empty()) {-                element = "n/a";-            }--            try {-                auto&& ty = typeAttributes.at(column);-                switch (ty[0]) {-                    case 's': tuple[column] = symbolTable.unsafeLookup(element); break;-                    case 'i':-                    case 'u':-                    case 'f':-                    case 'r': tuple[column] = RamSignedFromString(element); break;-                    default: fatal("invalid type attribute: `%c`", ty[0]);-                }-            } catch (...) {-                std::stringstream errorMessage;-                errorMessage << "Error converting number in column " << (column) + 1;-                throw std::invalid_argument(errorMessage.str());-            }-        }--        return tuple;-    }--    void executeSQL(const std::string& sql) {-        assert(db && "Database connection is closed");--        char* errorMessage = nullptr;-        /* Execute SQL statement */-        int rc = sqlite3_exec(db, sql.c_str(), nullptr, nullptr, &errorMessage);-        if (rc != SQLITE_OK) {-            std::stringstream error;-            error << "SQLite error in sqlite3_exec: " << sqlite3_errmsg(db) << "\n";-            error << "SQL error: " << errorMessage << "\n";-            error << "SQL: " << sql << "\n";-            sqlite3_free(errorMessage);-            throw std::invalid_argument(error.str());-        }-    }--    void throwError(const std::string& message) {-        std::stringstream error;-        error << message << sqlite3_errmsg(db) << "\n";-        throw std::invalid_argument(error.str());-    }--    void prepareSelectStatement() {-        std::stringstream selectSQL;-        selectSQL << "SELECT * FROM '" << relationName << "'";-        const char* tail = nullptr;-        if (sqlite3_prepare_v2(db, selectSQL.str().c_str(), -1, &selectStatement, &tail) != SQLITE_OK) {-            throwError("SQLite error in sqlite3_prepare_v2: ");-        }-    }--    void openDB() {-        if (sqlite3_open(dbFilename.c_str(), &db) != SQLITE_OK) {-            throwError("SQLite error in sqlite3_open: ");-        }-        sqlite3_extended_result_codes(db, 1);-        executeSQL("PRAGMA synchronous = OFF");-        executeSQL("PRAGMA journal_mode = MEMORY");-    }--    void checkTableExists() {-        sqlite3_stmt* tableStatement;-        std::stringstream selectSQL;-        selectSQL << "SELECT count(*) FROM sqlite_master WHERE type IN ('table', 'view') AND ";-        selectSQL << " name = '" << relationName << "';";-        const char* tail = nullptr;--        if (sqlite3_prepare_v2(db, selectSQL.str().c_str(), -1, &tableStatement, &tail) != SQLITE_OK) {-            throwError("SQLite error in sqlite3_prepare_v2: ");-        }--        if (sqlite3_step(tableStatement) == SQLITE_ROW) {-            int count = sqlite3_column_int(tableStatement, 0);-            if (count > 0) {-                sqlite3_finalize(tableStatement);-                return;-            }-        }-        sqlite3_finalize(tableStatement);-        throw std::invalid_argument(-                "Required table or view does not exist in " + dbFilename + " for relation " + relationName);-    }--    /**-     * Return given filename or construct from relation name.-     * Default name is [configured path]/[relation name].sqlite-     *-     * @param rwOperation map of IO configuration options-     * @return input filename-     */-    static std::string getFileName(const std::map<std::string, std::string>& rwOperation) {-        // legacy support for SQLite prior to 2020-03-18-        // convert dbname to filename-        auto name = getOr(rwOperation, "dbname", rwOperation.at("name") + ".sqlite");-        name = getOr(rwOperation, "filename", name);--        if (name.front() != '/') {-            name = getOr(rwOperation, "fact-dir", ".") + "/" + name;-        }-        return name;-    }--    const std::string dbFilename;-    const std::string relationName;-    sqlite3_stmt* selectStatement = nullptr;-    sqlite3* db = nullptr;-};--class ReadSQLiteFactory : public ReadStreamFactory {-public:-    std::unique_ptr<ReadStream> getReader(const std::map<std::string, std::string>& rwOperation,-            SymbolTable& symbolTable, RecordTable& recordTable) override {-        return std::make_unique<ReadStreamSQLite>(rwOperation, symbolTable, recordTable);-    }--    const std::string& getName() const override {-        static const std::string name = "sqlite";-        return name;-    }-    ~ReadSQLiteFactory() override = default;-};--} /* namespace souffle */
cbits/souffle/RecordTable.h view
@@ -17,8 +17,8 @@  #pragma once -#include "CompiledTuple.h"-#include "RamTypes.h"+#include "souffle/CompiledTuple.h"+#include "souffle/RamTypes.h" #include <cassert> #include <cstddef> #include <limits>@@ -70,7 +70,7 @@ #pragma omp critical(record_unpack)                 {                     indexToRecord.push_back(vector);-                    index = indexToRecord.size() - 1;+                    index = static_cast<RamDomain>(indexToRecord.size()) - 1;                     recordToIndex[vector] = index;                      // assert that new index is smaller than the range@@ -117,8 +117,13 @@     }     /** @brief convert record reference to a record */     const RamDomain* unpack(RamDomain ref, size_t arity) const {-        auto iter = maps.find(arity);-        assert(iter != maps.end() && "Attempting to unpack non-existing record");+        std::unordered_map<size_t, RecordMap>::const_iterator iter;+#pragma omp critical(RecordTableGetForArity)+        {+            // Find a previously emplaced map+            iter = maps.find(arity);+        }+        assert(iter != maps.end() && "Attempting to unpack record for non-existing arity");         return (iter->second).unpack(ref);     }
− cbits/souffle/SerialisationStream.h 
@@ -1,91 +0,0 @@-/*- * Souffle - A Datalog Compiler- * Copyright (c) 2020, The Souffle Developers. All rights reserved.- * Licensed under the Universal Permissive License v 1.0 as shown at:- * - https://opensource.org/licenses/UPL- * - <souffle root>/licenses/SOUFFLE-UPL.txt- */--/************************************************************************- *- * @file SerialisationStream.h- *- * Defines a common base class for relation serialisation streams.- *- ***********************************************************************/--#pragma once--#include "RamTypes.h"-#include "json11.h"--#include <cassert>-#include <cstddef>-#include <map>-#include <string>-#include <utility>-#include <vector>--namespace souffle {--class RecordTable;-class SymbolTable;--using json11::Json;--template <bool readOnlyTables>-class SerialisationStream {-public:-    virtual ~SerialisationStream() = default;--protected:-    template <typename A>-    using RO = std::conditional_t<readOnlyTables, const A, A>;--    SerialisationStream(RO<SymbolTable>& symTab, RO<RecordTable>& recTab, Json types,-            std::vector<std::string> relTypes, size_t auxArity = 0)-            : symbolTable(symTab), recordTable(recTab), types(std::move(types)),-              typeAttributes(std::move(relTypes)), arity(typeAttributes.size() - auxArity),-              auxiliaryArity(auxArity) {}--    SerialisationStream(RO<SymbolTable>& symTab, RO<RecordTable>& recTab, Json types)-            : symbolTable(symTab), recordTable(recTab), types(std::move(types)) {-        setupFromJson();-    }--    SerialisationStream(RO<SymbolTable>& symTab, RO<RecordTable>& recTab,-            const std::map<std::string, std::string>& rwOperation)-            : symbolTable(symTab), recordTable(recTab) {-        std::string parseErrors;-        types = Json::parse(rwOperation.at("types"), parseErrors);-        assert(parseErrors.size() == 0 && "Internal JSON parsing failed.");-        setupFromJson();-    }--    RO<SymbolTable>& symbolTable;-    RO<RecordTable>& recordTable;-    Json types;-    std::vector<std::string> typeAttributes;--    size_t arity = 0;-    size_t auxiliaryArity = 0;--private:-    void setupFromJson() {-        auto&& relInfo = types["relation"];-        arity = static_cast<size_t>(relInfo["arity"].long_value());-        auxiliaryArity = static_cast<size_t>(relInfo["auxArity"].long_value());--        assert(relInfo["types"].is_array());-        auto&& relTypes = relInfo["types"].array_items();-        assert(relTypes.size() == (arity + auxiliaryArity));--        for (size_t i = 0; i < arity + auxiliaryArity; ++i) {-            auto&& type = relTypes[i].string_value();-            assert(!type.empty() && "malformed types tag");-            typeAttributes.push_back(type);-        }-    }-};--}  // namespace souffle
cbits/souffle/SouffleInterface.h view
@@ -16,8 +16,9 @@  #pragma once -#include "RamTypes.h"-#include "SymbolTable.h"+#include "souffle/RamTypes.h"+#include "souffle/SymbolTable.h"+#include "souffle/utility/MiscUtil.h" #include <algorithm> #include <cassert> #include <cstddef>@@ -150,7 +151,7 @@          * iterator_base class pointer.          *          */-        std::unique_ptr<iterator_base> iter = nullptr;+        Own<iterator_base> iter = nullptr;      public:         /**@@ -752,12 +753,12 @@     /**      * Output all the input relations in stdout, without generating any files. (for debug purposes).      */-    virtual void dumpInputs(std::ostream& out = std::cout) = 0;+    virtual void dumpInputs() = 0;      /**      * Output all the output relations in stdout, without generating any files. (for debug purposes).      */-    virtual void dumpOutputs(std::ostream& out = std::cout) = 0;+    virtual void dumpOutputs() = 0;      /**      * Set the number of threads to be used
cbits/souffle/SymbolTable.h view
@@ -16,10 +16,10 @@  #pragma once -#include "RamTypes.h"-#include "utility/MiscUtil.h"-#include "utility/ParallelUtil.h"-#include "utility/StreamUtil.h"+#include "souffle/RamTypes.h"+#include "souffle/utility/MiscUtil.h"+#include "souffle/utility/ParallelUtil.h"+#include "souffle/utility/StreamUtil.h" #include <algorithm> #include <cstdlib> #include <deque>@@ -139,7 +139,7 @@     /** Find the index of a symbol in the table, inserting a new symbol if it does not exist there      * already. */     RamDomain unsafeLookup(const std::string& symbol) {-        return newSymbolOfIndex(symbol);+        return static_cast<RamDomain>(newSymbolOfIndex(symbol));     }      /** Find a symbol in the table by its index, note that this gives an error if the index is out of
− cbits/souffle/Table.h 
@@ -1,145 +0,0 @@-/*- * Souffle - A Datalog Compiler- * Copyright (c) 2013, 2015, Oracle and/or its affiliates. All rights reserved- * Licensed under the Universal Permissive License v 1.0 as shown at:- * - https://opensource.org/licenses/UPL- * - <souffle root>/licenses/SOUFFLE-UPL.txt- */--/************************************************************************- *- * @file Table.h- *- * An implementation of a generic Table storing a position-fixed collection- * of objects in main memory.- *- ***********************************************************************/--#pragma once--#include <iosfwd>-#include <iterator>--namespace souffle {--template <typename T, unsigned blockSize = 4096>-class Table {-    struct Block {-        Block* next;-        std::size_t used = 0;-        T data[blockSize];--        Block() : next(nullptr) {}--        bool isFull() const {-            return used == blockSize;-        }--        const T& append(const T& element) {-            const T& res = data[used];-            data[used] = element;-            used++;-            return res;-        }-    };--    Block* head;-    Block* tail;--    std::size_t count = 0;--public:-    class iterator : public std::iterator<std::forward_iterator_tag, T> {-        Block* block;-        unsigned pos;--    public:-        iterator(Block* block = nullptr, unsigned pos = 0) : block(block), pos(pos) {}--        iterator(const iterator&) = default;-        iterator(iterator&&) = default;-        iterator& operator=(const iterator&) = default;--        // the equality operator as required by the iterator concept-        bool operator==(const iterator& other) const {-            return (block == nullptr && other.block == nullptr) || (block == other.block && pos == other.pos);-        }--        // the not-equality operator as required by the iterator concept-        bool operator!=(const iterator& other) const {-            return !(*this == other);-        }--        // the deref operator as required by the iterator concept-        const T& operator*() const {-            return block->data[pos];-        }--        // the increment operator as required by the iterator concept-        iterator& operator++() {-            // move on in block-            if (++pos < block->used) {-                return *this;-            }-            // or to next block-            block = block->next;-            pos = 0;-            return *this;-        }-    };--    Table() : head(nullptr), tail(nullptr) {}--    ~Table() {-        clear();-    }--    bool empty() const {-        return (!head);-    }--    std::size_t size() const {-        return count;-    }--    const T& insert(const T& element) {-        // check whether the head is initialized-        if (!head) {-            head = new Block();-            tail = head;-        }--        // check whether tail is full-        if (tail->isFull()) {-            tail->next = new Block();-            tail = tail->next;-        }--        // increment counter-        count++;--        // add another element-        return tail->append(element);-    }--    iterator begin() const {-        return iterator(head);-    }--    iterator end() const {-        return iterator();-    }--    void clear() {-        while (head != nullptr) {-            auto cur = head;-            head = head->next;-            delete cur;-        }-        count = 0;-        head = nullptr;-        tail = nullptr;-    }-};--}  // end namespace souffle
− cbits/souffle/UnionFind.h 
@@ -1,356 +0,0 @@-/*- * Souffle - A Datalog Compiler- * Copyright (c) 2017 The Souffle Developers. All Rights reserved- * Licensed under the Universal Permissive License v 1.0 as shown at:- * - https://opensource.org/licenses/UPL- * - <souffle root>/licenses/SOUFFLE-UPL.txt- */--/************************************************************************- *- * @file UnionFind.h- *- * Defines a union-find data-structure- *- ***********************************************************************/--#pragma once--#include "LambdaBTree.h"-#include "PiggyList.h"-#include <atomic>-#include <cstddef>-#include <cstdint>-#include <functional>-#include <utility>--namespace souffle {--// branch predictor hacks-#define unlikely(x) __builtin_expect((x), 0)-#define likely(x) __builtin_expect((x), 1)--using rank_t = uint8_t;-/* technically uint56_t, but, doesn't exist. Just be careful about storing > 2^56 elements. */-using parent_t = uint64_t;--// number of bits that the rank is-constexpr uint8_t split_size = 8u;--// block_t stores parent in the upper half, rank in the lower half-using block_t = uint64_t;-// block_t & rank_mask extracts the rank-constexpr block_t rank_mask = (1ul << split_size) - 1;--/**- * Structure that emulates a Disjoint Set, i.e. a data structure that supports efficient union-find operations- */-class DisjointSet {-    template <typename TupleType>-    friend class EquivalenceRelation;--    PiggyList<std::atomic<block_t>> a_blocks;--public:-    DisjointSet() = default;--    // copy ctor-    DisjointSet(DisjointSet& other) = delete;-    // move ctor-    DisjointSet(DisjointSet&& other) = delete;--    // copy assign ctor-    DisjointSet& operator=(DisjointSet& ds) = delete;-    // move assign ctor-    DisjointSet& operator=(DisjointSet&& ds) = delete;--    /**-     * Return the number of elements in this disjoint set (not the number of pairs)-     */-    inline size_t size() {-        auto sz = a_blocks.size();-        return sz;-    };--    /**-     * Yield reference to the node by its node index-     * @param node node to be searched-     * @return the parent block of the specified node-     */-    inline std::atomic<block_t>& get(parent_t node) const {-        auto& ret = a_blocks.get(node);-        return ret;-    };--    /**-     * Equivalent to the find() function in union/find-     * Find the highest ancestor of the provided node - flattening as we go-     * @param x the node to find the parent of, whilst flattening its set-tree-     * @return The parent of x-     */-    parent_t findNode(parent_t x) {-        // while x's parent is not itself-        while (x != b2p(get(x))) {-            block_t xState = get(x);-            // yield x's parent's parent-            parent_t newParent = b2p(get(b2p(xState)));-            // construct block out of the original rank and the new parent-            block_t newState = pr2b(newParent, b2r(xState));--            this->get(x).compare_exchange_strong(xState, newState);--            x = newParent;-        }-        return x;-    }--private:-    /**-     * Update the root of the tree of which x is, to have y as the base instead-     * @param x : old root-     * @param oldrank : old root rank-     * @param y : new root-     * @param newrank : new root rank-     * @return Whether the update succeeded (fails if another root update/union has been perfomed in the-     * interim)-     */-    bool updateRoot(const parent_t x, const rank_t oldrank, const parent_t y, const rank_t newrank) {-        block_t oldState = get(x);-        parent_t nextN = b2p(oldState);-        rank_t rankN = b2r(oldState);--        if (nextN != x || rankN != oldrank) return false;-        // set the parent and rank of the new record-        block_t newVal = pr2b(y, newrank);--        return this->get(x).compare_exchange_strong(oldState, newVal);-    }--public:-    /**-     * Clears the DisjointSet of all nodes-     * Invalidates all iterators-     */-    void clear() {-        a_blocks.clear();-    }--    /**-     * Check whether the two indices are in the same set-     * @param x node to be checked-     * @param y node to be checked-     * @return where the two indices are in the same set-     */-    bool sameSet(parent_t x, parent_t y) {-        while (true) {-            x = findNode(x);-            y = findNode(y);-            if (x == y) return true;-            // if x's parent is itself, they are not the same set-            if (b2p(get(x)) == x) return false;-        }-    }--    /**-     * Union the two specified index nodes-     * @param x node to be unioned-     * @param y node to be unioned-     */-    void unionNodes(parent_t x, parent_t y) {-        while (true) {-            x = findNode(x);-            y = findNode(y);--            // no need to union if both already in same set-            if (x == y) return;--            rank_t xrank = b2r(get(x));-            rank_t yrank = b2r(get(y));--            // if x comes before y (better rank or earlier & equal node)-            if (xrank > yrank || ((xrank == yrank) && x > y)) {-                std::swap(x, y);-                std::swap(xrank, yrank);-            }-            // join the trees together-            // perhaps we can optimise the use of compare_exchange_strong here, as we're in a pessimistic loop-            if (!updateRoot(x, xrank, y, yrank)) {-                continue;-            }-            // make sure that the ranks are orderable-            if (xrank == yrank) {-                updateRoot(y, yrank, y, yrank + 1);-            }-            break;-        }-    }--    /**-     * Create a node with its parent as itself, rank 0-     * @return the newly created block-     */-    inline block_t makeNode() {-        // make node and find out where we've added it-        size_t nodeDetails = a_blocks.createNode();--        a_blocks.get(nodeDetails).store(pr2b(nodeDetails, 0));--        return a_blocks.get(nodeDetails).load();-    };--    /**-     * Extract parent from block-     * @param inblock the block to be masked-     * @return The parent_t contained in the upper half of block_t-     */-    static inline parent_t b2p(const block_t inblock) {-        return (parent_t)(inblock >> split_size);-    };--    /**-     * Extract rank from block-     * @param inblock the block to be masked-     * @return the rank_t contained in the lower half of block_t-     */-    static inline rank_t b2r(const block_t inblock) {-        return (rank_t)(inblock & rank_mask);-    };--    /**-     * Yield a block given parent and rank-     * @param parent the top half bits-     * @param rank the lower half bits-     * @return the resultant block after merge-     */-    static inline block_t pr2b(const parent_t parent, const rank_t rank) {-        return (((block_t)parent) << split_size) | rank;-    };-};--template <typename StorePair>-struct EqrelMapComparator {-    int operator()(const StorePair& a, const StorePair& b) {-        if (a.first < b.first) {-            return -1;-        } else if (b.first < a.first) {-            return 1;-        } else {-            return 0;-        }-    }--    bool less(const StorePair& a, const StorePair& b) {-        return operator()(a, b) < 0;-    }--    bool equal(const StorePair& a, const StorePair& b) {-        return operator()(a, b) == 0;-    }-};--template <typename SparseDomain>-class SparseDisjointSet {-    DisjointSet ds;--    template <typename TupleType>-    friend class EquivalenceRelation;--    using PairStore = std::pair<SparseDomain, parent_t>;-    using SparseMap =-            LambdaBTreeSet<PairStore, std::function<parent_t(PairStore&)>, EqrelMapComparator<PairStore>>;-    using DenseMap = RandomInsertPiggyList<SparseDomain>;--    typename SparseMap::operation_hints last_ins;--    SparseMap sparseToDenseMap;-    // mapping from union-find val to souffle, union-find encoded as index-    DenseMap denseToSparseMap;--public:-    /**-     * Retrieve dense encoding, adding it in if non-existent-     * @param in the sparse value-     * @return the corresponding dense value-     */-    parent_t toDense(const SparseDomain in) {-        // insert into the mapping - if the key doesn't exist (in), the function will be called-        // and a dense value will be created for it-        PairStore p = {in, -1};-        return sparseToDenseMap.insert(p, [&](PairStore& p) {-            parent_t c2 = DisjointSet::b2p(this->ds.makeNode());-            this->denseToSparseMap.insertAt(c2, p.first);-            p.second = c2;-            return c2;-        });-    }--public:-    SparseDisjointSet() = default;--    // copy ctor-    SparseDisjointSet(SparseDisjointSet& other) = delete;--    // move ctor-    SparseDisjointSet(SparseDisjointSet&& other) = delete;--    // copy assign ctor-    SparseDisjointSet& operator=(SparseDisjointSet& other) = delete;--    // move assign ctor-    SparseDisjointSet& operator=(SparseDisjointSet&& other) = delete;--    /**-     * For the given dense value, return the associated sparse value-     *   Undefined behaviour if dense value not in set-     * @param in the supplied dense value-     * @return the sparse value from the denseToSparseMap-     */-    inline const SparseDomain toSparse(const parent_t in) const {-        return denseToSparseMap.get(in);-    };--    /* a wrapper to enable checking in the sparse set - however also adds them if not already existing */-    inline bool sameSet(SparseDomain x, SparseDomain y) {-        return ds.sameSet(toDense(x), toDense(y));-    };-    /* finds the node in the underlying disjoint set, adding the node if non-existent */-    inline SparseDomain findNode(SparseDomain x) {-        return toSparse(ds.findNode(toDense(x)));-    };-    /* union the nodes, add if not existing */-    inline void unionNodes(SparseDomain x, SparseDomain y) {-        ds.unionNodes(toDense(x), toDense(y));-    };--    inline std::size_t size() {-        return ds.size();-    };--    /**-     * Remove all elements from this disjoint set-     */-    void clear() {-        ds.clear();-        sparseToDenseMap.clear();-        denseToSparseMap.clear();-    }--    /* wrapper for node creation */-    inline void makeNode(SparseDomain val) {-        // dense has the behaviour of creating if not exists.-        toDense(val);-    };--    /* whether we the supplied node exists */-    inline bool nodeExists(const SparseDomain val) const {-        return sparseToDenseMap.contains({val, -1});-    };--    inline bool contains(SparseDomain v1, SparseDomain v2) {-        if (nodeExists(v1) && nodeExists(v2)) {-            return sameSet(v1, v2);-        }-        return false;-    }-};-}  // namespace souffle
− cbits/souffle/WriteStream.h 
@@ -1,133 +0,0 @@-/*- * Souffle - A Datalog Compiler- * Copyright (c) 2013, 2014, Oracle and/or its affiliates. All rights reserved- * Licensed under the Universal Permissive License v 1.0 as shown at:- * - https://opensource.org/licenses/UPL- * - <souffle root>/licenses/SOUFFLE-UPL.txt- */--/************************************************************************- *- * @file WriteStream.h- *- ***********************************************************************/--#pragma once--#include "RamTypes.h"-#include "RecordTable.h"-#include "SerialisationStream.h"-#include "SymbolTable.h"-#include "json11.h"-#include "utility/MiscUtil.h"-#include <cassert>-#include <cstddef>-#include <map>-#include <memory>-#include <ostream>-#include <string>--namespace souffle {--using json11::Json;--class WriteStream : public SerialisationStream<true> {-public:-    WriteStream(const std::map<std::string, std::string>& rwOperation, const SymbolTable& symbolTable,-            const RecordTable& recordTable)-            : SerialisationStream(symbolTable, recordTable, rwOperation),-              summary(rwOperation.at("IO") == "stdoutprintsize") {}--    template <typename T>-    void writeAll(const T& relation) {-        if (summary) {-            return writeSize(relation.size());-        }-        auto lease = symbolTable.acquireLock();-        (void)lease;  // silence "unused variable" warning-        if (arity == 0) {-            if (relation.begin() != relation.end()) {-                writeNullary();-            }-            return;-        }-        for (const auto& current : relation) {-            writeNext(current);-        }-    }--    template <typename T>-    void writeSize(const T& relation) {-        writeSize(relation.size());-    }--protected:-    const bool summary;--    virtual void writeNullary() = 0;-    virtual void writeNextTuple(const RamDomain* tuple) = 0;-    virtual void writeSize(std::size_t) {-        fatal("attempting to print size of a write operation");-    }--    template <typename Tuple>-    void writeNext(const Tuple tuple) {-        writeNextTuple(tuple.data);-    }--    void outputRecord(std::ostream& destination, const RamDomain value, const std::string& name) {-        auto&& recordInfo = types["records"][name];--        // Check if record type information are present-        assert(!recordInfo.is_null() && "Missing record type information");--        // Check for nil-        if (value == 0) {-            destination << "nil";-            return;-        }--        auto&& recordTypes = recordInfo["types"];-        const size_t recordArity = recordInfo["arity"].long_value();--        const RamDomain* tuplePtr = recordTable.unpack(value, recordArity);--        destination << "[";--        // print record's elements-        for (size_t i = 0; i < recordArity; ++i) {-            if (i > 0) {-                destination << ", ";-            }--            const std::string& recordType = recordTypes[i].string_value();-            const RamDomain recordValue = tuplePtr[i];--            switch (recordType[0]) {-                case 'i': destination << recordValue; break;-                case 'f': destination << ramBitCast<RamFloat>(recordValue); break;-                case 'u': destination << ramBitCast<RamUnsigned>(recordValue); break;-                case 's': destination << symbolTable.unsafeResolve(recordValue); break;-                case 'r': outputRecord(destination, recordValue, recordType); break;-                default: fatal("Unsupported type attribute: `%c`", recordType[0]);-            }-        }-        destination << "]";-    }-};--class WriteStreamFactory {-public:-    virtual std::unique_ptr<WriteStream> getWriter(const std::map<std::string, std::string>& rwOperation,-            const SymbolTable& symbolTable, const RecordTable& recordTable) = 0;--    virtual const std::string& getName() const = 0;-    virtual ~WriteStreamFactory() = default;-};--template <>-inline void WriteStream::writeNext(const RamDomain* tuple) {-    writeNextTuple(tuple);-}--} /* namespace souffle */
− cbits/souffle/WriteStreamCSV.h 
@@ -1,252 +0,0 @@-/*- * Souffle - A Datalog Compiler- * Copyright (c) 2013, 2014, Oracle and/or its affiliates. All rights reserved- * Licensed under the Universal Permissive License v 1.0 as shown at:- * - https://opensource.org/licenses/UPL- * - <souffle root>/licenses/SOUFFLE-UPL.txt- */--/************************************************************************- *- * @file WriteStreamCSV.h- *- ***********************************************************************/--#pragma once--#include "RamTypes.h"-#include "SymbolTable.h"-#include "WriteStream.h"-#include "utility/ContainerUtil.h"-#include "utility/MiscUtil.h"-#include "utility/ParallelUtil.h"-#ifdef USE_LIBZ-#include "gzfstream.h"-#endif--#include <cstddef>-#include <iomanip>-#include <iostream>-#include <map>-#include <ostream>-#include <string>-#include <vector>--namespace souffle {--class RecordTable;--class WriteStreamCSV : public WriteStream {-protected:-    WriteStreamCSV(const std::map<std::string, std::string>& rwOperation, const SymbolTable& symbolTable,-            const RecordTable& recordTable)-            : WriteStream(rwOperation, symbolTable, recordTable),-              delimiter(getOr(rwOperation, "delimiter", "\t")){};--    const std::string delimiter;--    void writeNextTupleCSV(std::ostream& destination, const RamDomain* tuple) {-        writeNextTupleElement(destination, typeAttributes.at(0), tuple[0]);--        for (size_t col = 1; col < arity; ++col) {-            destination << delimiter;-            writeNextTupleElement(destination, typeAttributes.at(col), tuple[col]);-        }--        destination << "\n";-    }--    void writeNextTupleElement(std::ostream& destination, const std::string& type, RamDomain value) {-        switch (type[0]) {-            case 's': destination << symbolTable.unsafeResolve(value); break;-            case 'i': destination << value; break;-            case 'u': destination << ramBitCast<RamUnsigned>(value); break;-            case 'f': destination << ramBitCast<RamFloat>(value); break;-            case 'r': outputRecord(destination, value, type); break;-            default: fatal("unsupported type attribute: `%c`", type[0]);-        }-    }-};--class WriteFileCSV : public WriteStreamCSV {-public:-    WriteFileCSV(const std::map<std::string, std::string>& rwOperation, const SymbolTable& symbolTable,-            const RecordTable& recordTable)-            : WriteStreamCSV(rwOperation, symbolTable, recordTable),-              file(getFileName(rwOperation), std::ios::out | std::ios::binary) {-        if (getOr(rwOperation, "headers", "false") == "true") {-            file << rwOperation.at("attributeNames") << std::endl;-        }-        file << std::setprecision(std::numeric_limits<RamFloat>::max_digits10);-    }--    ~WriteFileCSV() override = default;--protected:-    std::ofstream file;--    void writeNullary() override {-        file << "()\n";-    }--    void writeNextTuple(const RamDomain* tuple) override {-        writeNextTupleCSV(file, tuple);-    }--    /**-     * Return given filename or construct from relation name.-     * Default name is [configured path]/[relation name].csv-     *-     * @param rwOperation map of IO configuration options-     * @return input filename-     */-    static std::string getFileName(const std::map<std::string, std::string>& rwOperation) {-        auto name = getOr(rwOperation, "filename", rwOperation.at("name") + ".csv");-        if (name.front() != '/') {-            name = getOr(rwOperation, "output-dir", ".") + "/" + name;-        }-        return name;-    }-};--#ifdef USE_LIBZ-class WriteGZipFileCSV : public WriteStreamCSV {-public:-    WriteGZipFileCSV(const std::map<std::string, std::string>& rwOperation, const SymbolTable& symbolTable,-            const RecordTable& recordTable)-            : WriteStreamCSV(rwOperation, symbolTable, recordTable),-              file(getFileName(rwOperation), std::ios::out | std::ios::binary) {-        if (getOr(rwOperation, "headers", "false") == "true") {-            file << rwOperation.at("attributeNames") << std::endl;-        }-        file << std::setprecision(std::numeric_limits<RamFloat>::max_digits10);-    }--    ~WriteGZipFileCSV() override = default;--protected:-    void writeNullary() override {-        file << "()\n";-    }--    void writeNextTuple(const RamDomain* tuple) override {-        writeNextTupleCSV(file, tuple);-    }--    /**-     * Return given filename or construct from relation name.-     * Default name is [configured path]/[relation name].csv-     *-     * @param rwOperation map of IO configuration options-     * @return input filename-     */-    static std::string getFileName(const std::map<std::string, std::string>& rwOperation) {-        auto name = getOr(rwOperation, "filename", rwOperation.at("name") + ".csv.gz");-        if (name.front() != '/') {-            name = getOr(rwOperation, "output-dir", ".") + "/" + name;-        }-        return name;-    }--    gzfstream::ogzfstream file;-};-#endif--class WriteCoutCSV : public WriteStreamCSV {-public:-    WriteCoutCSV(const std::map<std::string, std::string>& rwOperation, const SymbolTable& symbolTable,-            const RecordTable& recordTable)-            : WriteStreamCSV(rwOperation, symbolTable, recordTable) {-        std::cout << "---------------\n" << rwOperation.at("name");-        if (getOr(rwOperation, "headers", "false") == "true") {-            std::cout << "\n" << rwOperation.at("attributeNames");-        }-        std::cout << "\n===============\n";-        std::cout << std::setprecision(std::numeric_limits<RamFloat>::max_digits10);-    }--    ~WriteCoutCSV() override {-        std::cout << "===============\n";-    }--protected:-    void writeNullary() override {-        std::cout << "()\n";-    }--    void writeNextTuple(const RamDomain* tuple) override {-        writeNextTupleCSV(std::cout, tuple);-    }-};--class WriteCoutPrintSize : public WriteStream {-public:-    explicit WriteCoutPrintSize(const std::map<std::string, std::string>& rwOperation)-            : WriteStream(rwOperation, {}, {}), lease(souffle::getOutputLock().acquire()) {-        std::cout << rwOperation.at("name") << "\t";-    }--    ~WriteCoutPrintSize() override = default;--protected:-    void writeNullary() override {-        fatal("attempting to iterate over a print size operation");-    }--    void writeNextTuple(const RamDomain* /* tuple */) override {-        fatal("attempting to iterate over a print size operation");-    }--    void writeSize(std::size_t size) override {-        std::cout << size << "\n";-    }--    Lock::Lease lease;-};--class WriteFileCSVFactory : public WriteStreamFactory {-public:-    std::unique_ptr<WriteStream> getWriter(const std::map<std::string, std::string>& rwOperation,-            const SymbolTable& symbolTable, const RecordTable& recordTable) override {-#ifdef USE_LIBZ-        if (contains(rwOperation, "compress")) {-            return std::make_unique<WriteGZipFileCSV>(rwOperation, symbolTable, recordTable);-        }-#endif-        return std::make_unique<WriteFileCSV>(rwOperation, symbolTable, recordTable);-    }-    const std::string& getName() const override {-        static const std::string name = "file";-        return name;-    }-    ~WriteFileCSVFactory() override = default;-};--class WriteCoutCSVFactory : public WriteStreamFactory {-public:-    std::unique_ptr<WriteStream> getWriter(const std::map<std::string, std::string>& rwOperation,-            const SymbolTable& symbolTable, const RecordTable& recordTable) override {-        return std::make_unique<WriteCoutCSV>(rwOperation, symbolTable, recordTable);-    }--    const std::string& getName() const override {-        static const std::string name = "stdout";-        return name;-    }-    ~WriteCoutCSVFactory() override = default;-};--class WriteCoutPrintSizeFactory : public WriteStreamFactory {-public:-    std::unique_ptr<WriteStream> getWriter(const std::map<std::string, std::string>& rwOperation,-            const SymbolTable&, const RecordTable&) override {-        return std::make_unique<WriteCoutPrintSize>(rwOperation);-    }-    const std::string& getName() const override {-        static const std::string name = "stdoutprintsize";-        return name;-    }-    ~WriteCoutPrintSizeFactory() override = default;-};--} /* namespace souffle */
− cbits/souffle/WriteStreamJSON.h 
@@ -1,298 +0,0 @@-/*- * Souffle - A Datalog Compiler- * Copyright (c) 2020, The Souffle Developers. All rights reserved- * Licensed under the Universal Permissive License v 1.0 as shown at:- * - https://opensource.org/licenses/UPL- * - <souffle root>/licenses/SOUFFLE-UPL.txt- */--/************************************************************************- *- * @file WriteStreamJSON.h- *- ***********************************************************************/--#pragma once--#include "RamTypes.h"-#include "SymbolTable.h"-#include "WriteStream.h"-#include "json11.h"-#include "utility/ContainerUtil.h"--#include <map>-#include <ostream>-#include <queue>-#include <stack>-#include <string>-#include <variant>-#include <vector>--namespace souffle {--class WriteStreamJSON : public WriteStream {-protected:-    WriteStreamJSON(const std::map<std::string, std::string>& rwOperation, const SymbolTable& symbolTable,-            const RecordTable& recordTable)-            : WriteStream(rwOperation, symbolTable, recordTable),-              useObjects(getOr(rwOperation, "format", "list") == "object") {-        if (useObjects) {-            std::string err;-            params = Json::parse(rwOperation.at("params"), err);-            if (err.length() > 0) {-                fatal("cannot get internal param names: %s", err);-            }-        }-    };--    const bool useObjects;-    Json params;--    void writeNextTupleJSON(std::ostream& destination, const RamDomain* tuple) {-        std::vector<Json> result;--        if (useObjects)-            destination << "{";-        else-            destination << "[";--        for (size_t col = 0; col < arity; ++col) {-            if (col > 0) {-                destination << ", ";-            }--            if (useObjects) {-                destination << params["relation"]["params"][col].dump() << ": ";-                writeNextTupleObject(destination, typeAttributes.at(col), tuple[col]);-            } else {-                writeNextTupleList(destination, typeAttributes.at(col), tuple[col]);-            }-        }--        if (useObjects)-            destination << "}";-        else-            destination << "]";-    }--    void writeNextTupleList(std::ostream& destination, const std::string& name, const RamDomain value) {-        using ValueTuple = std::pair<const std::string, const RamDomain>;-        std::stack<std::variant<ValueTuple, std::string>> worklist;-        worklist.push(std::make_pair(name, value));--        // the Json11 output is not tail recursive, therefore highly inefficient for recursive record-        // in addition the JSON object is immutable, so has memory overhead-        while (!worklist.empty()) {-            std::variant<ValueTuple, std::string> curr = worklist.top();-            worklist.pop();--            if (std::holds_alternative<std::string>(curr)) {-                destination << std::get<std::string>(curr);-                continue;-            }--            const std::string& currType = std::get<ValueTuple>(curr).first;-            const RamDomain currValue = std::get<ValueTuple>(curr).second;-            assert(currType.length() > 2 && "Invalid type length");-            switch (currType[0]) {-                // since some strings may need to be escaped, we use dump here-                case 's': destination << Json(symbolTable.unsafeResolve(currValue)).dump(); break;-                case 'i': destination << currValue; break;-                case 'u': destination << (int)ramBitCast<RamUnsigned>(currValue); break;-                case 'f': destination << ramBitCast<RamFloat>(currValue); break;-                case 'r': {-                    auto&& recordInfo = types["records"][currType];-                    assert(!recordInfo.is_null() && "Missing record type information");-                    if (currValue == 0) {-                        destination << "null";-                        break;-                    }--                    auto&& recordTypes = recordInfo["types"];-                    const size_t recordArity = recordInfo["arity"].long_value();-                    const RamDomain* tuplePtr = recordTable.unpack(currValue, recordArity);-                    worklist.push("]");-                    for (auto i = (long long)(recordArity - 1); i >= 0; --i) {-                        if (i != (long long)(recordArity - 1)) {-                            worklist.push(", ");-                        }-                        const std::string& recordType = recordTypes[i].string_value();-                        const RamDomain recordValue = tuplePtr[i];-                        worklist.push(std::make_pair(recordType, recordValue));-                    }--                    worklist.push("[");-                    break;-                }-                default: fatal("unsupported type attribute: `%c`", currType[0]);-            }-        }-    }--    void writeNextTupleObject(std::ostream& destination, const std::string& name, const RamDomain value) {-        using ValueTuple = std::pair<const std::string, const RamDomain>;-        std::stack<std::variant<ValueTuple, std::string>> worklist;-        worklist.push(std::make_pair(name, value));--        // the Json11 output is not tail recursive, therefore highly inefficient for recursive record-        // in addition the JSON object is immutable, so has memory overhead-        while (!worklist.empty()) {-            std::variant<ValueTuple, std::string> curr = worklist.top();-            worklist.pop();--            if (std::holds_alternative<std::string>(curr)) {-                destination << std::get<std::string>(curr);-                continue;-            }--            const std::string& currType = std::get<ValueTuple>(curr).first;-            const RamDomain currValue = std::get<ValueTuple>(curr).second;-            const std::string& typeName = currType.substr(2);-            assert(currType.length() > 2 && "Invalid type length");-            switch (currType[0]) {-                // since some strings may need to be escaped, we use dump here-                case 's': destination << Json(symbolTable.unsafeResolve(currValue)).dump(); break;-                case 'i': destination << currValue; break;-                case 'u': destination << (int)ramBitCast<RamUnsigned>(currValue); break;-                case 'f': destination << ramBitCast<RamFloat>(currValue); break;-                case 'r': {-                    auto&& recordInfo = types["records"][currType];-                    assert(!recordInfo.is_null() && "Missing record type information");-                    if (currValue == 0) {-                        destination << "null";-                        break;-                    }--                    auto&& recordTypes = recordInfo["types"];-                    const size_t recordArity = recordInfo["arity"].long_value();-                    const RamDomain* tuplePtr = recordTable.unpack(currValue, recordArity);-                    worklist.push("}");-                    for (auto i = (long long)(recordArity - 1); i >= 0; --i) {-                        if (i != (long long)(recordArity - 1)) {-                            worklist.push(", ");-                        }-                        const std::string& recordType = recordTypes[i].string_value();-                        const RamDomain recordValue = tuplePtr[i];-                        worklist.push(std::make_pair(recordType, recordValue));-                        worklist.push(": ");--                        auto&& recordParam = params["records"][typeName]["params"][i];-                        assert(recordParam.is_string());-                        worklist.push(recordParam.dump());-                    }--                    worklist.push("{");-                    break;-                }-                default: fatal("unsupported type attribute: `%c`", currType[0]);-            }-        }-    }-};--class WriteFileJSON : public WriteStreamJSON {-public:-    WriteFileJSON(const std::map<std::string, std::string>& rwOperation, const SymbolTable& symbolTable,-            const RecordTable& recordTable)-            : WriteStreamJSON(rwOperation, symbolTable, recordTable), isFirst(true),-              file(getFileName(rwOperation), std::ios::out | std::ios::binary) {-        file << "[";-    }--    ~WriteFileJSON() override {-        file << "]\n";-        file.close();-    }--protected:-    bool isFirst;-    std::ofstream file;--    void writeNullary() override {-        file << "null\n";-    }--    void writeNextTuple(const RamDomain* tuple) override {-        if (!isFirst) {-            file << ",\n";-        } else {-            isFirst = false;-        }-        writeNextTupleJSON(file, tuple);-    }--    /**-     * Return given filename or construct from relation name.-     * Default name is [configured path]/[relation name].json-     *-     * @param rwOperation map of IO configuration options-     * @return input filename-     */-    static std::string getFileName(const std::map<std::string, std::string>& rwOperation) {-        auto name = getOr(rwOperation, "filename", rwOperation.at("name") + ".json");-        if (name.front() != '/') {-            name = getOr(rwOperation, "output-dir", ".") + "/" + name;-        }-        return name;-    }-};--class WriteCoutJSON : public WriteStreamJSON {-public:-    WriteCoutJSON(const std::map<std::string, std::string>& rwOperation, const SymbolTable& symbolTable,-            const RecordTable& recordTable)-            : WriteStreamJSON(rwOperation, symbolTable, recordTable), isFirst(true) {-        std::cout << "[";-    }--    ~WriteCoutJSON() override {-        std::cout << "]\n";-    };--protected:-    bool isFirst;--    void writeNullary() override {-        std::cout << "null\n";-    }--    void writeNextTuple(const RamDomain* tuple) override {-        if (!isFirst) {-            std::cout << ",\n";-        } else {-            isFirst = false;-        }-        writeNextTupleJSON(std::cout, tuple);-    }-};--class WriteFileJSONFactory : public WriteStreamFactory {-public:-    std::unique_ptr<WriteStream> getWriter(const std::map<std::string, std::string>& rwOperation,-            const SymbolTable& symbolTable, const RecordTable& recordTable) override {-        return std::make_unique<WriteFileJSON>(rwOperation, symbolTable, recordTable);-    }--    const std::string& getName() const override {-        static const std::string name = "jsonfile";-        return name;-    }--    ~WriteFileJSONFactory() override = default;-};--class WriteCoutJSONFactory : public WriteStreamFactory {-public:-    std::unique_ptr<WriteStream> getWriter(const std::map<std::string, std::string>& rwOperation,-            const SymbolTable& symbolTable, const RecordTable& recordTable) override {-        return std::make_unique<WriteCoutJSON>(rwOperation, symbolTable, recordTable);-    }--    const std::string& getName() const override {-        static const std::string name = "json";-        return name;-    }--    ~WriteCoutJSONFactory() override = default;-};-}  // namespace souffle
− cbits/souffle/WriteStreamSQLite.h 
@@ -1,297 +0,0 @@-/*- * Souffle - A Datalog Compiler- * Copyright (c) 2013, 2014, Oracle and/or its affiliates. All rights reserved- * Licensed under the Universal Permissive License v 1.0 as shown at:- * - https://opensource.org/licenses/UPL- * - <souffle root>/licenses/SOUFFLE-UPL.txt- */--/************************************************************************- *- * @file WriteStreamSQLite.h- *- ***********************************************************************/--#pragma once--#include "RamTypes.h"-#include "SymbolTable.h"-#include "WriteStream.h"-#include <cassert>-#include <cstddef>-#include <cstdint>-#include <map>-#include <memory>-#include <sstream>-#include <stdexcept>-#include <string>-#include <unordered_map>-#include <vector>-#include <sqlite3.h>--namespace souffle {--class RecordTable;--class WriteStreamSQLite : public WriteStream {-public:-    WriteStreamSQLite(const std::map<std::string, std::string>& rwOperation, const SymbolTable& symbolTable,-            const RecordTable& recordTable)-            : WriteStream(rwOperation, symbolTable, recordTable), dbFilename(getFileName(rwOperation)),-              relationName(rwOperation.at("name")) {-        openDB();-        createTables();-        prepareStatements();-        //        executeSQL("BEGIN TRANSACTION", db);-    }--    ~WriteStreamSQLite() override {-        sqlite3_finalize(insertStatement);-        sqlite3_finalize(symbolInsertStatement);-        sqlite3_finalize(symbolSelectStatement);-        sqlite3_close(db);-    }--protected:-    void writeNullary() override {}--    void writeNextTuple(const RamDomain* tuple) override {-        for (size_t i = 0; i < arity; i++) {-            RamDomain value = 0;  // Silence warning--            switch (typeAttributes.at(i)[0]) {-                case 's': value = getSymbolTableID(tuple[i]); break;-                default: value = tuple[i]; break;-            }--#if RAM_DOMAIN_SIZE == 64-            if (sqlite3_bind_int64(insertStatement, i + 1, value) != SQLITE_OK) {-#else-            if (sqlite3_bind_int(insertStatement, i + 1, value) != SQLITE_OK) {-#endif-                throwError("SQLite error in sqlite3_bind_text: ");-            }-        }-        if (sqlite3_step(insertStatement) != SQLITE_DONE) {-            throwError("SQLite error in sqlite3_step: ");-        }-        sqlite3_clear_bindings(insertStatement);-        sqlite3_reset(insertStatement);-    }--private:-    void executeSQL(const std::string& sql, sqlite3* db) {-        assert(db && "Database connection is closed");--        char* errorMessage = nullptr;-        /* Execute SQL statement */-        int rc = sqlite3_exec(db, sql.c_str(), nullptr, nullptr, &errorMessage);-        if (rc != SQLITE_OK) {-            std::stringstream error;-            error << "SQLite error in sqlite3_exec: " << sqlite3_errmsg(db) << "\n";-            error << "SQL error: " << errorMessage << "\n";-            error << "SQL: " << sql << "\n";-            sqlite3_free(errorMessage);-            throw std::invalid_argument(error.str());-        }-    }--    void throwError(const std::string& message) {-        std::stringstream error;-        error << message << sqlite3_errmsg(db) << "\n";-        throw std::invalid_argument(error.str());-    }--    uint64_t getSymbolTableIDFromDB(int index) {-        if (sqlite3_bind_text(symbolSelectStatement, 1, symbolTable.unsafeResolve(index).c_str(), -1,-                    SQLITE_TRANSIENT) != SQLITE_OK) {-            throwError("SQLite error in sqlite3_bind_text: ");-        }-        if (sqlite3_step(symbolSelectStatement) != SQLITE_ROW) {-            throwError("SQLite error in sqlite3_step: ");-        }-        uint64_t rowid = sqlite3_column_int64(symbolSelectStatement, 0);-        sqlite3_clear_bindings(symbolSelectStatement);-        sqlite3_reset(symbolSelectStatement);-        return rowid;-    }-    uint64_t getSymbolTableID(int index) {-        if (dbSymbolTable.count(index) != 0) {-            return dbSymbolTable[index];-        }--        if (sqlite3_bind_text(symbolInsertStatement, 1, symbolTable.unsafeResolve(index).c_str(), -1,-                    SQLITE_TRANSIENT) != SQLITE_OK) {-            throwError("SQLite error in sqlite3_bind_text: ");-        }-        // Either the insert succeeds and we have a new row id or it already exists and a select is needed.-        uint64_t rowid;-        if (sqlite3_step(symbolInsertStatement) != SQLITE_DONE) {-            // The symbol already exists so select it.-            rowid = getSymbolTableIDFromDB(index);-        } else {-            rowid = sqlite3_last_insert_rowid(db);-        }-        sqlite3_clear_bindings(symbolInsertStatement);-        sqlite3_reset(symbolInsertStatement);--        dbSymbolTable[index] = rowid;-        return rowid;-    }--    void openDB() {-        if (sqlite3_open(dbFilename.c_str(), &db) != SQLITE_OK) {-            throwError("SQLite error in sqlite3_open");-        }-        sqlite3_extended_result_codes(db, 1);-        executeSQL("PRAGMA synchronous = OFF", db);-        executeSQL("PRAGMA journal_mode = MEMORY", db);-    }--    void prepareStatements() {-        prepareInsertStatement();-        prepareSymbolInsertStatement();-        prepareSymbolSelectStatement();-    }-    void prepareSymbolInsertStatement() {-        std::stringstream insertSQL;-        insertSQL << "INSERT INTO " << symbolTableName;-        insertSQL << " VALUES(null,@V0);";-        const char* tail = nullptr;-        if (sqlite3_prepare_v2(db, insertSQL.str().c_str(), -1, &symbolInsertStatement, &tail) != SQLITE_OK) {-            throwError("SQLite error in sqlite3_prepare_v2: ");-        }-    }--    void prepareSymbolSelectStatement() {-        std::stringstream selectSQL;-        selectSQL << "SELECT id FROM " << symbolTableName;-        selectSQL << " WHERE symbol = @V0;";-        const char* tail = nullptr;-        if (sqlite3_prepare_v2(db, selectSQL.str().c_str(), -1, &symbolSelectStatement, &tail) != SQLITE_OK) {-            throwError("SQLite error in sqlite3_prepare_v2: ");-        }-    }--    void prepareInsertStatement() {-        std::stringstream insertSQL;-        insertSQL << "INSERT INTO '_" << relationName << "' VALUES ";-        insertSQL << "(@V0";-        for (unsigned int i = 1; i < arity; i++) {-            insertSQL << ",@V" << i;-        }-        insertSQL << ");";-        const char* tail = nullptr;-        if (sqlite3_prepare_v2(db, insertSQL.str().c_str(), -1, &insertStatement, &tail) != SQLITE_OK) {-            throwError("SQLite error in sqlite3_prepare_v2: ");-        }-    }--    void createTables() {-        createRelationTable();-        createRelationView();-        createSymbolTable();-    }--    void createRelationTable() {-        std::stringstream createTableText;-        createTableText << "CREATE TABLE IF NOT EXISTS '_" << relationName << "' (";-        if (arity > 0) {-            createTableText << "'0' INTEGER";-            for (unsigned int i = 1; i < arity; i++) {-                createTableText << ",'" << std::to_string(i) << "' ";-                createTableText << "INTEGER";-            }-        }-        createTableText << ");";-        executeSQL(createTableText.str(), db);-        executeSQL("DELETE FROM '_" + relationName + "';", db);-    }--    void createRelationView() {-        // Create view with symbol strings resolved-        std::stringstream createViewText;-        createViewText << "CREATE VIEW IF NOT EXISTS '" << relationName << "' AS ";-        std::stringstream projectionClause;-        std::stringstream fromClause;-        fromClause << "'_" << relationName << "'";-        std::stringstream whereClause;-        bool firstWhere = true;-        for (unsigned int i = 0; i < arity; i++) {-            std::string columnName = std::to_string(i);-            if (i != 0) {-                projectionClause << ",";-            }-            if (typeAttributes.at(i)[0] == 's') {-                projectionClause << "'_symtab_" << columnName << "'.symbol AS '" << columnName << "'";-                fromClause << ",'" << symbolTableName << "' AS '_symtab_" << columnName << "'";-                if (!firstWhere) {-                    whereClause << " AND ";-                } else {-                    firstWhere = false;-                }-                whereClause << "'_" << relationName << "'.'" << columnName << "' = "-                            << "'_symtab_" << columnName << "'.id";-            } else {-                projectionClause << "'_" << relationName << "'.'" << columnName << "'";-            }-        }-        createViewText << "SELECT " << projectionClause.str() << " FROM " << fromClause.str();-        if (!firstWhere) {-            createViewText << " WHERE " << whereClause.str();-        }-        createViewText << ";";-        executeSQL(createViewText.str(), db);-    }-    void createSymbolTable() {-        std::stringstream createTableText;-        createTableText << "CREATE TABLE IF NOT EXISTS '" << symbolTableName << "' ";-        createTableText << "(id INTEGER PRIMARY KEY, symbol TEXT UNIQUE);";-        executeSQL(createTableText.str(), db);-    }--    /**-     * Return given filename or construct from relation name.-     * Default name is [configured path]/[relation name].sqlite-     *-     * @param rwOperation map of IO configuration options-     * @return input filename-     */-    static std::string getFileName(const std::map<std::string, std::string>& rwOperation) {-        // legacy support for SQLite prior to 2020-03-18-        // convert dbname to filename-        auto name = getOr(rwOperation, "dbname", rwOperation.at("name") + ".sqlite");-        name = getOr(rwOperation, "filename", name);--        if (name.front() != '/') {-            name = getOr(rwOperation, "output-dir", ".") + "/" + name;-        }-        return name;-    }--    const std::string dbFilename;-    const std::string relationName;-    const std::string symbolTableName = "__SymbolTable";--    std::unordered_map<uint64_t, uint64_t> dbSymbolTable;-    sqlite3_stmt* insertStatement = nullptr;-    sqlite3_stmt* symbolInsertStatement = nullptr;-    sqlite3_stmt* symbolSelectStatement = nullptr;-    sqlite3* db = nullptr;-};--class WriteSQLiteFactory : public WriteStreamFactory {-public:-    std::unique_ptr<WriteStream> getWriter(const std::map<std::string, std::string>& rwOperation,-            const SymbolTable& symbolTable, const RecordTable& recordTable) override {-        return std::make_unique<WriteStreamSQLite>(rwOperation, symbolTable, recordTable);-    }--    const std::string& getName() const override {-        static const std::string name = "sqlite";-        return name;-    }-    ~WriteSQLiteFactory() override = default;-};--} /* namespace souffle */
+ cbits/souffle/datastructure/BTree.h view
@@ -0,0 +1,2344 @@+/*+ * Souffle - A Datalog Compiler+ * Copyright (c) 2013, 2015, Oracle and/or its affiliates. All rights reserved+ * Licensed under the Universal Permissive License v 1.0 as shown at:+ * - https://opensource.org/licenses/UPL+ * - <souffle root>/licenses/SOUFFLE-UPL.txt+ */++/************************************************************************+ *+ * @file BTree.h+ *+ * An implementation of a generic B-tree data structure including+ * interfaces for utilizing instances as set or multiset containers.+ *+ ***********************************************************************/++#pragma once++#include "souffle/utility/CacheUtil.h"+#include "souffle/utility/ContainerUtil.h"+#include "souffle/utility/ParallelUtil.h"+#include <algorithm>+#include <cassert>+#include <cstddef>+#include <cstdint>+#include <iostream>+#include <iterator>+#include <string>+#include <tuple>+#include <type_traits>+#include <typeinfo>+#include <vector>++namespace souffle {++namespace detail {++// ---------- comparators --------------++/**+ * A generic comparator implementation as it is used by+ * a b-tree based on types that can be less-than and+ * equality comparable.+ */+template <typename T>+struct comparator {+    /**+     * Compares the values of a and b and returns+     * -1 if a<b, 1 if a>b and 0 otherwise+     */+    int operator()(const T& a, const T& b) const {+        return (a > b) - (a < b);+    }+    bool less(const T& a, const T& b) const {+        return a < b;+    }+    bool equal(const T& a, const T& b) const {+        return a == b;+    }+};++// ---------- search strategies --------------++/**+ * A common base class for search strategies in b-trees.+ */+struct search_strategy {};++/**+ * A linear search strategy for looking up keys in b-tree nodes.+ */+struct linear_search : public search_strategy {+    /**+     * Required user-defined default constructor.+     */+    linear_search() = default;++    /**+     * Obtains an iterator referencing an element equivalent to the+     * given key in the given range. If no such element is present,+     * a reference to the first element not less than the given key+     * is returned.+     */+    template <typename Key, typename Iter, typename Comp>+    inline Iter operator()(const Key& k, Iter a, Iter b, Comp& comp) const {+        return lower_bound(k, a, b, comp);+    }++    /**+     * Obtains a reference to the first element in the given range that+     * is not less than the given key.+     */+    template <typename Key, typename Iter, typename Comp>+    inline Iter lower_bound(const Key& k, Iter a, Iter b, Comp& comp) const {+        auto c = a;+        while (c < b) {+            auto r = comp(*c, k);+            if (r >= 0) {+                return c;+            }+            ++c;+        }+        return b;+    }++    /**+     * Obtains a reference to the first element in the given range that+     * such that the given key is less than the referenced element.+     */+    template <typename Key, typename Iter, typename Comp>+    inline Iter upper_bound(const Key& k, Iter a, Iter b, Comp& comp) const {+        auto c = a;+        while (c < b) {+            if (comp(*c, k) > 0) {+                return c;+            }+            ++c;+        }+        return b;+    }+};++/**+ * A binary search strategy for looking up keys in b-tree nodes.+ */+struct binary_search : public search_strategy {+    /**+     * Required user-defined default constructor.+     */+    binary_search() = default;++    /**+     * Obtains an iterator pointing to some element within the given+     * range that is equal to the given key, if available. If multiple+     * elements are equal to the given key, an undefined instance will+     * be obtained (no guaranteed lower or upper boundary).  If no such+     * element is present, a reference to the first element not less than+     * the given key will be returned.+     */+    template <typename Key, typename Iter, typename Comp>+    Iter operator()(const Key& k, Iter a, Iter b, Comp& comp) const {+        Iter c;+        auto count = b - a;+        while (count > 0) {+            auto step = count >> 1;+            c = a + step;+            auto r = comp(*c, k);+            if (r == 0) {+                return c;+            }+            if (r < 0) {+                a = ++c;+                count -= step + 1;+            } else {+                count = step;+            }+        }+        return a;+    }++    /**+     * Obtains a reference to the first element in the given range that+     * is not less than the given key.+     */+    template <typename Key, typename Iter, typename Comp>+    Iter lower_bound(const Key& k, Iter a, Iter b, Comp& comp) const {+        Iter c;+        auto count = b - a;+        while (count > 0) {+            auto step = count >> 1;+            c = a + step;+            if (comp(*c, k) < 0) {+                a = ++c;+                count -= step + 1;+            } else {+                count = step;+            }+        }+        return a;+    }++    /**+     * Obtains a reference to the first element in the given range that+     * such that the given key is less than the referenced element.+     */+    template <typename Key, typename Iter, typename Comp>+    Iter upper_bound(const Key& k, Iter a, Iter b, Comp& comp) const {+        Iter c;+        auto count = b - a;+        while (count > 0) {+            auto step = count >> 1;+            c = a + step;+            if (comp(k, *c) >= 0) {+                a = ++c;+                count -= step + 1;+            } else {+                count = step;+            }+        }+        return a;+    }+};++// ---------- search strategies selection --------------++/**+ * A template-meta class to select search strategies for b-trees+ * depending on the key type.+ */+template <typename S>+struct strategy_selection {+    using type = S;+};++struct linear : public strategy_selection<linear_search> {};+struct binary : public strategy_selection<binary_search> {};++// by default every key utilizes binary search+template <typename Key>+struct default_strategy : public binary {};++template <>+struct default_strategy<int> : public linear {};++template <typename... Ts>+struct default_strategy<std::tuple<Ts...>> : public linear {};++/**+ * The default non-updater+ */+template <typename T>+struct updater {+    void update(T& /* old_t */, const T& /* new_t */) {}+};++/**+ * The actual implementation of a b-tree data structure.+ *+ * @tparam Key             .. the element type to be stored in this tree+ * @tparam Comparator     .. a class defining an order on the stored elements+ * @tparam Allocator     .. utilized for allocating memory for required nodes+ * @tparam blockSize    .. determines the number of bytes/block utilized by leaf nodes+ * @tparam SearchStrategy .. enables switching between linear, binary or any other search strategy+ * @tparam isSet        .. true = set, false = multiset+ */+template <typename Key, typename Comparator,+        typename Allocator,  // is ignored so far - TODO: add support+        unsigned blockSize, typename SearchStrategy, bool isSet, typename WeakComparator = Comparator,+        typename Updater = detail::updater<Key>>+class btree {+public:+    class iterator;+    using const_iterator = iterator;++    using key_type = Key;+    using element_type = Key;+    using chunk = range<iterator>;++protected:+    /* ------------- static utilities ----------------- */++    const static SearchStrategy search;++    /* ---------- comparison utilities ---------------- */++    mutable Comparator comp;++    bool less(const Key& a, const Key& b) const {+        return comp.less(a, b);+    }++    bool equal(const Key& a, const Key& b) const {+        return comp.equal(a, b);+    }++    mutable WeakComparator weak_comp;++    bool weak_less(const Key& a, const Key& b) const {+        return weak_comp.less(a, b);+    }++    bool weak_equal(const Key& a, const Key& b) const {+        return weak_comp.equal(a, b);+    }++    /* -------------- updater utilities ------------- */++    mutable Updater upd;+    void update(Key& old_k, const Key& new_k) {+        upd.update(old_k, new_k);+    }++    /* -------------- the node type ----------------- */++    using size_type = std::size_t;+    using field_index_type = uint8_t;+    using lock_type = OptimisticReadWriteLock;++    struct node;++    /**+     * The base type of all node types containing essential+     * book-keeping information.+     */+    struct base {+#ifdef IS_PARALLEL++        // the parent node+        node* volatile parent;++        // a lock for synchronizing parallel operations on this node+        lock_type lock;++        // the number of keys in this node+        volatile size_type numElements;++        // the position in the parent node+        volatile field_index_type position;+#else+        // the parent node+        node* parent;++        // the number of keys in this node+        size_type numElements;++        // the position in the parent node+        field_index_type position;+#endif++        // a flag indicating whether this is a inner node or not+        const bool inner;++        /**+         * A simple constructor for nodes+         */+        base(bool inner) : parent(nullptr), numElements(0), position(0), inner(inner) {}++        bool isLeaf() const {+            return !inner;+        }++        bool isInner() const {+            return inner;+        }++        node* getParent() const {+            return parent;+        }++        field_index_type getPositionInParent() const {+            return position;+        }++        size_type getNumElements() const {+            return numElements;+        }+    };++    struct inner_node;++    /**+     * The actual, generic node implementation covering the operations+     * for both, inner and leaf nodes.+     */+    struct node : public base {+        /**+         * The number of keys/node desired by the user.+         */+        static constexpr size_t desiredNumKeys =+                ((blockSize > sizeof(base)) ? blockSize - sizeof(base) : 0) / sizeof(Key);++        /**+         * The actual number of keys/node corrected by functional requirements.+         */+        static constexpr size_t maxKeys = (desiredNumKeys > 3) ? desiredNumKeys : 3;++        // the keys stored in this node+        Key keys[maxKeys];++        // a simple constructor+        node(bool inner) : base(inner) {}++        /**+         * A deep-copy operation creating a clone of this node.+         */+        node* clone() const {+            // create a clone of this node+            node* res = (this->isInner()) ? static_cast<node*>(new inner_node())+                                          : static_cast<node*>(new leaf_node());++            // copy basic fields+            res->position = this->position;+            res->numElements = this->numElements;++            for (size_type i = 0; i < this->numElements; ++i) {+                res->keys[i] = this->keys[i];+            }++            // if this is a leaf we are done+            if (this->isLeaf()) {+                return res;+            }++            // copy child nodes recursively+            auto* ires = (inner_node*)res;+            for (size_type i = 0; i <= this->numElements; ++i) {+                ires->children[i] = this->getChild(i)->clone();+                ires->children[i]->parent = res;+            }++            // that's it+            return res;+        }++        /**+         * A utility function providing a reference to this node as+         * an inner node.+         */+        inner_node& asInnerNode() {+            assert(this->inner && "Invalid cast!");+            return *static_cast<inner_node*>(this);+        }++        /**+         * A utility function providing a reference to this node as+         * a const inner node.+         */+        const inner_node& asInnerNode() const {+            assert(this->inner && "Invalid cast!");+            return *static_cast<const inner_node*>(this);+        }++        /**+         * Computes the number of nested levels of the tree rooted+         * by this node.+         */+        size_type getDepth() const {+            if (this->isLeaf()) {+                return 1;+            }+            return getChild(0)->getDepth() + 1;+        }++        /**+         * Counts the number of nodes contained in the sub-tree rooted+         * by this node.+         */+        size_type countNodes() const {+            if (this->isLeaf()) {+                return 1;+            }+            size_type sum = 1;+            for (unsigned i = 0; i <= this->numElements; ++i) {+                sum += getChild(i)->countNodes();+            }+            return sum;+        }++        /**+         * Counts the number of entries contained in the sub-tree rooted+         * by this node.+         */+        size_type countEntries() const {+            if (this->isLeaf()) {+                return this->numElements;+            }+            size_type sum = this->numElements;+            for (unsigned i = 0; i <= this->numElements; ++i) {+                sum += getChild(i)->countEntries();+            }+            return sum;+        }++        /**+         * Determines the amount of memory used by the sub-tree rooted+         * by this node.+         */+        size_type getMemoryUsage() const {+            if (this->isLeaf()) {+                return sizeof(leaf_node);+            }+            size_type res = sizeof(inner_node);+            for (unsigned i = 0; i <= this->numElements; ++i) {+                res += getChild(i)->getMemoryUsage();+            }+            return res;+        }++        /**+         * Obtains a pointer to the array of child-pointers+         * of this node -- if it is an inner node.+         */+        node** getChildren() {+            return asInnerNode().children;+        }++        /**+         * Obtains a pointer to the array of const child-pointers+         * of this node -- if it is an inner node.+         */+        node* const* getChildren() const {+            return asInnerNode().children;+        }++        /**+         * Obtains a reference to the child of the given index.+         */+        node* getChild(size_type s) const {+            return asInnerNode().children[s];+        }++        /**+         * Checks whether this node is empty -- can happen due to biased insertion.+         */+        bool isEmpty() const {+            return this->numElements == 0;+        }++        /**+         * Checks whether this node is full.+         */+        bool isFull() const {+            return this->numElements == maxKeys;+        }++        /**+         * Obtains the point at which full nodes should be split.+         * Conventional b-trees always split in half. However, in cases+         * where in-order insertions are frequent, a split assigning+         * larger portions to the right fragment provide higher performance+         * and a better node-filling rate.+         */+        int getSplitPoint(int /*unused*/) {+            return static_cast<int>(std::min(3 * maxKeys / 4, maxKeys - 2));+        }++        /**+         * Splits this node.+         *+         * @param root .. a pointer to the root-pointer of the enclosing b-tree+         *                 (might have to be updated if the root-node needs to be split)+         * @param idx  .. the position of the insert causing the split+         */+#ifdef IS_PARALLEL+        void split(node** root, lock_type& root_lock, int idx, std::vector<node*>& locked_nodes) {+            assert(this->lock.is_write_locked());+            assert(!this->parent || this->parent->lock.is_write_locked());+            assert((this->parent != nullptr) || root_lock.is_write_locked());+            assert(this->isLeaf() || souffle::contains(locked_nodes, this));+            assert(!this->parent || souffle::contains(locked_nodes, const_cast<node*>(this->parent)));+#else+        void split(node** root, lock_type& root_lock, int idx) {+#endif+            assert(this->numElements == maxKeys);++            // get middle element+            int split_point = getSplitPoint(idx);++            // create a new sibling node+            node* sibling = (this->inner) ? static_cast<node*>(new inner_node())+                                          : static_cast<node*>(new leaf_node());++#ifdef IS_PARALLEL+            // lock sibling+            sibling->lock.start_write();+            locked_nodes.push_back(sibling);+#endif++            // move data over to the new node+            for (unsigned i = split_point + 1, j = 0; i < maxKeys; ++i, ++j) {+                sibling->keys[j] = keys[i];+            }++            // move child pointers+            if (this->inner) {+                // move pointers to sibling+                auto* other = static_cast<inner_node*>(sibling);+                for (unsigned i = split_point + 1, j = 0; i <= maxKeys; ++i, ++j) {+                    other->children[j] = getChildren()[i];+                    other->children[j]->parent = other;+                    other->children[j]->position = static_cast<field_index_type>(j);+                }+            }++            // update number of elements+            this->numElements = split_point;+            sibling->numElements = maxKeys - split_point - 1;++            // update parent+#ifdef IS_PARALLEL+            grow_parent(root, root_lock, sibling, locked_nodes);+#else+            grow_parent(root, root_lock, sibling);+#endif+        }++        /**+         * Moves keys from this node to one of its siblings or splits+         * this node to make some space for the insertion of an element at+         * position idx.+         *+         * Returns the number of elements moved to the left side, 0 in case+         * of a split. The number of moved elements will be <= the given idx.+         *+         * @param root .. the root node of the b-tree being part of+         * @param idx  .. the position of the insert triggering this operation+         */+        // TODO: remove root_lock ... no longer needed+#ifdef IS_PARALLEL+        int rebalance_or_split(node** root, lock_type& root_lock, int idx, std::vector<node*>& locked_nodes) {+            assert(this->lock.is_write_locked());+            assert(!this->parent || this->parent->lock.is_write_locked());+            assert((this->parent != nullptr) || root_lock.is_write_locked());+            assert(this->isLeaf() || souffle::contains(locked_nodes, this));+            assert(!this->parent || souffle::contains(locked_nodes, const_cast<node*>(this->parent)));+#else+        int rebalance_or_split(node** root, lock_type& root_lock, int idx) {+#endif++            // this node is full ... and needs some space+            assert(this->numElements == maxKeys);++            // get snap-shot of parent+            auto parent = this->parent;+            auto pos = this->position;++            // Option A) re-balance data+            if (parent && pos > 0) {+                node* left = parent->getChild(pos - 1);++#ifdef IS_PARALLEL+                // lock access to left sibling+                if (!left->lock.try_start_write()) {+                    // left node is currently updated => skip balancing and split+                    split(root, root_lock, idx, locked_nodes);+                    return 0;+                }+#endif++                // compute number of elements to be movable to left+                //    space available in left vs. insertion index+                size_type num = static_cast<size_type>(+                        std::min<int>(static_cast<int>(maxKeys - left->numElements), idx));++                // if there are elements to move ..+                if (num > 0) {+                    Key* splitter = &(parent->keys[this->position - 1]);++                    // .. move keys to left node+                    left->keys[left->numElements] = *splitter;+                    for (size_type i = 0; i < num - 1; ++i) {+                        left->keys[left->numElements + 1 + i] = keys[i];+                    }+                    *splitter = keys[num - 1];++                    // shift keys in this node to the left+                    for (size_type i = 0; i < this->numElements - num; ++i) {+                        keys[i] = keys[i + num];+                    }++                    // .. and children if necessary+                    if (this->isInner()) {+                        auto* ileft = static_cast<inner_node*>(left);+                        auto* iright = static_cast<inner_node*>(this);++                        // move children+                        for (field_index_type i = 0; i < num; ++i) {+                            ileft->children[left->numElements + i + 1] = iright->children[i];+                        }++                        // update moved children+                        for (size_type i = 0; i < num; ++i) {+                            iright->children[i]->parent = ileft;+                            iright->children[i]->position =+                                    static_cast<field_index_type>(left->numElements + i) + 1;+                        }++                        // shift child-pointer to the left+                        for (size_type i = 0; i < this->numElements - num + 1; ++i) {+                            iright->children[i] = iright->children[i + num];+                        }++                        // update position of children+                        for (size_type i = 0; i < this->numElements - num + 1; ++i) {+                            iright->children[i]->position = static_cast<field_index_type>(i);+                        }+                    }++                    // update node sizes+                    left->numElements += num;+                    this->numElements -= num;++#ifdef IS_PARALLEL+                    left->lock.end_write();+#endif++                    // done+                    return static_cast<int>(num);+                }++#ifdef IS_PARALLEL+                left->lock.abort_write();+#endif+            }++            // Option B) split node+#ifdef IS_PARALLEL+            split(root, root_lock, idx, locked_nodes);+#else+            split(root, root_lock, idx);+#endif+            return 0;  // = no re-balancing+        }++    private:+        /**+         * Inserts a new sibling into the parent of this node utilizing+         * the last key of this node as a separation key. (for internal+         * use only)+         *+         * @param root .. a pointer to the root-pointer of the containing tree+         * @param sibling .. the new right-sibling to be add to the parent node+         */+#ifdef IS_PARALLEL+        void grow_parent(node** root, lock_type& root_lock, node* sibling, std::vector<node*>& locked_nodes) {+            assert(this->lock.is_write_locked());+            assert(!this->parent || this->parent->lock.is_write_locked());+            assert((this->parent != nullptr) || root_lock.is_write_locked());+            assert(this->isLeaf() || souffle::contains(locked_nodes, this));+            assert(!this->parent || souffle::contains(locked_nodes, const_cast<node*>(this->parent)));+#else+        void grow_parent(node** root, lock_type& root_lock, node* sibling) {+#endif++            if (this->parent == nullptr) {+                assert(*root == this);++                // create a new root node+                auto* new_root = new inner_node();+                new_root->numElements = 1;+                new_root->keys[0] = keys[this->numElements];++                new_root->children[0] = this;+                new_root->children[1] = sibling;++                // link this and the sibling node to new root+                this->parent = new_root;+                sibling->parent = new_root;+                sibling->position = 1;++                // switch root node+                *root = new_root;++            } else {+                // insert new element in parent element+                auto parent = this->parent;+                auto pos = this->position;++#ifdef IS_PARALLEL+                parent->insert_inner(+                        root, root_lock, pos, this, keys[this->numElements], sibling, locked_nodes);+#else+                parent->insert_inner(root, root_lock, pos, this, keys[this->numElements], sibling);+#endif+            }+        }++        /**+         * Inserts a new element into an inner node (for internal use only).+         *+         * @param root .. a pointer to the root-pointer of the containing tree+         * @param pos  .. the position to insert the new key+         * @param key  .. the key to insert+         * @param newNode .. the new right-child of the inserted key+         */+#ifdef IS_PARALLEL+        void insert_inner(node** root, lock_type& root_lock, unsigned pos, node* predecessor, const Key& key,+                node* newNode, std::vector<node*>& locked_nodes) {+            assert(this->lock.is_write_locked());+            assert(souffle::contains(locked_nodes, this));+#else+        void insert_inner(node** root, lock_type& root_lock, unsigned pos, node* predecessor, const Key& key,+                node* newNode) {+#endif++            // check capacity+            if (this->numElements >= maxKeys) {+#ifdef IS_PARALLEL+                assert(!this->parent || this->parent->lock.is_write_locked());+                assert((this->parent) || root_lock.is_write_locked());+                assert(!this->parent || souffle::contains(locked_nodes, const_cast<node*>(this->parent)));+#endif++                // split this node+#ifdef IS_PARALLEL+                pos -= rebalance_or_split(root, root_lock, pos, locked_nodes);+#else+                pos -= rebalance_or_split(root, root_lock, pos);+#endif++                // complete insertion within new sibling if necessary+                if (pos > this->numElements) {+                    // correct position+                    pos = pos - static_cast<unsigned int>(this->numElements) - 1;++                    // get new sibling+                    auto other = this->parent->getChild(this->position + 1);++#ifdef IS_PARALLEL+                    // make sure other side is write locked+                    assert(other->lock.is_write_locked());+                    assert(souffle::contains(locked_nodes, other));++                    // search for new position (since other may have been altered in the meanwhile)+                    size_type i = 0;+                    for (; i <= other->numElements; ++i) {+                        if (other->getChild(i) == predecessor) {+                            break;+                        }+                    }++                    pos = (i > other->numElements) ? 0 : i;+                    other->insert_inner(root, root_lock, pos, predecessor, key, newNode, locked_nodes);+#else+                    other->insert_inner(root, root_lock, pos, predecessor, key, newNode);+#endif+                    return;+                }+            }++            // move bigger keys one forward+            for (int i = static_cast<int>(this->numElements) - 1; i >= (int)pos; --i) {+                keys[i + 1] = keys[i];+                getChildren()[i + 2] = getChildren()[i + 1];+                ++getChildren()[i + 2]->position;+            }++            // ensure proper position+            assert(getChild(pos) == predecessor);++            // insert new element+            keys[pos] = key;+            getChildren()[pos + 1] = newNode;+            newNode->parent = this;+            newNode->position = static_cast<field_index_type>(pos) + 1;+            ++this->numElements;+        }++    public:+        /**+         * Prints a textual representation of this tree to the given output stream.+         * This feature is mainly intended for debugging and tuning purposes.+         *+         * @see btree::printTree+         */+        void printTree(std::ostream& out, const std::string& prefix) const {+            // print the header+            out << prefix << "@" << this << "[" << ((int)(this->position)) << "] - "+                << (this->inner ? "i" : "") << "node : " << this->numElements << "/" << maxKeys << " [";++            // print the keys+            for (unsigned i = 0; i < this->numElements; i++) {+                out << keys[i];+                if (i != this->numElements - 1) {+                    out << ",";+                }+            }+            out << "]";++            // print references to children+            if (this->inner) {+                out << " - [";+                for (unsigned i = 0; i <= this->numElements; i++) {+                    out << getChildren()[i];+                    if (i != this->numElements) {+                        out << ",";+                    }+                }+                out << "]";+            }++#ifdef IS_PARALLEL+            // print the lock state+            if (this->lock.is_write_locked()) {+                std::cout << " locked";+            }+#endif++            out << "\n";++            // print the children recursively+            if (this->inner) {+                for (unsigned i = 0; i < this->numElements + 1; ++i) {+                    static_cast<const inner_node*>(this)->children[i]->printTree(out, prefix + "    ");+                }+            }+        }++        /**+         * A function decomposing the sub-tree rooted by this node into approximately equally+         * sized chunks. To minimize computational overhead, no strict load balance nor limit+         * on the number of actual chunks is given.+         *+         * @see btree::getChunks()+         *+         * @param res   .. the list of chunks to be extended+         * @param num   .. the number of chunks to be produced+         * @param begin .. the iterator to start the first chunk with+         * @param end   .. the iterator to end the last chunk with+         * @return the handed in list of chunks extended by generated chunks+         */+        std::vector<chunk>& collectChunks(+                std::vector<chunk>& res, size_type num, const iterator& begin, const iterator& end) const {+            assert(num > 0);++            // special case: this node is empty+            if (isEmpty()) {+                if (begin != end) {+                    res.push_back(chunk(begin, end));+                }+                return res;+            }++            // special case: a single chunk is requested+            if (num == 1) {+                res.push_back(chunk(begin, end));+                return res;+            }++            // cut-off+            if (this->isLeaf() || num < (this->numElements + 1)) {+                auto step = this->numElements / num;+                if (step == 0) {+                    step = 1;+                }++                size_type i = 0;++                // the first chunk starts at the begin+                res.push_back(chunk(begin, iterator(this, static_cast<field_index_type>(step) - 1)));++                // split up the main part+                for (i = step - 1; i < this->numElements - step; i += step) {+                    res.push_back(chunk(iterator(this, static_cast<field_index_type>(i)),+                            iterator(this, static_cast<field_index_type>(i + step))));+                }++                // the last chunk runs to the end+                res.push_back(chunk(iterator(this, static_cast<field_index_type>(i)), end));++                // done+                return res;+            }++            // else: collect chunks of sub-set elements++            auto part = num / (this->numElements + 1);+            assert(part > 0);+            getChild(0)->collectChunks(res, part, begin, iterator(this, 0));+            for (size_type i = 1; i < this->numElements; i++) {+                getChild(i)->collectChunks(res, part, iterator(this, static_cast<field_index_type>(i - 1)),+                        iterator(this, static_cast<field_index_type>(i)));+            }+            getChild(this->numElements)+                    ->collectChunks(res, num - (part * this->numElements),+                            iterator(this, static_cast<field_index_type>(this->numElements) - 1), end);++            // done+            return res;+        }++        /**+         * A function to verify the consistency of this node.+         *+         * @param root ... a reference to the root of the enclosing tree.+         * @return true if valid, false otherwise+         */+        template <typename Comp>+        bool check(Comp& comp, const node* root) const {+            bool valid = true;++            // check fill-state+            if (this->numElements > maxKeys) {+                std::cout << "Node with " << this->numElements << "/" << maxKeys << " encountered!\n";+                valid = false;+            }++            // check root state+            if (root == this) {+                if (this->parent != nullptr) {+                    std::cout << "Root not properly linked!\n";+                    valid = false;+                }+            } else {+                // check parent relation+                if (!this->parent) {+                    std::cout << "Invalid null-parent!\n";+                    valid = false;+                } else {+                    if (this->parent->getChildren()[this->position] != this) {+                        std::cout << "Parent reference invalid!\n";+                        std::cout << "   Node:     " << this << "\n";+                        std::cout << "   Parent:   " << this->parent << "\n";+                        std::cout << "   Position: " << ((int)this->position) << "\n";+                        valid = false;+                    }++                    // check parent key+                    if (valid && this->position != 0 &&+                            !(comp(this->parent->keys[this->position - 1], keys[0]) < ((isSet) ? 0 : 1))) {+                        std::cout << "Left parent key not lower bound!\n";+                        std::cout << "   Node:     " << this << "\n";+                        std::cout << "   Parent:   " << this->parent << "\n";+                        std::cout << "   Position: " << ((int)this->position) << "\n";+                        std::cout << "   Key:   " << (this->parent->keys[this->position]) << "\n";+                        std::cout << "   Lower: " << (keys[0]) << "\n";+                        valid = false;+                    }++                    // check parent key+                    if (valid && this->position != this->parent->numElements &&+                            !(comp(keys[this->numElements - 1], this->parent->keys[this->position]) <+                                    ((isSet) ? 0 : 1))) {+                        std::cout << "Right parent key not lower bound!\n";+                        std::cout << "   Node:     " << this << "\n";+                        std::cout << "   Parent:   " << this->parent << "\n";+                        std::cout << "   Position: " << ((int)this->position) << "\n";+                        std::cout << "   Key:   " << (this->parent->keys[this->position]) << "\n";+                        std::cout << "   Upper: " << (keys[0]) << "\n";+                        valid = false;+                    }+                }+            }++            // check element order+            if (this->numElements > 0) {+                for (unsigned i = 0; i < this->numElements - 1; i++) {+                    if (valid && !(comp(keys[i], keys[i + 1]) < ((isSet) ? 0 : 1))) {+                        std::cout << "Element order invalid!\n";+                        std::cout << " @" << this << " key " << i << " is " << keys[i] << " vs "+                                  << keys[i + 1] << "\n";+                        valid = false;+                    }+                }+            }++            // check state of sub-nodes+            if (this->inner) {+                for (unsigned i = 0; i <= this->numElements; i++) {+                    valid &= getChildren()[i]->check(comp, root);+                }+            }++            return valid;+        }+    };  // namespace detail++    /**+     * The data type representing inner nodes of the b-tree. It extends+     * the generic implementation of a node by the storage locations+     * of child pointers.+     */+    struct inner_node : public node {+        // references to child nodes owned by this node+        node* children[node::maxKeys + 1];++        // a simple default constructor initializing member fields+        inner_node() : node(true) {}++        // clear up child nodes recursively+        ~inner_node() {+            for (unsigned i = 0; i <= this->numElements; ++i) {+                if (children[i] != nullptr) {+                    if (children[i]->isLeaf()) {+                        delete static_cast<leaf_node*>(children[i]);+                    } else {+                        delete static_cast<inner_node*>(children[i]);+                    }+                }+            }+        }+    };++    /**+     * The data type representing leaf nodes of the b-tree. It does not+     * add any capabilities to the generic node type.+     */+    struct leaf_node : public node {+        // a simple default constructor initializing member fields+        leaf_node() : node(false) {}+    };++    // ------------------- iterators ------------------------++public:+    /**+     * The iterator type to be utilized for scanning through btree instances.+     */+    class iterator {+        // a pointer to the node currently referred to+        node const* cur;++        // the index of the element currently addressed within the referenced node+        field_index_type pos = 0;++    public:+        typedef std::forward_iterator_tag iterator_category;+        typedef Key value_type;+        typedef ptrdiff_t difference_type;+        typedef value_type* pointer;+        typedef value_type& reference;++        // default constructor -- creating an end-iterator+        iterator() : cur(nullptr) {}++        // creates an iterator referencing a specific element within a given node+        iterator(node const* cur, field_index_type pos) : cur(cur), pos(pos) {}++        // a copy constructor+        iterator(const iterator& other) : cur(other.cur), pos(other.pos) {}++        // an assignment operator+        iterator& operator=(const iterator& other) {+            cur = other.cur;+            pos = other.pos;+            return *this;+        }++        // the equality operator as required by the iterator concept+        bool operator==(const iterator& other) const {+            return cur == other.cur && pos == other.pos;+        }++        // the not-equality operator as required by the iterator concept+        bool operator!=(const iterator& other) const {+            return !(*this == other);+        }++        // the deref operator as required by the iterator concept+        const Key& operator*() const {+            return cur->keys[pos];+        }++        // the increment operator as required by the iterator concept+        iterator& operator++() {+            // the quick mode -- if in a leaf and there are elements left+            if (cur->isLeaf() && ++pos < cur->getNumElements()) {+                return *this;+            }++            // otherwise it is a bit more tricky++            // A) currently in an inner node => go to the left-most child+            if (cur->isInner()) {+                cur = cur->getChildren()[pos + 1];+                while (!cur->isLeaf()) {+                    cur = cur->getChildren()[0];+                }+                pos = 0;++                // nodes may be empty due to biased insertion+                if (!cur->isEmpty()) {+                    return *this;+                }+            }++            // B) we are at the right-most element of a leaf => go to next inner node+            assert(cur->isLeaf());+            assert(pos == cur->getNumElements());++            while (cur != nullptr && pos == cur->getNumElements()) {+                pos = cur->getPositionInParent();+                cur = cur->getParent();+            }+            return *this;+        }++        // prints a textual representation of this iterator to the given stream (mainly for debugging)+        void print(std::ostream& out = std::cout) const {+            out << cur << "[" << (int)pos << "]";+        }+    };++    /**+     * A collection of operation hints speeding up some of the involved operations+     * by exploiting temporal locality.+     */+    template <unsigned size = 1>+    struct btree_operation_hints {+        using node_cache = LRUCache<node*, size>;++        // the node where the last insertion terminated+        node_cache last_insert;++        // the node where the last find-operation terminated+        node_cache last_find_end;++        // the node where the last lower-bound operation terminated+        node_cache last_lower_bound_end;++        // the node where the last upper-bound operation terminated+        node_cache last_upper_bound_end;++        // default constructor+        btree_operation_hints() = default;++        // resets all hints (to be triggered e.g. when deleting nodes)+        void clear() {+            last_insert.clear(nullptr);+            last_find_end.clear(nullptr);+            last_lower_bound_end.clear(nullptr);+            last_upper_bound_end.clear(nullptr);+        }+    };++    using operation_hints = btree_operation_hints<1>;++protected:+#ifdef IS_PARALLEL+    // a pointer to the root node of this tree+    node* volatile root;++    // a lock to synchronize update operations on the root pointer+    lock_type root_lock;+#else+    // a pointer to the root node of this tree+    node* root;++    // required to not duplicate too much code+    lock_type root_lock;+#endif++    // a pointer to the left-most node of this tree (initial note for iteration)+    leaf_node* leftmost;++    /* -------------- operator hint statistics ----------------- */++    // an aggregation of statistical values of the hint utilization+    struct hint_statistics {+        // the counter for insertion operations+        CacheAccessCounter inserts;++        // the counter for contains operations+        CacheAccessCounter contains;++        // the counter for lower_bound operations+        CacheAccessCounter lower_bound;++        // the counter for upper_bound operations+        CacheAccessCounter upper_bound;+    };++    // the hint statistic of this b-tree instance+    mutable hint_statistics hint_stats;++public:+    // the maximum number of keys stored per node+    static constexpr size_t max_keys_per_node = node::maxKeys;++    // -- ctors / dtors --++    // the default constructor creating an empty tree+    btree(Comparator comp = Comparator(), WeakComparator weak_comp = WeakComparator())+            : comp(std::move(comp)), weak_comp(std::move(weak_comp)), root(nullptr), leftmost(nullptr) {}++    // a constructor creating a tree from the given iterator range+    template <typename Iter>+    btree(const Iter& a, const Iter& b) : root(nullptr), leftmost(nullptr) {+        insert(a, b);+    }++    // a move constructor+    btree(btree&& other)+            : comp(other.comp), weak_comp(other.weak_comp), root(other.root), leftmost(other.leftmost) {+        other.root = nullptr;+        other.leftmost = nullptr;+    }++    // a copy constructor+    btree(const btree& set) : comp(set.comp), weak_comp(set.weak_comp), root(nullptr), leftmost(nullptr) {+        // use assignment operator for a deep copy+        *this = set;+    }++protected:+    /**+     * An internal constructor enabling the specific creation of a tree+     * based on internal parameters.+     */+    btree(size_type /* size */, node* root, leaf_node* leftmost) : root(root), leftmost(leftmost) {}++public:+    // the destructor freeing all contained nodes+    ~btree() {+        clear();+    }++    // -- mutators and observers --++    // emptiness check+    bool empty() const {+        return root == nullptr;+    }++    // determines the number of elements in this tree+    size_type size() const {+        return (root) ? root->countEntries() : 0;+    }++    /**+     * Inserts the given key into this tree.+     */+    bool insert(const Key& k) {+        operation_hints hints;+        return insert(k, hints);+    }++    /**+     * Inserts the given key into this tree.+     */+    bool insert(const Key& k, operation_hints& hints) {+#ifdef IS_PARALLEL++        // special handling for inserting first element+        while (root == nullptr) {+            // try obtaining root-lock+            if (!root_lock.try_start_write()) {+                // somebody else was faster => re-check+                continue;+            }++            // check loop condition again+            if (root != nullptr) {+                // somebody else was faster => normal insert+                root_lock.end_write();+                break;+            }++            // create new node+            leftmost = new leaf_node();+            leftmost->numElements = 1;+            leftmost->keys[0] = k;+            root = leftmost;++            // operation complete => we can release the root lock+            root_lock.end_write();++            hints.last_insert.access(leftmost);++            return true;+        }++        // insert using iterative implementation++        node* cur = nullptr;++        // test last insert hints+        lock_type::Lease cur_lease;++        auto checkHint = [&](node* last_insert) {+            // ignore null pointer+            if (!last_insert) return false;+            // get a read lease on indicated node+            auto hint_lease = last_insert->lock.start_read();+            // check whether it covers the key+            if (!weak_covers(last_insert, k)) return false;+            // and if there was no concurrent modification+            if (!last_insert->lock.validate(hint_lease)) return false;+            // use hinted location+            cur = last_insert;+            // and keep lease+            cur_lease = hint_lease;+            // we found a hit+            return true;+        };++        if (hints.last_insert.any(checkHint)) {+            // register this as a hit+            hint_stats.inserts.addHit();+        } else {+            // register this as a miss+            hint_stats.inserts.addMiss();+        }++        // if there is no valid hint ..+        if (!cur) {+            do {+                // get root - access lock+                auto root_lease = root_lock.start_read();++                // start with root+                cur = root;++                // get lease of the next node to be accessed+                cur_lease = cur->lock.start_read();++                // check validity of root pointer+                if (root_lock.end_read(root_lease)) {+                    break;+                }++            } while (true);+        }++        while (true) {+            // handle inner nodes+            if (cur->inner) {+                auto a = &(cur->keys[0]);+                auto b = &(cur->keys[cur->numElements]);++                auto pos = search.lower_bound(k, a, b, weak_comp);+                auto idx = pos - a;++                // early exit for sets+                if (isSet && pos != b && weak_equal(*pos, k)) {+                    // validate results+                    if (!cur->lock.validate(cur_lease)) {+                        // start over again+                        return insert(k, hints);+                    }++                    // update provenance information+                    if (typeid(Comparator) != typeid(WeakComparator) && less(k, *pos)) {+                        if (!cur->lock.try_upgrade_to_write(cur_lease)) {+                            // start again+                            return insert(k, hints);+                        }+                        update(*pos, k);+                        cur->lock.end_write();+                        return true;+                    }++                    // we found the element => no check of lock necessary+                    return false;+                }++                // get next pointer+                auto next = cur->getChild(idx);++                // get lease on next level+                auto next_lease = next->lock.start_read();++                // check whether there was a write+                if (!cur->lock.end_read(cur_lease)) {+                    // start over+                    return insert(k, hints);+                }++                // go to next+                cur = next;++                // move on lease+                cur_lease = next_lease;++                continue;+            }++            // the rest is for leaf nodes+            assert(!cur->inner);++            // -- insert node in leaf node --++            auto a = &(cur->keys[0]);+            auto b = &(cur->keys[cur->numElements]);++            auto pos = search.upper_bound(k, a, b, weak_comp);+            auto idx = pos - a;++            // early exit for sets+            if (isSet && pos != a && weak_equal(*(pos - 1), k)) {+                // validate result+                if (!cur->lock.validate(cur_lease)) {+                    // start over again+                    return insert(k, hints);+                }++                // update provenance information+                if (typeid(Comparator) != typeid(WeakComparator) && less(k, *(pos - 1))) {+                    if (!cur->lock.try_upgrade_to_write(cur_lease)) {+                        // start again+                        return insert(k, hints);+                    }+                    update(*(pos - 1), k);+                    cur->lock.end_write();+                    return true;+                }++                // we found the element => done+                return false;+            }++            // upgrade to write-permission+            if (!cur->lock.try_upgrade_to_write(cur_lease)) {+                // something has changed => restart+                hints.last_insert.access(cur);+                return insert(k, hints);+            }++            if (cur->numElements >= node::maxKeys) {+                // -- lock parents --+                auto priv = cur;+                auto parent = priv->parent;+                std::vector<node*> parents;+                do {+                    if (parent) {+                        parent->lock.start_write();+                        while (true) {+                            // check whether parent is correct+                            if (parent == priv->parent) {+                                break;+                            }+                            // switch parent+                            parent->lock.abort_write();+                            parent = priv->parent;+                            parent->lock.start_write();+                        }+                    } else {+                        // lock root lock => since cur is root+                        root_lock.start_write();+                    }++                    // record locked node+                    parents.push_back(parent);++                    // stop at "sphere of influence"+                    if (!parent || !parent->isFull()) {+                        break;+                    }++                    // go one step higher+                    priv = parent;+                    parent = parent->parent;++                } while (true);++                // split this node+                auto old_root = root;+                idx -= cur->rebalance_or_split(const_cast<node**>(&root), root_lock, idx, parents);++                // release parent lock+                for (auto it = parents.rbegin(); it != parents.rend(); ++it) {+                    auto parent = *it;++                    // release this lock+                    if (parent) {+                        parent->lock.end_write();+                    } else {+                        if (old_root != root) {+                            root_lock.end_write();+                        } else {+                            root_lock.abort_write();+                        }+                    }+                }++                // insert element in right fragment+                if (((size_type)idx) > cur->numElements) {+                    // release current lock+                    cur->lock.end_write();++                    // insert in sibling+                    return insert(k, hints);+                }+            }++            // ok - no split necessary+            assert(cur->numElements < node::maxKeys && "Split required!");++            // move keys+            for (int j = cur->numElements; j > idx; --j) {+                cur->keys[j] = cur->keys[j - 1];+            }++            // insert new element+            cur->keys[idx] = k;+            cur->numElements++;++            // release lock on current node+            cur->lock.end_write();++            // remember last insertion position+            hints.last_insert.access(cur);+            return true;+        }++#else+        // special handling for inserting first element+        if (empty()) {+            // create new node+            leftmost = new leaf_node();+            leftmost->numElements = 1;+            leftmost->keys[0] = k;+            root = leftmost;++            hints.last_insert.access(leftmost);++            return true;+        }++        // insert using iterative implementation+        node* cur = root;++        auto checkHints = [&](node* last_insert) {+            if (!last_insert) return false;+            if (!weak_covers(last_insert, k)) return false;+            cur = last_insert;+            return true;+        };++        // test last insert+        if (hints.last_insert.any(checkHints)) {+            hint_stats.inserts.addHit();+        } else {+            hint_stats.inserts.addMiss();+        }++        while (true) {+            // handle inner nodes+            if (cur->inner) {+                auto a = &(cur->keys[0]);+                auto b = &(cur->keys[cur->numElements]);++                auto pos = search.lower_bound(k, a, b, weak_comp);+                auto idx = pos - a;++                // early exit for sets+                if (isSet && pos != b && weak_equal(*pos, k)) {+                    // update provenance information+                    if (typeid(Comparator) != typeid(WeakComparator) && less(k, *pos)) {+                        update(*pos, k);+                        return true;+                    }++                    return false;+                }++                cur = cur->getChild(idx);+                continue;+            }++            // the rest is for leaf nodes+            assert(!cur->inner);++            // -- insert node in leaf node --++            auto a = &(cur->keys[0]);+            auto b = &(cur->keys[cur->numElements]);++            auto pos = search.upper_bound(k, a, b, weak_comp);+            auto idx = pos - a;++            // early exit for sets+            if (isSet && pos != a && weak_equal(*(pos - 1), k)) {+                // update provenance information+                if (typeid(Comparator) != typeid(WeakComparator) && less(k, *(pos - 1))) {+                    update(*(pos - 1), k);+                    return true;+                }++                return false;+            }++            if (cur->numElements >= node::maxKeys) {+                // split this node+                idx -= cur->rebalance_or_split(&root, root_lock, static_cast<int>(idx));++                // insert element in right fragment+                if (((size_type)idx) > cur->numElements) {+                    idx -= cur->numElements + 1;+                    cur = cur->parent->getChild(cur->position + 1);+                }+            }++            // ok - no split necessary+            assert(cur->numElements < node::maxKeys && "Split required!");++            // move keys+            for (int j = static_cast<int>(cur->numElements); j > idx; --j) {+                cur->keys[j] = cur->keys[j - 1];+            }++            // insert new element+            cur->keys[idx] = k;+            cur->numElements++;++            // remember last insertion position+            hints.last_insert.access(cur);++            return true;+        }+#endif+    }++    /**+     * Inserts the given range of elements into this tree.+     */+    template <typename Iter>+    void insert(const Iter& a, const Iter& b) {+        // TODO: improve this beyond a naive insert+        operation_hints hints;+        // a naive insert so far .. seems to work fine+        for (auto it = a; it != b; ++it) {+            // use insert with hint+            insert(*it, hints);+        }+    }++    // Obtains an iterator referencing the first element of the tree.+    iterator begin() const {+        return iterator(leftmost, 0);+    }++    // Obtains an iterator referencing the position after the last element of the tree.+    iterator end() const {+        return iterator();+    }++    /**+     * Partitions the full range of this set into up to a given number of chunks.+     * The chunks will cover approximately the same number of elements. Also, the+     * number of chunks will only approximate the desired number of chunks.+     *+     * @param num .. the number of chunks requested+     * @return a list of chunks partitioning this tree+     */+    std::vector<chunk> partition(size_type num) const {+        return getChunks(num);+    }++    std::vector<chunk> getChunks(size_type num) const {+        std::vector<chunk> res;+        if (empty()) {+            return res;+        }+        return root->collectChunks(res, num, begin(), end());+    }++    /**+     * Determines whether the given element is a member of this tree.+     */+    bool contains(const Key& k) const {+        operation_hints hints;+        return contains(k, hints);+    }++    /**+     * Determines whether the given element is a member of this tree.+     */+    bool contains(const Key& k, operation_hints& hints) const {+        return find(k, hints) != end();+    }++    /**+     * Locates the given key within this tree and returns an iterator+     * referencing its position. If not found, an end-iterator will be returned.+     */+    iterator find(const Key& k) const {+        operation_hints hints;+        return find(k, hints);+    }++    /**+     * Locates the given key within this tree and returns an iterator+     * referencing its position. If not found, an end-iterator will be returned.+     */+    iterator find(const Key& k, operation_hints& hints) const {+        if (empty()) {+            return end();+        }++        node* cur = root;++        auto checkHints = [&](node* last_find_end) {+            if (!last_find_end) return false;+            if (!covers(last_find_end, k)) return false;+            cur = last_find_end;+            return true;+        };++        // test last location searched (temporal locality)+        if (hints.last_find_end.any(checkHints)) {+            // register it as a hit+            hint_stats.contains.addHit();+        } else {+            // register it as a miss+            hint_stats.contains.addMiss();+        }++        // an iterative implementation (since 2/7 faster than recursive)++        while (true) {+            auto a = &(cur->keys[0]);+            auto b = &(cur->keys[cur->numElements]);++            auto pos = search(k, a, b, comp);++            if (pos < b && equal(*pos, k)) {+                hints.last_find_end.access(cur);+                return iterator(cur, static_cast<field_index_type>(pos - a));+            }++            if (!cur->inner) {+                hints.last_find_end.access(cur);+                return end();+            }++            // continue search in child node+            cur = cur->getChild(pos - a);+        }+    }++    /**+     * Obtains a lower boundary for the given key -- hence an iterator referencing+     * the smallest value that is not less the given key. If there is no such element,+     * an end-iterator will be returned.+     */+    iterator lower_bound(const Key& k) const {+        operation_hints hints;+        return lower_bound(k, hints);+    }++    /**+     * Obtains a lower boundary for the given key -- hence an iterator referencing+     * the smallest value that is not less the given key. If there is no such element,+     * an end-iterator will be returned.+     */+    iterator lower_bound(const Key& k, operation_hints& hints) const {+        if (empty()) {+            return end();+        }++        node* cur = root;++        auto checkHints = [&](node* last_lower_bound_end) {+            if (!last_lower_bound_end) return false;+            if (!covers(last_lower_bound_end, k)) return false;+            cur = last_lower_bound_end;+            return true;+        };++        // test last searched node+        if (hints.last_lower_bound_end.any(checkHints)) {+            hint_stats.lower_bound.addHit();+        } else {+            hint_stats.lower_bound.addMiss();+        }++        iterator res = end();+        while (true) {+            auto a = &(cur->keys[0]);+            auto b = &(cur->keys[cur->numElements]);++            auto pos = search.lower_bound(k, a, b, comp);+            auto idx = static_cast<field_index_type>(pos - a);++            if (!cur->inner) {+                hints.last_lower_bound_end.access(cur);+                return (pos != b) ? iterator(cur, idx) : res;+            }++            if (isSet && pos != b && equal(*pos, k)) {+                return iterator(cur, idx);+            }++            if (pos != b) {+                res = iterator(cur, idx);+            }++            cur = cur->getChild(idx);+        }+    }++    /**+     * Obtains an upper boundary for the given key -- hence an iterator referencing+     * the first element that the given key is less than the referenced value. If+     * there is no such element, an end-iterator will be returned.+     */+    iterator upper_bound(const Key& k) const {+        operation_hints hints;+        return upper_bound(k, hints);+    }++    /**+     * Obtains an upper boundary for the given key -- hence an iterator referencing+     * the first element that the given key is less than the referenced value. If+     * there is no such element, an end-iterator will be returned.+     */+    iterator upper_bound(const Key& k, operation_hints& hints) const {+        if (empty()) {+            return end();+        }++        node* cur = root;++        auto checkHints = [&](node* last_upper_bound_end) {+            if (!last_upper_bound_end) return false;+            if (!coversUpperBound(last_upper_bound_end, k)) return false;+            cur = last_upper_bound_end;+            return true;+        };++        // test last search node+        if (hints.last_upper_bound_end.any(checkHints)) {+            hint_stats.upper_bound.addHit();+        } else {+            hint_stats.upper_bound.addMiss();+        }++        iterator res = end();+        while (true) {+            auto a = &(cur->keys[0]);+            auto b = &(cur->keys[cur->numElements]);++            auto pos = search.upper_bound(k, a, b, comp);+            auto idx = static_cast<field_index_type>(pos - a);++            if (!cur->inner) {+                hints.last_upper_bound_end.access(cur);+                return (pos != b) ? iterator(cur, idx) : res;+            }++            if (pos != b) {+                res = iterator(cur, idx);+            }++            cur = cur->getChild(idx);+        }+    }++    /**+     * Clears this tree.+     */+    void clear() {+        if (root != nullptr) {+            if (root->isLeaf()) {+                delete static_cast<leaf_node*>(root);+            } else {+                delete static_cast<inner_node*>(root);+            }+        }+        root = nullptr;+        leftmost = nullptr;+    }++    /**+     * Swaps the content of this tree with the given tree. This+     * is a much more efficient operation than creating a copy and+     * realizing the swap utilizing assignment operations.+     */+    void swap(btree& other) {+        // swap the content+        std::swap(root, other.root);+        std::swap(leftmost, other.leftmost);+    }++    // Implementation of the assignment operation for trees.+    btree& operator=(const btree& other) {+        // check identity+        if (this == &other) {+            return *this;+        }++        // create a deep-copy of the content of the other tree+        // shortcut for empty sets+        if (other.empty()) {+            return *this;+        }++        // clone content (deep copy)+        root = other.root->clone();++        // update leftmost reference+        auto tmp = root;+        while (!tmp->isLeaf()) {+            tmp = tmp->getChild(0);+        }+        leftmost = static_cast<leaf_node*>(tmp);++        // done+        return *this;+    }++    // Implementation of an equality operation for trees.+    bool operator==(const btree& other) const {+        // check identity+        if (this == &other) {+            return true;+        }++        // check size+        if (size() != other.size()) {+            return false;+        }+        if (size() < other.size()) {+            return other == *this;+        }++        // check content+        for (const auto& key : other) {+            if (!contains(key)) {+                return false;+            }+        }+        return true;+    }++    // Implementation of an inequality operation for trees.+    bool operator!=(const btree& other) const {+        return !(*this == other);+    }++    // -- for debugging --++    // Determines the number of levels contained in this tree.+    size_type getDepth() const {+        return (empty()) ? 0 : root->getDepth();+    }++    // Determines the number of nodes contained in this tree.+    size_type getNumNodes() const {+        return (empty()) ? 0 : root->countNodes();+    }++    // Determines the amount of memory used by this data structure+    size_type getMemoryUsage() const {+        return sizeof(*this) + (empty() ? 0 : root->getMemoryUsage());+    }++    /*+     * Prints a textual representation of this tree to the given+     * output stream (mostly for debugging and tuning).+     */+    void printTree(std::ostream& out = std::cout) const {+        out << "B-Tree with " << size() << " elements:\n";+        if (empty()) {+            out << " - empty - \n";+        } else {+            root->printTree(out, "");+        }+    }++    /**+     * Prints a textual summary of statistical properties of this+     * tree to the given output stream (for debugging and tuning).+     */+    void printStats(std::ostream& out = std::cout) const {+        auto nodes = getNumNodes();+        out << " ---------------------------------\n";+        out << "  Elements: " << size() << "\n";+        out << "  Depth:    " << (empty() ? 0 : root->getDepth()) << "\n";+        out << "  Nodes:    " << nodes << "\n";+        out << " ---------------------------------\n";+        out << "  Size of inner node: " << sizeof(inner_node) << "\n";+        out << "  Size of leaf node:  " << sizeof(leaf_node) << "\n";+        out << "  Size of Key:        " << sizeof(Key) << "\n";+        out << "  max keys / node:  " << node::maxKeys << "\n";+        out << "  avg keys / node:  " << (size() / (double)nodes) << "\n";+        out << "  avg filling rate: " << ((size() / (double)nodes) / node::maxKeys) << "\n";+        out << " ---------------------------------\n";+        out << "  insert-hint (hits/misses/total): " << hint_stats.inserts.getHits() << "/"+            << hint_stats.inserts.getMisses() << "/" << hint_stats.inserts.getAccesses() << "\n";+        out << "  contains-hint(hits/misses/total):" << hint_stats.contains.getHits() << "/"+            << hint_stats.contains.getMisses() << "/" << hint_stats.contains.getAccesses() << "\n";+        out << "  lower-bound-hint (hits/misses/total):" << hint_stats.lower_bound.getHits() << "/"+            << hint_stats.lower_bound.getMisses() << "/" << hint_stats.lower_bound.getAccesses() << "\n";+        out << "  upper-bound-hint (hits/misses/total):" << hint_stats.upper_bound.getHits() << "/"+            << hint_stats.upper_bound.getMisses() << "/" << hint_stats.upper_bound.getAccesses() << "\n";+        out << " ---------------------------------\n";+    }++    /**+     * Checks the consistency of this tree.+     */+    bool check() {+        auto ok = empty() || root->check(comp, root);+        if (!ok) {+            printTree();+        }+        return ok;+    }++    /**+     * A static member enabling the bulk-load of ordered data into an empty+     * tree. This function is much more efficient in creating a index over+     * an ordered set of elements than an iterative insertion of values.+     *+     * @tparam Iter .. the type of iterator specifying the range+     *                     it must be a random-access iterator+     */+    template <typename R, typename Iter>+    static typename std::enable_if<std::is_same<typename std::iterator_traits<Iter>::iterator_category,+                                           std::random_access_iterator_tag>::value,+            R>::type+    load(const Iter& a, const Iter& b) {+        // quick exit - empty range+        if (a == b) {+            return R();+        }++        // resolve tree recursively+        auto root = buildSubTree(a, b - 1);++        // find leftmost node+        node* leftmost = root;+        while (!leftmost->isLeaf()) {+            leftmost = leftmost->getChild(0);+        }++        // build result+        return R(b - a, root, static_cast<leaf_node*>(leftmost));+    }++protected:+    /**+     * Determines whether the range covered by the given node is also+     * covering the given key value.+     */+    bool covers(const node* node, const Key& k) const {+        if (isSet) {+            // in sets we can include the ends as covered elements+            return !node->isEmpty() && !less(k, node->keys[0]) && !less(node->keys[node->numElements - 1], k);+        }+        // in multi-sets the ends may not be completely covered+        return !node->isEmpty() && less(node->keys[0], k) && less(k, node->keys[node->numElements - 1]);+    }++    /**+     * Determines whether the range covered by the given node is also+     * covering the given key value.+     */+    bool weak_covers(const node* node, const Key& k) const {+        if (isSet) {+            // in sets we can include the ends as covered elements+            return !node->isEmpty() && !weak_less(k, node->keys[0]) &&+                   !weak_less(node->keys[node->numElements - 1], k);+        }+        // in multi-sets the ends may not be completely covered+        return !node->isEmpty() && weak_less(node->keys[0], k) &&+               weak_less(k, node->keys[node->numElements - 1]);+    }++private:+    /**+     * Determines whether the range covered by this node covers+     * the upper bound of the given key.+     */+    bool coversUpperBound(const node* node, const Key& k) const {+        // ignore edges+        return !node->isEmpty() && !less(k, node->keys[0]) && less(k, node->keys[node->numElements - 1]);+    }++    // Utility function for the load operation above.+    template <typename Iter>+    static node* buildSubTree(const Iter& a, const Iter& b) {+        const int N = node::maxKeys;++        // divide range in N+1 sub-ranges+        int length = (b - a) + 1;++        // terminal case: length is less then maxKeys+        if (length <= N) {+            // create a leaf node+            node* res = new leaf_node();+            res->numElements = length;++            for (int i = 0; i < length; ++i) {+                res->keys[i] = a[i];+            }++            return res;+        }++        // recursive case - compute step size+        int numKeys = N;+        int step = ((length - numKeys) / (numKeys + 1));++        while (numKeys > 1 && (step < N / 2)) {+            numKeys--;+            step = ((length - numKeys) / (numKeys + 1));+        }++        // create inner node+        node* res = new inner_node();+        res->numElements = numKeys;++        Iter c = a;+        for (int i = 0; i < numKeys; i++) {+            // get dividing key+            res->keys[i] = c[step];++            // get sub-tree+            auto child = buildSubTree(c, c + (step - 1));+            child->parent = res;+            child->position = i;+            res->getChildren()[i] = child;++            c = c + (step + 1);+        }++        // and the remaining part+        auto child = buildSubTree(c, b);+        child->parent = res;+        child->position = numKeys;+        res->getChildren()[numKeys] = child;++        // done+        return res;+    }+};  // namespace souffle++// Instantiation of static member search.+template <typename Key, typename Comparator, typename Allocator, unsigned blockSize, typename SearchStrategy,+        bool isSet, typename WeakComparator, typename Updater>+const SearchStrategy+        btree<Key, Comparator, Allocator, blockSize, SearchStrategy, isSet, WeakComparator, Updater>::search;++}  // end namespace detail++/**+ * A b-tree based set implementation.+ *+ * @tparam Key             .. the element type to be stored in this set+ * @tparam Comparator     .. a class defining an order on the stored elements+ * @tparam Allocator     .. utilized for allocating memory for required nodes+ * @tparam blockSize    .. determines the number of bytes/block utilized by leaf nodes+ * @tparam SearchStrategy .. enables switching between linear, binary or any other search strategy+ */+template <typename Key, typename Comparator = detail::comparator<Key>,+        typename Allocator = std::allocator<Key>,  // is ignored so far+        unsigned blockSize = 256,+        typename SearchStrategy = typename souffle::detail::default_strategy<Key>::type,+        typename WeakComparator = Comparator, typename Updater = souffle::detail::updater<Key>>+class btree_set : public souffle::detail::btree<Key, Comparator, Allocator, blockSize, SearchStrategy, true,+                          WeakComparator, Updater> {+    using super = souffle::detail::btree<Key, Comparator, Allocator, blockSize, SearchStrategy, true,+            WeakComparator, Updater>;++    friend class souffle::detail::btree<Key, Comparator, Allocator, blockSize, SearchStrategy, true,+            WeakComparator, Updater>;++public:+    /**+     * A default constructor creating an empty set.+     */+    btree_set(const Comparator& comp = Comparator(), const WeakComparator& weak_comp = WeakComparator())+            : super(comp, weak_comp) {}++    /**+     * A constructor creating a set based on the given range.+     */+    template <typename Iter>+    btree_set(const Iter& a, const Iter& b) {+        this->insert(a, b);+    }++    // A copy constructor.+    btree_set(const btree_set& other) : super(other) {}++    // A move constructor.+    btree_set(btree_set&& other) : super(std::move(other)) {}++private:+    // A constructor required by the bulk-load facility.+    template <typename s, typename n, typename l>+    btree_set(s size, n* root, l* leftmost) : super(size, root, leftmost) {}++public:+    // Support for the assignment operator.+    btree_set& operator=(const btree_set& other) {+        super::operator=(other);+        return *this;+    }++    // Support for the bulk-load operator.+    template <typename Iter>+    static btree_set load(const Iter& a, const Iter& b) {+        return super::template load<btree_set>(a, b);+    }+};++/**+ * A b-tree based multi-set implementation.+ *+ * @tparam Key             .. the element type to be stored in this set+ * @tparam Comparator     .. a class defining an order on the stored elements+ * @tparam Allocator     .. utilized for allocating memory for required nodes+ * @tparam blockSize    .. determines the number of bytes/block utilized by leaf nodes+ * @tparam SearchStrategy .. enables switching between linear, binary or any other search strategy+ */+template <typename Key, typename Comparator = detail::comparator<Key>,+        typename Allocator = std::allocator<Key>,  // is ignored so far+        unsigned blockSize = 256,+        typename SearchStrategy = typename souffle::detail::default_strategy<Key>::type,+        typename WeakComparator = Comparator, typename Updater = souffle::detail::updater<Key>>+class btree_multiset : public souffle::detail::btree<Key, Comparator, Allocator, blockSize, SearchStrategy,+                               false, WeakComparator, Updater> {+    using super = souffle::detail::btree<Key, Comparator, Allocator, blockSize, SearchStrategy, false,+            WeakComparator, Updater>;++    friend class souffle::detail::btree<Key, Comparator, Allocator, blockSize, SearchStrategy, false,+            WeakComparator, Updater>;++public:+    /**+     * A default constructor creating an empty set.+     */+    btree_multiset(const Comparator& comp = Comparator(), const WeakComparator& weak_comp = WeakComparator())+            : super(comp, weak_comp) {}++    /**+     * A constructor creating a set based on the given range.+     */+    template <typename Iter>+    btree_multiset(const Iter& a, const Iter& b) {+        this->insert(a, b);+    }++    // A copy constructor.+    btree_multiset(const btree_multiset& other) : super(other) {}++    // A move constructor.+    btree_multiset(btree_multiset&& other) : super(std::move(other)) {}++private:+    // A constructor required by the bulk-load facility.+    template <typename s, typename n, typename l>+    btree_multiset(s size, n* root, l* leftmost) : super(size, root, leftmost) {}++public:+    // Support for the assignment operator.+    btree_multiset& operator=(const btree_multiset& other) {+        super::operator=(other);+        return *this;+    }++    // Support for the bulk-load operator.+    template <typename Iter>+    static btree_multiset load(const Iter& a, const Iter& b) {+        return super::template load<btree_multiset>(a, b);+    }+};++}  // end of namespace souffle
+ cbits/souffle/datastructure/Brie.h view
@@ -0,0 +1,3176 @@+/*+ * Souffle - A Datalog Compiler+ * Copyright (c) 2013, 2015, Oracle and/or its affiliates. All rights reserved+ * Licensed under the Universal Permissive License v 1.0 as shown at:+ * - https://opensource.org/licenses/UPL+ * - <souffle root>/licenses/SOUFFLE-UPL.txt+ */++/************************************************************************+ *+ * @file Brie.h+ *+ * This header file contains the implementation for a generic, fixed+ * length integer trie.+ *+ * Tries trie is utilized to store n-ary tuples of integers. Each level+ * is implemented via a sparse array (also covered by this header file),+ * referencing the following nested level. The leaf level is realized+ * by a sparse bit-map to minimize the memory footprint.+ *+ * Multiple insert operations can be be conducted concurrently on trie+ * structures. So can read-only operations. However, inserts and read+ * operations may not be conducted at the same time.+ *+ ***********************************************************************/++#pragma once++#include "souffle/CompiledTuple.h"+#include "souffle/RamTypes.h"+#include "souffle/utility/CacheUtil.h"+#include "souffle/utility/ContainerUtil.h"+#include "souffle/utility/StreamUtil.h"+#include <algorithm>+#include <atomic>+#include <bitset>+#include <cassert>+#include <cstdint>+#include <cstring>+#include <iostream>+#include <iterator>+#include <limits>+#include <utility>+#include <vector>++#ifdef _WIN32+/**+ * When compiling for windows, redefine the gcc builtins which are used to+ * their equivalents on the windows platform.+ */+#define __sync_synchronize MemoryBarrier+#define __sync_bool_compare_and_swap(ptr, oldval, newval) \+    (InterlockedCompareExchangePointer((void* volatile*)ptr, (void*)newval, (void*)oldval) == (void*)oldval)+#endif  // _WIN32++namespace souffle {++namespace detail {++/**+ * A templated functor to obtain default values for+ * unspecified elements of sparse array instances.+ */+template <typename T>+struct default_factory {+    T operator()() const {+        return T();  // just use the default constructor+    }+};++/**+ * A functor representing the identity function.+ */+template <typename T>+struct identity {+    T operator()(T v) const {+        return v;+    }+};++/**+ * A operation to be utilized by the sparse map when merging+ * elements associated to different values.+ */+template <typename T>+struct default_merge {+    /**+     * Merges two values a and b when merging spase maps.+     */+    T operator()(T a, T b) const {+        default_factory<T> def;+        // if a is the default => us b, else stick to a+        return (a != def()) ? a : b;+    }+};++}  // end namespace detail++/**+ * A sparse array simulates an array associating to every element+ * of uint32_t an element of a generic type T. Any non-defined element+ * will be default-initialized utilizing the detail::default_factory+ * functor.+ *+ * Internally the array is organized as a balanced tree. The leaf+ * level of the tree corresponds to the elements of the represented+ * array. Inner nodes utilize individual bits of the indices to reference+ * sub-trees. For efficiency reasons, only the minimal sub-tree required+ * to cover all non-null / non-default values stored in the array is+ * maintained. Furthermore, several levels of nodes are aggreated in a+ * B-tree like fashion to inprove cache utilization and reduce the number+ * of steps required for lookup and insert operations.+ *+ * @tparam T the type of the stored elements+ * @tparam BITS the number of bits consumed per node-level+ *              e.g. if it is set to 3, the resulting tree will be of a degree of+ *              2^3=8, and thus 8 child-pointers will be stored in each inner node+ *              and as many values will be stored in each leaf node.+ * @tparam merge_op the functor to be utilized when merging the content of two+ *              instances of this type.+ * @tparam copy_op a functor to be applied to each stored value when copying an+ *              instance of this array. For instance, this is utilized by the+ *              trie implementation to create a clone of each sub-tree instead+ *              of preserving the original pointer.+ */+template <typename T, unsigned BITS = 6, typename merge_op = detail::default_merge<T>,+        typename copy_op = detail::identity<T>>+class SparseArray {+    using key_type = uint64_t;++    // some internal constants+    static constexpr int BIT_PER_STEP = BITS;+    static constexpr int NUM_CELLS = 1 << BIT_PER_STEP;+    static constexpr key_type INDEX_MASK = NUM_CELLS - 1;++public:+    // the type utilized for indexing contained elements+    using index_type = key_type;++    // the type of value stored in this array+    using value_type = T;++    // the atomic view on stored values+    using atomic_value_type = std::atomic<value_type>;++private:+    struct Node;++    /**+     * The value stored in a single cell of a inner+     * or leaf node.+     */+    union Cell {+        // an atomic view on the pointer referencing a nested level+        std::atomic<Node*> aptr;++        // a pointer to the nested level (unsynchronized operations)+        Node* ptr{nullptr};++        // an atomic view on the value stored in this cell (leaf node)+        atomic_value_type avalue;++        // the value stored in this cell (unsynchronized access, leaf node)+        value_type value;+    };++    /**+     * The node type of the internally maintained tree.+     */+    struct Node {+        // a pointer to the parent node (for efficient iteration)+        const Node* parent;+        // the pointers to the child nodes (inner nodes) or the stored values (leaf nodes)+        Cell cell[NUM_CELLS];+    };++    /**+     * A struct describing all the information required by the container+     * class to manage the wrapped up tree.+     */+    struct RootInfo {+        // the root node of the tree+        Node* root;+        // the number of levels of the tree+        uint32_t levels;+        // the absolute offset of the theoretical first element in the tree+        index_type offset;++        // the first leaf node in the tree+        Node* first;+        // the absolute offset of the first element in the first leaf node+        index_type firstOffset;+    };++    union {+        RootInfo unsynced;         // for sequential operations+        volatile RootInfo synced;  // for synchronized operations+    };++public:+    /**+     * A default constructor creating an empty sparse array.+     */+    SparseArray() : unsynced(RootInfo{nullptr, 0, 0, nullptr, std::numeric_limits<index_type>::max()}) {}++    /**+     * A copy constructor for sparse arrays. It creates a deep+     * copy of the data structure maintained by the handed in+     * array instance.+     */+    SparseArray(const SparseArray& other)+            : unsynced(RootInfo{clone(other.unsynced.root, other.unsynced.levels), other.unsynced.levels,+                      other.unsynced.offset, nullptr, other.unsynced.firstOffset}) {+        if (unsynced.root) {+            unsynced.root->parent = nullptr;+            unsynced.first = findFirst(unsynced.root, unsynced.levels);+        }+    }++    /**+     * A r-value based copy constructor for sparse arrays. It+     * takes over ownership of the structure maintained by the+     * handed in array.+     */+    SparseArray(SparseArray&& other)+            : unsynced(RootInfo{other.unsynced.root, other.unsynced.levels, other.unsynced.offset,+                      other.unsynced.first, other.unsynced.firstOffset}) {+        other.unsynced.root = nullptr;+        other.unsynced.levels = 0;+        other.unsynced.first = nullptr;+    }++    /**+     * A destructor for sparse arrays clearing up the internally+     * maintained data structure.+     */+    ~SparseArray() {+        clean();+    }++    /**+     * An assignment creating a deep copy of the handed in+     * array structure (utilizing the copy functor provided+     * as a template parameter).+     */+    SparseArray& operator=(const SparseArray& other) {+        if (this == &other) return *this;++        // clean this one+        clean();++        // copy content+        unsynced.levels = other.unsynced.levels;+        unsynced.root = clone(other.unsynced.root, unsynced.levels);+        if (unsynced.root) {+            unsynced.root->parent = nullptr;+        }+        unsynced.offset = other.unsynced.offset;+        unsynced.first = (unsynced.root) ? findFirst(unsynced.root, unsynced.levels) : nullptr;+        unsynced.firstOffset = other.unsynced.firstOffset;++        // done+        return *this;+    }++    /**+     * An assignment operation taking over ownership+     * from a r-value reference to a sparse array.+     */+    SparseArray& operator=(SparseArray&& other) {+        // clean this one+        clean();++        // harvest content+        unsynced.root = other.unsynced.root;+        unsynced.levels = other.unsynced.levels;+        unsynced.offset = other.unsynced.offset;+        unsynced.first = other.unsynced.first;+        unsynced.firstOffset = other.unsynced.firstOffset;++        // reset other+        other.unsynced.root = nullptr;+        other.unsynced.levels = 0;+        other.unsynced.first = nullptr;++        // done+        return *this;+    }++    /**+     * Tests whether this sparse array is empty, thus it only+     * contains default-values, or not.+     */+    bool empty() const {+        return unsynced.root == nullptr;+    }++    /**+     * Computes the number of non-empty elements within this+     * sparse array.+     */+    std::size_t size() const {+        // quick one for the empty map+        if (empty()) return 0;++        // count elements -- since maintaining is making inserts more expensive+        std::size_t res = 0;+        for (auto it = begin(); it != end(); ++it) {+            ++res;+        }+        return res;+    }++private:+    /**+     * Computes the memory usage of the given sub-tree.+     */+    static std::size_t getMemoryUsage(const Node* node, int level) {+        // support null-nodes+        if (!node) return 0;++        // add size of current node+        std::size_t res = sizeof(Node);++        // sum up memory usage of child nodes+        if (level > 0) {+            for (int i = 0; i < NUM_CELLS; i++) {+                res += getMemoryUsage(node->cell[i].ptr, level - 1);+            }+        }++        // done+        return res;+    }++public:+    /**+     * Computes the total memory usage of this data structure.+     */+    std::size_t getMemoryUsage() const {+        // the memory of the wrapper class+        std::size_t res = sizeof(*this);++        // add nodes+        if (unsynced.root) {+            res += getMemoryUsage(unsynced.root, unsynced.levels);+        }++        // done+        return res;+    }++    /**+     * Resets the content of this array to default values for each contained+     * element.+     */+    void clear() {+        clean();+        unsynced.root = nullptr;+        unsynced.levels = 0;+        unsynced.first = nullptr;+        unsynced.firstOffset = std::numeric_limits<index_type>::max();+    }++    /**+     * A struct to be utilized as a local, temporal context by client code+     * to speed up the execution of various operations (optional parameter).+     */+    struct op_context {+        index_type lastIndex{0};+        Node* lastNode{nullptr};+        op_context() = default;+    };++private:+    // ---------------------------------------------------------------------+    //              Optimistic Locking of Root-Level Infos+    // ---------------------------------------------------------------------++    /**+     * A struct to cover a snapshot of the root node state.+     */+    struct RootInfoSnapshot {+        // the current pointer to a root node+        Node* root;+        // the current number of levels+        uint32_t levels;+        // the current offset of the first theoretical element+        index_type offset;+        // a version number for the optimistic locking+        uintptr_t version;+    };++    /**+     * Obtains the current version of the root.+     */+    uint64_t getRootVersion() const {+        // here it is assumed that the load of a 64-bit word is atomic+        return (uint64_t)synced.root;+    }++    /**+     * Obtains a snapshot of the current root information.+     */+    RootInfoSnapshot getRootInfo() const {+        RootInfoSnapshot res{};+        do {+            // first take the mod counter+            do {+                // if res.mod % 2 == 1 .. there is an update in progress+                res.version = getRootVersion();+            } while (res.version % 2);++            // then the rest+            res.root = synced.root;+            res.levels = synced.levels;+            res.offset = synced.offset;++            // check consistency of obtained data (optimistic locking)+        } while (res.version != getRootVersion());++        // got a consistent snapshot+        return res;+    }++    /**+     * Updates the current root information based on the handed in modified+     * snapshot instance if the version number of the snapshot still corresponds+     * to the current version. Otherwise a concurrent update took place and the+     * operation is aborted.+     *+     * @param info the updated information to be assigned to the active root-info data+     * @return true if successfully updated, false if aborted+     */+    bool tryUpdateRootInfo(const RootInfoSnapshot& info) {+        // check mod counter+        uintptr_t version = info.version;++        // update root to invalid pointer (ending with 1)+        if (!__sync_bool_compare_and_swap(&synced.root, (Node*)version, (Node*)(version + 1))) {+            return false;+        }++        // conduct update+        synced.levels = info.levels;+        synced.offset = info.offset;++        // update root (and thus the version to enable future retrievals)+        __sync_synchronize();+        synced.root = info.root;++        // done+        return true;+    }++    /**+     * A struct summarizing the state of the first node reference.+     */+    struct FirstInfoSnapshot {+        // the pointer to the first node+        Node* node;+        // the offset of the first node+        index_type offset;+        // the version number of the first node (for the optimistic locking)+        uintptr_t version;+    };++    /**+     * Obtains the current version number of the first node information.+     */+    uint64_t getFirstVersion() const {+        // here it is assumed that the load of a 64-bit word is atomic+        return (uint64_t)synced.first;+    }++    /**+     * Obtains a snapshot of the current first-node information.+     */+    FirstInfoSnapshot getFirstInfo() const {+        FirstInfoSnapshot res{};+        do {+            // first take the version+            do {+                res.version = getFirstVersion();+            } while (res.version % 2);++            // collect the values+            res.node = synced.first;+            res.offset = synced.firstOffset;++        } while (res.version != getFirstVersion());++        // we got a consistent snapshot+        return res;+    }++    /**+     * Updates the information stored regarding the first node in a+     * concurrent setting utilizing a optimistic locking approach.+     * This is identical to the approach utilized for the root info.+     */+    bool tryUpdateFirstInfo(const FirstInfoSnapshot& info) {+        // check mod counter+        uintptr_t version = info.version;++        // temporary update first pointer to point to uneven value (lock-out)+        if (!__sync_bool_compare_and_swap(&synced.first, (Node*)version, (Node*)(version + 1))) {+            return false;+        }++        // conduct update+        synced.firstOffset = info.offset;++        // update node pointer (and thus the version number)+        __sync_synchronize();+        synced.first = info.node;  // must be last (and atomic)++        // done+        return true;+    }++public:+    /**+     * Obtains a mutable reference to the value addressed by the given index.+     *+     * @param i the index of the element to be addressed+     * @return a mutable reference to the corresponding element+     */+    value_type& get(index_type i) {+        op_context ctxt;+        return get(i, ctxt);+    }++    /**+     * Obtains a mutable reference to the value addressed by the given index.+     *+     * @param i the index of the element to be addressed+     * @param ctxt a operation context to exploit state-less temporal locality+     * @return a mutable reference to the corresponding element+     */+    value_type& get(index_type i, op_context& ctxt) {+        return getLeaf(i, ctxt).value;+    }++    /**+     * Obtains a mutable reference to the atomic value addressed by the given index.+     *+     * @param i the index of the element to be addressed+     * @return a mutable reference to the corresponding element+     */+    atomic_value_type& getAtomic(index_type i) {+        op_context ctxt;+        return getAtomic(i, ctxt);+    }++    /**+     * Obtains a mutable reference to the atomic value addressed by the given index.+     *+     * @param i the index of the element to be addressed+     * @param ctxt a operation context to exploit state-less temporal locality+     * @return a mutable reference to the corresponding element+     */+    atomic_value_type& getAtomic(index_type i, op_context& ctxt) {+        return getLeaf(i, ctxt).avalue;+    }++private:+    /**+     * An internal function capable of navigating to a given leaf node entry.+     * If the cell does not exist yet it will be created as a side-effect.+     *+     * @param i the index of the requested cell+     * @param ctxt a operation context to exploit state-less temporal locality+     * @return a reference to the requested cell+     */+    inline Cell& getLeaf(index_type i, op_context& ctxt) {+        // check context+        if (ctxt.lastNode && (ctxt.lastIndex == (i & ~INDEX_MASK))) {+            // return reference to referenced+            return ctxt.lastNode->cell[i & INDEX_MASK];+        }++        // get snapshot of root+        auto info = getRootInfo();++        // check for emptiness+        if (info.root == nullptr) {+            // build new root node+            info.root = newNode();++            // initialize the new node+            info.root->parent = nullptr;+            info.offset = i & ~(INDEX_MASK);++            // try updating root information atomically+            if (tryUpdateRootInfo(info)) {+                // success -- finish get call++                // update first+                auto firstInfo = getFirstInfo();+                while (info.offset < firstInfo.offset) {+                    firstInfo.node = info.root;+                    firstInfo.offset = info.offset;+                    if (!tryUpdateFirstInfo(firstInfo)) {+                        // there was some concurrent update => check again+                        firstInfo = getFirstInfo();+                    }+                }++                // return reference to proper cell+                return info.root->cell[i & INDEX_MASK];+            }++            // somebody else was faster => use standard insertion procedure+            delete info.root;++            // retrieve new root info+            info = getRootInfo();++            // make sure there is a root+            assert(info.root);+        }++        // for all other inserts+        //   - check boundary+        //   - navigate to node+        //   - insert value++        // check boundaries+        while (!inBoundaries(i, info.levels, info.offset)) {+            // boundaries need to be expanded by growing upwards+            raiseLevel(info);  // try raising level unless someone else did already+            // update root info+            info = getRootInfo();+        }++        // navigate to node+        Node* node = info.root;+        unsigned level = info.levels;+        while (level != 0) {+            // get X coordinate+            auto x = getIndex(static_cast<RamDomain>(i), level);++            // decrease level counter+            --level;++            // check next node+            std::atomic<Node*>& aNext = node->cell[x].aptr;+            Node* next = aNext;+            if (!next) {+                // create new sub-tree+                Node* newNext = newNode();+                newNext->parent = node;++                // try to update next+                if (!aNext.compare_exchange_strong(next, newNext)) {+                    // some other thread was faster => use updated next+                    delete newNext;+                } else {+                    // the locally created next is the new next+                    next = newNext;++                    // update first+                    if (level == 0) {+                        // compute offset of this node+                        auto off = i & ~INDEX_MASK;++                        // fast over-approximation of whether a update is necessary+                        if (off < unsynced.firstOffset) {+                            // update first reference if this one is the smallest+                            auto first_info = getFirstInfo();+                            while (off < first_info.offset) {+                                first_info.node = next;+                                first_info.offset = off;+                                if (!tryUpdateFirstInfo(first_info)) {+                                    // there was some concurrent update => check again+                                    first_info = getFirstInfo();+                                }+                            }+                        }+                    }+                }++                // now next should be defined+                assert(next);+            }++            // continue one level below+            node = next;+        }++        // update context+        ctxt.lastIndex = (i & ~INDEX_MASK);+        ctxt.lastNode = node;++        // return reference to cell+        return node->cell[i & INDEX_MASK];+    }++public:+    /**+     * Updates the value stored in cell i by the given value.+     */+    void update(index_type i, const value_type& val) {+        op_context ctxt;+        update(i, val, ctxt);+    }++    /**+     * Updates the value stored in cell i by the given value. A operation+     * context can be provided for exploiting temporal locality.+     */+    void update(index_type i, const value_type& val, op_context& ctxt) {+        get(i, ctxt) = val;+    }++    /**+     * Obtains the value associated to index i -- which might be+     * the default value of the covered type if the value hasn't been+     * defined previously.+     */+    value_type operator[](index_type i) const {+        return lookup(i);+    }++    /**+     * Obtains the value associated to index i -- which might be+     * the default value of the covered type if the value hasn't been+     * defined previously.+     */+    value_type lookup(index_type i) const {+        op_context ctxt;+        return lookup(i, ctxt);+    }++    /**+     * Obtains the value associated to index i -- which might be+     * the default value of the covered type if the value hasn't been+     * defined previously. A operation context can be provided for+     * exploiting temporal locality.+     */+    value_type lookup(index_type i, op_context& ctxt) const {+        // check whether it is empty+        if (!unsynced.root) return souffle::detail::default_factory<value_type>()();++        // check boundaries+        if (!inBoundaries(i)) return souffle::detail::default_factory<value_type>()();++        // check context+        if (ctxt.lastNode && ctxt.lastIndex == (i & ~INDEX_MASK)) {+            return ctxt.lastNode->cell[i & INDEX_MASK].value;+        }++        // navigate to value+        Node* node = unsynced.root;+        unsigned level = unsynced.levels;+        while (level != 0) {+            // get X coordinate+            auto x = getIndex(static_cast<RamDomain>(i), level);++            // decrease level counter+            --level;++            // check next node+            Node* next = node->cell[x].ptr;++            // check next step+            if (!next) return souffle::detail::default_factory<value_type>()();++            // continue one level below+            node = next;+        }++        // remember context+        ctxt.lastIndex = (i & ~INDEX_MASK);+        ctxt.lastNode = node;++        // return reference to cell+        return node->cell[i & INDEX_MASK].value;+    }++private:+    /**+     * A static operation utilized internally for merging sub-trees recursively.+     *+     * @param parent the parent node of the current merge operation+     * @param trg a reference to the pointer the cloned node should be stored to+     * @param src the node to be cloned+     * @param levels the height of the cloned node+     */+    static void merge(const Node* parent, Node*& trg, const Node* src, int levels) {+        // if other side is null => done+        if (src == nullptr) {+            return;+        }++        // if the trg sub-tree is empty, clone the corresponding branch+        if (trg == nullptr) {+            trg = clone(src, levels);+            if (trg != nullptr) {+                trg->parent = parent;+            }+            return;  // done+        }++        // otherwise merge recursively++        // the leaf-node step+        if (levels == 0) {+            merge_op merg;+            for (int i = 0; i < NUM_CELLS; ++i) {+                trg->cell[i].value = merg(trg->cell[i].value, src->cell[i].value);+            }+            return;+        }++        // the recursive step+        for (int i = 0; i < NUM_CELLS; ++i) {+            merge(trg, trg->cell[i].ptr, src->cell[i].ptr, levels - 1);+        }+    }++public:+    /**+     * Adds all the values stored in the given array to this array.+     */+    void addAll(const SparseArray& other) {+        // skip if other is empty+        if (other.empty()) {+            return;+        }++        // special case: emptiness+        if (empty()) {+            // use assignment operator+            *this = other;+            return;+        }++        // adjust levels+        while (unsynced.levels < other.unsynced.levels || !inBoundaries(other.unsynced.offset)) {+            raiseLevel();+        }++        // navigate to root node equivalent of the other node in this tree+        auto level = unsynced.levels;+        Node** node = &unsynced.root;+        while (level > other.unsynced.levels) {+            // get X coordinate+            auto x = getIndex(static_cast<RamDomain>(other.unsynced.offset), level);++            // decrease level counter+            --level;++            // check next node+            Node*& next = (*node)->cell[x].ptr;+            if (!next) {+                // create new sub-tree+                next = newNode();+                next->parent = *node;+            }++            // continue one level below+            node = &next;+        }++        // merge sub-branches from here+        merge((*node)->parent, *node, other.unsynced.root, level);++        // update first+        if (unsynced.firstOffset > other.unsynced.firstOffset) {+            unsynced.first = findFirst(*node, level);+            unsynced.firstOffset = other.unsynced.firstOffset;+        }+    }++    // ---------------------------------------------------------------------+    //                           Iterator+    // ---------------------------------------------------------------------++    /**+     * The iterator type to be utilized to iterate over the non-default elements of this array.+     */+    class iterator {+        using pair_type = std::pair<index_type, value_type>;++        // a pointer to the leaf node currently processed or null (end)+        const Node* node;++        // the value currently pointed to+        pair_type value;++    public:+        // default constructor -- creating an end-iterator+        iterator() : node(nullptr) {}++        iterator(const Node* node, pair_type value) : node(node), value(std::move(value)) {}++        iterator(const Node* first, index_type firstOffset) : node(first), value(firstOffset, 0) {+            // if the start is the end => we are done+            if (!first) return;++            // load the value+            if (first->cell[0].value == value_type()) {+                ++(*this);  // walk to first element+            } else {+                value.second = first->cell[0].value;+            }+        }++        // a copy constructor+        iterator(const iterator& other) = default;++        // an assignment operator+        iterator& operator=(const iterator& other) = default;++        // the equality operator as required by the iterator concept+        bool operator==(const iterator& other) const {+            // only equivalent if pointing to the end+            return (node == nullptr && other.node == nullptr) ||+                   (node == other.node && value.first == other.value.first);+        }++        // the not-equality operator as required by the iterator concept+        bool operator!=(const iterator& other) const {+            return !(*this == other);+        }++        // the deref operator as required by the iterator concept+        const pair_type& operator*() const {+            return value;+        }++        // support for the pointer operator+        const pair_type* operator->() const {+            return &value;+        }++        // the increment operator as required by the iterator concept+        iterator& operator++() {+            // get current offset+            index_type x = value.first & INDEX_MASK;++            // go to next non-empty value in current node+            do {+                x++;+            } while (x < NUM_CELLS && node->cell[x].value == value_type());++            // check whether one has been found+            if (x < NUM_CELLS) {+                // update value and be done+                value.first = (value.first & ~INDEX_MASK) | x;+                value.second = node->cell[x].value;+                return *this;  // done+            }++            // go to parent+            node = node->parent;+            int level = 1;++            // get current index on this level+            x = getIndex(static_cast<RamDomain>(value.first), level);+            x++;++            while (level > 0 && node) {+                // search for next child+                while (x < NUM_CELLS) {+                    if (node->cell[x].ptr != nullptr) {+                        break;+                    }+                    x++;+                }++                // pick next step+                if (x < NUM_CELLS) {+                    // going down+                    node = node->cell[x].ptr;+                    value.first &= getLevelMask(level + 1);+                    value.first |= x << (BIT_PER_STEP * level);+                    level--;+                    x = 0;+                } else {+                    // going up+                    node = node->parent;+                    level++;++                    // get current index on this level+                    x = getIndex(static_cast<RamDomain>(value.first), level);+                    x++;  // go one step further+                }+            }++            // check whether it is the end of range+            if (node == nullptr) {+                return *this;+            }++            // search the first value in this node+            x = 0;+            while (node->cell[x].value == value_type()) {+                x++;+            }++            // update value+            value.first |= x;+            value.second = node->cell[x].value;++            // done+            return *this;+        }++        // True if this iterator is passed the last element.+        bool isEnd() const {+            return node == nullptr;+        }++        // enables this iterator core to be printed (for debugging)+        void print(std::ostream& out) const {+            out << "SparseArrayIter(" << node << " @ " << value << ")";+        }++        friend std::ostream& operator<<(std::ostream& out, const iterator& iter) {+            iter.print(out);+            return out;+        }+    };++    /**+     * Obtains an iterator referencing the first non-default element or end in+     * case there are no such elements.+     */+    iterator begin() const {+        return iterator(unsynced.first, unsynced.firstOffset);+    }++    /**+     * An iterator referencing the position after the last non-default element.+     */+    iterator end() const {+        return iterator();+    }++    /**+     * An operation to obtain an iterator referencing an element addressed by the+     * given index. If the corresponding element is a non-default value, a corresponding+     * iterator will be returned. Otherwise end() will be returned.+     */+    iterator find(index_type i) const {+        op_context ctxt;+        return find(i, ctxt);+    }++    /**+     * An operation to obtain an iterator referencing an element addressed by the+     * given index. If the corresponding element is a non-default value, a corresponding+     * iterator will be returned. Otherwise end() will be returned. A operation context+     * can be provided for exploiting temporal locality.+     */+    iterator find(index_type i, op_context& ctxt) const {+        // check whether it is empty+        if (!unsynced.root) return end();++        // check boundaries+        if (!inBoundaries(i)) return end();++        // check context+        if (ctxt.lastNode && ctxt.lastIndex == (i & ~INDEX_MASK)) {+            Node* node = ctxt.lastNode;++            // check whether there is a proper entry+            value_type value = node->cell[i & INDEX_MASK].value;+            if (value == value_type{}) {+                return end();+            }+            // return iterator pointing to value+            return iterator(node, std::make_pair(i, value));+        }++        // navigate to value+        Node* node = unsynced.root;+        unsigned level = unsynced.levels;+        while (level != 0) {+            // get X coordinate+            auto x = getIndex(i, level);++            // decrease level counter+            --level;++            // check next node+            Node* next = node->cell[x].ptr;++            // check next step+            if (!next) return end();++            // continue one level below+            node = next;+        }++        // register in context+        ctxt.lastNode = node;+        ctxt.lastIndex = (i & ~INDEX_MASK);++        // check whether there is a proper entry+        value_type value = node->cell[i & INDEX_MASK].value;+        if (value == value_type{}) {+            return end();+        }++        // return iterator pointing to cell+        return iterator(node, std::make_pair(i, value));+    }++    /**+     * An operation obtaining the smallest non-default element such that it's index is >=+     * the given index.+     */+    iterator lowerBound(index_type i) const {+        op_context ctxt;+        return lowerBound(i, ctxt);+    }++    /**+     * An operation obtaining the smallest non-default element such that it's index is >=+     * the given index. A operation context can be provided for exploiting temporal locality.+     */+    iterator lowerBound(index_type i, op_context&) const {+        // check whether it is empty+        if (!unsynced.root) return end();++        // check boundaries+        if (!inBoundaries(i)) {+            // if it is on the lower end, return minimum result+            if (i < unsynced.offset) {+                const auto& value = unsynced.first->cell[0].value;+                auto res = iterator(unsynced.first, std::make_pair(unsynced.offset, value));+                if (value == value_type()) {+                    ++res;+                }+                return res;+            }+            // otherwise it is on the high end, return end iterator+            return end();+        }++        // navigate to value+        Node* node = unsynced.root;+        unsigned level = unsynced.levels;+        while (true) {+            // get X coordinate+            auto x = getIndex(static_cast<RamDomain>(i), level);++            // check next node+            Node* next = node->cell[x].ptr;++            // check next step+            if (!next) {+                if (x == NUM_CELLS - 1) {+                    ++level;+                    node = const_cast<Node*>(node->parent);+                    if (!node) return end();+                }++                // continue search+                i = i & getLevelMask(level);++                // find next higher value+                i += 1ull << (BITS * level);++            } else {+                if (level == 0) {+                    // found boundary+                    return iterator(node, std::make_pair(i, node->cell[x].value));+                }++                // decrease level counter+                --level;++                // continue one level below+                node = next;+            }+        }+    }++    /**+     * An operation obtaining the smallest non-default element such that it's index is greater+     * the given index.+     */+    iterator upperBound(index_type i) const {+        op_context ctxt;+        return upperBound(i, ctxt);+    }++    /**+     * An operation obtaining the smallest non-default element such that it's index is greater+     * the given index. A operation context can be provided for exploiting temporal locality.+     */+    iterator upperBound(index_type i, op_context& ctxt) const {+        if (i == std::numeric_limits<index_type>::max()) {+            return end();+        }+        return lowerBound(i + 1, ctxt);+    }++private:+    /**+     * An internal debug utility printing the internal structure of this sparse array to the given output+     * stream.+     */+    void dump(bool detailed, std::ostream& out, const Node& node, int level, index_type offset,+            int indent = 0) const {+        auto x = getIndex(offset, level + 1);+        out << times("\t", indent) << x << ": Node " << &node << " on level " << level+            << " parent: " << node.parent << " -- range: " << offset << " - "+            << (offset + ~getLevelMask(level + 1)) << "\n";++        if (level == 0) {+            for (int i = 0; i < NUM_CELLS; i++) {+                if (detailed || node.cell[i].value != value_type()) {+                    out << times("\t", indent + 1) << i << ": [" << (offset + i) << "] " << node.cell[i].value+                        << "\n";+                }+            }+        } else {+            for (int i = 0; i < NUM_CELLS; i++) {+                if (node.cell[i].ptr) {+                    dump(detailed, out, *node.cell[i].ptr, level - 1,+                            offset + (i * (index_type(1) << (level * BIT_PER_STEP))), indent + 1);+                } else if (detailed) {+                    auto low = offset + (i * (1 << (level * BIT_PER_STEP)));+                    auto hig = low + ~getLevelMask(level);+                    out << times("\t", indent + 1) << i << ": empty range " << low << " - " << hig << "\n";+                }+            }+        }+        out << "\n";+    }++public:+    /**+     * A debug utility printing the internal structure of this sparse array to the given output stream.+     */+    void dump(bool detail = false, std::ostream& out = std::cout) const {+        if (!unsynced.root) {+            out << " - empty - \n";+            return;+        }+        out << "root:  " << unsynced.root << "\n";+        out << "offset: " << unsynced.offset << "\n";+        out << "first: " << unsynced.first << "\n";+        out << "fist offset: " << unsynced.firstOffset << "\n";+        dump(detail, out, *unsynced.root, unsynced.levels, unsynced.offset);+    }++private:+    // --------------------------------------------------------------------------+    //                                 Utilities+    // --------------------------------------------------------------------------++    /**+     * Creates new nodes and initializes them with 0.+     */+    static Node* newNode() {+        return new Node();+    }++    /**+     * Destroys a node and all its sub-nodes recursively.+     */+    static void freeNodes(Node* node, int level) {+        if (!node) return;+        if (level != 0) {+            for (int i = 0; i < NUM_CELLS; i++) {+                freeNodes(node->cell[i].ptr, level - 1);+            }+        }+        delete node;+    }++    /**+     * Conducts a cleanup of the internal tree structure.+     */+    void clean() {+        freeNodes(unsynced.root, unsynced.levels);+        unsynced.root = nullptr;+        unsynced.levels = 0;+    }++    /**+     * Clones the given node and all its sub-nodes.+     */+    static Node* clone(const Node* node, int level) {+        // support null-pointers+        if (node == nullptr) {+            return nullptr;+        }++        // create a clone+        auto* res = new Node();++        // handle leaf level+        if (level == 0) {+            copy_op copy;+            for (int i = 0; i < NUM_CELLS; i++) {+                res->cell[i].value = copy(node->cell[i].value);+            }+            return res;+        }++        // for inner nodes clone each child+        for (int i = 0; i < NUM_CELLS; i++) {+            auto cur = clone(node->cell[i].ptr, level - 1);+            if (cur != nullptr) {+                cur->parent = res;+            }+            res->cell[i].ptr = cur;+        }++        // done+        return res;+    }++    /**+     * Obtains the left-most leaf-node of the tree rooted by the given node+     * with the given level.+     */+    static Node* findFirst(Node* node, int level) {+        while (level > 0) {+            bool found = false;+            for (int i = 0; i < NUM_CELLS; i++) {+                Node* cur = node->cell[i].ptr;+                if (cur) {+                    node = cur;+                    --level;+                    found = true;+                    break;+                }+            }+            assert(found && "No first node!");+        }++        return node;+    }++    /**+     * Raises the level of this tree by one level. It does so by introducing+     * a new root node and inserting the current root node as a child node.+     */+    void raiseLevel() {+        // something went wrong when we pass that line+        assert(unsynced.levels < (sizeof(index_type) * 8 / BITS) + 1);++        // create new root+        Node* node = newNode();+        node->parent = nullptr;++        // insert existing root as child+        auto x = getIndex(static_cast<RamDomain>(unsynced.offset), unsynced.levels + 1);+        node->cell[x].ptr = unsynced.root;++        // swap the root+        unsynced.root->parent = node;++        // update root+        unsynced.root = node;+        ++unsynced.levels;++        // update offset be removing additional bits+        unsynced.offset &= getLevelMask(unsynced.levels + 1);+    }++    /**+     * Attempts to raise the height of this tree based on the given root node+     * information and updates the root-info snapshot correspondingly.+     */+    void raiseLevel(RootInfoSnapshot& info) {+        // something went wrong when we pass that line+        assert(info.levels < (sizeof(index_type) * 8 / BITS) + 1);++        // create new root+        Node* newRoot = newNode();+        newRoot->parent = nullptr;++        // insert existing root as child+        auto x = getIndex(static_cast<RamDomain>(info.offset), info.levels + 1);+        newRoot->cell[x].ptr = info.root;++        // exchange the root in the info struct+        auto oldRoot = info.root;+        info.root = newRoot;++        // update level counter+        ++info.levels;++        // update offset+        info.offset &= getLevelMask(info.levels + 1);++        // try exchanging root info+        if (tryUpdateRootInfo(info)) {+            // success => final step, update parent of old root+            oldRoot->parent = info.root;+        } else {+            // throw away temporary new node+            delete newRoot;+        }+    }++    /**+     * Tests whether the given index is covered by the boundaries defined+     * by the hight and offset of the internally maintained tree.+     */+    bool inBoundaries(index_type a) const {+        return inBoundaries(a, unsynced.levels, unsynced.offset);+    }++    /**+     * Tests whether the given index is within the boundaries defined by the+     * given tree hight and offset.+     */+    static bool inBoundaries(index_type a, uint32_t levels, index_type offset) {+        auto mask = getLevelMask(levels + 1);+        return (a & mask) == offset;+    }++    /**+     * Obtains the index within the arrays of cells of a given index on a given+     * level of the internally maintained tree.+     */+    static index_type getIndex(RamDomain a, unsigned level) {+        return (a & (INDEX_MASK << (level * BIT_PER_STEP))) >> (level * BIT_PER_STEP);+    }++    /**+     * Computes the bit-mask to be applicable to obtain the offset of a node on a+     * given tree level.+     */+    static index_type getLevelMask(unsigned level) {+        if (level > (sizeof(index_type) * 8 / BITS)) return 0;+        return (~(index_type(0)) << (level * BIT_PER_STEP));+    }+};++/**+ * A sparse bit-map is a bit map virtually assigning a bit value to every value if the+ * uint32_t domain. However, only 1-bits are stored utilizing a nested sparse array+ * structure.+ *+ * @tparam BITS similar to the BITS parameter of the sparse array type+ */+template <unsigned BITS = 4>+class SparseBitMap {+    // the element type stored in the nested sparse array+    using value_t = uint64_t;++    // define the bit-level merge operation+    struct merge_op {+        value_t operator()(value_t a, value_t b) const {+            return a | b;  // merging bit masks => bitwise or operation+        }+    };++    // the type of the internal data store+    using data_store_t = SparseArray<value_t, BITS, merge_op>;+    using atomic_value_t = typename data_store_t::atomic_value_type;++    // some constants for manipulating stored values+    static constexpr short BITS_PER_ENTRY = sizeof(value_t) * 8;+    static constexpr short LEAF_INDEX_WIDTH = static_cast<short>(__builtin_ctz(BITS_PER_ENTRY));+    static constexpr uint64_t LEAF_INDEX_MASK = BITS_PER_ENTRY - 1;++public:+    // the type to address individual entries+    using index_type = typename data_store_t::index_type;++private:+    // it utilizes a sparse map to store its data+    data_store_t store;++public:+    // a simple default constructor+    SparseBitMap() = default;++    // a default copy constructor+    SparseBitMap(const SparseBitMap&) = default;++    // a default r-value copy constructor+    SparseBitMap(SparseBitMap&&) = default;++    // a default assignment operator+    SparseBitMap& operator=(const SparseBitMap&) = default;++    // a default r-value assignment operator+    SparseBitMap& operator=(SparseBitMap&&) = default;++    // checks whether this bit-map is empty -- thus it does not have any 1-entries+    bool empty() const {+        return store.empty();+    }++    // the type utilized for recording context information for exploiting temporal locality+    using op_context = typename data_store_t::op_context;++    /**+     * Sets the bit addressed by i to 1.+     */+    bool set(index_type i) {+        op_context ctxt;+        return set(i, ctxt);+    }++    /**+     * Sets the bit addressed by i to 1. A context for exploiting temporal locality+     * can be provided.+     */+    bool set(index_type i, op_context& ctxt) {+        atomic_value_t& val = store.getAtomic(i >> LEAF_INDEX_WIDTH, ctxt);+        value_t bit = (1ull << (i & LEAF_INDEX_MASK));++#ifdef __GNUC__+#if __GNUC__ >= 7+        // In GCC >= 7 the usage of fetch_or causes a bug that needs further investigation+        // For now, this two-instruction based implementation provides a fix that does+        // not sacrifice too much performance.++        while (true) {+            auto order = std::memory_order::memory_order_relaxed;++            // load current value+            value_t old = val.load(order);++            // if bit is already set => we are done+            if (old & bit) return false;++            // set the bit, if failed, repeat+            if (!val.compare_exchange_strong(old, old | bit, order, order)) continue;++            // it worked, new bit added+            return true;+        }++#endif+#endif++        value_t old = val.fetch_or(bit, std::memory_order::memory_order_relaxed);+        return (old & bit) == 0u;+    }++    /**+     * Determines the whether the bit addressed by i is set or not.+     */+    bool test(index_type i) const {+        op_context ctxt;+        return test(i, ctxt);+    }++    /**+     * Determines the whether the bit addressed by i is set or not. A context for+     * exploiting temporal locality can be provided.+     */+    bool test(index_type i, op_context& ctxt) const {+        value_t bit = (1ull << (i & LEAF_INDEX_MASK));+        return store.lookup(i >> LEAF_INDEX_WIDTH, ctxt) & bit;+    }++    /**+     * Determines the whether the bit addressed by i is set or not.+     */+    bool operator[](index_type i) const {+        return test(i);+    }++    /**+     * Resets all contained bits to 0.+     */+    void clear() {+        store.clear();+    }++    /**+     * Determines the number of bits set.+     */+    std::size_t size() const {+        // this is computed on demand to keep the set operation simple.+        std::size_t res = 0;+        for (const auto& cur : store) {+            res += __builtin_popcountll(cur.second);+        }+        return res;+    }++    /**+     * Computes the total memory usage of this data structure.+     */+    std::size_t getMemoryUsage() const {+        // compute the total memory usage+        return sizeof(*this) - sizeof(data_store_t) + store.getMemoryUsage();+    }++    /**+     * Sets all bits set in other to 1 within this bit map.+     */+    void addAll(const SparseBitMap& other) {+        // nothing to do if it is a self-assignment+        if (this == &other) return;++        // merge the sparse store+        store.addAll(other.store);+    }++    // ---------------------------------------------------------------------+    //                           Iterator+    // ---------------------------------------------------------------------++    /**+     * An iterator iterating over all indices set to 1.+     */+    class iterator {+        using nested_iterator = typename data_store_t::iterator;++        // the iterator through the underlying sparse data structure+        nested_iterator iter;++        // the currently consumed mask+        uint64_t mask = 0;++        // the value currently pointed to+        index_type value{};++    public:+        typedef std::forward_iterator_tag iterator_category;+        typedef index_type value_type;+        typedef ptrdiff_t difference_type;+        typedef value_type* pointer;+        typedef value_type& reference;++        // default constructor -- creating an end-iterator+        iterator() = default;++        iterator(const nested_iterator& iter)+                : iter(iter), mask(toMask(iter->second)), value(iter->first << LEAF_INDEX_WIDTH) {+            moveToNextInMask();+        }++        iterator(const nested_iterator& iter, uint64_t m, index_type value)+                : iter(iter), mask(m), value(value) {}++        // a copy constructor+        iterator(const iterator& other) = default;++        // an assignment operator+        iterator& operator=(const iterator& other) = default;++        // the equality operator as required by the iterator concept+        bool operator==(const iterator& other) const {+            // only equivalent if pointing to the end+            return iter == other.iter && mask == other.mask;+        }++        // the not-equality operator as required by the iterator concept+        bool operator!=(const iterator& other) const {+            return !(*this == other);+        }++        // the deref operator as required by the iterator concept+        const index_type& operator*() const {+            return value;+        }++        // support for the pointer operator+        const index_type* operator->() const {+            return &value;+        }++        // the increment operator as required by the iterator concept+        iterator& operator++() {+            // progress in current mask+            if (moveToNextInMask()) return *this;++            // go to next entry+            ++iter;++            // update value+            if (!iter.isEnd()) {+                value = iter->first << LEAF_INDEX_WIDTH;+                mask = toMask(iter->second);+                moveToNextInMask();+            }++            // done+            return *this;+        }++        bool isEnd() const {+            return iter.isEnd();+        }++        void print(std::ostream& out) const {+            out << "SparseBitMapIter(" << iter << " -> " << std::bitset<64>(mask) << " @ " << value << ")";+        }++        // enables this iterator core to be printed (for debugging)+        friend std::ostream& operator<<(std::ostream& out, const iterator& iter) {+            iter.print(out);+            return out;+        }++        static uint64_t toMask(const value_t& value) {+            static_assert(sizeof(value_t) == sizeof(uint64_t), "Fixed for 64-bit compiler.");+            return reinterpret_cast<const uint64_t&>(value);+        }++    private:+        bool moveToNextInMask() {+            // check if there is something left+            if (mask == 0) return false;++            // get position of leading 1+            auto pos = __builtin_ctzll(mask);++            // consume this bit+            mask &= ~(1llu << pos);++            // update value+            value &= ~LEAF_INDEX_MASK;+            value |= pos;++            // done+            return true;+        }+    };++    /**+     * Obtains an iterator pointing to the first index set to 1. If there+     * is no such bit, end() will be returned.+     */+    iterator begin() const {+        auto it = store.begin();+        if (it.isEnd()) return end();+        return iterator(it);+    }++    /**+     * Returns an iterator referencing the position after the last set bit.+     */+    iterator end() const {+        return iterator();+    }++    /**+     * Obtains an iterator referencing the position i if the corresponding+     * bit is set, end() otherwise.+     */+    iterator find(index_type i) const {+        op_context ctxt;+        return find(i, ctxt);+    }++    /**+     * Obtains an iterator referencing the position i if the corresponding+     * bit is set, end() otherwise. An operation context can be provided+     * to exploit temporal locality.+     */+    iterator find(index_type i, op_context& ctxt) const {+        // check prefix part+        auto it = store.find(i >> LEAF_INDEX_WIDTH, ctxt);+        if (it.isEnd()) return end();++        // check bit-set part+        uint64_t mask = iterator::toMask(it->second);+        if (!(mask & (1llu << (i & LEAF_INDEX_MASK)))) return end();++        // OK, it is there => create iterator+        mask &= ((1ull << (i & LEAF_INDEX_MASK)) - 1);  // remove all bits before pos i+        return iterator(it, mask, i);+    }++    /**+     * Locates an iterator to the first element in this sparse bit map not less+     * than the given index.+     */+    iterator lower_bound(index_type i) const {+        auto it = store.lowerBound(i >> LEAF_INDEX_WIDTH);+        if (it.isEnd()) return end();++        // check bit-set part+        uint64_t mask = iterator::toMask(it->second);++        // if there is no bit remaining in this mask, check next mask.+        if (!(mask & ((~uint64_t(0)) << (i & LEAF_INDEX_MASK)))) {+            index_type next = ((i >> LEAF_INDEX_WIDTH) + 1) << LEAF_INDEX_WIDTH;+            if (next < i) return end();+            return lower_bound(next);+        }++        // there are bits left, use least significant bit of those+        if (it->first == i >> LEAF_INDEX_WIDTH) {+            mask &= ((~uint64_t(0)) << (i & LEAF_INDEX_MASK));  // remove all bits before pos i+        }++        // compute value represented by least significant bit+        index_type pos = __builtin_ctzll(mask);++        // remove this bit as well+        mask = mask & ~(1ull << pos);++        // construct value of this located bit+        index_type val = (it->first << LEAF_INDEX_WIDTH) | pos;+        return iterator(it, mask, val);+    }++    /**+     * Locates an iterator to the first element in this sparse bit map than is greater+     * than the given index.+     */+    iterator upper_bound(index_type i) const {+        if (i == std::numeric_limits<index_type>::max()) {+            return end();+        }+        return lower_bound(i + 1);+    }++    /**+     * A debugging utility printing the internal structure of this map to the+     * given output stream.+     */+    void dump(bool detail = false, std::ostream& out = std::cout) const {+        store.dump(detail, out);+    }++    /**+     * Provides write-protected access to the internal store for running+     * analysis on the data structure.+     */+    const data_store_t& getStore() const {+        return store;+    }+};++// ---------------------------------------------------------------------+//                              TRIE+// ---------------------------------------------------------------------++namespace detail {++/**+ * A base class for the Trie implementation allowing various+ * specializations of the Trie template to inherit common functionality.+ *+ * @tparam Dim the number of dimensions / arity of the stored tuples+ * @tparam Derived the type derived from this base class+ */+template <unsigned Dim, typename Derived>+class TrieBase {+public:+    /**+     * The type of the stored entries / tuples.+     */+    using entry_type = typename souffle::Tuple<RamDomain, Dim>;++    // -- operation wrappers --++    /**+     * A generic function enabling the insertion of tuple values in a user-friendly way.+     */+    template <typename... Values>+    bool insert(Values... values) {+        return static_cast<Derived&>(*this).insert(entry_type{{RamDomain(values)...}});+    }++    /**+     * A generic function enabling the convenient conduction of a membership check.+     */+    template <typename... Values>+    bool contains(Values... values) const {+        return static_cast<const Derived&>(*this).contains(entry_type{{RamDomain(values)...}});+    }++    // ---------------------------------------------------------------------+    //                           Iterator+    // ---------------------------------------------------------------------++    /**+     * An iterator over the stored entries.+     *+     * Iterators for tries consist of a top-level iterator maintaining the+     * master copy of a materialized tuple and a recursively nested iterator+     * core -- one for each nested trie level.+     */+    template <template <unsigned D> class IterCore>+    class iterator {+        template <unsigned Len, unsigned Pos, unsigned Dimensions>+        friend struct fix_binding;++        template <unsigned Pos, unsigned Dimensions>+        friend struct fix_lower_bound;++        template <unsigned Pos, unsigned Dimensions>+        friend struct fix_upper_bound;++        template <unsigned Pos, unsigned Dimensions>+        friend struct fix_first;++        // the iterator core of this level+        using iter_core_t = IterCore<0>;++        // the wrapped iterator+        iter_core_t iter_core;++        // the value currently pointed to+        entry_type value;++    public:+        typedef std::forward_iterator_tag iterator_category;+        typedef entry_type value_type;+        typedef ptrdiff_t difference_type;+        typedef value_type* pointer;+        typedef value_type& reference;++        // default constructor -- creating an end-iterator+        iterator() = default;++        // a copy constructor+        iterator(const iterator& other) = default;++        iterator(iterator&& other) = default;++        template <typename Param>+        explicit iterator(const Param& param) : iter_core(param, value) {}++        // an assignment operator+        iterator& operator=(const iterator& other) = default;++        // the equality operator as required by the iterator concept+        bool operator==(const iterator& other) const {+            // equivalent if pointing to the same value+            return iter_core == other.iter_core;+        }++        // the not-equality operator as required by the iterator concept+        bool operator!=(const iterator& other) const {+            return !(*this == other);+        }++        // the deref operator as required by the iterator concept+        const entry_type& operator*() const {+            return value;+        }++        // support for the pointer operator+        const entry_type* operator->() const {+            return &value;+        }++        // the increment operator as required by the iterator concept+        iterator& operator++() {+            iter_core.inc(value);+            return *this;+        }++        // enables this iterator to be printed (for debugging)+        void print(std::ostream& out) const {+            out << "iter(" << iter_core << " -> " << value << ")";+        }++        friend std::ostream& operator<<(std::ostream& out, const iterator& iter) {+            iter.print(out);+            return out;+        }+    };++    /* -------------- operator hint statistics ----------------- */++    // an aggregation of statistical values of the hint utilization+    struct hint_statistics {+        // the counter for insertion operations+        CacheAccessCounter inserts;++        // the counter for contains operations+        CacheAccessCounter contains;++        // the counter for get_boundaries operations+        CacheAccessCounter get_boundaries;+    };++protected:+    // the hint statistic of this b-tree instance+    mutable hint_statistics hint_stats;++public:+    void printStats(std::ostream& out) const {+        out << "---------------------------------\n";+        out << "  insert-hint (hits/misses/total): " << hint_stats.inserts.getHits() << "/"+            << hint_stats.inserts.getMisses() << "/" << hint_stats.inserts.getAccesses() << "\n";+        out << "  contains-hint (hits/misses/total):" << hint_stats.contains.getHits() << "/"+            << hint_stats.contains.getMisses() << "/" << hint_stats.contains.getAccesses() << "\n";+        out << "  get-boundaries-hint (hits/misses/total):" << hint_stats.get_boundaries.getHits() << "/"+            << hint_stats.get_boundaries.getMisses() << "/" << hint_stats.get_boundaries.getAccesses()+            << "\n";+        out << "---------------------------------\n";+    }+};++/**+ * A functor extracting a reference to a nested iterator core from an enclosing+ * iterator core.+ */+template <unsigned Level>+struct get_nested_iter_core {+    template <typename IterCore>+    auto operator()(IterCore& core) -> decltype(get_nested_iter_core<Level - 1>()(core.getNested())) {+        return get_nested_iter_core<Level - 1>()(core.getNested());+    }+};++template <>+struct get_nested_iter_core<0> {+    template <typename IterCore>+    IterCore& operator()(IterCore& core) {+        return core;+    }+};++/**+ * A functor initializing an iterator upon creation to reference the first+ * element in the associated Trie.+ */+template <unsigned Pos, unsigned Dim>+struct fix_first {+    template <unsigned bits, typename iterator>+    void operator()(const SparseBitMap<bits>& store, iterator& iter) const {+        // set iterator to first in store+        auto first = store.begin();+        get_nested_iter_core<Pos>()(iter.iter_core).setIterator(first);+        iter.value[Pos] = *first;+    }++    template <typename Store, typename iterator>+    void operator()(const Store& store, iterator& iter) const {+        // set iterator to first in store+        auto first = store.begin();+        get_nested_iter_core<Pos>()(iter.iter_core).setIterator(first);+        iter.value[Pos] = first->first;+        // and continue recursively+        fix_first<Pos + 1, Dim>()(first->second->getStore(), iter);+    }+};++template <unsigned Dim>+struct fix_first<Dim, Dim> {+    template <typename Store, typename iterator>+    void operator()(const Store&, iterator&) const {+        // terminal case => nothing to do+    }+};++/**+ * A functor initializing an iterator upon creation to reference the first element+ * exhibiting a given prefix within a given Trie.+ */+template <unsigned Len, unsigned Pos, unsigned Dim>+struct fix_binding {+    template <unsigned bits, typename iterator, typename entry_type>+    bool operator()(+            const SparseBitMap<bits>& store, iterator& begin, iterator& end, const entry_type& entry) const {+        // search in current level+        auto cur = store.find(entry[Pos]);++        // if not present => fail+        if (cur == store.end()) return false;++        // take current value+        get_nested_iter_core<Pos>()(begin.iter_core).setIterator(cur);+        ++cur;+        get_nested_iter_core<Pos>()(end.iter_core).setIterator(cur);++        // update iterator value+        begin.value[Pos] = entry[Pos];++        // no more remaining levels to fix+        return true;+    }++    template <typename Store, typename iterator, typename entry_type>+    bool operator()(const Store& store, iterator& begin, iterator& end, const entry_type& entry) const {+        // search in current level+        auto cur = store.find(entry[Pos]);++        // if not present => fail+        if (cur == store.end()) return false;++        // take current value as start+        get_nested_iter_core<Pos>()(begin.iter_core).setIterator(cur);++        // update iterator value+        begin.value[Pos] = entry[Pos];++        // fix remaining nested iterators+        auto res = fix_binding<Len - 1, Pos + 1, Dim>()(cur->second->getStore(), begin, end, entry);++        // update end of iterator+        if (get_nested_iter_core<Pos + 1>()(end.iter_core).getIterator() == cur->second->getStore().end()) {+            ++cur;+            if (cur != store.end()) {+                fix_first<Pos + 1, Dim>()(cur->second->getStore(), end);+            }+        }+        get_nested_iter_core<Pos>()(end.iter_core).setIterator(cur);++        // done+        return res;+    }+};++template <unsigned Pos, unsigned Dim>+struct fix_binding<0, Pos, Dim> {+    template <unsigned bits, typename iterator, typename entry_type>+    bool operator()(const SparseBitMap<bits>& store, iterator& begin, iterator& /* end */,+            const entry_type& /* entry */) const {+        // move begin to begin of store+        auto a = store.begin();+        get_nested_iter_core<Pos>()(begin.iter_core).setIterator(a);+        begin.value[Pos] = *a;++        return true;+    }++    template <typename Store, typename iterator, typename entry_type>+    bool operator()(const Store& store, iterator& begin, iterator& end, const entry_type& entry) const {+        // move begin to begin of store+        auto a = store.begin();+        get_nested_iter_core<Pos>()(begin.iter_core).setIterator(a);+        begin.value[Pos] = a->first;++        // continue recursively+        fix_binding<0, Pos + 1, Dim>()(a->second->getStore(), begin, end, entry);+        return true;+    }+};++template <unsigned Dim>+struct fix_binding<0, Dim, Dim> {+    template <typename Store, typename iterator, typename entry_type>+    bool operator()(const Store& /* store */, iterator& /* begin */, iterator& /* end */,+            const entry_type& /* entry */) const {+        // nothing more to do+        return true;+    }+};++/**+ * A functor initializing an iterator upon creation to reference the first element+ * within a given Trie being not less than a given value .+ */+template <unsigned Pos, unsigned Dim>+struct fix_lower_bound {+    template <unsigned bits, typename iterator, typename entry_type>+    bool operator()(const SparseBitMap<bits>& store, iterator& iter, const entry_type& entry) const {+        // search in current level+        auto cur = store.lower_bound(entry[Pos]);++        if (cur == store.end()) return false;++        get_nested_iter_core<Pos>()(iter.iter_core).setIterator(cur);++        assert(entry[Pos] <= RamDomain(*cur));+        iter.value[Pos] = *cur;++        // no more remaining levels to fix+        return true;+    }++    template <typename Store, typename iterator, typename entry_type>+    bool operator()(const Store& store, iterator& iter, const entry_type& entry) const {+        // search in current level+        auto cur = store.lowerBound(entry[Pos]);++        // if no lower boundary is found, be done+        if (cur == store.end()) return false;+        assert(RamDomain(cur->first) >= entry[Pos]);++        // if the lower bound is higher than the requested value, go to first in subtree+        if (RamDomain(cur->first) > entry[Pos]) {+            get_nested_iter_core<Pos>()(iter.iter_core).setIterator(cur);+            iter.value[Pos] = cur->first;+            fix_first<Pos + 1, Dim>()(cur->second->getStore(), iter);+            return true;+        }++        // attempt to fix the rest+        if (!fix_lower_bound<Pos + 1, Dim>()(cur->second->getStore(), iter, entry)) {+            // if it does not work, since there are no matching elements in this branch, go to next+            entry_type sub = entry;+            sub[Pos] += 1;+            for (size_t i = Pos + 1; i < Dim; ++i) {+                sub[i] = 0;+            }+            return (*this)(store, iter, sub);+        }++        // remember result+        get_nested_iter_core<Pos>()(iter.iter_core).setIterator(cur);++        // update iterator value+        iter.value[Pos] = cur->first;++        // done!+        return true;+    }+};++/**+ * A functor initializing an iterator upon creation to reference the first element+ * within a given Trie being greater than a given value .+ */+template <unsigned Pos, unsigned Dim>+struct fix_upper_bound {+    template <unsigned bits, typename iterator, typename entry_type>+    bool operator()(const SparseBitMap<bits>& store, iterator& iter, const entry_type& entry) const {+        // search in current level+        auto cur = store.upper_bound(entry[Pos]);++        if (cur == store.end()) {+            return false;+        }++        get_nested_iter_core<Pos>()(iter.iter_core).setIterator(cur);++        assert(entry[Pos] <= RamDomain(*cur));+        iter.value[Pos] = *cur;++        // no more remaining levels to fix+        return true;+    }++    template <typename Store, typename iterator, typename entry_type>+    bool operator()(const Store& store, iterator& iter, const entry_type& entry) const {+        // search in current level (if it is not the last level, we need a lower bound)+        auto cur = store.lowerBound(entry[Pos]);++        // if no lower boundary is found, be done+        if (cur == store.end()) {+            return false;+        }+        assert(RamDomain(cur->first) >= entry[Pos]);++        // if the lower bound is higher than the requested value, go to first in subtree+        if (RamDomain(cur->first) > entry[Pos]) {+            get_nested_iter_core<Pos>()(iter.iter_core).setIterator(cur);+            iter.value[Pos] = cur->first;+            fix_first<Pos + 1, Dim>()(cur->second->getStore(), iter);+            return true;+        }++        // attempt to fix the rest+        if (!fix_upper_bound<Pos + 1, Dim>()(cur->second->getStore(), iter, entry)) {+            // if it does not work, since there are no matching elements in this branch, go to next+            entry_type sub = entry;+            sub[Pos] += 1;+            for (size_t i = Pos + 1; i < Dim; ++i) {+                sub[i] = 0;+            }+            return (*this)(store, iter, sub);+        }++        // remember result+        get_nested_iter_core<Pos>()(iter.iter_core).setIterator(cur);++        // update iterator value+        iter.value[Pos] = cur->first;++        // done!+        return true;+    }+};++}  // namespace detail++/**+ * The most generic implementation of a Trie forming the top-level of any+ * Trie storing tuples of arity > 1.+ */+template <unsigned Dim>+class Trie : public souffle::detail::TrieBase<Dim, Trie<Dim>> {+    template <unsigned D>+    friend class Trie;++    template <unsigned D, typename Derived>+    friend class TrieBase;++    // a shortcut for the common base class type+    using base = typename souffle::detail::TrieBase<Dim, Trie<Dim>>;++    // the type of the nested tries (1 dimension less)+    using nested_trie_type = Trie<Dim - 1>;++    // the merge operation capable of merging two nested tries+    struct nested_trie_merger {+        nested_trie_type* operator()(nested_trie_type* a, const nested_trie_type* b) const {+            if (!b) return a;+            if (!a) return new nested_trie_type(*b);+            a->insertAll(*b);+            return a;+        }+    };++    // the operation capable of cloning a nested trie+    struct nested_trie_cloner {+        nested_trie_type* operator()(nested_trie_type* a) const {+            if (!a) return a;+            return new nested_trie_type(*a);+        }+    };++    // the data structure utilized for indexing nested tries+    using store_type = SparseArray<nested_trie_type*,+            6,  // = 2^6 entries per block+            nested_trie_merger, nested_trie_cloner>;++    // the actual data store+    store_type store;++public:+    using entry_type = typename souffle::Tuple<RamDomain, Dim>;+    using element_type = entry_type;++    // ---------------------------------------------------------------------+    //                           Iterator+    // ---------------------------------------------------------------------++    /**+     * The iterator core for trie iterators involving this level.+     */+    template <unsigned I = 0>+    class iterator_core {+        // the iterator for the current level+        using store_iter_t = typename store_type::iterator;++        // the type of the nested iterator+        using nested_iter_core = typename Trie<Dim - 1>::template iterator_core<I + 1>;++        store_iter_t iter;++        nested_iter_core nested;++    public:+        /** default end-iterator constructor */+        iterator_core() = default;++        template <typename Tuple>+        iterator_core(const store_iter_t& iter, Tuple& entry) : iter(iter) {+            entry[I] = iter->first;+            nested = iter->second->template getBeginCoreIterator<I + 1>(entry);+        }++        void setIterator(const store_iter_t& iter) {+            this->iter = iter;+        }++        store_iter_t& getIterator() {+            return this->iter;+        }++        nested_iter_core& getNested() {+            return nested;+        }++        template <typename Tuple>+        bool inc(Tuple& entry) {+            // increment nested iterator+            if (nested.inc(entry)) return true;++            // increment the iterator on this level+            ++iter;++            // check whether the end has been reached+            if (iter.isEnd()) return false;++            // otherwise update entry value+            entry[I] = iter->first;++            // and restart nested+            nested = iter->second->template getBeginCoreIterator<I + 1>(entry);+            return true;+        }++        bool operator==(const iterator_core& other) const {+            return nested == other.nested && iter == other.iter;+        }++        bool operator!=(const iterator_core& other) const {+            return !(*this == other);+        }++        // enables this iterator core to be printed (for debugging)+        void print(std::ostream& out) const {+            out << iter << " | " << nested;+        }++        friend std::ostream& operator<<(std::ostream& out, const iterator_core& iter) {+            iter.print(out);+            return out;+        }+    };++    // the type of iterator to be utilized when iterating of instances of this trie+    using iterator = typename base::template iterator<iterator_core>;++    // the operation context aggregating all operation contexts of nested structures+    struct op_context {+        using local_ctxt = typename store_type::op_context;+        using nested_ctxt = typename nested_trie_type::op_context;++        // for insert and contain+        local_ctxt local{};+        RamDomain lastQuery{};+        nested_trie_type* lastNested{nullptr};+        nested_ctxt nestedCtxt{};++        // for boundaries+        unsigned lastBoundaryLevels{Dim + 1};+        entry_type lastBoundaryRequest{};+        range<iterator> lastBoundaries{iterator(), iterator()};++        op_context() = default;+    };++    using operation_hints = op_context;++    using base::contains;+    using base::insert;++    /**+     * A simple destructore.+     */+    ~Trie() {+        for (auto& cur : store) {+            delete cur.second;  // clears all nested tries+        }+    }++    /**+     * Determines whether this trie is empty or not.+     */+    bool empty() const {+        return store.empty();+    }++    /**+     * Determines the number of entries in this trie.+     */+    std::size_t size() const {+        // the number of elements is lazy-evaluated+        std::size_t res = 0;+        for (const auto& cur : store) {+            res += cur.second->size();+        }+        return res;+    }++    /**+     * Computes the total memory usage of this data structure.+     */+    std::size_t getMemoryUsage() const {+        // compute the total memory usage of this level+        std::size_t res = sizeof(*this) - sizeof(store) + store.getMemoryUsage();++        // add the memory usage of sub-levels+        for (const auto& cur : store) {+            res += cur.second->getMemoryUsage();+        }++        // done+        return res;+    }++    /**+     * Removes all entries within this trie.+     */+    void clear() {+        // delete lower levels+        for (auto& cur : store) {+            delete cur.second;+        }++        // clear store+        store.clear();+    }++    /**+     * Inserts a new entry.+     *+     * @param tuple the entry to be added+     * @return true if the same tuple hasn't been present before, false otherwise+     */+    bool insert(const entry_type& tuple) {+        op_context ctxt;+        return insert(tuple, ctxt);+    }++    /**+     * Inserts a new entry. A operation context may be provided to exploit temporal+     * locality.+     *+     * @param tuple the entry to be added+     * @param ctxt the operation context to be utilized+     * @return true if the same tuple hasn't been present before, false otherwise+     */+    bool insert(const entry_type& tuple, op_context& ctxt) {+        return insert_internal<0>(tuple, ctxt);+    }++    /**+     * Determines whether a given tuple is present within the set specified+     * by this trie.+     *+     * @param tuple the tuple to be tested+     * @return true if present, false otherwise+     */+    bool contains(const entry_type& tuple) const {+        op_context ctxt;+        return contains(tuple, ctxt);+    }++    /**+     * Determines whether a given tuple is present within the set specified+     * by this trie. A operation context may be provided to exploit temporal+     * locality.+     *+     * @param tuple the entry to be added+     * @param ctxt the operation context to be utilized+     * @return true if the same tuple hasn't been present before, false otherwise+     */+    bool contains(const entry_type& tuple, op_context& ctxt) const {+        return contains_internal<0>(tuple, ctxt);+    }++    /**+     * Inserts all elements stored within the given trie into this trie.+     *+     * @param other the elements to be inserted into this trie+     */+    void insertAll(const Trie& other) {+        store.addAll(other.store);+    }++    /**+     * Obtains an iterator referencing the first element stored within this trie.+     */+    iterator begin() const {+        auto it = store.begin();+        if (it.isEnd()) return end();+        return iterator(it);+    }++    /**+     * Obtains an iterator referencing the position after the last element stored+     * within this trie.+     */+    iterator end() const {+        return iterator();+    }++    iterator find(const entry_type& entry) const {+        op_context ctxt;+        return find(entry, ctxt);+    }++    iterator find(const entry_type& entry, op_context& ctxt) const {+        auto range = getBoundaries<Dim>(entry, ctxt);+        return (!range.empty()) ? range.begin() : end();+    }++    /**+     * Obtains a range of elements matching the prefix of the given entry up to+     * levels elements.+     *+     * @tparam levels the length of the requested matching prefix+     * @param entry the entry to be looking for+     * @return the corresponding range of matching elements+     */+    template <unsigned levels>+    range<iterator> getBoundaries(const entry_type& entry) const {+        op_context ctxt;+        return getBoundaries<levels>(entry, ctxt);+    }++    /**+     * Obtains a range of elements matching the prefix of the given entry up to+     * levels elements. A operation context may be provided to exploit temporal+     * locality.+     *+     * @tparam levels the length of the requested matching prefix+     * @param entry the entry to be looking for+     * @param ctxt the operation context to be utilized+     * @return the corresponding range of matching elements+     */+    template <unsigned levels>+    range<iterator> getBoundaries(const entry_type& entry, op_context& ctxt) const {+        // if nothing is bound => just use begin and end+        if (levels == 0) return make_range(begin(), end());++        // check context+        if (ctxt.lastBoundaryLevels == levels) {+            bool fit = true;+            for (unsigned i = 0; i < levels; ++i) {+                fit = fit && (entry[i] == ctxt.lastBoundaryRequest[i]);+            }++            // if it fits => take it+            if (fit) {+                base::hint_stats.get_boundaries.addHit();+                return ctxt.lastBoundaries;+            }+        }++        // the hint has not been a hit+        base::hint_stats.get_boundaries.addMiss();++        // start with two end iterators+        iterator begin{};+        iterator end{};++        // adapt them level by level+        auto found = souffle::detail::fix_binding<levels, 0, Dim>()(store, begin, end, entry);+        if (!found) return make_range(iterator(), iterator());++        // update context+        ctxt.lastBoundaryLevels = levels;+        ctxt.lastBoundaryRequest = entry;+        ctxt.lastBoundaries = make_range(begin, end);++        // use the result+        return ctxt.lastBoundaries;+    }++    /**+     * Obtains an iterator to the first element not less than the given entry value.+     *+     * @param entry the lower bound for this search+     * @param ctxt the operation context to be utilized+     * @return an iterator addressing the first element in this structure not less than the given value+     */+    iterator lower_bound(const entry_type& entry, op_context& /* ctxt */) const {+        // start with a default-initialized iterator+        iterator res;++        // adapt it level by level+        bool found = detail::fix_lower_bound<0, Dim>()(store, res, entry);++        // use the result+        return found ? res : end();+    }++    /**+     * Obtains an iterator to the first element not less than the given entry value.+     *+     * @param entry the lower bound for this search+     * @return an iterator addressing the first element in this structure not less than the given value+     */+    iterator lower_bound(const entry_type& entry) const {+        op_context ctxt;+        return lower_bound(entry, ctxt);+    }++    /**+     * Obtains an iterator to the first element greater than the given entry value, or end if there is no such+     * element.+     *+     * @param entry the upper bound for this search+     * @param ctxt the operation context to be utilized+     * @return an iterator addressing the first element in this structure greater than the given value+     */+    iterator upper_bound(const entry_type& entry, op_context& /* ctxt */) const {+        // start with a default-initialized iterator+        iterator res;++        // adapt it level by level+        bool found = detail::fix_upper_bound<0, Dim>()(store, res, entry);++        // use the result+        return found ? res : end();+    }++    /**+     * Obtains an iterator to the first element greater than the given entry value, or end if there is no such+     * element.+     *+     * @param entry the upper bound for this search+     * @return an iterator addressing the first element in this structure greater than the given value+     */+    iterator upper_bound(const entry_type& entry) const {+        op_context ctxt;+        return upper_bound(entry, ctxt);+    }++    /**+     * Computes a partition of an approximate number of chunks of the content+     * of this trie. Thus, the union of the resulting set of disjoint ranges is+     * equivalent to the content of this trie.+     *+     * @param chunks the number of chunks requested+     * @return a list of sub-ranges forming a partition of the content of this trie+     */+    std::vector<range<iterator>> partition(unsigned chunks = 500) const {+        std::vector<range<iterator>> res;++        // shortcut for empty trie+        if (this->empty()) return res;++        // use top-level elements for partitioning+        int step = std::max(store.size() / chunks, size_t(1));++        int c = 1;+        auto priv = begin();+        for (auto it = store.begin(); it != store.end(); ++it, c++) {+            if (c % step != 0 || c == 1) {+                continue;+            }+            auto cur = iterator(it);+            res.push_back(make_range(priv, cur));+            priv = cur;+        }+        // add final chunk+        res.push_back(make_range(priv, end()));+        return res;+    }++    /**+     * Provides a protected access to the internally maintained store.+     */+    const store_type& getStore() const {+        return store;+    }++private:+    /**+     * Creates a core iterator for this trie level and updates component+     * I of the given entry to exhibit the corresponding first value.+     *+     * @tparam I the index of the tuple to be processed by the resulting iterator core+     * @tparam Tuple the type of the tuple to be processed by the resulting iterator core+     * @param entry a reference to the tuple to be updated to the first value+     * @return the requested iterator core instance+     */+    template <unsigned I, typename Tuple>+    iterator_core<I> getBeginCoreIterator(Tuple& entry) const {+        return iterator_core<I>(store.begin(), entry);+    }++    /**+     * The internally utilized implementation of the insert operation inserting+     * a given tuple into this sub-trie.+     *+     * @tparam I the component index associated to this level+     * @tparam Tuple the tuple type to be inserted+     * @param tuple the tuple to be inserted+     * @param ctxt a operation context to exploit temporal locality+     * @return true if this tuple wasn't contained before, false otherwise+     */+    template <unsigned I, typename Tuple>+    bool insert_internal(const Tuple& tuple, op_context& ctxt) {+        using value_t = typename store_type::value_type;+        using atomic_value_t = typename store_type::atomic_value_type;++        // check context+        if (ctxt.lastNested && ctxt.lastQuery == tuple[I]) {+            base::hint_stats.inserts.addHit();+            return ctxt.lastNested->template insert_internal<I + 1>(tuple, ctxt.nestedCtxt);+        } else {+            base::hint_stats.inserts.addMiss();+        }++        // lookup nested+        atomic_value_t& next = store.getAtomic(tuple[I], ctxt.local);++        // get pure pointer to next level+        value_t nextPtr = next;++        // conduct a lock-free lazy-creation of nested trees+        if (!nextPtr) {+            // create a new sub-tree+            auto newNested = new nested_trie_type();++            // register new sub-tree atomically+            if (next.compare_exchange_weak(nextPtr, newNested)) {+                nextPtr = newNested;  // worked+            } else {+                delete newNested;  // some other thread was faster => use its version+            }+        }++        // make sure a next has been established+        assert(nextPtr);++        // clear context if necessary+        if (nextPtr != ctxt.lastNested) {+            ctxt.lastQuery = tuple[I];+            ctxt.lastNested = nextPtr;+            ctxt.nestedCtxt = typename op_context::nested_ctxt();+        }++        // conduct recursive step+        return nextPtr->template insert_internal<I + 1>(tuple, ctxt.nestedCtxt);+    }++    /**+     * An internal implementation of the contains member function determining+     * whether a given tuple is present within this sub-trie or not.+     *+     * @tparam I the component index associated to this level+     * @tparam Tuple the tuple type to be checked+     * @param tuple the tuple to be checked+     * @param ctxt a operation context to exploit temporal locality+     * @return true if this tuple is present, false otherwise+     */+    template <unsigned I, typename Tuple>+    bool contains_internal(const Tuple& tuple, op_context& ctxt) const {+        // check context+        if (ctxt.lastNested && ctxt.lastQuery == tuple[I]) {+            base::hint_stats.contains.addHit();+            return ctxt.lastNested->template contains_internal<I + 1>(tuple, ctxt.nestedCtxt);+        } else {+            base::hint_stats.contains.addMiss();+        }++        // lookup next step+        auto next = store.lookup(tuple[I], ctxt.local);++        // clear context if necessary+        if (next != ctxt.lastNested) {+            ctxt.lastQuery = tuple[I];+            ctxt.lastNested = next;+            ctxt.nestedCtxt = typename op_context::nested_ctxt();+        }++        // conduct recursive step+        return next && next->template contains_internal<I + 1>(tuple, ctxt.nestedCtxt);+    }+};++/**+ * A template specialization for tries representing a set.+ * For improved memory efficiency, this level is the leaf-node level+ * of all tries exhibiting an arity >= 1. Internally, values are stored utilizing+ * sparse bit maps.+ */+template <>+class Trie<1u> : public detail::TrieBase<1u, Trie<1u>> {+    template <unsigned Dim>+    friend class Trie;++    template <unsigned Dim, typename Derived>+    friend class detail::TrieBase;++    // a shortcut for the base type+    using base = typename detail::TrieBase<1u, Trie<1u>>;++    // the map type utilized internally+    using map_type = SparseBitMap<>;++    // the internal data store+    map_type map;++public:+    using element_type = entry_type;+    using op_context = typename map_type::op_context;+    using operation_hints = op_context;++    using base::contains;+    using base::insert;++    /**+     * Determines whether this trie is empty or not.+     */+    bool empty() const {+        return map.empty();+    }++    /**+     * Determines the number of elements stored in this trie.+     */+    std::size_t size() const {+        return map.size();+    }++    /**+     * Computes the total memory usage of this data structure.+     */+    std::size_t getMemoryUsage() const {+        // compute the total memory usage+        return sizeof(*this) - sizeof(map_type) + map.getMemoryUsage();+    }++    /**+     * Removes all elements form this trie.+     */+    void clear() {+        map.clear();+    }++    /**+     * Inserts the given tuple into this trie.+     *+     * @param tuple the tuple to be inserted+     * @return true if the tuple has not been present before, false otherwise+     */+    bool insert(const entry_type& tuple) {+        op_context ctxt;+        return insert(tuple, ctxt);+    }++    /**+     * Inserts the given tuple into this trie.+     * An operation context can be provided to exploit temporal locality.+     *+     * @param tuple the tuple to be inserted+     * @param ctxt an operation context for exploiting temporal locality+     * @return true if the tuple has not been present before, false otherwise+     */+    bool insert(const entry_type& tuple, op_context& ctxt) {+        return insert_internal<0>(tuple, ctxt);+    }++    /**+     * Determines whether the given tuple is present in this trie or not.+     *+     * @param tuple the tuple to be tested+     * @return true if present, false otherwise+     */+    bool contains(const entry_type& tuple) const {+        op_context ctxt;+        return contains(tuple, ctxt);+    }++    /**+     * Determines whether the given tuple is present in this trie or not.+     * An operation context can be provided to exploit temporal locality.+     *+     * @param tuple the tuple to be tested+     * @param ctxt an operation context for exploiting temporal locality+     * @return true if present, false otherwise+     */+    bool contains(const entry_type& tuple, op_context& ctxt) const {+        return contains_internal<0>(tuple, ctxt);+    }++    /**+     * Inserts all tuples stored within the given trie into this trie.+     * This operation is considerably more efficient than the consecutive+     * insertion of the elements in other into this trie.+     */+    void insertAll(const Trie& other) {+        map.addAll(other.map);+    }++    // ---------------------------------------------------------------------+    //                           Iterator+    // ---------------------------------------------------------------------++    /**+     * The iterator core of this level contributing to the construction of+     * a composed trie iterator.+     */+    template <unsigned I = 0>+    class iterator_core {+        // the iterator for this level+        using iter_type = typename map_type::iterator;++        // the referenced bit-map iterator+        iter_type iter;++    public:+        /** default end-iterator constructor */+        iterator_core() = default;++        template <typename Tuple>+        iterator_core(const iter_type& iter, Tuple& entry) : iter(iter) {+            entry[I] = static_cast<RamDomain>(*iter);+        }++        void setIterator(const iter_type& iter) {+            this->iter = iter;+        }++        iter_type& getIterator() {+            return this->iter;+        }++        template <typename Tuple>+        bool inc(Tuple& entry) {+            // increment the iterator on this level+            ++iter;++            // check whether the end has been reached+            if (iter.isEnd()) return false;++            // otherwise update entry value+            entry[I] = *iter;+            return true;+        }++        bool operator==(const iterator_core& other) const {+            return iter == other.iter;+        }++        bool operator!=(const iterator_core& other) const {+            return !(*this == other);+        }++        // enables this iterator core to be printed (for debugging)+        void print(std::ostream& out) const {+            out << iter;+        }++        friend std::ostream& operator<<(std::ostream& out, const iterator_core& iter) {+            iter.print(out);+            return out;+        }+    };++    // the iterator type utilized by this trie type+    using iterator = typename base::template iterator<iterator_core>;++    /**+     * Obtains an iterator referencing the first element stored within this trie+     * or end() if this trie is empty.+     */+    iterator begin() const {+        if (map.empty()) return end();+        return iterator(map.begin());+    }++    /**+     * Obtains an iterator referencing the first position after the last element+     * within this trie.+     */+    iterator end() const {+        return iterator();+    }++    /**+     * Obtains a partition of this tire such that the resulting list of ranges+     * cover disjoint subsets of the elements stored in this trie. Their union+     * is equivalent to the content of this trie.+     */+    std::vector<range<iterator>> partition(unsigned chunks = 500) const {+        std::vector<range<iterator>> res;++        // shortcut for empty trie+        if (this->empty()) return res;++        // use top-level elements for partitioning+        int step = static_cast<int>(std::max(map.size() / chunks, size_t(1)));++        int c = 1;+        auto priv = begin();+        for (auto it = map.begin(); it != map.end(); ++it, c++) {+            if (c % step != 0 || c == 1) {+                continue;+            }+            auto cur = iterator(it);+            res.push_back(make_range(priv, cur));+            priv = cur;+        }+        // add final chunk+        res.push_back(make_range(priv, end()));+        return res;+    }++    /**+     * Obtains a range of elements matching the prefix of the given entry up to+     * levels elements.+     *+     * @tparam levels the length of the requested matching prefix+     * @param entry the entry to be looking for+     * @return the corresponding range of matching elements+     */+    template <unsigned levels>+    range<iterator> getBoundaries(const entry_type& entry) const {+        op_context ctxt;+        return getBoundaries<levels>(entry, ctxt);+    }++    /**+     * Obtains a range of elements matching the prefix of the given entry up to+     * levels elements. A operation context may be provided to exploit temporal+     * locality.+     *+     * @tparam levels the length of the requested matching prefix+     * @param entry the entry to be looking for+     * @param ctxt the operation context to be utilized+     * @return the corresponding range of matching elements+     */+    template <unsigned levels>+    range<iterator> getBoundaries(const entry_type& entry, op_context& ctxt) const {+        // for levels = 0+        if (levels == 0) return make_range(begin(), end());+        // for levels = 1+        auto pos = map.find(entry[0], ctxt);+        if (pos == map.end()) return make_range(end(), end());+        auto next = pos;+        ++next;+        return make_range(iterator(pos), iterator(next));+    }++    iterator lower_bound(const entry_type& entry, op_context&) const {+        return iterator(map.lower_bound(entry[0]));+    }++    iterator lower_bound(const entry_type& entry) const {+        op_context ctxt;+        return lower_bound(entry, ctxt);+    }++    iterator upper_bound(const entry_type& entry, op_context&) const {+        return iterator(map.upper_bound(entry[0]));+    }++    iterator upper_bound(const entry_type& entry) const {+        op_context ctxt;+        return upper_bound(entry, ctxt);+    }++    /**+     * Provides protected access to the internally maintained store.+     */+    const map_type& getStore() const {+        return map;+    }++private:+    /**+     * Creates a core iterator for this trie level and updates component+     * I of the given entry to exhibit the corresponding first value.+     *+     * @tparam I the index of the tuple to be processed by the resulting iterator core+     * @tparam Tuple the type of the tuple to be processed by the resulting iterator core+     * @param entry a reference to the tuple to be updated to the first value+     * @return the requested iterator core instance+     */+    template <unsigned I, typename Tuple>+    iterator_core<I> getBeginCoreIterator(Tuple& entry) const {+        return iterator_core<I>(map.begin(), entry);+    }++    /**+     * The internally utilized implementation of the insert operation inserting+     * a given tuple into this sub-trie.+     *+     * @tparam I the component index associated to this level+     * @tparam Tuple the tuple type to be inserted+     * @param tuple the tuple to be inserted+     * @param ctxt a operation context to exploit temporal locality+     * @return true if this tuple wasn't contained before, false otherwise+     */+    template <unsigned I, typename Tuple>+    bool insert_internal(const Tuple& tuple, op_context& ctxt) {+        return map.set(tuple[I], ctxt);+    }++    /**+     * An internal implementation of the contains member function determining+     * whether a given tuple is present within this sub-trie or not.+     *+     * @tparam I the component index associated to this level+     * @tparam Tuple the tuple type to be checked+     * @param tuple the tuple to be checked+     * @param ctxt a operation context to exploit temporal locality+     * @return true if this tuple is present, false otherwise+     */+    template <unsigned I, typename Tuple>+    bool contains_internal(const Tuple& tuple, op_context& ctxt) const {+        return map.test(tuple[I], ctxt);+    }+};++}  // end namespace souffle
+ cbits/souffle/datastructure/EquivalenceRelation.h view
@@ -0,0 +1,738 @@+/*+ * Souffle - A Datalog Compiler+ * Copyright (c) 2017 The Souffle Developers. All rights reserved+ * Licensed under the Universal Permissive License v 1.0 as shown at:+ * - https://opensource.org/licenses/UPL+ * - <souffle root>/licenses/SOUFFLE-UPL.txt+ */++/************************************************************************+ *+ * @file EquivalenceRelation.h+ *+ * Defines a binary relation interface to be used with Souffle as a relational store.+ * Pairs inserted into this relation implicitly store a reflexive, symmetric, and transitive relation+ * with each other.+ *+ ***********************************************************************/++#pragma once++#include "souffle/RamTypes.h"+#include "souffle/datastructure/LambdaBTree.h"+#include "souffle/datastructure/PiggyList.h"+#include "souffle/datastructure/UnionFind.h"+#include "souffle/utility/ContainerUtil.h"+#include "souffle/utility/ParallelUtil.h"+#include <atomic>+#include <cassert>+#include <cstddef>+#include <functional>+#include <iostream>+#include <iterator>+#include <set>+#include <shared_mutex>+#include <stdexcept>+#include <tuple>+#include <utility>+#include <vector>++namespace souffle {+template <typename TupleType>+class EquivalenceRelation {+    using value_type = typename TupleType::value_type;++    // mapping from representative to disjoint set+    // just a cache, essentially, used for iteration over+    using StatesList = souffle::PiggyList<value_type>;+    using StatesBucket = StatesList*;+    using StorePair = std::pair<value_type, StatesBucket>;+    using StatesMap = souffle::LambdaBTreeSet<StorePair, std::function<StatesBucket(StorePair&)>,+            souffle::EqrelMapComparator<StorePair>>;++public:+    using element_type = TupleType;++    EquivalenceRelation() : statesMapStale(false){};+    ~EquivalenceRelation() {+        emptyPartition();+    }++    /**+     * A collection of operation hints speeding up some of the involved operations+     * by exploiting temporal locality.+     * Unused in this class, as there is no speedup to be gained.+     * This is just defined as the class expects it.+     */+    struct operation_hints {+        // resets all hints (to be triggered e.g. when deleting nodes)+        void clear() {}+    };++    /**+     * Insert the two values symbolically as a binary relation+     * @param x node to be added/paired+     * @param y node to be added/paired+     * @return true if the pair is new to the data structure+     */+    bool insert(value_type x, value_type y) {+        operation_hints z;+        return insert(x, y, z);+    };++    /**+     * Insert the tuple symbolically.+     * @param tuple The tuple to be inserted+     * @return true if the tuple is new to the data structure+     */+    bool insert(const TupleType& tuple) {+        operation_hints hints;+        return insert(tuple[0], tuple[1], hints);+    };++    /**+     * Insert the two values symbolically as a binary relation+     * @param x node to be added/paired+     * @param y node to be added/paired+     * @param z the hints to where the pair should be inserted (not applicable atm)+     * @return true if the pair is new to the data structure+     */+    bool insert(value_type x, value_type y, operation_hints) {+        // indicate that iterators will have to generate on request+        this->statesMapStale.store(true, std::memory_order_relaxed);+        bool retval = contains(x, y);+        sds.unionNodes(x, y);+        return retval;+    }++    /**+     * inserts all nodes from the other relation into this one+     * @param other the binary relation from which to add elements from+     */+    void insertAll(const EquivalenceRelation<TupleType>& other) {+        other.genAllDisjointSetLists();++        // iterate over partitions at a time+        for (typename StatesMap::chunk it : other.equivalencePartition.getChunks(MAX_THREADS)) {+            for (auto& p : it) {+                value_type rep = p.first;+                StatesList& pl = *p.second;+                const size_t ksize = pl.size();+                for (size_t i = 0; i < ksize; ++i) {+                    this->sds.unionNodes(rep, pl.get(i));+                }+            }+        }+        // invalidate iterators unconditionally+        this->statesMapStale.store(true, std::memory_order_relaxed);+    }++    /**+     * Extend this relation with another relation, expanding this equivalence relation+     * The supplied relation is the old knowledge, whilst this relation only contains+     * explicitly new knowledge. After this operation the "implicitly new tuples" are now+     * explicitly inserted this relation.+     */+    void extend(const EquivalenceRelation<TupleType>& other) {+        // nothing to extend if there's no new/original knowledge+        if (other.size() == 0 || this->size() == 0) return;++        this->genAllDisjointSetLists();+        other.genAllDisjointSetLists();++        std::set<value_type> repsCovered;++        // find all the disjoint sets that need to be added to this relation+        // that exist in other (and exist in this)+        {+            auto it = this->sds.sparseToDenseMap.begin();+            auto end = this->sds.sparseToDenseMap.end();+            value_type el;+            for (; it != end; ++it) {+                std::tie(el, std::ignore) = *it;+                if (other.containsElement(el)) {+                    value_type rep = other.sds.findNode(el);+                    if (repsCovered.count(rep) == 0) {+                        repsCovered.emplace(rep);+                    }+                }+            }+        }++        // add the intersecting dj sets into this one+        {+            value_type el;+            value_type rep;+            auto it = other.sds.sparseToDenseMap.begin();+            auto end = other.sds.sparseToDenseMap.end();+            for (; it != end; ++it) {+                std::tie(el, std::ignore) = *it;+                rep = other.sds.findNode(el);+                if (repsCovered.count(rep) != 0) {+                    this->insert(el, rep);+                }+            }+        }+    }++    /**+     * Returns whether there exists a pair with these two nodes+     * @param x front of pair+     * @param y back of pair+     */+    bool contains(value_type x, value_type y) const {+        return sds.contains(x, y);+    }++    /**+     * Returns whether there exists given tuple.+     * @param tuple The tuple to search for.+     */+    bool contains(const TupleType& tuple, operation_hints&) const {+        return contains(tuple[0], tuple[1]);+    };++    void emptyPartition() const {+        // delete the beautiful values inside (they're raw ptrs, so they need to be.)+        for (auto& pair : equivalencePartition) {+            delete pair.second;+        }+        // invalidate it my dude+        this->statesMapStale.store(true, std::memory_order_relaxed);++        equivalencePartition.clear();+    }++    /**+     * Empty the relation+     */+    void clear() {+        statesLock.lock();++        sds.clear();+        emptyPartition();++        statesLock.unlock();+    }++    /**+     * Size of relation+     * @return the sum of the number of pairs per disjoint set+     */+    size_t size() const {+        genAllDisjointSetLists();++        statesLock.lock_shared();++        size_t retVal = 0;+        for (auto& e : this->equivalencePartition) {+            const size_t s = e.second->size();+            retVal += s * s;+        }++        statesLock.unlock_shared();+        return retVal;+    }++    // an almighty iterator for several types of iteration.+    // Unfortunately, subclassing isn't an option with souffle+    //   - we don't deal with pointers (so no virtual)+    //   - and a single iter type is expected (see Relation::iterator e.g.) (i think)+    class iterator {+    public:+        typedef std::forward_iterator_tag iterator_category;+        typedef TupleType value_type;+        typedef ptrdiff_t difference_type;+        typedef value_type* pointer;+        typedef value_type& reference;++        // one iterator for signalling the end (simplifies)+        explicit iterator(const EquivalenceRelation* br, bool /* signalIsEndIterator */)+                : br(br), isEndVal(true){};++        explicit iterator(const EquivalenceRelation* br)+                : br(br), ityp(IterType::ALL), djSetMapListIt(br->equivalencePartition.begin()),+                  djSetMapListEnd(br->equivalencePartition.end()) {+            // no need to fast forward if this iterator is empty+            if (djSetMapListIt == djSetMapListEnd) {+                isEndVal = true;+                return;+            }+            // grab the pointer to the list, and make it our current list+            djSetList = (*djSetMapListIt).second;+            assert(djSetList->size() != 0);++            updateAnterior();+            updatePosterior();+        }++        // WITHIN: iterator for everything within the same DJset (used for EquivalenceRelation.partition())+        explicit iterator(const EquivalenceRelation* br, const StatesBucket within)+                : br(br), ityp(IterType::WITHIN), djSetList(within) {+            // empty dj set+            if (djSetList->size() == 0) {+                isEndVal = true;+            }++            updateAnterior();+            updatePosterior();+        }++        // ANTERIOR: iterator that yields all (former, _) \in djset(former) (djset(former) === within)+        explicit iterator(const EquivalenceRelation* br, const typename TupleType::value_type former,+                const StatesBucket within)+                : br(br), ityp(IterType::ANTERIOR), djSetList(within) {+            if (djSetList->size() == 0) {+                isEndVal = true;+            }++            setAnterior(former);+            updatePosterior();+        }++        // ANTPOST: iterator that yields all (former, latter) \in djset(former), (djset(former) ==+        // djset(latter) == within)+        explicit iterator(const EquivalenceRelation* br, const typename TupleType::value_type former,+                typename TupleType::value_type latter, const StatesBucket within)+                : br(br), ityp(IterType::ANTPOST), djSetList(within) {+            if (djSetList->size() == 0) {+                isEndVal = true;+            }++            setAnterior(former);+            setPosterior(latter);+        }++        /** explicit set first half of cPair */+        inline void setAnterior(const typename TupleType::value_type a) {+            this->cPair[0] = a;+        }++        /** quick update to whatever the current index is pointing to */+        inline void updateAnterior() {+            this->cPair[0] = this->djSetList->get(this->cAnteriorIndex);+        }++        /** explicit set second half of cPair */+        inline void setPosterior(const typename TupleType::value_type b) {+            this->cPair[1] = b;+        }++        /** quick update to whatever the current index is pointing to */+        inline void updatePosterior() {+            this->cPair[1] = this->djSetList->get(this->cPosteriorIndex);+        }++        // copy ctor+        iterator(const iterator& other) = default;+        // move ctor+        iterator(iterator&& other) = default;+        // assign iter+        iterator& operator=(const iterator& other) = default;++        bool operator==(const iterator& other) const {+            if (isEndVal && other.isEndVal) return br == other.br;+            return isEndVal == other.isEndVal && cPair == other.cPair;+        }++        bool operator!=(const iterator& other) const {+            return !((*this) == other);+        }++        const TupleType& operator*() const {+            return cPair;+        }++        const TupleType* operator->() const {+            return &cPair;+        }++        /* pre-increment */+        iterator& operator++() {+            if (isEndVal) {+                throw std::out_of_range("error: incrementing an out of range iterator");+            }++            switch (ityp) {+                case IterType::ALL:+                    // move posterior along one+                    // see if we can't move the posterior along+                    if (++cPosteriorIndex == djSetList->size()) {+                        // move anterior along one+                        // see if we can't move the anterior along one+                        if (++cAnteriorIndex == djSetList->size()) {+                            // move the djset it along one+                            // see if we can't move it along one (we're at the end)+                            if (++djSetMapListIt == djSetMapListEnd) {+                                isEndVal = true;+                                return *this;+                            }++                            // we can't iterate along this djset if it is empty+                            djSetList = (*djSetMapListIt).second;+                            if (djSetList->size() == 0) {+                                throw std::out_of_range("error: encountered a zero size djset");+                            }++                            // update our cAnterior and cPosterior+                            cAnteriorIndex = 0;+                            cPosteriorIndex = 0;+                            updateAnterior();+                            updatePosterior();+                        }++                        // we moved our anterior along one+                        updateAnterior();++                        cPosteriorIndex = 0;+                        updatePosterior();+                    }+                    // we just moved our posterior along one+                    updatePosterior();++                    break;+                case IterType::ANTERIOR:+                    // step posterior along one, and if we can't, then we're done.+                    if (++cPosteriorIndex == djSetList->size()) {+                        isEndVal = true;+                        return *this;+                    }+                    updatePosterior();++                    break;+                case IterType::ANTPOST:+                    // fixed anterior and posterior literally only points to one, so if we increment, its the+                    // end+                    isEndVal = true;+                    break;+                case IterType::WITHIN:+                    // move posterior along one+                    // see if we can't move the posterior along+                    if (++cPosteriorIndex == djSetList->size()) {+                        // move anterior along one+                        // see if we can't move the anterior along one+                        if (++cAnteriorIndex == djSetList->size()) {+                            isEndVal = true;+                            return *this;+                        }++                        // we moved our anterior along one+                        updateAnterior();++                        cPosteriorIndex = 0;+                        updatePosterior();+                    }+                    // we just moved our posterior along one+                    updatePosterior();+                    break;+            }++            return *this;+        }++    private:+        const EquivalenceRelation* br = nullptr;+        // special tombstone value to notify that this iter represents the end+        bool isEndVal = false;++        // all the different types of iterator this can be+        enum IterType { ALL, ANTERIOR, ANTPOST, WITHIN };+        IterType ityp;++        TupleType cPair;++        // the disjoint set that we're currently iterating through+        StatesBucket djSetList;+        typename StatesMap::iterator djSetMapListIt;+        typename StatesMap::iterator djSetMapListEnd;++        // used for ALL, and POSTERIOR (just a current index in the cList)+        size_t cAnteriorIndex = 0;+        // used for ALL, and ANTERIOR (just a current index in the cList)+        size_t cPosteriorIndex = 0;+    };++public:+    /**+     * iterator pointing to the beginning of the tuples, with no restrictions+     * @return the iterator that corresponds to the beginning of the binary relation+     */+    iterator begin() const {+        genAllDisjointSetLists();+        return iterator(this);+    }++    /**+     * iterator pointing to the end of the tuples+     * @return the iterator which represents the end of the binary rel+     */+    iterator end() const {+        return iterator(this, true);+    }++    /**+     * Obtains a range of elements matching the prefix of the given entry up to+     * levels elements.+     *+     * @tparam levels the length of the requested matching prefix+     * @param entry the entry to be looking for+     * @return the corresponding range of matching elements+     */+    template <unsigned levels>+    range<iterator> getBoundaries(const TupleType& entry) const {+        operation_hints ctxt;+        return getBoundaries<levels>(entry, ctxt);+    }++    /**+     * Obtains a range of elements matching the prefix of the given entry up to+     * levels elements. A operation context may be provided to exploit temporal+     * locality.+     *+     * @tparam levels the length of the requested matching prefix+     * @param entry the entry to be looking for+     * @param ctxt the operation context to be utilized+     * @return the corresponding range of matching elements+     */+    template <unsigned levels>+    range<iterator> getBoundaries(const TupleType& entry, operation_hints&) const {+        // if nothing is bound => just use begin and end+        if (levels == 0) return make_range(begin(), end());++        // as disjoint set is exactly two args (equiv relation)+        // we only need to handle these cases++        if (levels == 1) {+            // need to test if the entry actually exists+            if (!sds.nodeExists(entry[0])) return make_range(end(), end());++            // return an iterator over all (entry[0], _)+            return make_range(anteriorIt(entry[0]), end());+        }++        if (levels == 2) {+            // need to test if the entry actually exists+            if (!sds.contains(entry[0], entry[1])) return make_range(end(), end());++            // if so return an iterator containing exactly that node+            return make_range(antpostit(entry[0], entry[1]), end());+        }++        std::cerr << "invalid state, cannot search for >2 arg start point in getBoundaries, in 2 arg tuple "+                     "store\n";+        throw "invalid state, cannot search for >2 arg start point in getBoundaries, in 2 arg tuple store";++        return make_range(end(), end());+    }++    /**+     * Act similar to getBoundaries. But non-static.+     * This function should be used ONLY by interpreter,+     * and its behavior is tightly coupling with InterpreterIndex.+     * Do Not rely on this interface outside the interpreter.+     *+     * @param entry the entry to be looking for+     * @return the corresponding range of matching elements+     */+    iterator lower_bound(const TupleType& entry, operation_hints&) const {+        if (entry[0] == MIN_RAM_SIGNED && entry[1] == MIN_RAM_SIGNED) {+            // Return an iterator over all tuples.+            return begin();+        }++        if (entry[0] != MIN_RAM_SIGNED && entry[1] == MIN_RAM_SIGNED) {+            // Return an iterator over all (entry[0], _)++            if (!sds.nodeExists(entry[0])) {+                return end();+            }+            return anteriorIt(entry[0]);+        }++        if (entry[0] != MIN_RAM_SIGNED && entry[1] != MIN_RAM_SIGNED) {+            // Return an iterator point to the exact same node.++            if (!sds.contains(entry[0], entry[1])) {+                return end();+            }+            return antpostit(entry[0], entry[1]);+        }++        return end();+    }++    /**+     * This function is only here in order to unify interfaces in InterpreterIndex.+     * Unlike the name suggestes, it omit the arguments and simply return the end+     * iterator of the relation.+     *+     * @param omitted+     * @return the end iterator.+     */+    iterator upper_bound(const TupleType&, operation_hints&) const {+        return end();+    }++    /**+     * Check emptiness.+     */+    bool empty() const {+        return this->size() == 0;+    }++    /**+     * Creates an iterator that generates all pairs (A, X)+     * for a given A, and X are elements within A's disjoint set.+     * @param anteriorVal: The first value of the tuple to be generated for+     * @return the iterator representing this.+     */+    iterator anteriorIt(value_type anteriorVal) const {+        genAllDisjointSetLists();++        // locate the blocklist that the anterior val resides in+        auto found = equivalencePartition.find({sds.findNode(anteriorVal), nullptr});+        assert(found != equivalencePartition.end() && "iterator called on partition that doesn't exist");++        return iterator(static_cast<const EquivalenceRelation*>(this),+                static_cast<const value_type>(anteriorVal), static_cast<const StatesBucket>((*found).second));+    }++    /**+     * Creates an iterator that generates the pair (A, B)+     * for a given A and B. If A and B don't exist, or aren't in the same set,+     * then the end() iterator is returned.+     * @param anteriorVal: the A value of the tuple+     * @param posteriorVal: the B value of the tuple+     * @return the iterator representing this+     */+    iterator antpostit(value_type anteriorVal, value_type posteriorVal) const {+        // obv if they're in diff sets, then iteration for this pair just ends.+        if (!sds.sameSet(anteriorVal, posteriorVal)) return end();++        genAllDisjointSetLists();++        // locate the blocklist that the val resides in+        auto found = equivalencePartition.find({sds.findNode(posteriorVal), nullptr});+        assert(found != equivalencePartition.end() && "iterator called on partition that doesn't exist");++        return iterator(this, anteriorVal, posteriorVal, (*found).second);+    }++    /**+     * Begin an iterator over all pairs within a single disjoint set - This is used for partition().+     * @param rep the representative of (or element within) a disjoint set of which to generate all pairs+     * @return an iterator that will generate all pairs within the disjoint set+     */+    iterator closure(value_type rep) const {+        genAllDisjointSetLists();++        // locate the blocklist that the val resides in+        auto found = equivalencePartition.find({sds.findNode(rep), nullptr});+        return iterator(this, (*found).second);+    }++    /**+     * Generate an approximate number of iterators for parallel iteration+     * The iterators returned are not necessarily equal in size, but in practise are approximately similarly+     * sized+     * Depending on the structure of the data, there can be more or less partitions returned than requested.+     * @param chunks the number of requested partitions+     * @return a list of the iterators as ranges+     */+    std::vector<souffle::range<iterator>> partition(size_t chunks) const {+        // generate all reps+        genAllDisjointSetLists();++        size_t numPairs = this->size();+        if (numPairs == 0) return {};+        if (numPairs == 1 || chunks <= 1) return {souffle::make_range(begin(), end())};++        // if there's more dj sets than requested chunks, then just return an iter per dj set+        std::vector<souffle::range<iterator>> ret;+        if (chunks <= equivalencePartition.size()) {+            for (auto& p : equivalencePartition) {+                ret.push_back(souffle::make_range(closure(p.first), end()));+            }+            return ret;+        }++        // keep it simple stupid+        // just go through and if the size of the binrel is > numpairs/chunks, then generate an anteriorIt for+        // each+        const size_t perchunk = numPairs / chunks;+        for (const auto& itp : equivalencePartition) {+            const size_t s = itp.second->size();+            if (s * s > perchunk) {+                for (const auto& i : *itp.second) {+                    ret.push_back(souffle::make_range(anteriorIt(i), end()));+                }+            } else {+                ret.push_back(souffle::make_range(closure(itp.first), end()));+            }+        }++        return ret;+    }++    iterator find(const TupleType&, operation_hints&) const {+        throw std::runtime_error("error: find() is not compatible with equivalence relations");+        return begin();+    }++    iterator find(const TupleType& t) const {+        operation_hints context;+        return find(t, context);+    }++protected:+    bool containsElement(value_type e) const {+        return this->sds.nodeExists(e);+    }++private:+    // marked as mutable due to difficulties with the const enforcement via the Relation API+    // const operations *may* safely change internal state (i.e. collapse djset forest)+    mutable souffle::SparseDisjointSet<value_type> sds;++    // read/write lock on equivalencePartition+    mutable std::shared_mutex statesLock;++    mutable StatesMap equivalencePartition;+    // whether the cache is stale+    mutable std::atomic<bool> statesMapStale;++    /**+     * Generate a cache of the sets such that they can be iterated over efficiently.+     * Each set is partitioned into a PiggyList.+     */+    void genAllDisjointSetLists() const {+        statesLock.lock();++        // no need to generate again, already done.+        if (!this->statesMapStale.load(std::memory_order_acquire)) {+            statesLock.unlock();+            return;+        }++        // btree version+        emptyPartition();++        size_t dSetSize = this->sds.ds.a_blocks.size();+        for (size_t i = 0; i < dSetSize; ++i) {+            typename TupleType::value_type sparseVal = this->sds.toSparse(i);+            parent_t rep = this->sds.findNode(sparseVal);++            StorePair p = {static_cast<value_type>(rep), nullptr};+            StatesList* mapList = equivalencePartition.insert(p, [&](StorePair& sp) {+                auto* r = new StatesList(1);+                sp.second = r;+                return r;+            });+            mapList->append(sparseVal);+        }++        statesMapStale.store(false, std::memory_order_release);+        statesLock.unlock();+    }+};+}  // namespace souffle
+ cbits/souffle/datastructure/LambdaBTree.h view
@@ -0,0 +1,620 @@+/*+ * Souffle - A Datalog Compiler+ * Copyright (c) 2018, Souffle Developers+ * Licensed under the Universal Permissive License v 1.0 as shown at:+ * - https://opensource.org/licenses/UPL+ * - <souffle root>/licenses/SOUFFLE-UPL.txt+ */++/************************************************************************+ *+ * @file LambdaBTree.h+ *+ * An implementation of a generic B-tree data structure including+ * interfaces for utilizing instances as set or multiset containers.+ * Allows the user to provide a function to execute on successful insert+ * Be careful using this, it currently expects a pair as the key.+ *+ ***********************************************************************/++#pragma once++#include "souffle/datastructure/BTree.h"+#include "souffle/utility/ContainerUtil.h"+#include "souffle/utility/ParallelUtil.h"+#include <atomic>+#include <cassert>+#include <typeinfo>+#include <vector>++namespace souffle {++namespace detail {+/**+ * The actual implementation of a b-tree data structure.+ *+ * @tparam Key             .. the element type to be stored in this tree+ * @tparam Comparator     .. a class defining an order on the stored elements+ * @tparam Allocator     .. utilized for allocating memory for required nodes+ * @tparam blockSize    .. determines the number of bytes/block utilized by leaf nodes+ * @tparam SearchStrategy .. enables switching between linear, binary or any other search strategy+ * @tparam isSet        .. true = set, false = multiset+ * @tparam Functor      .. a std::function that is called on successful (new) insert+ */+template <typename Key, typename Comparator,+        typename Allocator,  // is ignored so far - TODO: add support+        unsigned blockSize, typename SearchStrategy, bool isSet, typename Functor,+        typename WeakComparator = Comparator, typename Updater = detail::updater<Key>>+class LambdaBTree : public btree<Key, Comparator, Allocator, blockSize, SearchStrategy, isSet, WeakComparator,+                            Updater> {+public:+    using parenttype =+            btree<Key, Comparator, Allocator, blockSize, SearchStrategy, isSet, WeakComparator, Updater>;++    LambdaBTree(const Comparator& comp = Comparator(), const WeakComparator& weak_comp = WeakComparator())+            : parenttype(comp, weak_comp) {}++    /**+     * Inserts the given key into this tree.+     */+    typename Functor::result_type insert(Key& k, const Functor& f) {+        typename parenttype::operation_hints hints;+        return insert(k, hints, f);+    }++    // rewriting this because of david's changes+    typename Functor::result_type insert(+            Key& k, typename parenttype::operation_hints& hints, const Functor& f) {+#ifdef IS_PARALLEL++        // special handling for inserting first element+        while (this->root == nullptr) {+            // try obtaining root-lock+            if (!this->root_lock.try_start_write()) {+                // somebody else was faster => re-check+                continue;+            }++            // check loop condition again+            if (this->root != nullptr) {+                // somebody else was faster => normal insert+                this->root_lock.abort_write();+                break;+            }++            // create new node+            this->leftmost = new typename parenttype::leaf_node();+            this->leftmost->numElements = 1;+            // call the functor as we've successfully inserted+            typename Functor::result_type res = f(k);++            this->leftmost->keys[0] = k;+            this->root = this->leftmost;++            // operation complete => we can release the root lock+            this->root_lock.end_write();++            hints.last_insert.access(this->leftmost);++            return res;+        }++        // insert using iterative implementation++        typename parenttype::node* cur = nullptr;++        // test last insert hints+        typename parenttype::lock_type::Lease cur_lease;++        auto checkHint = [&](typename parenttype::node* last_insert) {+            // ignore null pointer+            if (!last_insert) return false;+            // get a read lease on indicated node+            auto hint_lease = last_insert->lock.start_read();+            // check whether it covers the key+            if (!this->weak_covers(last_insert, k)) return false;+            // and if there was no concurrent modification+            if (!last_insert->lock.validate(hint_lease)) return false;+            // use hinted location+            cur = last_insert;+            // and keep lease+            cur_lease = hint_lease;+            // we found a hit+            return true;+        };++        if (hints.last_insert.any(checkHint)) {+            // register this as a hit+            this->hint_stats.inserts.addHit();+        } else {+            // register this as a miss+            this->hint_stats.inserts.addMiss();+        }++        // if there is no valid hint ..+        if (!cur) {+            do {+                // get root - access lock+                auto root_lease = this->root_lock.start_read();++                // start with root+                cur = this->root;++                // get lease of the next node to be accessed+                cur_lease = cur->lock.start_read();++                // check validity of root pointer+                if (this->root_lock.end_read(root_lease)) {+                    break;+                }++            } while (true);+        }++        while (true) {+            // handle inner nodes+            if (cur->inner) {+                auto a = &(cur->keys[0]);+                auto b = &(cur->keys[cur->numElements]);++                auto pos = this->search.lower_bound(k, a, b, this->weak_comp);+                auto idx = pos - a;++                // early exit for sets+                if (isSet && pos != b && this->weak_equal(*pos, k)) {+                    // validate results+                    if (!cur->lock.validate(cur_lease)) {+                        // start over again+                        return insert(k, hints, f);+                    }++                    // update provenance information+                    if (typeid(Comparator) != typeid(WeakComparator) && this->less(k, *pos)) {+                        if (!cur->lock.try_upgrade_to_write(cur_lease)) {+                            // start again+                            return insert(k, hints, f);+                        }+                        this->update(*pos, k);++                        // get result before releasing lock+                        auto res = (*pos).second;++                        cur->lock.end_write();+                        return res;+                    }++                    // get the result before releasing lock+                    auto res = (*pos).second;++                    // check validity+                    if (!cur->lock.validate(cur_lease)) {+                        // start over again+                        return insert(k, hints, f);+                    }++                    // we found the element => return the result+                    return res;+                }++                // get next pointer+                auto next = cur->getChild(idx);++                // get lease on next level+                auto next_lease = next->lock.start_read();++                // check whether there was a write+                if (!cur->lock.end_read(cur_lease)) {+                    // start over+                    return insert(k, hints, f);+                }++                // go to next+                cur = next;++                // move on lease+                cur_lease = next_lease;++                continue;+            }++            // the rest is for leaf nodes+            assert(!cur->inner);++            // -- insert node in leaf node --++            auto a = &(cur->keys[0]);+            auto b = &(cur->keys[cur->numElements]);++            auto pos = this->search.upper_bound(k, a, b, this->weak_comp);+            auto idx = pos - a;++            // early exit for sets+            if (isSet && pos != a && this->weak_equal(*(pos - 1), k)) {+                // validate result+                if (!cur->lock.validate(cur_lease)) {+                    // start over again+                    return insert(k, hints, f);+                }++                // TODO (pnappa): remove provenance from LambdaBTree - no use for it+                // update provenance information+                if (typeid(Comparator) != typeid(WeakComparator) && this->less(k, *(pos - 1))) {+                    if (!cur->lock.try_upgrade_to_write(cur_lease)) {+                        // start again+                        return insert(k, hints, f);+                    }+                    this->update(*(pos - 1), k);++                    // retrieve result before releasing lock+                    auto res = (*(pos - 1)).second;++                    cur->lock.end_write();+                    return res;+                }++                // read result (atomic) -- just as a proof of concept, this is actually not valid!!+                std::atomic<typename Functor::result_type>& loc =+                        *reinterpret_cast<std::atomic<typename Functor::result_type>*>(&(*(pos - 1)).second);+                auto res = loc.load(std::memory_order_relaxed);++                // check validity+                if (!cur->lock.validate(cur_lease)) {+                    // start over again+                    return insert(k, hints, f);+                }++                // we found the element => done+                return res;+            }++            // upgrade to write-permission+            if (!cur->lock.try_upgrade_to_write(cur_lease)) {+                // something has changed => restart+                hints.last_insert.access(cur);+                return insert(k, hints, f);+            }++            if (cur->numElements >= parenttype::node::maxKeys) {+                // -- lock parents --+                auto priv = cur;+                auto parent = priv->parent;+                std::vector<typename parenttype::node*> parents;+                do {+                    if (parent) {+                        parent->lock.start_write();+                        while (true) {+                            // check whether parent is correct+                            if (parent == priv->parent) {+                                break;+                            }+                            // switch parent+                            parent->lock.abort_write();+                            parent = priv->parent;+                            parent->lock.start_write();+                        }+                    } else {+                        // lock root lock => since cur is root+                        this->root_lock.start_write();+                    }++                    // record locked node+                    parents.push_back(parent);++                    // stop at "sphere of influence"+                    if (!parent || !parent->isFull()) {+                        break;+                    }++                    // go one step higher+                    priv = parent;+                    parent = parent->parent;++                } while (true);++                // split this node+                auto old_root = this->root;+                idx -= cur->rebalance_or_split(+                        const_cast<typename parenttype::node**>(&this->root), this->root_lock, idx, parents);++                // release parent lock+                for (auto it = parents.rbegin(); it != parents.rend(); ++it) {+                    auto parent = *it;++                    // release this lock+                    if (parent) {+                        parent->lock.end_write();+                    } else {+                        if (old_root != this->root) {+                            this->root_lock.end_write();+                        } else {+                            this->root_lock.abort_write();+                        }+                    }+                }++                // insert element in right fragment+                if (((typename parenttype::size_type)idx) > cur->numElements) {+                    // release current lock+                    cur->lock.end_write();++                    // insert in sibling+                    return insert(k, hints, f);+                }+            }++            // ok - no split necessary+            assert(cur->numElements < parenttype::node::maxKeys && "Split required!");++            // move keys+            for (int j = cur->numElements; j > idx; --j) {+                cur->keys[j] = cur->keys[j - 1];+            }++            // insert new element+            typename Functor::result_type res = f(k);+            cur->keys[idx] = k;+            cur->numElements++;++            // release lock on current node+            cur->lock.end_write();++            // remember last insertion position+            hints.last_insert.access(cur);+            return res;+        }++#else+        // special handling for inserting first element+        if (this->empty()) {+            // create new node+            this->leftmost = new typename parenttype::leaf_node();+            this->leftmost->numElements = 1;+            // call the functor as we've successfully inserted+            typename Functor::result_type res = f(k);+            this->leftmost->keys[0] = k;+            this->root = this->leftmost;++            hints.last_insert.access(this->leftmost);++            return res;+        }++        // insert using iterative implementation+        typename parenttype::node* cur = this->root;++        auto checkHints = [&](typename parenttype::node* last_insert) {+            if (!last_insert) return false;+            if (!this->weak_covers(last_insert, k)) return false;+            cur = last_insert;+            return true;+        };++        // test last insert+        if (hints.last_insert.any(checkHints)) {+            this->hint_stats.inserts.addHit();+        } else {+            this->hint_stats.inserts.addMiss();+        }++        while (true) {+            // handle inner nodes+            if (cur->inner) {+                auto a = &(cur->keys[0]);+                auto b = &(cur->keys[cur->numElements]);++                auto pos = this->search.lower_bound(k, a, b, this->weak_comp);+                auto idx = pos - a;++                // early exit for sets+                if (isSet && pos != b && this->weak_equal(*pos, k)) {+                    // update provenance information+                    if (typeid(Comparator) != typeid(WeakComparator) && this->less(k, *pos)) {+                        this->update(*pos, k);+                        return (*pos).second;+                    }++                    return (*pos).second;+                }++                cur = cur->getChild(idx);+                continue;+            }++            // the rest is for leaf nodes+            assert(!cur->inner);++            // -- insert node in leaf node --++            auto a = &(cur->keys[0]);+            auto b = &(cur->keys[cur->numElements]);++            auto pos = this->search.upper_bound(k, a, b, this->weak_comp);+            auto idx = pos - a;++            // early exit for sets+            if (isSet && pos != a && this->weak_equal(*(pos - 1), k)) {+                // update provenance information+                if (typeid(Comparator) != typeid(WeakComparator) && this->less(k, *(pos - 1))) {+                    this->update(*(pos - 1), k);+                    return (*(pos - 1)).second;+                }++                return (*(pos - 1)).second;+            }++            if (cur->numElements >= parenttype::node::maxKeys) {+                // split this node+                idx -= cur->rebalance_or_split(+                        const_cast<typename parenttype::node**>(&this->root), this->root_lock, idx);++                // insert element in right fragment+                if (((typename parenttype::size_type)idx) > cur->numElements) {+                    idx -= cur->numElements + 1;+                    cur = cur->parent->getChild(cur->position + 1);+                }+            }++            // ok - no split necessary+            assert(cur->numElements < parenttype::node::maxKeys && "Split required!");++            // move keys+            for (int j = cur->numElements; j > idx; --j) {+                cur->keys[j] = cur->keys[j - 1];+            }++            // call the functor as we've successfully inserted+            typename Functor::result_type res = f(k);+            // insert new element+            cur->keys[idx] = k;+            cur->numElements++;++            // remember last insertion position+            hints.last_insert.access(cur);+            return res;+        }+#endif+    }++    /**+     * Inserts the given range of elements into this tree.+     */+    template <typename Iter>+    void insert(const Iter& a, const Iter& b) {+        // TODO: improve this beyond a naive insert+        typename parenttype::operation_hints hints;+        // a naive insert so far .. seems to work fine+        for (auto it = a; it != b; ++it) {+            // use insert with hint+            insert(*it, hints);+        }+    }++    /**+     * Swaps the content of this tree with the given tree. This+     * is a much more efficient operation than creating a copy and+     * realizing the swap utilizing assignment operations.+     */+    void swap(LambdaBTree& other) {+        // swap the content+        std::swap(this->root, other.root);+        std::swap(this->leftmost, other.leftmost);+    }++    // Implementation of the assignment operation for trees.+    LambdaBTree& operator=(const LambdaBTree& other) {+        // check identity+        if (this == &other) {+            return *this;+        }++        // create a deep-copy of the content of the other tree+        // shortcut for empty sets+        if (other.empty()) {+            return *this;+        }++        // clone content (deep copy)+        this->root = other.root->clone();++        // update leftmost reference+        auto tmp = this->root;+        while (!tmp->isLeaf()) {+            tmp = tmp->getChild(0);+        }+        this->leftmost = static_cast<typename parenttype::leaf_node*>(tmp);++        // done+        return *this;+    }++    // Implementation of an equality operation for trees.+    bool operator==(const LambdaBTree& other) const {+        // check identity+        if (this == &other) {+            return true;+        }++        // check size+        if (this->size() != other.size()) {+            return false;+        }+        if (this->size() < other.size()) {+            return other == *this;+        }++        // check content+        for (const auto& key : other) {+            if (!contains(key)) {+                return false;+            }+        }+        return true;+    }++    // Implementation of an inequality operation for trees.+    bool operator!=(const LambdaBTree& other) const {+        return !(*this == other);+    }+};++}  // end namespace detail++/**+ * A b-tree based set implementation.+ *+ * @tparam Key             .. the element type to be stored in this set+ * @tparam Functor         .. a std::function that is invoked on successful insert+ * @tparam Comparator     .. a class defining an order on the stored elements+ * @tparam Allocator     .. utilized for allocating memory for required nodes+ * @tparam blockSize    .. determines the number of bytes/block utilized by leaf nodes+ * @tparam SearchStrategy .. enables switching between linear, binary or any other search strategy+ */+template <typename Key, typename Functor, typename Comparator = detail::comparator<Key>,+        typename Allocator = std::allocator<Key>,  // is ignored so far+        unsigned blockSize = 256, typename SearchStrategy = typename detail::default_strategy<Key>::type>+class LambdaBTreeSet+        : public detail::LambdaBTree<Key, Comparator, Allocator, blockSize, SearchStrategy, true, Functor> {+    using super = detail::LambdaBTree<Key, Comparator, Allocator, blockSize, SearchStrategy, true, Functor>;++    friend class detail::LambdaBTree<Key, Comparator, Allocator, blockSize, SearchStrategy, true, Functor>;++public:+    /**+     * A default constructor creating an empty set.+     */+    LambdaBTreeSet(const Comparator& comp = Comparator()) : super(comp) {}++    /**+     * A constructor creating a set based on the given range.+     */+    template <typename Iter>+    LambdaBTreeSet(const Iter& a, const Iter& b) {+        this->insert(a, b);+    }++    // A copy constructor.+    LambdaBTreeSet(const LambdaBTreeSet& other) : super(other) {}++    // A move constructor.+    LambdaBTreeSet(LambdaBTreeSet&& other) : super(std::move(other)) {}++private:+    // A constructor required by the bulk-load facility.+    template <typename s, typename n, typename l>+    LambdaBTreeSet(s size, n* root, l* leftmost) : super::parenttype(size, root, leftmost) {}++public:+    // Support for the assignment operator.+    LambdaBTreeSet& operator=(const LambdaBTreeSet& other) {+        super::operator=(other);+        return *this;+    }++    // Support for the bulk-load operator.+    template <typename Iter>+    static LambdaBTreeSet load(const Iter& a, const Iter& b) {+        return super::template load<LambdaBTreeSet>(a, b);+    }+};++}  // end of namespace souffle
+ cbits/souffle/datastructure/PiggyList.h view
@@ -0,0 +1,329 @@+#pragma once++#include "souffle/utility/ParallelUtil.h"+#include <array>+#include <atomic>+#include <cstring>+#include <iostream>+#include <iterator>++#ifdef _WIN32+/**+ * Some versions of MSVC do not provide a builtin for counting leading zeroes+ * like gcc, so we have to implement it ourselves.+ */+#if _MSC_VER < 1924+unsigned long __inline __builtin_clzll(unsigned long long value) {+    unsigned long msb = 0;++    if (_BitScanReverse64(&msb, value))+        return 63 - msb;+    else+        return 64;+}+#endif  // _MSC_VER < 1924+#endif  // _WIN32++using std::size_t;+namespace souffle {++/**+ * A PiggyList that allows insertAt functionality.+ * This means we can't append, as we don't know the next available element.+ * insertAt is dangerous. You must be careful not to call it for the same index twice!+ */+template <class T>+class RandomInsertPiggyList {+public:+    RandomInsertPiggyList() = default;+    // an instance where the initial size is not 65k, and instead is user settable (to a power of+    // initialbitsize)+    RandomInsertPiggyList(size_t initialbitsize) : BLOCKBITS(initialbitsize) {}++    /** copy constructor */+    RandomInsertPiggyList(const RandomInsertPiggyList& other) : BLOCKBITS(other.BLOCKBITS) {+        this->numElements.store(other.numElements.load());++        // copy blocks from the old lookup table to this one+        for (size_t i = 0; i < maxContainers; ++i) {+            if (other.blockLookupTable[i].load() != nullptr) {+                // calculate the size of that block+                const size_t blockSize = INITIALBLOCKSIZE << i;++                // allocate that in the new container+                this->blockLookupTable[i].store(new T[blockSize]);++                // then copy the stuff over+                std::memcpy(this->blockLookupTable[i].load(), other.blockLookupTable[i].load(),+                        blockSize * sizeof(T));+            }+        }+    }++    // move ctr+    RandomInsertPiggyList(RandomInsertPiggyList&& other) = delete;+    // copy assign ctor+    RandomInsertPiggyList& operator=(RandomInsertPiggyList& other) = delete;+    // move assign ctor+    RandomInsertPiggyList& operator=(RandomInsertPiggyList&& other) = delete;++    ~RandomInsertPiggyList() {+        freeList();+    }++    inline size_t size() const {+        return numElements.load();+    }++    inline T* getBlock(size_t blockNum) const {+        return blockLookupTable[blockNum];+    }++    inline T& get(size_t index) const {+        size_t nindex = index + INITIALBLOCKSIZE;+        size_t blockNum = (63 - __builtin_clzll(nindex));+        size_t blockInd = (nindex) & ((1 << blockNum) - 1);+        return this->getBlock(blockNum - BLOCKBITS)[blockInd];+    }++    void insertAt(size_t index, T value) {+        // starting with an initial blocksize requires some shifting to transform into a nice powers of two+        // series+        size_t blockNum = (63 - __builtin_clzll(index + INITIALBLOCKSIZE)) - BLOCKBITS;++        // allocate the block if not allocated+        if (blockLookupTable[blockNum].load() == nullptr) {+            slock.lock();+            if (blockLookupTable[blockNum].load() == nullptr) {+                blockLookupTable[blockNum].store(new T[INITIALBLOCKSIZE << blockNum]);+            }+            slock.unlock();+        }++        this->get(index) = value;+        // we ALWAYS increment size, even if there was something there before (its impossible to tell!)+        // the onus is up to the user to not call this for an index twice+        ++numElements;+    }++    void clear() {+        freeList();+        numElements.store(0);+    }+    const size_t BLOCKBITS = 16ul;+    const size_t INITIALBLOCKSIZE = (1ul << BLOCKBITS);++    // number of elements currently stored within+    std::atomic<size_t> numElements{0};++    // 2^64 - 1 elements can be stored (default initialised to nullptrs)+    static constexpr size_t maxContainers = 64;+    std::array<std::atomic<T*>, maxContainers> blockLookupTable = {};++    // for parallel node insertions+    mutable SpinLock slock;++    /**+     * Free the arrays allocated within the linked list nodes+     */+    void freeList() {+        slock.lock();+        // delete all - deleting a nullptr is a no-op+        for (size_t i = 0; i < maxContainers; ++i) {+            delete[] blockLookupTable[i].load();+            // reset the container within to be empty.+            blockLookupTable[i].store(nullptr);+        }+        slock.unlock();+    }+};++template <class T>+class PiggyList {+public:+    PiggyList() : num_containers(0), container_size(0), m_size(0) {}+    PiggyList(size_t initialbitsize)+            : BLOCKBITS(initialbitsize), num_containers(0), container_size(0), m_size(0) {}++    /** copy constructor */+    PiggyList(const PiggyList& other) : BLOCKBITS(other.BLOCKBITS) {+        num_containers.store(other.num_containers.load());+        container_size.store(other.container_size.load());+        m_size.store(other.m_size.load());+        // copy each chunk from other into this+        // the size of the next container to allocate+        size_t cSize = BLOCKSIZE;+        for (size_t i = 0; i < other.num_containers; ++i) {+            this->blockLookupTable[i] = new T[cSize];+            std::memcpy(this->blockLookupTable[i], other.blockLookupTable[i], cSize * sizeof(T));+            cSize <<= 1;+        }+        // if this isn't the case, uhh+        assert((cSize >> 1) == container_size.load());+    }++    /** move constructor */+    PiggyList(PiggyList&& other) = delete;+    /** copy assign ctor **/+    PiggyList& operator=(const PiggyList& other) = delete;++    ~PiggyList() {+        freeList();+    }++    /**+     * Well, returns the number of nodes exist within the list + number of nodes queued to be inserted+     *  The reason for this, is that there may be many nodes queued up+     *  that haven't had time to had containers created and updated+     * @return the number of nodes exist within the list + number of nodes queued to be inserted+     */+    inline size_t size() const {+        return m_size.load();+    };++    inline T* getBlock(size_t blocknum) const {+        return this->blockLookupTable[blocknum];+    }++    size_t append(T element) {+        size_t new_index = m_size.fetch_add(1, std::memory_order_acquire);++        // will this not fit?+        if (container_size < new_index + 1) {+            sl.lock();+            // check and add as many containers as required+            while (container_size < new_index + 1) {+                blockLookupTable[num_containers] = new T[allocsize];+                num_containers += 1;+                container_size += allocsize;+                // double the number elements that will be allocated next time+                allocsize <<= 1;+            }+            sl.unlock();+        }++        this->get(new_index) = element;+        return new_index;+    }++    size_t createNode() {+        size_t new_index = m_size.fetch_add(1, std::memory_order_acquire);++        // will this not fit?+        if (container_size < new_index + 1) {+            sl.lock();+            // check and add as many containers as required+            while (container_size < new_index + 1) {+                blockLookupTable[num_containers] = new T[allocsize];+                num_containers += 1;+                container_size += allocsize;+                // double the number elements that will be allocated next time+                allocsize <<= 1;+            }+            sl.unlock();+        }++        return new_index;+    }++    /**+     * Retrieve a reference to the stored value at index+     * @param index position to search+     * @return the value at index+     */+    inline T& get(size_t index) const {+        // supa fast 2^16 size first block+        size_t nindex = index + BLOCKSIZE;+        size_t blockNum = (63 - __builtin_clzll(nindex));+        size_t blockInd = (nindex) & ((1 << blockNum) - 1);+        return this->getBlock(blockNum - BLOCKBITS)[blockInd];+    }++    /**+     * Clear all elements from the PiggyList+     */+    void clear() {+        freeList();+        m_size = 0;+        num_containers = 0;++        allocsize = BLOCKSIZE;+        container_size = 0;+    }++    class iterator : std::iterator<std::forward_iterator_tag, T> {+        size_t cIndex = 0;+        PiggyList* bl;++    public:+        // default ctor, to silence+        iterator() = default;++        /* begin iterator for iterating over all elements */+        iterator(PiggyList* bl) : bl(bl){};+        /* ender iterator for marking the end of the iteration */+        iterator(PiggyList* bl, size_t beginInd) : cIndex(beginInd), bl(bl){};++        T operator*() {+            return bl->get(cIndex);+        };+        const T operator*() const {+            return bl->get(cIndex);+        };++        iterator& operator++(int) {+            ++cIndex;+            return *this;+        };++        iterator operator++() {+            iterator ret(*this);+            ++cIndex;+            return ret;+        };++        bool operator==(const iterator& x) const {+            return x.cIndex == this->cIndex && x.bl == this->bl;+        };++        bool operator!=(const iterator& x) const {+            return !(x == *this);+        };+    };++    iterator begin() {+        return iterator(this);+    }+    iterator end() {+        return iterator(this, size());+    }+    const size_t BLOCKBITS = 16ul;+    const size_t BLOCKSIZE = (1ul << BLOCKBITS);++    // number of inserted+    std::atomic<size_t> num_containers = 0;+    size_t allocsize = BLOCKSIZE;+    std::atomic<size_t> container_size = 0;+    std::atomic<size_t> m_size = 0;++    // > 2^64 elements can be stored (default initialise to nullptrs)+    static constexpr size_t max_conts = 64;+    std::array<T*, max_conts> blockLookupTable = {};++    // for parallel node insertions+    mutable SpinLock sl;++    /**+     * Free the arrays allocated within the linked list nodes+     */+    void freeList() {+        sl.lock();+        // we don't know which ones are taken up!+        for (size_t i = 0; i < num_containers; ++i) {+            delete[] blockLookupTable[i];+        }+        sl.unlock();+    }+};++}  // namespace souffle
+ cbits/souffle/datastructure/Table.h view
@@ -0,0 +1,145 @@+/*+ * Souffle - A Datalog Compiler+ * Copyright (c) 2013, 2015, Oracle and/or its affiliates. All rights reserved+ * Licensed under the Universal Permissive License v 1.0 as shown at:+ * - https://opensource.org/licenses/UPL+ * - <souffle root>/licenses/SOUFFLE-UPL.txt+ */++/************************************************************************+ *+ * @file Table.h+ *+ * An implementation of a generic Table storing a position-fixed collection+ * of objects in main memory.+ *+ ***********************************************************************/++#pragma once++#include <iosfwd>+#include <iterator>++namespace souffle {++template <typename T, unsigned blockSize = 4096>+class Table {+    struct Block {+        Block* next;+        std::size_t used = 0;+        T data[blockSize];++        Block() : next(nullptr) {}++        bool isFull() const {+            return used == blockSize;+        }++        const T& append(const T& element) {+            const T& res = data[used];+            data[used] = element;+            used++;+            return res;+        }+    };++    Block* head;+    Block* tail;++    std::size_t count = 0;++public:+    class iterator : public std::iterator<std::forward_iterator_tag, T> {+        Block* block;+        unsigned pos;++    public:+        iterator(Block* block = nullptr, unsigned pos = 0) : block(block), pos(pos) {}++        iterator(const iterator&) = default;+        iterator(iterator&&) = default;+        iterator& operator=(const iterator&) = default;++        // the equality operator as required by the iterator concept+        bool operator==(const iterator& other) const {+            return (block == nullptr && other.block == nullptr) || (block == other.block && pos == other.pos);+        }++        // the not-equality operator as required by the iterator concept+        bool operator!=(const iterator& other) const {+            return !(*this == other);+        }++        // the deref operator as required by the iterator concept+        const T& operator*() const {+            return block->data[pos];+        }++        // the increment operator as required by the iterator concept+        iterator& operator++() {+            // move on in block+            if (++pos < block->used) {+                return *this;+            }+            // or to next block+            block = block->next;+            pos = 0;+            return *this;+        }+    };++    Table() : head(nullptr), tail(nullptr) {}++    ~Table() {+        clear();+    }++    bool empty() const {+        return (!head);+    }++    std::size_t size() const {+        return count;+    }++    const T& insert(const T& element) {+        // check whether the head is initialized+        if (!head) {+            head = new Block();+            tail = head;+        }++        // check whether tail is full+        if (tail->isFull()) {+            tail->next = new Block();+            tail = tail->next;+        }++        // increment counter+        count++;++        // add another element+        return tail->append(element);+    }++    iterator begin() const {+        return iterator(head);+    }++    iterator end() const {+        return iterator();+    }++    void clear() {+        while (head != nullptr) {+            auto cur = head;+            head = head->next;+            delete cur;+        }+        count = 0;+        head = nullptr;+        tail = nullptr;+    }+};++}  // end namespace souffle
+ cbits/souffle/datastructure/UnionFind.h view
@@ -0,0 +1,356 @@+/*+ * Souffle - A Datalog Compiler+ * Copyright (c) 2017 The Souffle Developers. All rights reserved+ * Licensed under the Universal Permissive License v 1.0 as shown at:+ * - https://opensource.org/licenses/UPL+ * - <souffle root>/licenses/SOUFFLE-UPL.txt+ */++/************************************************************************+ *+ * @file UnionFind.h+ *+ * Defines a union-find data-structure+ *+ ***********************************************************************/++#pragma once++#include "souffle/datastructure/LambdaBTree.h"+#include "souffle/datastructure/PiggyList.h"+#include <atomic>+#include <cstddef>+#include <cstdint>+#include <functional>+#include <utility>++namespace souffle {++// branch predictor hacks+#define unlikely(x) __builtin_expect((x), 0)+#define likely(x) __builtin_expect((x), 1)++using rank_t = uint8_t;+/* technically uint56_t, but, doesn't exist. Just be careful about storing > 2^56 elements. */+using parent_t = uint64_t;++// number of bits that the rank is+constexpr uint8_t split_size = 8u;++// block_t stores parent in the upper half, rank in the lower half+using block_t = uint64_t;+// block_t & rank_mask extracts the rank+constexpr block_t rank_mask = (1ul << split_size) - 1;++/**+ * Structure that emulates a Disjoint Set, i.e. a data structure that supports efficient union-find operations+ */+class DisjointSet {+    template <typename TupleType>+    friend class EquivalenceRelation;++    PiggyList<std::atomic<block_t>> a_blocks;++public:+    DisjointSet() = default;++    // copy ctor+    DisjointSet(DisjointSet& other) = delete;+    // move ctor+    DisjointSet(DisjointSet&& other) = delete;++    // copy assign ctor+    DisjointSet& operator=(DisjointSet& ds) = delete;+    // move assign ctor+    DisjointSet& operator=(DisjointSet&& ds) = delete;++    /**+     * Return the number of elements in this disjoint set (not the number of pairs)+     */+    inline size_t size() {+        auto sz = a_blocks.size();+        return sz;+    };++    /**+     * Yield reference to the node by its node index+     * @param node node to be searched+     * @return the parent block of the specified node+     */+    inline std::atomic<block_t>& get(parent_t node) const {+        auto& ret = a_blocks.get(node);+        return ret;+    };++    /**+     * Equivalent to the find() function in union/find+     * Find the highest ancestor of the provided node - flattening as we go+     * @param x the node to find the parent of, whilst flattening its set-tree+     * @return The parent of x+     */+    parent_t findNode(parent_t x) {+        // while x's parent is not itself+        while (x != b2p(get(x))) {+            block_t xState = get(x);+            // yield x's parent's parent+            parent_t newParent = b2p(get(b2p(xState)));+            // construct block out of the original rank and the new parent+            block_t newState = pr2b(newParent, b2r(xState));++            this->get(x).compare_exchange_strong(xState, newState);++            x = newParent;+        }+        return x;+    }++private:+    /**+     * Update the root of the tree of which x is, to have y as the base instead+     * @param x : old root+     * @param oldrank : old root rank+     * @param y : new root+     * @param newrank : new root rank+     * @return Whether the update succeeded (fails if another root update/union has been perfomed in the+     * interim)+     */+    bool updateRoot(const parent_t x, const rank_t oldrank, const parent_t y, const rank_t newrank) {+        block_t oldState = get(x);+        parent_t nextN = b2p(oldState);+        rank_t rankN = b2r(oldState);++        if (nextN != x || rankN != oldrank) return false;+        // set the parent and rank of the new record+        block_t newVal = pr2b(y, newrank);++        return this->get(x).compare_exchange_strong(oldState, newVal);+    }++public:+    /**+     * Clears the DisjointSet of all nodes+     * Invalidates all iterators+     */+    void clear() {+        a_blocks.clear();+    }++    /**+     * Check whether the two indices are in the same set+     * @param x node to be checked+     * @param y node to be checked+     * @return where the two indices are in the same set+     */+    bool sameSet(parent_t x, parent_t y) {+        while (true) {+            x = findNode(x);+            y = findNode(y);+            if (x == y) return true;+            // if x's parent is itself, they are not the same set+            if (b2p(get(x)) == x) return false;+        }+    }++    /**+     * Union the two specified index nodes+     * @param x node to be unioned+     * @param y node to be unioned+     */+    void unionNodes(parent_t x, parent_t y) {+        while (true) {+            x = findNode(x);+            y = findNode(y);++            // no need to union if both already in same set+            if (x == y) return;++            rank_t xrank = b2r(get(x));+            rank_t yrank = b2r(get(y));++            // if x comes before y (better rank or earlier & equal node)+            if (xrank > yrank || ((xrank == yrank) && x > y)) {+                std::swap(x, y);+                std::swap(xrank, yrank);+            }+            // join the trees together+            // perhaps we can optimise the use of compare_exchange_strong here, as we're in a pessimistic loop+            if (!updateRoot(x, xrank, y, yrank)) {+                continue;+            }+            // make sure that the ranks are orderable+            if (xrank == yrank) {+                updateRoot(y, yrank, y, yrank + 1);+            }+            break;+        }+    }++    /**+     * Create a node with its parent as itself, rank 0+     * @return the newly created block+     */+    inline block_t makeNode() {+        // make node and find out where we've added it+        size_t nodeDetails = a_blocks.createNode();++        a_blocks.get(nodeDetails).store(pr2b(nodeDetails, 0));++        return a_blocks.get(nodeDetails).load();+    };++    /**+     * Extract parent from block+     * @param inblock the block to be masked+     * @return The parent_t contained in the upper half of block_t+     */+    static inline parent_t b2p(const block_t inblock) {+        return (parent_t)(inblock >> split_size);+    };++    /**+     * Extract rank from block+     * @param inblock the block to be masked+     * @return the rank_t contained in the lower half of block_t+     */+    static inline rank_t b2r(const block_t inblock) {+        return (rank_t)(inblock & rank_mask);+    };++    /**+     * Yield a block given parent and rank+     * @param parent the top half bits+     * @param rank the lower half bits+     * @return the resultant block after merge+     */+    static inline block_t pr2b(const parent_t parent, const rank_t rank) {+        return (((block_t)parent) << split_size) | rank;+    };+};++template <typename StorePair>+struct EqrelMapComparator {+    int operator()(const StorePair& a, const StorePair& b) {+        if (a.first < b.first) {+            return -1;+        } else if (b.first < a.first) {+            return 1;+        } else {+            return 0;+        }+    }++    bool less(const StorePair& a, const StorePair& b) {+        return operator()(a, b) < 0;+    }++    bool equal(const StorePair& a, const StorePair& b) {+        return operator()(a, b) == 0;+    }+};++template <typename SparseDomain>+class SparseDisjointSet {+    DisjointSet ds;++    template <typename TupleType>+    friend class EquivalenceRelation;++    using PairStore = std::pair<SparseDomain, parent_t>;+    using SparseMap =+            LambdaBTreeSet<PairStore, std::function<parent_t(PairStore&)>, EqrelMapComparator<PairStore>>;+    using DenseMap = RandomInsertPiggyList<SparseDomain>;++    typename SparseMap::operation_hints last_ins;++    SparseMap sparseToDenseMap;+    // mapping from union-find val to souffle, union-find encoded as index+    DenseMap denseToSparseMap;++public:+    /**+     * Retrieve dense encoding, adding it in if non-existent+     * @param in the sparse value+     * @return the corresponding dense value+     */+    parent_t toDense(const SparseDomain in) {+        // insert into the mapping - if the key doesn't exist (in), the function will be called+        // and a dense value will be created for it+        PairStore p = {in, -1};+        return sparseToDenseMap.insert(p, [&](PairStore& p) {+            parent_t c2 = DisjointSet::b2p(this->ds.makeNode());+            this->denseToSparseMap.insertAt(c2, p.first);+            p.second = c2;+            return c2;+        });+    }++public:+    SparseDisjointSet() = default;++    // copy ctor+    SparseDisjointSet(SparseDisjointSet& other) = delete;++    // move ctor+    SparseDisjointSet(SparseDisjointSet&& other) = delete;++    // copy assign ctor+    SparseDisjointSet& operator=(SparseDisjointSet& other) = delete;++    // move assign ctor+    SparseDisjointSet& operator=(SparseDisjointSet&& other) = delete;++    /**+     * For the given dense value, return the associated sparse value+     *   Undefined behaviour if dense value not in set+     * @param in the supplied dense value+     * @return the sparse value from the denseToSparseMap+     */+    inline const SparseDomain toSparse(const parent_t in) const {+        return denseToSparseMap.get(in);+    };++    /* a wrapper to enable checking in the sparse set - however also adds them if not already existing */+    inline bool sameSet(SparseDomain x, SparseDomain y) {+        return ds.sameSet(toDense(x), toDense(y));+    };+    /* finds the node in the underlying disjoint set, adding the node if non-existent */+    inline SparseDomain findNode(SparseDomain x) {+        return toSparse(ds.findNode(toDense(x)));+    };+    /* union the nodes, add if not existing */+    inline void unionNodes(SparseDomain x, SparseDomain y) {+        ds.unionNodes(toDense(x), toDense(y));+    };++    inline std::size_t size() {+        return ds.size();+    };++    /**+     * Remove all elements from this disjoint set+     */+    void clear() {+        ds.clear();+        sparseToDenseMap.clear();+        denseToSparseMap.clear();+    }++    /* wrapper for node creation */+    inline void makeNode(SparseDomain val) {+        // dense has the behaviour of creating if not exists.+        toDense(val);+    };++    /* whether we the supplied node exists */+    inline bool nodeExists(const SparseDomain val) const {+        return sparseToDenseMap.contains({val, -1});+    };++    inline bool contains(SparseDomain v1, SparseDomain v2) {+        if (nodeExists(v1) && nodeExists(v2)) {+            return sameSet(v1, v2);+        }+        return false;+    }+};+}  // namespace souffle
− cbits/souffle/gzfstream.h 
@@ -1,235 +0,0 @@-/*- * Souffle - A Datalog Compiler- * Copyright (c) 2013, 2014, Oracle and/or its affiliates. All rights reserved- * Licensed under the Universal Permissive License v 1.0 as shown at:- * - https://opensource.org/licenses/UPL- * - <souffle root>/licenses/SOUFFLE-UPL.txt- */--/************************************************************************- *- * @file gzfstream.h- * A simple zlib wrapper to provide gzip file streams.- *- ***********************************************************************/--#pragma once--#include <cstdio>-#include <cstring>-#include <iostream>-#include <string>-#include <zlib.h>--namespace souffle {--namespace gzfstream {--namespace internal {--class gzfstreambuf : public std::streambuf {-public:-    gzfstreambuf() {-        setp(buffer, buffer + (bufferSize - 1));-        setg(buffer + reserveSize, buffer + reserveSize, buffer + reserveSize);-    }--    gzfstreambuf(const gzfstreambuf&) = delete;--    gzfstreambuf(gzfstreambuf&& old) = default;--    gzfstreambuf* open(const std::string& filename, std::ios_base::openmode mode) {-        if (is_open()) {-            return nullptr;-        }-        if ((mode ^ std::ios::in ^ std::ios::out) == 0) {-            return nullptr;-        }--        this->mode = mode;-        std::string gzmode((mode & std::ios::in) != 0 ? "rb" : "wb");-        fileHandle = gzopen(filename.c_str(), gzmode.c_str());--        if (fileHandle == nullptr) {-            return nullptr;-        }-        isOpen = true;--        return this;-    }--    gzfstreambuf* close() {-        if (is_open()) {-            sync();-            isOpen = false;-            if (gzclose(fileHandle) == Z_OK) {-                return this;-            }-        }-        return nullptr;-    }--    bool is_open() const {-        return isOpen;-    }--    ~gzfstreambuf() override {-        try {-            close();-        } catch (...) {-            // Don't throw exceptions.-        }-    }--protected:-    int_type overflow(int c = EOF) override {-        if (((mode & std::ios::out) == 0) || !isOpen) {-            return EOF;-        }--        if (c != EOF) {-            *pptr() = c;-            pbump(1);-        }-        int toWrite = pptr() - pbase();-        if (gzwrite(fileHandle, pbase(), toWrite) != toWrite) {-            return EOF;-        }-        pbump(-toWrite);--        return c;-    }--    int_type underflow() override {-        if (((mode & std::ios::in) == 0) || !isOpen) {-            return EOF;-        }-        if ((gptr() != nullptr) && (gptr() < egptr())) {-            return traits_type::to_int_type(*gptr());-        }--        unsigned charsPutBack = gptr() - eback();-        if (charsPutBack > reserveSize) {-            charsPutBack = reserveSize;-        }-        memcpy(buffer + reserveSize - charsPutBack, gptr() - charsPutBack, charsPutBack);--        int charsRead = gzread(fileHandle, buffer + reserveSize, bufferSize - reserveSize);-        if (charsRead <= 0) {-            return EOF;-        }--        setg(buffer + reserveSize - charsPutBack, buffer + reserveSize, buffer + reserveSize + charsRead);--        return traits_type::to_int_type(*gptr());-    }--    int sync() override {-        if ((pptr() != nullptr) && pptr() > pbase()) {-            int toWrite = pptr() - pbase();-            if (gzwrite(fileHandle, pbase(), toWrite) != toWrite) {-                return -1;-            }-            pbump(-toWrite);-        }-        return 0;-    }--private:-    static constexpr unsigned int bufferSize = 65536;-    static constexpr unsigned int reserveSize = 16;--    char buffer[bufferSize] = {};-    gzFile fileHandle = {};-    bool isOpen = false;-    std::ios_base::openmode mode = std::ios_base::in;-};--class gzfstream : virtual public std::ios {-public:-    gzfstream() {-        init(&buf);-    }--    gzfstream(const std::string& filename, std::ios_base::openmode mode) {-        init(&buf);-        open(filename, mode);-    }--    gzfstream(const gzfstream&) = delete;--    gzfstream(gzfstream&&) = delete;--    ~gzfstream() override = default;--    void open(const std::string& filename, std::ios_base::openmode mode) {-        if (buf.open(filename, mode) == nullptr) {-            clear(rdstate() | std::ios::badbit);-        }-    }--    bool is_open() {-        return buf.is_open();-    }--    void close() {-        if (buf.is_open()) {-            if (buf.close() == nullptr) {-                clear(rdstate() | std::ios::badbit);-            }-        }-    }--    gzfstreambuf* rdbuf() const {-        return &buf;-    }--protected:-    mutable gzfstreambuf buf;-};--}  // namespace internal--class igzfstream : public internal::gzfstream, public std::istream {-public:-    igzfstream() : internal::gzfstream(), std::istream(&buf) {}--    explicit igzfstream(const std::string& filename, std::ios_base::openmode mode = std::ios::in)-            : internal::gzfstream(filename, mode), std::istream(&buf) {}--    igzfstream(const igzfstream&) = delete;--    igzfstream(igzfstream&&) = delete;--    internal::gzfstreambuf* rdbuf() const {-        return internal::gzfstream::rdbuf();-    }--    void open(const std::string& filename, std::ios_base::openmode mode = std::ios::in) {-        internal::gzfstream::open(filename, mode);-    }-};--class ogzfstream : public internal::gzfstream, public std::ostream {-public:-    ogzfstream() : std::ostream(&buf) {}--    explicit ogzfstream(const std::string& filename, std::ios_base::openmode mode = std::ios::out)-            : internal::gzfstream(filename, mode), std::ostream(&buf) {}--    ogzfstream(const ogzfstream&) = delete;--    ogzfstream(ogzfstream&&) = delete;--    internal::gzfstreambuf* rdbuf() const {-        return internal::gzfstream::rdbuf();-    }--    void open(const std::string& filename, std::ios_base::openmode mode = std::ios::out) {-        internal::gzfstream::open(filename, mode);-    }-};--} /* namespace gzfstream */--} /* namespace souffle */
+ cbits/souffle/io/IOSystem.h view
@@ -0,0 +1,98 @@+/*+ * Souffle - A Datalog Compiler+ * Copyright (c) 2013, 2014, Oracle and/or its affiliates. All rights reserved+ * Licensed under the Universal Permissive License v 1.0 as shown at:+ * - https://opensource.org/licenses/UPL+ * - <souffle root>/licenses/SOUFFLE-UPL.txt+ */++/************************************************************************+ *+ * @file IOSystem.h+ *+ ***********************************************************************/++#pragma once++#include "souffle/RamTypes.h"+#include "souffle/SymbolTable.h"+#include "souffle/io/ReadStream.h"+#include "souffle/io/ReadStreamCSV.h"+#include "souffle/io/ReadStreamJSON.h"+#include "souffle/io/WriteStream.h"+#include "souffle/io/WriteStreamCSV.h"+#include "souffle/io/WriteStreamJSON.h"++#ifdef USE_SQLITE+#include "souffle/io/ReadStreamSQLite.h"+#include "souffle/io/WriteStreamSQLite.h"+#endif++#include <map>+#include <memory>+#include <stdexcept>+#include <string>++namespace souffle {+class RecordTable;++class IOSystem {+public:+    static IOSystem& getInstance() {+        static IOSystem singleton;+        return singleton;+    }++    void registerWriteStreamFactory(const std::shared_ptr<WriteStreamFactory>& factory) {+        outputFactories[factory->getName()] = factory;+    }++    void registerReadStreamFactory(const std::shared_ptr<ReadStreamFactory>& factory) {+        inputFactories[factory->getName()] = factory;+    }++    /**+     * Return a new WriteStream+     */+    Own<WriteStream> getWriter(const std::map<std::string, std::string>& rwOperation,+            const SymbolTable& symbolTable, const RecordTable& recordTable) const {+        std::string ioType = rwOperation.at("IO");+        if (outputFactories.count(ioType) == 0) {+            throw std::invalid_argument("Requested output type <" + ioType + "> is not supported.");+        }+        return outputFactories.at(ioType)->getWriter(rwOperation, symbolTable, recordTable);+    }+    /**+     * Return a new ReadStream+     */+    Own<ReadStream> getReader(const std::map<std::string, std::string>& rwOperation, SymbolTable& symbolTable,+            RecordTable& recordTable) const {+        std::string ioType = rwOperation.at("IO");+        if (inputFactories.count(ioType) == 0) {+            throw std::invalid_argument("Requested input type <" + ioType + "> is not supported.");+        }+        return inputFactories.at(ioType)->getReader(rwOperation, symbolTable, recordTable);+    }+    ~IOSystem() = default;++private:+    IOSystem() {+        registerReadStreamFactory(std::make_shared<ReadFileCSVFactory>());+        registerReadStreamFactory(std::make_shared<ReadCinCSVFactory>());+        registerReadStreamFactory(std::make_shared<ReadFileJSONFactory>());+        registerReadStreamFactory(std::make_shared<ReadCinJSONFactory>());+        registerWriteStreamFactory(std::make_shared<WriteFileCSVFactory>());+        registerWriteStreamFactory(std::make_shared<WriteCoutCSVFactory>());+        registerWriteStreamFactory(std::make_shared<WriteCoutPrintSizeFactory>());+        registerWriteStreamFactory(std::make_shared<WriteFileJSONFactory>());+        registerWriteStreamFactory(std::make_shared<WriteCoutJSONFactory>());+#ifdef USE_SQLITE+        registerReadStreamFactory(std::make_shared<ReadSQLiteFactory>());+        registerWriteStreamFactory(std::make_shared<WriteSQLiteFactory>());+#endif+    };+    std::map<std::string, std::shared_ptr<WriteStreamFactory>> outputFactories;+    std::map<std::string, std::shared_ptr<ReadStreamFactory>> inputFactories;+};++} /* namespace souffle */
+ cbits/souffle/io/ReadStream.h view
@@ -0,0 +1,307 @@+/*+ * Souffle - A Datalog Compiler+ * Copyright (c) 2013, 2014, Oracle and/or its affiliates. All rights reserved+ * Licensed under the Universal Permissive License v 1.0 as shown at:+ * - https://opensource.org/licenses/UPL+ * - <souffle root>/licenses/SOUFFLE-UPL.txt+ */++/************************************************************************+ *+ * @file ReadStream.h+ *+ ***********************************************************************/++#pragma once++#include "souffle/RamTypes.h"+#include "souffle/RecordTable.h"+#include "souffle/SymbolTable.h"+#include "souffle/io/SerialisationStream.h"+#include "souffle/utility/ContainerUtil.h"+#include "souffle/utility/MiscUtil.h"+#include "souffle/utility/StringUtil.h"+#include "souffle/utility/json11.h"+#include <cctype>+#include <cstddef>+#include <map>+#include <memory>+#include <ostream>+#include <stdexcept>+#include <string>+#include <vector>++namespace souffle {++class ReadStream : public SerialisationStream<false> {+protected:+    ReadStream(+            const std::map<std::string, std::string>& rwOperation, SymbolTable& symTab, RecordTable& recTab)+            : SerialisationStream(symTab, recTab, rwOperation) {}++public:+    template <typename T>+    void readAll(T& relation) {+        auto lease = symbolTable.acquireLock();+        (void)lease;+        while (const auto next = readNextTuple()) {+            const RamDomain* ramDomain = next.get();+            relation.insert(ramDomain);+        }+    }++protected:+    /**+     * Read a record from a string.+     *+     * @param source - string containing a record+     * @param recordTypeName - record type.+     * @parem pos - start parsing from this position.+     * @param consumed - if not nullptr: number of characters read.+     *+     */+    RamDomain readRecord(const std::string& source, const std::string& recordTypeName, size_t pos = 0,+            size_t* charactersRead = nullptr) {+        const size_t initial_position = pos;++        // Check if record type information are present+        auto&& recordInfo = types["records"][recordTypeName];+        if (recordInfo.is_null()) {+            throw std::invalid_argument("Missing record type information: " + recordTypeName);+        }++        // Handle nil case+        consumeWhiteSpace(source, pos);+        if (source.substr(pos, 3) == "nil") {+            if (charactersRead != nullptr) {+                *charactersRead = 3;+            }+            return 0;+        }++        auto&& recordTypes = recordInfo["types"];+        const size_t recordArity = recordInfo["arity"].long_value();++        std::vector<RamDomain> recordValues(recordArity);++        consumeChar(source, '[', pos);++        for (size_t i = 0; i < recordArity; ++i) {+            const std::string& recordType = recordTypes[i].string_value();+            size_t consumed = 0;++            if (i > 0) {+                consumeChar(source, ',', pos);+            }+            consumeWhiteSpace(source, pos);+            switch (recordType[0]) {+                case 's': {+                    recordValues[i] = symbolTable.unsafeLookup(readUntil(source, ",]", pos, &consumed));+                    break;+                }+                case 'i': {+                    recordValues[i] = RamSignedFromString(source.substr(pos), &consumed);+                    break;+                }+                case 'u': {+                    recordValues[i] = ramBitCast(RamUnsignedFromString(source.substr(pos), &consumed));+                    break;+                }+                case 'f': {+                    recordValues[i] = ramBitCast(RamFloatFromString(source.substr(pos), &consumed));+                    break;+                }+                case 'r': {+                    recordValues[i] = readRecord(source, recordType, pos, &consumed);+                    break;+                }+                case '+': {+                    recordValues[i] = readADT(source, recordType, pos, &consumed);+                    break;+                }+                default: fatal("Invalid type attribute");+            }+            pos += consumed;+        }+        consumeChar(source, ']', pos);++        if (charactersRead != nullptr) {+            *charactersRead = pos - initial_position;+        }++        return recordTable.pack(recordValues.data(), recordValues.size());+    }++    RamDomain readADT(const std::string& source, const std::string& adtName, size_t pos = 0,+            size_t* charactersRead = nullptr) {+        const size_t initial_position = pos;++        // Branch will are encoded as [branchIdx, [branchValues...]].+        RamDomain branchIdx = -1;++        auto&& adtInfo = types["ADTs"][adtName];+        const auto& branches = adtInfo["branches"];++        if (adtInfo.is_null() || !branches.is_array()) {+            throw std::invalid_argument("Missing ADT information: " + adtName);+        }++        // Consume initial character+        consumeChar(source, '$', pos);+        std::string constructor = readAlphanumeric(source, pos);++        json11::Json branchInfo = [&]() -> json11::Json {+            for (auto branch : branches.array_items()) {+                ++branchIdx;+                if (branch["name"].string_value() == constructor) {+                    return branch;+                }+            }++            throw std::invalid_argument("Missing branch information: " + constructor);+        }();++        assert(branchInfo["types"].is_array());+        auto branchTypes = branchInfo["types"].array_items();++        // Handle a branch without arguments.+        if (branchTypes.empty()) {+            if (charactersRead != nullptr) {+                *charactersRead = pos - initial_position;+            }+            RamDomain emptyArgs = recordTable.pack(toVector<RamDomain>().data(), 0);+            return recordTable.pack(toVector<RamDomain>(branchIdx, emptyArgs).data(), 2);+        }++        consumeChar(source, '(', pos);++        std::vector<RamDomain> branchArgs(branchTypes.size());++        for (size_t i = 0; i < branchTypes.size(); ++i) {+            auto argType = branchTypes[i].string_value();+            assert(!argType.empty());++            size_t consumed = 0;++            if (i > 0) {+                consumeChar(source, ',', pos);+            }+            consumeWhiteSpace(source, pos);++            switch (argType[0]) {+                case 's': {+                    branchArgs[i] = symbolTable.unsafeLookup(readUntil(source, ",)", pos, &consumed));+                    break;+                }+                case 'i': {+                    branchArgs[i] = RamSignedFromString(source.substr(pos), &consumed);+                    break;+                }+                case 'u': {+                    branchArgs[i] = ramBitCast(RamUnsignedFromString(source.substr(pos), &consumed));+                    break;+                }+                case 'f': {+                    branchArgs[i] = ramBitCast(RamFloatFromString(source.substr(pos), &consumed));+                    break;+                }+                case 'r': {+                    branchArgs[i] = readRecord(source, argType, pos, &consumed);+                    break;+                }+                case '+': {+                    branchArgs[i] = readADT(source, argType, pos, &consumed);+                    break;+                }+                default: fatal("Invalid type attribute");+            }+            pos += consumed;+        }++        consumeChar(source, ')', pos);++        if (charactersRead != nullptr) {+            *charactersRead = pos - initial_position;+        }++        // Store branch either as [branch_id, [arguments]] or [branch_id, argument].+        RamDomain branchValue = [&]() -> RamDomain {+            if (branchArgs.size() != 1) {+                return recordTable.pack(branchArgs.data(), branchArgs.size());+            } else {+                return branchArgs[0];+            }+        }();++        return recordTable.pack(toVector<RamDomain>(branchIdx, branchValue).data(), 2);+    }++    /**+     * Read the next alphanumeric sequence (corresponding to IDENT).+     * Consume preceding whitespace.+     * TODO (darth_tytus): use std::string_view?+     */+    std::string readAlphanumeric(const std::string& source, size_t& pos) {+        consumeWhiteSpace(source, pos);+        if (pos >= source.length()) {+            throw std::invalid_argument("Unexpected end of input");+        }++        const size_t bgn = pos;+        while (pos < source.length() && std::isalnum(static_cast<unsigned char>(source[pos]))) {+            ++pos;+        }++        return source.substr(bgn, pos - bgn);+    }++    std::string readUntil(const std::string& source, const std::string stopChars, const size_t pos,+            size_t* charactersRead) {+        size_t endOfSymbol = source.find_first_of(stopChars, pos);++        if (endOfSymbol == std::string::npos) {+            throw std::invalid_argument("Unexpected end of input");+        }++        *charactersRead = endOfSymbol - pos;++        return source.substr(pos, *charactersRead);+    }++    /**+     * Read past given character, consuming any preceding whitespace.+     */+    void consumeChar(const std::string& str, char c, size_t& pos) {+        consumeWhiteSpace(str, pos);+        if (pos >= str.length()) {+            throw std::invalid_argument("Unexpected end of input");+        }+        if (str[pos] != c) {+            std::stringstream error;+            error << "Expected: \'" << c << "\', got: " << str[pos];+            throw std::invalid_argument(error.str());+        }+        ++pos;+    }++    /**+     * Advance position in the string until first non-whitespace character.+     */+    void consumeWhiteSpace(const std::string& str, size_t& pos) {+        while (pos < str.length() && std::isspace(static_cast<unsigned char>(str[pos]))) {+            ++pos;+        }+    }++    virtual Own<RamDomain[]> readNextTuple() = 0;+};++class ReadStreamFactory {+public:+    virtual Own<ReadStream> getReader(+            const std::map<std::string, std::string>&, SymbolTable&, RecordTable&) = 0;+    virtual const std::string& getName() const = 0;+    virtual ~ReadStreamFactory() = default;+};++} /* namespace souffle */
+ cbits/souffle/io/ReadStreamCSV.h view
@@ -0,0 +1,332 @@+/*+ * Souffle - A Datalog Compiler+ * Copyright (c) 2013, 2014, Oracle and/or its affiliates. All rights reserved+ * Licensed under the Universal Permissive License v 1.0 as shown at:+ * - https://opensource.org/licenses/UPL+ * - <souffle root>/licenses/SOUFFLE-UPL.txt+ */++/************************************************************************+ *+ * @file ReadStreamCSV.h+ *+ ***********************************************************************/++#pragma once++#include "souffle/RamTypes.h"+#include "souffle/SymbolTable.h"+#include "souffle/io/ReadStream.h"+#include "souffle/utility/ContainerUtil.h"+#include "souffle/utility/FileUtil.h"+#include "souffle/utility/StringUtil.h"++#ifdef USE_LIBZ+#include "souffle/io/gzfstream.h"+#else+#include <fstream>+#endif++#include <algorithm>+#include <cassert>+#include <cstddef>+#include <cstdint>+#include <iostream>+#include <map>+#include <memory>+#include <sstream>+#include <stdexcept>+#include <string>+#include <vector>++namespace souffle {+class RecordTable;++class ReadStreamCSV : public ReadStream {+public:+    ReadStreamCSV(std::istream& file, const std::map<std::string, std::string>& rwOperation,+            SymbolTable& symbolTable, RecordTable& recordTable)+            : ReadStream(rwOperation, symbolTable, recordTable),+              delimiter(getOr(rwOperation, "delimiter", "\t")), file(file), lineNumber(0),+              inputMap(getInputColumnMap(rwOperation, static_cast<unsigned int>(arity))) {+        while (inputMap.size() < arity) {+            int size = static_cast<int>(inputMap.size());+            inputMap[size] = size;+        }+    }++protected:+    /**+     * Read and return the next tuple.+     *+     * Returns nullptr if no tuple was readable.+     * @return+     */+    Own<RamDomain[]> readNextTuple() override {+        if (file.eof()) {+            return nullptr;+        }+        std::string line;+        Own<RamDomain[]> tuple = std::make_unique<RamDomain[]>(typeAttributes.size());++        if (!getline(file, line)) {+            return nullptr;+        }+        // Handle Windows line endings on non-Windows systems+        if (!line.empty() && line.back() == '\r') {+            line = line.substr(0, line.length() - 1);+        }+        ++lineNumber;++        size_t start = 0;+        size_t end = 0;+        size_t columnsFilled = 0;+        for (uint32_t column = 0; columnsFilled < arity; column++) {+            size_t charactersRead = 0;+            std::string element = nextElement(line, start, end);+            if (inputMap.count(column) == 0) {+                continue;+            }+            ++columnsFilled;++            try {+                auto&& ty = typeAttributes.at(inputMap[column]);+                switch (ty[0]) {+                    case 's': {+                        tuple[inputMap[column]] = symbolTable.unsafeLookup(element);+                        charactersRead = element.size();+                        break;+                    }+                    case 'r': {+                        tuple[inputMap[column]] = readRecord(element, ty, 0, &charactersRead);+                        break;+                    }+                    case '+': {+                        tuple[inputMap[column]] = readADT(element, ty, 0, &charactersRead);+                        break;+                    }+                    case 'i': {+                        tuple[inputMap[column]] = RamSignedFromString(element, &charactersRead);+                        break;+                    }+                    case 'u': {+                        tuple[inputMap[column]] = ramBitCast(readRamUnsigned(element, charactersRead));+                        break;+                    }+                    case 'f': {+                        tuple[inputMap[column]] = ramBitCast(RamFloatFromString(element, &charactersRead));+                        break;+                    }+                    default: fatal("invalid type attribute: `%c`", ty[0]);+                }+                // Check if everything was read.+                if (charactersRead != element.size()) {+                    throw std::invalid_argument(+                            "Expected: " + delimiter + " or \\n. Got: " + element[charactersRead]);+                }+            } catch (...) {+                std::stringstream errorMessage;+                errorMessage << "Error converting <" + element + "> in column " << column + 1 << " in line "+                             << lineNumber << "; ";+                throw std::invalid_argument(errorMessage.str());+            }+        }++        return tuple;+    }++    /**+     * Read an unsigned element. Possible bases are 2, 10, 16+     * Base is indicated by the first two chars.+     */+    RamUnsigned readRamUnsigned(const std::string& element, size_t& charactersRead) {+        // Sanity check+        assert(element.size() > 0);++        RamSigned value = 0;++        // Check prefix and parse the input.+        if (isPrefix("0b", element)) {+            value = RamUnsignedFromString(element, &charactersRead, 2);+        } else if (isPrefix("0x", element)) {+            value = RamUnsignedFromString(element, &charactersRead, 16);+        } else {+            value = RamUnsignedFromString(element, &charactersRead);+        }+        return value;+    }++    std::string nextElement(const std::string& line, size_t& start, size_t& end) {+        std::string element;++        // Handle record/tuple delimiter coincidence.+        if (delimiter.find(',') != std::string::npos) {+            int record_parens = 0;+            size_t next_delimiter = line.find(delimiter, start);++            // Find first delimiter after the record.+            while (end < std::min(next_delimiter, line.length()) || record_parens != 0) {+                // Track the number of parenthesis.+                if (line[end] == '[') {+                    ++record_parens;+                } else if (line[end] == ']') {+                    --record_parens;+                }++                // Check for unbalanced parenthesis.+                if (record_parens < 0) {+                    break;+                };++                ++end;++                // Find a next delimiter if the old one is invalid.+                // But only if inside the unbalance parenthesis.+                if (end == next_delimiter && record_parens != 0) {+                    next_delimiter = line.find(delimiter, end);+                }+            }++            // Handle the end-of-the-line case where parenthesis are unbalanced.+            if (record_parens != 0) {+                std::stringstream errorMessage;+                errorMessage << "Unbalanced record parenthesis " << lineNumber << "; ";+                throw std::invalid_argument(errorMessage.str());+            }+        } else {+            end = std::min(line.find(delimiter, start), line.length());+        }++        // Check for missing value.+        if (start > end) {+            std::stringstream errorMessage;+            errorMessage << "Values missing in line " << lineNumber << "; ";+            throw std::invalid_argument(errorMessage.str());+        }++        element = line.substr(start, end - start);+        start = end + delimiter.size();++        return element;+    }++    std::map<int, int> getInputColumnMap(+            const std::map<std::string, std::string>& rwOperation, const unsigned arity_) const {+        std::string columnString = getOr(rwOperation, "columns", "");+        std::map<int, int> inputColumnMap;++        if (!columnString.empty()) {+            std::istringstream iss(columnString);+            std::string mapping;+            int index = 0;+            while (std::getline(iss, mapping, ':')) {+                inputColumnMap[stoi(mapping)] = index++;+            }+            if (inputColumnMap.size() < arity_) {+                throw std::invalid_argument("Invalid column set was given: <" + columnString + ">");+            }+        } else {+            while (inputColumnMap.size() < arity_) {+                int size = static_cast<int>(inputColumnMap.size());+                inputColumnMap[size] = size;+            }+        }+        return inputColumnMap;+    }++    const std::string delimiter;+    std::istream& file;+    size_t lineNumber;+    std::map<int, int> inputMap;+};++class ReadFileCSV : public ReadStreamCSV {+public:+    ReadFileCSV(const std::map<std::string, std::string>& rwOperation, SymbolTable& symbolTable,+            RecordTable& recordTable)+            : ReadStreamCSV(fileHandle, rwOperation, symbolTable, recordTable),+              baseName(souffle::baseName(getFileName(rwOperation))),+              fileHandle(getFileName(rwOperation), std::ios::in | std::ios::binary) {+        if (!fileHandle.is_open()) {+            throw std::invalid_argument("Cannot open fact file " + baseName + "\n");+        }+        // Strip headers if we're using them+        if (getOr(rwOperation, "headers", "false") == "true") {+            std::string line;+            getline(file, line);+        }+    }++    /**+     * Read and return the next tuple.+     *+     * Returns nullptr if no tuple was readable.+     * @return+     */+    Own<RamDomain[]> readNextTuple() override {+        try {+            return ReadStreamCSV::readNextTuple();+        } catch (std::exception& e) {+            std::stringstream errorMessage;+            errorMessage << e.what();+            errorMessage << "cannot parse fact file " << baseName << "!\n";+            throw std::invalid_argument(errorMessage.str());+        }+    }++    ~ReadFileCSV() override = default;++protected:+    /**+     * Return given filename or construct from relation name.+     * Default name is [configured path]/[relation name].facts+     *+     * @param rwOperation map of IO configuration options+     * @return input filename+     */+    static std::string getFileName(const std::map<std::string, std::string>& rwOperation) {+        auto name = getOr(rwOperation, "filename", rwOperation.at("name") + ".facts");+        if (name.front() != '/') {+            name = getOr(rwOperation, "fact-dir", ".") + "/" + name;+        }+        return name;+    }++    std::string baseName;+#ifdef USE_LIBZ+    gzfstream::igzfstream fileHandle;+#else+    std::ifstream fileHandle;+#endif+};++class ReadCinCSVFactory : public ReadStreamFactory {+public:+    Own<ReadStream> getReader(const std::map<std::string, std::string>& rwOperation, SymbolTable& symbolTable,+            RecordTable& recordTable) override {+        return mk<ReadStreamCSV>(std::cin, rwOperation, symbolTable, recordTable);+    }++    const std::string& getName() const override {+        static const std::string name = "stdin";+        return name;+    }+    ~ReadCinCSVFactory() override = default;+};++class ReadFileCSVFactory : public ReadStreamFactory {+public:+    Own<ReadStream> getReader(const std::map<std::string, std::string>& rwOperation, SymbolTable& symbolTable,+            RecordTable& recordTable) override {+        return mk<ReadFileCSV>(rwOperation, symbolTable, recordTable);+    }++    const std::string& getName() const override {+        static const std::string name = "file";+        return name;+    }++    ~ReadFileCSVFactory() override = default;+};++} /* namespace souffle */
+ cbits/souffle/io/ReadStreamJSON.h view
@@ -0,0 +1,368 @@+/*+ * Souffle - A Datalog Compiler+ * Copyright (c) 2020, The Souffle Developers. All rights reserved+ * Licensed under the Universal Permissive License v 1.0 as shown at:+ * - https://opensource.org/licenses/UPL+ * - <souffle root>/licenses/SOUFFLE-UPL.txt+ */++/************************************************************************+ *+ * @file ReadStreamJSON.h+ *+ ***********************************************************************/++#pragma once++#include "souffle/RamTypes.h"+#include "souffle/SymbolTable.h"+#include "souffle/io/ReadStream.h"+#include "souffle/utility/ContainerUtil.h"+#include "souffle/utility/FileUtil.h"+#include "souffle/utility/StringUtil.h"++#include <algorithm>+#include <cassert>+#include <cstddef>+#include <cstdint>+#include <fstream>+#include <iostream>+#include <map>+#include <memory>+#include <queue>+#include <sstream>+#include <stdexcept>+#include <string>+#include <tuple>+#include <vector>++namespace souffle {+class RecordTable;++class ReadStreamJSON : public ReadStream {+public:+    ReadStreamJSON(std::istream& file, const std::map<std::string, std::string>& rwOperation,+            SymbolTable& symbolTable, RecordTable& recordTable)+            : ReadStream(rwOperation, symbolTable, recordTable), file(file), pos(0), isInitialized(false) {+        std::string err;+        params = Json::parse(rwOperation.at("params"), err);+        if (err.length() > 0) {+            fatal("cannot get internal params: %s", err);+        }+    }++protected:+    std::istream& file;+    size_t pos;+    Json jsonSource;+    Json params;+    bool isInitialized;+    bool useObjects;+    std::map<const std::string, const size_t> paramIndex;++    Own<RamDomain[]> readNextTuple() override {+        // for some reasons we cannot initalized our json objects in constructor+        // otherwise it will segfault, so we initialize in the first call+        if (!isInitialized) {+            isInitialized = true;+            std::string error = "";+            std::string source(std::istreambuf_iterator<char>(file), {});++            jsonSource = Json::parse(source, error);+            // it should be wrapped by an extra array+            if (error.length() > 0 || !jsonSource.is_array()) {+                fatal("cannot deserialize json because %s:\n%s", error, source);+            }++            // we only check the first one, since there are extra checks+            // in readNextTupleObject/readNextTupleList+            if (jsonSource[0].is_array()) {+                useObjects = false;+            } else if (jsonSource[0].is_object()) {+                useObjects = true;+                size_t index_pos = 0;+                for (auto param : params["relation"]["params"].array_items()) {+                    paramIndex.insert(std::make_pair(param.string_value(), index_pos));+                    index_pos++;+                }+            } else {+                fatal("the input is neither list nor object format");+            }+        }++        if (useObjects) {+            return readNextTupleObject();+        } else {+            return readNextTupleList();+        }+    }++    Own<RamDomain[]> readNextTupleList() {+        if (pos >= jsonSource.array_items().size()) {+            return nullptr;+        }++        Own<RamDomain[]> tuple = std::make_unique<RamDomain[]>(typeAttributes.size());+        const Json& jsonObj = jsonSource[pos];+        assert(jsonObj.is_array() && "the input is not json array");+        pos++;+        for (size_t i = 0; i < typeAttributes.size(); ++i) {+            try {+                auto&& ty = typeAttributes.at(i);+                switch (ty[0]) {+                    case 's': {+                        tuple[i] = symbolTable.unsafeLookup(jsonObj[i].string_value());+                        break;+                    }+                    case 'r': {+                        tuple[i] = readNextElementList(jsonObj[i], ty);+                        break;+                    }+                    case 'i': {+                        tuple[i] = jsonObj[i].int_value();+                        break;+                    }+                    case 'u': {+                        tuple[i] = jsonObj[i].int_value();+                        break;+                    }+                    case 'f': {+                        tuple[i] = static_cast<RamDomain>(jsonObj[i].number_value());+                        break;+                    }+                    default: fatal("invalid type attribute: `%c`", ty[0]);+                }+            } catch (...) {+                std::stringstream errorMessage;+                if (jsonObj.is_array() && i < jsonObj.array_items().size()) {+                    errorMessage << "Error converting: " << jsonObj[i].dump();+                } else {+                    errorMessage << "Invalid index: " << i;+                }+                throw std::invalid_argument(errorMessage.str());+            }+        }++        return tuple;+    }++    RamDomain readNextElementList(const Json& source, const std::string& recordTypeName) {+        auto&& recordInfo = types["records"][recordTypeName];++        if (recordInfo.is_null()) {+            throw std::invalid_argument("Missing record type information: " + recordTypeName);+        }++        // Handle null case+        if (source.is_null()) {+            return 0;+        }++        assert(source.is_array() && "the input is not json array");+        auto&& recordTypes = recordInfo["types"];+        const size_t recordArity = recordInfo["arity"].long_value();+        std::vector<RamDomain> recordValues(recordArity);+        for (size_t i = 0; i < recordArity; ++i) {+            const std::string& recordType = recordTypes[i].string_value();+            switch (recordType[0]) {+                case 's': {+                    recordValues[i] = symbolTable.unsafeLookup(source[i].string_value());+                    break;+                }+                case 'r': {+                    recordValues[i] = readNextElementList(source[i], recordType);+                    break;+                }+                case 'i': {+                    recordValues[i] = source[i].int_value();+                    break;+                }+                case 'u': {+                    recordValues[i] = source[i].int_value();+                    break;+                }+                case 'f': {+                    recordValues[i] = static_cast<RamDomain>(source[i].number_value());+                    break;+                }+                default: fatal("invalid type attribute");+            }+        }++        return recordTable.pack(recordValues.data(), recordValues.size());+    }++    Own<RamDomain[]> readNextTupleObject() {+        if (pos >= jsonSource.array_items().size()) {+            return nullptr;+        }++        Own<RamDomain[]> tuple = std::make_unique<RamDomain[]>(typeAttributes.size());+        const Json& jsonObj = jsonSource[pos];+        assert(jsonObj.is_object() && "the input is not json object");+        pos++;+        for (auto p : jsonObj.object_items()) {+            try {+                // get the corresponding position by parameter name+                if (paramIndex.find(p.first) == paramIndex.end()) {+                    fatal("invalid parameter: %s", p.first);+                }+                size_t i = paramIndex.at(p.first);+                auto&& ty = typeAttributes.at(i);+                switch (ty[0]) {+                    case 's': {+                        tuple[i] = symbolTable.unsafeLookup(p.second.string_value());+                        break;+                    }+                    case 'r': {+                        tuple[i] = readNextElementObject(p.second, ty);+                        break;+                    }+                    case 'i': {+                        tuple[i] = p.second.int_value();+                        break;+                    }+                    case 'u': {+                        tuple[i] = p.second.int_value();+                        break;+                    }+                    case 'f': {+                        tuple[i] = static_cast<RamDomain>(p.second.number_value());+                        break;+                    }+                    default: fatal("invalid type attribute: `%c`", ty[0]);+                }+            } catch (...) {+                std::stringstream errorMessage;+                errorMessage << "Error converting: " << p.second.dump();+                throw std::invalid_argument(errorMessage.str());+            }+        }++        return tuple;+    }++    RamDomain readNextElementObject(const Json& source, const std::string& recordTypeName) {+        auto&& recordInfo = types["records"][recordTypeName];+        const std::string recordName = recordTypeName.substr(2);+        std::map<const std::string, const size_t> recordIndex;++        size_t index_pos = 0;+        for (auto param : params["records"][recordName]["params"].array_items()) {+            recordIndex.insert(std::make_pair(param.string_value(), index_pos));+            index_pos++;+        }++        if (recordInfo.is_null()) {+            throw std::invalid_argument("Missing record type information: " + recordTypeName);+        }++        // Handle null case+        if (source.is_null()) {+            return 0;+        }++        assert(source.is_object() && "the input is not json object");+        auto&& recordTypes = recordInfo["types"];+        const size_t recordArity = recordInfo["arity"].long_value();+        std::vector<RamDomain> recordValues(recordArity);+        recordValues.reserve(recordIndex.size());+        for (auto readParam : source.object_items()) {+            // get the corresponding position by parameter name+            if (recordIndex.find(readParam.first) == recordIndex.end()) {+                fatal("invalid parameter: %s", readParam.first);+            }+            size_t i = recordIndex.at(readParam.first);+            auto&& type = recordTypes[i].string_value();+            switch (type[0]) {+                case 's': {+                    recordValues[i] = symbolTable.unsafeLookup(readParam.second.string_value());+                    break;+                }+                case 'r': {+                    recordValues[i] = readNextElementObject(readParam.second, type);+                    break;+                }+                case 'i': {+                    recordValues[i] = readParam.second.int_value();+                    break;+                }+                case 'u': {+                    recordValues[i] = readParam.second.int_value();+                    break;+                }+                case 'f': {+                    recordValues[i] = static_cast<RamDomain>(readParam.second.number_value());+                    break;+                }+                default: fatal("invalid type attribute: `%c`", type[0]);+            }+        }++        return recordTable.pack(recordValues.data(), recordValues.size());+    }+};++class ReadFileJSON : public ReadStreamJSON {+public:+    ReadFileJSON(const std::map<std::string, std::string>& rwOperation, SymbolTable& symbolTable,+            RecordTable& recordTable)+            : ReadStreamJSON(fileHandle, rwOperation, symbolTable, recordTable),+              baseName(souffle::baseName(getFileName(rwOperation))),+              fileHandle(getFileName(rwOperation), std::ios::in | std::ios::binary) {+        if (!fileHandle.is_open()) {+            throw std::invalid_argument("Cannot open json file " + baseName + "\n");+        }+    }++    ~ReadFileJSON() override = default;++protected:+    /**+     * Return given filename or construct from relation name.+     * Default name is [configured path]/[relation name].json+     *+     * @param rwOperation map of IO configuration options+     * @return input filename+     */+    static std::string getFileName(const std::map<std::string, std::string>& rwOperation) {+        auto name = getOr(rwOperation, "filename", rwOperation.at("name") + ".json");+        if (name.front() != '/') {+            name = getOr(rwOperation, "fact-dir", ".") + "/" + name;+        }+        return name;+    }++    std::string baseName;+    std::ifstream fileHandle;+};++class ReadCinJSONFactory : public ReadStreamFactory {+public:+    Own<ReadStream> getReader(const std::map<std::string, std::string>& rwOperation, SymbolTable& symbolTable,+            RecordTable& recordTable) override {+        return mk<ReadStreamJSON>(std::cin, rwOperation, symbolTable, recordTable);+    }++    const std::string& getName() const override {+        static const std::string name = "json";+        return name;+    }+    ~ReadCinJSONFactory() override = default;+};++class ReadFileJSONFactory : public ReadStreamFactory {+public:+    Own<ReadStream> getReader(const std::map<std::string, std::string>& rwOperation, SymbolTable& symbolTable,+            RecordTable& recordTable) override {+        return mk<ReadFileJSON>(rwOperation, symbolTable, recordTable);+    }++    const std::string& getName() const override {+        static const std::string name = "jsonfile";+        return name;+    }++    ~ReadFileJSONFactory() override = default;+};+}  // namespace souffle
+ cbits/souffle/io/ReadStreamSQLite.h view
@@ -0,0 +1,195 @@+/*+ * Souffle - A Datalog Compiler+ * Copyright (c) 2013, 2014, Oracle and/or its affiliates. All rights reserved+ * Licensed under the Universal Permissive License v 1.0 as shown at:+ * - https://opensource.org/licenses/UPL+ * - <souffle root>/licenses/SOUFFLE-UPL.txt+ */++/************************************************************************+ *+ * @file ReadStreamSQLite.h+ *+ ***********************************************************************/++#pragma once++#include "souffle/RamTypes.h"+#include "souffle/SymbolTable.h"+#include "souffle/io/ReadStream.h"+#include "souffle/utility/MiscUtil.h"+#include "souffle/utility/StringUtil.h"+#include <cassert>+#include <cstdint>+#include <fstream>+#include <map>+#include <memory>+#include <stdexcept>+#include <string>+#include <vector>+#include <sqlite3.h>++namespace souffle {+class RecordTable;++class ReadStreamSQLite : public ReadStream {+public:+    ReadStreamSQLite(const std::map<std::string, std::string>& rwOperation, SymbolTable& symbolTable,+            RecordTable& recordTable)+            : ReadStream(rwOperation, symbolTable, recordTable), dbFilename(getFileName(rwOperation)),+              relationName(rwOperation.at("name")) {+        openDB();+        checkTableExists();+        prepareSelectStatement();+    }++    ~ReadStreamSQLite() override {+        sqlite3_finalize(selectStatement);+        sqlite3_close(db);+    }++protected:+    /**+     * Read and return the next tuple.+     *+     * Returns nullptr if no tuple was readable.+     * @return+     */+    Own<RamDomain[]> readNextTuple() override {+        if (sqlite3_step(selectStatement) != SQLITE_ROW) {+            return nullptr;+        }++        Own<RamDomain[]> tuple = std::make_unique<RamDomain[]>(arity + auxiliaryArity);++        uint32_t column;+        for (column = 0; column < arity; column++) {+            std::string element(reinterpret_cast<const char*>(sqlite3_column_text(selectStatement, column)));++            if (element.empty()) {+                element = "n/a";+            }++            try {+                auto&& ty = typeAttributes.at(column);+                switch (ty[0]) {+                    case 's': tuple[column] = symbolTable.unsafeLookup(element); break;+                    case 'i':+                    case 'u':+                    case 'f':+                    case 'r': tuple[column] = RamSignedFromString(element); break;+                    default: fatal("invalid type attribute: `%c`", ty[0]);+                }+            } catch (...) {+                std::stringstream errorMessage;+                errorMessage << "Error converting number in column " << (column) + 1;+                throw std::invalid_argument(errorMessage.str());+            }+        }++        return tuple;+    }++    void executeSQL(const std::string& sql) {+        assert(db && "Database connection is closed");++        char* errorMessage = nullptr;+        /* Execute SQL statement */+        int rc = sqlite3_exec(db, sql.c_str(), nullptr, nullptr, &errorMessage);+        if (rc != SQLITE_OK) {+            std::stringstream error;+            error << "SQLite error in sqlite3_exec: " << sqlite3_errmsg(db) << "\n";+            error << "SQL error: " << errorMessage << "\n";+            error << "SQL: " << sql << "\n";+            sqlite3_free(errorMessage);+            throw std::invalid_argument(error.str());+        }+    }++    void throwError(const std::string& message) {+        std::stringstream error;+        error << message << sqlite3_errmsg(db) << "\n";+        throw std::invalid_argument(error.str());+    }++    void prepareSelectStatement() {+        std::stringstream selectSQL;+        selectSQL << "SELECT * FROM '" << relationName << "'";+        const char* tail = nullptr;+        if (sqlite3_prepare_v2(db, selectSQL.str().c_str(), -1, &selectStatement, &tail) != SQLITE_OK) {+            throwError("SQLite error in sqlite3_prepare_v2: ");+        }+    }++    void openDB() {+        if (sqlite3_open(dbFilename.c_str(), &db) != SQLITE_OK) {+            throwError("SQLite error in sqlite3_open: ");+        }+        sqlite3_extended_result_codes(db, 1);+        executeSQL("PRAGMA synchronous = OFF");+        executeSQL("PRAGMA journal_mode = MEMORY");+    }++    void checkTableExists() {+        sqlite3_stmt* tableStatement;+        std::stringstream selectSQL;+        selectSQL << "SELECT count(*) FROM sqlite_master WHERE type IN ('table', 'view') AND ";+        selectSQL << " name = '" << relationName << "';";+        const char* tail = nullptr;++        if (sqlite3_prepare_v2(db, selectSQL.str().c_str(), -1, &tableStatement, &tail) != SQLITE_OK) {+            throwError("SQLite error in sqlite3_prepare_v2: ");+        }++        if (sqlite3_step(tableStatement) == SQLITE_ROW) {+            int count = sqlite3_column_int(tableStatement, 0);+            if (count > 0) {+                sqlite3_finalize(tableStatement);+                return;+            }+        }+        sqlite3_finalize(tableStatement);+        throw std::invalid_argument(+                "Required table or view does not exist in " + dbFilename + " for relation " + relationName);+    }++    /**+     * Return given filename or construct from relation name.+     * Default name is [configured path]/[relation name].sqlite+     *+     * @param rwOperation map of IO configuration options+     * @return input filename+     */+    static std::string getFileName(const std::map<std::string, std::string>& rwOperation) {+        // legacy support for SQLite prior to 2020-03-18+        // convert dbname to filename+        auto name = getOr(rwOperation, "dbname", rwOperation.at("name") + ".sqlite");+        name = getOr(rwOperation, "filename", name);++        if (name.front() != '/') {+            name = getOr(rwOperation, "fact-dir", ".") + "/" + name;+        }+        return name;+    }++    const std::string dbFilename;+    const std::string relationName;+    sqlite3_stmt* selectStatement = nullptr;+    sqlite3* db = nullptr;+};++class ReadSQLiteFactory : public ReadStreamFactory {+public:+    Own<ReadStream> getReader(const std::map<std::string, std::string>& rwOperation, SymbolTable& symbolTable,+            RecordTable& recordTable) override {+        return mk<ReadStreamSQLite>(rwOperation, symbolTable, recordTable);+    }++    const std::string& getName() const override {+        static const std::string name = "sqlite";+        return name;+    }+    ~ReadSQLiteFactory() override = default;+};++} /* namespace souffle */
+ cbits/souffle/io/SerialisationStream.h view
@@ -0,0 +1,91 @@+/*+ * Souffle - A Datalog Compiler+ * Copyright (c) 2020, The Souffle Developers. All rights reserved.+ * Licensed under the Universal Permissive License v 1.0 as shown at:+ * - https://opensource.org/licenses/UPL+ * - <souffle root>/licenses/SOUFFLE-UPL.txt+ */++/************************************************************************+ *+ * @file SerialisationStream.h+ *+ * Defines a common base class for relation serialisation streams.+ *+ ***********************************************************************/++#pragma once++#include "souffle/RamTypes.h"++#include "souffle/utility/json11.h"+#include <cassert>+#include <cstddef>+#include <map>+#include <string>+#include <utility>+#include <vector>++namespace souffle {++class RecordTable;+class SymbolTable;++using json11::Json;++template <bool readOnlyTables>+class SerialisationStream {+public:+    virtual ~SerialisationStream() = default;++protected:+    template <typename A>+    using RO = std::conditional_t<readOnlyTables, const A, A>;++    SerialisationStream(RO<SymbolTable>& symTab, RO<RecordTable>& recTab, Json types,+            std::vector<std::string> relTypes, size_t auxArity = 0)+            : symbolTable(symTab), recordTable(recTab), types(std::move(types)),+              typeAttributes(std::move(relTypes)), arity(typeAttributes.size() - auxArity),+              auxiliaryArity(auxArity) {}++    SerialisationStream(RO<SymbolTable>& symTab, RO<RecordTable>& recTab, Json types)+            : symbolTable(symTab), recordTable(recTab), types(std::move(types)) {+        setupFromJson();+    }++    SerialisationStream(RO<SymbolTable>& symTab, RO<RecordTable>& recTab,+            const std::map<std::string, std::string>& rwOperation)+            : symbolTable(symTab), recordTable(recTab) {+        std::string parseErrors;+        types = Json::parse(rwOperation.at("types"), parseErrors);+        assert(parseErrors.size() == 0 && "Internal JSON parsing failed.");+        setupFromJson();+    }++    RO<SymbolTable>& symbolTable;+    RO<RecordTable>& recordTable;+    Json types;+    std::vector<std::string> typeAttributes;++    size_t arity = 0;+    size_t auxiliaryArity = 0;++private:+    void setupFromJson() {+        auto&& relInfo = types["relation"];+        arity = static_cast<size_t>(relInfo["arity"].long_value());+        auxiliaryArity = static_cast<size_t>(relInfo["auxArity"].long_value());++        assert(relInfo["types"].is_array());+        auto&& relTypes = relInfo["types"].array_items();+        assert(relTypes.size() == (arity + auxiliaryArity));++        for (size_t i = 0; i < arity + auxiliaryArity; ++i) {+            auto&& type = relTypes[i].string_value();+            assert(!type.empty() && "malformed types tag");+            typeAttributes.push_back(type);+        }+    }+};++}  // namespace souffle
+ cbits/souffle/io/WriteStream.h view
@@ -0,0 +1,190 @@+/*+ * Souffle - A Datalog Compiler+ * Copyright (c) 2013, 2014, Oracle and/or its affiliates. All rights reserved+ * Licensed under the Universal Permissive License v 1.0 as shown at:+ * - https://opensource.org/licenses/UPL+ * - <souffle root>/licenses/SOUFFLE-UPL.txt+ */++/************************************************************************+ *+ * @file WriteStream.h+ *+ ***********************************************************************/++#pragma once++#include "souffle/RamTypes.h"+#include "souffle/RecordTable.h"+#include "souffle/SymbolTable.h"+#include "souffle/io/SerialisationStream.h"+#include "souffle/utility/MiscUtil.h"+#include "souffle/utility/json11.h"+#include <cassert>+#include <cstddef>+#include <map>+#include <memory>+#include <ostream>+#include <string>++namespace souffle {++using json11::Json;++class WriteStream : public SerialisationStream<true> {+public:+    WriteStream(const std::map<std::string, std::string>& rwOperation, const SymbolTable& symbolTable,+            const RecordTable& recordTable)+            : SerialisationStream(symbolTable, recordTable, rwOperation),+              summary(rwOperation.at("IO") == "stdoutprintsize") {}++    template <typename T>+    void writeAll(const T& relation) {+        if (summary) {+            return writeSize(relation.size());+        }+        auto lease = symbolTable.acquireLock();+        (void)lease;  // silence "unused variable" warning+        if (arity == 0) {+            if (relation.begin() != relation.end()) {+                writeNullary();+            }+            return;+        }+        for (const auto& current : relation) {+            writeNext(current);+        }+    }++    template <typename T>+    void writeSize(const T& relation) {+        writeSize(relation.size());+    }++protected:+    const bool summary;++    virtual void writeNullary() = 0;+    virtual void writeNextTuple(const RamDomain* tuple) = 0;+    virtual void writeSize(std::size_t) {+        fatal("attempting to print size of a write operation");+    }++    template <typename Tuple>+    void writeNext(const Tuple tuple) {+        writeNextTuple(tuple.data);+    }++    void outputRecord(std::ostream& destination, const RamDomain value, const std::string& name) {+        auto&& recordInfo = types["records"][name];++        // Check if record type information are present+        assert(!recordInfo.is_null() && "Missing record type information");++        // Check for nil+        if (value == 0) {+            destination << "nil";+            return;+        }++        auto&& recordTypes = recordInfo["types"];+        const size_t recordArity = recordInfo["arity"].long_value();++        const RamDomain* tuplePtr = recordTable.unpack(value, recordArity);++        destination << "[";++        // print record's elements+        for (size_t i = 0; i < recordArity; ++i) {+            if (i > 0) {+                destination << ", ";+            }++            const std::string& recordType = recordTypes[i].string_value();+            const RamDomain recordValue = tuplePtr[i];++            switch (recordType[0]) {+                case 'i': destination << recordValue; break;+                case 'f': destination << ramBitCast<RamFloat>(recordValue); break;+                case 'u': destination << ramBitCast<RamUnsigned>(recordValue); break;+                case 's': destination << symbolTable.unsafeResolve(recordValue); break;+                case 'r': outputRecord(destination, recordValue, recordType); break;+                case '+': outputADT(destination, recordValue, recordType); break;+                default: fatal("Unsupported type attribute: `%c`", recordType[0]);+            }+        }+        destination << "]";+    }++    void outputADT(std::ostream& destination, const RamDomain value, const std::string& name) {+        auto&& adtInfo = types["ADTs"][name];++        assert(!adtInfo.is_null() && "Missing adt type information");++        const size_t numBranches = adtInfo["arity"].long_value();+        assert(numBranches > 0);++        // adt is encoded as [branchID, [branch_args]] when |branch_args| != 1+        // and as [branchID, arg] when a branch takes a single argument.+        const RamDomain* tuplePtr = recordTable.unpack(value, 2);++        const RamDomain branchId = tuplePtr[0];+        const RamDomain rawBranchArgs = tuplePtr[1];++        auto branchInfo = adtInfo["branches"][branchId];+        auto branchTypes = branchInfo["types"].array_items();++        // Prepare branch's arguments for output.+        const RamDomain* branchArgs = [&]() -> const RamDomain* {+            if (branchTypes.size() > 1) {+                return recordTable.unpack(rawBranchArgs, branchTypes.size());+            } else {+                return &rawBranchArgs;+            }+        }();++        destination << "$" << branchInfo["name"].string_value();++        if (branchTypes.size() > 0) {+            destination << "(";+        }++        // Print arguments+        for (size_t i = 0; i < branchTypes.size(); ++i) {+            if (i > 0) {+                destination << ", ";+            }++            auto argType = branchTypes[i].string_value();+            switch (argType[0]) {+                case 'i': destination << branchArgs[i]; break;+                case 'f': destination << ramBitCast<RamFloat>(branchArgs[i]); break;+                case 'u': destination << ramBitCast<RamUnsigned>(branchArgs[i]); break;+                case 's': destination << symbolTable.unsafeResolve(branchArgs[i]); break;+                case 'r': outputRecord(destination, branchArgs[i], argType); break;+                case '+': outputADT(destination, branchArgs[i], argType); break;+                default: fatal("Unsupported type attribute: `%c`", argType[0]);+            }+        }++        if (branchTypes.size() > 0) {+            destination << ")";+        }+    }+};++class WriteStreamFactory {+public:+    virtual Own<WriteStream> getWriter(const std::map<std::string, std::string>& rwOperation,+            const SymbolTable& symbolTable, const RecordTable& recordTable) = 0;++    virtual const std::string& getName() const = 0;+    virtual ~WriteStreamFactory() = default;+};++template <>+inline void WriteStream::writeNext(const RamDomain* tuple) {+    writeNextTuple(tuple);+}++} /* namespace souffle */
+ cbits/souffle/io/WriteStreamCSV.h view
@@ -0,0 +1,254 @@+/*+ * Souffle - A Datalog Compiler+ * Copyright (c) 2013, 2014, Oracle and/or its affiliates. All rights reserved+ * Licensed under the Universal Permissive License v 1.0 as shown at:+ * - https://opensource.org/licenses/UPL+ * - <souffle root>/licenses/SOUFFLE-UPL.txt+ */++/************************************************************************+ *+ * @file WriteStreamCSV.h+ *+ ***********************************************************************/++#pragma once++#include "souffle/RamTypes.h"+#include "souffle/SymbolTable.h"+#include "souffle/io/WriteStream.h"+#include "souffle/utility/ContainerUtil.h"+#include "souffle/utility/MiscUtil.h"+#include "souffle/utility/ParallelUtil.h"+#ifdef USE_LIBZ+#include "souffle/io/gzfstream.h"+#endif++#include <cstddef>+#include <fstream>+#include <iomanip>+#include <iostream>+#include <map>+#include <ostream>+#include <string>+#include <vector>++namespace souffle {++class RecordTable;++class WriteStreamCSV : public WriteStream {+protected:+    WriteStreamCSV(const std::map<std::string, std::string>& rwOperation, const SymbolTable& symbolTable,+            const RecordTable& recordTable)+            : WriteStream(rwOperation, symbolTable, recordTable),+              delimiter(getOr(rwOperation, "delimiter", "\t")){};++    const std::string delimiter;++    void writeNextTupleCSV(std::ostream& destination, const RamDomain* tuple) {+        writeNextTupleElement(destination, typeAttributes.at(0), tuple[0]);++        for (size_t col = 1; col < arity; ++col) {+            destination << delimiter;+            writeNextTupleElement(destination, typeAttributes.at(col), tuple[col]);+        }++        destination << "\n";+    }++    void writeNextTupleElement(std::ostream& destination, const std::string& type, RamDomain value) {+        switch (type[0]) {+            case 's': destination << symbolTable.unsafeResolve(value); break;+            case 'i': destination << value; break;+            case 'u': destination << ramBitCast<RamUnsigned>(value); break;+            case 'f': destination << ramBitCast<RamFloat>(value); break;+            case 'r': outputRecord(destination, value, type); break;+            case '+': outputADT(destination, value, type); break;+            default: fatal("unsupported type attribute: `%c`", type[0]);+        }+    }+};++class WriteFileCSV : public WriteStreamCSV {+public:+    WriteFileCSV(const std::map<std::string, std::string>& rwOperation, const SymbolTable& symbolTable,+            const RecordTable& recordTable)+            : WriteStreamCSV(rwOperation, symbolTable, recordTable),+              file(getFileName(rwOperation), std::ios::out | std::ios::binary) {+        if (getOr(rwOperation, "headers", "false") == "true") {+            file << rwOperation.at("attributeNames") << std::endl;+        }+        file << std::setprecision(std::numeric_limits<RamFloat>::max_digits10);+    }++    ~WriteFileCSV() override = default;++protected:+    std::ofstream file;++    void writeNullary() override {+        file << "()\n";+    }++    void writeNextTuple(const RamDomain* tuple) override {+        writeNextTupleCSV(file, tuple);+    }++    /**+     * Return given filename or construct from relation name.+     * Default name is [configured path]/[relation name].csv+     *+     * @param rwOperation map of IO configuration options+     * @return input filename+     */+    static std::string getFileName(const std::map<std::string, std::string>& rwOperation) {+        auto name = getOr(rwOperation, "filename", rwOperation.at("name") + ".csv");+        if (name.front() != '/') {+            name = getOr(rwOperation, "output-dir", ".") + "/" + name;+        }+        return name;+    }+};++#ifdef USE_LIBZ+class WriteGZipFileCSV : public WriteStreamCSV {+public:+    WriteGZipFileCSV(const std::map<std::string, std::string>& rwOperation, const SymbolTable& symbolTable,+            const RecordTable& recordTable)+            : WriteStreamCSV(rwOperation, symbolTable, recordTable),+              file(getFileName(rwOperation), std::ios::out | std::ios::binary) {+        if (getOr(rwOperation, "headers", "false") == "true") {+            file << rwOperation.at("attributeNames") << std::endl;+        }+        file << std::setprecision(std::numeric_limits<RamFloat>::max_digits10);+    }++    ~WriteGZipFileCSV() override = default;++protected:+    void writeNullary() override {+        file << "()\n";+    }++    void writeNextTuple(const RamDomain* tuple) override {+        writeNextTupleCSV(file, tuple);+    }++    /**+     * Return given filename or construct from relation name.+     * Default name is [configured path]/[relation name].csv+     *+     * @param rwOperation map of IO configuration options+     * @return input filename+     */+    static std::string getFileName(const std::map<std::string, std::string>& rwOperation) {+        auto name = getOr(rwOperation, "filename", rwOperation.at("name") + ".csv.gz");+        if (name.front() != '/') {+            name = getOr(rwOperation, "output-dir", ".") + "/" + name;+        }+        return name;+    }++    gzfstream::ogzfstream file;+};+#endif++class WriteCoutCSV : public WriteStreamCSV {+public:+    WriteCoutCSV(const std::map<std::string, std::string>& rwOperation, const SymbolTable& symbolTable,+            const RecordTable& recordTable)+            : WriteStreamCSV(rwOperation, symbolTable, recordTable) {+        std::cout << "---------------\n" << rwOperation.at("name");+        if (getOr(rwOperation, "headers", "false") == "true") {+            std::cout << "\n" << rwOperation.at("attributeNames");+        }+        std::cout << "\n===============\n";+        std::cout << std::setprecision(std::numeric_limits<RamFloat>::max_digits10);+    }++    ~WriteCoutCSV() override {+        std::cout << "===============\n";+    }++protected:+    void writeNullary() override {+        std::cout << "()\n";+    }++    void writeNextTuple(const RamDomain* tuple) override {+        writeNextTupleCSV(std::cout, tuple);+    }+};++class WriteCoutPrintSize : public WriteStream {+public:+    explicit WriteCoutPrintSize(const std::map<std::string, std::string>& rwOperation)+            : WriteStream(rwOperation, {}, {}), lease(souffle::getOutputLock().acquire()) {+        std::cout << rwOperation.at("name") << "\t";+    }++    ~WriteCoutPrintSize() override = default;++protected:+    void writeNullary() override {+        fatal("attempting to iterate over a print size operation");+    }++    void writeNextTuple(const RamDomain* /* tuple */) override {+        fatal("attempting to iterate over a print size operation");+    }++    void writeSize(std::size_t size) override {+        std::cout << size << "\n";+    }++    Lock::Lease lease;+};++class WriteFileCSVFactory : public WriteStreamFactory {+public:+    Own<WriteStream> getWriter(const std::map<std::string, std::string>& rwOperation,+            const SymbolTable& symbolTable, const RecordTable& recordTable) override {+#ifdef USE_LIBZ+        if (contains(rwOperation, "compress")) {+            return mk<WriteGZipFileCSV>(rwOperation, symbolTable, recordTable);+        }+#endif+        return mk<WriteFileCSV>(rwOperation, symbolTable, recordTable);+    }+    const std::string& getName() const override {+        static const std::string name = "file";+        return name;+    }+    ~WriteFileCSVFactory() override = default;+};++class WriteCoutCSVFactory : public WriteStreamFactory {+public:+    Own<WriteStream> getWriter(const std::map<std::string, std::string>& rwOperation,+            const SymbolTable& symbolTable, const RecordTable& recordTable) override {+        return mk<WriteCoutCSV>(rwOperation, symbolTable, recordTable);+    }++    const std::string& getName() const override {+        static const std::string name = "stdout";+        return name;+    }+    ~WriteCoutCSVFactory() override = default;+};++class WriteCoutPrintSizeFactory : public WriteStreamFactory {+public:+    Own<WriteStream> getWriter(const std::map<std::string, std::string>& rwOperation, const SymbolTable&,+            const RecordTable&) override {+        return mk<WriteCoutPrintSize>(rwOperation);+    }+    const std::string& getName() const override {+        static const std::string name = "stdoutprintsize";+        return name;+    }+    ~WriteCoutPrintSizeFactory() override = default;+};++} /* namespace souffle */
+ cbits/souffle/io/WriteStreamJSON.h view
@@ -0,0 +1,298 @@+/*+ * Souffle - A Datalog Compiler+ * Copyright (c) 2020, The Souffle Developers. All rights reserved+ * Licensed under the Universal Permissive License v 1.0 as shown at:+ * - https://opensource.org/licenses/UPL+ * - <souffle root>/licenses/SOUFFLE-UPL.txt+ */++/************************************************************************+ *+ * @file WriteStreamJSON.h+ *+ ***********************************************************************/++#pragma once++#include "souffle/RamTypes.h"+#include "souffle/SymbolTable.h"+#include "souffle/io/WriteStream.h"+#include "souffle/utility/ContainerUtil.h"+#include "souffle/utility/json11.h"++#include <map>+#include <ostream>+#include <queue>+#include <stack>+#include <string>+#include <variant>+#include <vector>++namespace souffle {++class WriteStreamJSON : public WriteStream {+protected:+    WriteStreamJSON(const std::map<std::string, std::string>& rwOperation, const SymbolTable& symbolTable,+            const RecordTable& recordTable)+            : WriteStream(rwOperation, symbolTable, recordTable),+              useObjects(getOr(rwOperation, "format", "list") == "object") {+        if (useObjects) {+            std::string err;+            params = Json::parse(rwOperation.at("params"), err);+            if (err.length() > 0) {+                fatal("cannot get internal param names: %s", err);+            }+        }+    };++    const bool useObjects;+    Json params;++    void writeNextTupleJSON(std::ostream& destination, const RamDomain* tuple) {+        std::vector<Json> result;++        if (useObjects)+            destination << "{";+        else+            destination << "[";++        for (size_t col = 0; col < arity; ++col) {+            if (col > 0) {+                destination << ", ";+            }++            if (useObjects) {+                destination << params["relation"]["params"][col].dump() << ": ";+                writeNextTupleObject(destination, typeAttributes.at(col), tuple[col]);+            } else {+                writeNextTupleList(destination, typeAttributes.at(col), tuple[col]);+            }+        }++        if (useObjects)+            destination << "}";+        else+            destination << "]";+    }++    void writeNextTupleList(std::ostream& destination, const std::string& name, const RamDomain value) {+        using ValueTuple = std::pair<const std::string, const RamDomain>;+        std::stack<std::variant<ValueTuple, std::string>> worklist;+        worklist.push(std::make_pair(name, value));++        // the Json11 output is not tail recursive, therefore highly inefficient for recursive record+        // in addition the JSON object is immutable, so has memory overhead+        while (!worklist.empty()) {+            std::variant<ValueTuple, std::string> curr = worklist.top();+            worklist.pop();++            if (std::holds_alternative<std::string>(curr)) {+                destination << std::get<std::string>(curr);+                continue;+            }++            const std::string& currType = std::get<ValueTuple>(curr).first;+            const RamDomain currValue = std::get<ValueTuple>(curr).second;+            assert(currType.length() > 2 && "Invalid type length");+            switch (currType[0]) {+                // since some strings may need to be escaped, we use dump here+                case 's': destination << Json(symbolTable.unsafeResolve(currValue)).dump(); break;+                case 'i': destination << currValue; break;+                case 'u': destination << (int)ramBitCast<RamUnsigned>(currValue); break;+                case 'f': destination << ramBitCast<RamFloat>(currValue); break;+                case 'r': {+                    auto&& recordInfo = types["records"][currType];+                    assert(!recordInfo.is_null() && "Missing record type information");+                    if (currValue == 0) {+                        destination << "null";+                        break;+                    }++                    auto&& recordTypes = recordInfo["types"];+                    const size_t recordArity = recordInfo["arity"].long_value();+                    const RamDomain* tuplePtr = recordTable.unpack(currValue, recordArity);+                    worklist.push("]");+                    for (auto i = (long long)(recordArity - 1); i >= 0; --i) {+                        if (i != (long long)(recordArity - 1)) {+                            worklist.push(", ");+                        }+                        const std::string& recordType = recordTypes[i].string_value();+                        const RamDomain recordValue = tuplePtr[i];+                        worklist.push(std::make_pair(recordType, recordValue));+                    }++                    worklist.push("[");+                    break;+                }+                default: fatal("unsupported type attribute: `%c`", currType[0]);+            }+        }+    }++    void writeNextTupleObject(std::ostream& destination, const std::string& name, const RamDomain value) {+        using ValueTuple = std::pair<const std::string, const RamDomain>;+        std::stack<std::variant<ValueTuple, std::string>> worklist;+        worklist.push(std::make_pair(name, value));++        // the Json11 output is not tail recursive, therefore highly inefficient for recursive record+        // in addition the JSON object is immutable, so has memory overhead+        while (!worklist.empty()) {+            std::variant<ValueTuple, std::string> curr = worklist.top();+            worklist.pop();++            if (std::holds_alternative<std::string>(curr)) {+                destination << std::get<std::string>(curr);+                continue;+            }++            const std::string& currType = std::get<ValueTuple>(curr).first;+            const RamDomain currValue = std::get<ValueTuple>(curr).second;+            const std::string& typeName = currType.substr(2);+            assert(currType.length() > 2 && "Invalid type length");+            switch (currType[0]) {+                // since some strings may need to be escaped, we use dump here+                case 's': destination << Json(symbolTable.unsafeResolve(currValue)).dump(); break;+                case 'i': destination << currValue; break;+                case 'u': destination << (int)ramBitCast<RamUnsigned>(currValue); break;+                case 'f': destination << ramBitCast<RamFloat>(currValue); break;+                case 'r': {+                    auto&& recordInfo = types["records"][currType];+                    assert(!recordInfo.is_null() && "Missing record type information");+                    if (currValue == 0) {+                        destination << "null";+                        break;+                    }++                    auto&& recordTypes = recordInfo["types"];+                    const size_t recordArity = recordInfo["arity"].long_value();+                    const RamDomain* tuplePtr = recordTable.unpack(currValue, recordArity);+                    worklist.push("}");+                    for (auto i = (long long)(recordArity - 1); i >= 0; --i) {+                        if (i != (long long)(recordArity - 1)) {+                            worklist.push(", ");+                        }+                        const std::string& recordType = recordTypes[i].string_value();+                        const RamDomain recordValue = tuplePtr[i];+                        worklist.push(std::make_pair(recordType, recordValue));+                        worklist.push(": ");++                        auto&& recordParam = params["records"][typeName]["params"][i];+                        assert(recordParam.is_string());+                        worklist.push(recordParam.dump());+                    }++                    worklist.push("{");+                    break;+                }+                default: fatal("unsupported type attribute: `%c`", currType[0]);+            }+        }+    }+};++class WriteFileJSON : public WriteStreamJSON {+public:+    WriteFileJSON(const std::map<std::string, std::string>& rwOperation, const SymbolTable& symbolTable,+            const RecordTable& recordTable)+            : WriteStreamJSON(rwOperation, symbolTable, recordTable), isFirst(true),+              file(getFileName(rwOperation), std::ios::out | std::ios::binary) {+        file << "[";+    }++    ~WriteFileJSON() override {+        file << "]\n";+        file.close();+    }++protected:+    bool isFirst;+    std::ofstream file;++    void writeNullary() override {+        file << "null\n";+    }++    void writeNextTuple(const RamDomain* tuple) override {+        if (!isFirst) {+            file << ",\n";+        } else {+            isFirst = false;+        }+        writeNextTupleJSON(file, tuple);+    }++    /**+     * Return given filename or construct from relation name.+     * Default name is [configured path]/[relation name].json+     *+     * @param rwOperation map of IO configuration options+     * @return input filename+     */+    static std::string getFileName(const std::map<std::string, std::string>& rwOperation) {+        auto name = getOr(rwOperation, "filename", rwOperation.at("name") + ".json");+        if (name.front() != '/') {+            name = getOr(rwOperation, "output-dir", ".") + "/" + name;+        }+        return name;+    }+};++class WriteCoutJSON : public WriteStreamJSON {+public:+    WriteCoutJSON(const std::map<std::string, std::string>& rwOperation, const SymbolTable& symbolTable,+            const RecordTable& recordTable)+            : WriteStreamJSON(rwOperation, symbolTable, recordTable), isFirst(true) {+        std::cout << "[";+    }++    ~WriteCoutJSON() override {+        std::cout << "]\n";+    };++protected:+    bool isFirst;++    void writeNullary() override {+        std::cout << "null\n";+    }++    void writeNextTuple(const RamDomain* tuple) override {+        if (!isFirst) {+            std::cout << ",\n";+        } else {+            isFirst = false;+        }+        writeNextTupleJSON(std::cout, tuple);+    }+};++class WriteFileJSONFactory : public WriteStreamFactory {+public:+    Own<WriteStream> getWriter(const std::map<std::string, std::string>& rwOperation,+            const SymbolTable& symbolTable, const RecordTable& recordTable) override {+        return mk<WriteFileJSON>(rwOperation, symbolTable, recordTable);+    }++    const std::string& getName() const override {+        static const std::string name = "jsonfile";+        return name;+    }++    ~WriteFileJSONFactory() override = default;+};++class WriteCoutJSONFactory : public WriteStreamFactory {+public:+    Own<WriteStream> getWriter(const std::map<std::string, std::string>& rwOperation,+            const SymbolTable& symbolTable, const RecordTable& recordTable) override {+        return mk<WriteCoutJSON>(rwOperation, symbolTable, recordTable);+    }++    const std::string& getName() const override {+        static const std::string name = "json";+        return name;+    }++    ~WriteCoutJSONFactory() override = default;+};+}  // namespace souffle
+ cbits/souffle/io/WriteStreamSQLite.h view
@@ -0,0 +1,297 @@+/*+ * Souffle - A Datalog Compiler+ * Copyright (c) 2013, 2014, Oracle and/or its affiliates. All rights reserved+ * Licensed under the Universal Permissive License v 1.0 as shown at:+ * - https://opensource.org/licenses/UPL+ * - <souffle root>/licenses/SOUFFLE-UPL.txt+ */++/************************************************************************+ *+ * @file WriteStreamSQLite.h+ *+ ***********************************************************************/++#pragma once++#include "souffle/RamTypes.h"+#include "souffle/SymbolTable.h"+#include "souffle/io/WriteStream.h"+#include <cassert>+#include <cstddef>+#include <cstdint>+#include <map>+#include <memory>+#include <sstream>+#include <stdexcept>+#include <string>+#include <unordered_map>+#include <vector>+#include <sqlite3.h>++namespace souffle {++class RecordTable;++class WriteStreamSQLite : public WriteStream {+public:+    WriteStreamSQLite(const std::map<std::string, std::string>& rwOperation, const SymbolTable& symbolTable,+            const RecordTable& recordTable)+            : WriteStream(rwOperation, symbolTable, recordTable), dbFilename(getFileName(rwOperation)),+              relationName(rwOperation.at("name")) {+        openDB();+        createTables();+        prepareStatements();+        //        executeSQL("BEGIN TRANSACTION", db);+    }++    ~WriteStreamSQLite() override {+        sqlite3_finalize(insertStatement);+        sqlite3_finalize(symbolInsertStatement);+        sqlite3_finalize(symbolSelectStatement);+        sqlite3_close(db);+    }++protected:+    void writeNullary() override {}++    void writeNextTuple(const RamDomain* tuple) override {+        for (size_t i = 0; i < arity; i++) {+            RamDomain value = 0;  // Silence warning++            switch (typeAttributes.at(i)[0]) {+                case 's': value = getSymbolTableID(tuple[i]); break;+                default: value = tuple[i]; break;+            }++#if RAM_DOMAIN_SIZE == 64+            if (sqlite3_bind_int64(insertStatement, i + 1, value) != SQLITE_OK) {+#else+            if (sqlite3_bind_int(insertStatement, i + 1, value) != SQLITE_OK) {+#endif+                throwError("SQLite error in sqlite3_bind_text: ");+            }+        }+        if (sqlite3_step(insertStatement) != SQLITE_DONE) {+            throwError("SQLite error in sqlite3_step: ");+        }+        sqlite3_clear_bindings(insertStatement);+        sqlite3_reset(insertStatement);+    }++private:+    void executeSQL(const std::string& sql, sqlite3* db) {+        assert(db && "Database connection is closed");++        char* errorMessage = nullptr;+        /* Execute SQL statement */+        int rc = sqlite3_exec(db, sql.c_str(), nullptr, nullptr, &errorMessage);+        if (rc != SQLITE_OK) {+            std::stringstream error;+            error << "SQLite error in sqlite3_exec: " << sqlite3_errmsg(db) << "\n";+            error << "SQL error: " << errorMessage << "\n";+            error << "SQL: " << sql << "\n";+            sqlite3_free(errorMessage);+            throw std::invalid_argument(error.str());+        }+    }++    void throwError(const std::string& message) {+        std::stringstream error;+        error << message << sqlite3_errmsg(db) << "\n";+        throw std::invalid_argument(error.str());+    }++    uint64_t getSymbolTableIDFromDB(int index) {+        if (sqlite3_bind_text(symbolSelectStatement, 1, symbolTable.unsafeResolve(index).c_str(), -1,+                    SQLITE_TRANSIENT) != SQLITE_OK) {+            throwError("SQLite error in sqlite3_bind_text: ");+        }+        if (sqlite3_step(symbolSelectStatement) != SQLITE_ROW) {+            throwError("SQLite error in sqlite3_step: ");+        }+        uint64_t rowid = sqlite3_column_int64(symbolSelectStatement, 0);+        sqlite3_clear_bindings(symbolSelectStatement);+        sqlite3_reset(symbolSelectStatement);+        return rowid;+    }+    uint64_t getSymbolTableID(int index) {+        if (dbSymbolTable.count(index) != 0) {+            return dbSymbolTable[index];+        }++        if (sqlite3_bind_text(symbolInsertStatement, 1, symbolTable.unsafeResolve(index).c_str(), -1,+                    SQLITE_TRANSIENT) != SQLITE_OK) {+            throwError("SQLite error in sqlite3_bind_text: ");+        }+        // Either the insert succeeds and we have a new row id or it already exists and a select is needed.+        uint64_t rowid;+        if (sqlite3_step(symbolInsertStatement) != SQLITE_DONE) {+            // The symbol already exists so select it.+            rowid = getSymbolTableIDFromDB(index);+        } else {+            rowid = sqlite3_last_insert_rowid(db);+        }+        sqlite3_clear_bindings(symbolInsertStatement);+        sqlite3_reset(symbolInsertStatement);++        dbSymbolTable[index] = rowid;+        return rowid;+    }++    void openDB() {+        if (sqlite3_open(dbFilename.c_str(), &db) != SQLITE_OK) {+            throwError("SQLite error in sqlite3_open");+        }+        sqlite3_extended_result_codes(db, 1);+        executeSQL("PRAGMA synchronous = OFF", db);+        executeSQL("PRAGMA journal_mode = MEMORY", db);+    }++    void prepareStatements() {+        prepareInsertStatement();+        prepareSymbolInsertStatement();+        prepareSymbolSelectStatement();+    }+    void prepareSymbolInsertStatement() {+        std::stringstream insertSQL;+        insertSQL << "INSERT INTO " << symbolTableName;+        insertSQL << " VALUES(null,@V0);";+        const char* tail = nullptr;+        if (sqlite3_prepare_v2(db, insertSQL.str().c_str(), -1, &symbolInsertStatement, &tail) != SQLITE_OK) {+            throwError("SQLite error in sqlite3_prepare_v2: ");+        }+    }++    void prepareSymbolSelectStatement() {+        std::stringstream selectSQL;+        selectSQL << "SELECT id FROM " << symbolTableName;+        selectSQL << " WHERE symbol = @V0;";+        const char* tail = nullptr;+        if (sqlite3_prepare_v2(db, selectSQL.str().c_str(), -1, &symbolSelectStatement, &tail) != SQLITE_OK) {+            throwError("SQLite error in sqlite3_prepare_v2: ");+        }+    }++    void prepareInsertStatement() {+        std::stringstream insertSQL;+        insertSQL << "INSERT INTO '_" << relationName << "' VALUES ";+        insertSQL << "(@V0";+        for (unsigned int i = 1; i < arity; i++) {+            insertSQL << ",@V" << i;+        }+        insertSQL << ");";+        const char* tail = nullptr;+        if (sqlite3_prepare_v2(db, insertSQL.str().c_str(), -1, &insertStatement, &tail) != SQLITE_OK) {+            throwError("SQLite error in sqlite3_prepare_v2: ");+        }+    }++    void createTables() {+        createRelationTable();+        createRelationView();+        createSymbolTable();+    }++    void createRelationTable() {+        std::stringstream createTableText;+        createTableText << "CREATE TABLE IF NOT EXISTS '_" << relationName << "' (";+        if (arity > 0) {+            createTableText << "'0' INTEGER";+            for (unsigned int i = 1; i < arity; i++) {+                createTableText << ",'" << std::to_string(i) << "' ";+                createTableText << "INTEGER";+            }+        }+        createTableText << ");";+        executeSQL(createTableText.str(), db);+        executeSQL("DELETE FROM '_" + relationName + "';", db);+    }++    void createRelationView() {+        // Create view with symbol strings resolved+        std::stringstream createViewText;+        createViewText << "CREATE VIEW IF NOT EXISTS '" << relationName << "' AS ";+        std::stringstream projectionClause;+        std::stringstream fromClause;+        fromClause << "'_" << relationName << "'";+        std::stringstream whereClause;+        bool firstWhere = true;+        for (unsigned int i = 0; i < arity; i++) {+            std::string columnName = std::to_string(i);+            if (i != 0) {+                projectionClause << ",";+            }+            if (typeAttributes.at(i)[0] == 's') {+                projectionClause << "'_symtab_" << columnName << "'.symbol AS '" << columnName << "'";+                fromClause << ",'" << symbolTableName << "' AS '_symtab_" << columnName << "'";+                if (!firstWhere) {+                    whereClause << " AND ";+                } else {+                    firstWhere = false;+                }+                whereClause << "'_" << relationName << "'.'" << columnName << "' = "+                            << "'_symtab_" << columnName << "'.id";+            } else {+                projectionClause << "'_" << relationName << "'.'" << columnName << "'";+            }+        }+        createViewText << "SELECT " << projectionClause.str() << " FROM " << fromClause.str();+        if (!firstWhere) {+            createViewText << " WHERE " << whereClause.str();+        }+        createViewText << ";";+        executeSQL(createViewText.str(), db);+    }+    void createSymbolTable() {+        std::stringstream createTableText;+        createTableText << "CREATE TABLE IF NOT EXISTS '" << symbolTableName << "' ";+        createTableText << "(id INTEGER PRIMARY KEY, symbol TEXT UNIQUE);";+        executeSQL(createTableText.str(), db);+    }++    /**+     * Return given filename or construct from relation name.+     * Default name is [configured path]/[relation name].sqlite+     *+     * @param rwOperation map of IO configuration options+     * @return input filename+     */+    static std::string getFileName(const std::map<std::string, std::string>& rwOperation) {+        // legacy support for SQLite prior to 2020-03-18+        // convert dbname to filename+        auto name = getOr(rwOperation, "dbname", rwOperation.at("name") + ".sqlite");+        name = getOr(rwOperation, "filename", name);++        if (name.front() != '/') {+            name = getOr(rwOperation, "output-dir", ".") + "/" + name;+        }+        return name;+    }++    const std::string dbFilename;+    const std::string relationName;+    const std::string symbolTableName = "__SymbolTable";++    std::unordered_map<uint64_t, uint64_t> dbSymbolTable;+    sqlite3_stmt* insertStatement = nullptr;+    sqlite3_stmt* symbolInsertStatement = nullptr;+    sqlite3_stmt* symbolSelectStatement = nullptr;+    sqlite3* db = nullptr;+};++class WriteSQLiteFactory : public WriteStreamFactory {+public:+    Own<WriteStream> getWriter(const std::map<std::string, std::string>& rwOperation,+            const SymbolTable& symbolTable, const RecordTable& recordTable) override {+        return mk<WriteStreamSQLite>(rwOperation, symbolTable, recordTable);+    }++    const std::string& getName() const override {+        static const std::string name = "sqlite";+        return name;+    }+    ~WriteSQLiteFactory() override = default;+};++} /* namespace souffle */
+ cbits/souffle/io/gzfstream.h view
@@ -0,0 +1,235 @@+/*+ * Souffle - A Datalog Compiler+ * Copyright (c) 2013, 2014, Oracle and/or its affiliates. All rights reserved+ * Licensed under the Universal Permissive License v 1.0 as shown at:+ * - https://opensource.org/licenses/UPL+ * - <souffle root>/licenses/SOUFFLE-UPL.txt+ */++/************************************************************************+ *+ * @file gzfstream.h+ * A simple zlib wrapper to provide gzip file streams.+ *+ ***********************************************************************/++#pragma once++#include <cstdio>+#include <cstring>+#include <iostream>+#include <string>+#include <zlib.h>++namespace souffle {++namespace gzfstream {++namespace internal {++class gzfstreambuf : public std::streambuf {+public:+    gzfstreambuf() {+        setp(buffer, buffer + (bufferSize - 1));+        setg(buffer + reserveSize, buffer + reserveSize, buffer + reserveSize);+    }++    gzfstreambuf(const gzfstreambuf&) = delete;++    gzfstreambuf(gzfstreambuf&& old) = default;++    gzfstreambuf* open(const std::string& filename, std::ios_base::openmode mode) {+        if (is_open()) {+            return nullptr;+        }+        if ((mode ^ std::ios::in ^ std::ios::out) == 0) {+            return nullptr;+        }++        this->mode = mode;+        std::string gzmode((mode & std::ios::in) != 0 ? "rb" : "wb");+        fileHandle = gzopen(filename.c_str(), gzmode.c_str());++        if (fileHandle == nullptr) {+            return nullptr;+        }+        isOpen = true;++        return this;+    }++    gzfstreambuf* close() {+        if (is_open()) {+            sync();+            isOpen = false;+            if (gzclose(fileHandle) == Z_OK) {+                return this;+            }+        }+        return nullptr;+    }++    bool is_open() const {+        return isOpen;+    }++    ~gzfstreambuf() override {+        try {+            close();+        } catch (...) {+            // Don't throw exceptions.+        }+    }++protected:+    int_type overflow(int c = EOF) override {+        if (((mode & std::ios::out) == 0) || !isOpen) {+            return EOF;+        }++        if (c != EOF) {+            *pptr() = c;+            pbump(1);+        }+        int toWrite = pptr() - pbase();+        if (gzwrite(fileHandle, pbase(), toWrite) != toWrite) {+            return EOF;+        }+        pbump(-toWrite);++        return c;+    }++    int_type underflow() override {+        if (((mode & std::ios::in) == 0) || !isOpen) {+            return EOF;+        }+        if ((gptr() != nullptr) && (gptr() < egptr())) {+            return traits_type::to_int_type(*gptr());+        }++        unsigned charsPutBack = gptr() - eback();+        if (charsPutBack > reserveSize) {+            charsPutBack = reserveSize;+        }+        memcpy(buffer + reserveSize - charsPutBack, gptr() - charsPutBack, charsPutBack);++        int charsRead = gzread(fileHandle, buffer + reserveSize, bufferSize - reserveSize);+        if (charsRead <= 0) {+            return EOF;+        }++        setg(buffer + reserveSize - charsPutBack, buffer + reserveSize, buffer + reserveSize + charsRead);++        return traits_type::to_int_type(*gptr());+    }++    int sync() override {+        if ((pptr() != nullptr) && pptr() > pbase()) {+            int toWrite = pptr() - pbase();+            if (gzwrite(fileHandle, pbase(), toWrite) != toWrite) {+                return -1;+            }+            pbump(-toWrite);+        }+        return 0;+    }++private:+    static constexpr unsigned int bufferSize = 65536;+    static constexpr unsigned int reserveSize = 16;++    char buffer[bufferSize] = {};+    gzFile fileHandle = {};+    bool isOpen = false;+    std::ios_base::openmode mode = std::ios_base::in;+};++class gzfstream : virtual public std::ios {+public:+    gzfstream() {+        init(&buf);+    }++    gzfstream(const std::string& filename, std::ios_base::openmode mode) {+        init(&buf);+        open(filename, mode);+    }++    gzfstream(const gzfstream&) = delete;++    gzfstream(gzfstream&&) = delete;++    ~gzfstream() override = default;++    void open(const std::string& filename, std::ios_base::openmode mode) {+        if (buf.open(filename, mode) == nullptr) {+            clear(rdstate() | std::ios::badbit);+        }+    }++    bool is_open() {+        return buf.is_open();+    }++    void close() {+        if (buf.is_open()) {+            if (buf.close() == nullptr) {+                clear(rdstate() | std::ios::badbit);+            }+        }+    }++    gzfstreambuf* rdbuf() const {+        return &buf;+    }++protected:+    mutable gzfstreambuf buf;+};++}  // namespace internal++class igzfstream : public internal::gzfstream, public std::istream {+public:+    igzfstream() : internal::gzfstream(), std::istream(&buf) {}++    explicit igzfstream(const std::string& filename, std::ios_base::openmode mode = std::ios::in)+            : internal::gzfstream(filename, mode), std::istream(&buf) {}++    igzfstream(const igzfstream&) = delete;++    igzfstream(igzfstream&&) = delete;++    internal::gzfstreambuf* rdbuf() const {+        return internal::gzfstream::rdbuf();+    }++    void open(const std::string& filename, std::ios_base::openmode mode = std::ios::in) {+        internal::gzfstream::open(filename, mode);+    }+};++class ogzfstream : public internal::gzfstream, public std::ostream {+public:+    ogzfstream() : std::ostream(&buf) {}++    explicit ogzfstream(const std::string& filename, std::ios_base::openmode mode = std::ios::out)+            : internal::gzfstream(filename, mode), std::ostream(&buf) {}++    ogzfstream(const ogzfstream&) = delete;++    ogzfstream(ogzfstream&&) = delete;++    internal::gzfstreambuf* rdbuf() const {+        return internal::gzfstream::rdbuf();+    }++    void open(const std::string& filename, std::ios_base::openmode mode = std::ios::out) {+        internal::gzfstream::open(filename, mode);+    }+};++} /* namespace gzfstream */++} /* namespace souffle */
− cbits/souffle/json11.h 
@@ -1,1107 +0,0 @@-/* json11- *- * json11 is a tiny JSON library for C++11, providing JSON parsing and serialization.- *- * The core object provided by the library is json11::Json. A Json object represents any JSON- * value: null, bool, number (int or double), string (std::string), array (std::vector), or- * object (std::map).- *- * Json objects act like values: they can be assigned, copied, moved, compared for equality or- * order, etc. There are also helper methods Json::dump, to serialize a Json to a string, and- * Json::parse (static) to parse a std::string as a Json object.- *- * Internally, the various types of Json object are represented by the JsonValue class- * hierarchy.- *- * A note on numbers - JSON specifies the syntax of number formatting but not its semantics,- * so some JSON implementations distinguish between integers and floating-point numbers, while- * some don't. In json11, we choose the latter. Because some JSON implementations (namely- * Javascript itself) treat all numbers as the same type, distinguishing the two leads- * to JSON that will be *silently* changed by a round-trip through those implementations.- * Dangerous! To avoid that risk, json11 stores all numbers as double internally, but also- * provides integer helpers.- *- * Fortunately, double-precision IEEE754 ('double') can precisely store any integer in the- * range +/-2^53, which includes every 'int' on most systems. (Timestamps often use int64- * or long long to avoid the Y2038K problem; a double storing microseconds since some epoch- * will be exact for +/- 275 years.)- */--/* Copyright (c) 2013 Dropbox, Inc.- *- * Permission is hereby granted, free of charge, to any person obtaining a copy- * of this software and associated documentation files (the "Software"), to deal- * in the Software without restriction, including without limitation the rights- * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell- * copies of the Software, and to permit persons to whom the Software is- * furnished to do so, subject to the following conditions:- *- * The above copyright notice and this permission notice shall be included in- * all copies or substantial portions of the Software.- *- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN- * THE SOFTWARE.- */--#pragma once--#include <cassert>-#include <cmath>-#include <cstdint>-#include <cstdio>-#include <cstdlib>-#include <initializer_list>-#include <iosfwd>-#include <limits>-#include <map>-#include <memory>-#include <string>-#include <type_traits>-#include <utility>-#include <vector>--#ifdef _MSC_VER-#if _MSC_VER <= 1800  // VS 2013-#ifndef noexcept-#define noexcept throw()-#endif--#ifndef snprintf-#define snprintf _snprintf_s-#endif-#endif-#endif--namespace json11 {--enum JsonParse { STANDARD, COMMENTS };--class JsonValue;--class Json final {-public:-    // Types-    enum Type { NUL, NUMBER, BOOL, STRING, ARRAY, OBJECT };--    // Array and object typedefs-    using array = std::vector<Json>;-    using object = std::map<std::string, Json>;--    // Constructors for the various types of JSON value.-    Json() noexcept;                 // NUL-    Json(std::nullptr_t) noexcept;   // NUL-    Json(double value);              // NUMBER-    Json(long long value);           // NUMBER-    Json(bool value);                // BOOL-    Json(const std::string& value);  // STRING-    Json(std::string&& value);       // STRING-    Json(const char* value);         // STRING-    Json(const array& values);       // ARRAY-    Json(array&& values);            // ARRAY-    Json(const object& values);      // OBJECT-    Json(object&& values);           // OBJECT--    // Implicit constructor: anything with a to_json() function.-    template <class T, class = decltype(&T::to_json)>-    Json(const T& t) : Json(t.to_json()) {}--    // Implicit constructor: map-like objects (std::map, std::unordered_map, etc)-    template <class M,-            typename std::enable_if<-                    std::is_constructible<std::string, decltype(std::declval<M>().begin()->first)>::value &&-                            std::is_constructible<Json, decltype(std::declval<M>().begin()->second)>::value,-                    int>::type = 0>-    Json(const M& m) : Json(object(m.begin(), m.end())) {}--    // Implicit constructor: vector-like objects (std::list, std::vector, std::set, etc)-    template <class V,-            typename std::enable_if<std::is_constructible<Json, decltype(*std::declval<V>().begin())>::value,-                    int>::type = 0>-    Json(const V& v) : Json(array(v.begin(), v.end())) {}--    // This prevents Json(some_pointer) from accidentally producing a bool. Use-    // Json(bool(some_pointer)) if that behavior is desired.-    Json(void*) = delete;--    // Accessors-    Type type() const;--    bool is_null() const {-        return type() == NUL;-    }-    bool is_number() const {-        return type() == NUMBER;-    }-    bool is_bool() const {-        return type() == BOOL;-    }-    bool is_string() const {-        return type() == STRING;-    }-    bool is_array() const {-        return type() == ARRAY;-    }-    bool is_object() const {-        return type() == OBJECT;-    }--    // Return the enclosed value if this is a number, 0 otherwise. Note that json11 does not-    // distinguish between integer and non-integer numbers - number_value() and int_value()-    // can both be applied to a NUMBER-typed object.-    double number_value() const;-    int int_value() const;-    long long long_value() const;--    // Return the enclosed value if this is a boolean, false otherwise.-    bool bool_value() const;-    // Return the enclosed string if this is a string, "" otherwise.-    const std::string& string_value() const;-    // Return the enclosed std::vector if this is an array, or an empty vector otherwise.-    const array& array_items() const;-    // Return the enclosed std::map if this is an object, or an empty map otherwise.-    const object& object_items() const;--    // Return a reference to arr[i] if this is an array, Json() otherwise.-    const Json& operator[](size_t i) const;-    // Return a reference to obj[key] if this is an object, Json() otherwise.-    const Json& operator[](const std::string& key) const;--    // Serialize.-    void dump(std::string& out) const;-    std::string dump() const {-        std::string out;-        dump(out);-        return out;-    }--    // Parse. If parse fails, return Json() and assign an error message to err.-    static Json parse(const std::string& in, std::string& err, JsonParse strategy = JsonParse::STANDARD);--    static Json parse(const char* in, std::string& err, JsonParse strategy = JsonParse::STANDARD) {-        if (in == nullptr) {-            err = "null input";-            return nullptr;-        }-        return parse(std::string(in), err, strategy);-    }-    // Parse multiple objects, concatenated or separated by whitespace-    static std::vector<Json> parse_multi(const std::string& in, std::string::size_type& parser_stop_pos,-            std::string& err, JsonParse strategy = JsonParse::STANDARD);--    static inline std::vector<Json> parse_multi(-            const std::string& in, std::string& err, JsonParse strategy = JsonParse::STANDARD) {-        std::string::size_type parser_stop_pos;-        return parse_multi(in, parser_stop_pos, err, strategy);-    }--    bool operator==(const Json& rhs) const;-    bool operator<(const Json& rhs) const;-    bool operator!=(const Json& rhs) const {-        return !(*this == rhs);-    }-    bool operator<=(const Json& rhs) const {-        return !(rhs < *this);-    }-    bool operator>(const Json& rhs) const {-        return (rhs < *this);-    }-    bool operator>=(const Json& rhs) const {-        return !(*this < rhs);-    }--    /* has_shape(types, err)-     *-     * Return true if this is a JSON object and, for each item in types, has a field of-     * the given type. If not, return false and set err to a descriptive message.-     */-    using shape = std::initializer_list<std::pair<std::string, Type>>;-    bool has_shape(const shape& types, std::string& err) const {-        if (!is_object()) {-            err = "expected JSON object, got " + dump();-            return false;-        }--        for (auto& item : types) {-            if ((*this)[item.first].type() != item.second) {-                err = "bad type for " + item.first + " in " + dump();-                return false;-            }-        }--        return true;-    }--private:-    std::shared_ptr<JsonValue> m_ptr;-};--// Internal class hierarchy - JsonValue objects are not exposed to users of this API.-class JsonValue {-protected:-    friend class Json;-    friend class JsonInt;-    friend class JsonDouble;-    virtual Json::Type type() const = 0;-    virtual bool equals(const JsonValue* other) const = 0;-    virtual bool less(const JsonValue* other) const = 0;-    virtual void dump(std::string& out) const = 0;-    virtual double number_value() const;-    virtual int int_value() const;-    virtual long long long_value() const;-    virtual bool bool_value() const;-    virtual const std::string& string_value() const;-    virtual const Json::array& array_items() const;-    virtual const Json& operator[](size_t i) const;-    virtual const Json::object& object_items() const;-    virtual const Json& operator[](const std::string& key) const;-    virtual ~JsonValue() = default;-};--static const int max_depth = 200;--/* Helper for representing null - just a do-nothing struct, plus comparison- * operators so the helpers in JsonValue work. We can't use nullptr_t because- * it may not be orderable.- */-struct NullStruct {-    bool operator==(NullStruct) const {-        return true;-    }-    bool operator<(NullStruct) const {-        return false;-    }-};--/* * * * * * * * * * * * * * * * * * * *- * Serialization- */--static void dump(NullStruct, std::string& out) {-    out += "null";-}--static void dump(double value, std::string& out) {-    if (std::isfinite(value)) {-        char buf[32];-        snprintf(buf, sizeof buf, "%.17g", value);-        out += buf;-    } else {-        out += "null";-    }-}--static void dump(long long value, std::string& out) {-    char buf[32];-    snprintf(buf, sizeof buf, "%lld", value);-    out += buf;-}--static void dump(bool value, std::string& out) {-    out += value ? "true" : "false";-}--static void dump(const std::string& value, std::string& out) {-    out += '"';-    for (size_t i = 0; i < value.length(); i++) {-        const char ch = value[i];-        if (ch == '\\') {-            out += "\\\\";-        } else if (ch == '"') {-            out += "\\\"";-        } else if (ch == '\b') {-            out += "\\b";-        } else if (ch == '\f') {-            out += "\\f";-        } else if (ch == '\n') {-            out += "\\n";-        } else if (ch == '\r') {-            out += "\\r";-        } else if (ch == '\t') {-            out += "\\t";-        } else if (static_cast<uint8_t>(ch) <= 0x1f) {-            char buf[8];-            snprintf(buf, sizeof buf, "\\u%04x", ch);-            out += buf;-        } else if (static_cast<uint8_t>(ch) == 0xe2 && static_cast<uint8_t>(value[i + 1]) == 0x80 &&-                   static_cast<uint8_t>(value[i + 2]) == 0xa8) {-            out += "\\u2028";-            i += 2;-        } else if (static_cast<uint8_t>(ch) == 0xe2 && static_cast<uint8_t>(value[i + 1]) == 0x80 &&-                   static_cast<uint8_t>(value[i + 2]) == 0xa9) {-            out += "\\u2029";-            i += 2;-        } else {-            out += ch;-        }-    }-    out += '"';-}--static void dump(const Json::array& values, std::string& out) {-    bool first = true;-    out += "[";-    for (const auto& value : values) {-        if (!first) out += ", ";-        value.dump(out);-        first = false;-    }-    out += "]";-}--static void dump(const Json::object& values, std::string& out) {-    bool first = true;-    out += "{";-    for (const auto& kv : values) {-        if (!first) out += ", ";-        dump(kv.first, out);-        out += ": ";-        kv.second.dump(out);-        first = false;-    }-    out += "}";-}--inline void Json::dump(std::string& out) const {-    m_ptr->dump(out);-}--/* * * * * * * * * * * * * * * * * * * *- * Value wrappers- */--template <Json::Type tag, typename T>-class Value : public JsonValue {-protected:-    // Constructors-    explicit Value(T value) : m_value(std::move(value)) {}--    // Get type tag-    Json::Type type() const override {-        return tag;-    }--    // Comparisons-    bool equals(const JsonValue* other) const override {-        return m_value == static_cast<const Value<tag, T>*>(other)->m_value;-    }-    bool less(const JsonValue* other) const override {-        return m_value < static_cast<const Value<tag, T>*>(other)->m_value;-    }--    const T m_value;-    void dump(std::string& out) const override {-        json11::dump(m_value, out);-    }-};--class JsonDouble final : public Value<Json::NUMBER, double> {-    double number_value() const override {-        return m_value;-    }-    int int_value() const override {-        return static_cast<int>(m_value);-    }-    long long long_value() const override {-        return static_cast<long long>(m_value);-    }-    bool equals(const JsonValue* other) const override {-        return m_value == other->number_value();-    }-    bool less(const JsonValue* other) const override {-        return m_value < other->number_value();-    }--public:-    explicit JsonDouble(double value) : Value(value) {}-};--class JsonInt final : public Value<Json::NUMBER, long long> {-    double number_value() const override {-        return m_value;-    }-    int int_value() const override {-        return m_value;-    }-    long long long_value() const override {-        return static_cast<long long>(m_value);-    }-    bool equals(const JsonValue* other) const override {-        return m_value == other->number_value();-    }-    bool less(const JsonValue* other) const override {-        return m_value < other->number_value();-    }--public:-    explicit JsonInt(int value) : Value(value) {}-};--class JsonBoolean final : public Value<Json::BOOL, bool> {-    bool bool_value() const override {-        return m_value;-    }--public:-    explicit JsonBoolean(bool value) : Value(value) {}-};--class JsonString final : public Value<Json::STRING, std::string> {-    const std::string& string_value() const override {-        return m_value;-    }--public:-    explicit JsonString(const std::string& value) : Value(value) {}-    explicit JsonString(std::string&& value) : Value(std::move(value)) {}-};--class JsonArray final : public Value<Json::ARRAY, Json::array> {-    const Json::array& array_items() const override {-        return m_value;-    }-    const Json& operator[](size_t i) const override;--public:-    explicit JsonArray(const Json::array& value) : Value(value) {}-    explicit JsonArray(Json::array&& value) : Value(std::move(value)) {}-};--class JsonObject final : public Value<Json::OBJECT, Json::object> {-    const Json::object& object_items() const override {-        return m_value;-    }-    const Json& operator[](const std::string& key) const override;--public:-    explicit JsonObject(const Json::object& value) : Value(value) {}-    explicit JsonObject(Json::object&& value) : Value(std::move(value)) {}-};--class JsonNull final : public Value<Json::NUL, NullStruct> {-public:-    JsonNull() : Value({}) {}-};--/* * * * * * * * * * * * * * * * * * * *- * Static globals - static-init-safe- */-struct Statics {-    const std::shared_ptr<JsonValue> null = std::make_shared<JsonNull>();-    const std::shared_ptr<JsonValue> t = std::make_shared<JsonBoolean>(true);-    const std::shared_ptr<JsonValue> f = std::make_shared<JsonBoolean>(false);-    const std::string empty_string{};-    const std::vector<Json> empty_vector{};-    const std::map<std::string, Json> empty_map{};-    Statics() = default;-};--static const Statics& statics() {-    static const Statics s{};-    return s;-}--static const Json& static_null() {-    // This has to be separate, not in Statics, because Json() accesses statics().null.-    static const Json json_null;-    return json_null;-}--/* * * * * * * * * * * * * * * * * * * *- * Constructors- */--inline Json::Json() noexcept : m_ptr(statics().null) {}-inline Json::Json(std::nullptr_t) noexcept : m_ptr(statics().null) {}-inline Json::Json(double value) : m_ptr(std::make_shared<JsonDouble>(value)) {}-inline Json::Json(long long value) : m_ptr(std::make_shared<JsonInt>(value)) {}-inline Json::Json(bool value) : m_ptr(value ? statics().t : statics().f) {}-inline Json::Json(const std::string& value) : m_ptr(std::make_shared<JsonString>(value)) {}-inline Json::Json(std::string&& value) : m_ptr(std::make_shared<JsonString>(std::move(value))) {}-inline Json::Json(const char* value) : m_ptr(std::make_shared<JsonString>(value)) {}-inline Json::Json(const Json::array& values) : m_ptr(std::make_shared<JsonArray>(values)) {}-inline Json::Json(Json::array&& values) : m_ptr(std::make_shared<JsonArray>(std::move(values))) {}-inline Json::Json(const Json::object& values) : m_ptr(std::make_shared<JsonObject>(values)) {}-inline Json::Json(Json::object&& values) : m_ptr(std::make_shared<JsonObject>(std::move(values))) {}--/* * * * * * * * * * * * * * * * * * * *- * Accessors- */--inline Json::Type Json::type() const {-    return m_ptr->type();-}-inline double Json::number_value() const {-    return m_ptr->number_value();-}-inline int Json::int_value() const {-    return m_ptr->int_value();-}-inline long long Json::long_value() const {-    return m_ptr->long_value();-}-inline bool Json::bool_value() const {-    return m_ptr->bool_value();-}-inline const std::string& Json::string_value() const {-    return m_ptr->string_value();-}-inline const std::vector<Json>& Json::array_items() const {-    return m_ptr->array_items();-}-inline const std::map<std::string, Json>& Json::object_items() const {-    return m_ptr->object_items();-}-inline const Json& Json::operator[](size_t i) const {-    return (*m_ptr)[i];-}-inline const Json& Json::operator[](const std::string& key) const {-    return (*m_ptr)[key];-}--inline double JsonValue::number_value() const {-    return 0;-}-inline int JsonValue::int_value() const {-    return 0;-}-inline long long JsonValue::long_value() const {-    return 0;-}-inline bool JsonValue::bool_value() const {-    return false;-}-inline const std::string& JsonValue::string_value() const {-    return statics().empty_string;-}-inline const std::vector<Json>& JsonValue::array_items() const {-    return statics().empty_vector;-}-inline const std::map<std::string, Json>& JsonValue::object_items() const {-    return statics().empty_map;-}-inline const Json& JsonValue::operator[](size_t) const {-    return static_null();-}-inline const Json& JsonValue::operator[](const std::string&) const {-    return static_null();-}--inline const Json& JsonObject::operator[](const std::string& key) const {-    auto iter = m_value.find(key);-    return (iter == m_value.end()) ? static_null() : iter->second;-}-inline const Json& JsonArray::operator[](size_t i) const {-    if (i >= m_value.size()) {-        return static_null();-    }-    return m_value[i];-}--/* * * * * * * * * * * * * * * * * * * *- * Comparison- */--inline bool Json::operator==(const Json& other) const {-    if (m_ptr == other.m_ptr) {-        return true;-    }-    if (m_ptr->type() != other.m_ptr->type()) {-        return false;-    }--    return m_ptr->equals(other.m_ptr.get());-}--inline bool Json::operator<(const Json& other) const {-    if (m_ptr == other.m_ptr) {-        return false;-    }-    if (m_ptr->type() != other.m_ptr->type()) {-        return m_ptr->type() < other.m_ptr->type();-    }--    return m_ptr->less(other.m_ptr.get());-}--/* * * * * * * * * * * * * * * * * * * *- * Parsing- */--/* esc(c)- *- * Format char c suitable for printing in an error message.- */-static inline std::string esc(char c) {-    char buf[12];-    if (static_cast<uint8_t>(c) >= 0x20 && static_cast<uint8_t>(c) <= 0x7f) {-        snprintf(buf, sizeof buf, "'%c' (%d)", c, c);-    } else {-        snprintf(buf, sizeof buf, "(%d)", c);-    }-    return std::string(buf);-}--static inline bool in_range(long x, long lower, long upper) {-    return (x >= lower && x <= upper);-}--namespace {-/* JsonParser- *- * Object that tracks all state of an in-progress parse.- */-struct JsonParser final {-    /* State-     */-    const std::string& str;-    size_t i;-    std::string& err;-    bool failed;-    const JsonParse strategy;--    /* fail(msg, err_ret = Json())-     *-     * Mark this parse as failed.-     */-    Json fail(std::string&& msg) {-        return fail(std::move(msg), Json());-    }--    template <typename T>-    T fail(std::string&& msg, T err_ret) {-        if (!failed) {-            err = std::move(msg);-        }-        failed = true;-        return err_ret;-    }--    /* consume_whitespace()-     *-     * Advance until the current character is non-whitespace.-     */-    void consume_whitespace() {-        while (str[i] == ' ' || str[i] == '\r' || str[i] == '\n' || str[i] == '\t') {-            i++;-        }-    }--    /* consume_comment()-     *-     * Advance comments (c-style inline and multiline).-     */-    bool consume_comment() {-        bool comment_found = false;-        if (str[i] == '/') {-            i++;-            if (i == str.size()) {-                return fail("unexpected end of input after start of comment", false);-            }-            if (str[i] == '/') {  // inline comment-                i++;-                // advance until next line, or end of input-                while (i < str.size() && str[i] != '\n') {-                    i++;-                }-                comment_found = true;-            } else if (str[i] == '*') {  // multiline comment-                i++;-                if (i > str.size() - 2)-                    return fail("unexpected end of input inside multi-line comment", false);-                // advance until closing tokens-                while (!(str[i] == '*' && str[i + 1] == '/')) {-                    i++;-                    if (i > str.size() - 2)-                        return fail("unexpected end of input inside multi-line comment", false);-                }-                i += 2;-                comment_found = true;-            } else {-                return fail("malformed comment", false);-            }-        }-        return comment_found;-    }--    /* consume_garbage()-     *-     * Advance until the current character is non-whitespace and non-comment.-     */-    void consume_garbage() {-        consume_whitespace();-        if (strategy == JsonParse::COMMENTS) {-            bool comment_found = false;-            do {-                comment_found = consume_comment();-                if (failed) {-                    return;-                }-                consume_whitespace();-            } while (comment_found);-        }-    }--    /* get_next_token()-     *-     * Return the next non-whitespace character. If the end of the input is reached,-     * flag an error and return 0.-     */-    char get_next_token() {-        consume_garbage();-        if (failed) {-            return static_cast<char>(0);-        }-        if (i == str.size()) {-            return fail("unexpected end of input", static_cast<char>(0));-        }--        return str[i++];-    }--    /* encode_utf8(pt, out)-     *-     * Encode pt as UTF-8 and add it to out.-     */-    void encode_utf8(long pt, std::string& out) {-        if (pt < 0) {-            return;-        }--        if (pt < 0x80) {-            out += static_cast<char>(pt);-        } else if (pt < 0x800) {-            out += static_cast<char>((pt >> 6) | 0xC0);-            out += static_cast<char>((pt & 0x3F) | 0x80);-        } else if (pt < 0x10000) {-            out += static_cast<char>((pt >> 12) | 0xE0);-            out += static_cast<char>(((pt >> 6) & 0x3F) | 0x80);-            out += static_cast<char>((pt & 0x3F) | 0x80);-        } else {-            out += static_cast<char>((pt >> 18) | 0xF0);-            out += static_cast<char>(((pt >> 12) & 0x3F) | 0x80);-            out += static_cast<char>(((pt >> 6) & 0x3F) | 0x80);-            out += static_cast<char>((pt & 0x3F) | 0x80);-        }-    }--    /* parse_string()-     *-     * Parse a std::string, starting at the current position.-     */-    std::string parse_string() {-        std::string out;-        long last_escaped_codepoint = -1;-        while (true) {-            if (i == str.size()) return fail("unexpected end of input in std::string", "");--            char ch = str[i++];--            if (ch == '"') {-                encode_utf8(last_escaped_codepoint, out);-                return out;-            }--            if (in_range(ch, 0, 0x1f)) {-                return fail("unescaped " + esc(ch) + " in std::string", "");-            }--            // The usual case: non-escaped characters-            if (ch != '\\') {-                encode_utf8(last_escaped_codepoint, out);-                last_escaped_codepoint = -1;-                out += ch;-                continue;-            }--            // Handle escapes-            if (i == str.size()) {-                return fail("unexpected end of input in std::string", "");-            }--            ch = str[i++];--            if (ch == 'u') {-                // Extract 4-byte escape sequence-                std::string esc = str.substr(i, 4);-                // Explicitly check length of the substring. The following loop-                // relies on std::string returning the terminating NUL when-                // accessing str[length]. Checking here reduces brittleness.-                if (esc.length() < 4) {-                    return fail("bad \\u escape: " + esc, "");-                }-                for (size_t j = 0; j < 4; j++) {-                    if (!in_range(esc[j], 'a', 'f') && !in_range(esc[j], 'A', 'F') &&-                            !in_range(esc[j], '0', '9'))-                        return fail("bad \\u escape: " + esc, "");-                }--                long codepoint = strtol(esc.data(), nullptr, 16);--                // JSON specifies that characters outside the BMP shall be encoded as a pair-                // of 4-hex-digit \u escapes encoding their surrogate pair components. Check-                // whether we're in the middle of such a beast: the previous codepoint was an-                // escaped lead (high) surrogate, and this is a trail (low) surrogate.-                if (in_range(last_escaped_codepoint, 0xD800, 0xDBFF) && in_range(codepoint, 0xDC00, 0xDFFF)) {-                    // Reassemble the two surrogate pairs into one astral-plane character, per-                    // the UTF-16 algorithm.-                    encode_utf8((((last_escaped_codepoint - 0xD800) << 10) | (codepoint - 0xDC00)) + 0x10000,-                            out);-                    last_escaped_codepoint = -1;-                } else {-                    encode_utf8(last_escaped_codepoint, out);-                    last_escaped_codepoint = codepoint;-                }--                i += 4;-                continue;-            }--            encode_utf8(last_escaped_codepoint, out);-            last_escaped_codepoint = -1;--            if (ch == 'b') {-                out += '\b';-            } else if (ch == 'f') {-                out += '\f';-            } else if (ch == 'n') {-                out += '\n';-            } else if (ch == 'r') {-                out += '\r';-            } else if (ch == 't') {-                out += '\t';-            } else if (ch == '"' || ch == '\\' || ch == '/') {-                out += ch;-            } else {-                return fail("invalid escape character " + esc(ch), "");-            }-        }-    }--    /* parse_number()-     *-     * Parse a double.-     */-    Json parse_number() {-        size_t start_pos = i;--        if (str[i] == '-') {-            i++;-        }--        // Integer part-        if (str[i] == '0') {-            i++;-            if (in_range(str[i], '0', '9')) {-                return fail("leading 0s not permitted in numbers");-            }-        } else if (in_range(str[i], '1', '9')) {-            i++;-            while (in_range(str[i], '0', '9')) {-                i++;-            }-        } else {-            return fail("invalid " + esc(str[i]) + " in number");-        }--        if (str[i] != '.' && str[i] != 'e' && str[i] != 'E' &&-                (i - start_pos) <= static_cast<size_t>(std::numeric_limits<int>::digits10)) {-            return std::atoll(str.c_str() + start_pos);-        }--        // Decimal part-        if (str[i] == '.') {-            i++;-            if (!in_range(str[i], '0', '9')) {-                return fail("at least one digit required in fractional part");-            }--            while (in_range(str[i], '0', '9')) {-                i++;-            }-        }--        // Exponent part-        if (str[i] == 'e' || str[i] == 'E') {-            i++;--            if (str[i] == '+' || str[i] == '-') {-                i++;-            }--            if (!in_range(str[i], '0', '9')) {-                return fail("at least one digit required in exponent");-            }--            while (in_range(str[i], '0', '9')) {-                i++;-            }-        }--        return std::strtod(str.c_str() + start_pos, nullptr);-    }--    /* expect(str, res)-     *-     * Expect that 'str' starts at the character that was just read. If it does, advance-     * the input and return res. If not, flag an error.-     */-    Json expect(const std::string& expected, Json res) {-        assert(i != 0);-        i--;-        if (str.compare(i, expected.length(), expected) == 0) {-            i += expected.length();-            return res;-        } else {-            return fail("parse error: expected " + expected + ", got " + str.substr(i, expected.length()));-        }-    }--    /* parse_json()-     *-     * Parse a JSON object.-     */-    Json parse_json(int depth) {-        if (depth > max_depth) {-            return fail("exceeded maximum nesting depth");-        }--        char ch = get_next_token();-        if (failed) {-            return Json();-        }--        if (ch == '-' || (ch >= '0' && ch <= '9')) {-            i--;-            return parse_number();-        }--        if (ch == 't') {-            return expect("true", true);-        }--        if (ch == 'f') {-            return expect("false", false);-        }--        if (ch == 'n') {-            return expect("null", Json());-        }--        if (ch == '"') {-            return parse_string();-        }--        if (ch == '{') {-            std::map<std::string, Json> data;-            ch = get_next_token();-            if (ch == '}') {-                return data;-            }--            while (true) {-                if (ch != '"') return fail("expected '\"' in object, got " + esc(ch));--                std::string key = parse_string();-                if (failed) {-                    return Json();-                }--                ch = get_next_token();-                if (ch != ':') {-                    return fail("expected ':' in object, got " + esc(ch));-                }--                data[std::move(key)] = parse_json(depth + 1);-                if (failed) {-                    return Json();-                }--                ch = get_next_token();-                if (ch == '}') {-                    break;-                }-                if (ch != ',') {-                    return fail("expected ',' in object, got " + esc(ch));-                }--                ch = get_next_token();-            }-            return data;-        }--        if (ch == '[') {-            std::vector<Json> data;-            ch = get_next_token();-            if (ch == ']') {-                return data;-            }--            while (true) {-                i--;-                data.push_back(parse_json(depth + 1));-                if (failed) {-                    return Json();-                }--                ch = get_next_token();-                if (ch == ']') {-                    break;-                }-                if (ch != ',') {-                    return fail("expected ',' in list, got " + esc(ch));-                }--                ch = get_next_token();-                (void)ch;-            }-            return data;-        }--        return fail("expected value, got " + esc(ch));-    }-};-}  // namespace--inline Json Json::parse(const std::string& in, std::string& err, JsonParse strategy) {-    JsonParser parser{in, 0, err, false, strategy};-    Json result = parser.parse_json(0);--    // Check for any trailing garbage-    parser.consume_garbage();-    if (parser.failed) {-        return Json();-    }-    if (parser.i != in.size()) {-        return parser.fail("unexpected trailing " + esc(in[parser.i]));-    }--    return result;-}--inline std::vector<Json> parse_multi(const std::string& in, std::string::size_type& parser_stop_pos,-        std::string& err, JsonParse strategy) {-    JsonParser parser{in, 0, err, false, strategy};-    parser_stop_pos = 0;-    std::vector<Json> json_vec;-    while (parser.i != in.size() && !parser.failed) {-        json_vec.push_back(parser.parse_json(0));-        if (parser.failed) {-            break;-        }--        // Check for another object-        parser.consume_garbage();-        if (parser.failed) {-            break;-        }-        parser_stop_pos = parser.i;-    }-    return json_vec;-}--}  // namespace json11
− cbits/souffle/profile/CellInterface.h 
@@ -1,33 +0,0 @@-/*- * Souffle - A Datalog Compiler- * Copyright (c) 2016, The Souffle Developers. All rights reserved- * Licensed under the Universal Permissive License v 1.0 as shown at:- * - https://opensource.org/licenses/UPL- * - <souffle root>/licenses/SOUFFLE-UPL.txt- */--#pragma once--#include <chrono>-#include <string>--namespace souffle {-namespace profile {--class CellInterface {-public:-    virtual std::string toString(int precision) const = 0;--    virtual double getDoubleVal() const = 0;--    virtual long getLongVal() const = 0;--    virtual std::string getStringVal() const = 0;--    virtual std::chrono::microseconds getTimeVal() const = 0;--    virtual ~CellInterface() = default;-};--}  // namespace profile-}  // namespace souffle
− cbits/souffle/profile/DataComparator.h 
@@ -1,66 +0,0 @@-/*- * Souffle - A Datalog Compiler- * Copyright (c) 2016, The Souffle Developers. All rights reserved- * Licensed under the Universal Permissive License v 1.0 as shown at:- * - https://opensource.org/licenses/UPL- * - <souffle root>/licenses/SOUFFLE-UPL.txt- */--#pragma once--#include "CellInterface.h"-#include "Row.h"--#include <cmath>-#include <memory>-#include <vector>--namespace souffle {-namespace profile {--/*- * Data comparison functions for sorting tables- *- * Will sort the values of only one column, in descending order- *- */-class DataComparator {-public:-    /** Sort by total time. */-    static bool TIME(const std::shared_ptr<Row>& a, const std::shared_ptr<Row>& b) {-        return a->cells[0]->getDoubleVal() > b->cells[0]->getDoubleVal();-    }--    /** Sort by non-recursive time. */-    static bool NR_T(const std::shared_ptr<Row>& a, const std::shared_ptr<Row>& b) {-        return a->cells[1]->getDoubleVal() > b->cells[1]->getDoubleVal();-    }--    /** Sort by recursive time. */-    static bool R_T(const std::shared_ptr<Row>& a, const std::shared_ptr<Row>& b) {-        return a->cells[2]->getDoubleVal() > b->cells[2]->getDoubleVal();-    }--    /** Sort by copy time. */-    static bool C_T(const std::shared_ptr<Row>& a, const std::shared_ptr<Row>& b) {-        return a->cells[3]->getDoubleVal() > b->cells[3]->getDoubleVal();-    }--    /** Sort by tuple count. */-    static bool TUP(const std::shared_ptr<Row>& a, const std::shared_ptr<Row>& b) {-        return b->cells[4]->getLongVal() < a->cells[4]->getLongVal();-    }--    /** Sort by name. */-    static bool NAME(const std::shared_ptr<Row>& a, const std::shared_ptr<Row>& b) {-        return b->cells[5]->getStringVal() > a->cells[5]->getStringVal();-    }--    /** Sort by ID. */-    static bool ID(const std::shared_ptr<Row>& a, const std::shared_ptr<Row>& b) {-        return b->cells[6]->getStringVal() > a->cells[6]->getStringVal();-    }-};--}  // namespace profile-}  // namespace souffle
− cbits/souffle/profile/Row.h 
@@ -1,48 +0,0 @@-/*- * Souffle - A Datalog Compiler- * Copyright (c) 2016, The Souffle Developers. All rights reserved- * Licensed under the Universal Permissive License v 1.0 as shown at:- * - https://opensource.org/licenses/UPL- * - <souffle root>/licenses/SOUFFLE-UPL.txt- */--#pragma once--#include "CellInterface.h"--#include <iostream>-#include <memory>-#include <string>-#include <vector>--namespace souffle {-namespace profile {--/*- * Row class for Tables, holds a vector of cells.- */-class Row {-public:-    std::vector<std::shared_ptr<CellInterface>> cells;--    Row(unsigned long size) : cells() {-        for (unsigned long i = 0; i < size; i++) {-            cells.emplace_back(std::shared_ptr<CellInterface>(nullptr));-        }-    }--    std::shared_ptr<CellInterface>& operator[](unsigned long i) {-        return cells.at(i);-    }--    //    void addCell(int location, std::shared_ptr<CellInterface> cell) {-    //        cells[location] = cell;-    //    }--    inline std::vector<std::shared_ptr<CellInterface>> getCells() {-        return cells;-    }-};--}  // namespace profile-}  // namespace souffle
− cbits/souffle/profile/Table.h 
@@ -1,66 +0,0 @@-/*- * Souffle - A Datalog Compiler- * Copyright (c) 2016, The Souffle Developers. All rights reserved- * Licensed under the Universal Permissive License v 1.0 as shown at:- * - https://opensource.org/licenses/UPL- * - <souffle root>/licenses/SOUFFLE-UPL.txt- */--#pragma once--#include "DataComparator.h"-#include "Row.h"-#include <algorithm>-#include <vector>--namespace souffle {-namespace profile {--/*- * Table class for holding a vector of rows- * And sorting the rows based on a datacomparator function- */-class Table {-public:-    std::vector<std::shared_ptr<Row>> rows;--    Table() : rows() {}--    void addRow(std::shared_ptr<Row> row) {-        rows.push_back(row);-    }--    inline std::vector<std::shared_ptr<Row>> getRows() {-        return rows;-    }--    void sort(int col_num) {-        switch (col_num) {-            case 1:-                std::sort(rows.begin(), rows.end(), DataComparator::NR_T);-                break;-            case 2:-                std::sort(rows.begin(), rows.end(), DataComparator::R_T);-                break;-            case 3:-                std::sort(rows.begin(), rows.end(), DataComparator::C_T);-                break;-            case 4:-                std::sort(rows.begin(), rows.end(), DataComparator::TUP);-                break;-            case 5:-                std::sort(rows.begin(), rows.end(), DataComparator::ID);-                break;-            case 6:-                std::sort(rows.begin(), rows.end(), DataComparator::NAME);-                break;-            case 0:-            default:  // if the col_num isn't defined use TIME-                std::sort(rows.begin(), rows.end(), DataComparator::TIME);-                break;-        }-    }-};--}  // namespace profile-}  // namespace souffle
cbits/souffle/utility/ContainerUtil.h view
@@ -163,15 +163,25 @@ // -------------------------------------------------------------------------------  template <typename A>-auto clone(const std::vector<A>& xs) {-    std::vector<decltype(clone(xs[0]))> ys;+auto clone(const std::vector<A*>& xs) {+    std::vector<std::unique_ptr<A>> ys;     ys.reserve(xs.size());     for (auto&& x : xs) {-        ys.emplace_back(clone(x));+        ys.emplace_back(x ? std::unique_ptr<A>(x->clone()) : nullptr);     }     return ys; } +template <typename A>+auto clone(const std::vector<std::unique_ptr<A>>& xs) {+    std::vector<std::unique_ptr<A>> ys;+    ys.reserve(xs.size());+    for (auto&& x : xs) {+        ys.emplace_back(x ? std::unique_ptr<A>(x->clone()) : nullptr);+    }+    return ys;+}+ // ------------------------------------------------------------- //                            Iterators // -------------------------------------------------------------@@ -457,64 +467,6 @@     auto comp = comp_deref<std::unique_ptr<Value>>();     return equal_targets(             a, b, [&comp](auto& a, auto& b) { return a.first == b.first && comp(a.second, b.second); });-}--/**- * Compares two values referenced by a pointer where the case where both- * pointers are null is also considered equivalent.- */-template <typename T>-bool equal_ptr(const T* a, const T* b) {-    if (a == nullptr && b == nullptr) {-        return true;-    }-    if (a != nullptr && b != nullptr) {-        return *a == *b;-    }-    return false;-}--/**- * Compares two values referenced by a pointer where the case where both- * pointers are null is also considered equivalent.- */-template <typename T>-bool equal_ptr(const std::unique_ptr<T>& a, const std::unique_ptr<T>& b) {-    return equal_ptr(a.get(), b.get());-}--template <typename A, typename B>-using copy_const_t = std::conditional_t<std::is_const_v<A>, const B, B>;--/**- * Helpers for `dynamic_cast`ing without having to specify redundant type qualifiers.- * e.g. `as<AstLiteral>(p)` instead of `dynamic_cast<const AstLiteral*>(p.get())`.- */-template <typename B, typename A>-auto as(A* x) {-    static_assert(std::is_base_of_v<A, B>,-            "`as<B, A>` does not allow cross-type dyn casts. "-            "(i.e. `as<B, A>` where `B <: A` is not true.) "-            "Such a cast is likely a mistake or typo.");-    return dynamic_cast<copy_const_t<A, B>*>(x);-}--template <typename B, typename A>-std::enable_if_t<std::is_base_of_v<A, B>, copy_const_t<A, B>*> as(A& x) {-    return as<B>(&x);-}--template <typename B, typename A>-B* as(const Own<A>& x) {-    return as<B>(x.get());-}--/**- * Checks if the object of type Source can be casted to type Destination.- */-template <typename Destination, typename Source>-bool isA(Source&& src) {-    return as<Destination>(std::forward<Source>(src)) != nullptr; }  }  // namespace souffle
cbits/souffle/utility/EvaluatorUtil.h view
@@ -16,8 +16,8 @@  #pragma once -#include "../CompiledTuple.h"-#include "../RamTypes.h"+#include "souffle/CompiledTuple.h"+#include "souffle/RamTypes.h" #include "tinyformat.h"  namespace souffle::evaluator {@@ -58,7 +58,7 @@     } catch (...) {         tfm::format(std::cerr, "error: wrong string provided by `to_number(\"%s\")` functor.\n", src);         raise(SIGFPE);-        fatal("");  // UNREACHABLE: `raise` lacks a no-return attribute+        abort();  // UNREACHABLE: `raise` lacks a no-return attribute     } };
cbits/souffle/utility/FileUtil.h view
@@ -50,6 +50,11 @@     return _fullpath(resolved_path, path, PATH_MAX); } +/**+ * Define an alias for the popen and pclose functions on windows+ */+#define popen _popen+#define pclose _pclose #endif  namespace souffle {@@ -264,11 +269,11 @@     FILE* in = popen(cmd, "r");     std::stringstream data;     while (in != nullptr) {-        char c = fgetc(in);+        int c = fgetc(in);         if (feof(in) != 0) {             break;         }-        data << c;+        data << static_cast<char>(c);     }     pclose(in);     return data;
cbits/souffle/utility/FunctionalUtil.h view
@@ -18,6 +18,8 @@  #include <algorithm> #include <functional>+#include <utility>+#include <vector>  namespace souffle {
cbits/souffle/utility/MiscUtil.h view
@@ -48,6 +48,7 @@  */ #define __builtin_popcountll __popcnt64 +#if _MSC_VER < 1924 constexpr unsigned long __builtin_ctz(unsigned long value) {     unsigned long trailing_zeroes = 0;     while ((value = value >> 1) ^ 1) {@@ -65,6 +66,7 @@         return 64;     } }+#endif  // _MSC_VER < 1924 #endif  // -------------------------------------------------------------------------------@@ -109,6 +111,77 @@ template <typename A, typename B> auto clone(const std::pair<A, B>& p) {     return std::make_pair(clone(p.first), clone(p.second));+}++// -------------------------------------------------------------------------------+//                             Comparison Utilities+// -------------------------------------------------------------------------------+/**+ * Compares two values referenced by a pointer where the case where both+ * pointers are null is also considered equivalent.+ */+template <typename T>+bool equal_ptr(const T* a, const T* b) {+    if (a == nullptr && b == nullptr) {+        return true;+    }+    if (a != nullptr && b != nullptr) {+        return *a == *b;+    }+    return false;+}++/**+ * Compares two values referenced by a pointer where the case where both+ * pointers are null is also considered equivalent.+ */+template <typename T>+bool equal_ptr(const std::unique_ptr<T>& a, const std::unique_ptr<T>& b) {+    return equal_ptr(a.get(), b.get());+}++template <typename A, typename B>+using copy_const_t = std::conditional_t<std::is_const_v<A>, const B, B>;++/**+ * Helpers for `dynamic_cast`ing without having to specify redundant type qualifiers.+ * e.g. `as<AstLiteral>(p)` instead of `dynamic_cast<const AstLiteral*>(p.get())`.+ */+template <typename B, typename A>+auto as(A* x) {+    static_assert(std::is_base_of_v<A, B>,+            "`as<B, A>` does not allow cross-type dyn casts. "+            "(i.e. `as<B, A>` where `B <: A` is not true.) "+            "Such a cast is likely a mistake or typo.");+    return dynamic_cast<copy_const_t<A, B>*>(x);+}++template <typename B, typename A>+std::enable_if_t<std::is_base_of_v<A, B>, copy_const_t<A, B>*> as(A& x) {+    return as<B>(&x);+}++template <typename B, typename A>+B* as(const std::unique_ptr<A>& x) {+    return as<B>(x.get());+}++/**+ * Checks if the object of type Source can be casted to type Destination.+ */+template <typename B, typename A>+bool isA(A* x) {+    return dynamic_cast<copy_const_t<A, B>*>(x) != nullptr;+}++template <typename B, typename A>+std::enable_if_t<std::is_base_of_v<A, B>, bool> isA(A& x) {+    return isA<B>(&x);+}++template <typename B, typename A>+bool isA(const std::unique_ptr<A>& x) {+    return isA<B>(x.get()); }  // -------------------------------------------------------------------------------
cbits/souffle/utility/StreamUtil.h view
@@ -25,6 +25,8 @@ #include <utility> #include <vector> +#include "souffle/utility/ContainerUtil.h"+ // ------------------------------------------------------------------------------- //                           General Print Utilities // -------------------------------------------------------------------------------@@ -34,7 +36,7 @@ template <typename A> struct IsPtrLike : std::is_pointer<A> {}; template <typename A>-struct IsPtrLike<std::unique_ptr<A>> : std::true_type {};+struct IsPtrLike<Own<A>> : std::true_type {}; template <typename A> struct IsPtrLike<std::shared_ptr<A>> : std::true_type {}; template <typename A>@@ -226,6 +228,15 @@  */ template <typename K, typename C, typename A> ostream& operator<<(ostream& out, const set<K, C, A>& s) {+    return out << "{" << souffle::join(s) << "}";+}++/**+ * Enables the generic printing of multisets assuming their element types+ * are printable.+ */+template <typename K, typename C, typename A>+ostream& operator<<(ostream& out, const multiset<K, C, A>& s) {     return out << "{" << souffle::join(s) << "}"; }
cbits/souffle/utility/StringUtil.h view
@@ -16,7 +16,7 @@  #pragma once -#include "../RamTypes.h"+#include "souffle/RamTypes.h" #include <algorithm> #include <cctype> #include <cstdlib>
+ cbits/souffle/utility/json11.h view
@@ -0,0 +1,1112 @@+/* json11+ *+ * json11 is a tiny JSON library for C++11, providing JSON parsing and serialization.+ *+ * The core object provided by the library is json11::Json. A Json object represents any JSON+ * value: null, bool, number (int or double), string (std::string), array (std::vector), or+ * object (std::map).+ *+ * Json objects act like values: they can be assigned, copied, moved, compared for equality or+ * order, etc. There are also helper methods Json::dump, to serialize a Json to a string, and+ * Json::parse (static) to parse a std::string as a Json object.+ *+ * Internally, the various types of Json object are represented by the JsonValue class+ * hierarchy.+ *+ * A note on numbers - JSON specifies the syntax of number formatting but not its semantics,+ * so some JSON implementations distinguish between integers and floating-point numbers, while+ * some don't. In json11, we choose the latter. Because some JSON implementations (namely+ * Javascript itself) treat all numbers as the same type, distinguishing the two leads+ * to JSON that will be *silently* changed by a round-trip through those implementations.+ * Dangerous! To avoid that risk, json11 stores all numbers as double internally, but also+ * provides integer helpers.+ *+ * Fortunately, double-precision IEEE754 ('double') can precisely store any integer in the+ * range +/-2^53, which includes every 'int' on most systems. (Timestamps often use int64+ * or long long to avoid the Y2038K problem; a double storing microseconds since some epoch+ * will be exact for +/- 275 years.)+ */++/* Copyright (c) 2013 Dropbox, Inc.+ *+ * Permission is hereby granted, free of charge, to any person obtaining a copy+ * of this software and associated documentation files (the "Software"), to deal+ * in the Software without restriction, including without limitation the rights+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell+ * copies of the Software, and to permit persons to whom the Software is+ * furnished to do so, subject to the following conditions:+ *+ * The above copyright notice and this permission notice shall be included in+ * all copies or substantial portions of the Software.+ *+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN+ * THE SOFTWARE.+ */++#pragma once++#include <cassert>+#include <cmath>+#include <cstdint>+#include <cstdio>+#include <cstdlib>+#include <initializer_list>+#include <iosfwd>+#include <limits>+#include <map>+#include <memory>+#include <string>+#include <type_traits>+#include <utility>+#include <vector>++#ifdef _MSC_VER+#pragma warning(disable : 4244)+#if _MSC_VER <= 1800  // VS 2013+#ifndef noexcept+#define noexcept throw()+#endif++#ifndef snprintf+#define snprintf _snprintf_s+#endif+#endif+#endif++namespace json11 {++enum JsonParse { STANDARD, COMMENTS };++class JsonValue;++class Json final {+public:+    // Types+    enum Type { NUL, NUMBER, BOOL, STRING, ARRAY, OBJECT };++    // Array and object typedefs+    using array = std::vector<Json>;+    using object = std::map<std::string, Json>;++    // Constructors for the various types of JSON value.+    Json() noexcept;                 // NUL+    Json(std::nullptr_t) noexcept;   // NUL+    Json(double value);              // NUMBER+    Json(long long value);           // NUMBER+    Json(bool value);                // BOOL+    Json(const std::string& value);  // STRING+    Json(std::string&& value);       // STRING+    Json(const char* value);         // STRING+    Json(const array& values);       // ARRAY+    Json(array&& values);            // ARRAY+    Json(const object& values);      // OBJECT+    Json(object&& values);           // OBJECT++    // Implicit constructor: anything with a to_json() function.+    template <class T, class = decltype(&T::to_json)>+    Json(const T& t) : Json(t.to_json()) {}++    // Implicit constructor: map-like objects (std::map, std::unordered_map, etc)+    template <class M,+            typename std::enable_if<+                    std::is_constructible<std::string, decltype(std::declval<M>().begin()->first)>::value &&+                            std::is_constructible<Json, decltype(std::declval<M>().begin()->second)>::value,+                    int>::type = 0>+    Json(const M& m) : Json(object(m.begin(), m.end())) {}++    // Implicit constructor: vector-like objects (std::list, std::vector, std::set, etc)+    template <class V,+            typename std::enable_if<std::is_constructible<Json, decltype(*std::declval<V>().begin())>::value,+                    int>::type = 0>+    Json(const V& v) : Json(array(v.begin(), v.end())) {}++    // This prevents Json(some_pointer) from accidentally producing a bool. Use+    // Json(bool(some_pointer)) if that behavior is desired.+    Json(void*) = delete;++    // Accessors+    Type type() const;++    bool is_null() const {+        return type() == NUL;+    }+    bool is_number() const {+        return type() == NUMBER;+    }+    bool is_bool() const {+        return type() == BOOL;+    }+    bool is_string() const {+        return type() == STRING;+    }+    bool is_array() const {+        return type() == ARRAY;+    }+    bool is_object() const {+        return type() == OBJECT;+    }++    // Return the enclosed value if this is a number, 0 otherwise. Note that json11 does not+    // distinguish between integer and non-integer numbers - number_value() and int_value()+    // can both be applied to a NUMBER-typed object.+    double number_value() const;+    int int_value() const;+    long long long_value() const;++    // Return the enclosed value if this is a boolean, false otherwise.+    bool bool_value() const;+    // Return the enclosed string if this is a string, "" otherwise.+    const std::string& string_value() const;+    // Return the enclosed std::vector if this is an array, or an empty vector otherwise.+    const array& array_items() const;+    // Return the enclosed std::map if this is an object, or an empty map otherwise.+    const object& object_items() const;++    // Return a reference to arr[i] if this is an array, Json() otherwise.+    const Json& operator[](size_t i) const;+    // Return a reference to obj[key] if this is an object, Json() otherwise.+    const Json& operator[](const std::string& key) const;++    // Serialize.+    void dump(std::string& out) const;+    std::string dump() const {+        std::string out;+        dump(out);+        return out;+    }++    // Parse. If parse fails, return Json() and assign an error message to err.+    static Json parse(const std::string& in, std::string& err, JsonParse strategy = JsonParse::STANDARD);++    static Json parse(const char* in, std::string& err, JsonParse strategy = JsonParse::STANDARD) {+        if (in == nullptr) {+            err = "null input";+            return nullptr;+        }+        return parse(std::string(in), err, strategy);+    }+    // Parse multiple objects, concatenated or separated by whitespace+    static std::vector<Json> parse_multi(const std::string& in, std::string::size_type& parser_stop_pos,+            std::string& err, JsonParse strategy = JsonParse::STANDARD);++    static inline std::vector<Json> parse_multi(+            const std::string& in, std::string& err, JsonParse strategy = JsonParse::STANDARD) {+        std::string::size_type parser_stop_pos;+        return parse_multi(in, parser_stop_pos, err, strategy);+    }++    bool operator==(const Json& rhs) const;+    bool operator<(const Json& rhs) const;+    bool operator!=(const Json& rhs) const {+        return !(*this == rhs);+    }+    bool operator<=(const Json& rhs) const {+        return !(rhs < *this);+    }+    bool operator>(const Json& rhs) const {+        return (rhs < *this);+    }+    bool operator>=(const Json& rhs) const {+        return !(*this < rhs);+    }++    /* has_shape(types, err)+     *+     * Return true if this is a JSON object and, for each item in types, has a field of+     * the given type. If not, return false and set err to a descriptive message.+     */+    using shape = std::initializer_list<std::pair<std::string, Type>>;+    bool has_shape(const shape& types, std::string& err) const {+        if (!is_object()) {+            err = "expected JSON object, got " + dump();+            return false;+        }++        for (auto& item : types) {+            if ((*this)[item.first].type() != item.second) {+                err = "bad type for " + item.first + " in " + dump();+                return false;+            }+        }++        return true;+    }++private:+    std::shared_ptr<JsonValue> m_ptr;+};++// Internal class hierarchy - JsonValue objects are not exposed to users of this API.+class JsonValue {+protected:+    friend class Json;+    friend class JsonInt;+    friend class JsonDouble;+    virtual Json::Type type() const = 0;+    virtual bool equals(const JsonValue* other) const = 0;+    virtual bool less(const JsonValue* other) const = 0;+    virtual void dump(std::string& out) const = 0;+    virtual double number_value() const;+    virtual int int_value() const;+    virtual long long long_value() const;+    virtual bool bool_value() const;+    virtual const std::string& string_value() const;+    virtual const Json::array& array_items() const;+    virtual const Json& operator[](size_t i) const;+    virtual const Json::object& object_items() const;+    virtual const Json& operator[](const std::string& key) const;+    virtual ~JsonValue() = default;+};++static const int max_depth = 200;++/* Helper for representing null - just a do-nothing struct, plus comparison+ * operators so the helpers in JsonValue work. We can't use nullptr_t because+ * it may not be orderable.+ */+struct NullStruct {+    bool operator==(NullStruct) const {+        return true;+    }+    bool operator<(NullStruct) const {+        return false;+    }+};++/* * * * * * * * * * * * * * * * * * * *+ * Serialization+ */++static void dump(NullStruct, std::string& out) {+    out += "null";+}++static void dump(double value, std::string& out) {+    if (std::isfinite(value)) {+        char buf[32];+        snprintf(buf, sizeof buf, "%.17g", value);+        out += buf;+    } else {+        out += "null";+    }+}++static void dump(long long value, std::string& out) {+    char buf[32];+    snprintf(buf, sizeof buf, "%lld", value);+    out += buf;+}++static void dump(bool value, std::string& out) {+    out += value ? "true" : "false";+}++static void dump(const std::string& value, std::string& out) {+    out += '"';+    for (size_t i = 0; i < value.length(); i++) {+        const char ch = value[i];+        if (ch == '\\') {+            out += "\\\\";+        } else if (ch == '"') {+            out += "\\\"";+        } else if (ch == '\b') {+            out += "\\b";+        } else if (ch == '\f') {+            out += "\\f";+        } else if (ch == '\n') {+            out += "\\n";+        } else if (ch == '\r') {+            out += "\\r";+        } else if (ch == '\t') {+            out += "\\t";+        } else if (static_cast<uint8_t>(ch) <= 0x1f) {+            char buf[8];+            snprintf(buf, sizeof buf, "\\u%04x", ch);+            out += buf;+        } else if (static_cast<uint8_t>(ch) == 0xe2 && static_cast<uint8_t>(value[i + 1]) == 0x80 &&+                   static_cast<uint8_t>(value[i + 2]) == 0xa8) {+            out += "\\u2028";+            i += 2;+        } else if (static_cast<uint8_t>(ch) == 0xe2 && static_cast<uint8_t>(value[i + 1]) == 0x80 &&+                   static_cast<uint8_t>(value[i + 2]) == 0xa9) {+            out += "\\u2029";+            i += 2;+        } else {+            out += ch;+        }+    }+    out += '"';+}++static void dump(const Json::array& values, std::string& out) {+    bool first = true;+    out += "[";+    for (const auto& value : values) {+        if (!first) out += ", ";+        value.dump(out);+        first = false;+    }+    out += "]";+}++static void dump(const Json::object& values, std::string& out) {+    bool first = true;+    out += "{";+    for (const auto& kv : values) {+        if (!first) out += ", ";+        dump(kv.first, out);+        out += ": ";+        kv.second.dump(out);+        first = false;+    }+    out += "}";+}++inline void Json::dump(std::string& out) const {+    m_ptr->dump(out);+}++/* * * * * * * * * * * * * * * * * * * *+ * Value wrappers+ */++template <Json::Type tag, typename T>+class Value : public JsonValue {+protected:+    // Constructors+    explicit Value(T value) : m_value(std::move(value)) {}++    // Get type tag+    Json::Type type() const override {+        return tag;+    }++    // Comparisons+    bool equals(const JsonValue* other) const override {+        return m_value == static_cast<const Value<tag, T>*>(other)->m_value;+    }+    bool less(const JsonValue* other) const override {+        return m_value < static_cast<const Value<tag, T>*>(other)->m_value;+    }++    const T m_value;+    void dump(std::string& out) const override {+        json11::dump(m_value, out);+    }+};++class JsonDouble final : public Value<Json::NUMBER, double> {+    double number_value() const override {+        return m_value;+    }+    int int_value() const override {+        return static_cast<int>(m_value);+    }+    long long long_value() const override {+        return static_cast<long long>(m_value);+    }+    bool equals(const JsonValue* other) const override {+        return m_value == other->number_value();+    }+    bool less(const JsonValue* other) const override {+        return m_value < other->number_value();+    }++public:+    explicit JsonDouble(double value) : Value(value) {}+};++class JsonInt final : public Value<Json::NUMBER, long long> {+    double number_value() const override {+        return m_value;+    }+    int int_value() const override {+        return m_value;+    }+    long long long_value() const override {+        return static_cast<long long>(m_value);+    }+    bool equals(const JsonValue* other) const override {+        return m_value == other->number_value();+    }+    bool less(const JsonValue* other) const override {+        return m_value < other->number_value();+    }++public:+    explicit JsonInt(int value) : Value(value) {}+};++class JsonBoolean final : public Value<Json::BOOL, bool> {+    bool bool_value() const override {+        return m_value;+    }++public:+    explicit JsonBoolean(bool value) : Value(value) {}+};++class JsonString final : public Value<Json::STRING, std::string> {+    const std::string& string_value() const override {+        return m_value;+    }++public:+    explicit JsonString(const std::string& value) : Value(value) {}+    explicit JsonString(std::string&& value) : Value(std::move(value)) {}+};++class JsonArray final : public Value<Json::ARRAY, Json::array> {+    const Json::array& array_items() const override {+        return m_value;+    }+    const Json& operator[](size_t i) const override;++public:+    explicit JsonArray(const Json::array& value) : Value(value) {}+    explicit JsonArray(Json::array&& value) : Value(std::move(value)) {}+};++class JsonObject final : public Value<Json::OBJECT, Json::object> {+    const Json::object& object_items() const override {+        return m_value;+    }+    const Json& operator[](const std::string& key) const override;++public:+    explicit JsonObject(const Json::object& value) : Value(value) {}+    explicit JsonObject(Json::object&& value) : Value(std::move(value)) {}+};++class JsonNull final : public Value<Json::NUL, NullStruct> {+public:+    JsonNull() : Value({}) {}+};++/* * * * * * * * * * * * * * * * * * * *+ * Static globals - static-init-safe+ */+struct Statics {+    const std::shared_ptr<JsonValue> null = std::make_shared<JsonNull>();+    const std::shared_ptr<JsonValue> t = std::make_shared<JsonBoolean>(true);+    const std::shared_ptr<JsonValue> f = std::make_shared<JsonBoolean>(false);+    const std::string empty_string{};+    const std::vector<Json> empty_vector{};+    const std::map<std::string, Json> empty_map{};+    Statics() = default;+};++static const Statics& statics() {+    static const Statics s{};+    return s;+}++static const Json& static_null() {+    // This has to be separate, not in Statics, because Json() accesses statics().null.+    static const Json json_null;+    return json_null;+}++/* * * * * * * * * * * * * * * * * * * *+ * Constructors+ */++inline Json::Json() noexcept : m_ptr(statics().null) {}+inline Json::Json(std::nullptr_t) noexcept : m_ptr(statics().null) {}+inline Json::Json(double value) : m_ptr(std::make_shared<JsonDouble>(value)) {}+inline Json::Json(long long value) : m_ptr(std::make_shared<JsonInt>(value)) {}+inline Json::Json(bool value) : m_ptr(value ? statics().t : statics().f) {}+inline Json::Json(const std::string& value) : m_ptr(std::make_shared<JsonString>(value)) {}+inline Json::Json(std::string&& value) : m_ptr(std::make_shared<JsonString>(std::move(value))) {}+inline Json::Json(const char* value) : m_ptr(std::make_shared<JsonString>(value)) {}+inline Json::Json(const Json::array& values) : m_ptr(std::make_shared<JsonArray>(values)) {}+inline Json::Json(Json::array&& values) : m_ptr(std::make_shared<JsonArray>(std::move(values))) {}+inline Json::Json(const Json::object& values) : m_ptr(std::make_shared<JsonObject>(values)) {}+inline Json::Json(Json::object&& values) : m_ptr(std::make_shared<JsonObject>(std::move(values))) {}++/* * * * * * * * * * * * * * * * * * * *+ * Accessors+ */++inline Json::Type Json::type() const {+    return m_ptr->type();+}+inline double Json::number_value() const {+    return m_ptr->number_value();+}+inline int Json::int_value() const {+    return m_ptr->int_value();+}+inline long long Json::long_value() const {+    return m_ptr->long_value();+}+inline bool Json::bool_value() const {+    return m_ptr->bool_value();+}+inline const std::string& Json::string_value() const {+    return m_ptr->string_value();+}+inline const std::vector<Json>& Json::array_items() const {+    return m_ptr->array_items();+}+inline const std::map<std::string, Json>& Json::object_items() const {+    return m_ptr->object_items();+}+inline const Json& Json::operator[](size_t i) const {+    return (*m_ptr)[i];+}+inline const Json& Json::operator[](const std::string& key) const {+    return (*m_ptr)[key];+}++inline double JsonValue::number_value() const {+    return 0;+}+inline int JsonValue::int_value() const {+    return 0;+}+inline long long JsonValue::long_value() const {+    return 0;+}+inline bool JsonValue::bool_value() const {+    return false;+}+inline const std::string& JsonValue::string_value() const {+    return statics().empty_string;+}+inline const std::vector<Json>& JsonValue::array_items() const {+    return statics().empty_vector;+}+inline const std::map<std::string, Json>& JsonValue::object_items() const {+    return statics().empty_map;+}+inline const Json& JsonValue::operator[](size_t) const {+    return static_null();+}+inline const Json& JsonValue::operator[](const std::string&) const {+    return static_null();+}++inline const Json& JsonObject::operator[](const std::string& key) const {+    auto iter = m_value.find(key);+    return (iter == m_value.end()) ? static_null() : iter->second;+}+inline const Json& JsonArray::operator[](size_t i) const {+    if (i >= m_value.size()) {+        return static_null();+    }+    return m_value[i];+}++/* * * * * * * * * * * * * * * * * * * *+ * Comparison+ */++inline bool Json::operator==(const Json& other) const {+    if (m_ptr == other.m_ptr) {+        return true;+    }+    if (m_ptr->type() != other.m_ptr->type()) {+        return false;+    }++    return m_ptr->equals(other.m_ptr.get());+}++inline bool Json::operator<(const Json& other) const {+    if (m_ptr == other.m_ptr) {+        return false;+    }+    if (m_ptr->type() != other.m_ptr->type()) {+        return m_ptr->type() < other.m_ptr->type();+    }++    return m_ptr->less(other.m_ptr.get());+}++/* * * * * * * * * * * * * * * * * * * *+ * Parsing+ */++/* esc(c)+ *+ * Format char c suitable for printing in an error message.+ */+static inline std::string esc(char c) {+    char buf[12];+    if (static_cast<uint8_t>(c) >= 0x20 && static_cast<uint8_t>(c) <= 0x7f) {+        snprintf(buf, sizeof buf, "'%c' (%d)", c, c);+    } else {+        snprintf(buf, sizeof buf, "(%d)", c);+    }+    return std::string(buf);+}++static inline bool in_range(long x, long lower, long upper) {+    return (x >= lower && x <= upper);+}++namespace {+/* JsonParser+ *+ * Object that tracks all state of an in-progress parse.+ */+struct JsonParser final {+    /* State+     */+    const std::string& str;+    size_t i;+    std::string& err;+    bool failed;+    const JsonParse strategy;++    /* fail(msg, err_ret = Json())+     *+     * Mark this parse as failed.+     */+    Json fail(std::string&& msg) {+        return fail(std::move(msg), Json());+    }++    template <typename T>+    T fail(std::string&& msg, T err_ret) {+        if (!failed) {+            err = std::move(msg);+        }+        failed = true;+        return err_ret;+    }++    /* consume_whitespace()+     *+     * Advance until the current character is non-whitespace.+     */+    void consume_whitespace() {+        while (str[i] == ' ' || str[i] == '\r' || str[i] == '\n' || str[i] == '\t') {+            i++;+        }+    }++    /* consume_comment()+     *+     * Advance comments (c-style inline and multiline).+     */+    bool consume_comment() {+        bool comment_found = false;+        if (str[i] == '/') {+            i++;+            if (i == str.size()) {+                return fail("unexpected end of input after start of comment", false);+            }+            if (str[i] == '/') {  // inline comment+                i++;+                // advance until next line, or end of input+                while (i < str.size() && str[i] != '\n') {+                    i++;+                }+                comment_found = true;+            } else if (str[i] == '*') {  // multiline comment+                i++;+                if (i > str.size() - 2)+                    return fail("unexpected end of input inside multi-line comment", false);+                // advance until closing tokens+                while (!(str[i] == '*' && str[i + 1] == '/')) {+                    i++;+                    if (i > str.size() - 2)+                        return fail("unexpected end of input inside multi-line comment", false);+                }+                i += 2;+                comment_found = true;+            } else {+                return fail("malformed comment", false);+            }+        }+        return comment_found;+    }++    /* consume_garbage()+     *+     * Advance until the current character is non-whitespace and non-comment.+     */+    void consume_garbage() {+        consume_whitespace();+        if (strategy == JsonParse::COMMENTS) {+            bool comment_found = false;+            do {+                comment_found = consume_comment();+                if (failed) {+                    return;+                }+                consume_whitespace();+            } while (comment_found);+        }+    }++    /* get_next_token()+     *+     * Return the next non-whitespace character. If the end of the input is reached,+     * flag an error and return 0.+     */+    char get_next_token() {+        consume_garbage();+        if (failed) {+            return static_cast<char>(0);+        }+        if (i == str.size()) {+            return fail("unexpected end of input", static_cast<char>(0));+        }++        return str[i++];+    }++    /* encode_utf8(pt, out)+     *+     * Encode pt as UTF-8 and add it to out.+     */+    void encode_utf8(long pt, std::string& out) {+        if (pt < 0) {+            return;+        }++        if (pt < 0x80) {+            out += static_cast<char>(pt);+        } else if (pt < 0x800) {+            out += static_cast<char>((pt >> 6) | 0xC0);+            out += static_cast<char>((pt & 0x3F) | 0x80);+        } else if (pt < 0x10000) {+            out += static_cast<char>((pt >> 12) | 0xE0);+            out += static_cast<char>(((pt >> 6) & 0x3F) | 0x80);+            out += static_cast<char>((pt & 0x3F) | 0x80);+        } else {+            out += static_cast<char>((pt >> 18) | 0xF0);+            out += static_cast<char>(((pt >> 12) & 0x3F) | 0x80);+            out += static_cast<char>(((pt >> 6) & 0x3F) | 0x80);+            out += static_cast<char>((pt & 0x3F) | 0x80);+        }+    }++    /* parse_string()+     *+     * Parse a std::string, starting at the current position.+     */+    std::string parse_string() {+        std::string out;+        long last_escaped_codepoint = -1;+        while (true) {+            if (i == str.size()) return fail("unexpected end of input in std::string", "");++            char ch = str[i++];++            if (ch == '"') {+                encode_utf8(last_escaped_codepoint, out);+                return out;+            }++            if (in_range(ch, 0, 0x1f)) {+                return fail("unescaped " + esc(ch) + " in std::string", "");+            }++            // The usual case: non-escaped characters+            if (ch != '\\') {+                encode_utf8(last_escaped_codepoint, out);+                last_escaped_codepoint = -1;+                out += ch;+                continue;+            }++            // Handle escapes+            if (i == str.size()) {+                return fail("unexpected end of input in std::string", "");+            }++            ch = str[i++];++            if (ch == 'u') {+                // Extract 4-byte escape sequence+                std::string esc = str.substr(i, 4);+                // Explicitly check length of the substring. The following loop+                // relies on std::string returning the terminating NUL when+                // accessing str[length]. Checking here reduces brittleness.+                if (esc.length() < 4) {+                    return fail("bad \\u escape: " + esc, "");+                }+                for (size_t j = 0; j < 4; j++) {+                    if (!in_range(esc[j], 'a', 'f') && !in_range(esc[j], 'A', 'F') &&+                            !in_range(esc[j], '0', '9'))+                        return fail("bad \\u escape: " + esc, "");+                }++                long codepoint = strtol(esc.data(), nullptr, 16);++                // JSON specifies that characters outside the BMP shall be encoded as a pair+                // of 4-hex-digit \u escapes encoding their surrogate pair components. Check+                // whether we're in the middle of such a beast: the previous codepoint was an+                // escaped lead (high) surrogate, and this is a trail (low) surrogate.+                if (in_range(last_escaped_codepoint, 0xD800, 0xDBFF) && in_range(codepoint, 0xDC00, 0xDFFF)) {+                    // Reassemble the two surrogate pairs into one astral-plane character, per+                    // the UTF-16 algorithm.+                    encode_utf8((((last_escaped_codepoint - 0xD800) << 10) | (codepoint - 0xDC00)) + 0x10000,+                            out);+                    last_escaped_codepoint = -1;+                } else {+                    encode_utf8(last_escaped_codepoint, out);+                    last_escaped_codepoint = codepoint;+                }++                i += 4;+                continue;+            }++            encode_utf8(last_escaped_codepoint, out);+            last_escaped_codepoint = -1;++            if (ch == 'b') {+                out += '\b';+            } else if (ch == 'f') {+                out += '\f';+            } else if (ch == 'n') {+                out += '\n';+            } else if (ch == 'r') {+                out += '\r';+            } else if (ch == 't') {+                out += '\t';+            } else if (ch == '"' || ch == '\\' || ch == '/') {+                out += ch;+            } else {+                return fail("invalid escape character " + esc(ch), "");+            }+        }+    }++    /* parse_number()+     *+     * Parse a double.+     */+    Json parse_number() {+        size_t start_pos = i;++        if (str[i] == '-') {+            i++;+        }++        // Integer part+        if (str[i] == '0') {+            i++;+            if (in_range(str[i], '0', '9')) {+                return fail("leading 0s not permitted in numbers");+            }+        } else if (in_range(str[i], '1', '9')) {+            i++;+            while (in_range(str[i], '0', '9')) {+                i++;+            }+        } else {+            return fail("invalid " + esc(str[i]) + " in number");+        }++        if (str[i] != '.' && str[i] != 'e' && str[i] != 'E' &&+                (i - start_pos) <= static_cast<size_t>(std::numeric_limits<int>::digits10)) {+            return std::atoll(str.c_str() + start_pos);+        }++        // Decimal part+        if (str[i] == '.') {+            i++;+            if (!in_range(str[i], '0', '9')) {+                return fail("at least one digit required in fractional part");+            }++            while (in_range(str[i], '0', '9')) {+                i++;+            }+        }++        // Exponent part+        if (str[i] == 'e' || str[i] == 'E') {+            i++;++            if (str[i] == '+' || str[i] == '-') {+                i++;+            }++            if (!in_range(str[i], '0', '9')) {+                return fail("at least one digit required in exponent");+            }++            while (in_range(str[i], '0', '9')) {+                i++;+            }+        }++        return std::strtod(str.c_str() + start_pos, nullptr);+    }++    /* expect(str, res)+     *+     * Expect that 'str' starts at the character that was just read. If it does, advance+     * the input and return res. If not, flag an error.+     */+    Json expect(const std::string& expected, Json res) {+        assert(i != 0);+        i--;+        if (str.compare(i, expected.length(), expected) == 0) {+            i += expected.length();+            return res;+        } else {+            return fail("parse error: expected " + expected + ", got " + str.substr(i, expected.length()));+        }+    }++    /* parse_json()+     *+     * Parse a JSON object.+     */+    Json parse_json(int depth) {+        if (depth > max_depth) {+            return fail("exceeded maximum nesting depth");+        }++        char ch = get_next_token();+        if (failed) {+            return Json();+        }++        if (ch == '-' || (ch >= '0' && ch <= '9')) {+            i--;+            return parse_number();+        }++        if (ch == 't') {+            return expect("true", true);+        }++        if (ch == 'f') {+            return expect("false", false);+        }++        if (ch == 'n') {+            return expect("null", Json());+        }++        if (ch == '"') {+            return parse_string();+        }++        if (ch == '{') {+            std::map<std::string, Json> data;+            ch = get_next_token();+            if (ch == '}') {+                return data;+            }++            while (true) {+                if (ch != '"') return fail("expected '\"' in object, got " + esc(ch));++                std::string key = parse_string();+                if (failed) {+                    return Json();+                }++                ch = get_next_token();+                if (ch != ':') {+                    return fail("expected ':' in object, got " + esc(ch));+                }++                data[std::move(key)] = parse_json(depth + 1);+                if (failed) {+                    return Json();+                }++                ch = get_next_token();+                if (ch == '}') {+                    break;+                }+                if (ch != ',') {+                    return fail("expected ',' in object, got " + esc(ch));+                }++                ch = get_next_token();+            }+            return data;+        }++        if (ch == '[') {+            std::vector<Json> data;+            ch = get_next_token();+            if (ch == ']') {+                return data;+            }++            while (true) {+                i--;+                data.push_back(parse_json(depth + 1));+                if (failed) {+                    return Json();+                }++                ch = get_next_token();+                if (ch == ']') {+                    break;+                }+                if (ch != ',') {+                    return fail("expected ',' in list, got " + esc(ch));+                }++                ch = get_next_token();+                (void)ch;+            }+            return data;+        }++        return fail("expected value, got " + esc(ch));+    }+};+}  // namespace++inline Json Json::parse(const std::string& in, std::string& err, JsonParse strategy) {+    JsonParser parser{in, 0, err, false, strategy};+    Json result = parser.parse_json(0);++    // Check for any trailing garbage+    parser.consume_garbage();+    if (parser.failed) {+        return Json();+    }+    if (parser.i != in.size()) {+        return parser.fail("unexpected trailing " + esc(in[parser.i]));+    }++    return result;+}++inline std::vector<Json> parse_multi(const std::string& in, std::string::size_type& parser_stop_pos,+        std::string& err, JsonParse strategy) {+    JsonParser parser{in, 0, err, false, strategy};+    parser_stop_pos = 0;+    std::vector<Json> json_vec;+    while (parser.i != in.size() && !parser.failed) {+        json_vec.push_back(parser.parse_json(0));+        if (parser.failed) {+            break;+        }++        // Check for another object+        parser.consume_garbage();+        if (parser.failed) {+            break;+        }+        parser_stop_pos = parser.i;+    }+    return json_vec;+}++#ifdef _MSC_VER+#pragma warning(default : 4244)+#endif  // _MSC_VER++}  // namespace json11
cbits/souffle/utility/tinyformat.h view
@@ -179,6 +179,10 @@ #   define TINYFORMAT_HIDDEN #endif +#ifdef _MSC_VER+#pragma warning(disable : 4127)+#endif  // _MSC_VER+ namespace tinyformat {  //------------------------------------------------------------------------------@@ -1141,6 +1145,10 @@   } // namespace tinyformat++#ifdef _MSC_VER+#pragma warning(default : 4127)+#endif  // _MSC_VER  #endif // TINYFORMAT_H_INCLUDED
lib/Language/Souffle/Compiled.hs view
@@ -50,7 +50,8 @@  -- | A monad for executing Souffle-related actions in. newtype SouffleM a = SouffleM (IO a)-  deriving ( Functor, Applicative, Monad, MonadIO ) via IO+  deriving (Functor, Applicative, Monad, MonadIO) via IO+  deriving (Semigroup, Monoid) via (IO a)  {- | Initializes and runs a Souffle program.
lib/Language/Souffle/Experimental.hs view
@@ -1,7 +1,7 @@-{-# LANGUAGE GADTs, RankNTypes, TypeFamilies, DataKinds, TypeOperators, ConstraintKinds #-}+{-# LANGUAGE GADTs, RankNTypes, DataKinds, TypeOperators, ConstraintKinds #-} {-# LANGUAGE UndecidableInstances, UndecidableSuperClasses, FlexibleContexts #-} {-# LANGUAGE MultiParamTypeClasses, FlexibleInstances, DerivingVia, ScopedTypeVariables #-}-{-# LANGUAGE PolyKinds #-}+{-# LANGUAGE PolyKinds, TypeFamilyDependencies #-} {-# OPTIONS_GHC -Wno-redundant-constraints #-}  {-| This module provides an experimental DSL for generating Souffle Datalog code,
lib/Language/Souffle/Interpreted.hs view
@@ -32,7 +32,6 @@ import Control.DeepSeq (deepseq) import Control.Exception (ErrorCall(..), throwIO, bracket) import Control.Monad.State.Strict-import Control.Monad.Reader import Data.IORef import Data.Foldable (traverse_) import Data.List hiding (init)@@ -56,10 +55,9 @@   -- | A monad for executing Souffle-related actions in.-newtype SouffleM a-  = SouffleM (ReaderT Config IO a)-  deriving (Functor, Applicative, Monad, MonadIO)-  via (ReaderT Config IO)+newtype SouffleM a = SouffleM (IO a)+  deriving (Functor, Applicative, Monad, MonadIO) via IO+  deriving (Semigroup, Monoid) via (IO a)  -- | A helper data type for storing the configurable settings of the --   interpreter.@@ -135,7 +133,7 @@   :: Program prog => Config -> prog -> (Maybe (Handle prog) -> SouffleM a) -> IO a runSouffleWith cfg program f = bracket initialize maybeCleanup $ \handle -> do   let (SouffleM action) = f handle-  runReaderT action cfg+  action   where     initialize = datalogProgramFile program (cfgDatalogDir cfg) >>= \case       Nothing -> pure Nothing
lib/Language/Souffle/Marshal.hs view
@@ -42,9 +42,9 @@ See also: 'MonadPush', 'Marshal'. -} class Monad m => MonadPop m where-  -- | Unmarshals a signed 32 bir integer from the datalog side.+  -- | Unmarshals a signed 32 bit integer from the datalog side.   popInt32 :: m Int32-  -- | Unmarshals an unsigned 32 bir integer from the datalog side.+  -- | Unmarshals an unsigned 32 bit integer from the datalog side.   popUInt32 :: m Word32   -- | Unmarshals a float from the datalog side.   popFloat :: m Float
− scripts/import_souffle_headers.hs 
@@ -1,180 +0,0 @@-{-# LANGUAGE DataKinds, TypeFamilies, DeriveGeneric, DeriveAnyClass, TypeApplications #-}--module Main ( main ) where---- NOTE: This is a helper script for importing all Souffle headers into--- this repository. This is done in order to make Haskell libraries that--- use souffle-haskell "self-contained", meaning users of the packages--- using those libraries are not required to have souffle (headers) installed.--import System.Directory-import System.FilePath-import System.Process-import Control.Monad-import Control.Monad.Extra-import Control.Applicative-import Data.List-import Data.Maybe-import Data.Void-import GHC.Generics-import qualified Text.Megaparsec as P-import qualified Text.Megaparsec.Char as P-import qualified Language.Souffle.Interpreted as Souffle-import Language.Souffle.Experimental---data Includes = Includes FilePath FilePath-  deriving (Eq, Show, Generic, Souffle.Marshal, FactMetadata)--data TransitivelyIncludes = TransitivelyIncludes FilePath FilePath-  deriving (Eq, Show, Generic, Souffle.Marshal, FactMetadata)--newtype TopLevelInclude = TopLevelInclude FilePath-  deriving (Eq, Show, Generic, Souffle.Marshal, FactMetadata)--newtype RequiredInclude = RequiredInclude FilePath-  deriving (Eq, Show, Generic, Souffle.Marshal, FactMetadata)--data Handle = Handle--instance Souffle.Program Handle where-  type ProgramFacts Handle =-    [ TopLevelInclude-    , Includes-    , TransitivelyIncludes-    , RequiredInclude-    ]-  programName = const "required_include"--instance Souffle.Fact Includes where-  type FactDirection Includes = 'Souffle.Input-  factName = const "includes"--instance Souffle.Fact TransitivelyIncludes where-  type FactDirection TransitivelyIncludes = 'Souffle.Internal-  factName = const "transitively_includes"--instance Souffle.Fact TopLevelInclude where-  type FactDirection TopLevelInclude = 'Souffle.Input-  factName = const "top_level_include"--instance Souffle.Fact RequiredInclude where-  type FactDirection RequiredInclude = 'Souffle.Output-  factName = const "required_include"--dlProgram :: DSL Handle 'Definition ()-dlProgram = do-  Predicate includes <- predicateFor @Includes-  Predicate transitivelyIncludes <- predicateFor @TransitivelyIncludes-  Predicate topLevelInclude <- predicateFor @TopLevelInclude-  Predicate requiredInclude <- predicateFor @RequiredInclude--  file1 <- var "file1"-  file2 <- var "file2"-  file3 <- var "file3"--  requiredInclude(file1) |--    topLevelInclude(file1)-  requiredInclude(file1) |- do-    topLevelInclude(file2)-    transitivelyIncludes(file2, file1)--  transitivelyIncludes(file1, file2) |--    includes(file1, file2)-  transitivelyIncludes(file1, file2) |- do-    includes(file1, file3)-    transitivelyIncludes(file3, file2)---run :: String -> IO ()-run = callCommand--runWithResult :: String -> IO String-runWithResult s = case words s of-  (program:args) -> readProcess program args ""-  _ -> error "Passed empty string to 'runWithResult'. Aborting."--headerDir :: FilePath-headerDir = "cbits/souffle/"--main :: IO ()-main = do-  pwd <- getCurrentDirectory-  gitRoot <- getGitRootDirectory-  when (pwd /= gitRoot) $-    putStrLn "You need to run this script in the root directory of this repo! Aborting."--  run "git submodule update --init --recursive"-  run $ "rm -rf " <> headerDir <> "*.h"-  run $ "cp souffle/LICENSE " <> headerDir-  files <- copyHeaders-  putStrLn "Replace 'install-includes' in your package.yaml with the following:"-  putStrLn . unlines $ map ("  - souffle/" <>) files--getGitRootDirectory :: IO FilePath-getGitRootDirectory =-  filter (/= '\n') <$> runWithResult "git rev-parse --show-toplevel"--parseIncludes :: String -> [FilePath]-parseIncludes s = either (const []) catMaybes $ P.runParser parser "" s where-  parser :: P.Parsec Void String [Maybe FilePath]-  parser = many (includeParser <|> skipRestOfLine) <* P.eof-  includeParser = do-    P.chunk "#include" *> P.space1-    P.lookAhead (P.char '"' <|> P.char '<') >>= \case-      '"' -> do-        include <- P.between quotes quotes $ P.takeWhile1P Nothing (/= '"')-        void skipRestOfLine-        pure $ Just include-      _ -> pure Nothing-  quotes = P.char '"'-  skipRestOfLine = Nothing <$ (P.takeWhileP Nothing (/= '\n') *> P.newline)--parseIncludesInHeader :: FilePath -> IO [Includes]-parseIncludesInHeader file = f <$> readFile file where-  f = map ((file `Includes`) . normalizeFilePath dir) . parseIncludes-  dir = takeDirectory file--normalizeFilePath :: FilePath -> FilePath -> FilePath-normalizeFilePath dir file = normalize $ dir </> file' where-  file' = if "souffle/" `isPrefixOf` file then file \\ "souffle/" else file-  normalize = withExplodedPath (reverse . removeParentDirRefs . reverse)-  removeParentDirRefs = \case-    ("../":_:xs) -> removeParentDirRefs xs-    (x:xs) -> x:removeParentDirRefs xs-    [] -> []--copyHeaders :: IO [FilePath]-copyHeaders = do-  headers <- filter (".h" `isSuffixOf`) . lines-          <$> runWithResult "find souffle -type f"-  includes <- concatMapM parseIncludesInHeader headers-  traverse copyHeader =<< computeRequiredIncludes includes--computeRequiredIncludes :: [Includes] -> IO [FilePath]-computeRequiredIncludes includes = do-  requiredIncludes <- runSouffleInterpreted Handle dlProgram $ \case-    Nothing -> error "Failed to load Souffle program. Aborting."-    Just prog -> do-      Souffle.addFacts prog [ TopLevelInclude "souffle/src/SouffleInterface.h"-                            , TopLevelInclude "souffle/src/CompiledSouffle.h"-                            ]-      Souffle.addFacts prog includes-      Souffle.run prog-      Souffle.getFacts prog-  pure $ map (\(RequiredInclude include) -> include) requiredIncludes--copyHeader :: FilePath -> IO FilePath-copyHeader file = do-  header <- head . filter (file `isSuffixOf`) . lines-        <$> runWithResult "find souffle/ -type f"-  let header' = withExplodedPath (drop 2) header-      dir = headerDir </> takeDirectory header'-      destination = replaceDirectory header' dir-  createDirectoryIfMissing True dir-  copyFile header destination-  pure header'--withExplodedPath :: ([FilePath] -> [FilePath]) -> FilePath -> FilePath-withExplodedPath f = joinPath . f . splitPath-
souffle-haskell.cabal view
@@ -4,10 +4,10 @@ -- -- see: https://github.com/sol/hpack ----- hash: a4e6da289ecdafada838f9dc73d43b5f69a8411c5c7adb647efaefe5426dff62+-- hash: 282b7f644a46aaacc10fd325f80d88a1529d0ea5737403640996967d6b523a4c  name:           souffle-haskell-version:        2.0.1+version:        2.1.0 synopsis:       Souffle Datalog bindings for Haskell description:    Souffle Datalog bindings for Haskell. category:       Logic Programming, Foreign Binding, Bindings@@ -24,51 +24,42 @@     CHANGELOG.md     LICENSE     cbits/souffle.h-    cbits/souffle/Brie.h-    cbits/souffle/BTree.h-    cbits/souffle/CompiledOptions.h     cbits/souffle/CompiledSouffle.h     cbits/souffle/CompiledTuple.h-    cbits/souffle/EquivalenceRelation.h-    cbits/souffle/EventProcessor.h-    cbits/souffle/gzfstream.h-    cbits/souffle/IOSystem.h-    cbits/souffle/json11.h-    cbits/souffle/LambdaBTree.h-    cbits/souffle/Logger.h-    cbits/souffle/PiggyList.h-    cbits/souffle/profile/CellInterface.h-    cbits/souffle/profile/DataComparator.h-    cbits/souffle/profile/Row.h-    cbits/souffle/profile/Table.h-    cbits/souffle/ProfileDatabase.h-    cbits/souffle/ProfileEvent.h+    cbits/souffle/datastructure/Brie.h+    cbits/souffle/datastructure/BTree.h+    cbits/souffle/datastructure/EquivalenceRelation.h+    cbits/souffle/datastructure/LambdaBTree.h+    cbits/souffle/datastructure/PiggyList.h+    cbits/souffle/datastructure/Table.h+    cbits/souffle/datastructure/UnionFind.h+    cbits/souffle/io/gzfstream.h+    cbits/souffle/io/IOSystem.h+    cbits/souffle/io/ReadStream.h+    cbits/souffle/io/ReadStreamCSV.h+    cbits/souffle/io/ReadStreamJSON.h+    cbits/souffle/io/ReadStreamSQLite.h+    cbits/souffle/io/SerialisationStream.h+    cbits/souffle/io/WriteStream.h+    cbits/souffle/io/WriteStreamCSV.h+    cbits/souffle/io/WriteStreamJSON.h+    cbits/souffle/io/WriteStreamSQLite.h     cbits/souffle/RamTypes.h-    cbits/souffle/ReadStream.h-    cbits/souffle/ReadStreamCSV.h-    cbits/souffle/ReadStreamJSON.h-    cbits/souffle/ReadStreamSQLite.h     cbits/souffle/RecordTable.h-    cbits/souffle/SerialisationStream.h     cbits/souffle/SignalHandler.h     cbits/souffle/SouffleInterface.h     cbits/souffle/SymbolTable.h-    cbits/souffle/Table.h-    cbits/souffle/UnionFind.h     cbits/souffle/utility/CacheUtil.h     cbits/souffle/utility/ContainerUtil.h     cbits/souffle/utility/EvaluatorUtil.h     cbits/souffle/utility/FileUtil.h     cbits/souffle/utility/FunctionalUtil.h+    cbits/souffle/utility/json11.h     cbits/souffle/utility/MiscUtil.h     cbits/souffle/utility/ParallelUtil.h     cbits/souffle/utility/StreamUtil.h     cbits/souffle/utility/StringUtil.h     cbits/souffle/utility/tinyformat.h-    cbits/souffle/WriteStream.h-    cbits/souffle/WriteStreamCSV.h-    cbits/souffle/WriteStreamJSON.h-    cbits/souffle/WriteStreamSQLite.h     cbits/souffle/LICENSE  source-repository head@@ -92,53 +83,48 @@   hs-source-dirs:       lib   default-extensions: OverloadedStrings LambdaCase ScopedTypeVariables-  ghc-options: -Wall -Weverything -Wno-safe -Wno-unsafe -Wno-implicit-prelude -Wno-missed-specializations -Wno-all-missed-specializations -Wno-missing-import-lists -Wno-type-defaults -Wno-missing-local-signatures -Wno-monomorphism-restriction -Wno-missing-deriving-strategies -optP-Wno-nonportable-include-path -fhide-source-paths -fno-show-valid-hole-fits -fno-sort-valid-hole-fits+  ghc-options: -Wall -Weverything -Wno-safe -Wno-unsafe -Wno-implicit-prelude -Wno-missed-specializations -Wno-all-missed-specializations -Wno-missing-import-lists -Wno-type-defaults -Wno-missing-local-signatures -Wno-monomorphism-restriction -Wno-missing-deriving-strategies -Wno-prepositive-qualified-module -Wno-missing-safe-haskell-mode -optP-Wno-nonportable-include-path -fhide-source-paths -fno-show-valid-hole-fits -fno-sort-valid-hole-fits   cxx-options: -std=c++17 -Wall   include-dirs:       cbits       cbits/souffle   install-includes:+      souffle/CompiledSouffle.h+      souffle/CompiledTuple.h       souffle/RamTypes.h-      souffle/utility/MiscUtil.h+      souffle/RecordTable.h+      souffle/SignalHandler.h+      souffle/SouffleInterface.h       souffle/SymbolTable.h-      souffle/json11.h-      souffle/utility/FunctionalUtil.h+      souffle/utility/MiscUtil.h+      souffle/utility/tinyformat.h+      souffle/utility/ParallelUtil.h       souffle/utility/StreamUtil.h-      souffle/utility/StringUtil.h-      souffle/ReadStreamCSV.h-      souffle/ReadStream.h       souffle/utility/ContainerUtil.h-      souffle/utility/FileUtil.h-      souffle/gzfstream.h-      souffle/CompiledTuple.h-      souffle/RecordTable.h-      souffle/Brie.h+      souffle/datastructure/Brie.h       souffle/utility/CacheUtil.h-      souffle/SouffleInterface.h-      souffle/EventProcessor.h-      souffle/ProfileDatabase.h-      souffle/WriteStreamSQLite.h-      souffle/WriteStream.h-      souffle/LambdaBTree.h-      souffle/BTree.h-      souffle/utility/ParallelUtil.h-      souffle/WriteStreamJSON.h-      souffle/WriteStreamCSV.h-      souffle/IOSystem.h-      souffle/ReadStreamJSON.h-      souffle/ReadStreamSQLite.h-      souffle/UnionFind.h-      souffle/PiggyList.h-      souffle/ProfileEvent.h-      souffle/SerialisationStream.h-      souffle/CompiledSouffle.h-      souffle/EquivalenceRelation.h-      souffle/SignalHandler.h-      souffle/Table.h+      souffle/datastructure/EquivalenceRelation.h+      souffle/datastructure/LambdaBTree.h+      souffle/datastructure/BTree.h+      souffle/datastructure/PiggyList.h+      souffle/datastructure/UnionFind.h+      souffle/datastructure/Table.h+      souffle/io/IOSystem.h+      souffle/io/ReadStream.h+      souffle/io/SerialisationStream.h+      souffle/utility/json11.h+      souffle/utility/StringUtil.h+      souffle/io/ReadStreamCSV.h+      souffle/utility/FileUtil.h+      souffle/io/gzfstream.h+      souffle/io/ReadStreamJSON.h+      souffle/io/WriteStream.h+      souffle/io/WriteStreamCSV.h+      souffle/io/WriteStreamJSON.h+      souffle/io/ReadStreamSQLite.h+      souffle/io/WriteStreamSQLite.h       souffle/utility/EvaluatorUtil.h-      souffle/CompiledOptions.h-      souffle/Logger.h-      souffle/utility/tinyformat.h+      souffle/utility/FunctionalUtil.h   cxx-sources:       cbits/souffle.cpp   build-depends:@@ -160,79 +146,6 @@         stdc++   default-language: Haskell2010 -executable import-souffle-headers-  main-is: import_souffle_headers.hs-  other-modules:-      Paths_souffle_haskell-  hs-source-dirs:-      scripts-  default-extensions: OverloadedStrings LambdaCase ScopedTypeVariables-  ghc-options: -Wall -Weverything -Wno-safe -Wno-unsafe -Wno-implicit-prelude -Wno-missed-specializations -Wno-all-missed-specializations -Wno-missing-import-lists -Wno-type-defaults -Wno-missing-local-signatures -Wno-monomorphism-restriction -Wno-missing-deriving-strategies -optP-Wno-nonportable-include-path -fhide-source-paths -fno-show-valid-hole-fits -fno-sort-valid-hole-fits-  cxx-options: -std=c++17-  include-dirs:-      cbits-      cbits/souffle-  install-includes:-      souffle/RamTypes.h-      souffle/utility/MiscUtil.h-      souffle/SymbolTable.h-      souffle/json11.h-      souffle/utility/FunctionalUtil.h-      souffle/utility/StreamUtil.h-      souffle/utility/StringUtil.h-      souffle/ReadStreamCSV.h-      souffle/ReadStream.h-      souffle/utility/ContainerUtil.h-      souffle/utility/FileUtil.h-      souffle/gzfstream.h-      souffle/CompiledTuple.h-      souffle/RecordTable.h-      souffle/Brie.h-      souffle/utility/CacheUtil.h-      souffle/SouffleInterface.h-      souffle/EventProcessor.h-      souffle/ProfileDatabase.h-      souffle/WriteStreamSQLite.h-      souffle/WriteStream.h-      souffle/LambdaBTree.h-      souffle/BTree.h-      souffle/utility/ParallelUtil.h-      souffle/WriteStreamJSON.h-      souffle/WriteStreamCSV.h-      souffle/IOSystem.h-      souffle/ReadStreamJSON.h-      souffle/ReadStreamSQLite.h-      souffle/UnionFind.h-      souffle/PiggyList.h-      souffle/ProfileEvent.h-      souffle/SerialisationStream.h-      souffle/CompiledSouffle.h-      souffle/EquivalenceRelation.h-      souffle/SignalHandler.h-      souffle/Table.h-      souffle/utility/EvaluatorUtil.h-      souffle/CompiledOptions.h-      souffle/Logger.h-      souffle/utility/tinyformat.h-  build-depends:-      array <=1.0-    , base >=4.12 && <5-    , containers >=0.6.2.1 && <1-    , deepseq >=1.4.4 && <2-    , directory >=1.3.3 && <2-    , extra >=1.6.18 && <2-    , filepath >=1.4.2 && <2-    , megaparsec >=7.0.5 && <8-    , mtl >=2.0 && <3-    , process >=1.6 && <2-    , souffle-haskell-    , template-haskell >=2 && <3-    , temporary >=1.3 && <2-    , text >=1.0 && <2-    , type-errors-pretty >=0.0.1.0 && <1-    , vector <=1.0-  default-language: Haskell2010- test-suite souffle-haskell-test   type: exitcode-stdio-1.0   main-is: test.hs@@ -247,53 +160,48 @@   hs-source-dirs:       tests   default-extensions: OverloadedStrings LambdaCase ScopedTypeVariables-  ghc-options: -Wall -Weverything -Wno-safe -Wno-unsafe -Wno-implicit-prelude -Wno-missed-specializations -Wno-all-missed-specializations -Wno-missing-import-lists -Wno-type-defaults -Wno-missing-local-signatures -Wno-monomorphism-restriction -Wno-missing-deriving-strategies -optP-Wno-nonportable-include-path -fhide-source-paths -fno-show-valid-hole-fits -fno-sort-valid-hole-fits+  ghc-options: -Wall -Weverything -Wno-safe -Wno-unsafe -Wno-implicit-prelude -Wno-missed-specializations -Wno-all-missed-specializations -Wno-missing-import-lists -Wno-type-defaults -Wno-missing-local-signatures -Wno-monomorphism-restriction -Wno-missing-deriving-strategies -Wno-prepositive-qualified-module -Wno-missing-safe-haskell-mode -optP-Wno-nonportable-include-path -fhide-source-paths -fno-show-valid-hole-fits -fno-sort-valid-hole-fits   cxx-options: -std=c++17 -D__EMBEDDED_SOUFFLE__   include-dirs:       cbits       cbits/souffle   install-includes:+      souffle/CompiledSouffle.h+      souffle/CompiledTuple.h       souffle/RamTypes.h-      souffle/utility/MiscUtil.h+      souffle/RecordTable.h+      souffle/SignalHandler.h+      souffle/SouffleInterface.h       souffle/SymbolTable.h-      souffle/json11.h-      souffle/utility/FunctionalUtil.h+      souffle/utility/MiscUtil.h+      souffle/utility/tinyformat.h+      souffle/utility/ParallelUtil.h       souffle/utility/StreamUtil.h-      souffle/utility/StringUtil.h-      souffle/ReadStreamCSV.h-      souffle/ReadStream.h       souffle/utility/ContainerUtil.h-      souffle/utility/FileUtil.h-      souffle/gzfstream.h-      souffle/CompiledTuple.h-      souffle/RecordTable.h-      souffle/Brie.h+      souffle/datastructure/Brie.h       souffle/utility/CacheUtil.h-      souffle/SouffleInterface.h-      souffle/EventProcessor.h-      souffle/ProfileDatabase.h-      souffle/WriteStreamSQLite.h-      souffle/WriteStream.h-      souffle/LambdaBTree.h-      souffle/BTree.h-      souffle/utility/ParallelUtil.h-      souffle/WriteStreamJSON.h-      souffle/WriteStreamCSV.h-      souffle/IOSystem.h-      souffle/ReadStreamJSON.h-      souffle/ReadStreamSQLite.h-      souffle/UnionFind.h-      souffle/PiggyList.h-      souffle/ProfileEvent.h-      souffle/SerialisationStream.h-      souffle/CompiledSouffle.h-      souffle/EquivalenceRelation.h-      souffle/SignalHandler.h-      souffle/Table.h+      souffle/datastructure/EquivalenceRelation.h+      souffle/datastructure/LambdaBTree.h+      souffle/datastructure/BTree.h+      souffle/datastructure/PiggyList.h+      souffle/datastructure/UnionFind.h+      souffle/datastructure/Table.h+      souffle/io/IOSystem.h+      souffle/io/ReadStream.h+      souffle/io/SerialisationStream.h+      souffle/utility/json11.h+      souffle/utility/StringUtil.h+      souffle/io/ReadStreamCSV.h+      souffle/utility/FileUtil.h+      souffle/io/gzfstream.h+      souffle/io/ReadStreamJSON.h+      souffle/io/WriteStream.h+      souffle/io/WriteStreamCSV.h+      souffle/io/WriteStreamJSON.h+      souffle/io/ReadStreamSQLite.h+      souffle/io/WriteStreamSQLite.h       souffle/utility/EvaluatorUtil.h-      souffle/CompiledOptions.h-      souffle/Logger.h-      souffle/utility/tinyformat.h+      souffle/utility/FunctionalUtil.h   cxx-sources:       tests/fixtures/path.cpp       tests/fixtures/round_trip.cpp
tests/Test/Language/Souffle/CompiledSpec.hs view
@@ -20,7 +20,7 @@   deriving (Eq, Show, Generic)  instance Souffle.Program Path where-  type ProgramFacts Path = [Edge, Reachable]+  type ProgramFacts Path = '[Edge, Reachable]   programName = const "path"  instance Souffle.Fact Edge where@@ -172,3 +172,40 @@       reachable `shouldBe` Just (Reachable "a" "c")    -- TODO writeFiles / loadFiles++  describe "Semigroup and Monoid instances" $ parallel $ do+    it "combines Souffle actions into one using (<>)" $ do+      edges <- Souffle.runSouffle Path $ \handle -> do+        let prog = fromJust handle+            action1 = Souffle.addFact prog $ Edge "e" "f"+            action2 = Souffle.addFact prog $ Edge "f" "g"+            action = action1 <> action2+        action+        Souffle.run prog+        Souffle.getFacts prog+      edges `shouldBe` [ Edge "f" "g", Edge "e" "f"+                       , Edge "b" "c", Edge "a" "b"+                       ]++    it "supports mempty" $ do+      edges <- Souffle.runSouffle Path $ \handle -> do+        let prog = fromJust handle+            action = Souffle.addFact prog $ Edge "e" "f"+            action' = action <> mempty+        action'+        Souffle.run prog+        Souffle.getFacts prog+      edges `shouldBe` [Edge "e" "f", Edge "b" "c", Edge "a" "b"]++    it "supports foldMap" $ do+      edges <- Souffle.runSouffle Path $ \handle -> do+        let prog = fromJust handle+            fact1 = Edge "e" "f"+            fact2 = Edge "f" "g"+            action = foldMap (Souffle.addFact prog) [fact1, fact2]+        action+        Souffle.run prog+        Souffle.getFacts prog+      edges `shouldBe` [ Edge "f" "g", Edge "e" "f"+                       , Edge "b" "c", Edge "a" "b"+                       ]
tests/Test/Language/Souffle/InterpretedSpec.hs view
@@ -218,3 +218,41 @@         pure (e, r)       edge `shouldBe` Just (Edge "a" "b")       reachable `shouldBe` Just (Reachable "a" "c")++  describe "Semigroup and Monoid instances" $ parallel $ do+    it "combines Souffle actions into one using (<>)" $ do+      edges <- Souffle.runSouffle Path $ \handle -> do+        let prog = fromJust handle+            action1 = Souffle.addFact prog $ Edge "e" "f"+            action2 = Souffle.addFact prog $ Edge "f" "g"+            action = action1 <> action2+        action+        Souffle.run prog+        Souffle.getFacts prog+      edges `shouldBe` [ Edge "a" "b", Edge "b" "c"+                       , Edge "e" "f", Edge "f" "g"+                       ]++    it "supports mempty" $ do+      edges <- Souffle.runSouffle Path $ \handle -> do+        let prog = fromJust handle+            action = Souffle.addFact prog $ Edge "e" "f"+            action' = action <> mempty+        action'+        Souffle.run prog+        Souffle.getFacts prog+      edges `shouldBe` [Edge "a" "b", Edge "b" "c", Edge "e" "f"]++    it "supports foldMap" $ do+      edges <- Souffle.runSouffle Path $ \handle -> do+        let prog = fromJust handle+            fact1 = Edge "e" "f"+            fact2 = Edge "f" "g"+            action = foldMap (Souffle.addFact prog) [fact1, fact2]+        action+        Souffle.run prog+        Souffle.getFacts prog+      edges `shouldBe` [ Edge "a" "b", Edge "b" "c"+                       , Edge "e" "f", Edge "f" "g"+                       ]+
tests/fixtures/path.cpp view
@@ -6,24 +6,25 @@  namespace souffle { static const RamDomain RAM_BIT_SHIFT_MASK = RAM_DOMAIN_SIZE - 1;-struct t_btree_2__0_1__01__11 {+struct t_btree_ii__0_1__11__10 { using t_tuple = Tuple<RamDomain, 2>; struct t_comparator_0{  int operator()(const t_tuple& a, const t_tuple& b) const {-  return (a[0] < b[0]) ? -1 : ((a[0] > b[0]) ? 1 :((a[1] < b[1]) ? -1 : ((a[1] > b[1]) ? 1 :(0))));+  return (ramBitCast<RamSigned>(a[0]) < ramBitCast<RamSigned>(b[0])) ? -1 : (ramBitCast<RamSigned>(a[0]) > ramBitCast<RamSigned>(b[0])) ? 1 :((ramBitCast<RamSigned>(a[1]) < ramBitCast<RamSigned>(b[1])) ? -1 : (ramBitCast<RamSigned>(a[1]) > ramBitCast<RamSigned>(b[1])) ? 1 :(0));  } bool less(const t_tuple& a, const t_tuple& b) const {-  return  a[0] < b[0]|| (a[0] == b[0] && ( a[1] < b[1]));+  return (ramBitCast<RamSigned>(a[0]) < ramBitCast<RamSigned>(b[0]))|| (ramBitCast<RamSigned>(a[0]) == ramBitCast<RamSigned>(b[0])) && ((ramBitCast<RamSigned>(a[1]) < ramBitCast<RamSigned>(b[1])));  } bool equal(const t_tuple& a, const t_tuple& b) const {-return a[0] == b[0]&&a[1] == b[1];+return (ramBitCast<RamSigned>(a[0]) == ramBitCast<RamSigned>(b[0]))&&(ramBitCast<RamSigned>(a[1]) == ramBitCast<RamSigned>(b[1]));  } }; using t_ind_0 = btree_set<t_tuple,t_comparator_0>; t_ind_0 ind_0; using iterator = t_ind_0::iterator; struct context {-t_ind_0::operation_hints hints_0;+t_ind_0::operation_hints hints_0_lower;+t_ind_0::operation_hints hints_0_upper; }; context createContext() { return context(); } bool insert(const t_tuple& t) {@@ -31,7 +32,7 @@ return insert(t, h); } bool insert(const t_tuple& t, context& h) {-if (ind_0.insert(t, h.hints_0)) {+if (ind_0.insert(t, h.hints_0_lower)) { return true; } else return false; }@@ -47,7 +48,7 @@ return insert(data); } bool contains(const t_tuple& t, context& h) const {-return ind_0.contains(t, h.hints_0);+return ind_0.contains(t, h.hints_0_lower); } bool contains(const t_tuple& t) const { context h;@@ -57,38 +58,48 @@ return ind_0.size(); } iterator find(const t_tuple& t, context& h) const {-return ind_0.find(t, h.hints_0);+return ind_0.find(t, h.hints_0_lower); } iterator find(const t_tuple& t) const { context h; return find(t, h); }-range<iterator> lowerUpperRange_00(const t_tuple& lower, const t_tuple& upper, context& h) const {+range<iterator> lowerUpperRange_00(const t_tuple& /* lower */, const t_tuple& /* upper */, context& /* h */) const { return range<iterator>(ind_0.begin(),ind_0.end()); }-range<iterator> lowerUpperRange_00(const t_tuple& lower, const t_tuple& upper) const {+range<iterator> lowerUpperRange_00(const t_tuple& /* lower */, const t_tuple& /* upper */) const { return range<iterator>(ind_0.begin(),ind_0.end()); }-range<t_ind_0::iterator> lowerUpperRange_01(const t_tuple& lower, const t_tuple& upper, context& h) const {-t_tuple low(lower); t_tuple high(lower);-low[1] = MIN_RAM_SIGNED;-high[1] = MAX_RAM_SIGNED;-return make_range(ind_0.lower_bound(low, h.hints_0), ind_0.upper_bound(high, h.hints_0));+range<t_ind_0::iterator> lowerUpperRange_11(const t_tuple& lower, const t_tuple& upper, context& h) const {+t_comparator_0 comparator;+int cmp = comparator(lower, upper);+if (cmp == 0) {+    auto pos = ind_0.find(lower, h.hints_0_lower);+    auto fin = ind_0.end();+    if (pos != fin) {fin = pos; ++fin;}+    return make_range(pos, fin); }-range<t_ind_0::iterator> lowerUpperRange_01(const t_tuple& lower, const t_tuple& upper) const {-context h;-return lowerUpperRange_01(lower,upper,h);+if (cmp > 0) {+    return make_range(ind_0.end(), ind_0.end()); }-range<t_ind_0::iterator> lowerUpperRange_11(const t_tuple& lower, const t_tuple& upper, context& h) const {-auto pos = ind_0.find(lower, h.hints_0);-auto fin = ind_0.end();-if (pos != fin) {fin = pos; ++fin;}-return make_range(pos, fin);+return make_range(ind_0.lower_bound(lower, h.hints_0_lower), ind_0.upper_bound(upper, h.hints_0_upper)); } range<t_ind_0::iterator> lowerUpperRange_11(const t_tuple& lower, const t_tuple& upper) const { context h; return lowerUpperRange_11(lower,upper,h); }+range<t_ind_0::iterator> lowerUpperRange_10(const t_tuple& lower, const t_tuple& upper, context& h) const {+t_comparator_0 comparator;+int cmp = comparator(lower, upper);+if (cmp > 0) {+    return make_range(ind_0.end(), ind_0.end());+}+return make_range(ind_0.lower_bound(lower, h.hints_0_lower), ind_0.upper_bound(upper, h.hints_0_upper));+}+range<t_ind_0::iterator> lowerUpperRange_10(const t_tuple& lower, const t_tuple& upper) const {+context h;+return lowerUpperRange_10(lower,upper,h);+} bool empty() const { return ind_0.empty(); }@@ -109,24 +120,25 @@ ind_0.printStats(o); } };-struct t_btree_2__0_1__11 {+struct t_btree_ii__0_1__11 { using t_tuple = Tuple<RamDomain, 2>; struct t_comparator_0{  int operator()(const t_tuple& a, const t_tuple& b) const {-  return (a[0] < b[0]) ? -1 : ((a[0] > b[0]) ? 1 :((a[1] < b[1]) ? -1 : ((a[1] > b[1]) ? 1 :(0))));+  return (ramBitCast<RamSigned>(a[0]) < ramBitCast<RamSigned>(b[0])) ? -1 : (ramBitCast<RamSigned>(a[0]) > ramBitCast<RamSigned>(b[0])) ? 1 :((ramBitCast<RamSigned>(a[1]) < ramBitCast<RamSigned>(b[1])) ? -1 : (ramBitCast<RamSigned>(a[1]) > ramBitCast<RamSigned>(b[1])) ? 1 :(0));  } bool less(const t_tuple& a, const t_tuple& b) const {-  return  a[0] < b[0]|| (a[0] == b[0] && ( a[1] < b[1]));+  return (ramBitCast<RamSigned>(a[0]) < ramBitCast<RamSigned>(b[0]))|| (ramBitCast<RamSigned>(a[0]) == ramBitCast<RamSigned>(b[0])) && ((ramBitCast<RamSigned>(a[1]) < ramBitCast<RamSigned>(b[1])));  } bool equal(const t_tuple& a, const t_tuple& b) const {-return a[0] == b[0]&&a[1] == b[1];+return (ramBitCast<RamSigned>(a[0]) == ramBitCast<RamSigned>(b[0]))&&(ramBitCast<RamSigned>(a[1]) == ramBitCast<RamSigned>(b[1]));  } }; using t_ind_0 = btree_set<t_tuple,t_comparator_0>; t_ind_0 ind_0; using iterator = t_ind_0::iterator; struct context {-t_ind_0::operation_hints hints_0;+t_ind_0::operation_hints hints_0_lower;+t_ind_0::operation_hints hints_0_upper; }; context createContext() { return context(); } bool insert(const t_tuple& t) {@@ -134,7 +146,7 @@ return insert(t, h); } bool insert(const t_tuple& t, context& h) {-if (ind_0.insert(t, h.hints_0)) {+if (ind_0.insert(t, h.hints_0_lower)) { return true; } else return false; }@@ -150,7 +162,7 @@ return insert(data); } bool contains(const t_tuple& t, context& h) const {-return ind_0.contains(t, h.hints_0);+return ind_0.contains(t, h.hints_0_lower); } bool contains(const t_tuple& t) const { context h;@@ -160,24 +172,32 @@ return ind_0.size(); } iterator find(const t_tuple& t, context& h) const {-return ind_0.find(t, h.hints_0);+return ind_0.find(t, h.hints_0_lower); } iterator find(const t_tuple& t) const { context h; return find(t, h); }-range<iterator> lowerUpperRange_00(const t_tuple& lower, const t_tuple& upper, context& h) const {+range<iterator> lowerUpperRange_00(const t_tuple& /* lower */, const t_tuple& /* upper */, context& /* h */) const { return range<iterator>(ind_0.begin(),ind_0.end()); }-range<iterator> lowerUpperRange_00(const t_tuple& lower, const t_tuple& upper) const {+range<iterator> lowerUpperRange_00(const t_tuple& /* lower */, const t_tuple& /* upper */) const { return range<iterator>(ind_0.begin(),ind_0.end()); } range<t_ind_0::iterator> lowerUpperRange_11(const t_tuple& lower, const t_tuple& upper, context& h) const {-auto pos = ind_0.find(lower, h.hints_0);-auto fin = ind_0.end();-if (pos != fin) {fin = pos; ++fin;}-return make_range(pos, fin);+t_comparator_0 comparator;+int cmp = comparator(lower, upper);+if (cmp == 0) {+    auto pos = ind_0.find(lower, h.hints_0_lower);+    auto fin = ind_0.end();+    if (pos != fin) {fin = pos; ++fin;}+    return make_range(pos, fin); }+if (cmp > 0) {+    return make_range(ind_0.end(), ind_0.end());+}+return make_range(ind_0.lower_bound(lower, h.hints_0_lower), ind_0.upper_bound(upper, h.hints_0_upper));+} range<t_ind_0::iterator> lowerUpperRange_11(const t_tuple& lower, const t_tuple& upper) const { context h; return lowerUpperRange_11(lower,upper,h);@@ -206,16 +226,16 @@ class Sf_path : public SouffleProgram { private: static inline bool regex_wrapper(const std::string& pattern, const std::string& text) {-   bool result = false;-   try { result = std::regex_match(text, std::regex(pattern)); } catch(...) {+   bool result = false; +   try { result = std::regex_match(text, std::regex(pattern)); } catch(...) {       std::cerr << "warning: wrong pattern provided for match(\"" << pattern << "\",\"" << text << "\").\n"; }    return result; } private: static inline std::string substr_wrapper(const std::string& str, size_t idx, size_t len) {-   std::string result;-   try { result = str.substr(idx,len); } catch(...) {+   std::string result; +   try { result = str.substr(idx,len); } catch(...) {       std::cerr << "warning: wrong index position provided by substr(\"";      std::cerr << str << "\"," << (int32_t)idx << "," << (int32_t)len << ") functor.\n";    } return result;@@ -229,17 +249,17 @@ };// -- initialize record table -- RecordTable recordTable; // -- Table: @delta_reachable-std::unique_ptr<t_btree_2__0_1__01__11> rel_1_delta_reachable = std::make_unique<t_btree_2__0_1__01__11>();+Own<t_btree_ii__0_1__11__10> rel_1_delta_reachable = mk<t_btree_ii__0_1__11__10>(); // -- Table: @new_reachable-std::unique_ptr<t_btree_2__0_1__01__11> rel_2_new_reachable = std::make_unique<t_btree_2__0_1__01__11>();+Own<t_btree_ii__0_1__11__10> rel_2_new_reachable = mk<t_btree_ii__0_1__11__10>(); // -- Table: edge-std::unique_ptr<t_btree_2__0_1__11> rel_3_edge = std::make_unique<t_btree_2__0_1__11>();-souffle::RelationWrapper<0,t_btree_2__0_1__11,Tuple<RamDomain,2>,2,0> wrapper_rel_3_edge;+Own<t_btree_ii__0_1__11> rel_3_edge = mk<t_btree_ii__0_1__11>();+souffle::RelationWrapper<0,t_btree_ii__0_1__11,Tuple<RamDomain,2>,2,0> wrapper_rel_3_edge; // -- Table: reachable-std::unique_ptr<t_btree_2__0_1__11> rel_4_reachable = std::make_unique<t_btree_2__0_1__11>();-souffle::RelationWrapper<1,t_btree_2__0_1__11,Tuple<RamDomain,2>,2,0> wrapper_rel_4_reachable;+Own<t_btree_ii__0_1__11> rel_4_reachable = mk<t_btree_ii__0_1__11>();+souffle::RelationWrapper<1,t_btree_ii__0_1__11,Tuple<RamDomain,2>,2,0> wrapper_rel_4_reachable; public:-Sf_path() :+Sf_path() :  wrapper_rel_3_edge(*rel_3_edge,symTable,"edge",std::array<const char *,2>{{"s:symbol","s:symbol"}},std::array<const char *,2>{{"n","m"}}),  wrapper_rel_4_reachable(*rel_4_reachable,symTable,"reachable",std::array<const char *,2>{{"s:symbol","s:symbol"}},std::array<const char *,2>{{"n","m"}}){@@ -255,10 +275,10 @@ std::atomic<RamDomain> ctr{};  std::atomic<size_t> iter{};-void runFunction(std::string inputDirectory = ".", std::string outputDirectory = ".", bool performIO = false) {-this->inputDirectory = inputDirectory;-this->outputDirectory = outputDirectory;-this->performIO = performIO;+void runFunction(std::string inputDirectoryArg = "", std::string outputDirectoryArg = "", bool performIOArg = false) {+this->inputDirectory = inputDirectoryArg;+this->outputDirectory = outputDirectoryArg;+this->performIO = performIOArg; SignalHandler::instance()->set(); #if defined(_OPENMP) if (getNumThreads() > 0) {omp_set_num_threads(getNumThreads());}@@ -278,49 +298,49 @@ SignalHandler::instance()->reset(); } public:-void run() override { runFunction(".", ".", false); }+void run() override { runFunction("", "", false); } public:-void runAll(std::string inputDirectory = ".", std::string outputDirectory = ".") override { runFunction(inputDirectory, outputDirectory, true);+void runAll(std::string inputDirectoryArg = "", std::string outputDirectoryArg = "") override { runFunction(inputDirectoryArg, outputDirectoryArg, true); } public:-void printAll(std::string outputDirectory = ".") override {-try {std::map<std::string, std::string> directiveMap({{"IO","file"},{"attributeNames","n\tm"},{"filename","./edge.csv"},{"name","edge"},{"operation","output"},{"types","{\"edge\": {\"arity\": 2, \"auxArity\": 0, \"types\": [\"s:symbol\", \"s:symbol\"]}, \"records\": {}}"}});-if (!outputDirectory.empty() && directiveMap["IO"] == "file" && directiveMap["filename"].front() != '/') {directiveMap["filename"] = outputDirectory + "/" + directiveMap["filename"];}+void printAll(std::string outputDirectoryArg = "") override {+try {std::map<std::string, std::string> directiveMap({{"IO","file"},{"attributeNames","n\tm"},{"name","edge"},{"operation","output"},{"output-dir","."},{"params","{\"records\": {}, \"relation\": {\"arity\": 2, \"auxArity\": 0, \"params\": [\"n\", \"m\"]}}"},{"types","{\"ADTs\": {}, \"records\": {}, \"relation\": {\"arity\": 2, \"auxArity\": 0, \"types\": [\"s:symbol\", \"s:symbol\"]}}"}});+if (!outputDirectoryArg.empty()) {directiveMap["output-dir"] = outputDirectoryArg;} IOSystem::getInstance().getWriter(directiveMap, symTable, recordTable)->writeAll(*rel_3_edge); } catch (std::exception& e) {std::cerr << e.what();exit(1);}-try {std::map<std::string, std::string> directiveMap({{"IO","file"},{"attributeNames","n\tm"},{"filename","./reachable.csv"},{"name","reachable"},{"operation","output"},{"types","{\"reachable\": {\"arity\": 2, \"auxArity\": 0, \"types\": [\"s:symbol\", \"s:symbol\"]}, \"records\": {}}"}});-if (!outputDirectory.empty() && directiveMap["IO"] == "file" && directiveMap["filename"].front() != '/') {directiveMap["filename"] = outputDirectory + "/" + directiveMap["filename"];}+try {std::map<std::string, std::string> directiveMap({{"IO","file"},{"attributeNames","n\tm"},{"name","reachable"},{"operation","output"},{"output-dir","."},{"params","{\"records\": {}, \"relation\": {\"arity\": 2, \"auxArity\": 0, \"params\": [\"n\", \"m\"]}}"},{"types","{\"ADTs\": {}, \"records\": {}, \"relation\": {\"arity\": 2, \"auxArity\": 0, \"types\": [\"s:symbol\", \"s:symbol\"]}}"}});+if (!outputDirectoryArg.empty()) {directiveMap["output-dir"] = outputDirectoryArg;} IOSystem::getInstance().getWriter(directiveMap, symTable, recordTable)->writeAll(*rel_4_reachable); } catch (std::exception& e) {std::cerr << e.what();exit(1);} } public:-void loadAll(std::string inputDirectory = ".") override {-try {std::map<std::string, std::string> directiveMap({{"IO","file"},{"filename","./edge.facts"},{"name","edge"},{"operation","input"},{"types","{\"edge\": {\"arity\": 2, \"auxArity\": 0, \"types\": [\"s:symbol\", \"s:symbol\"]}, \"records\": {}}"}});-if (!inputDirectory.empty() && directiveMap["IO"] == "file" && directiveMap["filename"].front() != '/') {directiveMap["filename"] = inputDirectory + "/" + directiveMap["filename"];}+void loadAll(std::string inputDirectoryArg = "") override {+try {std::map<std::string, std::string> directiveMap({{"IO","file"},{"attributeNames","n\tm"},{"fact-dir","."},{"name","edge"},{"operation","input"},{"params","{\"records\": {}, \"relation\": {\"arity\": 2, \"auxArity\": 0, \"params\": [\"n\", \"m\"]}}"},{"types","{\"ADTs\": {}, \"records\": {}, \"relation\": {\"arity\": 2, \"auxArity\": 0, \"types\": [\"s:symbol\", \"s:symbol\"]}}"}});+if (!inputDirectoryArg.empty()) {directiveMap["fact-dir"] = inputDirectoryArg;} IOSystem::getInstance().getReader(directiveMap, symTable, recordTable)->readAll(*rel_3_edge); } catch (std::exception& e) {std::cerr << "Error loading data: " << e.what() << '\n';} } public:-void dumpInputs(std::ostream& out = std::cout) override {+void dumpInputs() override { try {std::map<std::string, std::string> rwOperation; rwOperation["IO"] = "stdout"; rwOperation["name"] = "edge";-rwOperation["types"] = "{\"edge\": {\"arity\": 2, \"auxArity\": 0, \"types\": [\"s:symbol\", \"s:symbol\"]}}";+rwOperation["types"] = "{\"relation\": {\"arity\": 2, \"auxArity\": 0, \"types\": [\"s:symbol\", \"s:symbol\"]}}"; IOSystem::getInstance().getWriter(rwOperation, symTable, recordTable)->writeAll(*rel_3_edge); } catch (std::exception& e) {std::cerr << e.what();exit(1);} } public:-void dumpOutputs(std::ostream& out = std::cout) override {+void dumpOutputs() override { try {std::map<std::string, std::string> rwOperation; rwOperation["IO"] = "stdout"; rwOperation["name"] = "edge";-rwOperation["types"] = "{\"edge\": {\"arity\": 2, \"auxArity\": 0, \"types\": [\"s:symbol\", \"s:symbol\"]}}";+rwOperation["types"] = "{\"relation\": {\"arity\": 2, \"auxArity\": 0, \"types\": [\"s:symbol\", \"s:symbol\"]}}"; IOSystem::getInstance().getWriter(rwOperation, symTable, recordTable)->writeAll(*rel_3_edge); } catch (std::exception& e) {std::cerr << e.what();exit(1);} try {std::map<std::string, std::string> rwOperation; rwOperation["IO"] = "stdout"; rwOperation["name"] = "reachable";-rwOperation["types"] = "{\"reachable\": {\"arity\": 2, \"auxArity\": 0, \"types\": [\"s:symbol\", \"s:symbol\"]}}";+rwOperation["types"] = "{\"relation\": {\"arity\": 2, \"auxArity\": 0, \"types\": [\"s:symbol\", \"s:symbol\"]}}"; IOSystem::getInstance().getWriter(rwOperation, symTable, recordTable)->writeAll(*rel_4_reachable); } catch (std::exception& e) {std::cerr << e.what();exit(1);} }@@ -337,10 +357,13 @@ return;} fatal("unknown subroutine"); }+#ifdef _MSC_VER+#pragma warning(disable: 4100)+#endif // _MSC_VER void subroutine_0(const std::vector<RamDomain>& args, std::vector<RamDomain>& ret) { if (performIO) {-try {std::map<std::string, std::string> directiveMap({{"IO","file"},{"filename","./edge.facts"},{"name","edge"},{"operation","input"},{"types","{\"edge\": {\"arity\": 2, \"auxArity\": 0, \"types\": [\"s:symbol\", \"s:symbol\"]}, \"records\": {}}"}});-if (!inputDirectory.empty() && directiveMap["filename"].front() != '/') {directiveMap["filename"] = inputDirectory + "/" + directiveMap["filename"];}+try {std::map<std::string, std::string> directiveMap({{"IO","file"},{"attributeNames","n\tm"},{"fact-dir","."},{"name","edge"},{"operation","input"},{"params","{\"records\": {}, \"relation\": {\"arity\": 2, \"auxArity\": 0, \"params\": [\"n\", \"m\"]}}"},{"types","{\"ADTs\": {}, \"records\": {}, \"relation\": {\"arity\": 2, \"auxArity\": 0, \"types\": [\"s:symbol\", \"s:symbol\"]}}"}});+if (!inputDirectory.empty()) {directiveMap["fact-dir"] = inputDirectory;} IOSystem::getInstance().getReader(directiveMap, symTable, recordTable)->readAll(*rel_3_edge); } catch (std::exception& e) {std::cerr << "Error loading data: " << e.what() << '\n';} }@@ -359,14 +382,20 @@ rel_3_edge->insert(tuple,READ_OP_CONTEXT(rel_3_edge_op_ctxt)); } ();if (performIO) {-try {std::map<std::string, std::string> directiveMap({{"IO","file"},{"attributeNames","n\tm"},{"filename","./edge.csv"},{"name","edge"},{"operation","output"},{"types","{\"edge\": {\"arity\": 2, \"auxArity\": 0, \"types\": [\"s:symbol\", \"s:symbol\"]}, \"records\": {}}"}});-if (!outputDirectory.empty() && directiveMap["filename"].front() != '/') {directiveMap["filename"] = outputDirectory + "/" + directiveMap["filename"];}+try {std::map<std::string, std::string> directiveMap({{"IO","file"},{"attributeNames","n\tm"},{"name","edge"},{"operation","output"},{"output-dir","."},{"params","{\"records\": {}, \"relation\": {\"arity\": 2, \"auxArity\": 0, \"params\": [\"n\", \"m\"]}}"},{"types","{\"ADTs\": {}, \"records\": {}, \"relation\": {\"arity\": 2, \"auxArity\": 0, \"types\": [\"s:symbol\", \"s:symbol\"]}}"}});+if (!outputDirectory.empty()) {directiveMap["output-dir"] = outputDirectory;} IOSystem::getInstance().getWriter(directiveMap, symTable, recordTable)->writeAll(*rel_3_edge); } catch (std::exception& e) {std::cerr << e.what();exit(1);} } }+#ifdef _MSC_VER+#pragma warning(default: 4100)+#endif // _MSC_VER+#ifdef _MSC_VER+#pragma warning(disable: 4100)+#endif // _MSC_VER void subroutine_1(const std::vector<RamDomain>& args, std::vector<RamDomain>& ret) {-SignalHandler::instance()->setMsg(R"_(reachable(x,y) :-+SignalHandler::instance()->setMsg(R"_(reachable(x,y) :-     edge(x,y). in file /Users/luc/personal/souffle-hs/tests/fixtures/path.dl [14:1-14:31])_"); if(!(rel_3_edge->empty())) {@@ -389,7 +418,7 @@ } ();iter = 0; for(;;) {-SignalHandler::instance()->setMsg(R"_(reachable(x,z) :-+SignalHandler::instance()->setMsg(R"_(reachable(x,z) :-     edge(x,y),    reachable(y,z). in file /Users/luc/personal/souffle-hs/tests/fixtures/path.dl [15:1-15:48])_");@@ -400,9 +429,7 @@ CREATE_OP_CONTEXT(rel_1_delta_reachable_op_ctxt,rel_1_delta_reachable->createContext()); CREATE_OP_CONTEXT(rel_2_new_reachable_op_ctxt,rel_2_new_reachable->createContext()); for(const auto& env0 : *rel_3_edge) {-const Tuple<RamDomain,2> lower{{ramBitCast(env0[1]),0}};-const Tuple<RamDomain,2> upper{{ramBitCast(env0[1]),0}};-auto range = rel_1_delta_reachable->lowerUpperRange_01(lower, upper,READ_OP_CONTEXT(rel_1_delta_reachable_op_ctxt));+auto range = rel_1_delta_reachable->lowerUpperRange_10(Tuple<RamDomain,2>{{ramBitCast(env0[1]), ramBitCast<RamDomain>(MIN_RAM_SIGNED)}},Tuple<RamDomain,2>{{ramBitCast(env0[1]), ramBitCast<RamDomain>(MAX_RAM_SIGNED)}},READ_OP_CONTEXT(rel_1_delta_reachable_op_ctxt)); for(const auto& env1 : range) { if( !(rel_4_reachable->contains(Tuple<RamDomain,2>{{ramBitCast(env0[0]),ramBitCast(env1[1])}},READ_OP_CONTEXT(rel_4_reachable_op_ctxt)))) { Tuple<RamDomain,2> tuple{{ramBitCast(env0[0]),ramBitCast(env1[1])}};@@ -429,14 +456,17 @@ rel_1_delta_reachable->purge(); rel_2_new_reachable->purge(); if (performIO) {-try {std::map<std::string, std::string> directiveMap({{"IO","file"},{"attributeNames","n\tm"},{"filename","./reachable.csv"},{"name","reachable"},{"operation","output"},{"types","{\"reachable\": {\"arity\": 2, \"auxArity\": 0, \"types\": [\"s:symbol\", \"s:symbol\"]}, \"records\": {}}"}});-if (!outputDirectory.empty() && directiveMap["filename"].front() != '/') {directiveMap["filename"] = outputDirectory + "/" + directiveMap["filename"];}+try {std::map<std::string, std::string> directiveMap({{"IO","file"},{"attributeNames","n\tm"},{"name","reachable"},{"operation","output"},{"output-dir","."},{"params","{\"records\": {}, \"relation\": {\"arity\": 2, \"auxArity\": 0, \"params\": [\"n\", \"m\"]}}"},{"types","{\"ADTs\": {}, \"records\": {}, \"relation\": {\"arity\": 2, \"auxArity\": 0, \"types\": [\"s:symbol\", \"s:symbol\"]}}"}});+if (!outputDirectory.empty()) {directiveMap["output-dir"] = outputDirectory;} IOSystem::getInstance().getWriter(directiveMap, symTable, recordTable)->writeAll(*rel_4_reachable); } catch (std::exception& e) {std::cerr << e.what();exit(1);} }-if (performIO) rel_3_edge->purge(); if (performIO) rel_4_reachable->purge();+if (performIO) rel_3_edge->purge(); }+#ifdef _MSC_VER+#pragma warning(default: 4100)+#endif // _MSC_VER }; SouffleProgram *newInstance_path(){return new Sf_path;} SymbolTable *getST_path(SouffleProgram *p){return &reinterpret_cast<Sf_path*>(p)->symTable;}@@ -459,14 +489,14 @@ { try{ souffle::CmdOptions opt(R"(path.dl)",-R"(.)",-R"(.)",+R"()",+R"()", false, R"()", 1); if (!opt.parse(argc,argv)) return 1; souffle::Sf_path obj;-#if defined(_OPENMP)+#if defined(_OPENMP)  obj.setNumThreads(opt.getNumJobs());  #endif
tests/fixtures/round_trip.cpp view
@@ -6,24 +6,25 @@  namespace souffle { static const RamDomain RAM_BIT_SHIFT_MASK = RAM_DOMAIN_SIZE - 1;-struct t_btree_1__0__1 {+struct t_btree_f__0__1 { using t_tuple = Tuple<RamDomain, 1>; struct t_comparator_0{  int operator()(const t_tuple& a, const t_tuple& b) const {-  return (a[0] < b[0]) ? -1 : ((a[0] > b[0]) ? 1 :(0));+  return (ramBitCast<RamFloat>(a[0]) < ramBitCast<RamFloat>(b[0])) ? -1 : (ramBitCast<RamFloat>(a[0]) > ramBitCast<RamFloat>(b[0])) ? 1 :(0);  } bool less(const t_tuple& a, const t_tuple& b) const {-  return  a[0] < b[0];+  return (ramBitCast<RamFloat>(a[0]) < ramBitCast<RamFloat>(b[0]));  } bool equal(const t_tuple& a, const t_tuple& b) const {-return a[0] == b[0];+return (ramBitCast<RamFloat>(a[0]) == ramBitCast<RamFloat>(b[0]));  } }; using t_ind_0 = btree_set<t_tuple,t_comparator_0>; t_ind_0 ind_0; using iterator = t_ind_0::iterator; struct context {-t_ind_0::operation_hints hints_0;+t_ind_0::operation_hints hints_0_lower;+t_ind_0::operation_hints hints_0_upper; }; context createContext() { return context(); } bool insert(const t_tuple& t) {@@ -31,7 +32,7 @@ return insert(t, h); } bool insert(const t_tuple& t, context& h) {-if (ind_0.insert(t, h.hints_0)) {+if (ind_0.insert(t, h.hints_0_lower)) { return true; } else return false; }@@ -47,7 +48,7 @@ return insert(data); } bool contains(const t_tuple& t, context& h) const {-return ind_0.contains(t, h.hints_0);+return ind_0.contains(t, h.hints_0_lower); } bool contains(const t_tuple& t) const { context h;@@ -57,24 +58,32 @@ return ind_0.size(); } iterator find(const t_tuple& t, context& h) const {-return ind_0.find(t, h.hints_0);+return ind_0.find(t, h.hints_0_lower); } iterator find(const t_tuple& t) const { context h; return find(t, h); }-range<iterator> lowerUpperRange_0(const t_tuple& lower, const t_tuple& upper, context& h) const {+range<iterator> lowerUpperRange_0(const t_tuple& /* lower */, const t_tuple& /* upper */, context& /* h */) const { return range<iterator>(ind_0.begin(),ind_0.end()); }-range<iterator> lowerUpperRange_0(const t_tuple& lower, const t_tuple& upper) const {+range<iterator> lowerUpperRange_0(const t_tuple& /* lower */, const t_tuple& /* upper */) const { return range<iterator>(ind_0.begin(),ind_0.end()); } range<t_ind_0::iterator> lowerUpperRange_1(const t_tuple& lower, const t_tuple& upper, context& h) const {-auto pos = ind_0.find(lower, h.hints_0);-auto fin = ind_0.end();-if (pos != fin) {fin = pos; ++fin;}-return make_range(pos, fin);+t_comparator_0 comparator;+int cmp = comparator(lower, upper);+if (cmp == 0) {+    auto pos = ind_0.find(lower, h.hints_0_lower);+    auto fin = ind_0.end();+    if (pos != fin) {fin = pos; ++fin;}+    return make_range(pos, fin); }+if (cmp > 0) {+    return make_range(ind_0.end(), ind_0.end());+}+return make_range(ind_0.lower_bound(lower, h.hints_0_lower), ind_0.upper_bound(upper, h.hints_0_upper));+} range<t_ind_0::iterator> lowerUpperRange_1(const t_tuple& lower, const t_tuple& upper) const { context h; return lowerUpperRange_1(lower,upper,h);@@ -99,6 +108,210 @@ ind_0.printStats(o); } };+struct t_btree_i__0__1 {+using t_tuple = Tuple<RamDomain, 1>;+struct t_comparator_0{+ int operator()(const t_tuple& a, const t_tuple& b) const {+  return (ramBitCast<RamSigned>(a[0]) < ramBitCast<RamSigned>(b[0])) ? -1 : (ramBitCast<RamSigned>(a[0]) > ramBitCast<RamSigned>(b[0])) ? 1 :(0);+ }+bool less(const t_tuple& a, const t_tuple& b) const {+  return (ramBitCast<RamSigned>(a[0]) < ramBitCast<RamSigned>(b[0]));+ }+bool equal(const t_tuple& a, const t_tuple& b) const {+return (ramBitCast<RamSigned>(a[0]) == ramBitCast<RamSigned>(b[0]));+ }+};+using t_ind_0 = btree_set<t_tuple,t_comparator_0>;+t_ind_0 ind_0;+using iterator = t_ind_0::iterator;+struct context {+t_ind_0::operation_hints hints_0_lower;+t_ind_0::operation_hints hints_0_upper;+};+context createContext() { return context(); }+bool insert(const t_tuple& t) {+context h;+return insert(t, h);+}+bool insert(const t_tuple& t, context& h) {+if (ind_0.insert(t, h.hints_0_lower)) {+return true;+} else return false;+}+bool insert(const RamDomain* ramDomain) {+RamDomain data[1];+std::copy(ramDomain, ramDomain + 1, data);+const t_tuple& tuple = reinterpret_cast<const t_tuple&>(data);+context h;+return insert(tuple, h);+}+bool insert(RamDomain a0) {+RamDomain data[1] = {a0};+return insert(data);+}+bool contains(const t_tuple& t, context& h) const {+return ind_0.contains(t, h.hints_0_lower);+}+bool contains(const t_tuple& t) const {+context h;+return contains(t, h);+}+std::size_t size() const {+return ind_0.size();+}+iterator find(const t_tuple& t, context& h) const {+return ind_0.find(t, h.hints_0_lower);+}+iterator find(const t_tuple& t) const {+context h;+return find(t, h);+}+range<iterator> lowerUpperRange_0(const t_tuple& /* lower */, const t_tuple& /* upper */, context& /* h */) const {+return range<iterator>(ind_0.begin(),ind_0.end());+}+range<iterator> lowerUpperRange_0(const t_tuple& /* lower */, const t_tuple& /* upper */) const {+return range<iterator>(ind_0.begin(),ind_0.end());+}+range<t_ind_0::iterator> lowerUpperRange_1(const t_tuple& lower, const t_tuple& upper, context& h) const {+t_comparator_0 comparator;+int cmp = comparator(lower, upper);+if (cmp == 0) {+    auto pos = ind_0.find(lower, h.hints_0_lower);+    auto fin = ind_0.end();+    if (pos != fin) {fin = pos; ++fin;}+    return make_range(pos, fin);+}+if (cmp > 0) {+    return make_range(ind_0.end(), ind_0.end());+}+return make_range(ind_0.lower_bound(lower, h.hints_0_lower), ind_0.upper_bound(upper, h.hints_0_upper));+}+range<t_ind_0::iterator> lowerUpperRange_1(const t_tuple& lower, const t_tuple& upper) const {+context h;+return lowerUpperRange_1(lower,upper,h);+}+bool empty() const {+return ind_0.empty();+}+std::vector<range<iterator>> partition() const {+return ind_0.getChunks(400);+}+void purge() {+ind_0.clear();+}+iterator begin() const {+return ind_0.begin();+}+iterator end() const {+return ind_0.end();+}+void printStatistics(std::ostream& o) const {+o << " arity 1 direct b-tree index 0 lex-order [0]\n";+ind_0.printStats(o);+}+};+struct t_btree_u__0__1 {+using t_tuple = Tuple<RamDomain, 1>;+struct t_comparator_0{+ int operator()(const t_tuple& a, const t_tuple& b) const {+  return (ramBitCast<RamUnsigned>(a[0]) < ramBitCast<RamUnsigned>(b[0])) ? -1 : (ramBitCast<RamUnsigned>(a[0]) > ramBitCast<RamUnsigned>(b[0])) ? 1 :(0);+ }+bool less(const t_tuple& a, const t_tuple& b) const {+  return (ramBitCast<RamUnsigned>(a[0]) < ramBitCast<RamUnsigned>(b[0]));+ }+bool equal(const t_tuple& a, const t_tuple& b) const {+return (ramBitCast<RamUnsigned>(a[0]) == ramBitCast<RamUnsigned>(b[0]));+ }+};+using t_ind_0 = btree_set<t_tuple,t_comparator_0>;+t_ind_0 ind_0;+using iterator = t_ind_0::iterator;+struct context {+t_ind_0::operation_hints hints_0_lower;+t_ind_0::operation_hints hints_0_upper;+};+context createContext() { return context(); }+bool insert(const t_tuple& t) {+context h;+return insert(t, h);+}+bool insert(const t_tuple& t, context& h) {+if (ind_0.insert(t, h.hints_0_lower)) {+return true;+} else return false;+}+bool insert(const RamDomain* ramDomain) {+RamDomain data[1];+std::copy(ramDomain, ramDomain + 1, data);+const t_tuple& tuple = reinterpret_cast<const t_tuple&>(data);+context h;+return insert(tuple, h);+}+bool insert(RamDomain a0) {+RamDomain data[1] = {a0};+return insert(data);+}+bool contains(const t_tuple& t, context& h) const {+return ind_0.contains(t, h.hints_0_lower);+}+bool contains(const t_tuple& t) const {+context h;+return contains(t, h);+}+std::size_t size() const {+return ind_0.size();+}+iterator find(const t_tuple& t, context& h) const {+return ind_0.find(t, h.hints_0_lower);+}+iterator find(const t_tuple& t) const {+context h;+return find(t, h);+}+range<iterator> lowerUpperRange_0(const t_tuple& /* lower */, const t_tuple& /* upper */, context& /* h */) const {+return range<iterator>(ind_0.begin(),ind_0.end());+}+range<iterator> lowerUpperRange_0(const t_tuple& /* lower */, const t_tuple& /* upper */) const {+return range<iterator>(ind_0.begin(),ind_0.end());+}+range<t_ind_0::iterator> lowerUpperRange_1(const t_tuple& lower, const t_tuple& upper, context& h) const {+t_comparator_0 comparator;+int cmp = comparator(lower, upper);+if (cmp == 0) {+    auto pos = ind_0.find(lower, h.hints_0_lower);+    auto fin = ind_0.end();+    if (pos != fin) {fin = pos; ++fin;}+    return make_range(pos, fin);+}+if (cmp > 0) {+    return make_range(ind_0.end(), ind_0.end());+}+return make_range(ind_0.lower_bound(lower, h.hints_0_lower), ind_0.upper_bound(upper, h.hints_0_upper));+}+range<t_ind_0::iterator> lowerUpperRange_1(const t_tuple& lower, const t_tuple& upper) const {+context h;+return lowerUpperRange_1(lower,upper,h);+}+bool empty() const {+return ind_0.empty();+}+std::vector<range<iterator>> partition() const {+return ind_0.getChunks(400);+}+void purge() {+ind_0.clear();+}+iterator begin() const {+return ind_0.begin();+}+iterator end() const {+return ind_0.end();+}+void printStatistics(std::ostream& o) const {+o << " arity 1 direct b-tree index 0 lex-order [0]\n";+ind_0.printStats(o);+}+};  class Sf_round_trip : public SouffleProgram { private:@@ -122,17 +335,17 @@ SymbolTable symTable;// -- initialize record table -- RecordTable recordTable; // -- Table: float_fact-std::unique_ptr<t_btree_1__0__1> rel_1_float_fact = std::make_unique<t_btree_1__0__1>();-souffle::RelationWrapper<0,t_btree_1__0__1,Tuple<RamDomain,1>,1,0> wrapper_rel_1_float_fact;+Own<t_btree_f__0__1> rel_1_float_fact = mk<t_btree_f__0__1>();+souffle::RelationWrapper<0,t_btree_f__0__1,Tuple<RamDomain,1>,1,0> wrapper_rel_1_float_fact; // -- Table: number_fact-std::unique_ptr<t_btree_1__0__1> rel_2_number_fact = std::make_unique<t_btree_1__0__1>();-souffle::RelationWrapper<1,t_btree_1__0__1,Tuple<RamDomain,1>,1,0> wrapper_rel_2_number_fact;+Own<t_btree_i__0__1> rel_2_number_fact = mk<t_btree_i__0__1>();+souffle::RelationWrapper<1,t_btree_i__0__1,Tuple<RamDomain,1>,1,0> wrapper_rel_2_number_fact; // -- Table: string_fact-std::unique_ptr<t_btree_1__0__1> rel_3_string_fact = std::make_unique<t_btree_1__0__1>();-souffle::RelationWrapper<2,t_btree_1__0__1,Tuple<RamDomain,1>,1,0> wrapper_rel_3_string_fact;+Own<t_btree_i__0__1> rel_3_string_fact = mk<t_btree_i__0__1>();+souffle::RelationWrapper<2,t_btree_i__0__1,Tuple<RamDomain,1>,1,0> wrapper_rel_3_string_fact; // -- Table: unsigned_fact-std::unique_ptr<t_btree_1__0__1> rel_4_unsigned_fact = std::make_unique<t_btree_1__0__1>();-souffle::RelationWrapper<3,t_btree_1__0__1,Tuple<RamDomain,1>,1,0> wrapper_rel_4_unsigned_fact;+Own<t_btree_u__0__1> rel_4_unsigned_fact = mk<t_btree_u__0__1>();+souffle::RelationWrapper<3,t_btree_u__0__1,Tuple<RamDomain,1>,1,0> wrapper_rel_4_unsigned_fact; public: Sf_round_trip() :  wrapper_rel_1_float_fact(*rel_1_float_fact,symTable,"float_fact",std::array<const char *,1>{{"f:float"}},std::array<const char *,1>{{"x"}}),@@ -156,10 +369,10 @@ std::atomic<RamDomain> ctr{};  std::atomic<size_t> iter{};-void runFunction(std::string inputDirectory = ".", std::string outputDirectory = ".", bool performIO = false) {-this->inputDirectory = inputDirectory;-this->outputDirectory = outputDirectory;-this->performIO = performIO;+void runFunction(std::string inputDirectoryArg = "", std::string outputDirectoryArg = "", bool performIOArg = false) {+this->inputDirectory = inputDirectoryArg;+this->outputDirectory = outputDirectoryArg;+this->performIO = performIOArg; SignalHandler::instance()->set(); #if defined(_OPENMP) if (getNumThreads() > 0) {omp_set_num_threads(getNumThreads());}@@ -187,99 +400,99 @@ SignalHandler::instance()->reset(); } public:-void run() override { runFunction(".", ".", false); }+void run() override { runFunction("", "", false); } public:-void runAll(std::string inputDirectory = ".", std::string outputDirectory = ".") override { runFunction(inputDirectory, outputDirectory, true);+void runAll(std::string inputDirectoryArg = "", std::string outputDirectoryArg = "") override { runFunction(inputDirectoryArg, outputDirectoryArg, true); } public:-void printAll(std::string outputDirectory = ".") override {-try {std::map<std::string, std::string> directiveMap({{"IO","file"},{"attributeNames","x"},{"filename","./string_fact.csv"},{"name","string_fact"},{"operation","output"},{"types","{\"records\": {}, \"string_fact\": {\"arity\": 1, \"auxArity\": 0, \"types\": [\"s:symbol\"]}}"}});-if (!outputDirectory.empty() && directiveMap["IO"] == "file" && directiveMap["filename"].front() != '/') {directiveMap["filename"] = outputDirectory + "/" + directiveMap["filename"];}-IOSystem::getInstance().getWriter(directiveMap, symTable, recordTable)->writeAll(*rel_3_string_fact);+void printAll(std::string outputDirectoryArg = "") override {+try {std::map<std::string, std::string> directiveMap({{"IO","file"},{"attributeNames","x"},{"name","number_fact"},{"operation","output"},{"output-dir","."},{"params","{\"records\": {}, \"relation\": {\"arity\": 1, \"auxArity\": 0, \"params\": [\"x\"]}}"},{"types","{\"ADTs\": {}, \"records\": {}, \"relation\": {\"arity\": 1, \"auxArity\": 0, \"types\": [\"i:number\"]}}"}});+if (!outputDirectoryArg.empty()) {directiveMap["output-dir"] = outputDirectoryArg;}+IOSystem::getInstance().getWriter(directiveMap, symTable, recordTable)->writeAll(*rel_2_number_fact); } catch (std::exception& e) {std::cerr << e.what();exit(1);}-try {std::map<std::string, std::string> directiveMap({{"IO","file"},{"attributeNames","x"},{"filename","./unsigned_fact.csv"},{"name","unsigned_fact"},{"operation","output"},{"types","{\"records\": {}, \"unsigned_fact\": {\"arity\": 1, \"auxArity\": 0, \"types\": [\"u:unsigned\"]}}"}});-if (!outputDirectory.empty() && directiveMap["IO"] == "file" && directiveMap["filename"].front() != '/') {directiveMap["filename"] = outputDirectory + "/" + directiveMap["filename"];}+try {std::map<std::string, std::string> directiveMap({{"IO","file"},{"attributeNames","x"},{"name","unsigned_fact"},{"operation","output"},{"output-dir","."},{"params","{\"records\": {}, \"relation\": {\"arity\": 1, \"auxArity\": 0, \"params\": [\"x\"]}}"},{"types","{\"ADTs\": {}, \"records\": {}, \"relation\": {\"arity\": 1, \"auxArity\": 0, \"types\": [\"u:unsigned\"]}}"}});+if (!outputDirectoryArg.empty()) {directiveMap["output-dir"] = outputDirectoryArg;} IOSystem::getInstance().getWriter(directiveMap, symTable, recordTable)->writeAll(*rel_4_unsigned_fact); } catch (std::exception& e) {std::cerr << e.what();exit(1);}-try {std::map<std::string, std::string> directiveMap({{"IO","file"},{"attributeNames","x"},{"filename","./number_fact.csv"},{"name","number_fact"},{"operation","output"},{"types","{\"number_fact\": {\"arity\": 1, \"auxArity\": 0, \"types\": [\"i:number\"]}, \"records\": {}}"}});-if (!outputDirectory.empty() && directiveMap["IO"] == "file" && directiveMap["filename"].front() != '/') {directiveMap["filename"] = outputDirectory + "/" + directiveMap["filename"];}-IOSystem::getInstance().getWriter(directiveMap, symTable, recordTable)->writeAll(*rel_2_number_fact);+try {std::map<std::string, std::string> directiveMap({{"IO","file"},{"attributeNames","x"},{"name","string_fact"},{"operation","output"},{"output-dir","."},{"params","{\"records\": {}, \"relation\": {\"arity\": 1, \"auxArity\": 0, \"params\": [\"x\"]}}"},{"types","{\"ADTs\": {}, \"records\": {}, \"relation\": {\"arity\": 1, \"auxArity\": 0, \"types\": [\"s:symbol\"]}}"}});+if (!outputDirectoryArg.empty()) {directiveMap["output-dir"] = outputDirectoryArg;}+IOSystem::getInstance().getWriter(directiveMap, symTable, recordTable)->writeAll(*rel_3_string_fact); } catch (std::exception& e) {std::cerr << e.what();exit(1);}-try {std::map<std::string, std::string> directiveMap({{"IO","file"},{"attributeNames","x"},{"filename","./float_fact.csv"},{"name","float_fact"},{"operation","output"},{"types","{\"float_fact\": {\"arity\": 1, \"auxArity\": 0, \"types\": [\"f:float\"]}, \"records\": {}}"}});-if (!outputDirectory.empty() && directiveMap["IO"] == "file" && directiveMap["filename"].front() != '/') {directiveMap["filename"] = outputDirectory + "/" + directiveMap["filename"];}+try {std::map<std::string, std::string> directiveMap({{"IO","file"},{"attributeNames","x"},{"name","float_fact"},{"operation","output"},{"output-dir","."},{"params","{\"records\": {}, \"relation\": {\"arity\": 1, \"auxArity\": 0, \"params\": [\"x\"]}}"},{"types","{\"ADTs\": {}, \"records\": {}, \"relation\": {\"arity\": 1, \"auxArity\": 0, \"types\": [\"f:float\"]}}"}});+if (!outputDirectoryArg.empty()) {directiveMap["output-dir"] = outputDirectoryArg;} IOSystem::getInstance().getWriter(directiveMap, symTable, recordTable)->writeAll(*rel_1_float_fact); } catch (std::exception& e) {std::cerr << e.what();exit(1);} } public:-void loadAll(std::string inputDirectory = ".") override {-try {std::map<std::string, std::string> directiveMap({{"IO","file"},{"filename","./string_fact.facts"},{"name","string_fact"},{"operation","input"},{"types","{\"records\": {}, \"string_fact\": {\"arity\": 1, \"auxArity\": 0, \"types\": [\"s:symbol\"]}}"}});-if (!inputDirectory.empty() && directiveMap["IO"] == "file" && directiveMap["filename"].front() != '/') {directiveMap["filename"] = inputDirectory + "/" + directiveMap["filename"];}-IOSystem::getInstance().getReader(directiveMap, symTable, recordTable)->readAll(*rel_3_string_fact);+void loadAll(std::string inputDirectoryArg = "") override {+try {std::map<std::string, std::string> directiveMap({{"IO","file"},{"attributeNames","x"},{"fact-dir","."},{"name","float_fact"},{"operation","input"},{"params","{\"records\": {}, \"relation\": {\"arity\": 1, \"auxArity\": 0, \"params\": [\"x\"]}}"},{"types","{\"ADTs\": {}, \"records\": {}, \"relation\": {\"arity\": 1, \"auxArity\": 0, \"types\": [\"f:float\"]}}"}});+if (!inputDirectoryArg.empty()) {directiveMap["fact-dir"] = inputDirectoryArg;}+IOSystem::getInstance().getReader(directiveMap, symTable, recordTable)->readAll(*rel_1_float_fact); } catch (std::exception& e) {std::cerr << "Error loading data: " << e.what() << '\n';}-try {std::map<std::string, std::string> directiveMap({{"IO","file"},{"filename","./number_fact.facts"},{"name","number_fact"},{"operation","input"},{"types","{\"number_fact\": {\"arity\": 1, \"auxArity\": 0, \"types\": [\"i:number\"]}, \"records\": {}}"}});-if (!inputDirectory.empty() && directiveMap["IO"] == "file" && directiveMap["filename"].front() != '/') {directiveMap["filename"] = inputDirectory + "/" + directiveMap["filename"];}+try {std::map<std::string, std::string> directiveMap({{"IO","file"},{"attributeNames","x"},{"fact-dir","."},{"name","number_fact"},{"operation","input"},{"params","{\"records\": {}, \"relation\": {\"arity\": 1, \"auxArity\": 0, \"params\": [\"x\"]}}"},{"types","{\"ADTs\": {}, \"records\": {}, \"relation\": {\"arity\": 1, \"auxArity\": 0, \"types\": [\"i:number\"]}}"}});+if (!inputDirectoryArg.empty()) {directiveMap["fact-dir"] = inputDirectoryArg;} IOSystem::getInstance().getReader(directiveMap, symTable, recordTable)->readAll(*rel_2_number_fact); } catch (std::exception& e) {std::cerr << "Error loading data: " << e.what() << '\n';}-try {std::map<std::string, std::string> directiveMap({{"IO","file"},{"filename","./unsigned_fact.facts"},{"name","unsigned_fact"},{"operation","input"},{"types","{\"records\": {}, \"unsigned_fact\": {\"arity\": 1, \"auxArity\": 0, \"types\": [\"u:unsigned\"]}}"}});-if (!inputDirectory.empty() && directiveMap["IO"] == "file" && directiveMap["filename"].front() != '/') {directiveMap["filename"] = inputDirectory + "/" + directiveMap["filename"];}-IOSystem::getInstance().getReader(directiveMap, symTable, recordTable)->readAll(*rel_4_unsigned_fact);+try {std::map<std::string, std::string> directiveMap({{"IO","file"},{"attributeNames","x"},{"fact-dir","."},{"name","string_fact"},{"operation","input"},{"params","{\"records\": {}, \"relation\": {\"arity\": 1, \"auxArity\": 0, \"params\": [\"x\"]}}"},{"types","{\"ADTs\": {}, \"records\": {}, \"relation\": {\"arity\": 1, \"auxArity\": 0, \"types\": [\"s:symbol\"]}}"}});+if (!inputDirectoryArg.empty()) {directiveMap["fact-dir"] = inputDirectoryArg;}+IOSystem::getInstance().getReader(directiveMap, symTable, recordTable)->readAll(*rel_3_string_fact); } catch (std::exception& e) {std::cerr << "Error loading data: " << e.what() << '\n';}-try {std::map<std::string, std::string> directiveMap({{"IO","file"},{"filename","./float_fact.facts"},{"name","float_fact"},{"operation","input"},{"types","{\"float_fact\": {\"arity\": 1, \"auxArity\": 0, \"types\": [\"f:float\"]}, \"records\": {}}"}});-if (!inputDirectory.empty() && directiveMap["IO"] == "file" && directiveMap["filename"].front() != '/') {directiveMap["filename"] = inputDirectory + "/" + directiveMap["filename"];}-IOSystem::getInstance().getReader(directiveMap, symTable, recordTable)->readAll(*rel_1_float_fact);+try {std::map<std::string, std::string> directiveMap({{"IO","file"},{"attributeNames","x"},{"fact-dir","."},{"name","unsigned_fact"},{"operation","input"},{"params","{\"records\": {}, \"relation\": {\"arity\": 1, \"auxArity\": 0, \"params\": [\"x\"]}}"},{"types","{\"ADTs\": {}, \"records\": {}, \"relation\": {\"arity\": 1, \"auxArity\": 0, \"types\": [\"u:unsigned\"]}}"}});+if (!inputDirectoryArg.empty()) {directiveMap["fact-dir"] = inputDirectoryArg;}+IOSystem::getInstance().getReader(directiveMap, symTable, recordTable)->readAll(*rel_4_unsigned_fact); } catch (std::exception& e) {std::cerr << "Error loading data: " << e.what() << '\n';} } public:-void dumpInputs(std::ostream& out = std::cout) override {+void dumpInputs() override { try {std::map<std::string, std::string> rwOperation; rwOperation["IO"] = "stdout";-rwOperation["name"] = "string_fact";-rwOperation["types"] = "{\"string_fact\": {\"arity\": 1, \"auxArity\": 0, \"types\": [\"s:symbol\"]}}";-IOSystem::getInstance().getWriter(rwOperation, symTable, recordTable)->writeAll(*rel_3_string_fact);+rwOperation["name"] = "float_fact";+rwOperation["types"] = "{\"relation\": {\"arity\": 1, \"auxArity\": 0, \"types\": [\"f:float\"]}}";+IOSystem::getInstance().getWriter(rwOperation, symTable, recordTable)->writeAll(*rel_1_float_fact); } catch (std::exception& e) {std::cerr << e.what();exit(1);} try {std::map<std::string, std::string> rwOperation; rwOperation["IO"] = "stdout"; rwOperation["name"] = "number_fact";-rwOperation["types"] = "{\"number_fact\": {\"arity\": 1, \"auxArity\": 0, \"types\": [\"i:number\"]}}";+rwOperation["types"] = "{\"relation\": {\"arity\": 1, \"auxArity\": 0, \"types\": [\"i:number\"]}}"; IOSystem::getInstance().getWriter(rwOperation, symTable, recordTable)->writeAll(*rel_2_number_fact); } catch (std::exception& e) {std::cerr << e.what();exit(1);} try {std::map<std::string, std::string> rwOperation; rwOperation["IO"] = "stdout";-rwOperation["name"] = "unsigned_fact";-rwOperation["types"] = "{\"unsigned_fact\": {\"arity\": 1, \"auxArity\": 0, \"types\": [\"u:unsigned\"]}}";-IOSystem::getInstance().getWriter(rwOperation, symTable, recordTable)->writeAll(*rel_4_unsigned_fact);+rwOperation["name"] = "string_fact";+rwOperation["types"] = "{\"relation\": {\"arity\": 1, \"auxArity\": 0, \"types\": [\"s:symbol\"]}}";+IOSystem::getInstance().getWriter(rwOperation, symTable, recordTable)->writeAll(*rel_3_string_fact); } catch (std::exception& e) {std::cerr << e.what();exit(1);} try {std::map<std::string, std::string> rwOperation; rwOperation["IO"] = "stdout";-rwOperation["name"] = "float_fact";-rwOperation["types"] = "{\"float_fact\": {\"arity\": 1, \"auxArity\": 0, \"types\": [\"f:float\"]}}";-IOSystem::getInstance().getWriter(rwOperation, symTable, recordTable)->writeAll(*rel_1_float_fact);+rwOperation["name"] = "unsigned_fact";+rwOperation["types"] = "{\"relation\": {\"arity\": 1, \"auxArity\": 0, \"types\": [\"u:unsigned\"]}}";+IOSystem::getInstance().getWriter(rwOperation, symTable, recordTable)->writeAll(*rel_4_unsigned_fact); } catch (std::exception& e) {std::cerr << e.what();exit(1);} } public:-void dumpOutputs(std::ostream& out = std::cout) override {+void dumpOutputs() override { try {std::map<std::string, std::string> rwOperation; rwOperation["IO"] = "stdout";-rwOperation["name"] = "string_fact";-rwOperation["types"] = "{\"string_fact\": {\"arity\": 1, \"auxArity\": 0, \"types\": [\"s:symbol\"]}}";-IOSystem::getInstance().getWriter(rwOperation, symTable, recordTable)->writeAll(*rel_3_string_fact);+rwOperation["name"] = "number_fact";+rwOperation["types"] = "{\"relation\": {\"arity\": 1, \"auxArity\": 0, \"types\": [\"i:number\"]}}";+IOSystem::getInstance().getWriter(rwOperation, symTable, recordTable)->writeAll(*rel_2_number_fact); } catch (std::exception& e) {std::cerr << e.what();exit(1);} try {std::map<std::string, std::string> rwOperation; rwOperation["IO"] = "stdout"; rwOperation["name"] = "unsigned_fact";-rwOperation["types"] = "{\"unsigned_fact\": {\"arity\": 1, \"auxArity\": 0, \"types\": [\"u:unsigned\"]}}";+rwOperation["types"] = "{\"relation\": {\"arity\": 1, \"auxArity\": 0, \"types\": [\"u:unsigned\"]}}"; IOSystem::getInstance().getWriter(rwOperation, symTable, recordTable)->writeAll(*rel_4_unsigned_fact); } catch (std::exception& e) {std::cerr << e.what();exit(1);} try {std::map<std::string, std::string> rwOperation; rwOperation["IO"] = "stdout";-rwOperation["name"] = "number_fact";-rwOperation["types"] = "{\"number_fact\": {\"arity\": 1, \"auxArity\": 0, \"types\": [\"i:number\"]}}";-IOSystem::getInstance().getWriter(rwOperation, symTable, recordTable)->writeAll(*rel_2_number_fact);+rwOperation["name"] = "string_fact";+rwOperation["types"] = "{\"relation\": {\"arity\": 1, \"auxArity\": 0, \"types\": [\"s:symbol\"]}}";+IOSystem::getInstance().getWriter(rwOperation, symTable, recordTable)->writeAll(*rel_3_string_fact); } catch (std::exception& e) {std::cerr << e.what();exit(1);} try {std::map<std::string, std::string> rwOperation; rwOperation["IO"] = "stdout"; rwOperation["name"] = "float_fact";-rwOperation["types"] = "{\"float_fact\": {\"arity\": 1, \"auxArity\": 0, \"types\": [\"f:float\"]}}";+rwOperation["types"] = "{\"relation\": {\"arity\": 1, \"auxArity\": 0, \"types\": [\"f:float\"]}}"; IOSystem::getInstance().getWriter(rwOperation, symTable, recordTable)->writeAll(*rel_1_float_fact); } catch (std::exception& e) {std::cerr << e.what();exit(1);} }@@ -302,62 +515,86 @@ return;} fatal("unknown subroutine"); }+#ifdef _MSC_VER+#pragma warning(disable: 4100)+#endif // _MSC_VER void subroutine_0(const std::vector<RamDomain>& args, std::vector<RamDomain>& ret) { if (performIO) {-try {std::map<std::string, std::string> directiveMap({{"IO","file"},{"filename","./string_fact.facts"},{"name","string_fact"},{"operation","input"},{"types","{\"records\": {}, \"string_fact\": {\"arity\": 1, \"auxArity\": 0, \"types\": [\"s:symbol\"]}}"}});-if (!inputDirectory.empty() && directiveMap["filename"].front() != '/') {directiveMap["filename"] = inputDirectory + "/" + directiveMap["filename"];}+try {std::map<std::string, std::string> directiveMap({{"IO","file"},{"attributeNames","x"},{"fact-dir","."},{"name","string_fact"},{"operation","input"},{"params","{\"records\": {}, \"relation\": {\"arity\": 1, \"auxArity\": 0, \"params\": [\"x\"]}}"},{"types","{\"ADTs\": {}, \"records\": {}, \"relation\": {\"arity\": 1, \"auxArity\": 0, \"types\": [\"s:symbol\"]}}"}});+if (!inputDirectory.empty()) {directiveMap["fact-dir"] = inputDirectory;} IOSystem::getInstance().getReader(directiveMap, symTable, recordTable)->readAll(*rel_3_string_fact); } catch (std::exception& e) {std::cerr << "Error loading data: " << e.what() << '\n';} } if (performIO) {-try {std::map<std::string, std::string> directiveMap({{"IO","file"},{"attributeNames","x"},{"filename","./string_fact.csv"},{"name","string_fact"},{"operation","output"},{"types","{\"records\": {}, \"string_fact\": {\"arity\": 1, \"auxArity\": 0, \"types\": [\"s:symbol\"]}}"}});-if (!outputDirectory.empty() && directiveMap["filename"].front() != '/') {directiveMap["filename"] = outputDirectory + "/" + directiveMap["filename"];}+try {std::map<std::string, std::string> directiveMap({{"IO","file"},{"attributeNames","x"},{"name","string_fact"},{"operation","output"},{"output-dir","."},{"params","{\"records\": {}, \"relation\": {\"arity\": 1, \"auxArity\": 0, \"params\": [\"x\"]}}"},{"types","{\"ADTs\": {}, \"records\": {}, \"relation\": {\"arity\": 1, \"auxArity\": 0, \"types\": [\"s:symbol\"]}}"}});+if (!outputDirectory.empty()) {directiveMap["output-dir"] = outputDirectory;} IOSystem::getInstance().getWriter(directiveMap, symTable, recordTable)->writeAll(*rel_3_string_fact); } catch (std::exception& e) {std::cerr << e.what();exit(1);} } }+#ifdef _MSC_VER+#pragma warning(default: 4100)+#endif // _MSC_VER+#ifdef _MSC_VER+#pragma warning(disable: 4100)+#endif // _MSC_VER void subroutine_1(const std::vector<RamDomain>& args, std::vector<RamDomain>& ret) { if (performIO) {-try {std::map<std::string, std::string> directiveMap({{"IO","file"},{"filename","./number_fact.facts"},{"name","number_fact"},{"operation","input"},{"types","{\"number_fact\": {\"arity\": 1, \"auxArity\": 0, \"types\": [\"i:number\"]}, \"records\": {}}"}});-if (!inputDirectory.empty() && directiveMap["filename"].front() != '/') {directiveMap["filename"] = inputDirectory + "/" + directiveMap["filename"];}+try {std::map<std::string, std::string> directiveMap({{"IO","file"},{"attributeNames","x"},{"fact-dir","."},{"name","number_fact"},{"operation","input"},{"params","{\"records\": {}, \"relation\": {\"arity\": 1, \"auxArity\": 0, \"params\": [\"x\"]}}"},{"types","{\"ADTs\": {}, \"records\": {}, \"relation\": {\"arity\": 1, \"auxArity\": 0, \"types\": [\"i:number\"]}}"}});+if (!inputDirectory.empty()) {directiveMap["fact-dir"] = inputDirectory;} IOSystem::getInstance().getReader(directiveMap, symTable, recordTable)->readAll(*rel_2_number_fact); } catch (std::exception& e) {std::cerr << "Error loading data: " << e.what() << '\n';} } if (performIO) {-try {std::map<std::string, std::string> directiveMap({{"IO","file"},{"attributeNames","x"},{"filename","./number_fact.csv"},{"name","number_fact"},{"operation","output"},{"types","{\"number_fact\": {\"arity\": 1, \"auxArity\": 0, \"types\": [\"i:number\"]}, \"records\": {}}"}});-if (!outputDirectory.empty() && directiveMap["filename"].front() != '/') {directiveMap["filename"] = outputDirectory + "/" + directiveMap["filename"];}+try {std::map<std::string, std::string> directiveMap({{"IO","file"},{"attributeNames","x"},{"name","number_fact"},{"operation","output"},{"output-dir","."},{"params","{\"records\": {}, \"relation\": {\"arity\": 1, \"auxArity\": 0, \"params\": [\"x\"]}}"},{"types","{\"ADTs\": {}, \"records\": {}, \"relation\": {\"arity\": 1, \"auxArity\": 0, \"types\": [\"i:number\"]}}"}});+if (!outputDirectory.empty()) {directiveMap["output-dir"] = outputDirectory;} IOSystem::getInstance().getWriter(directiveMap, symTable, recordTable)->writeAll(*rel_2_number_fact); } catch (std::exception& e) {std::cerr << e.what();exit(1);} } }+#ifdef _MSC_VER+#pragma warning(default: 4100)+#endif // _MSC_VER+#ifdef _MSC_VER+#pragma warning(disable: 4100)+#endif // _MSC_VER void subroutine_2(const std::vector<RamDomain>& args, std::vector<RamDomain>& ret) { if (performIO) {-try {std::map<std::string, std::string> directiveMap({{"IO","file"},{"filename","./unsigned_fact.facts"},{"name","unsigned_fact"},{"operation","input"},{"types","{\"records\": {}, \"unsigned_fact\": {\"arity\": 1, \"auxArity\": 0, \"types\": [\"u:unsigned\"]}}"}});-if (!inputDirectory.empty() && directiveMap["filename"].front() != '/') {directiveMap["filename"] = inputDirectory + "/" + directiveMap["filename"];}+try {std::map<std::string, std::string> directiveMap({{"IO","file"},{"attributeNames","x"},{"fact-dir","."},{"name","unsigned_fact"},{"operation","input"},{"params","{\"records\": {}, \"relation\": {\"arity\": 1, \"auxArity\": 0, \"params\": [\"x\"]}}"},{"types","{\"ADTs\": {}, \"records\": {}, \"relation\": {\"arity\": 1, \"auxArity\": 0, \"types\": [\"u:unsigned\"]}}"}});+if (!inputDirectory.empty()) {directiveMap["fact-dir"] = inputDirectory;} IOSystem::getInstance().getReader(directiveMap, symTable, recordTable)->readAll(*rel_4_unsigned_fact); } catch (std::exception& e) {std::cerr << "Error loading data: " << e.what() << '\n';} } if (performIO) {-try {std::map<std::string, std::string> directiveMap({{"IO","file"},{"attributeNames","x"},{"filename","./unsigned_fact.csv"},{"name","unsigned_fact"},{"operation","output"},{"types","{\"records\": {}, \"unsigned_fact\": {\"arity\": 1, \"auxArity\": 0, \"types\": [\"u:unsigned\"]}}"}});-if (!outputDirectory.empty() && directiveMap["filename"].front() != '/') {directiveMap["filename"] = outputDirectory + "/" + directiveMap["filename"];}+try {std::map<std::string, std::string> directiveMap({{"IO","file"},{"attributeNames","x"},{"name","unsigned_fact"},{"operation","output"},{"output-dir","."},{"params","{\"records\": {}, \"relation\": {\"arity\": 1, \"auxArity\": 0, \"params\": [\"x\"]}}"},{"types","{\"ADTs\": {}, \"records\": {}, \"relation\": {\"arity\": 1, \"auxArity\": 0, \"types\": [\"u:unsigned\"]}}"}});+if (!outputDirectory.empty()) {directiveMap["output-dir"] = outputDirectory;} IOSystem::getInstance().getWriter(directiveMap, symTable, recordTable)->writeAll(*rel_4_unsigned_fact); } catch (std::exception& e) {std::cerr << e.what();exit(1);} } }+#ifdef _MSC_VER+#pragma warning(default: 4100)+#endif // _MSC_VER+#ifdef _MSC_VER+#pragma warning(disable: 4100)+#endif // _MSC_VER void subroutine_3(const std::vector<RamDomain>& args, std::vector<RamDomain>& ret) { if (performIO) {-try {std::map<std::string, std::string> directiveMap({{"IO","file"},{"filename","./float_fact.facts"},{"name","float_fact"},{"operation","input"},{"types","{\"float_fact\": {\"arity\": 1, \"auxArity\": 0, \"types\": [\"f:float\"]}, \"records\": {}}"}});-if (!inputDirectory.empty() && directiveMap["filename"].front() != '/') {directiveMap["filename"] = inputDirectory + "/" + directiveMap["filename"];}+try {std::map<std::string, std::string> directiveMap({{"IO","file"},{"attributeNames","x"},{"fact-dir","."},{"name","float_fact"},{"operation","input"},{"params","{\"records\": {}, \"relation\": {\"arity\": 1, \"auxArity\": 0, \"params\": [\"x\"]}}"},{"types","{\"ADTs\": {}, \"records\": {}, \"relation\": {\"arity\": 1, \"auxArity\": 0, \"types\": [\"f:float\"]}}"}});+if (!inputDirectory.empty()) {directiveMap["fact-dir"] = inputDirectory;} IOSystem::getInstance().getReader(directiveMap, symTable, recordTable)->readAll(*rel_1_float_fact); } catch (std::exception& e) {std::cerr << "Error loading data: " << e.what() << '\n';} } if (performIO) {-try {std::map<std::string, std::string> directiveMap({{"IO","file"},{"attributeNames","x"},{"filename","./float_fact.csv"},{"name","float_fact"},{"operation","output"},{"types","{\"float_fact\": {\"arity\": 1, \"auxArity\": 0, \"types\": [\"f:float\"]}, \"records\": {}}"}});-if (!outputDirectory.empty() && directiveMap["filename"].front() != '/') {directiveMap["filename"] = outputDirectory + "/" + directiveMap["filename"];}+try {std::map<std::string, std::string> directiveMap({{"IO","file"},{"attributeNames","x"},{"name","float_fact"},{"operation","output"},{"output-dir","."},{"params","{\"records\": {}, \"relation\": {\"arity\": 1, \"auxArity\": 0, \"params\": [\"x\"]}}"},{"types","{\"ADTs\": {}, \"records\": {}, \"relation\": {\"arity\": 1, \"auxArity\": 0, \"types\": [\"f:float\"]}}"}});+if (!outputDirectory.empty()) {directiveMap["output-dir"] = outputDirectory;} IOSystem::getInstance().getWriter(directiveMap, symTable, recordTable)->writeAll(*rel_1_float_fact); } catch (std::exception& e) {std::cerr << e.what();exit(1);} } }+#ifdef _MSC_VER+#pragma warning(default: 4100)+#endif // _MSC_VER }; SouffleProgram *newInstance_round_trip(){return new Sf_round_trip;} SymbolTable *getST_round_trip(SouffleProgram *p){return &reinterpret_cast<Sf_round_trip*>(p)->symTable;}@@ -380,8 +617,8 @@ { try{ souffle::CmdOptions opt(R"(round_trip.dl)",-R"(.)",-R"(.)",+R"()",+R"()", false, R"()", 1);