packages feed

souffle-haskell 2.1.0 → 4.0.0

raw patch · 74 files changed

Files

CHANGELOG.md view
@@ -3,12 +3,109 @@ All notable changes to this project (as seen by library users) will be documented in this file. The CHANGELOG is available on [Github](https://github.com/luc-tielen/souffle-haskell.git/CHANGELOG.md). +## [4.0.0] - 2024-01-03++### Added++- Support for GHC 9.6++### Removed++- Support for marshalling of Text.Short values. Only UTF8 Text values are+  supported.++## [3.5.1] - 2022-11-07++### Added++- Instances for strict `State` and `RWS` monads for `MonadSouffle`.++### Changed++- Added support for GHC 9.2++## [3.5.0] - 2022-06-04++### Changed++- souffle-haskell now supports Souffle version 2.3.++## [3.4.0] - 2022-05-15++### Changed++- Loosen the constraint that only types that are simple products that only+  contain directly marshallable types like `Int32`, `Word32`, ... Now also+  newtypes are allowed to be serialized, as well as product types inside other+  product types (as long as all the fields implement the 'Marshal' typeclass).++### Fixed++- Check if a type is a simple product type consisting of only types supported by+  Datalog. (This had a small bug for facts with more than 4 arguments.)++## [3.3.0] - 2022-02-27++### Added++- New `DerivingVia`-style API for binding to a Datalog program.++## [3.2.0] - 2022-02-20++### Added++- Add `Analysis` type for composing multiple Datalog programs.++### Changed++- souffle-haskell now supports Souffle version 2.2.++## [3.1.0] - 2021-09-30++### Changed++- souffle-haskell now supports Souffle version 2.1.++### Fixed++- Bug in some C++ assertions that caused the actual assertion message to be+  wrongly computed.++## [3.0.0] - 2021-05-03++### Changed++- Optimized the underlying way of transferring facts between Haskell and C++.+- Add an extra `Submit a` constraint on `addFacts`, `addFact` and `containsFact`+  when using compiled mode, similar to `Collect c` when using `getFacts`.++### Fixed++- Potential memory leak if an async exception occurred between the point of+  allocating a pointer and wrapping it in a `ForeignPtr`.++### Removed++- Souffle EDSL (this is now available in the+  [souffle-dsl](https://github.com/luc-tielen/souffle-dsl)+  package). This is mostly done to improve compile times of this package.+ ## [2.1.0] - 2021-01-03 -- souffle-haskell now supports Souffle version 2.0.2.-- Fix GHC 8.10 specific warnings and compile error.+### Added+ - Support Semigroup and Monoid instances for composing Souffle actions in   other ways.+- Add role annotations to handle types to avoid using `coerce` to change+  the type of a Souffle handle.++### Changed++- souffle-haskell now supports Souffle version 2.0.2.++### Fixed++- Fix GHC 8.10 specific warnings and compile error.  ## [2.0.1] - 2020-09-05
README.md view
@@ -1,7 +1,7 @@ # Souffle-haskell  [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://github.com/luc-tielen/souffle-haskell/blob/master/LICENSE)-[![CircleCI](https://circleci.com/gh/luc-tielen/souffle-haskell.svg?style=svg&circle-token=07fcf633c70820100c529dda8869baa60d4b6dd8)](https://circleci.com/gh/luc-tielen/souffle-haskell)+[![CI](https://github.com/luc-tielen/souffle-haskell/actions/workflows/build.yml/badge.svg)](https://github.com/luc-tielen/souffle-haskell/actions/workflows/build.yml) [![Hackage](https://img.shields.io/hackage/v/souffle-haskell?style=flat-square)](https://hackage.haskell.org/package/souffle-haskell)  This repo provides Haskell bindings for performing analyses with the@@ -43,8 +43,11 @@  ```haskell -- Enable some necessary extensions:-{-# LANGUAGE ScopedTypeVariables, DataKinds, TypeFamilies, DeriveGeneric #-}+{-# LANGUAGE DeriveGeneric, DeriveAnyClass, DerivingVia, DataKinds, UndecidableInstances #-} +-- NOTE: The usage of "deriving stock", "deriving anyclass" and "deriving via" in the+-- examples below matters in order for the library to work correctly!+ module Main ( main ) where  import Data.Foldable ( traverse_ )@@ -54,40 +57,33 @@ import qualified Language.Souffle.Compiled as Souffle  --- We define a data type representing our datalog program.+-- First, we define a data type representing our datalog program. data Path = Path+  -- By making Path an instance of Program, we provide Haskell with information+  -- about the datalog program. It uses this to perform compile-time checks to+  -- limit the amount of possible programmer errors to a minimum.+  deriving Souffle.Program+  via Souffle.ProgramOptions Path "path" '[Edge, Reachable]  -- Facts are represented in Haskell as simple product types, -- Numbers map to Int32, unsigned to Word32, floats to Float, -- symbols to Strings / Text.  data Edge = Edge String String-  deriving (Eq, Show, Generic)+  deriving stock (Eq, Show, Generic)+  -- For simple product types, we can automatically generate the+  -- marshalling/unmarshalling code of data between Haskell and datalog.+  deriving anyclass Souffle.Marshal+  -- By making a data type an instance of Fact, we give Haskell the+  -- necessary information to bind to the datalog fact.+  deriving Souffle.Fact+  via Souffle.FactOptions Edge "edge" 'Souffle.Input  data Reachable = Reachable String String-  deriving (Eq, Show, Generic)---- By making Path an instance of Program, we provide Haskell with information--- about the datalog program. It uses this to perform compile-time checks to--- limit the amount of possible programmer errors to a minimum.-instance Souffle.Program Path where-  type ProgramFacts Path = [Edge, Reachable]-  programName = const "path"---- By making a data type an instance of Fact, we give Haskell the--- necessary information to bind to the datalog fact.-instance Souffle.Fact Edge where-  type FactDirection Edge = 'Souffle.Input-  factName = const "edge"--instance Souffle.Fact Reachable where-  type FactDirection Reachable = 'Souffle.Output-  factName = const "reachable"---- For simple product types, we can automatically generate the--- marshalling/unmarshalling code of data between Haskell and datalog.-instance Souffle.Marshal Edge-instance Souffle.Marshal Reachable+  deriving stock (Eq, Show, Generic)+  deriving anyclass Souffle.Marshal+  deriving Souffle.Fact+  via Souffle.FactOptions Reachable "reachable" 'Souffle.Output   main :: IO ()@@ -159,6 +155,12 @@ ghc-options:   souffle-haskell: -optcxx-std=c++17 ```++## Souffle EDSL++This package previously contained a Haskell EDSL for writing Souffle code+directly in Haskell. This has now been moved to a separate+[package](https://github.com/luc-tielen/souffle-dsl.git).  ## Documentation
+ benchmarks/bench.hs view
@@ -0,0 +1,183 @@+{-# LANGUAGE TypeFamilies #-}+{-# LANGUAGE DataKinds #-}+{-# LANGUAGE DeriveGeneric #-}+{-# LANGUAGE DeriveAnyClass #-}+{-# LANGUAGE FlexibleContexts #-}+module Main ( main ) where++import Criterion.Main+import qualified Language.Souffle.Compiled as S+import qualified Data.Text as T+import qualified Data.Vector as V+import GHC.Generics+import Data.Word+import Data.Int+import Control.Monad+import Control.Monad.IO.Class+import Control.DeepSeq+++data Benchmarks = Benchmarks++data NumbersFact+  = NumbersFact Word32 Int32 Float+  deriving (Generic, NFData)++data StringsFact+  = StringsFact Word32 T.Text Int32 Float+  deriving (Generic, NFData)++newtype FromDatalogFact+  = FromDatalogFact Int32+  deriving (Generic, NFData)++data FromDatalogStringFact+  = FromDatalogStringFact Int32 T.Text+  deriving (Generic, NFData)++instance S.Program Benchmarks where+  type ProgramFacts Benchmarks =+      '[NumbersFact, StringsFact, FromDatalogFact, FromDatalogStringFact]+  programName = const "bench"++instance S.Fact NumbersFact where+  type FactDirection NumbersFact = 'S.InputOutput+  factName = const "numbers_fact"++instance S.Fact StringsFact where+  type FactDirection StringsFact = 'S.InputOutput+  factName = const "strings_fact"++instance S.Fact FromDatalogFact where+  type FactDirection FromDatalogFact = 'S.InputOutput+  factName = const "from_datalog_fact"++instance S.Fact FromDatalogStringFact where+  type FactDirection FromDatalogStringFact = 'S.InputOutput+  factName = const "from_datalog_string_fact"++instance S.Marshal NumbersFact+instance S.Marshal StringsFact+instance S.Marshal FromDatalogFact+instance S.Marshal FromDatalogStringFact+++-- TODO: fix cases with larger numbers (crashes due to large memory allocations?)+main :: IO ()+main = defaultMain+     $ roundTripBenchmarks+    ++ serializationBenchmarks+    ++ deserializationBenchmarks++roundTripBenchmarks :: [Benchmark]+roundTripBenchmarks =+  [ bgroup "round trip facts (without strings)"+    [ bench "1"      $ nfIO $ roundTrip $ mkVec 1+    , bench "10"     $ nfIO $ roundTrip $ mkVec 10+    , bench "100"    $ nfIO $ roundTrip $ mkVec 100+    , bench "1000"   $ nfIO $ roundTrip $ mkVec 1000+    , bench "10000"  $ nfIO $ roundTrip $ mkVec 10000+    --, bench "100000" $ nfIO $ roundTrip $ mkVec 100000+    ]+  , bgroup "round trip facts (with strings)"+    [ bench "1"      $ nfIO $ roundTrip $ mkVecStr 1+    , bench "10"     $ nfIO $ roundTrip $ mkVecStr 10+    , bench "100"    $ nfIO $ roundTrip $ mkVecStr 100+    , bench "1000"   $ nfIO $ roundTrip $ mkVecStr 1000+    --, bench "10000"  $ nfIO $ roundTrip $ mkVecStr 10000+    --, bench "100000" $ nfIO $ roundTrip $ mkVecStr 100000+    ]+  , bgroup "round trip facts (with long strings)"+    [ bench "1"      $ nfIO $ roundTrip $ mkVecLongStr 1+    , bench "10"     $ nfIO $ roundTrip $ mkVecLongStr 10+    , bench "100"    $ nfIO $ roundTrip $ mkVecLongStr 100+    --, bench "1000"   $ nfIO $ roundTrip $ mkVecLongStr 1000+    --, bench "10000"  $ nfIO $ roundTrip $ mkVecLongStr 10000+    --, bench "100000" $ nfIO $ roundTrip $ mkVecLongStr 100000+    ]+  ]+  where mkVec count = V.generate count $ \i -> NumbersFact (fromIntegral i) (-42) 3.14+        mkVecStr count = V.generate count $ \i -> StringsFact (fromIntegral i) "abcdef" (-42) 3.14+        mkVecLongStr count = V.generate count $ \i -> StringsFact (fromIntegral i) (T.replicate 10 "abcdef") (-42) 3.14++roundTrip :: (S.ContainsInputFact Benchmarks a, S.ContainsOutputFact Benchmarks a, S.Fact a, S.Submit a)+          => V.Vector a -> IO (V.Vector a)+roundTrip vec = S.runSouffle Benchmarks $ \case+  Nothing -> do+    liftIO $ print "Failed to load roundtrip benchmarks!"+    pure V.empty+  Just prog -> do+    S.addFacts prog vec+    -- No run needed+    S.getFacts prog+++serializeNumbers :: Int -> IO ()+serializeNumbers iterationCount = S.runSouffle Benchmarks $ \case+  Nothing -> liftIO $ print "Failed to load serialize benchmarks!"+  Just prog ->+    replicateM_ iterationCount $ S.addFacts prog vec+    -- No run needed+  where vec = V.generate 100 $ \i -> NumbersFact (fromIntegral i) (-42) 3.14++deserializeNumbers :: Int -> IO ()+deserializeNumbers iterationCount = S.runSouffle Benchmarks $ \case+  Nothing -> liftIO $ print "Failed to load deserialize benchmarks!"+  Just prog -> do+    S.run prog+    replicateM_ iterationCount $ do+      fs <- S.getFacts prog+      pure (fs :: V.Vector FromDatalogFact)++serializeWithStrings :: Int -> IO ()+serializeWithStrings iterationCount = S.runSouffle Benchmarks $ \case+  Nothing -> liftIO $ print "Failed to load serialize benchmarks!"+  Just prog ->+    replicateM_ iterationCount $ S.addFacts prog vec+    -- No run needed+  where vec = V.generate 100 $ \i -> StringsFact (fromIntegral i) "abcdef" (-42) 3.14++deserializeWithStrings :: Int -> IO ()+deserializeWithStrings iterationCount = S.runSouffle Benchmarks $ \case+  Nothing -> liftIO $ print "Failed to load deserialize benchmarks!"+  Just prog -> do+    S.run prog+    replicateM_ iterationCount $ do+      fs <- S.getFacts prog+      pure (fs :: V.Vector FromDatalogStringFact)++serializationBenchmarks :: [Benchmark]+serializationBenchmarks =+  [ bgroup "serializing facts (without strings)"+    [ bench "1"      $ nfIO $ serializeNumbers 1+    , bench "10"     $ nfIO $ serializeNumbers 10+    , bench "100"    $ nfIO $ serializeNumbers 100+    , bench "1000"   $ nfIO $ serializeNumbers 1000+    , bench "10000"  $ nfIO $ serializeNumbers 10000+    ]+  , bgroup "serializing facts (with strings)"+    [ bench "1"      $ nfIO $ serializeWithStrings 1+    , bench "10"     $ nfIO $ serializeWithStrings 10+    , bench "100"    $ nfIO $ serializeWithStrings 100+    , bench "1000"   $ nfIO $ serializeWithStrings 1000+    , bench "10000"  $ nfIO $ serializeWithStrings 10000+    ]+  ]++deserializationBenchmarks :: [Benchmark]+deserializationBenchmarks =+  [ bgroup "deserializing facts (without strings)"+    [ bench "1"      $ nfIO $ deserializeNumbers 1+    , bench "10"     $ nfIO $ deserializeNumbers 10+    , bench "100"    $ nfIO $ deserializeNumbers 100+    , bench "1000"   $ nfIO $ deserializeNumbers 1000+    , bench "10000"  $ nfIO $ deserializeNumbers 10000+    ]+  , bgroup "deserializing facts (with strings)"+    [ bench "1"      $ nfIO $ deserializeWithStrings 1+    , bench "10"     $ nfIO $ deserializeWithStrings 10+    , bench "100"    $ nfIO $ deserializeWithStrings 100+    , bench "1000"   $ nfIO $ deserializeWithStrings 1000+    , bench "10000"  $ nfIO $ deserializeWithStrings 10000+    ]+  ]
+ benchmarks/fixtures/bench.cpp view
@@ -0,0 +1,891 @@++#include "souffle/CompiledSouffle.h"++extern "C" {+}++namespace souffle {+static const RamDomain RAM_BIT_SHIFT_MASK = RAM_DOMAIN_SIZE - 1;+struct t_btree_u__0__2__1 {+using t_tuple = Tuple<RamDomain, 1>;+struct t_comparator_0{+ int operator()(const t_tuple& a, const t_tuple& b) const {+  return (ramBitCast<RamUnsigned>(a[0]) < ramBitCast<RamUnsigned>(b[0])) ? -1 : (ramBitCast<RamUnsigned>(a[0]) > ramBitCast<RamUnsigned>(b[0])) ? 1 :(0);+ }+bool less(const t_tuple& a, const t_tuple& b) const {+  return (ramBitCast<RamUnsigned>(a[0]) < ramBitCast<RamUnsigned>(b[0]));+ }+bool equal(const t_tuple& a, const t_tuple& b) const {+return (ramBitCast<RamUnsigned>(a[0]) == ramBitCast<RamUnsigned>(b[0]));+ }+};+using t_ind_0 = btree_set<t_tuple,t_comparator_0>;+t_ind_0 ind_0;+using iterator = t_ind_0::iterator;+struct context {+t_ind_0::operation_hints hints_0_lower;+t_ind_0::operation_hints hints_0_upper;+};+context createContext() { return context(); }+bool insert(const t_tuple& t) {+context h;+return insert(t, h);+}+bool insert(const t_tuple& t, context& h) {+if (ind_0.insert(t, h.hints_0_lower)) {+return true;+} else return false;+}+bool insert(const RamDomain* ramDomain) {+RamDomain data[1];+std::copy(ramDomain, ramDomain + 1, data);+const t_tuple& tuple = reinterpret_cast<const t_tuple&>(data);+context h;+return insert(tuple, h);+}+bool insert(RamDomain a0) {+RamDomain data[1] = {a0};+return insert(data);+}+bool contains(const t_tuple& t, context& h) const {+return ind_0.contains(t, h.hints_0_lower);+}+bool contains(const t_tuple& t) const {+context h;+return contains(t, h);+}+std::size_t size() const {+return ind_0.size();+}+iterator find(const t_tuple& t, context& h) const {+return ind_0.find(t, h.hints_0_lower);+}+iterator find(const t_tuple& t) const {+context h;+return find(t, h);+}+range<iterator> lowerUpperRange_0(const t_tuple& /* lower */, const t_tuple& /* upper */, context& /* h */) const {+return range<iterator>(ind_0.begin(),ind_0.end());+}+range<iterator> lowerUpperRange_0(const t_tuple& /* lower */, const t_tuple& /* upper */) const {+return range<iterator>(ind_0.begin(),ind_0.end());+}+range<t_ind_0::iterator> lowerUpperRange_2(const t_tuple& lower, const t_tuple& upper, context& h) const {+t_comparator_0 comparator;+int cmp = comparator(lower, upper);+if (cmp > 0) {+    return make_range(ind_0.end(), ind_0.end());+}+return make_range(ind_0.lower_bound(lower, h.hints_0_lower), ind_0.upper_bound(upper, h.hints_0_upper));+}+range<t_ind_0::iterator> lowerUpperRange_2(const t_tuple& lower, const t_tuple& upper) const {+context h;+return lowerUpperRange_2(lower,upper,h);+}+range<t_ind_0::iterator> lowerUpperRange_1(const t_tuple& lower, const t_tuple& upper, context& h) const {+t_comparator_0 comparator;+int cmp = comparator(lower, upper);+if (cmp == 0) {+    auto pos = ind_0.find(lower, h.hints_0_lower);+    auto fin = ind_0.end();+    if (pos != fin) {fin = pos; ++fin;}+    return make_range(pos, fin);+}+if (cmp > 0) {+    return make_range(ind_0.end(), ind_0.end());+}+return make_range(ind_0.lower_bound(lower, h.hints_0_lower), ind_0.upper_bound(upper, h.hints_0_upper));+}+range<t_ind_0::iterator> lowerUpperRange_1(const t_tuple& lower, const t_tuple& upper) const {+context h;+return lowerUpperRange_1(lower,upper,h);+}+bool empty() const {+return ind_0.empty();+}+std::vector<range<iterator>> partition() const {+return ind_0.getChunks(400);+}+void purge() {+ind_0.clear();+}+iterator begin() const {+return ind_0.begin();+}+iterator end() const {+return ind_0.end();+}+void printStatistics(std::ostream& o) const {+o << " arity 1 direct b-tree index 0 lex-order [0]\n";+ind_0.printStats(o);+}+};+struct t_btree_u__0__1 {+using t_tuple = Tuple<RamDomain, 1>;+struct t_comparator_0{+ int operator()(const t_tuple& a, const t_tuple& b) const {+  return (ramBitCast<RamUnsigned>(a[0]) < ramBitCast<RamUnsigned>(b[0])) ? -1 : (ramBitCast<RamUnsigned>(a[0]) > ramBitCast<RamUnsigned>(b[0])) ? 1 :(0);+ }+bool less(const t_tuple& a, const t_tuple& b) const {+  return (ramBitCast<RamUnsigned>(a[0]) < ramBitCast<RamUnsigned>(b[0]));+ }+bool equal(const t_tuple& a, const t_tuple& b) const {+return (ramBitCast<RamUnsigned>(a[0]) == ramBitCast<RamUnsigned>(b[0]));+ }+};+using t_ind_0 = btree_set<t_tuple,t_comparator_0>;+t_ind_0 ind_0;+using iterator = t_ind_0::iterator;+struct context {+t_ind_0::operation_hints hints_0_lower;+t_ind_0::operation_hints hints_0_upper;+};+context createContext() { return context(); }+bool insert(const t_tuple& t) {+context h;+return insert(t, h);+}+bool insert(const t_tuple& t, context& h) {+if (ind_0.insert(t, h.hints_0_lower)) {+return true;+} else return false;+}+bool insert(const RamDomain* ramDomain) {+RamDomain data[1];+std::copy(ramDomain, ramDomain + 1, data);+const t_tuple& tuple = reinterpret_cast<const t_tuple&>(data);+context h;+return insert(tuple, h);+}+bool insert(RamDomain a0) {+RamDomain data[1] = {a0};+return insert(data);+}+bool contains(const t_tuple& t, context& h) const {+return ind_0.contains(t, h.hints_0_lower);+}+bool contains(const t_tuple& t) const {+context h;+return contains(t, h);+}+std::size_t size() const {+return ind_0.size();+}+iterator find(const t_tuple& t, context& h) const {+return ind_0.find(t, h.hints_0_lower);+}+iterator find(const t_tuple& t) const {+context h;+return find(t, h);+}+range<iterator> lowerUpperRange_0(const t_tuple& /* lower */, const t_tuple& /* upper */, context& /* h */) const {+return range<iterator>(ind_0.begin(),ind_0.end());+}+range<iterator> lowerUpperRange_0(const t_tuple& /* lower */, const t_tuple& /* upper */) const {+return range<iterator>(ind_0.begin(),ind_0.end());+}+range<t_ind_0::iterator> lowerUpperRange_1(const t_tuple& lower, const t_tuple& upper, context& h) const {+t_comparator_0 comparator;+int cmp = comparator(lower, upper);+if (cmp == 0) {+    auto pos = ind_0.find(lower, h.hints_0_lower);+    auto fin = ind_0.end();+    if (pos != fin) {fin = pos; ++fin;}+    return make_range(pos, fin);+}+if (cmp > 0) {+    return make_range(ind_0.end(), ind_0.end());+}+return make_range(ind_0.lower_bound(lower, h.hints_0_lower), ind_0.upper_bound(upper, h.hints_0_upper));+}+range<t_ind_0::iterator> lowerUpperRange_1(const t_tuple& lower, const t_tuple& upper) const {+context h;+return lowerUpperRange_1(lower,upper,h);+}+bool empty() const {+return ind_0.empty();+}+std::vector<range<iterator>> partition() const {+return ind_0.getChunks(400);+}+void purge() {+ind_0.clear();+}+iterator begin() const {+return ind_0.begin();+}+iterator end() const {+return ind_0.end();+}+void printStatistics(std::ostream& o) const {+o << " arity 1 direct b-tree index 0 lex-order [0]\n";+ind_0.printStats(o);+}+};+struct t_btree_ui__0_1__11 {+using t_tuple = Tuple<RamDomain, 2>;+struct t_comparator_0{+ int operator()(const t_tuple& a, const t_tuple& b) const {+  return (ramBitCast<RamUnsigned>(a[0]) < ramBitCast<RamUnsigned>(b[0])) ? -1 : (ramBitCast<RamUnsigned>(a[0]) > ramBitCast<RamUnsigned>(b[0])) ? 1 :((ramBitCast<RamSigned>(a[1]) < ramBitCast<RamSigned>(b[1])) ? -1 : (ramBitCast<RamSigned>(a[1]) > ramBitCast<RamSigned>(b[1])) ? 1 :(0));+ }+bool less(const t_tuple& a, const t_tuple& b) const {+  return (ramBitCast<RamUnsigned>(a[0]) < ramBitCast<RamUnsigned>(b[0]))|| (ramBitCast<RamUnsigned>(a[0]) == ramBitCast<RamUnsigned>(b[0])) && ((ramBitCast<RamSigned>(a[1]) < ramBitCast<RamSigned>(b[1])));+ }+bool equal(const t_tuple& a, const t_tuple& b) const {+return (ramBitCast<RamUnsigned>(a[0]) == ramBitCast<RamUnsigned>(b[0]))&&(ramBitCast<RamSigned>(a[1]) == ramBitCast<RamSigned>(b[1]));+ }+};+using t_ind_0 = btree_set<t_tuple,t_comparator_0>;+t_ind_0 ind_0;+using iterator = t_ind_0::iterator;+struct context {+t_ind_0::operation_hints hints_0_lower;+t_ind_0::operation_hints hints_0_upper;+};+context createContext() { return context(); }+bool insert(const t_tuple& t) {+context h;+return insert(t, h);+}+bool insert(const t_tuple& t, context& h) {+if (ind_0.insert(t, h.hints_0_lower)) {+return true;+} else return false;+}+bool insert(const RamDomain* ramDomain) {+RamDomain data[2];+std::copy(ramDomain, ramDomain + 2, data);+const t_tuple& tuple = reinterpret_cast<const t_tuple&>(data);+context h;+return insert(tuple, h);+}+bool insert(RamDomain a0,RamDomain a1) {+RamDomain data[2] = {a0,a1};+return insert(data);+}+bool contains(const t_tuple& t, context& h) const {+return ind_0.contains(t, h.hints_0_lower);+}+bool contains(const t_tuple& t) const {+context h;+return contains(t, h);+}+std::size_t size() const {+return ind_0.size();+}+iterator find(const t_tuple& t, context& h) const {+return ind_0.find(t, h.hints_0_lower);+}+iterator find(const t_tuple& t) const {+context h;+return find(t, h);+}+range<iterator> lowerUpperRange_00(const t_tuple& /* lower */, const t_tuple& /* upper */, context& /* h */) const {+return range<iterator>(ind_0.begin(),ind_0.end());+}+range<iterator> lowerUpperRange_00(const t_tuple& /* lower */, const t_tuple& /* upper */) const {+return range<iterator>(ind_0.begin(),ind_0.end());+}+range<t_ind_0::iterator> lowerUpperRange_11(const t_tuple& lower, const t_tuple& upper, context& h) const {+t_comparator_0 comparator;+int cmp = comparator(lower, upper);+if (cmp == 0) {+    auto pos = ind_0.find(lower, h.hints_0_lower);+    auto fin = ind_0.end();+    if (pos != fin) {fin = pos; ++fin;}+    return make_range(pos, fin);+}+if (cmp > 0) {+    return make_range(ind_0.end(), ind_0.end());+}+return make_range(ind_0.lower_bound(lower, h.hints_0_lower), ind_0.upper_bound(upper, h.hints_0_upper));+}+range<t_ind_0::iterator> lowerUpperRange_11(const t_tuple& lower, const t_tuple& upper) const {+context h;+return lowerUpperRange_11(lower,upper,h);+}+bool empty() const {+return ind_0.empty();+}+std::vector<range<iterator>> partition() const {+return ind_0.getChunks(400);+}+void purge() {+ind_0.clear();+}+iterator begin() const {+return ind_0.begin();+}+iterator end() const {+return ind_0.end();+}+void printStatistics(std::ostream& o) const {+o << " arity 2 direct b-tree index 0 lex-order [0,1]\n";+ind_0.printStats(o);+}+};+struct t_btree_uif__0_1_2__111 {+using t_tuple = Tuple<RamDomain, 3>;+struct t_comparator_0{+ int operator()(const t_tuple& a, const t_tuple& b) const {+  return (ramBitCast<RamUnsigned>(a[0]) < ramBitCast<RamUnsigned>(b[0])) ? -1 : (ramBitCast<RamUnsigned>(a[0]) > ramBitCast<RamUnsigned>(b[0])) ? 1 :((ramBitCast<RamSigned>(a[1]) < ramBitCast<RamSigned>(b[1])) ? -1 : (ramBitCast<RamSigned>(a[1]) > ramBitCast<RamSigned>(b[1])) ? 1 :((ramBitCast<RamFloat>(a[2]) < ramBitCast<RamFloat>(b[2])) ? -1 : (ramBitCast<RamFloat>(a[2]) > ramBitCast<RamFloat>(b[2])) ? 1 :(0)));+ }+bool less(const t_tuple& a, const t_tuple& b) const {+  return (ramBitCast<RamUnsigned>(a[0]) < ramBitCast<RamUnsigned>(b[0]))|| (ramBitCast<RamUnsigned>(a[0]) == ramBitCast<RamUnsigned>(b[0])) && ((ramBitCast<RamSigned>(a[1]) < ramBitCast<RamSigned>(b[1]))|| (ramBitCast<RamSigned>(a[1]) == ramBitCast<RamSigned>(b[1])) && ((ramBitCast<RamFloat>(a[2]) < ramBitCast<RamFloat>(b[2]))));+ }+bool equal(const t_tuple& a, const t_tuple& b) const {+return (ramBitCast<RamUnsigned>(a[0]) == ramBitCast<RamUnsigned>(b[0]))&&(ramBitCast<RamSigned>(a[1]) == ramBitCast<RamSigned>(b[1]))&&(ramBitCast<RamFloat>(a[2]) == ramBitCast<RamFloat>(b[2]));+ }+};+using t_ind_0 = btree_set<t_tuple,t_comparator_0>;+t_ind_0 ind_0;+using iterator = t_ind_0::iterator;+struct context {+t_ind_0::operation_hints hints_0_lower;+t_ind_0::operation_hints hints_0_upper;+};+context createContext() { return context(); }+bool insert(const t_tuple& t) {+context h;+return insert(t, h);+}+bool insert(const t_tuple& t, context& h) {+if (ind_0.insert(t, h.hints_0_lower)) {+return true;+} else return false;+}+bool insert(const RamDomain* ramDomain) {+RamDomain data[3];+std::copy(ramDomain, ramDomain + 3, data);+const t_tuple& tuple = reinterpret_cast<const t_tuple&>(data);+context h;+return insert(tuple, h);+}+bool insert(RamDomain a0,RamDomain a1,RamDomain a2) {+RamDomain data[3] = {a0,a1,a2};+return insert(data);+}+bool contains(const t_tuple& t, context& h) const {+return ind_0.contains(t, h.hints_0_lower);+}+bool contains(const t_tuple& t) const {+context h;+return contains(t, h);+}+std::size_t size() const {+return ind_0.size();+}+iterator find(const t_tuple& t, context& h) const {+return ind_0.find(t, h.hints_0_lower);+}+iterator find(const t_tuple& t) const {+context h;+return find(t, h);+}+range<iterator> lowerUpperRange_000(const t_tuple& /* lower */, const t_tuple& /* upper */, context& /* h */) const {+return range<iterator>(ind_0.begin(),ind_0.end());+}+range<iterator> lowerUpperRange_000(const t_tuple& /* lower */, const t_tuple& /* upper */) const {+return range<iterator>(ind_0.begin(),ind_0.end());+}+range<t_ind_0::iterator> lowerUpperRange_111(const t_tuple& lower, const t_tuple& upper, context& h) const {+t_comparator_0 comparator;+int cmp = comparator(lower, upper);+if (cmp == 0) {+    auto pos = ind_0.find(lower, h.hints_0_lower);+    auto fin = ind_0.end();+    if (pos != fin) {fin = pos; ++fin;}+    return make_range(pos, fin);+}+if (cmp > 0) {+    return make_range(ind_0.end(), ind_0.end());+}+return make_range(ind_0.lower_bound(lower, h.hints_0_lower), ind_0.upper_bound(upper, h.hints_0_upper));+}+range<t_ind_0::iterator> lowerUpperRange_111(const t_tuple& lower, const t_tuple& upper) const {+context h;+return lowerUpperRange_111(lower,upper,h);+}+bool empty() const {+return ind_0.empty();+}+std::vector<range<iterator>> partition() const {+return ind_0.getChunks(400);+}+void purge() {+ind_0.clear();+}+iterator begin() const {+return ind_0.begin();+}+iterator end() const {+return ind_0.end();+}+void printStatistics(std::ostream& o) const {+o << " arity 3 direct b-tree index 0 lex-order [0,1,2]\n";+ind_0.printStats(o);+}+};+struct t_btree_uiif__0_1_2_3__1111 {+using t_tuple = Tuple<RamDomain, 4>;+struct t_comparator_0{+ int operator()(const t_tuple& a, const t_tuple& b) const {+  return (ramBitCast<RamUnsigned>(a[0]) < ramBitCast<RamUnsigned>(b[0])) ? -1 : (ramBitCast<RamUnsigned>(a[0]) > ramBitCast<RamUnsigned>(b[0])) ? 1 :((ramBitCast<RamSigned>(a[1]) < ramBitCast<RamSigned>(b[1])) ? -1 : (ramBitCast<RamSigned>(a[1]) > ramBitCast<RamSigned>(b[1])) ? 1 :((ramBitCast<RamSigned>(a[2]) < ramBitCast<RamSigned>(b[2])) ? -1 : (ramBitCast<RamSigned>(a[2]) > ramBitCast<RamSigned>(b[2])) ? 1 :((ramBitCast<RamFloat>(a[3]) < ramBitCast<RamFloat>(b[3])) ? -1 : (ramBitCast<RamFloat>(a[3]) > ramBitCast<RamFloat>(b[3])) ? 1 :(0))));+ }+bool less(const t_tuple& a, const t_tuple& b) const {+  return (ramBitCast<RamUnsigned>(a[0]) < ramBitCast<RamUnsigned>(b[0]))|| (ramBitCast<RamUnsigned>(a[0]) == ramBitCast<RamUnsigned>(b[0])) && ((ramBitCast<RamSigned>(a[1]) < ramBitCast<RamSigned>(b[1]))|| (ramBitCast<RamSigned>(a[1]) == ramBitCast<RamSigned>(b[1])) && ((ramBitCast<RamSigned>(a[2]) < ramBitCast<RamSigned>(b[2]))|| (ramBitCast<RamSigned>(a[2]) == ramBitCast<RamSigned>(b[2])) && ((ramBitCast<RamFloat>(a[3]) < ramBitCast<RamFloat>(b[3])))));+ }+bool equal(const t_tuple& a, const t_tuple& b) const {+return (ramBitCast<RamUnsigned>(a[0]) == ramBitCast<RamUnsigned>(b[0]))&&(ramBitCast<RamSigned>(a[1]) == ramBitCast<RamSigned>(b[1]))&&(ramBitCast<RamSigned>(a[2]) == ramBitCast<RamSigned>(b[2]))&&(ramBitCast<RamFloat>(a[3]) == ramBitCast<RamFloat>(b[3]));+ }+};+using t_ind_0 = btree_set<t_tuple,t_comparator_0>;+t_ind_0 ind_0;+using iterator = t_ind_0::iterator;+struct context {+t_ind_0::operation_hints hints_0_lower;+t_ind_0::operation_hints hints_0_upper;+};+context createContext() { return context(); }+bool insert(const t_tuple& t) {+context h;+return insert(t, h);+}+bool insert(const t_tuple& t, context& h) {+if (ind_0.insert(t, h.hints_0_lower)) {+return true;+} else return false;+}+bool insert(const RamDomain* ramDomain) {+RamDomain data[4];+std::copy(ramDomain, ramDomain + 4, data);+const t_tuple& tuple = reinterpret_cast<const t_tuple&>(data);+context h;+return insert(tuple, h);+}+bool insert(RamDomain a0,RamDomain a1,RamDomain a2,RamDomain a3) {+RamDomain data[4] = {a0,a1,a2,a3};+return insert(data);+}+bool contains(const t_tuple& t, context& h) const {+return ind_0.contains(t, h.hints_0_lower);+}+bool contains(const t_tuple& t) const {+context h;+return contains(t, h);+}+std::size_t size() const {+return ind_0.size();+}+iterator find(const t_tuple& t, context& h) const {+return ind_0.find(t, h.hints_0_lower);+}+iterator find(const t_tuple& t) const {+context h;+return find(t, h);+}+range<iterator> lowerUpperRange_0000(const t_tuple& /* lower */, const t_tuple& /* upper */, context& /* h */) const {+return range<iterator>(ind_0.begin(),ind_0.end());+}+range<iterator> lowerUpperRange_0000(const t_tuple& /* lower */, const t_tuple& /* upper */) const {+return range<iterator>(ind_0.begin(),ind_0.end());+}+range<t_ind_0::iterator> lowerUpperRange_1111(const t_tuple& lower, const t_tuple& upper, context& h) const {+t_comparator_0 comparator;+int cmp = comparator(lower, upper);+if (cmp == 0) {+    auto pos = ind_0.find(lower, h.hints_0_lower);+    auto fin = ind_0.end();+    if (pos != fin) {fin = pos; ++fin;}+    return make_range(pos, fin);+}+if (cmp > 0) {+    return make_range(ind_0.end(), ind_0.end());+}+return make_range(ind_0.lower_bound(lower, h.hints_0_lower), ind_0.upper_bound(upper, h.hints_0_upper));+}+range<t_ind_0::iterator> lowerUpperRange_1111(const t_tuple& lower, const t_tuple& upper) const {+context h;+return lowerUpperRange_1111(lower,upper,h);+}+bool empty() const {+return ind_0.empty();+}+std::vector<range<iterator>> partition() const {+return ind_0.getChunks(400);+}+void purge() {+ind_0.clear();+}+iterator begin() const {+return ind_0.begin();+}+iterator end() const {+return ind_0.end();+}+void printStatistics(std::ostream& o) const {+o << " arity 4 direct b-tree index 0 lex-order [0,1,2,3]\n";+ind_0.printStats(o);+}+};++class Sf_bench : public SouffleProgram {+private:+static inline bool regex_wrapper(const std::string& pattern, const std::string& text) {+   bool result = false; +   try { result = std::regex_match(text, std::regex(pattern)); } catch(...) { +     std::cerr << "warning: wrong pattern provided for match(\"" << pattern << "\",\"" << text << "\").\n";+}+   return result;+}+private:+static inline std::string substr_wrapper(const std::string& str, size_t idx, size_t len) {+   std::string result; +   try { result = str.substr(idx,len); } catch(...) { +     std::cerr << "warning: wrong index position provided by substr(\"";+     std::cerr << str << "\"," << (int32_t)idx << "," << (int32_t)len << ") functor.\n";+   } return result;+}+public:+// -- initialize symbol table --+SymbolTable symTable{+	R"_(abcdef)_",+};// -- initialize record table --+RecordTable recordTable;+// -- Table: @delta_from_datalog_fact+Own<t_btree_u__0__2__1> rel_1_delta_from_datalog_fact = mk<t_btree_u__0__2__1>();+// -- Table: @new_from_datalog_fact+Own<t_btree_u__0__2__1> rel_2_new_from_datalog_fact = mk<t_btree_u__0__2__1>();+// -- Table: from_datalog_fact+Own<t_btree_u__0__1> rel_3_from_datalog_fact = mk<t_btree_u__0__1>();+souffle::RelationWrapper<0,t_btree_u__0__1,Tuple<RamDomain,1>,1,0> wrapper_rel_3_from_datalog_fact;+// -- Table: from_datalog_string_fact+Own<t_btree_ui__0_1__11> rel_4_from_datalog_string_fact = mk<t_btree_ui__0_1__11>();+souffle::RelationWrapper<1,t_btree_ui__0_1__11,Tuple<RamDomain,2>,2,0> wrapper_rel_4_from_datalog_string_fact;+// -- Table: numbers_fact+Own<t_btree_uif__0_1_2__111> rel_5_numbers_fact = mk<t_btree_uif__0_1_2__111>();+souffle::RelationWrapper<2,t_btree_uif__0_1_2__111,Tuple<RamDomain,3>,3,0> wrapper_rel_5_numbers_fact;+// -- Table: strings_fact+Own<t_btree_uiif__0_1_2_3__1111> rel_6_strings_fact = mk<t_btree_uiif__0_1_2_3__1111>();+souffle::RelationWrapper<3,t_btree_uiif__0_1_2_3__1111,Tuple<RamDomain,4>,4,0> wrapper_rel_6_strings_fact;+public:+Sf_bench() : +wrapper_rel_3_from_datalog_fact(*rel_3_from_datalog_fact,symTable,"from_datalog_fact",std::array<const char *,1>{{"u:unsigned"}},std::array<const char *,1>{{"u"}}),++wrapper_rel_4_from_datalog_string_fact(*rel_4_from_datalog_string_fact,symTable,"from_datalog_string_fact",std::array<const char *,2>{{"u:unsigned","s:symbol"}},std::array<const char *,2>{{"u","s"}}),++wrapper_rel_5_numbers_fact(*rel_5_numbers_fact,symTable,"numbers_fact",std::array<const char *,3>{{"u:unsigned","i:number","f:float"}},std::array<const char *,3>{{"u","n","f"}}),++wrapper_rel_6_strings_fact(*rel_6_strings_fact,symTable,"strings_fact",std::array<const char *,4>{{"u:unsigned","s:symbol","i:number","f:float"}},std::array<const char *,4>{{"u","s","n","f"}}){+addRelation("from_datalog_fact",&wrapper_rel_3_from_datalog_fact,false,true);+addRelation("from_datalog_string_fact",&wrapper_rel_4_from_datalog_string_fact,false,true);+addRelation("numbers_fact",&wrapper_rel_5_numbers_fact,true,true);+addRelation("strings_fact",&wrapper_rel_6_strings_fact,true,true);+}+~Sf_bench() {+}+private:+std::string inputDirectory;+std::string outputDirectory;+bool performIO;+std::atomic<RamDomain> ctr{};++std::atomic<size_t> iter{};+void runFunction(std::string inputDirectoryArg = "", std::string outputDirectoryArg = "", bool performIOArg = false) {+this->inputDirectory = inputDirectoryArg;+this->outputDirectory = outputDirectoryArg;+this->performIO = performIOArg;+SignalHandler::instance()->set();+#if defined(_OPENMP)+if (getNumThreads() > 0) {omp_set_num_threads(getNumThreads());}+#endif++// -- query evaluation --+{+ std::vector<RamDomain> args, ret;+subroutine_0(args, ret);+}+{+ std::vector<RamDomain> args, ret;+subroutine_1(args, ret);+}+{+ std::vector<RamDomain> args, ret;+subroutine_2(args, ret);+}+{+ std::vector<RamDomain> args, ret;+subroutine_3(args, ret);+}++// -- relation hint statistics --+SignalHandler::instance()->reset();+}+public:+void run() override { runFunction("", "", false); }+public:+void runAll(std::string inputDirectoryArg = "", std::string outputDirectoryArg = "") override { runFunction(inputDirectoryArg, outputDirectoryArg, true);+}+public:+void printAll(std::string outputDirectoryArg = "") override {+try {std::map<std::string, std::string> directiveMap({{"IO","file"},{"attributeNames","u"},{"name","from_datalog_fact"},{"operation","output"},{"output-dir","."},{"params","{\"records\": {}, \"relation\": {\"arity\": 1, \"auxArity\": 0, \"params\": [\"u\"]}}"},{"types","{\"ADTs\": {}, \"records\": {}, \"relation\": {\"arity\": 1, \"auxArity\": 0, \"types\": [\"u:unsigned\"]}}"}});+if (!outputDirectoryArg.empty()) {directiveMap["output-dir"] = outputDirectoryArg;}+IOSystem::getInstance().getWriter(directiveMap, symTable, recordTable)->writeAll(*rel_3_from_datalog_fact);+} catch (std::exception& e) {std::cerr << e.what();exit(1);}+try {std::map<std::string, std::string> directiveMap({{"IO","file"},{"attributeNames","u\tn\tf"},{"name","numbers_fact"},{"operation","output"},{"output-dir","."},{"params","{\"records\": {}, \"relation\": {\"arity\": 3, \"auxArity\": 0, \"params\": [\"u\", \"n\", \"f\"]}}"},{"types","{\"ADTs\": {}, \"records\": {}, \"relation\": {\"arity\": 3, \"auxArity\": 0, \"types\": [\"u:unsigned\", \"i:number\", \"f:float\"]}}"}});+if (!outputDirectoryArg.empty()) {directiveMap["output-dir"] = outputDirectoryArg;}+IOSystem::getInstance().getWriter(directiveMap, symTable, recordTable)->writeAll(*rel_5_numbers_fact);+} catch (std::exception& e) {std::cerr << e.what();exit(1);}+try {std::map<std::string, std::string> directiveMap({{"IO","file"},{"attributeNames","u\ts\tn\tf"},{"name","strings_fact"},{"operation","output"},{"output-dir","."},{"params","{\"records\": {}, \"relation\": {\"arity\": 4, \"auxArity\": 0, \"params\": [\"u\", \"s\", \"n\", \"f\"]}}"},{"types","{\"ADTs\": {}, \"records\": {}, \"relation\": {\"arity\": 4, \"auxArity\": 0, \"types\": [\"u:unsigned\", \"s:symbol\", \"i:number\", \"f:float\"]}}"}});+if (!outputDirectoryArg.empty()) {directiveMap["output-dir"] = outputDirectoryArg;}+IOSystem::getInstance().getWriter(directiveMap, symTable, recordTable)->writeAll(*rel_6_strings_fact);+} catch (std::exception& e) {std::cerr << e.what();exit(1);}+try {std::map<std::string, std::string> directiveMap({{"IO","file"},{"attributeNames","u\ts"},{"name","from_datalog_string_fact"},{"operation","output"},{"output-dir","."},{"params","{\"records\": {}, \"relation\": {\"arity\": 2, \"auxArity\": 0, \"params\": [\"u\", \"s\"]}}"},{"types","{\"ADTs\": {}, \"records\": {}, \"relation\": {\"arity\": 2, \"auxArity\": 0, \"types\": [\"u:unsigned\", \"s:symbol\"]}}"}});+if (!outputDirectoryArg.empty()) {directiveMap["output-dir"] = outputDirectoryArg;}+IOSystem::getInstance().getWriter(directiveMap, symTable, recordTable)->writeAll(*rel_4_from_datalog_string_fact);+} catch (std::exception& e) {std::cerr << e.what();exit(1);}+}+public:+void loadAll(std::string inputDirectoryArg = "") override {+try {std::map<std::string, std::string> directiveMap({{"IO","file"},{"attributeNames","u\tn\tf"},{"fact-dir","."},{"name","numbers_fact"},{"operation","input"},{"params","{\"records\": {}, \"relation\": {\"arity\": 3, \"auxArity\": 0, \"params\": [\"u\", \"n\", \"f\"]}}"},{"types","{\"ADTs\": {}, \"records\": {}, \"relation\": {\"arity\": 3, \"auxArity\": 0, \"types\": [\"u:unsigned\", \"i:number\", \"f:float\"]}}"}});+if (!inputDirectoryArg.empty()) {directiveMap["fact-dir"] = inputDirectoryArg;}+IOSystem::getInstance().getReader(directiveMap, symTable, recordTable)->readAll(*rel_5_numbers_fact);+} catch (std::exception& e) {std::cerr << "Error loading data: " << e.what() << '\n';}+try {std::map<std::string, std::string> directiveMap({{"IO","file"},{"attributeNames","u\ts\tn\tf"},{"fact-dir","."},{"name","strings_fact"},{"operation","input"},{"params","{\"records\": {}, \"relation\": {\"arity\": 4, \"auxArity\": 0, \"params\": [\"u\", \"s\", \"n\", \"f\"]}}"},{"types","{\"ADTs\": {}, \"records\": {}, \"relation\": {\"arity\": 4, \"auxArity\": 0, \"types\": [\"u:unsigned\", \"s:symbol\", \"i:number\", \"f:float\"]}}"}});+if (!inputDirectoryArg.empty()) {directiveMap["fact-dir"] = inputDirectoryArg;}+IOSystem::getInstance().getReader(directiveMap, symTable, recordTable)->readAll(*rel_6_strings_fact);+} catch (std::exception& e) {std::cerr << "Error loading data: " << e.what() << '\n';}+}+public:+void dumpInputs() override {+try {std::map<std::string, std::string> rwOperation;+rwOperation["IO"] = "stdout";+rwOperation["name"] = "numbers_fact";+rwOperation["types"] = "{\"relation\": {\"arity\": 3, \"auxArity\": 0, \"types\": [\"u:unsigned\", \"i:number\", \"f:float\"]}}";+IOSystem::getInstance().getWriter(rwOperation, symTable, recordTable)->writeAll(*rel_5_numbers_fact);+} catch (std::exception& e) {std::cerr << e.what();exit(1);}+try {std::map<std::string, std::string> rwOperation;+rwOperation["IO"] = "stdout";+rwOperation["name"] = "strings_fact";+rwOperation["types"] = "{\"relation\": {\"arity\": 4, \"auxArity\": 0, \"types\": [\"u:unsigned\", \"s:symbol\", \"i:number\", \"f:float\"]}}";+IOSystem::getInstance().getWriter(rwOperation, symTable, recordTable)->writeAll(*rel_6_strings_fact);+} catch (std::exception& e) {std::cerr << e.what();exit(1);}+}+public:+void dumpOutputs() override {+try {std::map<std::string, std::string> rwOperation;+rwOperation["IO"] = "stdout";+rwOperation["name"] = "from_datalog_fact";+rwOperation["types"] = "{\"relation\": {\"arity\": 1, \"auxArity\": 0, \"types\": [\"u:unsigned\"]}}";+IOSystem::getInstance().getWriter(rwOperation, symTable, recordTable)->writeAll(*rel_3_from_datalog_fact);+} catch (std::exception& e) {std::cerr << e.what();exit(1);}+try {std::map<std::string, std::string> rwOperation;+rwOperation["IO"] = "stdout";+rwOperation["name"] = "numbers_fact";+rwOperation["types"] = "{\"relation\": {\"arity\": 3, \"auxArity\": 0, \"types\": [\"u:unsigned\", \"i:number\", \"f:float\"]}}";+IOSystem::getInstance().getWriter(rwOperation, symTable, recordTable)->writeAll(*rel_5_numbers_fact);+} catch (std::exception& e) {std::cerr << e.what();exit(1);}+try {std::map<std::string, std::string> rwOperation;+rwOperation["IO"] = "stdout";+rwOperation["name"] = "strings_fact";+rwOperation["types"] = "{\"relation\": {\"arity\": 4, \"auxArity\": 0, \"types\": [\"u:unsigned\", \"s:symbol\", \"i:number\", \"f:float\"]}}";+IOSystem::getInstance().getWriter(rwOperation, symTable, recordTable)->writeAll(*rel_6_strings_fact);+} catch (std::exception& e) {std::cerr << e.what();exit(1);}+try {std::map<std::string, std::string> rwOperation;+rwOperation["IO"] = "stdout";+rwOperation["name"] = "from_datalog_string_fact";+rwOperation["types"] = "{\"relation\": {\"arity\": 2, \"auxArity\": 0, \"types\": [\"u:unsigned\", \"s:symbol\"]}}";+IOSystem::getInstance().getWriter(rwOperation, symTable, recordTable)->writeAll(*rel_4_from_datalog_string_fact);+} catch (std::exception& e) {std::cerr << e.what();exit(1);}+}+public:+SymbolTable& getSymbolTable() override {+return symTable;+}+void executeSubroutine(std::string name, const std::vector<RamDomain>& args, std::vector<RamDomain>& ret) override {+if (name == "stratum_0") {+subroutine_0(args, ret);+return;}+if (name == "stratum_1") {+subroutine_1(args, ret);+return;}+if (name == "stratum_2") {+subroutine_2(args, ret);+return;}+if (name == "stratum_3") {+subroutine_3(args, ret);+return;}+fatal("unknown subroutine");+}+#ifdef _MSC_VER+#pragma warning(disable: 4100)+#endif // _MSC_VER+void subroutine_0(const std::vector<RamDomain>& args, std::vector<RamDomain>& ret) {+if (performIO) {+try {std::map<std::string, std::string> directiveMap({{"IO","file"},{"attributeNames","u\tn\tf"},{"fact-dir","."},{"name","numbers_fact"},{"operation","input"},{"params","{\"records\": {}, \"relation\": {\"arity\": 3, \"auxArity\": 0, \"params\": [\"u\", \"n\", \"f\"]}}"},{"types","{\"ADTs\": {}, \"records\": {}, \"relation\": {\"arity\": 3, \"auxArity\": 0, \"types\": [\"u:unsigned\", \"i:number\", \"f:float\"]}}"}});+if (!inputDirectory.empty()) {directiveMap["fact-dir"] = inputDirectory;}+IOSystem::getInstance().getReader(directiveMap, symTable, recordTable)->readAll(*rel_5_numbers_fact);+} catch (std::exception& e) {std::cerr << "Error loading data: " << e.what() << '\n';}+}+if (performIO) {+try {std::map<std::string, std::string> directiveMap({{"IO","file"},{"attributeNames","u\tn\tf"},{"name","numbers_fact"},{"operation","output"},{"output-dir","."},{"params","{\"records\": {}, \"relation\": {\"arity\": 3, \"auxArity\": 0, \"params\": [\"u\", \"n\", \"f\"]}}"},{"types","{\"ADTs\": {}, \"records\": {}, \"relation\": {\"arity\": 3, \"auxArity\": 0, \"types\": [\"u:unsigned\", \"i:number\", \"f:float\"]}}"}});+if (!outputDirectory.empty()) {directiveMap["output-dir"] = outputDirectory;}+IOSystem::getInstance().getWriter(directiveMap, symTable, recordTable)->writeAll(*rel_5_numbers_fact);+} catch (std::exception& e) {std::cerr << e.what();exit(1);}+}+}+#ifdef _MSC_VER+#pragma warning(default: 4100)+#endif // _MSC_VER+#ifdef _MSC_VER+#pragma warning(disable: 4100)+#endif // _MSC_VER+void subroutine_1(const std::vector<RamDomain>& args, std::vector<RamDomain>& ret) {+if (performIO) {+try {std::map<std::string, std::string> directiveMap({{"IO","file"},{"attributeNames","u\ts\tn\tf"},{"fact-dir","."},{"name","strings_fact"},{"operation","input"},{"params","{\"records\": {}, \"relation\": {\"arity\": 4, \"auxArity\": 0, \"params\": [\"u\", \"s\", \"n\", \"f\"]}}"},{"types","{\"ADTs\": {}, \"records\": {}, \"relation\": {\"arity\": 4, \"auxArity\": 0, \"types\": [\"u:unsigned\", \"s:symbol\", \"i:number\", \"f:float\"]}}"}});+if (!inputDirectory.empty()) {directiveMap["fact-dir"] = inputDirectory;}+IOSystem::getInstance().getReader(directiveMap, symTable, recordTable)->readAll(*rel_6_strings_fact);+} catch (std::exception& e) {std::cerr << "Error loading data: " << e.what() << '\n';}+}+if (performIO) {+try {std::map<std::string, std::string> directiveMap({{"IO","file"},{"attributeNames","u\ts\tn\tf"},{"name","strings_fact"},{"operation","output"},{"output-dir","."},{"params","{\"records\": {}, \"relation\": {\"arity\": 4, \"auxArity\": 0, \"params\": [\"u\", \"s\", \"n\", \"f\"]}}"},{"types","{\"ADTs\": {}, \"records\": {}, \"relation\": {\"arity\": 4, \"auxArity\": 0, \"types\": [\"u:unsigned\", \"s:symbol\", \"i:number\", \"f:float\"]}}"}});+if (!outputDirectory.empty()) {directiveMap["output-dir"] = outputDirectory;}+IOSystem::getInstance().getWriter(directiveMap, symTable, recordTable)->writeAll(*rel_6_strings_fact);+} catch (std::exception& e) {std::cerr << e.what();exit(1);}+}+}+#ifdef _MSC_VER+#pragma warning(default: 4100)+#endif // _MSC_VER+#ifdef _MSC_VER+#pragma warning(disable: 4100)+#endif // _MSC_VER+void subroutine_2(const std::vector<RamDomain>& args, std::vector<RamDomain>& ret) {+SignalHandler::instance()->setMsg(R"_(from_datalog_fact(0).+in file /home/luc/souffle-haskell/benchmarks/fixtures/bench.dl [18:1-18:22])_");+[&](){+CREATE_OP_CONTEXT(rel_3_from_datalog_fact_op_ctxt,rel_3_from_datalog_fact->createContext());+Tuple<RamDomain,1> tuple{{ramBitCast(RamUnsigned(0))}};+rel_3_from_datalog_fact->insert(tuple,READ_OP_CONTEXT(rel_3_from_datalog_fact_op_ctxt));+}+();[&](){+CREATE_OP_CONTEXT(rel_3_from_datalog_fact_op_ctxt,rel_3_from_datalog_fact->createContext());+CREATE_OP_CONTEXT(rel_1_delta_from_datalog_fact_op_ctxt,rel_1_delta_from_datalog_fact->createContext());+for(const auto& env0 : *rel_3_from_datalog_fact) {+Tuple<RamDomain,1> tuple{{ramBitCast(env0[0])}};+rel_1_delta_from_datalog_fact->insert(tuple,READ_OP_CONTEXT(rel_1_delta_from_datalog_fact_op_ctxt));+}+}+();iter = 0;+for(;;) {+SignalHandler::instance()->setMsg(R"_(from_datalog_fact((x+1)) :- +   from_datalog_fact(x),+   x < 100.+in file /home/luc/souffle-haskell/benchmarks/fixtures/bench.dl [19:1-21:11])_");+if(!(rel_1_delta_from_datalog_fact->empty())) {+[&](){+CREATE_OP_CONTEXT(rel_3_from_datalog_fact_op_ctxt,rel_3_from_datalog_fact->createContext());+CREATE_OP_CONTEXT(rel_1_delta_from_datalog_fact_op_ctxt,rel_1_delta_from_datalog_fact->createContext());+CREATE_OP_CONTEXT(rel_2_new_from_datalog_fact_op_ctxt,rel_2_new_from_datalog_fact->createContext());+auto range = rel_1_delta_from_datalog_fact->lowerUpperRange_2(Tuple<RamDomain,1>{{ramBitCast<RamDomain>(MIN_RAM_UNSIGNED)}},Tuple<RamDomain,1>{{ramBitCast(RamUnsigned(100))}},READ_OP_CONTEXT(rel_1_delta_from_datalog_fact_op_ctxt));+for(const auto& env0 : range) {+if( (ramBitCast<RamDomain>(env0[0]) != ramBitCast<RamDomain>(RamUnsigned(100))) && !(rel_3_from_datalog_fact->contains(Tuple<RamDomain,1>{{ramBitCast((ramBitCast<RamUnsigned>(env0[0]) + ramBitCast<RamUnsigned>(RamUnsigned(1))))}},READ_OP_CONTEXT(rel_3_from_datalog_fact_op_ctxt)))) {+Tuple<RamDomain,1> tuple{{ramBitCast((ramBitCast<RamUnsigned>(env0[0]) + ramBitCast<RamUnsigned>(RamUnsigned(1))))}};+rel_2_new_from_datalog_fact->insert(tuple,READ_OP_CONTEXT(rel_2_new_from_datalog_fact_op_ctxt));+}+}+}+();}+if(rel_2_new_from_datalog_fact->empty()) break;+[&](){+CREATE_OP_CONTEXT(rel_3_from_datalog_fact_op_ctxt,rel_3_from_datalog_fact->createContext());+CREATE_OP_CONTEXT(rel_2_new_from_datalog_fact_op_ctxt,rel_2_new_from_datalog_fact->createContext());+for(const auto& env0 : *rel_2_new_from_datalog_fact) {+Tuple<RamDomain,1> tuple{{ramBitCast(env0[0])}};+rel_3_from_datalog_fact->insert(tuple,READ_OP_CONTEXT(rel_3_from_datalog_fact_op_ctxt));+}+}+();std::swap(rel_1_delta_from_datalog_fact, rel_2_new_from_datalog_fact);+rel_2_new_from_datalog_fact->purge();+iter++;+}+iter = 0;+rel_1_delta_from_datalog_fact->purge();+rel_2_new_from_datalog_fact->purge();+if (performIO) {+try {std::map<std::string, std::string> directiveMap({{"IO","file"},{"attributeNames","u"},{"name","from_datalog_fact"},{"operation","output"},{"output-dir","."},{"params","{\"records\": {}, \"relation\": {\"arity\": 1, \"auxArity\": 0, \"params\": [\"u\"]}}"},{"types","{\"ADTs\": {}, \"records\": {}, \"relation\": {\"arity\": 1, \"auxArity\": 0, \"types\": [\"u:unsigned\"]}}"}});+if (!outputDirectory.empty()) {directiveMap["output-dir"] = outputDirectory;}+IOSystem::getInstance().getWriter(directiveMap, symTable, recordTable)->writeAll(*rel_3_from_datalog_fact);+} catch (std::exception& e) {std::cerr << e.what();exit(1);}+}+}+#ifdef _MSC_VER+#pragma warning(default: 4100)+#endif // _MSC_VER+#ifdef _MSC_VER+#pragma warning(disable: 4100)+#endif // _MSC_VER+void subroutine_3(const std::vector<RamDomain>& args, std::vector<RamDomain>& ret) {+SignalHandler::instance()->setMsg(R"_(from_datalog_string_fact(x,"abcdef") :- +   from_datalog_fact(x).+in file /home/luc/souffle-haskell/benchmarks/fixtures/bench.dl [23:1-24:24])_");+if(!(rel_3_from_datalog_fact->empty())) {+[&](){+CREATE_OP_CONTEXT(rel_3_from_datalog_fact_op_ctxt,rel_3_from_datalog_fact->createContext());+CREATE_OP_CONTEXT(rel_4_from_datalog_string_fact_op_ctxt,rel_4_from_datalog_string_fact->createContext());+for(const auto& env0 : *rel_3_from_datalog_fact) {+Tuple<RamDomain,2> tuple{{ramBitCast(env0[0]),ramBitCast(RamSigned(0))}};+rel_4_from_datalog_string_fact->insert(tuple,READ_OP_CONTEXT(rel_4_from_datalog_string_fact_op_ctxt));+}+}+();}+if (performIO) {+try {std::map<std::string, std::string> directiveMap({{"IO","file"},{"attributeNames","u\ts"},{"name","from_datalog_string_fact"},{"operation","output"},{"output-dir","."},{"params","{\"records\": {}, \"relation\": {\"arity\": 2, \"auxArity\": 0, \"params\": [\"u\", \"s\"]}}"},{"types","{\"ADTs\": {}, \"records\": {}, \"relation\": {\"arity\": 2, \"auxArity\": 0, \"types\": [\"u:unsigned\", \"s:symbol\"]}}"}});+if (!outputDirectory.empty()) {directiveMap["output-dir"] = outputDirectory;}+IOSystem::getInstance().getWriter(directiveMap, symTable, recordTable)->writeAll(*rel_4_from_datalog_string_fact);+} catch (std::exception& e) {std::cerr << e.what();exit(1);}+}+if (performIO) rel_3_from_datalog_fact->purge();+}+#ifdef _MSC_VER+#pragma warning(default: 4100)+#endif // _MSC_VER+};+SouffleProgram *newInstance_bench(){return new Sf_bench;}+SymbolTable *getST_bench(SouffleProgram *p){return &reinterpret_cast<Sf_bench*>(p)->symTable;}++#ifdef __EMBEDDED_SOUFFLE__+class factory_Sf_bench: public souffle::ProgramFactory {+SouffleProgram *newInstance() {+return new Sf_bench();+};+public:+factory_Sf_bench() : ProgramFactory("bench"){}+};+extern "C" {+factory_Sf_bench __factory_Sf_bench_instance;+}+}+#else+}+int main(int argc, char** argv)+{+try{+souffle::CmdOptions opt(R"(bench.dl)",+R"()",+R"()",+false,+R"()",+1);+if (!opt.parse(argc,argv)) return 1;+souffle::Sf_bench obj;+#if defined(_OPENMP) +obj.setNumThreads(opt.getNumJobs());++#endif+obj.runAll(opt.getInputFileDir(), opt.getOutputFileDir());+return 0;+} catch(std::exception &e) { souffle::SignalHandler::instance()->error(e.what());}+}++#endif
cbits/souffle.cpp view
@@ -1,196 +1,509 @@ #include "souffle/SouffleInterface.h" #include "souffle.h"+#include <algorithm>+#include <string>+#include <unordered_map>+#include <vector>+#include <memory> +#ifndef ESTIMATED_AVERAGE_STRING_SIZE+#define ESTIMATED_AVERAGE_STRING_SIZE 32+#endif+#ifndef GROW_FACTOR+#define GROW_FACTOR 2+#endif++extern "C"+{++struct buf_data+{+private:+    std::unique_ptr<char[]> m_data;+    size_t m_size;++public:+    buf_data(size_t size)+        : m_data(std::make_unique<char[]>(size))+        , m_size(size)+    {+        assert(size);+    };++    void resize(size_t num_bytes)+    {+        m_data = std::make_unique<char[]>(num_bytes);+        m_size = num_bytes;+    }++    auto size() const+    {+        return m_size;+    }++    auto data() const+    {+        return m_data.get();+    }+};++struct souffle_interface+{+    std::unique_ptr<souffle::SouffleProgram> m_prog;+    buf_data m_buf;++    souffle_interface(souffle::SouffleProgram *prog)+        : m_prog(prog)+        , m_buf(4)+    {+        assert(prog);+    };++    char *get_buf(size_t num_bytes)+    {+        if (num_bytes > m_buf.size()) m_buf.resize(num_bytes);++        return m_buf.data();+    }+};++}++namespace helpers+{+using souffle_type = char;++inline auto parse_signature(const souffle::Relation& relation)+{+    const auto arity = relation.getArity();++    std::vector<souffle_type> types;+    types.reserve(arity);++    for (size_t i = 0; i < arity; ++i)+    {+        types.push_back(*relation.getAttrType(i));+    }++    return types;+}++inline bool relation_contains_strings(const souffle::Relation& relation)+{+    const auto types = parse_signature(relation);+    return std::any_of(types.begin(), types.end(), [](auto x) { return x == 's'; });+}++using offset_t = uint32_t;++using number_t = int32_t;+using unsigned_t = uint32_t;+using float_t = float;+ template <typename T>-void tuple_push_value(tuple_t *tuple, const T &value)+inline void serialize_value(souffle::tuple& tuple, char* buf, offset_t& offset) {-    auto t = reinterpret_cast<souffle::tuple *>(tuple);-    assert(t);-    *t << value;+    auto ptr = reinterpret_cast<T*>(buf);+    tuple >> *ptr;+    offset += sizeof(T); }  template <typename T>-void tuple_pop_value(tuple_t *tuple, T *result)+inline void deserialize_value(souffle::tuple& tuple, char* buf, offset_t& offset) {-    auto t = reinterpret_cast<souffle::tuple *>(tuple);-    assert(t);-    assert(result);-    *t >> *result;+    auto ptr = reinterpret_cast<T*>(buf);+    tuple << *ptr;+    offset += sizeof(T); } -extern "C"+inline void deserialize_symbol(souffle::tuple& tuple, char* buf, offset_t& offset) {-    souffle_t *souffle_init(const char *progName)-    {-        auto prog = souffle::ProgramFactory::newInstance(progName);-        return reinterpret_cast<souffle_t *>(prog);+    auto ptr = reinterpret_cast<uint32_t*>(buf);+    const auto num_bytes = *ptr;+    if (num_bytes == 0) {+        tuple << "";+        offset += sizeof(uint32_t);+        return;     } -    void souffle_free(souffle_t *program)+    auto string_ptr = reinterpret_cast<const char*>(buf) + sizeof(uint32_t);+    std::string str(string_ptr, num_bytes);+    tuple << str;+    offset += sizeof(uint32_t) + num_bytes;+}++using deserializer_t = void(*)(souffle::tuple&, char*, offset_t&);+using deserializer_map = std::unordered_map<souffle_type, deserializer_t>;++static const deserializer_map deserializers_map = {+    {'s', deserialize_symbol},+    {'i', deserialize_value<number_t>},+    {'u', deserialize_value<unsigned_t>},+    {'f', deserialize_value<float_t>}+};++using serializer_t = void(*)(souffle::tuple&, char*, offset_t&);+using serializer_map = std::unordered_map<souffle_type, serializer_t>;++static const serializer_map serializers_map = {+    {'i', serialize_value<number_t>},+    {'u', serialize_value<unsigned_t>},+    {'f', serialize_value<float_t>}+};++inline std::string unknown_souffle_type(souffle_type ty)+{+    std::string base_message = "Found unknown Souffle primitive type: ";+    return base_message + std::string(1, ty);+}++inline auto types_to_deserializer(const std::vector<souffle_type>& types)+{+    std::vector<deserializer_t> deserializers;+    deserializers.reserve(types.size());++    for (const auto& type: types)     {-        auto prog = reinterpret_cast<souffle::SouffleProgram *>(program);-        assert(prog);-        delete prog;+        const auto match = deserializers_map.find(type);+        assert(match != deserializers_map.end() && unknown_souffle_type(match->first).c_str());+        deserializers.push_back(match->second);     } -    void souffle_set_num_threads(souffle_t *program, size_t num_cores)+    return [deserializers = std::move(deserializers)](souffle::tuple& tuple, char* buf, offset_t& offset)     {-        auto prog = reinterpret_cast<souffle::SouffleProgram *>(program);-        assert(prog);-        prog->setNumThreads(num_cores);+        for (const auto& deserializer : deserializers)+        {+            deserializer(tuple, buf + offset, offset);+        }+    };+}++inline auto types_to_serializer(const std::vector<souffle_type>& types)+{+    std::vector<serializer_t> serializers;+    serializers.reserve(types.size());++    for (const auto& type: types)+    {+        const auto match = serializers_map.find(type);+        assert(match != serializers_map.end() && unknown_souffle_type(match->first).c_str());+        serializers.push_back(match->second);     } -    size_t souffle_get_num_threads(souffle_t *program)+    return [serializers = std::move(serializers)](souffle::tuple& tuple, char* buf, offset_t& offset)     {-        auto prog = reinterpret_cast<souffle::SouffleProgram *>(program);-        assert(prog);-        return prog->getNumThreads();+        for (const auto& serializer : serializers)+        {+            serializer(tuple, buf + offset, offset);+        }+    };+}++inline auto guess_tuple_size(const std::vector<souffle_type>& types)+{+    size_t size = 0;++    for (const auto& type : types)+    {+        size += type == 's'+             ? ESTIMATED_AVERAGE_STRING_SIZE+             : sizeof(number_t);     } -    void souffle_run(souffle_t *program)+    return size;+}++struct Serializer+{+public:+    inline Serializer(souffle_t *prog, const souffle::Relation& relation)+        : m_relation(relation)+        , m_types(parse_signature(relation))+        , m_buf(prog->m_buf)     {-        auto prog = reinterpret_cast<souffle::SouffleProgram *>(program);-        assert(prog);-        prog->run();+        auto tuple_size = guess_tuple_size(m_types);++        m_fact_count = relation.size();+        m_num_bytes = sizeof(uint32_t) + m_fact_count * tuple_size;+        m_offset = 0;++        // NOTE: we need to have atleast `m_num_bytes` large buffer, to make+        // memcpy later not write beyond the buffer.+        if (m_num_bytes > m_buf.size()) m_buf.resize(m_num_bytes);     } -    void souffle_load_all(souffle_t *program, const char *input_directory)+    inline const Serializer& serialize()     {-        auto prog = reinterpret_cast<souffle::SouffleProgram *>(program);-        assert(prog);-        assert(input_directory);-        prog->loadAll(input_directory);+        using serializer_t = void(*)(Serializer*, souffle::tuple&);+        using serializer_map_t = std::unordered_map<souffle_type, serializer_t>;++        serializer_t do_serialize_symbol = [](auto s, auto& t) {+            s->serialize_symbol(t);+        };+        serializer_t do_serialize_number = [](auto s, auto& t) {+            s->serialize_number(t);+        };+        serializer_t do_serialize_unsigned = [](auto s, auto& t) {+            s->serialize_unsigned(t);+        };+        serializer_t do_serialize_float = [](auto s, auto& t) {+            s->serialize_float(t);+        };++        static const serializer_map_t serializers_map = {+            {'s', do_serialize_symbol},+            {'i', do_serialize_number},+            {'u', do_serialize_unsigned},+            {'f', do_serialize_float},+        };++        std::vector<serializer_t> serializers;+        serializers.reserve(m_types.size());++        for (const auto& type: m_types)+        {+            const auto match = serializers_map.find(type);+            assert(match != serializers_map.end() && unknown_souffle_type(match->first).c_str());+            serializers.push_back(match->second);+        }++        const auto serialize = [this, serializers = std::move(serializers)](auto& tuple)+        {+            for (const auto& serializer : serializers)+            {+                serializer(this, tuple);+            }+        };++        auto buf = reinterpret_cast<uint32_t*>(m_buf.data());+        *buf = m_fact_count;+        m_offset += sizeof(uint32_t);++        for (auto& tuple: m_relation)+        {+            serialize(tuple);+        }++        return *this;     } -    void souffle_print_all(souffle_t *program, const char *output_directory)+    inline void serialize_number(souffle::tuple& tuple)     {-        auto prog = reinterpret_cast<souffle::SouffleProgram *>(program);-        assert(prog);-        assert(output_directory);-        prog->printAll(output_directory);+        constexpr auto byte_count = sizeof(number_t);+        if (!has_remaining_bytes(byte_count)) {+            resize_buf(byte_count);+        }++        serialize_value<number_t>(tuple, m_buf.data() + m_offset, m_offset);     } -    relation_t *souffle_relation(souffle_t *program, const char *relation_name)+    inline void serialize_unsigned(souffle::tuple& tuple)     {-        auto prog = reinterpret_cast<souffle::SouffleProgram *>(program);-        assert(prog);-        assert(relation_name);-        auto relation = prog->getRelation(relation_name);-        assert(relation);-        return reinterpret_cast<relation_t *>(relation);+        constexpr auto byte_count = sizeof(unsigned_t);+        if (!has_remaining_bytes(byte_count)) {+            resize_buf(byte_count);+        }++        serialize_value<unsigned_t>(tuple, m_buf.data() + m_offset, m_offset);     } -    size_t souffle_relation_tuple_count(relation_t *relation)+    inline void serialize_float(souffle::tuple& tuple)     {-        auto rel = reinterpret_cast<souffle::Relation *>(relation);-        assert(rel);-        return rel->size();+        constexpr auto byte_count = sizeof(float_t);+        if (!has_remaining_bytes(byte_count)) {+            resize_buf(byte_count);+        }++        serialize_value<float_t>(tuple, m_buf.data() + m_offset, m_offset);     } -    struct relation_iterator+    inline void serialize_symbol(souffle::tuple& tuple)     {-        using iterator_t = souffle::Relation::iterator;-        iterator_t iterator;+        std::string str;+        tuple >> str;+        const uint32_t num_bytes = str.length(); -        relation_iterator(const iterator_t &it)-            : iterator(it) {}-    };+        auto total_byte_count = sizeof(uint32_t) + num_bytes;+        if (!has_remaining_bytes(total_byte_count)) {+            resize_buf(total_byte_count);+        } -    relation_iterator_t *souffle_relation_iterator(relation_t *relation)+        auto buf = m_buf.data() + m_offset;+        auto ptr = reinterpret_cast<uint32_t*>(buf);+        *ptr = num_bytes;++        // TODO: check if we can directly write into byte buf?+        auto string_ptr = reinterpret_cast<char*>(buf) + sizeof(uint32_t);+        std::copy(str.begin(), str.end(), string_ptr);+        m_offset += sizeof(uint32_t) + num_bytes;+    }++    inline bool has_remaining_bytes(size_t count) const     {-        auto rel = reinterpret_cast<souffle::Relation *>(relation);-        assert(rel);-        relation_iterator_t *it = new relation_iterator_t(rel->begin());-        return it;+        return m_num_bytes >= m_offset + count;     } -    void souffle_relation_iterator_free(relation_iterator_t *iterator)+    inline void resize_buf(size_t byte_count)     {-        assert(iterator);-        delete iterator;+        size_t grow_factor = GROW_FACTOR;+        while (m_offset + byte_count > m_num_bytes * grow_factor) {+            grow_factor *= 2;+        }+        const auto new_num_bytes = m_num_bytes * grow_factor;+        m_num_bytes = new_num_bytes;++        buf_data new_buf(new_num_bytes);+        memcpy(new_buf.data(), m_buf.data(), m_offset);+        std::swap(m_buf, new_buf);     } -    tuple_t *souffle_relation_iterator_next(relation_iterator_t *iterator)+    inline byte_buf_t *to_buf() const     {-        assert(iterator);-        auto tuple = reinterpret_cast<tuple_t *>(&*iterator->iterator);-        ++iterator->iterator;-        return tuple;+        return reinterpret_cast<byte_buf_t*>(m_buf.data());     } -    bool souffle_contains_tuple(relation_t *relation, tuple_t *tuple)+private:+    const souffle::Relation& m_relation;+    std::vector<souffle_type> m_types;+    size_t m_fact_count;+    buf_data& m_buf;+    size_t m_num_bytes;+    offset_t m_offset;+};++inline byte_buf_t *serialize_slow(souffle_t *prog, const souffle::Relation& relation)+{+    Serializer s(prog, relation);+    return s.serialize().to_buf();+}++inline byte_buf_t *serialize_fast(souffle_t *prog, const souffle::Relation& relation)+{+    const auto types = parse_signature(relation);+    const auto serialize = types_to_serializer(types);++    const auto fact_count = relation.size();+    const auto tuple_size = guess_tuple_size(types);+    const auto num_bytes = sizeof(uint32_t) + fact_count * tuple_size;+    auto buf = prog->get_buf(num_bytes);+    const auto start_ptr = buf;++    offset_t offset = 0;++    auto ptr = reinterpret_cast<uint32_t*>(buf);+    *ptr = fact_count;+    offset += 4;++    for (auto& tuple: relation)     {-        auto rel = reinterpret_cast<souffle::Relation *>(relation);-        auto t = reinterpret_cast<souffle::tuple *>(tuple);-        assert(rel);-        assert(t);-        return rel->contains(*t);+        serialize(tuple, buf, offset);     } -    tuple_t *souffle_tuple_alloc(relation_t *relation)+    return reinterpret_cast<byte_buf_t*>(start_ptr);+}++}  // namespace helpers++extern "C"+{+    souffle_t *souffle_init(const char *progName)     {-        auto rel = reinterpret_cast<souffle::Relation *>(relation);-        assert(rel);-        auto tuple = new souffle::tuple(rel);-        return reinterpret_cast<tuple_t *>(tuple);+        auto prog = souffle::ProgramFactory::newInstance(progName);+        return prog ? new souffle_interface(prog) : nullptr;     } -    void souffle_tuple_free(tuple_t *tuple)+    void souffle_free(souffle_t *program)     {-        auto t = reinterpret_cast<souffle::tuple *>(tuple);-        assert(t);-        delete t;+        assert(program);+        delete program;     } -    void souffle_tuple_push_int32(tuple_t *tuple, int32_t value)+    void souffle_set_num_threads(souffle_t *program, size_t num_cores)     {-        tuple_push_value(tuple, value);+        assert(program);+        program->m_prog->setNumThreads(num_cores);     } -    void souffle_tuple_push_uint32(tuple_t *tuple, uint32_t value)+    size_t souffle_get_num_threads(souffle_t *program)     {-        tuple_push_value(tuple, value);+        assert(program);+        return program->m_prog->getNumThreads();     } -    void souffle_tuple_push_float(tuple_t *tuple, float value)+    void souffle_run(souffle_t *program)     {-        tuple_push_value(tuple, value);+        assert(program);+        program->m_prog->run();     } -    void souffle_tuple_push_string(tuple_t *tuple, const char *value)+    void souffle_load_all(souffle_t *program, const char *input_directory)     {-        tuple_push_value(tuple, value);+        assert(program);+        assert(input_directory);+        program->m_prog->loadAll(input_directory);     } -    void souffle_tuple_add(relation_t *relation, tuple_t *tuple)+    void souffle_print_all(souffle_t *program, const char *output_directory)     {-        auto rel = reinterpret_cast<souffle::Relation *>(relation);-        auto t = reinterpret_cast<souffle::tuple *>(tuple);-        assert(rel);-        assert(t);-        rel->insert(*t);+        assert(program);+        assert(output_directory);+        program->m_prog->printAll(output_directory);     } -    void souffle_tuple_pop_int32(tuple_t *tuple, int32_t *result)+    relation_t *souffle_relation(souffle_t *program, const char *relation_name)     {-        tuple_pop_value(tuple, result);+        assert(program);+        assert(relation_name);+        auto relation = program->m_prog->getRelation(relation_name);+        assert(relation);+        return reinterpret_cast<relation_t *>(relation);     } -    void souffle_tuple_pop_uint32(tuple_t *tuple, uint32_t *result)+    bool souffle_contains_tuple(relation_t *rel, byte_buf_t *buf)     {-        tuple_pop_value(tuple, result);+        auto relation = reinterpret_cast<souffle::Relation *>(rel);+        auto data = reinterpret_cast<char*>(buf);+        assert(relation && "Relation is NULL in souffle_contains_tuple");+        assert(data && "byte buf is NULL in souffle_contains_tuple");++        auto& r = *relation;+        const auto types = helpers::parse_signature(r);+        const auto deserialize_tuple = helpers::types_to_deserializer(types);++        souffle::tuple tuple(relation);+        helpers::offset_t offset = 0;+        deserialize_tuple(tuple, data, offset);+        return r.contains(tuple);     } -    void souffle_tuple_pop_float(tuple_t *tuple, float *result)+    void souffle_tuple_push_many(relation_t *rel, byte_buf_t *buf, size_t size)     {-        tuple_pop_value(tuple, result);+        auto relation = reinterpret_cast<souffle::Relation*>(rel);+        auto data = reinterpret_cast<char*>(buf);+        assert(data && "byte buf is NULL in souffle_tuple_push_many");+        assert(relation && "Relation is NULL in souffle_tuple_push_many");++        auto& r = *relation;+        const auto types = helpers::parse_signature(r);+        const auto deserialize_tuple = helpers::types_to_deserializer(types);++        helpers::offset_t offset = 0;+        for (size_t i = 0; i < size; ++i)+        {+            souffle::tuple tuple(relation);+            deserialize_tuple(tuple, data, offset);+            r.insert(tuple);+        }     } -    void souffle_tuple_pop_string(tuple_t *tuple, char **result)+    byte_buf_t *souffle_tuple_pop_many(souffle_t *prog, relation_t *rel)     {-        assert(result);-        std::string value;-        tuple_pop_value(tuple, &value);-        *result = strdup(value.c_str());+        auto relation = reinterpret_cast<souffle::Relation*>(rel);+        assert(prog && "Program is NULL in souffle_tuple_pop_many");+        assert(relation && "Relation is NULL in souffle_tuple_pop_many");+        auto& r = *relation;+        return helpers::relation_contains_strings(r)+            ? helpers::serialize_slow(prog, r)+            : helpers::serialize_fast(prog, r);     } }
cbits/souffle.h view
@@ -13,10 +13,8 @@     typedef struct souffle_interface souffle_t;     // Opaque struct representing a Souffle relation     typedef struct relation relation_t;-    // Opaque struct representing an iterator to a Souffle relation-    typedef struct relation_iterator relation_iterator_t;-    // Opaque struct representing a Souffle tuple (fact).-    typedef struct tuple tuple_t;+    // Opaque struct representing a byte array filled with data.+    typedef struct byte_buf byte_buf_t;      /*      * Initializes a Souffle program. The name of the program should be the@@ -84,159 +82,33 @@     relation_t *souffle_relation(souffle_t *program, const char *relation_name);      /*-     * Gets the amount of tuples found in a relation.-     * You need to check if the passed pointer is non-NULL before passing it-     * to this function. Not doing so results in undefined behavior.-     *-     * Returns the amount of tuples found in a relation.-     */-    size_t souffle_relation_tuple_count(relation_t *relation);--    /*-     * Create an iterator for iterating over the facts of a relation.-     * You need to check if the passed pointer is non-NULL before passing it-     * to this function. Not doing so results in undefined behavior.-     *-     * The returned pointer needs to be freed up with-     * "souffle_relation_iterator_free" after it is no longer needed.-     */-    relation_iterator_t *souffle_relation_iterator(relation_t *relation);--    /*-     * Frees a relation_iterator pointer.-     * You need to check if the passed pointer is non-NULL before passing it-     * to this function. Not doing so results in undefined behavior.-     */-    void souffle_relation_iterator_free(relation_iterator_t *iterator);--    /*-     * Advances the relation iterator by 1 position.-     * You need to check if the passed pointer is non-NULL before passing it-     * to this function. Not doing so results in undefined behavior.-     * Calling this function when there are no more tuples to be returned-     * will result in a crash.-     *-     * Returns a pointer to the next record. This pointer is not allowed to be freed.-     */-    tuple_t *souffle_relation_iterator_next(relation_iterator_t *iterator);--    /*      * Checks if a relation contains a certain tuple.      * You need to check if the passed pointers are non-NULL before passing it      * to this function. Not doing so results in undefined behavior.      *      * Returns true if the tuple was found in the relation; otherwise false.      */-    bool souffle_contains_tuple(relation_t *relation, tuple_t *tuple);--    /*-     * Allocates memory for a tuple to be added to a relation.-     * You need to check if the passed pointer is non-NULL before passing it-     * to this function. Not doing so results in undefined behavior.-     *-     * Returns a pointer to a new tuple. Use "souffle_tuple_free" when tuple-     * is no longer required.-     */-    tuple_t *souffle_tuple_alloc(relation_t *relation);--    /*-     * Frees memory of a tuple that was previously allocated.-     * You need to check if the passed pointer is non-NULL before passing it-     * to this function. Not doing so results in undefined behavior.-     */-    void souffle_tuple_free(tuple_t *tuple);--    /*-     * Adds a tuple to a relation.-     * You need to check if both passed pointers are non-NULL before passing it-     * to this function. Not doing so results in undefined behavior.-     */-    void souffle_tuple_add(relation_t *relation, tuple_t *tuple);--    /*-     * Pushes a 32 bit signed integer value into a tuple.-     * You need to check if the passed pointer is non-NULL before passing it-     * to this function. Not doing so results in undefined behavior.-     *-     * Pushing an integer value onto a tuple that expects another type results-     * in a crash.-     */-    void souffle_tuple_push_int32(tuple_t *tuple, int32_t value);--    /*-     * Pushes a 32 bit unsigned integer value into a tuple.-     * You need to check if the passed pointer is non-NULL before passing it-     * to this function. Not doing so results in undefined behavior.-     *-     * Pushing an integer value onto a tuple that expects another type results-     * in a crash.-     */-    void souffle_tuple_push_uint32(tuple_t *tuple, uint32_t value);--    /*-     * Pushes a float value into a tuple.-     * You need to check if the passed pointer is non-NULL before passing it-     * to this function. Not doing so results in undefined behavior.-     *-     * Pushing a float value onto a tuple that expects another type results-     * in a crash.-     */-    void souffle_tuple_push_float(tuple_t *tuple, float value);--    /*-     * Pushes a string value into a tuple.-     * You need to check if the passed pointer is non-NULL before passing it-     * to this function. Not doing so results in undefined behavior.-     *-     * Pushing a string value onto a tuple that expects another type results-     * in a crash.-     */-    void souffle_tuple_push_string(tuple_t *tuple, const char *value);--    /*-     * Extracts a 32 bit signed integer value from a tuple.-     * You need to check if the passed pointer is non-NULL before passing it-     * to this function. Not doing so results in undefined behavior.-     *-     * Extracting an integer value from a tuple that expects another type results-     * in a crash.-     * The popped integer will be stored in the result pointer.-     */-    void souffle_tuple_pop_int32(tuple_t *tuple, int32_t *result);--    /*-     * Extracts a 32 bit unsigned integer value from a tuple.-     * You need to check if the passed pointer is non-NULL before passing it-     * to this function. Not doing so results in undefined behavior.-     *-     * Extracting an integer value from a tuple that expects another type results-     * in a crash.-     * The popped integer will be stored in the result pointer.-     */-    void souffle_tuple_pop_uint32(tuple_t *tuple, uint32_t *result);+    bool souffle_contains_tuple(relation_t *relation, byte_buf_t *buf); -    /*-     * Extracts a float value from a tuple.-     * You need to check if the passed pointer is non-NULL before passing it+    /**+     * Pushes many Datalog facts from Haskell to Datalog.+     * You need to check if the passed pointers are non-NULL before passing it      * to this function. Not doing so results in undefined behavior.-     *-     * Extracting a float value from a tuple that expects another type results-     * in a crash.-     * The popped integer will be stored in the result pointer.+     * Passing in a different count of objects to what is actually inside the+     * byte buffer will crash.      */-    void souffle_tuple_pop_float(tuple_t *tuple, float *result);+    void souffle_tuple_push_many(relation_t *relation, byte_buf_t *buf, size_t size); -    /*-     * Extracts a string value from a tuple.-     * You need to check if the passed pointer is non-NULL before passing it+    /**+     * Pops many Datalog facts from Datalog to Haskell.+     * You need to check if the passed pointers are non-NULL before passing it      * to this function. Not doing so results in undefined behavior.      *-     * Extracting a string value from a tuple that expects another type results-     * in a crash.-     * The popped string will be stored in the result pointer.+     * Returns the byte buffer that contains the serialized Datalog facts.+     * This byte buffer is automatically managed by the C++ side and does not+     * need to be cleaned up.      */-    void souffle_tuple_pop_string(tuple_t *tuple, char **result);-+    byte_buf_t *souffle_tuple_pop_many(souffle_t *program, relation_t *relation); #ifdef __cplusplus } #endif
cbits/souffle/CompiledSouffle.h view
@@ -16,45 +16,23 @@  #pragma once -#include "souffle/CompiledTuple.h" #include "souffle/RamTypes.h" #include "souffle/RecordTable.h" #include "souffle/SignalHandler.h" #include "souffle/SouffleInterface.h" #include "souffle/SymbolTable.h"+#include "souffle/datastructure/BTreeDelete.h" #include "souffle/datastructure/Brie.h" #include "souffle/datastructure/EquivalenceRelation.h"+#include "souffle/datastructure/RecordTableImpl.h"+#include "souffle/datastructure/SymbolTableImpl.h" #include "souffle/datastructure/Table.h" #include "souffle/io/IOSystem.h" #include "souffle/io/WriteStream.h"-#include "souffle/utility/CacheUtil.h"-#include "souffle/utility/ContainerUtil.h" #include "souffle/utility/EvaluatorUtil.h"-#include "souffle/utility/FileUtil.h"-#include "souffle/utility/FunctionalUtil.h"-#include "souffle/utility/MiscUtil.h"-#include "souffle/utility/ParallelUtil.h"-#include "souffle/utility/StreamUtil.h"-#include "souffle/utility/StringUtil.h" #ifndef __EMBEDDED_SOUFFLE__ #include "souffle/CompiledOptions.h"-#include "souffle/profile/Logger.h"-#include "souffle/profile/ProfileEvent.h" #endif-#include <array>-#include <atomic>-#include <cassert>-#include <cmath>-#include <cstdint>-#include <cstdlib>-#include <exception>-#include <iostream>-#include <iterator>-#include <memory>-#include <regex>-#include <string>-#include <utility>-#include <vector>  #if defined(_OPENMP) #include <omp.h>@@ -71,31 +49,39 @@ /**  * Relation wrapper used internally in the generated Datalog program  */-template <uint32_t id, class RelType, class TupleType, size_t Arity, size_t NumAuxAttributes>+template <class RelType> class RelationWrapper : public souffle::Relation {+public:+    static constexpr arity_type Arity = RelType::Arity;+    using TupleType = Tuple<RamDomain, Arity>;+    using AttrStrSeq = std::array<const char*, Arity>;+ private:     RelType& relation;-    SymbolTable& symTable;+    SouffleProgram& program;     std::string name;-    std::array<const char*, Arity> tupleType;-    std::array<const char*, Arity> tupleName;+    AttrStrSeq attrTypes;+    AttrStrSeq attrNames;+    const uint32_t id;+    const arity_type numAuxAttribs; +    // NB: internal wrapper. does not satisfy the `iterator` concept.     class iterator_wrapper : public iterator_base {         typename RelType::iterator it;         const Relation* relation;         tuple t;      public:-        iterator_wrapper(uint32_t arg_id, const Relation* rel, const typename RelType::iterator& arg_it)-                : iterator_base(arg_id), it(arg_it), relation(rel), t(rel) {}+        iterator_wrapper(uint32_t arg_id, const Relation* rel, typename RelType::iterator arg_it)+                : iterator_base(arg_id), it(std::move(arg_it)), relation(rel), t(rel) {}         void operator++() override {             ++it;         }         tuple& operator*() override {+            auto&& value = *it;             t.rewind();-            for (size_t i = 0; i < Arity; i++) {-                t[i] = (*it)[i];-            }+            for (std::size_t i = 0; i < Arity; i++)+                t[i] = value[i];             return t;         }         iterator_base* clone() const override {@@ -104,26 +90,29 @@      protected:         bool equal(const iterator_base& o) const override {-            const auto& casted = static_cast<const iterator_wrapper&>(o);+            const auto& casted = asAssert<iterator_wrapper>(o);             return it == casted.it;         }     };  public:-    RelationWrapper(RelType& r, SymbolTable& s, std::string name, const std::array<const char*, Arity>& t,-            const std::array<const char*, Arity>& n)-            : relation(r), symTable(s), name(std::move(name)), tupleType(t), tupleName(n) {}+    RelationWrapper(uint32_t id, RelType& r, SouffleProgram& p, std::string name, const AttrStrSeq& t,+            const AttrStrSeq& n, arity_type numAuxAttribs)+            : relation(r), program(p), name(std::move(name)), attrTypes(t), attrNames(n), id(id),+              numAuxAttribs(numAuxAttribs) {}+     iterator begin() const override {-        return iterator(new iterator_wrapper(id, this, relation.begin()));+        return iterator(mk<iterator_wrapper>(id, this, relation.begin()));     }     iterator end() const override {-        return iterator(new iterator_wrapper(id, this, relation.end()));+        return iterator(mk<iterator_wrapper>(id, this, relation.end()));     }+     void insert(const tuple& arg) override {         TupleType t;         assert(&arg.getRelation() == this && "wrong relation");         assert(arg.size() == Arity && "wrong tuple arity");-        for (size_t i = 0; i < Arity; i++) {+        for (std::size_t i = 0; i < Arity; i++) {             t[i] = arg[i];         }         relation.insert(t);@@ -131,7 +120,7 @@     bool contains(const tuple& arg) const override {         TupleType t;         assert(arg.size() == Arity && "wrong tuple arity");-        for (size_t i = 0; i < Arity; i++) {+        for (std::size_t i = 0; i < Arity; i++) {             t[i] = arg[i];         }         return relation.contains(t);@@ -142,22 +131,22 @@     std::string getName() const override {         return name;     }-    const char* getAttrType(size_t arg) const override {+    const char* getAttrType(std::size_t arg) const override {         assert(arg < Arity && "attribute out of bound");-        return tupleType[arg];+        return attrTypes[arg];     }-    const char* getAttrName(size_t arg) const override {+    const char* getAttrName(std::size_t arg) const override {         assert(arg < Arity && "attribute out of bound");-        return tupleName[arg];+        return attrNames[arg];     }-    size_t getArity() const override {+    arity_type getArity() const override {         return Arity;     }-    size_t getAuxiliaryArity() const override {-        return NumAuxAttributes;+    arity_type getAuxiliaryArity() const override {+        return numAuxAttribs;     }     SymbolTable& getSymbolTable() const override {-        return symTable;+        return program.getSymbolTable();     }      /** Eliminate all the tuples in relation*/@@ -172,6 +161,8 @@     std::atomic<bool> data{false};  public:+    static constexpr Relation::arity_type Arity = 0;+     t_nullaries() = default;     using t_tuple = Tuple<RamDomain, 0>;     struct context {};@@ -182,11 +173,11 @@         bool value;      public:-        typedef std::forward_iterator_tag iterator_category;-        typedef RamDomain* value_type;-        typedef ptrdiff_t difference_type;-        typedef value_type* pointer;-        typedef value_type& reference;+        using iterator_category = std::forward_iterator_tag;+        using value_type = RamDomain*;+        using difference_type = ptrdiff_t;+        using pointer = value_type*;+        using reference = value_type&;          iterator(bool v = false) : value(v) {} @@ -247,14 +238,12 @@     void printStatistics(std::ostream& /* o */) const {} }; -/** info relations */-template <int Arity>+/** Info relations */+template <Relation::arity_type Arity_> class t_info {-private:-    std::vector<Tuple<RamDomain, Arity>> data;-    Lock insert_lock;- public:+    static constexpr Relation::arity_type Arity = Arity_;+     t_info() = default;     using t_tuple = Tuple<RamDomain, Arity>;     struct context {};@@ -303,7 +292,7 @@     void insert(const RamDomain* ramDomain) {         insert_lock.lock();         t_tuple t;-        for (size_t i = 0; i < Arity; ++i) {+        for (std::size_t i = 0; i < Arity; ++i) {             t.data[i] = ramDomain[i];         }         data.push_back(t);@@ -328,6 +317,164 @@     }     void purge() {         data.clear();+    }+    void printStatistics(std::ostream& /* o */) const {}++private:+    std::vector<Tuple<RamDomain, Arity>> data;+    Lock insert_lock;+};++/** Equivalence relations */+struct t_eqrel {+    static constexpr Relation::arity_type Arity = 2;+    using t_tuple = Tuple<RamDomain, 2>;+    using t_ind = EquivalenceRelation<t_tuple>;+    t_ind ind;+    class iterator_0 : public std::iterator<std::forward_iterator_tag, t_tuple> {+        using nested_iterator = typename t_ind::iterator;+        nested_iterator nested;+        t_tuple value;++    public:+        iterator_0(const nested_iterator& iter) : nested(iter), value(*iter) {}+        iterator_0(const iterator_0& other) = default;+        iterator_0& operator=(const iterator_0& other) = default;+        bool operator==(const iterator_0& other) const {+            return nested == other.nested;+        }+        bool operator!=(const iterator_0& other) const {+            return !(*this == other);+        }+        const t_tuple& operator*() const {+            return value;+        }+        const t_tuple* operator->() const {+            return &value;+        }+        iterator_0& operator++() {+            ++nested;+            value = *nested;+            return *this;+        }+    };+    class iterator_1 : public std::iterator<std::forward_iterator_tag, t_tuple> {+        using nested_iterator = typename t_ind::iterator;+        nested_iterator nested;+        t_tuple value;++    public:+        iterator_1(const nested_iterator& iter) : nested(iter), value(reorder(*iter)) {}+        iterator_1(const iterator_1& other) = default;+        iterator_1& operator=(const iterator_1& other) = default;+        bool operator==(const iterator_1& other) const {+            return nested == other.nested;+        }+        bool operator!=(const iterator_1& other) const {+            return !(*this == other);+        }+        const t_tuple& operator*() const {+            return value;+        }+        const t_tuple* operator->() const {+            return &value;+        }+        iterator_1& operator++() {+            ++nested;+            value = reorder(*nested);+            return *this;+        }+    };+    using iterator = iterator_0;+    struct context {+        t_ind::operation_hints hints;+    };+    context createContext() {+        return context();+    }+    bool insert(const t_tuple& t) {+        return ind.insert(t[0], t[1]);+    }+    bool insert(const t_tuple& t, context& h) {+        return ind.insert(t[0], t[1], h.hints);+    }+    bool insert(const RamDomain* ramDomain) {+        RamDomain data[2];+        std::copy(ramDomain, ramDomain + 2, data);+        auto& tuple = reinterpret_cast<const t_tuple&>(data);+        context h;+        return insert(tuple, h);+    }+    bool insert(RamDomain a1, RamDomain a2) {+        RamDomain data[2] = {a1, a2};+        return insert(data);+    }+    void extendAndInsert(t_eqrel& other) {+        ind.extendAndInsert(other.ind);+    }+    bool contains(const t_tuple& t) const {+        return ind.contains(t[0], t[1]);+    }+    bool contains(const t_tuple& t, context&) const {+        return ind.contains(t[0], t[1]);+    }+    std::size_t size() const {+        return ind.size();+    }+    iterator find(const t_tuple& t) const {+        return ind.find(t);+    }+    iterator find(const t_tuple& t, context&) const {+        return ind.find(t);+    }+    range<iterator> lowerUpperRange_10(const t_tuple& lower, const t_tuple& /*upper*/, context& h) const {+        auto r = ind.template getBoundaries<1>((lower), h.hints);+        return make_range(iterator(r.begin()), iterator(r.end()));+    }+    range<iterator> lowerUpperRange_10(const t_tuple& lower, const t_tuple& upper) const {+        context h;+        return lowerUpperRange_10(lower, upper, h);+    }+    range<iterator_1> lowerUpperRange_01(const t_tuple& lower, const t_tuple& /*upper*/, context& h) const {+        auto r = ind.template getBoundaries<1>(reorder(lower), h.hints);+        return make_range(iterator_1(r.begin()), iterator_1(r.end()));+    }+    range<iterator_1> lowerUpperRange_01(const t_tuple& lower, const t_tuple& upper) const {+        context h;+        return lowerUpperRange_01(lower, upper, h);+    }+    range<iterator> lowerUpperRange_11(const t_tuple& lower, const t_tuple& /*upper*/, context& h) const {+        auto r = ind.template getBoundaries<2>((lower), h.hints);+        return make_range(iterator(r.begin()), iterator(r.end()));+    }+    range<iterator> lowerUpperRange_11(const t_tuple& lower, const t_tuple& upper) const {+        context h;+        return lowerUpperRange_11(lower, upper, h);+    }+    bool empty() const {+        return ind.size() == 0;+    }+    std::vector<range<iterator>> partition() const {+        std::vector<range<iterator>> res;+        for (const auto& cur : ind.partition(10000)) {+            res.push_back(make_range(iterator(cur.begin()), iterator(cur.end())));+        }+        return res;+    }+    void purge() {+        ind.clear();+    }+    iterator begin() const {+        return iterator(ind.begin());+    }+    iterator end() const {+        return iterator(ind.end());+    }+    static t_tuple reorder(const t_tuple& t) {+        t_tuple res;+        res[0] = t[1];+        res[1] = t[0];+        return res;     }     void printStatistics(std::ostream& /* o */) const {} };
− cbits/souffle/CompiledTuple.h 
@@ -1,186 +0,0 @@-/*- * Souffle - A Datalog Compiler- * Copyright (c) 2013, 2015, Oracle and/or its affiliates. All rights reserved- * Licensed under the Universal Permissive License v 1.0 as shown at:- * - https://opensource.org/licenses/UPL- * - <souffle root>/licenses/SOUFFLE-UPL.txt- */--/************************************************************************- *- * @file CompiledTuple.h- *- * The central file covering the data structure utilized by- * the souffle compiler for representing relations in compiled queries.- *- ***********************************************************************/--#pragma once--#include <cstddef>-#include <functional>-#include <iostream>-#include <system_error>--namespace souffle {--/**- * The type of object stored within relations representing the actual- * tuple value. Each tuple consists of a constant number of components.- *- * @tparam Domain the domain of the component values- * @tparam arity the number of components within an instance- */-template <typename Domain, std::size_t _arity>-struct Tuple {-    // some features for template meta programming-    using value_type = Domain;-    static constexpr size_t arity = _arity;--    // the stored data-    Domain data[arity];--    // constructors, destructors and assignment are default--    // provide access to components-    const Domain& operator[](std::size_t index) const {-        return data[index];-    }--    // provide access to components-    Domain& operator[](std::size_t index) {-        return data[index];-    }--    // a comparison operation-    bool operator==(const Tuple& other) const {-        for (std::size_t i = 0; i < arity; i++) {-            if (data[i] != other.data[i]) return false;-        }-        return true;-    }--    // inequality comparison-    bool operator!=(const Tuple& other) const {-        return !(*this == other);-    }--    // required to put tuples into e.g. a std::set container-    bool operator<(const Tuple& other) const {-        for (std::size_t i = 0; i < arity; ++i) {-            if (data[i] < other.data[i]) return true;-            if (data[i] > other.data[i]) return false;-        }-        return false;-    }--    // required to put tuples into e.g. a btree container-    bool operator>(const Tuple& other) const {-        for (std::size_t i = 0; i < arity; ++i) {-            if (data[i] > other.data[i]) return true;-            if (data[i] < other.data[i]) return false;-        }-        return false;-    }--    // allow tuples to be printed-    friend std::ostream& operator<<(std::ostream& out, const Tuple& tuple) {-        if (arity == 0) return out << "[]";-        out << "[";-        for (std::size_t i = 0; i < (std::size_t)(arity - 1); ++i) {-            out << tuple.data[i];-            out << ",";-        }-        return out << tuple.data[arity - 1] << "]";-    }-};--#ifdef _MSC_VER-/**- * A template specialization for 0-arity tuples when compiling with microsoft's- * compiler, because it doesn't like the 0 length array even though it is the- * last member of the struct.- */-template <typename Domain>-struct Tuple<Domain, 0> {-    // some features for template meta programming-    using value_type = Domain;-    enum { arity = 0 };--    // the stored data-    Domain data[1];--    // constructores, destructors and assignment are default--    // provide access to components-    const Domain& operator[](std::size_t index) const {-        return data[index];-    }--    // provide access to components-    Domain& operator[](std::size_t index) {-        return data[index];-    }--    // a comparison operation-    bool operator==(const Tuple& other) const {-        for (std::size_t i = 0; i < arity; i++) {-            if (data[i] != other.data[i]) return false;-        }-        return true;-    }--    // inequality comparison-    bool operator!=(const Tuple& other) const {-        return !(*this == other);-    }--    // required to put tuples into e.g. a std::set container-    bool operator<(const Tuple& other) const {-        for (std::size_t i = 0; i < arity; ++i) {-            if (data[i] < other.data[i]) return true;-            if (data[i] > other.data[i]) return false;-        }-        return false;-    }--    // required to put tuples into e.g. a btree container-    bool operator>(const Tuple& other) const {-        for (std::size_t i = 0; i < arity; ++i) {-            if (data[i] > other.data[i]) return true;-            if (data[i] < other.data[i]) return false;-        }-        return false;-    }--    // allow tuples to be printed-    friend std::ostream& operator<<(std::ostream& out, const Tuple& tuple) {-        if (arity == 0) return out << "[]";-        out << "[";-        for (std::size_t i = 0; i < (std::size_t)(arity - 1); ++i) {-            out << tuple.data[i];-            out << ",";-        }-        return out << tuple.data[arity - 1] << "]";-    }-};-#endif  // _MSC_VER-}  // end of namespace souffle--// -- add hashing support ----namespace std {--template <typename Domain, std::size_t arity>-struct hash<souffle::Tuple<Domain, arity>> {-    size_t operator()(const souffle::Tuple<Domain, arity>& value) const {-        std::hash<Domain> hash;-        size_t res = 0;-        for (unsigned i = 0; i < arity; i++) {-            // from boost hash combine-            res ^= hash(value[i]) + 0x9e3779b9 + (res << 6) + (res >> 2);-        }-        return res;-    }-};-}  // namespace std
cbits/souffle/RamTypes.h view
@@ -16,6 +16,7 @@  #pragma once +#include <array> #include <cstdint> #include <cstring> #include <iostream>@@ -23,6 +24,10 @@ #include <type_traits>  namespace souffle {++// deprecated. use `std::array` directly.+template <typename A, std::size_t N>+using Tuple = std::array<A, N>;  /**  * Types of elements in a tuple.
cbits/souffle/RecordTable.h view
@@ -1,6 +1,6 @@ /*  * Souffle - A Datalog Compiler- * Copyright (c) 2020, The Souffle Developers. All rights reserved.+ * Copyright (c) 2013, 2014, Oracle and/or its affiliates. All rights reserved.  * Licensed under the Universal Permissive License v 1.0 as shown at:  * - https://opensource.org/licenses/UPL  * - <souffle root>/licenses/SOUFFLE-UPL.txt@@ -17,136 +17,42 @@  #pragma once -#include "souffle/CompiledTuple.h" #include "souffle/RamTypes.h"-#include <cassert>-#include <cstddef>-#include <limits>-#include <memory>-#include <unordered_map>-#include <utility>-#include <vector>+#include "souffle/utility/span.h"+#include <initializer_list>  namespace souffle { -/** @brief Bidirectional mappping between records and record references */-class RecordMap {-    /** arity of record */-    const size_t arity;--    /** hash function for unordered record map */-    struct RecordHash {-        std::size_t operator()(std::vector<RamDomain> record) const {-            std::size_t seed = 0;-            std::hash<RamDomain> domainHash;-            for (RamDomain value : record) {-                seed ^= domainHash(value) + 0x9e3779b9 + (seed << 6) + (seed >> 2);-            }-            return seed;-        }-    };--    /** map from records to references */-    // TODO (b-scholz): replace vector<RamDomain> with something more memory-frugal-    std::unordered_map<std::vector<RamDomain>, RamDomain, RecordHash> recordToIndex;--    /** array of records; index represents record reference */-    // TODO (b-scholz): replace vector<RamDomain> with something more memory-frugal-    std::vector<std::vector<RamDomain>> indexToRecord;-+/** The interface of any Record Table. */+class RecordTable { public:-    explicit RecordMap(size_t arity) : arity(arity), indexToRecord(1) {}  // note: index 0 element left free+    virtual ~RecordTable() {} -    /** @brief converts record to a record reference */-    // TODO (b-scholz): replace vector<RamDomain> with something more memory-frugal-    RamDomain pack(const std::vector<RamDomain>& vector) {-        RamDomain index;-#pragma omp critical(record_pack)-        {-            auto pos = recordToIndex.find(vector);-            if (pos != recordToIndex.end()) {-                index = pos->second;-            } else {-#pragma omp critical(record_unpack)-                {-                    indexToRecord.push_back(vector);-                    index = static_cast<RamDomain>(indexToRecord.size()) - 1;-                    recordToIndex[vector] = index;+    virtual void setNumLanes(const std::size_t NumLanes) = 0; -                    // assert that new index is smaller than the range-                    assert(index != std::numeric_limits<RamDomain>::max());-                }-            }-        }-        return index;-    }+    virtual RamDomain pack(const RamDomain* Tuple, const std::size_t Arity) = 0; -    /** @brief convert record pointer to a record reference */-    RamDomain pack(const RamDomain* tuple) {-        // TODO (b-scholz): data is unnecessarily copied-        // for a successful lookup. To avoid this, we should-        // compute a hash of the pointer-array and traverse through-        // the bucket list of the unordered map finding the record.-        // Note that in case of non-existence, the record still needs to be-        // copied for the newly created entry but this will be the less-        // frequent case.-        std::vector<RamDomain> tmp(arity);-        for (size_t i = 0; i < arity; i++) {-            tmp[i] = tuple[i];-        }-        return pack(tmp);-    }+    virtual RamDomain pack(const std::initializer_list<RamDomain>& List) = 0; -    /** @brief convert record reference to a record pointer */-    const RamDomain* unpack(RamDomain index) const {-        const RamDomain* res;-#pragma omp critical(record_unpack)-        res = indexToRecord[index].data();-        return res;-    }+    virtual const RamDomain* unpack(const RamDomain Ref, const std::size_t Arity) const = 0; }; -class RecordTable {-public:-    RecordTable() = default;-    virtual ~RecordTable() = default;--    /** @brief convert record to record reference */-    RamDomain pack(RamDomain* tuple, size_t arity) {-        return lookupArity(arity).pack(tuple);-    }-    /** @brief convert record reference to a record */-    const RamDomain* unpack(RamDomain ref, size_t arity) const {-        std::unordered_map<size_t, RecordMap>::const_iterator iter;-#pragma omp critical(RecordTableGetForArity)-        {-            // Find a previously emplaced map-            iter = maps.find(arity);-        }-        assert(iter != maps.end() && "Attempting to unpack record for non-existing arity");-        return (iter->second).unpack(ref);-    }--private:-    /** @brief lookup RecordMap for a given arity; if it does not exist, create new RecordMap */-    RecordMap& lookupArity(size_t arity) {-        std::unordered_map<size_t, RecordMap>::iterator mapsIterator;-#pragma omp critical(RecordTableGetForArity)-        {-            // This will create a new map if it doesn't exist yet.-            mapsIterator = maps.emplace(arity, arity).first;-        }-        return mapsIterator->second;-    }--    /** Arity/RecordMap association */-    std::unordered_map<size_t, RecordMap> maps;-};+/** @brief helper to convert tuple to record reference for the synthesiser */+template <class RecordTableT, std::size_t Arity>+RamDomain pack(RecordTableT&& recordTab, Tuple<RamDomain, Arity> const& tuple) {+    return recordTab.pack(tuple.data(), Arity);+}  /** @brief helper to convert tuple to record reference for the synthesiser */-template <std::size_t Arity>-inline RamDomain pack(RecordTable& recordTab, Tuple<RamDomain, Arity> tuple) {-    return recordTab.pack(static_cast<RamDomain*>(tuple.data), Arity);+template <class RecordTableT, std::size_t Arity>+RamDomain pack(RecordTableT&& recordTab, span<const RamDomain, Arity> tuple) {+    return recordTab.pack(tuple.data(), Arity);+}++/** @brief helper to pack using an initialization-list of RamDomain values. */+template <class RecordTableT>+RamDomain pack(RecordTableT&& recordTab, const std::initializer_list<RamDomain>&& initlist) {+    return recordTab.pack(std::data(initlist), initlist.size()); }  }  // namespace souffle
cbits/souffle/SignalHandler.h view
@@ -21,10 +21,18 @@ #include <cstdio> #include <cstdlib> #include <cstring>+#include <initializer_list> #include <iostream> #include <mutex> #include <string> +#ifndef _MSC_VER+#include <unistd.h>+#else+#include <io.h>+#define STDERR_FILENO 2+#endif+ namespace souffle {  /**@@ -135,6 +143,7 @@ private:     // signal context information     std::atomic<const char*> msg;+    static_assert(decltype(msg)::is_always_lock_free, "cannot safely use in signal handler");      // state of signal handler     bool isSet = false;@@ -150,20 +159,44 @@      * Signal handler for various types of signals.      */     static void handler(int signal) {-        const char* msg = instance()->msg;-        std::string error;+        // Signal handlers have extreme restrictions on what stdlib/OS facilities are available.+        // This is b/c signals are async on most platforms.+        // See: https://en.cppreference.com/w/cpp/utility/program/signal+        // See: `man 7 signal`++        const char* error;         switch (signal) {-            case SIGINT: error = "Interrupt"; break;+            case SIGABRT: error = "Abort"; break;             case SIGFPE: error = "Floating-point arithmetic exception"; break;+            case SIGILL: error = "Illegal instruction"; break;+            case SIGINT: error = "Interrupt"; break;             case SIGSEGV: error = "Segmentation violation"; break;+            case SIGTERM: error = "Terminate"; break;             default: error = "Unknown"; break;         }-        if (msg != nullptr) {-            std::cerr << error << " signal in rule:\n" << msg << std::endl;-        } else {-            std::cerr << error << " signal." << std::endl;-        }-        exit(1);++        auto write = [](std::initializer_list<char const*> const& msgs) {+            for (auto&& msg : msgs) {+                // assign to variable to suppress ignored-return-value error.+                // I don't think we care enough to handle this fringe failure mode.+                // Worse case we don't get an error message.+#ifdef _MSC_VER+                [[maybe_unused]] auto _ =+                        ::_write(STDERR_FILENO, msg, static_cast<unsigned int>(::strlen(msg)));+#else+                [[maybe_unused]] auto _ =+                        ::write(STDERR_FILENO, msg, static_cast<unsigned int>(::strlen(msg)));+#endif+            }+        };++        // `instance()` is okay. Static `singleton` must already be constructed if we got here.+        if (const char* msg = instance()->msg)+            write({error, " signal in rule:\n", msg, "\n"});+        else+            write({error, " signal.\n"});++        std::_Exit(EXIT_FAILURE);     }      SignalHandler() : msg(nullptr) {}
cbits/souffle/SouffleInterface.h view
@@ -8,7 +8,7 @@  /************************************************************************  *- * @file CompiledSouffle.h+ * @file SouffleInterface.h  *  * Main include file for generated C++ classes of Souffle  *@@ -17,6 +17,7 @@ #pragma once  #include "souffle/RamTypes.h"+#include "souffle/RecordTable.h" #include "souffle/SymbolTable.h" #include "souffle/utility/MiscUtil.h" #include <algorithm>@@ -27,6 +28,7 @@ #include <iostream> #include <map> #include <memory>+#include <optional> #include <string> #include <tuple> #include <utility>@@ -40,6 +42,9 @@  * Object-oriented wrapper class for Souffle's templatized relations.  */ class Relation {+public:+    using arity_type = std::size_t;+ protected:     /**      * Abstract iterator class.@@ -57,17 +62,20 @@          * Required for identifying type of iterator          * (NB: LLVM has no typeinfo).          *+         * Note: The above statement is not true anymore - should this be made to work the same+         * as Node::operator==?+         *          * TODO (Honghyw) : Provide a clear documentation of what id is used for.          */-        uint32_t id;+        std::size_t id;      public:         /**          * Get the ID of the iterator_base object.          *-         * @return ID of the iterator_base object (unit32_t)+         * @return ID of the iterator_base object (std::size_t)          */-        virtual uint32_t getId() const {+        virtual std::size_t getId() const {             return id;         } @@ -76,9 +84,9 @@          *          * Create an instance of iterator_base and set its ID to be arg_id.          *-         * @param arg_id ID of an iterator object (unit32_t)+         * @param arg_id ID of an iterator object (std::size_t)          */-        iterator_base(uint32_t arg_id) : id(arg_id) {}+        iterator_base(std::size_t arg_id) : id(arg_id) {}          /**          * Destructor.@@ -151,7 +159,7 @@          * iterator_base class pointer.          *          */-        Own<iterator_base> iter = nullptr;+        std::unique_ptr<iterator_base> iter = nullptr;      public:         /**@@ -177,7 +185,7 @@          *          * @param arg An iterator_base class pointer          */-        iterator(iterator_base* arg) : iter(arg) {}+        iterator(std::unique_ptr<iterator_base> it) : iter(std::move(it)) {}          /**          * Destructor.@@ -224,6 +232,20 @@         }          /**+         * Overload the "++" operator.+         *+         * Copies the iterator, increments itself, and returns the (pre-increment) copy.+         * WARNING: Expensive due to copy! Included for API compatibility.+         *+         * @return Pre-increment copy of `this`.+         */+        iterator operator++(int) {+            auto cpy = *this;+            ++(*this);+            return cpy;+        }++        /**          * Overload the "*" operator.          *          * This will return the tuple that the iterator is pointing to.@@ -310,43 +332,57 @@     /**      * Get the attribute type of a relation at the column specified by the parameter.      * The attribute type is in the form "<primitive type>:<type name>".-     * <primitive type> can be s, f, u, or i standing for symbol, float, unsigned, and integer respectively,+     * <primitive type> can be s, f, u, i, r, or + standing for symbol, float,+     * unsigned, integer, record, and ADT respectively,      * which are the primitive types in Souffle.      * <type name> is the name given by the user in the Souffle program      *-     * @param The index of the column starting starting from 0 (size_t)+     * @param The index of the column starting starting from 0 (std::size_t)      * @return The constant string of the attribute type      */-    virtual const char* getAttrType(size_t) const = 0;+    virtual const char* getAttrType(std::size_t) const = 0;      /**      * Get the attribute name of a relation at the column specified by the parameter.      * The attribute name is the name given to the type by the user in the .decl statement. For example, for      * ".decl edge (node1:Node, node2:Node)", the attribute names are node1 and node2.      *-     * @param The index of the column starting starting from 0 (size_t)+     * @param The index of the column starting starting from 0 (std::size_t)      * @return The constant string of the attribute name      */-    virtual const char* getAttrName(size_t) const = 0;+    virtual const char* getAttrName(std::size_t) const = 0;      /**      * Return the arity of a relation.      * For example for a tuple (1 2) the arity is 2 and for a tuple (1 2 3) the arity is 3.      *-     * @return Arity of a relation (size_t)+     * @return Arity of a relation (`arity_type`)      */-    virtual size_t getArity() const = 0;+    virtual arity_type getArity() const = 0;      /**      * Return the number of auxiliary attributes. Auxiliary attributes      * are used for provenance and and other alternative evaluation      * strategies. They are stored as the last attributes of a tuple.      *-     * @return Number of auxiliary attributes of a relation (size_t)+     * @return Number of auxiliary attributes of a relation (`arity_type`)      */-    virtual size_t getAuxiliaryArity() const = 0;+    virtual arity_type getAuxiliaryArity() const = 0;      /**+     * Return the number of non-auxiliary attributes.+     * Auxiliary attributes are used for provenance and and other alternative+     * evaluation strategies.+     * They are stored as the last attributes of a tuple.+     *+     * @return Number of non-auxiliary attributes of a relation (`arity_type`)+     */+    arity_type getPrimaryArity() const {+        assert(getAuxiliaryArity() <= getArity());+        return getArity() - getAuxiliaryArity();+    }++    /**      * Get the symbol table of a relation.      * The symbols in a tuple to be stored into a relation are stored and assigned with a number in a table      * called symbol table. For example, to insert ("John","Student") to a relation, "John" and "Student" are@@ -374,7 +410,7 @@         }          std::string signature = "<" + std::string(getAttrType(0));-        for (size_t i = 1; i < getArity(); i++) {+        for (arity_type i = 1; i < getArity(); i++) {             signature += "," + std::string(getAttrType(i));         }         signature += ">";@@ -423,7 +459,7 @@      * helps to make sure we access an insert a tuple within the bound by making sure pos never exceeds the      * arity of the relation.      */-    size_t pos;+    std::size_t pos;  public:     /**@@ -472,10 +508,11 @@     /**      * Return the number of elements in the tuple.      *-     * @return the number of elements in the tuple (size_t).+     * @return the number of elements in the tuple (std::size_t).      */-    size_t size() const {-        return array.size();+    Relation::arity_type size() const {+        assert(array.size() <= std::numeric_limits<Relation::arity_type>::max());+        return Relation::arity_type(array.size());     }      /**@@ -488,9 +525,9 @@      * only be used by friendly classes such as      * iterators; users should not use this interface.      *-     * @param idx This is the idx of element in a tuple (size_t).+     * @param idx This is the idx of element in a tuple (std::size_t).      */-    RamDomain& operator[](size_t idx) {+    RamDomain& operator[](std::size_t idx) {         return array[idx];     } @@ -504,9 +541,9 @@      * only be used by friendly classes such as      * iterators; users should not use this interface.      *-     * @param idx This is the idx of element in a tuple (size_t).+     * @param idx This is the idx of element in a tuple (std::size_t).      */-    const RamDomain& operator[](size_t idx) const {+    const RamDomain& operator[](std::size_t idx) const {         return array[idx];     } @@ -527,7 +564,7 @@     tuple& operator<<(const std::string& str) {         assert(pos < size() && "exceeded tuple's size");         assert(*relation.getAttrType(pos) == 's' && "wrong element type");-        array[pos++] = relation.getSymbolTable().lookup(str);+        array[pos++] = relation.getSymbolTable().encode(str);         return *this;     } @@ -540,7 +577,8 @@      */     tuple& operator<<(RamSigned integer) {         assert(pos < size() && "exceeded tuple's size");-        assert((*relation.getAttrType(pos) == 'i' || *relation.getAttrType(pos) == 'r') &&+        assert((*relation.getAttrType(pos) == 'i' || *relation.getAttrType(pos) == 'r' ||+                       *relation.getAttrType(pos) == '+') &&                 "wrong element type");         array[pos++] = integer;         return *this;@@ -584,7 +622,7 @@     tuple& operator>>(std::string& str) {         assert(pos < size() && "exceeded tuple's size");         assert(*relation.getAttrType(pos) == 's' && "wrong element type");-        str = relation.getSymbolTable().resolve(array[pos++]);+        str = relation.getSymbolTable().decode(array[pos++]);         return *this;     } @@ -597,7 +635,8 @@      */     tuple& operator>>(RamSigned& integer) {         assert(pos < size() && "exceeded tuple's size");-        assert((*relation.getAttrType(pos) == 'i' || *relation.getAttrType(pos) == 'r') &&+        assert((*relation.getAttrType(pos) == 'i' || *relation.getAttrType(pos) == 'r' ||+                       *relation.getAttrType(pos) == '+') &&                 "wrong element type");         integer = ramBitCast<RamSigned>(array[pos++]);         return *this;@@ -653,7 +692,7 @@  * Abstract base class for generated Datalog programs.  */ class SouffleProgram {-private:+protected:     /**      * Define a relation map for external access, when getRelation(name) is called,      * the relation with the given name will be returned from this map,@@ -681,37 +720,53 @@      * allRelations store all the relation in a vector.      */     std::vector<Relation*> allRelations;+     /**      * The number of threads used by OpenMP      */     std::size_t numThreads = 1; -protected:     /**+     * Enable I/O+     */+    bool performIO = false;++    /**+     * Prune Intermediate Relations when there is no further use for them.+     */+    bool pruneImdtRels = true;++    /**      * Add the relation to relationMap (with its name) and allRelations,      * depends on the properties of the relation, if the relation is an input relation, it will be added to      * inputRelations, else if the relation is an output relation, it will be added to outputRelations,      * otherwise will add to internalRelations. (a relation could be both input and output at the same time.)      *      * @param name the name of the relation (std::string)-     * @param rel a pointer to the relation (std::string)+     * @param rel a reference to the relation      * @param isInput a bool argument, true if the relation is a input relation, else false (bool)      * @param isOnput a bool argument, true if the relation is a ouput relation, else false (bool)      */-    void addRelation(const std::string& name, Relation* rel, bool isInput, bool isOutput) {-        relationMap[name] = rel;-        allRelations.push_back(rel);+    void addRelation(const std::string& name, Relation& rel, bool isInput, bool isOutput) {+        relationMap[name] = &rel;+        allRelations.push_back(&rel);         if (isInput) {-            inputRelations.push_back(rel);+            inputRelations.push_back(&rel);         }         if (isOutput) {-            outputRelations.push_back(rel);+            outputRelations.push_back(&rel);         }         if (!isInput && !isOutput) {-            internalRelations.push_back(rel);+            internalRelations.push_back(&rel);         }     } +    [[deprecated("pass `rel` by reference; `rel` may not be null"), maybe_unused]] void addRelation(+            const std::string& name, Relation* rel, bool isInput, bool isOutput) {+        assert(rel && "`rel` may not be null");+        addRelation(name, *rel, isInput, isOutput);+    }+ public:     /**      * Destructor.@@ -721,7 +776,8 @@     virtual ~SouffleProgram() = default;      /**-     * Execute the souffle program, without any loads or stores.+     * Execute the souffle program, without any loads or stores, and live-profiling (in case it is switched+     * on).      */     virtual void run() {} @@ -731,8 +787,11 @@      *      * @param inputDirectory If non-empty, specifies the input directory      * @param outputDirectory If non-empty, specifies the output directory+     * @param performIO Enable I/O operations+     * @param pruneImdtRels Prune intermediate relations      */-    virtual void runAll(std::string inputDirectory = "", std::string outputDirectory = "") = 0;+    virtual void runAll(std::string inputDirectory = "", std::string outputDirectory = "",+            bool performIO = false, bool pruneImdtRels = true) = 0;      /**      * Read all input relations.@@ -763,7 +822,7 @@     /**      * Set the number of threads to be used      */-    void setNumThreads(std::size_t numThreadsValue) {+    virtual void setNumThreads(std::size_t numThreadsValue) {         this->numThreads = numThreadsValue;     } @@ -795,8 +854,12 @@      * @param name The name of the target relation (const std::string)      * @return The size of the target relation (std::size_t)      */-    std::size_t getRelationSize(const std::string& name) const {-        return getRelation(name)->size();+    std::optional<std::size_t> getRelationSize(const std::string& name) const {+        if (auto* rel = getRelation(name)) {+            return rel->size();+        }++        return std::nullopt;     }      /**@@ -805,8 +868,12 @@      * @param name The name of the target relation (const std::string)      * @return The name of the target relation (std::string)      */-    std::string getRelationName(const std::string& name) const {-        return getRelation(name)->getName();+    std::optional<std::string> getRelationName(const std::string& name) const {+        if (auto* rel = getRelation(name)) {+            return rel->getName();+        }++        return std::nullopt;     }      /**@@ -867,6 +934,11 @@     virtual SymbolTable& getSymbolTable() = 0;      /**+     * Get the record table of the program.+     */+    virtual RecordTable& getRecordTable() = 0;++    /**      * Remove all the tuples from the outputRelations, calling the purge method of each.      *      * @see Relation::purge()@@ -902,7 +974,7 @@     /**      * Helper function for the wrapper function Relation::insert() and Relation::contains().      */-    template <typename Tuple, size_t N>+    template <typename Tuple, std::size_t N>     struct tuple_insert {         static void add(const Tuple& t, souffle::tuple& t1) {             tuple_insert<Tuple, N - 1>::add(t, t1);@@ -948,6 +1020,20 @@         tuple t1(relation);         tuple_insert<decltype(t), sizeof...(Args)>::add(t, t1);         return relation->contains(t1);+    }++    /**+     * Set perform-I/O flag+     */+    void setPerformIO(bool performIOArg) {+        performIO = performIOArg;+    }++    /**+     * Set prune-intermediate-relations flag+     */+    void setPruneImdtRels(bool pruneImdtRelsArg) {+        pruneImdtRels = pruneImdtRelsArg;     } };
cbits/souffle/SymbolTable.h view
@@ -1,6 +1,6 @@ /*  * Souffle - A Datalog Compiler- * Copyright (c) 2013, Oracle and/or its affiliates. All rights reserved+ * Copyright (c) 2013, 2014, 2015 Oracle and/or its affiliates. All rights reserved  * Licensed under the Universal Permissive License v 1.0 as shown at:  * - https://opensource.org/licenses/UPL  * - <souffle root>/licenses/SOUFFLE-UPL.txt@@ -10,234 +10,120 @@  *  * @file SymbolTable.h  *- * Data container to store symbols of the Datalog program.+ * Encodes/decodes symbols to numbers (and vice versa).  *  ***********************************************************************/  #pragma once  #include "souffle/RamTypes.h"-#include "souffle/utility/MiscUtil.h"-#include "souffle/utility/ParallelUtil.h"-#include "souffle/utility/StreamUtil.h"-#include <algorithm>-#include <cstdlib>-#include <deque>-#include <initializer_list>-#include <iostream>++#include <memory> #include <string>-#include <unordered_map>-#include <utility>-#include <vector>  namespace souffle { +/** Interface of a generic SymbolTable iterator. */+class SymbolTableIteratorInterface {+public:+    virtual ~SymbolTableIteratorInterface() {}++    virtual const std::pair<const std::string, const std::size_t>& get() const = 0;++    virtual bool equals(const SymbolTableIteratorInterface& other) = 0;++    virtual SymbolTableIteratorInterface& incr() = 0;++    virtual std::unique_ptr<SymbolTableIteratorInterface> copy() const = 0;+};+ /**  * @class SymbolTable  *- * Global pool of re-usable strings- *- * SymbolTable stores Datalog symbols and converts them to numbers and vice versa.+ * SymbolTable encodes symbols to numbers and decodes numbers to symbols.  */ class SymbolTable {-private:-    /** A lock to synchronize parallel accesses */-    mutable Lock access;--    /** Map indices to strings. */-    std::deque<std::string> numToStr;--    /** Map strings to indices. */-    std::unordered_map<std::string, size_t> strToNum;--    /** Convenience method to place a new symbol in the table, if it does not exist, and return the index of-     * it. */-    inline size_t newSymbolOfIndex(const std::string& symbol) {-        size_t index;-        auto it = strToNum.find(symbol);-        if (it == strToNum.end()) {-            index = numToStr.size();-            strToNum[symbol] = index;-            numToStr.push_back(symbol);-        } else {-            index = it->second;-        }-        return index;-    }+public:+    virtual ~SymbolTable() {} -    /** Convenience method to place a new symbol in the table, if it does not exist. */-    inline void newSymbol(const std::string& symbol) {-        if (strToNum.find(symbol) == strToNum.end()) {-            strToNum[symbol] = numToStr.size();-            numToStr.push_back(symbol);-        }-    }+    /**+     * @brief Iterator on a symbol table.+     *+     * Iterator over pairs of a symbol and its encoding index.+     */+    class Iterator {+    public:+        using value_type = const std::pair<const std::string, const std::size_t>;+        using reference = value_type&;+        using pointer = value_type*; -public:-    /** Empty constructor. */-    SymbolTable() = default;+        Iterator(std::unique_ptr<SymbolTableIteratorInterface> ptr) : impl(std::move(ptr)) {} -    /** Copy constructor, performs a deep copy. */-    SymbolTable(const SymbolTable& other) : numToStr(other.numToStr), strToNum(other.strToNum) {}+        Iterator(const Iterator& it) : impl(it.impl->copy()) {} -    /** Copy constructor for r-value reference. */-    SymbolTable(SymbolTable&& other) noexcept {-        numToStr.swap(other.numToStr);-        strToNum.swap(other.strToNum);-    }+        Iterator(Iterator&& it) : impl(std::move(it.impl)) {} -    SymbolTable(std::initializer_list<std::string> symbols) {-        strToNum.reserve(symbols.size());-        for (const auto& symbol : symbols) {-            newSymbol(symbol);+        reference operator*() const {+            return impl->get();         }-    } -    /** Destructor, frees memory allocated for all strings. */-    virtual ~SymbolTable() = default;+        pointer operator->() const {+            return &impl->get();+        } -    /** Assignment operator, performs a deep copy and frees memory allocated for all strings. */-    SymbolTable& operator=(const SymbolTable& other) {-        if (this == &other) {+        Iterator& operator++() {+            impl->incr();             return *this;         }-        numToStr = other.numToStr;-        strToNum = other.strToNum;-        return *this;-    } -    /** Assignment operator for r-value references. */-    SymbolTable& operator=(SymbolTable&& other) noexcept {-        numToStr.swap(other.numToStr);-        strToNum.swap(other.strToNum);-        return *this;-    }--    /** Find the index of a symbol in the table, inserting a new symbol if it does not exist there-     * already. */-    RamDomain lookup(const std::string& symbol) {-        {-            auto lease = access.acquire();-            (void)lease;  // avoid warning;-            return static_cast<RamDomain>(newSymbolOfIndex(symbol));+        Iterator operator++(int) {+            Iterator prev(impl->copy());+            impl->incr();+            return prev;         }-    } -    /** Finds the index of a symbol in the table, giving an error if it's not found */-    RamDomain lookupExisting(const std::string& symbol) const {-        {-            auto lease = access.acquire();-            (void)lease;  // avoid warning;-            auto result = strToNum.find(symbol);-            if (result == strToNum.end()) {-                fatal("Error string not found in call to `SymbolTable::lookupExisting`: `%s`", symbol);-            }-            return static_cast<RamDomain>(result->second);+        bool operator==(const Iterator& I) const {+            return impl->equals(*I.impl);         }-    } -    /** Find the index of a symbol in the table, inserting a new symbol if it does not exist there-     * already. */-    RamDomain unsafeLookup(const std::string& symbol) {-        return static_cast<RamDomain>(newSymbolOfIndex(symbol));-    }--    /** Find a symbol in the table by its index, note that this gives an error if the index is out of-     * bounds.-     */-    const std::string& resolve(const RamDomain index) const {-        {-            auto lease = access.acquire();-            (void)lease;  // avoid warning;-            auto pos = static_cast<size_t>(index);-            if (pos >= size()) {-                // TODO: use different error reporting here!!-                fatal("Error index out of bounds in call to `SymbolTable::resolve`. index = `%d`", index);-            }-            return numToStr[pos];+        bool operator!=(const Iterator& I) const {+            return !impl->equals(*I.impl);         }-    } -    const std::string& unsafeResolve(const RamDomain index) const {-        return numToStr[static_cast<size_t>(index)];-    }+    private:+        std::unique_ptr<SymbolTableIteratorInterface> impl;+    }; -    /* Return the size of the symbol table, being the number of symbols it currently holds. */-    size_t size() const {-        return numToStr.size();-    }+    using iterator = Iterator; -    /** Bulk insert symbols into the table, note that this operation is more efficient than repeated-     * inserts-     * of single symbols. */-    void insert(const std::vector<std::string>& symbols) {-        {-            auto lease = access.acquire();-            (void)lease;  // avoid warning;-            strToNum.reserve(size() + symbols.size());-            for (auto& symbol : symbols) {-                newSymbol(symbol);-            }-        }-    }+    /** @brief Return an iterator on the first symbol. */+    virtual iterator begin() const = 0; -    /** Insert a single symbol into the table, not that this operation should not be used if inserting-     * symbols-     * in bulk. */-    void insert(const std::string& symbol) {-        {-            auto lease = access.acquire();-            (void)lease;  // avoid warning;-            newSymbol(symbol);-        }-    }+    /** @brief Return an iterator past the last symbol. */+    virtual iterator end() const = 0; -    /** Print the symbol table to the given stream. */-    void print(std::ostream& out) const {-        {-            out << "SymbolTable: {\n\t";-            out << join(strToNum, "\n\t",-                           [](std::ostream& out, const std::pair<std::string, std::size_t>& entry) {-                               out << entry.first << "\t => " << entry.second;-                           })-                << "\n";-            out << "}\n";-        }-    }+    /** @brief Check if the given symbol exist. */+    virtual bool weakContains(const std::string& symbol) const = 0; -    /** Check if the symbol table contains a string */-    bool contains(const std::string& symbol) const {-        auto lease = access.acquire();-        (void)lease;  // avoid warning;-        auto result = strToNum.find(symbol);-        if (result == strToNum.end()) {-            return false;-        } else {-            return true;-        }-    }+    /** @brief Encode a symbol to a symbol index. */+    virtual RamDomain encode(const std::string& symbol) = 0; -    /** Check if the symbol table contains an index */-    bool contains(const RamDomain index) const {-        auto lease = access.acquire();-        (void)lease;  // avoid warning;-        auto pos = static_cast<size_t>(index);-        if (pos >= size()) {-            return false;-        } else {-            return true;-        }-    }+    /** @brief Decode a symbol index to a symbol. */+    virtual const std::string& decode(const RamDomain index) const = 0; -    Lock::Lease acquireLock() const {-        return access.acquire();-    }+    /** @brief Encode a symbol to a symbol index; aliases encode. */+    virtual RamDomain unsafeEncode(const std::string& symbol) = 0; -    /** Stream operator, used as a convenience for print. */-    friend std::ostream& operator<<(std::ostream& out, const SymbolTable& table) {-        table.print(out);-        return out;-    }+    /** @brief Decode a symbol index to a symbol; aliases decode. */+    virtual const std::string& unsafeDecode(const RamDomain index) const = 0;++    /**+     * @brief Encode the symbol, it is inserted if it does not exist.+     *+     * @return the symbol index and a boolean indicating if an insertion+     * happened.+     */+    virtual std::pair<RamDomain, bool> findOrInsert(const std::string& symbol) = 0; };  }  // namespace souffle
cbits/souffle/datastructure/BTree.h view
@@ -17,8 +17,10 @@  #pragma once +#include "souffle/datastructure/BTreeUtil.h" #include "souffle/utility/CacheUtil.h" #include "souffle/utility/ContainerUtil.h"+#include "souffle/utility/MiscUtil.h" #include "souffle/utility/ParallelUtil.h" #include <algorithm> #include <cassert>@@ -36,205 +38,7 @@  namespace detail { -// ---------- comparators --------------- /**- * A generic comparator implementation as it is used by- * a b-tree based on types that can be less-than and- * equality comparable.- */-template <typename T>-struct comparator {-    /**-     * Compares the values of a and b and returns-     * -1 if a<b, 1 if a>b and 0 otherwise-     */-    int operator()(const T& a, const T& b) const {-        return (a > b) - (a < b);-    }-    bool less(const T& a, const T& b) const {-        return a < b;-    }-    bool equal(const T& a, const T& b) const {-        return a == b;-    }-};--// ---------- search strategies ----------------/**- * A common base class for search strategies in b-trees.- */-struct search_strategy {};--/**- * A linear search strategy for looking up keys in b-tree nodes.- */-struct linear_search : public search_strategy {-    /**-     * Required user-defined default constructor.-     */-    linear_search() = default;--    /**-     * Obtains an iterator referencing an element equivalent to the-     * given key in the given range. If no such element is present,-     * a reference to the first element not less than the given key-     * is returned.-     */-    template <typename Key, typename Iter, typename Comp>-    inline Iter operator()(const Key& k, Iter a, Iter b, Comp& comp) const {-        return lower_bound(k, a, b, comp);-    }--    /**-     * Obtains a reference to the first element in the given range that-     * is not less than the given key.-     */-    template <typename Key, typename Iter, typename Comp>-    inline Iter lower_bound(const Key& k, Iter a, Iter b, Comp& comp) const {-        auto c = a;-        while (c < b) {-            auto r = comp(*c, k);-            if (r >= 0) {-                return c;-            }-            ++c;-        }-        return b;-    }--    /**-     * Obtains a reference to the first element in the given range that-     * such that the given key is less than the referenced element.-     */-    template <typename Key, typename Iter, typename Comp>-    inline Iter upper_bound(const Key& k, Iter a, Iter b, Comp& comp) const {-        auto c = a;-        while (c < b) {-            if (comp(*c, k) > 0) {-                return c;-            }-            ++c;-        }-        return b;-    }-};--/**- * A binary search strategy for looking up keys in b-tree nodes.- */-struct binary_search : public search_strategy {-    /**-     * Required user-defined default constructor.-     */-    binary_search() = default;--    /**-     * Obtains an iterator pointing to some element within the given-     * range that is equal to the given key, if available. If multiple-     * elements are equal to the given key, an undefined instance will-     * be obtained (no guaranteed lower or upper boundary).  If no such-     * element is present, a reference to the first element not less than-     * the given key will be returned.-     */-    template <typename Key, typename Iter, typename Comp>-    Iter operator()(const Key& k, Iter a, Iter b, Comp& comp) const {-        Iter c;-        auto count = b - a;-        while (count > 0) {-            auto step = count >> 1;-            c = a + step;-            auto r = comp(*c, k);-            if (r == 0) {-                return c;-            }-            if (r < 0) {-                a = ++c;-                count -= step + 1;-            } else {-                count = step;-            }-        }-        return a;-    }--    /**-     * Obtains a reference to the first element in the given range that-     * is not less than the given key.-     */-    template <typename Key, typename Iter, typename Comp>-    Iter lower_bound(const Key& k, Iter a, Iter b, Comp& comp) const {-        Iter c;-        auto count = b - a;-        while (count > 0) {-            auto step = count >> 1;-            c = a + step;-            if (comp(*c, k) < 0) {-                a = ++c;-                count -= step + 1;-            } else {-                count = step;-            }-        }-        return a;-    }--    /**-     * Obtains a reference to the first element in the given range that-     * such that the given key is less than the referenced element.-     */-    template <typename Key, typename Iter, typename Comp>-    Iter upper_bound(const Key& k, Iter a, Iter b, Comp& comp) const {-        Iter c;-        auto count = b - a;-        while (count > 0) {-            auto step = count >> 1;-            c = a + step;-            if (comp(k, *c) >= 0) {-                a = ++c;-                count -= step + 1;-            } else {-                count = step;-            }-        }-        return a;-    }-};--// ---------- search strategies selection ----------------/**- * A template-meta class to select search strategies for b-trees- * depending on the key type.- */-template <typename S>-struct strategy_selection {-    using type = S;-};--struct linear : public strategy_selection<linear_search> {};-struct binary : public strategy_selection<binary_search> {};--// by default every key utilizes binary search-template <typename Key>-struct default_strategy : public binary {};--template <>-struct default_strategy<int> : public linear {};--template <typename... Ts>-struct default_strategy<std::tuple<Ts...>> : public linear {};--/**- * The default non-updater- */-template <typename T>-struct updater {-    void update(T& /* old_t */, const T& /* new_t */) {}-};--/**  * The actual implementation of a b-tree data structure.  *  * @tparam Key             .. the element type to be stored in this tree@@ -367,13 +171,13 @@         /**          * The number of keys/node desired by the user.          */-        static constexpr size_t desiredNumKeys =+        static constexpr std::size_t desiredNumKeys =                 ((blockSize > sizeof(base)) ? blockSize - sizeof(base) : 0) / sizeof(Key);          /**          * The actual number of keys/node corrected by functional requirements.          */-        static constexpr size_t maxKeys = (desiredNumKeys > 3) ? desiredNumKeys : 3;+        static constexpr std::size_t maxKeys = (desiredNumKeys > 3) ? desiredNumKeys : 3;          // the keys stored in this node         Key keys[maxKeys];@@ -820,7 +624,7 @@                         }                     } -                    pos = (i > other->numElements) ? 0 : i;+                    pos = (i > static_cast<unsigned>(other->numElements)) ? 0 : static_cast<unsigned>(i);                     other->insert_inner(root, root_lock, pos, predecessor, key, newNode, locked_nodes); #else                     other->insert_inner(root, root_lock, pos, predecessor, key, newNode);@@ -1105,11 +909,11 @@         field_index_type pos = 0;      public:-        typedef std::forward_iterator_tag iterator_category;-        typedef Key value_type;-        typedef ptrdiff_t difference_type;-        typedef value_type* pointer;-        typedef value_type& reference;+        using iterator_category = std::forward_iterator_tag;+        using value_type = Key;+        using difference_type = ptrdiff_t;+        using pointer = value_type*;+        using reference = value_type&;          // default constructor -- creating an end-iterator         iterator() : cur(nullptr) {}@@ -1256,7 +1060,7 @@  public:     // the maximum number of keys stored per node-    static constexpr size_t max_keys_per_node = node::maxKeys;+    static constexpr std::size_t max_keys_per_node = node::maxKeys;      // -- ctors / dtors -- @@ -1536,7 +1340,8 @@                  // split this node                 auto old_root = root;-                idx -= cur->rebalance_or_split(const_cast<node**>(&root), root_lock, idx, parents);+                idx -= cur->rebalance_or_split(+                        const_cast<node**>(&root), root_lock, static_cast<int>(idx), parents);                  // release parent lock                 for (auto it = parents.rbegin(); it != parents.rend(); ++it) {@@ -1568,7 +1373,7 @@             assert(cur->numElements < node::maxKeys && "Split required!");              // move keys-            for (int j = cur->numElements; j > idx; --j) {+            for (int j = static_cast<int>(cur->numElements); j > static_cast<int>(idx); --j) {                 cur->keys[j] = cur->keys[j - 1];             } @@ -2156,7 +1961,7 @@         const int N = node::maxKeys;          // divide range in N+1 sub-ranges-        int length = (b - a) + 1;+        int64_t length = (b - a) + 1;          // terminal case: length is less then maxKeys         if (length <= N) {@@ -2173,7 +1978,7 @@          // recursive case - compute step size         int numKeys = N;-        int step = ((length - numKeys) / (numKeys + 1));+        int64_t step = ((length - numKeys) / (numKeys + 1));          while (numKeys > 1 && (step < N / 2)) {             numKeys--;
+ cbits/souffle/datastructure/BTreeDelete.h view
@@ -0,0 +1,2701 @@+/*+ * Souffle - A Datalog Compiler+ * Copyright (c) 2013, 2015, Oracle and/or its affiliates. All rights reserved+ * Licensed under the Universal Permissive License v 1.0 as shown at:+ * - https://opensource.org/licenses/UPL+ * - <souffle root>/licenses/SOUFFLE-UPL.txt+ */++/************************************************************************+ *+ * @file BTreeDelete.h+ *+ * An implementation of a generic B-tree data structure including+ * interfaces for utilizing instances as set or multiset containers+ * and deletion.+ *+ ***********************************************************************/++#pragma once++#include "souffle/datastructure/BTreeUtil.h"+#include "souffle/utility/CacheUtil.h"+#include "souffle/utility/ContainerUtil.h"+#include "souffle/utility/MiscUtil.h"+#include "souffle/utility/ParallelUtil.h"+#include <algorithm>+#include <cassert>+#include <cstddef>+#include <cstdint>+#include <iostream>+#include <iterator>+#include <string>+#include <tuple>+#include <type_traits>+#include <typeinfo>+#include <vector>++namespace souffle {++namespace detail {++/**+ * The actual implementation of a b-tree data structure.+ *+ * @tparam Key             .. the element type to be stored in this tree+ * @tparam Comparator     .. a class defining an order on the stored elements+ * @tparam Allocator     .. utilized for allocating memory for required nodes+ * @tparam blockSize    .. determines the number of bytes/block utilized by leaf nodes+ * @tparam SearchStrategy .. enables switching between linear, binary or any other search strategy+ * @tparam isSet        .. true = set, false = multiset+ */+template <typename Key, typename Comparator,+        typename Allocator,  // is ignored so far - TODO: add support+        unsigned blockSize, typename SearchStrategy, bool isSet, typename WeakComparator = Comparator,+        typename Updater = detail::updater<Key>>+class btree_delete {+public:+    class iterator;+    using const_iterator = iterator;++    using key_type = Key;+    using element_type = Key;+    using chunk = range<iterator>;++protected:+    /* ------------- static utilities ----------------- */++    const static SearchStrategy search;++    /* ---------- comparison utilities ---------------- */++    mutable Comparator comp;++    bool less(const Key& a, const Key& b) const {+        return comp.less(a, b);+    }++    bool equal(const Key& a, const Key& b) const {+        return comp.equal(a, b);+    }++    mutable WeakComparator weak_comp;++    bool weak_less(const Key& a, const Key& b) const {+        return weak_comp.less(a, b);+    }++    bool weak_equal(const Key& a, const Key& b) const {+        return weak_comp.equal(a, b);+    }++    /* -------------- updater utilities ------------- */++    mutable Updater upd;+    void update(Key& old_k, const Key& new_k) {+        upd.update(old_k, new_k);+    }++    /* -------------- the node type ----------------- */++    using size_type = std::size_t;+    using field_index_type = uint8_t;+    using lock_type = OptimisticReadWriteLock;++    struct node;++    /**+     * The base type of all node types containing essential+     * book-keeping information.+     */+    struct base {+#ifdef IS_PARALLEL++        // the parent node+        node* volatile parent;++        // a lock for synchronizing parallel operations on this node+        lock_type lock;++        // the number of keys in this node+        volatile size_type numElements;++        // the position in the parent node+        volatile field_index_type position;+#else+        // the parent node+        node* parent;++        // the number of keys in this node+        size_type numElements;++        // the position in the parent node+        field_index_type position;+#endif++        // a flag indicating whether this is a inner node or not+        const bool inner;++        /**+         * A simple constructor for nodes+         */+        base(bool inner) : parent(nullptr), numElements(0), position(0), inner(inner) {}++        bool isLeaf() const {+            return !inner;+        }++        bool isInner() const {+            return inner;+        }++        node* getParent() const {+            return parent;+        }++        field_index_type getPositionInParent() const {+            return position;+        }++        size_type getNumElements() const {+            return numElements;+        }+    };++    struct inner_node;++    /**+     * The actual, generic node implementation covering the operations+     * for both, inner and leaf nodes.+     */+    struct node : public base {+        /**+         * The number of keys/node desired by the user.+         */+        static constexpr std::size_t desiredNumKeys =+                ((blockSize > sizeof(base)) ? blockSize - sizeof(base) : 0) / sizeof(Key);++        /**+         * The actual number of keys/node corrected by functional requirements.+         */+        static constexpr std::size_t maxKeys = (desiredNumKeys > 3) ? desiredNumKeys : 3;+        static constexpr std::size_t split_point = std::min(3 * maxKeys / 4, maxKeys - 2);+        static constexpr std::size_t minKeys = std::min(maxKeys - (split_point + 1), split_point + 1);++        // the keys stored in this node+        Key keys[maxKeys];++        // a simple constructor+        node(bool inner) : base(inner) {}++        /**+         * A deep-copy operation creating a clone of this node.+         */+        node* clone() const {+            // create a clone of this node+            node* res = (this->isInner()) ? static_cast<node*>(new inner_node())+                                          : static_cast<node*>(new leaf_node());++            // copy basic fields+            res->position = this->position;+            res->numElements = this->numElements;++            for (size_type i = 0; i < this->numElements; ++i) {+                res->keys[i] = this->keys[i];+            }++            // if this is a leaf we are done+            if (this->isLeaf()) {+                return res;+            }++            // copy child nodes recursively+            auto* ires = (inner_node*)res;+            for (size_type i = 0; i <= this->numElements; ++i) {+                ires->children[i] = this->getChild(i)->clone();+                ires->children[i]->parent = res;+            }++            // that's it+            return res;+        }++        /**+         * A utility function providing a reference to this node as+         * an inner node.+         */+        inner_node& asInnerNode() {+            assert(this->inner && "Invalid cast!");+            return *static_cast<inner_node*>(this);+        }++        /**+         * A utility function providing a reference to this node as+         * a const inner node.+         */+        const inner_node& asInnerNode() const {+            assert(this->inner && "Invalid cast!");+            return *static_cast<const inner_node*>(this);+        }++        /**+         * Computes the number of nested levels of the tree rooted+         * by this node.+         */+        size_type getDepth() const {+            if (this->isLeaf()) {+                return 1;+            }+            return getChild(0)->getDepth() + 1;+        }++        /**+         * Counts the number of nodes contained in the sub-tree rooted+         * by this node.+         */+        size_type countNodes() const {+            if (this->isLeaf()) {+                return 1;+            }+            size_type sum = 1;+            for (unsigned i = 0; i <= this->numElements; ++i) {+                sum += getChild(i)->countNodes();+            }+            return sum;+        }++        /**+         * Counts the number of entries contained in the sub-tree rooted+         * by this node.+         */+        size_type countEntries() const {+            if (this->isLeaf()) {+                return this->numElements;+            }+            size_type sum = this->numElements;+            for (unsigned i = 0; i <= this->numElements; ++i) {+                sum += getChild(i)->countEntries();+            }+            return sum;+        }++        /**+         * Determines the amount of memory used by the sub-tree rooted+         * by this node.+         */+        size_type getMemoryUsage() const {+            if (this->isLeaf()) {+                return sizeof(leaf_node);+            }+            size_type res = sizeof(inner_node);+            for (unsigned i = 0; i <= this->numElements; ++i) {+                res += getChild(i)->getMemoryUsage();+            }+            return res;+        }++        /**+         * Obtains a pointer to the array of child-pointers+         * of this node -- if it is an inner node.+         */+        node** getChildren() {+            return asInnerNode().children;+        }++        /**+         * Obtains a pointer to the array of const child-pointers+         * of this node -- if it is an inner node.+         */+        node* const* getChildren() const {+            return asInnerNode().children;+        }++        /**+         * Obtains a reference to the child of the given index.+         */+        node* getChild(size_type s) const {+            return asInnerNode().children[s];+        }++        /**+         * Checks whether this node is empty -- can happen due to biased insertion.+         */+        bool isEmpty() const {+            return this->numElements == 0;+        }++        /**+         * Checks whether this node is full.+         */+        bool isFull() const {+            return this->numElements == maxKeys;+        }++        /**+         * Obtains the point at which full nodes should be split.+         * Conventional b-trees always split in half. However, in cases+         * where in-order insertions are frequent, a split assigning+         * larger portions to the right fragment provide higher performance+         * and a better node-filling rate.+         */+        int getSplitPoint(int /*unused*/) {+            return static_cast<int>(split_point);+        }++        /**+         * Splits this node.+         *+         * @param root .. a pointer to the root-pointer of the enclosing b-tree+         *                 (might have to be updated if the root-node needs to be split)+         * @param idx  .. the position of the insert causing the split+         */+#ifdef IS_PARALLEL+        void split(node** root, lock_type& root_lock, int idx, std::vector<node*>& locked_nodes) {+            assert(this->lock.is_write_locked());+            assert(!this->parent || this->parent->lock.is_write_locked());+            assert((this->parent != nullptr) || root_lock.is_write_locked());+            assert(this->isLeaf() || souffle::contains(locked_nodes, this));+            assert(!this->parent || souffle::contains(locked_nodes, const_cast<node*>(this->parent)));+#else+        void split(node** root, lock_type& root_lock, int idx) {+#endif+            assert(this->numElements == maxKeys);++            // get middle element+            int split_point = getSplitPoint(idx);++            // create a new sibling node+            node* sibling = (this->inner) ? static_cast<node*>(new inner_node())+                                          : static_cast<node*>(new leaf_node());++#ifdef IS_PARALLEL+            // lock sibling+            sibling->lock.start_write();+            locked_nodes.push_back(sibling);+#endif++            // move data over to the new node+            for (unsigned i = split_point + 1, j = 0; i < maxKeys; ++i, ++j) {+                sibling->keys[j] = keys[i];+            }++            // move child pointers+            if (this->inner) {+                // move pointers to sibling+                auto* other = static_cast<inner_node*>(sibling);+                for (unsigned i = split_point + 1, j = 0; i <= maxKeys; ++i, ++j) {+                    other->children[j] = getChildren()[i];+                    other->children[j]->parent = other;+                    other->children[j]->position = static_cast<field_index_type>(j);+                }+            }++            // update number of elements+            this->numElements = split_point;+            sibling->numElements = maxKeys - split_point - 1;++            // update parent+#ifdef IS_PARALLEL+            grow_parent(root, root_lock, sibling, locked_nodes);+#else+            grow_parent(root, root_lock, sibling);+#endif+        }++        /**+         * Moves keys from this node to one of its siblings or splits+         * this node to make some space for the insertion of an element at+         * position idx.+         *+         * Returns the number of elements moved to the left side, 0 in case+         * of a split. The number of moved elements will be <= the given idx.+         *+         * @param root .. the root node of the b-tree being part of+         * @param idx  .. the position of the insert triggering this operation+         */+        // TODO: remove root_lock ... no longer needed+#ifdef IS_PARALLEL+        int rebalance_or_split(node** root, lock_type& root_lock, int idx, std::vector<node*>& locked_nodes) {+            assert(this->lock.is_write_locked());+            assert(!this->parent || this->parent->lock.is_write_locked());+            assert((this->parent != nullptr) || root_lock.is_write_locked());+            assert(this->isLeaf() || souffle::contains(locked_nodes, this));+            assert(!this->parent || souffle::contains(locked_nodes, const_cast<node*>(this->parent)));+#else+        int rebalance_or_split(node** root, lock_type& root_lock, int idx) {+#endif++            // this node is full ... and needs some space+            assert(this->numElements == maxKeys);++            // get snap-shot of parent+            auto parent = this->parent;+            auto pos = this->position;++            // Option A) re-balance data+            if (parent && pos > 0) {+                node* left = parent->getChild(pos - 1);++#ifdef IS_PARALLEL+                // lock access to left sibling+                if (!left->lock.try_start_write()) {+                    // left node is currently updated => skip balancing and split+                    split(root, root_lock, idx, locked_nodes);+                    return 0;+                }+#endif++                // compute number of elements to be movable to left+                //    space available in left vs. insertion index+                size_type num = static_cast<size_type>(+                        std::min<int>(static_cast<int>(maxKeys - left->numElements), idx));++                // if there are elements to move ..+                if (num > 0) {+                    Key* splitter = &(parent->keys[this->position - 1]);++                    // .. move keys to left node+                    left->keys[left->numElements] = *splitter;+                    for (size_type i = 0; i < num - 1; ++i) {+                        left->keys[left->numElements + 1 + i] = keys[i];+                    }+                    *splitter = keys[num - 1];++                    // shift keys in this node to the left+                    for (size_type i = 0; i < this->numElements - num; ++i) {+                        keys[i] = keys[i + num];+                    }++                    // .. and children if necessary+                    if (this->isInner()) {+                        auto* ileft = static_cast<inner_node*>(left);+                        auto* iright = static_cast<inner_node*>(this);++                        // move children+                        for (field_index_type i = 0; i < num; ++i) {+                            ileft->children[left->numElements + i + 1] = iright->children[i];+                        }++                        // update moved children+                        for (size_type i = 0; i < num; ++i) {+                            iright->children[i]->parent = ileft;+                            iright->children[i]->position =+                                    static_cast<field_index_type>(left->numElements + i) + 1;+                        }++                        // shift child-pointer to the left+                        for (size_type i = 0; i < this->numElements - num + 1; ++i) {+                            iright->children[i] = iright->children[i + num];+                        }++                        // update position of children+                        for (size_type i = 0; i < this->numElements - num + 1; ++i) {+                            iright->children[i]->position = static_cast<field_index_type>(i);+                        }+                    }++                    // update node sizes+                    left->numElements += num;+                    this->numElements -= num;++#ifdef IS_PARALLEL+                    left->lock.end_write();+#endif++                    // done+                    return static_cast<int>(num);+                }++#ifdef IS_PARALLEL+                left->lock.abort_write();+#endif+            }++            // Option B) split node+#ifdef IS_PARALLEL+            split(root, root_lock, idx, locked_nodes);+#else+            split(root, root_lock, idx);+#endif+            return 0;  // = no re-balancing+        }++    private:+        /**+         * Inserts a new sibling into the parent of this node utilizing+         * the last key of this node as a separation key. (for internal+         * use only)+         *+         * @param root .. a pointer to the root-pointer of the containing tree+         * @param sibling .. the new right-sibling to be add to the parent node+         */+#ifdef IS_PARALLEL+        void grow_parent(node** root, lock_type& root_lock, node* sibling, std::vector<node*>& locked_nodes) {+            assert(this->lock.is_write_locked());+            assert(!this->parent || this->parent->lock.is_write_locked());+            assert((this->parent != nullptr) || root_lock.is_write_locked());+            assert(this->isLeaf() || souffle::contains(locked_nodes, this));+            assert(!this->parent || souffle::contains(locked_nodes, const_cast<node*>(this->parent)));+#else+        void grow_parent(node** root, lock_type& root_lock, node* sibling) {+#endif++            if (this->parent == nullptr) {+                assert(*root == this);++                // create a new root node+                auto* new_root = new inner_node();+                new_root->numElements = 1;+                new_root->keys[0] = keys[this->numElements];++                new_root->children[0] = this;+                new_root->children[1] = sibling;++                // link this and the sibling node to new root+                this->parent = new_root;+                sibling->parent = new_root;+                sibling->position = 1;++                // switch root node+                *root = new_root;++            } else {+                // insert new element in parent element+                auto parent = this->parent;+                auto pos = this->position;++#ifdef IS_PARALLEL+                parent->insert_inner(+                        root, root_lock, pos, this, keys[this->numElements], sibling, locked_nodes);+#else+                parent->insert_inner(root, root_lock, pos, this, keys[this->numElements], sibling);+#endif+            }+        }++        /**+         * Inserts a new element into an inner node (for internal use only).+         *+         * @param root .. a pointer to the root-pointer of the containing tree+         * @param pos  .. the position to insert the new key+         * @param key  .. the key to insert+         * @param newNode .. the new right-child of the inserted key+         */+#ifdef IS_PARALLEL+        void insert_inner(node** root, lock_type& root_lock, unsigned pos, node* predecessor, const Key& key,+                node* newNode, std::vector<node*>& locked_nodes) {+            assert(this->lock.is_write_locked());+            assert(souffle::contains(locked_nodes, this));+#else+        void insert_inner(node** root, lock_type& root_lock, unsigned pos, node* predecessor, const Key& key,+                node* newNode) {+#endif++            // check capacity+            if (this->numElements >= maxKeys) {+#ifdef IS_PARALLEL+                assert(!this->parent || this->parent->lock.is_write_locked());+                assert((this->parent) || root_lock.is_write_locked());+                assert(!this->parent || souffle::contains(locked_nodes, const_cast<node*>(this->parent)));+#endif++                // split this node+#ifdef IS_PARALLEL+                pos -= rebalance_or_split(root, root_lock, pos, locked_nodes);+#else+                pos -= rebalance_or_split(root, root_lock, pos);+#endif++                // complete insertion within new sibling if necessary+                if (pos > this->numElements) {+                    // correct position+                    pos = pos - static_cast<unsigned int>(this->numElements) - 1;++                    // get new sibling+                    auto other = this->parent->getChild(this->position + 1);++#ifdef IS_PARALLEL+                    // make sure other side is write locked+                    assert(other->lock.is_write_locked());+                    assert(souffle::contains(locked_nodes, other));++                    // search for new position (since other may have been altered in the meanwhile)+                    size_type i = 0;+                    for (; i <= other->numElements; ++i) {+                        if (other->getChild(i) == predecessor) {+                            break;+                        }+                    }++                    pos = (i > static_cast<unsigned>(other->numElements)) ? 0 : static_cast<unsigned>(i);+                    other->insert_inner(root, root_lock, pos, predecessor, key, newNode, locked_nodes);+#else+                    other->insert_inner(root, root_lock, pos, predecessor, key, newNode);+#endif+                    return;+                }+            }++            // move bigger keys one forward+            for (int i = static_cast<int>(this->numElements) - 1; i >= (int)pos; --i) {+                keys[i + 1] = keys[i];+                getChildren()[i + 2] = getChildren()[i + 1];+                ++getChildren()[i + 2]->position;+            }++            // ensure proper position+            assert(getChild(pos) == predecessor);++            // insert new element+            keys[pos] = key;+            getChildren()[pos + 1] = newNode;+            newNode->parent = this;+            newNode->position = static_cast<field_index_type>(pos) + 1;+            ++this->numElements;+        }++    public:+        /**+         * Prints a textual representation of this tree to the given output stream.+         * This feature is mainly intended for debugging and tuning purposes.+         *+         * @see btree::printTree+         */+        void printTree(std::ostream& out, const std::string& prefix) const {+            // print the header+            out << prefix << "@" << this << "[" << ((int)(this->position)) << "] - "+                << (this->inner ? "i" : "") << "node : " << this->numElements << "/" << maxKeys << " [";++            // print the keys+            for (unsigned i = 0; i < this->numElements; i++) {+                out << keys[i];+                if (i != this->numElements - 1) {+                    out << ",";+                }+            }+            out << "]";++            // print references to children+            if (this->inner) {+                out << " - [";+                for (unsigned i = 0; i <= this->numElements; i++) {+                    out << getChildren()[i];+                    if (i != this->numElements) {+                        out << ",";+                    }+                }+                out << "]";+            }++#ifdef IS_PARALLEL+            // print the lock state+            if (this->lock.is_write_locked()) {+                std::cout << " locked";+            }+#endif++            out << "\n";++            // print the children recursively+            if (this->inner) {+                for (unsigned i = 0; i < this->numElements + 1; ++i) {+                    static_cast<const inner_node*>(this)->children[i]->printTree(out, prefix + "    ");+                }+            }+        }++        /**+         * A function decomposing the sub-tree rooted by this node into approximately equally+         * sized chunks. To minimize computational overhead, no strict load balance nor limit+         * on the number of actual chunks is given.+         *+         * @see btree::getChunks()+         *+         * @param res   .. the list of chunks to be extended+         * @param num   .. the number of chunks to be produced+         * @param begin .. the iterator to start the first chunk with+         * @param end   .. the iterator to end the last chunk with+         * @return the handed in list of chunks extended by generated chunks+         */+        std::vector<chunk>& collectChunks(+                std::vector<chunk>& res, size_type num, const iterator& begin, const iterator& end) const {+            assert(num > 0);++            // special case: this node is empty+            if (isEmpty()) {+                if (begin != end) {+                    res.push_back(chunk(begin, end));+                }+                return res;+            }++            // special case: a single chunk is requested+            if (num == 1) {+                res.push_back(chunk(begin, end));+                return res;+            }++            // cut-off+            if (this->isLeaf() || num < (this->numElements + 1)) {+                auto step = this->numElements / num;+                if (step == 0) {+                    step = 1;+                }++                size_type i = 0;++                // the first chunk starts at the begin+                res.push_back(chunk(begin, iterator(this, static_cast<field_index_type>(step) - 1)));++                // split up the main part+                for (i = step - 1; i < this->numElements - step; i += step) {+                    res.push_back(chunk(iterator(this, static_cast<field_index_type>(i)),+                            iterator(this, static_cast<field_index_type>(i + step))));+                }++                // the last chunk runs to the end+                res.push_back(chunk(iterator(this, static_cast<field_index_type>(i)), end));++                // done+                return res;+            }++            // else: collect chunks of sub-set elements++            auto part = num / (this->numElements + 1);+            assert(part > 0);+            getChild(0)->collectChunks(res, part, begin, iterator(this, 0));+            for (size_type i = 1; i < this->numElements; i++) {+                getChild(i)->collectChunks(res, part, iterator(this, static_cast<field_index_type>(i - 1)),+                        iterator(this, static_cast<field_index_type>(i)));+            }+            getChild(this->numElements)+                    ->collectChunks(res, num - (part * this->numElements),+                            iterator(this, static_cast<field_index_type>(this->numElements) - 1), end);++            // done+            return res;+        }++        /**+         * A function to verify the consistency of this node.+         *+         * @param root ... a reference to the root of the enclosing tree.+         * @return true if valid, false otherwise+         */+        template <typename Comp>+        bool check(Comp& comp, const node* root) const {+            bool valid = true;++            // check fill-state+            if (this->numElements > maxKeys || (this->parent != nullptr && this->numElements < minKeys)) {+                std::cout << "Node with " << this->numElements << "/" << maxKeys << " encountered!\n";+                valid = false;+            }++            // check root state+            if (root == this) {+                if (this->parent != nullptr) {+                    std::cout << "Root not properly linked!\n";+                    valid = false;+                }+            } else {+                // check parent relation+                if (!this->parent) {+                    std::cout << "Invalid null-parent!\n";+                    valid = false;+                } else {+                    if (this->parent->getChildren()[this->position] != this) {+                        std::cout << "Parent reference invalid!\n";+                        std::cout << "   Node:     " << this << "\n";+                        std::cout << "   Parent:   " << this->parent << "\n";+                        std::cout << "   Position: " << ((int)this->position) << "\n";+                        valid = false;+                    }++                    // check parent key+                    if (valid && this->position != 0 &&+                            !(comp(this->parent->keys[this->position - 1], keys[0]) < ((isSet) ? 0 : 1))) {+                        std::cout << "Left parent key not lower bound!\n";+                        std::cout << "   Node:     " << this << "\n";+                        std::cout << "   Parent:   " << this->parent << "\n";+                        std::cout << "   Position: " << ((int)this->position) << "\n";+                        std::cout << "   Key:   " << (this->parent->keys[this->position]) << "\n";+                        std::cout << "   Lower: " << (keys[0]) << "\n";+                        valid = false;+                    }++                    // check parent key+                    if (valid && this->position != this->parent->numElements &&+                            !(comp(keys[this->numElements - 1], this->parent->keys[this->position]) <+                                    ((isSet) ? 0 : 1))) {+                        std::cout << "Right parent key not lower bound!\n";+                        std::cout << "   Node:     " << this << "\n";+                        std::cout << "   Parent:   " << this->parent << "\n";+                        std::cout << "   Position: " << ((int)this->position) << "\n";+                        std::cout << "   Key:   " << (this->parent->keys[this->position]) << "\n";+                        std::cout << "   Upper: " << (keys[0]) << "\n";+                        valid = false;+                    }+                }+            }++            // check element order+            if (this->numElements > 0) {+                for (unsigned i = 0; i < this->numElements - 1; i++) {+                    if (valid && !(comp(keys[i], keys[i + 1]) < ((isSet) ? 0 : 1))) {+                        std::cout << "Element order invalid!\n";+                        std::cout << " @" << this << " key " << i << " is " << keys[i] << " vs "+                                  << keys[i + 1] << "\n";+                        valid = false;+                    }+                }+            }++            // check state of sub-nodes+            if (this->inner) {+                for (unsigned i = 0; i <= this->numElements; i++) {+                    valid &= getChildren()[i]->check(comp, root);+                }+            }++            return valid;+        }+    };  // namespace detail++    /**+     * The data type representing inner nodes of the b-tree. It extends+     * the generic implementation of a node by the storage locations+     * of child pointers.+     */+    struct inner_node : public node {+        // references to child nodes owned by this node+        node* children[node::maxKeys + 1];++        // a simple default constructor initializing member fields+        inner_node() : node(true) {}++        // clear up child nodes recursively+        ~inner_node() {+            for (unsigned i = 0; i <= this->numElements; ++i) {+                if (children[i] != nullptr) {+                    if (children[i]->isLeaf()) {+                        delete static_cast<leaf_node*>(children[i]);+                    } else {+                        delete static_cast<inner_node*>(children[i]);+                    }+                }+            }+        }+    };++    /**+     * The data type representing leaf nodes of the b-tree. It does not+     * add any capabilities to the generic node type.+     */+    struct leaf_node : public node {+        // a simple default constructor initializing member fields+        leaf_node() : node(false) {}+    };++    // ------------------- iterators ------------------------++public:+    /**+     * The iterator type to be utilized for scanning through btree instances.+     */+    class iterator {+        friend class souffle::detail::btree_delete<Key, Comparator, Allocator, blockSize, SearchStrategy,+                true, WeakComparator, Updater>;++        // a pointer to the node currently referred to+        // node const* cur;+        node* cur;++        // the index of the element currently addressed within the referenced node+        field_index_type pos = 0;++    public:+        using iterator_category = std::forward_iterator_tag;+        using value_type = Key;+        using difference_type = ptrdiff_t;+        using pointer = value_type*;+        using reference = value_type&;++        // default constructor -- creating an end-iterator+        iterator() : cur(nullptr) {}++        // creates an iterator referencing a specific element within a given node+        // iterator(node const* cur, field_index_type pos) : cur(cur), pos(pos) {}+        // iterator(node* cur, field_index_type pos) : cur(cur), pos(pos) {}+        iterator(node const* cur, field_index_type pos) : cur(const_cast<node*>(cur)), pos(pos) {}++        // a copy constructor+        iterator(const iterator& other) : cur(other.cur), pos(other.pos) {}++        // an assignment operator+        iterator& operator=(const iterator& other) {+            cur = other.cur;+            pos = other.pos;+            return *this;+        }++        // the equality operator as required by the iterator concept+        bool operator==(const iterator& other) const {+            return cur == other.cur && pos == other.pos;+        }++        // the not-equality operator as required by the iterator concept+        bool operator!=(const iterator& other) const {+            return !(*this == other);+        }++        // the deref operator as required by the iterator concept+        const Key& operator*() const {+            return cur->keys[pos];+        }++        // Resolve an ambiguous position at the end of a leaf by moving forward.+        // Must be called from the end of a leaf or behaviour is undefined.+        void resolvePosition() {+            // Save current node in case we are at the end of the tree+            auto temp = cur;++            // While we are at the end of node, move up to parent+            do {+                pos = cur->getPositionInParent();+                cur = cur->getParent();+            } while (cur && pos == cur->getNumElements());++            // Check if we were at the end of the tree.+            // If so, reset iterator+            if (!cur) {+                cur = temp;+                pos = static_cast<field_index_type>(cur->getNumElements());+            }+        }++        // the increment operator as required by the iterator concept+        iterator& operator++() {+            if (cur) {+                if (cur->isInner()) {+                    // Currently in an inner node so move forward one place+                    cur = cur->getChild(pos + 1);+                    while (cur->isInner()) {+                        cur = cur->getChild(0);+                    }+                    pos = 0;+                } else {+                    // In a leaf so just increment the position+                    ++pos;+                    // If we have reached the end of the leaf walk up to an inner node+                    if (pos == cur->getNumElements()) {+                        resolvePosition();+                    }+                }+            }+            return *this;+        }++        // the decrement operator as required by the iterator concept+        iterator& operator--() {+            if (cur) {+                if (cur->isInner()) {+                    // Currently in an inner node so move back one place+                    cur = cur->getChild(pos);+                    while (cur->isInner()) {+                        cur = cur->getChild(cur->getNumElements());+                    }+                    pos = static_cast<field_index_type>(cur->getNumElements()) - 1;+                } else {+                    // In a leaf so decrement the position+                    // Check if pos > 0 to avoid unsigned wrap-around issues+                    if (pos > 0) {+                        --pos;+                    } else {+                        // Save the current node in case we are at the beginning+                        auto temp = cur;++                        // Walk back up the tree.+                        do {+                            pos = cur->getPositionInParent();+                            cur = cur->getParent();+                        } while (cur && pos == 0);++                        // If we were at the beginning of the tree, reset the iterator+                        if (!cur) {+                            cur = temp;+                        }+                    }+                }+            }+            return *this;+        }++        // prints a textual representation of this iterator to the given stream (mainly for debugging)+        void print(std::ostream& out = std::cout) const {+            out << cur << "[" << (int)pos << "]";+        }+    };++    /**+     * A collection of operation hints speeding up some of the involved operations+     * by exploiting temporal locality.+     */+    template <unsigned size = 1>+    struct btree_operation_hints {+        using node_cache = LRUCache<node*, size>;++        // the node where the last insertion terminated+        node_cache last_insert;++        // the node where the last find-operation terminated+        node_cache last_find_end;++        // the node where the last lower-bound operation terminated+        node_cache last_lower_bound_end;++        // the node where the last upper-bound operation terminated+        node_cache last_upper_bound_end;++        // default constructor+        btree_operation_hints() = default;++        // resets all hints (to be triggered e.g. when deleting nodes)+        void clear() {+            last_insert.clear(nullptr);+            last_find_end.clear(nullptr);+            last_lower_bound_end.clear(nullptr);+            last_upper_bound_end.clear(nullptr);+        }+    };++    using operation_hints = btree_operation_hints<1>;++protected:+#ifdef IS_PARALLEL+    // a pointer to the root node of this tree+    node* volatile root;++    // a lock to synchronize update operations on the root pointer+    lock_type root_lock;+#else+    // a pointer to the root node of this tree+    node* root;++    // required to not duplicate too much code+    lock_type root_lock;+#endif++    // a pointer to the left-most node of this tree (initial note for iteration)+    leaf_node* leftmost;++    /* -------------- operator hint statistics ----------------- */++    // an aggregation of statistical values of the hint utilization+    struct hint_statistics {+        // the counter for insertion operations+        CacheAccessCounter inserts;++        // the counter for contains operations+        CacheAccessCounter contains;++        // the counter for lower_bound operations+        CacheAccessCounter lower_bound;++        // the counter for upper_bound operations+        CacheAccessCounter upper_bound;+    };++    // the hint statistic of this b-tree instance+    mutable hint_statistics hint_stats;++public:+    // the maximum number of keys stored per node+    static constexpr std::size_t max_keys_per_node = node::maxKeys;++    // -- ctors / dtors --++    // the default constructor creating an empty tree+    btree_delete(Comparator comp = Comparator(), WeakComparator weak_comp = WeakComparator())+            : comp(std::move(comp)), weak_comp(std::move(weak_comp)), root(nullptr), leftmost(nullptr) {}++    // a constructor creating a tree from the given iterator range+    template <typename Iter>+    btree_delete(const Iter& a, const Iter& b) : root(nullptr), leftmost(nullptr) {+        insert(a, b);+    }++    // a move constructor+    btree_delete(btree_delete&& other)+            : comp(other.comp), weak_comp(other.weak_comp), root(other.root), leftmost(other.leftmost) {+        other.root = nullptr;+        other.leftmost = nullptr;+    }++    // a copy constructor+    btree_delete(const btree_delete& set)+            : comp(set.comp), weak_comp(set.weak_comp), root(nullptr), leftmost(nullptr) {+        // use assignment operator for a deep copy+        *this = set;+    }++protected:+    /**+     * An internal constructor enabling the specific creation of a tree+     * based on internal parameters.+     */+    btree_delete(size_type /* size */, node* root, leaf_node* leftmost) : root(root), leftmost(leftmost) {}++public:+    // the destructor freeing all contained nodes+    ~btree_delete() {+        clear();+    }++    // -- mutators and observers --++    // emptiness check+    bool empty() const {+        return root == nullptr;+    }++    // determines the number of elements in this tree+    size_type size() const {+        return (root) ? root->countEntries() : 0;+    }++    /**+     * Inserts the given key into this tree.+     */+    bool insert(const Key& k) {+        operation_hints hints;+        return insert(k, hints);+    }++    /**+     * Inserts the given key into this tree.+     */+    bool insert(const Key& k, operation_hints& hints) {+#ifdef IS_PARALLEL++        // special handling for inserting first element+        while (root == nullptr) {+            // try obtaining root-lock+            if (!root_lock.try_start_write()) {+                // somebody else was faster => re-check+                continue;+            }++            // check loop condition again+            if (root != nullptr) {+                // somebody else was faster => normal insert+                root_lock.end_write();+                break;+            }++            // create new node+            leftmost = new leaf_node();+            leftmost->numElements = 1;+            leftmost->keys[0] = k;+            root = leftmost;++            // operation complete => we can release the root lock+            root_lock.end_write();++            hints.last_insert.access(leftmost);++            return true;+        }++        // insert using iterative implementation++        node* cur = nullptr;++        // test last insert hints+        lock_type::Lease cur_lease;++        auto checkHint = [&](node* last_insert) {+            // ignore null pointer+            if (!last_insert) return false;+            // get a read lease on indicated node+            auto hint_lease = last_insert->lock.start_read();+            // check whether it covers the key+            if (!weak_covers(last_insert, k)) return false;+            // and if there was no concurrent modification+            if (!last_insert->lock.validate(hint_lease)) return false;+            // use hinted location+            cur = last_insert;+            // and keep lease+            cur_lease = hint_lease;+            // we found a hit+            return true;+        };++        if (hints.last_insert.any(checkHint)) {+            // register this as a hit+            hint_stats.inserts.addHit();+        } else {+            // register this as a miss+            hint_stats.inserts.addMiss();+        }++        // if there is no valid hint ..+        if (!cur) {+            do {+                // get root - access lock+                auto root_lease = root_lock.start_read();++                // start with root+                cur = root;++                // get lease of the next node to be accessed+                cur_lease = cur->lock.start_read();++                // check validity of root pointer+                if (root_lock.end_read(root_lease)) {+                    break;+                }++            } while (true);+        }++        while (true) {+            // handle inner nodes+            if (cur->inner) {+                auto a = &(cur->keys[0]);+                auto b = &(cur->keys[cur->numElements]);++                auto pos = search.lower_bound(k, a, b, weak_comp);+                auto idx = pos - a;++                // early exit for sets+                if (isSet && pos != b && weak_equal(*pos, k)) {+                    // validate results+                    if (!cur->lock.validate(cur_lease)) {+                        // start over again+                        return insert(k, hints);+                    }++                    // update provenance information+                    if (typeid(Comparator) != typeid(WeakComparator) && less(k, *pos)) {+                        if (!cur->lock.try_upgrade_to_write(cur_lease)) {+                            // start again+                            return insert(k, hints);+                        }+                        update(*pos, k);+                        cur->lock.end_write();+                        return true;+                    }++                    // we found the element => no check of lock necessary+                    return false;+                }++                // get next pointer+                auto next = cur->getChild(idx);++                // get lease on next level+                auto next_lease = next->lock.start_read();++                // check whether there was a write+                if (!cur->lock.end_read(cur_lease)) {+                    // start over+                    return insert(k, hints);+                }++                // go to next+                cur = next;++                // move on lease+                cur_lease = next_lease;++                continue;+            }++            // the rest is for leaf nodes+            assert(!cur->inner);++            // -- insert node in leaf node --++            auto a = &(cur->keys[0]);+            auto b = &(cur->keys[cur->numElements]);++            auto pos = search.upper_bound(k, a, b, weak_comp);+            auto idx = pos - a;++            // early exit for sets+            if (isSet && pos != a && weak_equal(*(pos - 1), k)) {+                // validate result+                if (!cur->lock.validate(cur_lease)) {+                    // start over again+                    return insert(k, hints);+                }++                // update provenance information+                if (typeid(Comparator) != typeid(WeakComparator) && less(k, *(pos - 1))) {+                    if (!cur->lock.try_upgrade_to_write(cur_lease)) {+                        // start again+                        return insert(k, hints);+                    }+                    update(*(pos - 1), k);+                    cur->lock.end_write();+                    return true;+                }++                // we found the element => done+                return false;+            }++            // upgrade to write-permission+            if (!cur->lock.try_upgrade_to_write(cur_lease)) {+                // something has changed => restart+                hints.last_insert.access(cur);+                return insert(k, hints);+            }++            if (cur->numElements >= node::maxKeys) {+                // -- lock parents --+                auto priv = cur;+                auto parent = priv->parent;+                std::vector<node*> parents;+                do {+                    if (parent) {+                        parent->lock.start_write();+                        while (true) {+                            // check whether parent is correct+                            if (parent == priv->parent) {+                                break;+                            }+                            // switch parent+                            parent->lock.abort_write();+                            parent = priv->parent;+                            parent->lock.start_write();+                        }+                    } else {+                        // lock root lock => since cur is root+                        root_lock.start_write();+                    }++                    // record locked node+                    parents.push_back(parent);++                    // stop at "sphere of influence"+                    if (!parent || !parent->isFull()) {+                        break;+                    }++                    // go one step higher+                    priv = parent;+                    parent = parent->parent;++                } while (true);++                // split this node+                auto old_root = root;+                idx -= cur->rebalance_or_split(+                        const_cast<node**>(&root), root_lock, static_cast<int>(idx), parents);++                // release parent lock+                for (auto it = parents.rbegin(); it != parents.rend(); ++it) {+                    auto parent = *it;++                    // release this lock+                    if (parent) {+                        parent->lock.end_write();+                    } else {+                        if (old_root != root) {+                            root_lock.end_write();+                        } else {+                            root_lock.abort_write();+                        }+                    }+                }++                // insert element in right fragment+                if (((size_type)idx) > cur->numElements) {+                    // release current lock+                    cur->lock.end_write();++                    // insert in sibling+                    return insert(k, hints);+                }+            }++            // ok - no split necessary+            assert(cur->numElements < node::maxKeys && "Split required!");++            // move keys+            for (int j = static_cast<int>(cur->numElements); j > static_cast<int>(idx); --j) {+                cur->keys[j] = cur->keys[j - 1];+            }++            // insert new element+            cur->keys[idx] = k;+            cur->numElements++;++            // release lock on current node+            cur->lock.end_write();++            // remember last insertion position+            hints.last_insert.access(cur);+            return true;+        }++#else+        // special handling for inserting first element+        if (empty()) {+            // create new node+            leftmost = new leaf_node();+            leftmost->numElements = 1;+            leftmost->keys[0] = k;+            root = leftmost;++            hints.last_insert.access(leftmost);++            return true;+        }++        // insert using iterative implementation+        node* cur = root;++        auto checkHints = [&](node* last_insert) {+            if (!last_insert) return false;+            if (!weak_covers(last_insert, k)) return false;+            cur = last_insert;+            return true;+        };++        // test last insert+        if (hints.last_insert.any(checkHints)) {+            hint_stats.inserts.addHit();+        } else {+            hint_stats.inserts.addMiss();+        }++        while (true) {+            // handle inner nodes+            if (cur->inner) {+                auto a = &(cur->keys[0]);+                auto b = &(cur->keys[cur->numElements]);++                auto pos = search.lower_bound(k, a, b, weak_comp);+                auto idx = pos - a;++                // early exit for sets+                if (isSet && pos != b && weak_equal(*pos, k)) {+                    // update provenance information+                    if (typeid(Comparator) != typeid(WeakComparator) && less(k, *pos)) {+                        update(*pos, k);+                        return true;+                    }++                    return false;+                }++                cur = cur->getChild(idx);+                continue;+            }++            // the rest is for leaf nodes+            assert(!cur->inner);++            // -- insert node in leaf node --++            auto a = &(cur->keys[0]);+            auto b = &(cur->keys[cur->numElements]);++            auto pos = search.upper_bound(k, a, b, weak_comp);+            auto idx = pos - a;++            // early exit for sets+            if (isSet && pos != a && weak_equal(*(pos - 1), k)) {+                // update provenance information+                if (typeid(Comparator) != typeid(WeakComparator) && less(k, *(pos - 1))) {+                    update(*(pos - 1), k);+                    return true;+                }++                return false;+            }++            if (cur->numElements >= node::maxKeys) {+                // split this node+                idx -= cur->rebalance_or_split(&root, root_lock, static_cast<int>(idx));++                // insert element in right fragment+                if (((size_type)idx) > cur->numElements) {+                    idx -= cur->numElements + 1;+                    cur = cur->parent->getChild(cur->position + 1);+                }+            }++            // ok - no split necessary+            assert(cur->numElements < node::maxKeys && "Split required!");++            // move keys+            for (int j = static_cast<int>(cur->numElements); j > idx; --j) {+                cur->keys[j] = cur->keys[j - 1];+            }++            // insert new element+            cur->keys[idx] = k;+            cur->numElements++;++            // remember last insertion position+            hints.last_insert.access(cur);++            return true;+        }+#endif+    }++    /**+     * Inserts the given range of elements into this tree.+     */+    template <typename Iter>+    void insert(const Iter& a, const Iter& b) {+        // TODO: improve this beyond a naive insert+        operation_hints hints;+        // a naive insert so far .. seems to work fine+        for (auto it = a; it != b; ++it) {+            // use insert with hint+            insert(*it, hints);+        }+    }++    /**+     * Compute the number of instances of a key in the tree+     */+    size_type get_count(const Key& k) const {+        if (empty()) {+            return 0;+        }+        if (isSet) {+            auto iter = internal_find(k);+            if (iter != end()) {+                return 1;+            } else {+                return 0;+            }+        } else {+            auto lower_iter = internal_lower_bound(k);+            if (lower_iter != end() && equal(*lower_iter, k)) {+                return std::distance(lower_iter, internal_upper_bound(k));+            } else {+                return 0;+            }+        }+    }++    /**+     * Erase the given key from the tree.+     * Return the number of erased keys.+     */+    size_type erase(const Key& k) {+        if (empty()) {+            return 0;+        }+        if (isSet) {+            iterator iter = internal_find(k);+            if (iter == end()) {+                // Key not found+                return 0;+            } else {+                erase(iter);+                return 1;+            }+        } else {+            iterator lower_iter = internal_lower_bound(k);+            if (lower_iter != end() && equal(*lower_iter, k)) {+                size_type count = std::distance(lower_iter, internal_upper_bound(k));+                for (size_type i = 0; i < count; i++) {+                    erase(lower_iter);+                }+                return count;+            } else {+                return 0;+            }+        }+    }++    /**+     * Erase the key pointed to by the iterator.+     * Advance the iterator to the next position.+     */+    void erase(iterator& iter) {+        bool internal_delete = false;+        // @julienhenry+        // iter.cur->lock.start_write();+        if (iter.cur->isInner()) {+            // In an inner node so swap key with previous key+            iterator temp_iter(iter);+            --iter;+            Key temp_key = temp_iter.cur->keys[temp_iter.pos];+            temp_iter.cur->keys[temp_iter.pos] = iter.cur->keys[iter.pos];+            iter.cur->keys[iter.pos] = temp_key;+            internal_delete = true;+        }+        // Now on a leaf node+        assert(iter.cur->isLeaf());++        // Delete the key, move other keys backwards and update size+        iter.cur->keys[iter.pos].~Key();+        for (size_type i = iter.pos + 1; i < iter.cur->getNumElements(); ++i) {+            iter.cur->keys[i - 1] = iter.cur->keys[i];+        }+        iter.cur->numElements--;++        // Next, ensure nodes have not become too small+        iterator res(iter);+        while (true) {+            auto parent = iter.cur->parent;+            if (!parent) {+                // cur is root+                if (iter.cur->getNumElements() == 0) {+                    // Root has become empty+                    if (iter.cur->isLeaf()) {+                        // Whole tree has become empty+                        root = nullptr;+                        leftmost = nullptr;+                        res.cur = nullptr;+                        res.pos = 0;+                    } else {+                        // Whole tree now contained in child at position 0+                        root = iter.cur->getChild(0);+                        root->parent = nullptr;+                    }+                    delete iter.cur;+                }+                break;+            }+            if (iter.cur->getNumElements() >= node::minKeys) {+                break;+            }+            bool merged = merge_or_rebalance(iter);+            if (iter.cur->isLeaf()) {+                res = iter;+            }+            if (!merged) {+                break;+            }+            iter.cur = iter.cur->getParent();+        }+        iter = res;++        // Finally, check the iterator points to the right position+        if (iter.cur) {+            // Tree hasn't become empty+            // If iterator is at end of node, resolve the position+            if (iter.pos == iter.cur->getNumElements()) {+                iter.resolvePosition();+            }++            // If we deleted internally, increment the iterator+            if (internal_delete) {+                ++iter;+            }+        }+        // iter.cur->lock.end_write(); //@julienhenry+    }++private:+    /**+     * Find the given key in a non-empty tree.+     * If found, return an iterator pointing to the key.+     * Otherwise, return end()+     */+    iterator internal_find(const Key& k) const {+        auto iter = iterator(root, 0);+        while (true) {+            auto a = &(iter.cur->keys[0]);+            auto b = &(iter.cur->keys[iter.cur->numElements]);++            auto pos = search(k, a, b, comp);+            iter.pos = static_cast<field_index_type>(pos - a);++            if (pos < b && equal(*pos, k)) {+                return iter;+            }++            if (!iter.cur->inner) {+                return end();+            }++            // continue search in child node+            iter.cur = iter.cur->getChild(iter.pos);+        }+    }++    /**+     * Find the first key in the tree greater or equal to the given key.+     * If found, return an iterator pointing to the key.+     * Otherwise, return end().+     */+    iterator internal_lower_bound(const Key& k) const {+        iterator iter = iterator(root, 0);+        iterator res;+        while (true) {+            auto a = &(iter.cur->keys[0]);+            auto b = &(iter.cur->keys[iter.cur->numElements]);++            auto pos = search.lower_bound(k, a, b, comp);+            iter.pos = static_cast<field_index_type>(pos - a);++            if (pos < b) {+                res = iter;+                if (isSet && equal(*pos, k)) {+                    // Early exit for sets+                    break;+                }+            }++            if (!iter.cur->inner) {+                break;+            }++            iter.cur = iter.cur->getChild(iter.pos);+        }+        if (!res.cur) {+            res = iter;+        }+        return res;+    }++    /**+     * Find the first key in the tree strictly greater than the given key.+     * If found, return an iterator pointing to the key.+     * Otherwise, return end().+     */+    iterator internal_upper_bound(const Key& k) const {+        iterator iter = iterator(root, 0);+        iterator res;+        while (true) {+            auto a = &(iter.cur->keys[0]);+            auto b = &(iter.cur->keys[iter.cur->numElements]);++            auto pos = search.upper_bound(k, a, b, comp);++            iter.pos = static_cast<field_index_type>(pos - a);++            if (pos < b) {+                res = iter;+            }++            if (!iter.cur->inner) {+                break;+            }++            iter.cur = iter.cur->getChild(iter.pos);+        }+        if (!res.cur) {+            res = iter;+        }+        return res;+    }++    /**+     * Merge or rebalance the current node of the iterator.+     * Update the iterator to point to its new position.+     * Return true if a merge occured, else false.+     */+    bool merge_or_rebalance(iterator& iter) {+        // Only called when the current node is too small+        assert(iter.cur->getNumElements() < node::minKeys);++        auto parent = iter.cur->getParent();+        auto siblings = parent->getChildren();+        auto pos = iter.cur->getPositionInParent();+        if (pos < parent->getNumElements()) {+            // Has right sibling+            auto right = siblings[pos + 1];+            if (iter.cur->getNumElements() + right->getNumElements() + 1 <= node::maxKeys) {+                // Merge with right sibling+                merge_with_right_sibling(iter, right);+                return true;+            } else if (pos > 0) {+                // Has a left sibling+                auto left = siblings[pos - 1];+                if (left->getNumElements() + iter.cur->getNumElements() + 1 <= node::maxKeys) {+                    // Merge into left sibling+                    merge_into_left_sibling(left, iter);+                    return true;+                } else {+                    // Rebalance from left sibling+                    rebalance_from_left_sibling(left, iter);+                    return false;+                }+            } else {+                // Can't merge with right and no left sibling so must rebalance from right+                rebalance_from_right_sibling(iter, right);+                return false;+            }+        } else {+            // No right sibling, so must have a left sibling+            assert(pos > 0);+            auto left = siblings[pos - 1];+            if (left->getNumElements() + iter.cur->getNumElements() + 1 <= node::maxKeys) {+                // Merge into left sibling+                merge_into_left_sibling(left, iter);+                return true;+            } else {+                // Rebalance from left sibling+                rebalance_from_left_sibling(left, iter);+                return false;+            }+        }+    }++    /**+     * Merge an iterator with its right sibling, updating the position of the iterator+     */+    void merge_with_right_sibling(iterator& iter, node* right) {+        merge(iter.cur, right);+    }++    /**+     * Merge an iterator into its left sibling, updating the position of the iterator+     */+    void merge_into_left_sibling(node* left, iterator& iter) {+        auto left_size = left->getNumElements();+        merge(left, iter.cur);+        iter.cur = left;+        iter.pos += static_cast<field_index_type>(left_size) + 1;+    }++    /**+     * Merge nodes, destroying the sibling on the right.+     */+    void merge(node* left, node* right) {+        auto parent = left->getParent();+        // Left must have a parent+        assert(parent);+        auto siblings = parent->getChildren();++        auto pos = left->getPositionInParent();+        // Node isn't the right-most node in its parent+        assert(pos < parent->getNumElements());++        // Update the parent node by:+        // 1. Moving dividing key to left node+        left->keys[left->getNumElements()] = parent->keys[pos];+        // 2. Moving keys and siblings found to the right of the+        // dividing key back one place+        for (size_type i = pos + 1; i < parent->getNumElements(); ++i) {+            parent->keys[i - 1] = parent->keys[i];+            auto sibling = siblings[i + 1];+            sibling->position--;+            siblings[i] = sibling;+        }+        // 3. Decrementing its size+        parent->numElements--;++        // Move keys from right node to left+        for (size_type i = left->getNumElements() + 1, j = 0; j < right->getNumElements(); ++i, ++j) {+            left->keys[i] = right->keys[j];+        }++        // If left is an inner node, move children from right to left+        if (left->isInner()) {+            // Right must also be an inner node+            assert(right->isInner());+            auto left_children = left->getChildren();+            auto right_children = right->getChildren();+            for (size_type i = left->getNumElements() + 1, j = 0; j <= right->getNumElements(); ++i, ++j) {+                auto child = right_children[j];+                child->parent = left;+                child->position = static_cast<field_index_type>(i);+                left_children[i] = child;+            }+        }++        // Update the number of elements in the left+        left->numElements += right->getNumElements() + 1;++        // Delete the right node+        delete right;+    }++    /**+     * Rebalance the given iterator node by moving keys from the right.+     * Update the iterator position.+     */+    void rebalance_from_right_sibling(iterator& iter, node* right) {+        auto left = iter.cur;+        auto parent = left->getParent();+        // Left must have a parent+        assert(parent);++        auto pos = left->getPositionInParent();+        // Node isn't the right-most node in its parent+        assert(pos < parent->getNumElements());++        // Number of keys to move+        size_type to_move = (right->getNumElements() - node::minKeys) / 2 + 1;++        // Move down dividing key from parent+        left->keys[left->getNumElements()] = parent->keys[pos];++        // Move keys from right sibling+        for (size_type i = left->getNumElements() + 1, j = 0; j < to_move - 1; ++i, ++j) {+            left->keys[i] = right->keys[j];+        }++        // Move key up to dividing position+        parent->keys[pos] = right->keys[to_move - 1];++        // Move remaining keys in right node back+        for (size_type i = to_move; i < right->getNumElements(); ++i) {+            right->keys[i - to_move] = right->keys[i];+        }++        // If left is an inner node, move children+        if (left->isInner()) {+            // Right must also be an inner node+            assert(right->isInner());+            auto left_children = left->getChildren();+            auto right_children = right->getChildren();++            // Move children from right node to left+            for (size_type i = left->getNumElements() + 1, j = 0; j < to_move; ++i, ++j) {+                auto child = right_children[j];+                child->parent = left;+                child->position = static_cast<field_index_type>(i);+                left_children[i] = child;+            }++            // Move right children back+            for (size_type i = to_move; i <= right->getNumElements(); ++i) {+                auto child = right_children[i];+                child->position = static_cast<field_index_type>(i - to_move);+                right_children[i - to_move] = child;+            }+        }++        // Update sizes+        left->numElements += to_move;+        right->numElements -= to_move;+    }++    /**+     * Rebalance the given iterator node by moving keys from the left sibling.+     * Update the iterator to its new position.+     */+    void rebalance_from_left_sibling(node* left, iterator& iter) {+        auto right = iter.cur;+        auto parent = right->getParent();+        // Left must have a parent+        assert(parent);++        auto pos = right->getPositionInParent();+        // Node isn't the left-most node in its parent+        assert(pos > 0);++        // Number of keys to move+        size_type to_move = (left->getNumElements() - node::minKeys) / 2 + 1;++        // Move keys in right node along+        for (size_type i = right->getNumElements() + to_move - 1; i >= to_move; --i) {+            right->keys[i] = right->keys[i - to_move];+        }++        // Move down dividing key from parent+        right->keys[to_move - 1] = parent->keys[pos - 1];++        // Move keys from left sibling+        for (size_type i = left->getNumElements() - to_move + 1, j = 0; j < to_move - 1; ++i, ++j) {+            right->keys[j] = left->keys[i];+        }++        // Move key up to dividing position+        parent->keys[pos - 1] = left->keys[left->getNumElements() - to_move];++        // If right is an inner node, move children+        if (right->isInner()) {+            // Left must also be an inner node+            assert(left->isInner());+            auto left_children = left->getChildren();+            auto right_children = right->getChildren();++            // Move right children along+            for (size_type i = right->getNumElements() + to_move; i >= to_move; --i) {+                auto child = right_children[i - to_move];+                child->position = static_cast<field_index_type>(i);+                right_children[i] = child;+            }++            // Move children from left node to right+            for (size_type i = left->getNumElements() - to_move + 1, j = 0; j < to_move; ++i, ++j) {+                auto child = left_children[i];+                child->parent = right;+                child->position = static_cast<field_index_type>(j);+                right_children[j] = child;+            }+        }++        // Update iterator position+        iter.pos += static_cast<field_index_type>(to_move);++        // Update sizes+        left->numElements -= to_move;+        right->numElements += to_move;+    }++public:+    /**+     * Return the rightmost node in the tree.+     * Currently inefficient as tree contains no reference to rightmost.+     */+    node* rightmost() const {+        auto rightmost = root;+        if (rightmost) {+            while (rightmost->isInner()) {+                rightmost = rightmost->getChild(rightmost->getNumElements());+            }+        }+        return rightmost;+    }++    // Obtains an iterator referencing the first element of the tree.+    iterator begin() const {+        return iterator(leftmost, 0);+    }++    // Obtains an iterator referencing the position after the last element of the tree.+    iterator end() const {+        node* rightmost = this->rightmost();+        if (rightmost) {+            return iterator(rightmost, static_cast<field_index_type>(rightmost->getNumElements()));+        } else {+            return iterator();+        }+    }++    /**+     * Partitions the full range of this set into up to a given number of chunks.+     * The chunks will cover approximately the same number of elements. Also, the+     * number of chunks will only approximate the desired number of chunks.+     *+     * @param num .. the number of chunks requested+     * @return a list of chunks partitioning this tree+     */+    std::vector<chunk> partition(size_type num) const {+        return getChunks(num);+    }++    std::vector<chunk> getChunks(size_type num) const {+        std::vector<chunk> res;+        if (empty()) {+            return res;+        }+        return root->collectChunks(res, num, begin(), end());+    }++    /**+     * Determines whether the given element is a member of this tree.+     */+    bool contains(const Key& k) const {+        operation_hints hints;+        return contains(k, hints);+    }++    /**+     * Determines whether the given element is a member of this tree.+     */+    bool contains(const Key& k, operation_hints& hints) const {+        return find(k, hints) != end();+    }++    /**+     * Locates the given key within this tree and returns an iterator+     * referencing its position. If not found, an end-iterator will be returned.+     */+    iterator find(const Key& k) const {+        operation_hints hints;+        return find(k, hints);+    }++    /**+     * Locates the given key within this tree and returns an iterator+     * referencing its position. If not found, an end-iterator will be returned.+     */+    iterator find(const Key& k, operation_hints& hints) const {+        if (empty()) {+            return end();+        }++        node* cur = root;++        auto checkHints = [&](node* last_find_end) {+            if (!last_find_end) return false;+            if (!covers(last_find_end, k)) return false;+            cur = last_find_end;+            return true;+        };++        // test last location searched (temporal locality)+        if (hints.last_find_end.any(checkHints)) {+            // register it as a hit+            hint_stats.contains.addHit();+        } else {+            // register it as a miss+            hint_stats.contains.addMiss();+        }++        // an iterative implementation (since 2/7 faster than recursive)++        while (true) {+            auto a = &(cur->keys[0]);+            auto b = &(cur->keys[cur->numElements]);++            auto pos = search(k, a, b, comp);++            if (pos < b && equal(*pos, k)) {+                hints.last_find_end.access(cur);+                return iterator(cur, static_cast<field_index_type>(pos - a));+            }++            if (!cur->inner) {+                hints.last_find_end.access(cur);+                return end();+            }++            // continue search in child node+            cur = cur->getChild(pos - a);+        }+    }++    /**+     * Obtains a lower boundary for the given key -- hence an iterator referencing+     * the smallest value that is not less the given key. If there is no such element,+     * an end-iterator will be returned.+     */+    iterator lower_bound(const Key& k) const {+        operation_hints hints;+        return lower_bound(k, hints);+    }++    /**+     * Obtains a lower boundary for the given key -- hence an iterator referencing+     * the smallest value that is not less the given key. If there is no such element,+     * an end-iterator will be returned.+     */+    iterator lower_bound(const Key& k, operation_hints& hints) const {+        if (empty()) {+            return end();+        }++        node* cur = root;++        auto checkHints = [&](node* last_lower_bound_end) {+            if (!last_lower_bound_end) return false;+            if (!covers(last_lower_bound_end, k)) return false;+            cur = last_lower_bound_end;+            return true;+        };++        // test last searched node+        if (hints.last_lower_bound_end.any(checkHints)) {+            hint_stats.lower_bound.addHit();+        } else {+            hint_stats.lower_bound.addMiss();+        }++        iterator res = end();+        while (true) {+            auto a = &(cur->keys[0]);+            auto b = &(cur->keys[cur->numElements]);++            auto pos = search.lower_bound(k, a, b, comp);+            auto idx = static_cast<field_index_type>(pos - a);++            if (!cur->inner) {+                hints.last_lower_bound_end.access(cur);+                return (pos != b) ? iterator(cur, idx) : res;+            }++            if (isSet && pos != b && equal(*pos, k)) {+                return iterator(cur, idx);+            }++            if (pos != b) {+                res = iterator(cur, idx);+            }++            cur = cur->getChild(idx);+        }+    }++    /**+     * Obtains an upper boundary for the given key -- hence an iterator referencing+     * the first element that the given key is less than the referenced value. If+     * there is no such element, an end-iterator will be returned.+     */+    iterator upper_bound(const Key& k) const {+        operation_hints hints;+        return upper_bound(k, hints);+    }++    /**+     * Obtains an upper boundary for the given key -- hence an iterator referencing+     * the first element that the given key is less than the referenced value. If+     * there is no such element, an end-iterator will be returned.+     */+    iterator upper_bound(const Key& k, operation_hints& hints) const {+        if (empty()) {+            return end();+        }++        node* cur = root;++        auto checkHints = [&](node* last_upper_bound_end) {+            if (!last_upper_bound_end) return false;+            if (!coversUpperBound(last_upper_bound_end, k)) return false;+            cur = last_upper_bound_end;+            return true;+        };++        // test last search node+        if (hints.last_upper_bound_end.any(checkHints)) {+            hint_stats.upper_bound.addHit();+        } else {+            hint_stats.upper_bound.addMiss();+        }++        iterator res = end();+        while (true) {+            auto a = &(cur->keys[0]);+            auto b = &(cur->keys[cur->numElements]);++            auto pos = search.upper_bound(k, a, b, comp);+            auto idx = static_cast<field_index_type>(pos - a);++            if (!cur->inner) {+                hints.last_upper_bound_end.access(cur);+                return (pos != b) ? iterator(cur, idx) : res;+            }++            if (pos != b) {+                res = iterator(cur, idx);+            }++            cur = cur->getChild(idx);+        }+    }++    /**+     * Clears this tree.+     */+    void clear() {+        if (root != nullptr) {+            if (root->isLeaf()) {+                delete static_cast<leaf_node*>(root);+            } else {+                delete static_cast<inner_node*>(root);+            }+        }+        root = nullptr;+        leftmost = nullptr;+    }++    /**+     * Swaps the content of this tree with the given tree. This+     * is a much more efficient operation than creating a copy and+     * realizing the swap utilizing assignment operations.+     */+    void swap(btree_delete& other) {+        // swap the content+        std::swap(root, other.root);+        std::swap(leftmost, other.leftmost);+    }++    // Implementation of the assignment operation for trees.+    btree_delete& operator=(const btree_delete& other) {+        // check identity+        if (this == &other) {+            return *this;+        }++        // create a deep-copy of the content of the other tree+        // shortcut for empty sets+        if (other.empty()) {+            return *this;+        }++        // clone content (deep copy)+        root = other.root->clone();++        // update leftmost reference+        auto tmp = root;+        while (!tmp->isLeaf()) {+            tmp = tmp->getChild(0);+        }+        leftmost = static_cast<leaf_node*>(tmp);++        // done+        return *this;+    }++    // Implementation of an equality operation for trees.+    bool operator==(const btree_delete& other) const {+        // check identity+        if (this == &other) {+            return true;+        }++        // check size+        if (size() != other.size()) {+            return false;+        }+        if (size() < other.size()) {+            return other == *this;+        }++        // check content+        for (const auto& key : other) {+            if (!contains(key)) {+                return false;+            }+        }+        return true;+    }++    // Implementation of an inequality operation for trees.+    bool operator!=(const btree_delete& other) const {+        return !(*this == other);+    }++    // -- for debugging --++    // Determines the number of levels contained in this tree.+    size_type getDepth() const {+        return (empty()) ? 0 : root->getDepth();+    }++    // Determines the number of nodes contained in this tree.+    size_type getNumNodes() const {+        return (empty()) ? 0 : root->countNodes();+    }++    // Determines the amount of memory used by this data structure+    size_type getMemoryUsage() const {+        return sizeof(*this) + (empty() ? 0 : root->getMemoryUsage());+    }++    /*+     * Prints a textual representation of this tree to the given+     * output stream (mostly for debugging and tuning).+     */+    void printTree(std::ostream& out = std::cout) const {+        out << "B-Tree with " << size() << " elements:\n";+        if (empty()) {+            out << " - empty - \n";+        } else {+            root->printTree(out, "");+        }+    }++    /**+     * Prints a textual summary of statistical properties of this+     * tree to the given output stream (for debugging and tuning).+     */+    void printStats(std::ostream& out = std::cout) const {+        auto nodes = getNumNodes();+        out << " ---------------------------------\n";+        out << "  Elements: " << size() << "\n";+        out << "  Depth:    " << (empty() ? 0 : root->getDepth()) << "\n";+        out << "  Nodes:    " << nodes << "\n";+        out << " ---------------------------------\n";+        out << "  Size of inner node: " << sizeof(inner_node) << "\n";+        out << "  Size of leaf node:  " << sizeof(leaf_node) << "\n";+        out << "  Size of Key:        " << sizeof(Key) << "\n";+        out << "  max keys / node:  " << node::maxKeys << "\n";+        out << "  avg keys / node:  " << (size() / (double)nodes) << "\n";+        out << "  avg filling rate: " << ((size() / (double)nodes) / node::maxKeys) << "\n";+        out << " ---------------------------------\n";+        out << "  insert-hint (hits/misses/total): " << hint_stats.inserts.getHits() << "/"+            << hint_stats.inserts.getMisses() << "/" << hint_stats.inserts.getAccesses() << "\n";+        out << "  contains-hint(hits/misses/total):" << hint_stats.contains.getHits() << "/"+            << hint_stats.contains.getMisses() << "/" << hint_stats.contains.getAccesses() << "\n";+        out << "  lower-bound-hint (hits/misses/total):" << hint_stats.lower_bound.getHits() << "/"+            << hint_stats.lower_bound.getMisses() << "/" << hint_stats.lower_bound.getAccesses() << "\n";+        out << "  upper-bound-hint (hits/misses/total):" << hint_stats.upper_bound.getHits() << "/"+            << hint_stats.upper_bound.getMisses() << "/" << hint_stats.upper_bound.getAccesses() << "\n";+        out << " ---------------------------------\n";+    }++    /**+     * Checks the consistency of this tree.+     */+    bool check() {+        auto ok = empty() || root->check(comp, root);+        if (!ok) {+            printTree();+        }+        return ok;+    }++    /**+     * A static member enabling the bulk-load of ordered data into an empty+     * tree. This function is much more efficient in creating a index over+     * an ordered set of elements than an iterative insertion of values.+     *+     * @tparam Iter .. the type of iterator specifying the range+     *                     it must be a random-access iterator+     */+    template <typename R, typename Iter>+    static typename std::enable_if<std::is_same<typename std::iterator_traits<Iter>::iterator_category,+                                           std::random_access_iterator_tag>::value,+            R>::type+    load(const Iter& a, const Iter& b) {+        // quick exit - empty range+        if (a == b) {+            return R();+        }++        // resolve tree recursively+        auto root = buildSubTree(a, b - 1);++        // find leftmost node+        node* leftmost = root;+        while (!leftmost->isLeaf()) {+            leftmost = leftmost->getChild(0);+        }++        // build result+        return R(b - a, root, static_cast<leaf_node*>(leftmost));+    }++protected:+    /**+     * Determines whether the range covered by the given node is also+     * covering the given key value.+     */+    bool covers(const node* node, const Key& k) const {+        if (isSet) {+            // in sets we can include the ends as covered elements+            return !node->isEmpty() && !less(k, node->keys[0]) && !less(node->keys[node->numElements - 1], k);+        }+        // in multi-sets the ends may not be completely covered+        return !node->isEmpty() && less(node->keys[0], k) && less(k, node->keys[node->numElements - 1]);+    }++    /**+     * Determines whether the range covered by the given node is also+     * covering the given key value.+     */+    bool weak_covers(const node* node, const Key& k) const {+        if (isSet) {+            // in sets we can include the ends as covered elements+            return !node->isEmpty() && !weak_less(k, node->keys[0]) &&+                   !weak_less(node->keys[node->numElements - 1], k);+        }+        // in multi-sets the ends may not be completely covered+        return !node->isEmpty() && weak_less(node->keys[0], k) &&+               weak_less(k, node->keys[node->numElements - 1]);+    }++private:+    /**+     * Determines whether the range covered by this node covers+     * the upper bound of the given key.+     */+    bool coversUpperBound(const node* node, const Key& k) const {+        // ignore edges+        return !node->isEmpty() && !less(k, node->keys[0]) && less(k, node->keys[node->numElements - 1]);+    }++    // Utility function for the load operation above.+    template <typename Iter>+    static node* buildSubTree(const Iter& a, const Iter& b) {+        const int N = node::maxKeys;++        // divide range in N+1 sub-ranges+        int length = (b - a) + 1;++        // terminal case: length is less then maxKeys+        if (length <= N) {+            // create a leaf node+            node* res = new leaf_node();+            res->numElements = length;++            for (int i = 0; i < length; ++i) {+                res->keys[i] = a[i];+            }++            return res;+        }++        // recursive case - compute step size+        int numKeys = N;+        int step = ((length - numKeys) / (numKeys + 1));++        while (numKeys > 1 && (step < N / 2)) {+            numKeys--;+            step = ((length - numKeys) / (numKeys + 1));+        }++        // create inner node+        node* res = new inner_node();+        res->numElements = numKeys;++        Iter c = a;+        for (int i = 0; i < numKeys; i++) {+            // get dividing key+            res->keys[i] = c[step];++            // get sub-tree+            auto child = buildSubTree(c, c + (step - 1));+            child->parent = res;+            child->position = i;+            res->getChildren()[i] = child;++            c = c + (step + 1);+        }++        // and the remaining part+        auto child = buildSubTree(c, b);+        child->parent = res;+        child->position = numKeys;+        res->getChildren()[numKeys] = child;++        // done+        return res;+    }+};  // namespace souffle++// Instantiation of static member search.+template <typename Key, typename Comparator, typename Allocator, unsigned blockSize, typename SearchStrategy,+        bool isSet, typename WeakComparator, typename Updater>+const SearchStrategy btree_delete<Key, Comparator, Allocator, blockSize, SearchStrategy, isSet,+        WeakComparator, Updater>::search;++}  // end namespace detail++/**+ * A b-tree based set implementation.+ *+ * @tparam Key             .. the element type to be stored in this set+ * @tparam Comparator     .. a class defining an order on the stored elements+ * @tparam Allocator     .. utilized for allocating memory for required nodes+ * @tparam blockSize    .. determines the number of bytes/block utilized by leaf nodes+ * @tparam SearchStrategy .. enables switching between linear, binary or any other search strategy+ */+template <typename Key, typename Comparator = detail::comparator<Key>,+        typename Allocator = std::allocator<Key>,  // is ignored so far+        unsigned blockSize = 256,+        typename SearchStrategy = typename souffle::detail::default_strategy<Key>::type,+        typename WeakComparator = Comparator, typename Updater = souffle::detail::updater<Key>>+class btree_delete_set : public souffle::detail::btree_delete<Key, Comparator, Allocator, blockSize,+                                 SearchStrategy, true, WeakComparator, Updater> {+    using super = souffle::detail::btree_delete<Key, Comparator, Allocator, blockSize, SearchStrategy, true,+            WeakComparator, Updater>;++    friend class souffle::detail::btree_delete<Key, Comparator, Allocator, blockSize, SearchStrategy, true,+            WeakComparator, Updater>;++public:+    /**+     * A default constructor creating an empty set.+     */+    btree_delete_set(+            const Comparator& comp = Comparator(), const WeakComparator& weak_comp = WeakComparator())+            : super(comp, weak_comp) {}++    /**+     * A constructor creating a set based on the given range.+     */+    template <typename Iter>+    btree_delete_set(const Iter& a, const Iter& b) {+        this->insert(a, b);+    }++    // A copy constructor.+    btree_delete_set(const btree_delete_set& other) : super(other) {}++    // A move constructor.+    btree_delete_set(btree_delete_set&& other) : super(std::move(other)) {}++private:+    // A constructor required by the bulk-load facility.+    template <typename s, typename n, typename l>+    btree_delete_set(s size, n* root, l* leftmost) : super(size, root, leftmost) {}++public:+    // Support for the assignment operator.+    btree_delete_set& operator=(const btree_delete_set& other) {+        super::operator=(other);+        return *this;+    }++    // Support for the bulk-load operator.+    template <typename Iter>+    static btree_delete_set load(const Iter& a, const Iter& b) {+        return super::template load<btree_delete_set>(a, b);+    }+};++/**+ * A b-tree based multi-set implementation.+ *+ * @tparam Key             .. the element type to be stored in this set+ * @tparam Comparator     .. a class defining an order on the stored elements+ * @tparam Allocator     .. utilized for allocating memory for required nodes+ * @tparam blockSize    .. determines the number of bytes/block utilized by leaf nodes+ * @tparam SearchStrategy .. enables switching between linear, binary or any other search strategy+ */+template <typename Key, typename Comparator = detail::comparator<Key>,+        typename Allocator = std::allocator<Key>,  // is ignored so far+        unsigned blockSize = 256,+        typename SearchStrategy = typename souffle::detail::default_strategy<Key>::type,+        typename WeakComparator = Comparator, typename Updater = souffle::detail::updater<Key>>+class btree_delete_multiset : public souffle::detail::btree_delete<Key, Comparator, Allocator, blockSize,+                                      SearchStrategy, false, WeakComparator, Updater> {+    using super = souffle::detail::btree_delete<Key, Comparator, Allocator, blockSize, SearchStrategy, false,+            WeakComparator, Updater>;++    friend class souffle::detail::btree_delete<Key, Comparator, Allocator, blockSize, SearchStrategy, false,+            WeakComparator, Updater>;++public:+    /**+     * A default constructor creating an empty set.+     */+    btree_delete_multiset(+            const Comparator& comp = Comparator(), const WeakComparator& weak_comp = WeakComparator())+            : super(comp, weak_comp) {}++    /**+     * A constructor creating a set based on the given range.+     */+    template <typename Iter>+    btree_delete_multiset(const Iter& a, const Iter& b) {+        this->insert(a, b);+    }++    // A copy constructor.+    btree_delete_multiset(const btree_delete_multiset& other) : super(other) {}++    // A move constructor.+    btree_delete_multiset(btree_delete_multiset&& other) : super(std::move(other)) {}++private:+    // A constructor required by the bulk-load facility.+    template <typename s, typename n, typename l>+    btree_delete_multiset(s size, n* root, l* leftmost) : super(size, root, leftmost) {}++public:+    // Support for the assignment operator.+    btree_delete_multiset& operator=(const btree_delete_multiset& other) {+        super::operator=(other);+        return *this;+    }++    // Support for the bulk-load operator.+    template <typename Iter>+    static btree_delete_multiset load(const Iter& a, const Iter& b) {+        return super::template load<btree_delete_multiset>(a, b);+    }+};++}  // end of namespace souffle
+ cbits/souffle/datastructure/BTreeUtil.h view
@@ -0,0 +1,222 @@+/*+ * Souffle - A Datalog Compiler+ * Copyright (c) 2013, 2015, Oracle and/or its affiliates. All rights reserved+ * Licensed under the Universal Permissive License v 1.0 as shown at:+ * - https://opensource.org/licenses/UPL+ * - <souffle root>/licenses/SOUFFLE-UPL.txt+ */++/************************************************************************+ *+ * @file BTreeUtil.h+ *+ * Utilities for a generic B-tree data structure+ *+ ***********************************************************************/++#pragma once++namespace souffle {++namespace detail {++// ---------- comparators --------------++/**+ * A generic comparator implementation as it is used by+ * a b-tree based on types that can be less-than and+ * equality comparable.+ */+template <typename T>+struct comparator {+    /**+     * Compares the values of a and b and returns+     * -1 if a<b, 1 if a>b and 0 otherwise+     */+    int operator()(const T& a, const T& b) const {+        return (a > b) - (a < b);+    }+    bool less(const T& a, const T& b) const {+        return a < b;+    }+    bool equal(const T& a, const T& b) const {+        return a == b;+    }+};++// ---------- search strategies --------------++/**+ * A common base class for search strategies in b-trees.+ */+struct search_strategy {};++/**+ * A linear search strategy for looking up keys in b-tree nodes.+ */+struct linear_search : public search_strategy {+    /**+     * Required user-defined default constructor.+     */+    linear_search() = default;++    /**+     * Obtains an iterator referencing an element equivalent to the+     * given key in the given range. If no such element is present,+     * a reference to the first element not less than the given key+     * is returned.+     */+    template <typename Key, typename Iter, typename Comp>+    inline Iter operator()(const Key& k, Iter a, Iter b, Comp& comp) const {+        return lower_bound(k, a, b, comp);+    }++    /**+     * Obtains a reference to the first element in the given range that+     * is not less than the given key.+     */+    template <typename Key, typename Iter, typename Comp>+    inline Iter lower_bound(const Key& k, Iter a, Iter b, Comp& comp) const {+        auto c = a;+        while (c < b) {+            auto r = comp(*c, k);+            if (r >= 0) {+                return c;+            }+            ++c;+        }+        return b;+    }++    /**+     * Obtains a reference to the first element in the given range that+     * such that the given key is less than the referenced element.+     */+    template <typename Key, typename Iter, typename Comp>+    inline Iter upper_bound(const Key& k, Iter a, Iter b, Comp& comp) const {+        auto c = a;+        while (c < b) {+            if (comp(*c, k) > 0) {+                return c;+            }+            ++c;+        }+        return b;+    }+};++/**+ * A binary search strategy for looking up keys in b-tree nodes.+ */+struct binary_search : public search_strategy {+    /**+     * Required user-defined default constructor.+     */+    binary_search() = default;++    /**+     * Obtains an iterator pointing to some element within the given+     * range that is equal to the given key, if available. If multiple+     * elements are equal to the given key, an undefined instance will+     * be obtained (no guaranteed lower or upper boundary).  If no such+     * element is present, a reference to the first element not less than+     * the given key will be returned.+     */+    template <typename Key, typename Iter, typename Comp>+    Iter operator()(const Key& k, Iter a, Iter b, Comp& comp) const {+        Iter c;+        auto count = b - a;+        while (count > 0) {+            auto step = count >> 1;+            c = a + step;+            auto r = comp(*c, k);+            if (r == 0) {+                return c;+            }+            if (r < 0) {+                a = ++c;+                count -= step + 1;+            } else {+                count = step;+            }+        }+        return a;+    }++    /**+     * Obtains a reference to the first element in the given range that+     * is not less than the given key.+     */+    template <typename Key, typename Iter, typename Comp>+    Iter lower_bound(const Key& k, Iter a, Iter b, Comp& comp) const {+        Iter c;+        auto count = b - a;+        while (count > 0) {+            auto step = count >> 1;+            c = a + step;+            if (comp(*c, k) < 0) {+                a = ++c;+                count -= step + 1;+            } else {+                count = step;+            }+        }+        return a;+    }++    /**+     * Obtains a reference to the first element in the given range that+     * such that the given key is less than the referenced element.+     */+    template <typename Key, typename Iter, typename Comp>+    Iter upper_bound(const Key& k, Iter a, Iter b, Comp& comp) const {+        Iter c;+        auto count = b - a;+        while (count > 0) {+            auto step = count >> 1;+            c = a + step;+            if (comp(k, *c) >= 0) {+                a = ++c;+                count -= step + 1;+            } else {+                count = step;+            }+        }+        return a;+    }+};++// ---------- search strategies selection --------------++/**+ * A template-meta class to select search strategies for b-trees+ * depending on the key type.+ */+template <typename S>+struct strategy_selection {+    using type = S;+};++struct linear : public strategy_selection<linear_search> {};+struct binary : public strategy_selection<binary_search> {};++// by default every key utilizes binary search+template <typename Key>+struct default_strategy : public binary {};++template <>+struct default_strategy<int> : public linear {};++template <typename... Ts>+struct default_strategy<std::tuple<Ts...>> : public linear {};++/**+ * The default non-updater+ */+template <typename T>+struct updater {+    void update(T& /* old_t */, const T& /* new_t */) {}+};++}  // end of namespace detail+}  // end of namespace souffle
cbits/souffle/datastructure/Brie.h view
@@ -26,3151 +26,3050 @@  #pragma once -#include "souffle/CompiledTuple.h"-#include "souffle/RamTypes.h"-#include "souffle/utility/CacheUtil.h"-#include "souffle/utility/ContainerUtil.h"-#include "souffle/utility/StreamUtil.h"-#include <algorithm>-#include <atomic>-#include <bitset>-#include <cassert>-#include <cstdint>-#include <cstring>-#include <iostream>-#include <iterator>-#include <limits>-#include <utility>-#include <vector>--#ifdef _WIN32-/**- * When compiling for windows, redefine the gcc builtins which are used to- * their equivalents on the windows platform.- */-#define __sync_synchronize MemoryBarrier-#define __sync_bool_compare_and_swap(ptr, oldval, newval) \-    (InterlockedCompareExchangePointer((void* volatile*)ptr, (void*)newval, (void*)oldval) == (void*)oldval)-#endif  // _WIN32--namespace souffle {--namespace detail {--/**- * A templated functor to obtain default values for- * unspecified elements of sparse array instances.- */-template <typename T>-struct default_factory {-    T operator()() const {-        return T();  // just use the default constructor-    }-};--/**- * A functor representing the identity function.- */-template <typename T>-struct identity {-    T operator()(T v) const {-        return v;-    }-};--/**- * A operation to be utilized by the sparse map when merging- * elements associated to different values.- */-template <typename T>-struct default_merge {-    /**-     * Merges two values a and b when merging spase maps.-     */-    T operator()(T a, T b) const {-        default_factory<T> def;-        // if a is the default => us b, else stick to a-        return (a != def()) ? a : b;-    }-};--}  // end namespace detail--/**- * A sparse array simulates an array associating to every element- * of uint32_t an element of a generic type T. Any non-defined element- * will be default-initialized utilizing the detail::default_factory- * functor.- *- * Internally the array is organized as a balanced tree. The leaf- * level of the tree corresponds to the elements of the represented- * array. Inner nodes utilize individual bits of the indices to reference- * sub-trees. For efficiency reasons, only the minimal sub-tree required- * to cover all non-null / non-default values stored in the array is- * maintained. Furthermore, several levels of nodes are aggreated in a- * B-tree like fashion to inprove cache utilization and reduce the number- * of steps required for lookup and insert operations.- *- * @tparam T the type of the stored elements- * @tparam BITS the number of bits consumed per node-level- *              e.g. if it is set to 3, the resulting tree will be of a degree of- *              2^3=8, and thus 8 child-pointers will be stored in each inner node- *              and as many values will be stored in each leaf node.- * @tparam merge_op the functor to be utilized when merging the content of two- *              instances of this type.- * @tparam copy_op a functor to be applied to each stored value when copying an- *              instance of this array. For instance, this is utilized by the- *              trie implementation to create a clone of each sub-tree instead- *              of preserving the original pointer.- */-template <typename T, unsigned BITS = 6, typename merge_op = detail::default_merge<T>,-        typename copy_op = detail::identity<T>>-class SparseArray {-    using key_type = uint64_t;--    // some internal constants-    static constexpr int BIT_PER_STEP = BITS;-    static constexpr int NUM_CELLS = 1 << BIT_PER_STEP;-    static constexpr key_type INDEX_MASK = NUM_CELLS - 1;--public:-    // the type utilized for indexing contained elements-    using index_type = key_type;--    // the type of value stored in this array-    using value_type = T;--    // the atomic view on stored values-    using atomic_value_type = std::atomic<value_type>;--private:-    struct Node;--    /**-     * The value stored in a single cell of a inner-     * or leaf node.-     */-    union Cell {-        // an atomic view on the pointer referencing a nested level-        std::atomic<Node*> aptr;--        // a pointer to the nested level (unsynchronized operations)-        Node* ptr{nullptr};--        // an atomic view on the value stored in this cell (leaf node)-        atomic_value_type avalue;--        // the value stored in this cell (unsynchronized access, leaf node)-        value_type value;-    };--    /**-     * The node type of the internally maintained tree.-     */-    struct Node {-        // a pointer to the parent node (for efficient iteration)-        const Node* parent;-        // the pointers to the child nodes (inner nodes) or the stored values (leaf nodes)-        Cell cell[NUM_CELLS];-    };--    /**-     * A struct describing all the information required by the container-     * class to manage the wrapped up tree.-     */-    struct RootInfo {-        // the root node of the tree-        Node* root;-        // the number of levels of the tree-        uint32_t levels;-        // the absolute offset of the theoretical first element in the tree-        index_type offset;--        // the first leaf node in the tree-        Node* first;-        // the absolute offset of the first element in the first leaf node-        index_type firstOffset;-    };--    union {-        RootInfo unsynced;         // for sequential operations-        volatile RootInfo synced;  // for synchronized operations-    };--public:-    /**-     * A default constructor creating an empty sparse array.-     */-    SparseArray() : unsynced(RootInfo{nullptr, 0, 0, nullptr, std::numeric_limits<index_type>::max()}) {}--    /**-     * A copy constructor for sparse arrays. It creates a deep-     * copy of the data structure maintained by the handed in-     * array instance.-     */-    SparseArray(const SparseArray& other)-            : unsynced(RootInfo{clone(other.unsynced.root, other.unsynced.levels), other.unsynced.levels,-                      other.unsynced.offset, nullptr, other.unsynced.firstOffset}) {-        if (unsynced.root) {-            unsynced.root->parent = nullptr;-            unsynced.first = findFirst(unsynced.root, unsynced.levels);-        }-    }--    /**-     * A r-value based copy constructor for sparse arrays. It-     * takes over ownership of the structure maintained by the-     * handed in array.-     */-    SparseArray(SparseArray&& other)-            : unsynced(RootInfo{other.unsynced.root, other.unsynced.levels, other.unsynced.offset,-                      other.unsynced.first, other.unsynced.firstOffset}) {-        other.unsynced.root = nullptr;-        other.unsynced.levels = 0;-        other.unsynced.first = nullptr;-    }--    /**-     * A destructor for sparse arrays clearing up the internally-     * maintained data structure.-     */-    ~SparseArray() {-        clean();-    }--    /**-     * An assignment creating a deep copy of the handed in-     * array structure (utilizing the copy functor provided-     * as a template parameter).-     */-    SparseArray& operator=(const SparseArray& other) {-        if (this == &other) return *this;--        // clean this one-        clean();--        // copy content-        unsynced.levels = other.unsynced.levels;-        unsynced.root = clone(other.unsynced.root, unsynced.levels);-        if (unsynced.root) {-            unsynced.root->parent = nullptr;-        }-        unsynced.offset = other.unsynced.offset;-        unsynced.first = (unsynced.root) ? findFirst(unsynced.root, unsynced.levels) : nullptr;-        unsynced.firstOffset = other.unsynced.firstOffset;--        // done-        return *this;-    }--    /**-     * An assignment operation taking over ownership-     * from a r-value reference to a sparse array.-     */-    SparseArray& operator=(SparseArray&& other) {-        // clean this one-        clean();--        // harvest content-        unsynced.root = other.unsynced.root;-        unsynced.levels = other.unsynced.levels;-        unsynced.offset = other.unsynced.offset;-        unsynced.first = other.unsynced.first;-        unsynced.firstOffset = other.unsynced.firstOffset;--        // reset other-        other.unsynced.root = nullptr;-        other.unsynced.levels = 0;-        other.unsynced.first = nullptr;--        // done-        return *this;-    }--    /**-     * Tests whether this sparse array is empty, thus it only-     * contains default-values, or not.-     */-    bool empty() const {-        return unsynced.root == nullptr;-    }--    /**-     * Computes the number of non-empty elements within this-     * sparse array.-     */-    std::size_t size() const {-        // quick one for the empty map-        if (empty()) return 0;--        // count elements -- since maintaining is making inserts more expensive-        std::size_t res = 0;-        for (auto it = begin(); it != end(); ++it) {-            ++res;-        }-        return res;-    }--private:-    /**-     * Computes the memory usage of the given sub-tree.-     */-    static std::size_t getMemoryUsage(const Node* node, int level) {-        // support null-nodes-        if (!node) return 0;--        // add size of current node-        std::size_t res = sizeof(Node);--        // sum up memory usage of child nodes-        if (level > 0) {-            for (int i = 0; i < NUM_CELLS; i++) {-                res += getMemoryUsage(node->cell[i].ptr, level - 1);-            }-        }--        // done-        return res;-    }--public:-    /**-     * Computes the total memory usage of this data structure.-     */-    std::size_t getMemoryUsage() const {-        // the memory of the wrapper class-        std::size_t res = sizeof(*this);--        // add nodes-        if (unsynced.root) {-            res += getMemoryUsage(unsynced.root, unsynced.levels);-        }--        // done-        return res;-    }--    /**-     * Resets the content of this array to default values for each contained-     * element.-     */-    void clear() {-        clean();-        unsynced.root = nullptr;-        unsynced.levels = 0;-        unsynced.first = nullptr;-        unsynced.firstOffset = std::numeric_limits<index_type>::max();-    }--    /**-     * A struct to be utilized as a local, temporal context by client code-     * to speed up the execution of various operations (optional parameter).-     */-    struct op_context {-        index_type lastIndex{0};-        Node* lastNode{nullptr};-        op_context() = default;-    };--private:-    // ----------------------------------------------------------------------    //              Optimistic Locking of Root-Level Infos-    // -----------------------------------------------------------------------    /**-     * A struct to cover a snapshot of the root node state.-     */-    struct RootInfoSnapshot {-        // the current pointer to a root node-        Node* root;-        // the current number of levels-        uint32_t levels;-        // the current offset of the first theoretical element-        index_type offset;-        // a version number for the optimistic locking-        uintptr_t version;-    };--    /**-     * Obtains the current version of the root.-     */-    uint64_t getRootVersion() const {-        // here it is assumed that the load of a 64-bit word is atomic-        return (uint64_t)synced.root;-    }--    /**-     * Obtains a snapshot of the current root information.-     */-    RootInfoSnapshot getRootInfo() const {-        RootInfoSnapshot res{};-        do {-            // first take the mod counter-            do {-                // if res.mod % 2 == 1 .. there is an update in progress-                res.version = getRootVersion();-            } while (res.version % 2);--            // then the rest-            res.root = synced.root;-            res.levels = synced.levels;-            res.offset = synced.offset;--            // check consistency of obtained data (optimistic locking)-        } while (res.version != getRootVersion());--        // got a consistent snapshot-        return res;-    }--    /**-     * Updates the current root information based on the handed in modified-     * snapshot instance if the version number of the snapshot still corresponds-     * to the current version. Otherwise a concurrent update took place and the-     * operation is aborted.-     *-     * @param info the updated information to be assigned to the active root-info data-     * @return true if successfully updated, false if aborted-     */-    bool tryUpdateRootInfo(const RootInfoSnapshot& info) {-        // check mod counter-        uintptr_t version = info.version;--        // update root to invalid pointer (ending with 1)-        if (!__sync_bool_compare_and_swap(&synced.root, (Node*)version, (Node*)(version + 1))) {-            return false;-        }--        // conduct update-        synced.levels = info.levels;-        synced.offset = info.offset;--        // update root (and thus the version to enable future retrievals)-        __sync_synchronize();-        synced.root = info.root;--        // done-        return true;-    }--    /**-     * A struct summarizing the state of the first node reference.-     */-    struct FirstInfoSnapshot {-        // the pointer to the first node-        Node* node;-        // the offset of the first node-        index_type offset;-        // the version number of the first node (for the optimistic locking)-        uintptr_t version;-    };--    /**-     * Obtains the current version number of the first node information.-     */-    uint64_t getFirstVersion() const {-        // here it is assumed that the load of a 64-bit word is atomic-        return (uint64_t)synced.first;-    }--    /**-     * Obtains a snapshot of the current first-node information.-     */-    FirstInfoSnapshot getFirstInfo() const {-        FirstInfoSnapshot res{};-        do {-            // first take the version-            do {-                res.version = getFirstVersion();-            } while (res.version % 2);--            // collect the values-            res.node = synced.first;-            res.offset = synced.firstOffset;--        } while (res.version != getFirstVersion());--        // we got a consistent snapshot-        return res;-    }--    /**-     * Updates the information stored regarding the first node in a-     * concurrent setting utilizing a optimistic locking approach.-     * This is identical to the approach utilized for the root info.-     */-    bool tryUpdateFirstInfo(const FirstInfoSnapshot& info) {-        // check mod counter-        uintptr_t version = info.version;--        // temporary update first pointer to point to uneven value (lock-out)-        if (!__sync_bool_compare_and_swap(&synced.first, (Node*)version, (Node*)(version + 1))) {-            return false;-        }--        // conduct update-        synced.firstOffset = info.offset;--        // update node pointer (and thus the version number)-        __sync_synchronize();-        synced.first = info.node;  // must be last (and atomic)--        // done-        return true;-    }--public:-    /**-     * Obtains a mutable reference to the value addressed by the given index.-     *-     * @param i the index of the element to be addressed-     * @return a mutable reference to the corresponding element-     */-    value_type& get(index_type i) {-        op_context ctxt;-        return get(i, ctxt);-    }--    /**-     * Obtains a mutable reference to the value addressed by the given index.-     *-     * @param i the index of the element to be addressed-     * @param ctxt a operation context to exploit state-less temporal locality-     * @return a mutable reference to the corresponding element-     */-    value_type& get(index_type i, op_context& ctxt) {-        return getLeaf(i, ctxt).value;-    }--    /**-     * Obtains a mutable reference to the atomic value addressed by the given index.-     *-     * @param i the index of the element to be addressed-     * @return a mutable reference to the corresponding element-     */-    atomic_value_type& getAtomic(index_type i) {-        op_context ctxt;-        return getAtomic(i, ctxt);-    }--    /**-     * Obtains a mutable reference to the atomic value addressed by the given index.-     *-     * @param i the index of the element to be addressed-     * @param ctxt a operation context to exploit state-less temporal locality-     * @return a mutable reference to the corresponding element-     */-    atomic_value_type& getAtomic(index_type i, op_context& ctxt) {-        return getLeaf(i, ctxt).avalue;-    }--private:-    /**-     * An internal function capable of navigating to a given leaf node entry.-     * If the cell does not exist yet it will be created as a side-effect.-     *-     * @param i the index of the requested cell-     * @param ctxt a operation context to exploit state-less temporal locality-     * @return a reference to the requested cell-     */-    inline Cell& getLeaf(index_type i, op_context& ctxt) {-        // check context-        if (ctxt.lastNode && (ctxt.lastIndex == (i & ~INDEX_MASK))) {-            // return reference to referenced-            return ctxt.lastNode->cell[i & INDEX_MASK];-        }--        // get snapshot of root-        auto info = getRootInfo();--        // check for emptiness-        if (info.root == nullptr) {-            // build new root node-            info.root = newNode();--            // initialize the new node-            info.root->parent = nullptr;-            info.offset = i & ~(INDEX_MASK);--            // try updating root information atomically-            if (tryUpdateRootInfo(info)) {-                // success -- finish get call--                // update first-                auto firstInfo = getFirstInfo();-                while (info.offset < firstInfo.offset) {-                    firstInfo.node = info.root;-                    firstInfo.offset = info.offset;-                    if (!tryUpdateFirstInfo(firstInfo)) {-                        // there was some concurrent update => check again-                        firstInfo = getFirstInfo();-                    }-                }--                // return reference to proper cell-                return info.root->cell[i & INDEX_MASK];-            }--            // somebody else was faster => use standard insertion procedure-            delete info.root;--            // retrieve new root info-            info = getRootInfo();--            // make sure there is a root-            assert(info.root);-        }--        // for all other inserts-        //   - check boundary-        //   - navigate to node-        //   - insert value--        // check boundaries-        while (!inBoundaries(i, info.levels, info.offset)) {-            // boundaries need to be expanded by growing upwards-            raiseLevel(info);  // try raising level unless someone else did already-            // update root info-            info = getRootInfo();-        }--        // navigate to node-        Node* node = info.root;-        unsigned level = info.levels;-        while (level != 0) {-            // get X coordinate-            auto x = getIndex(static_cast<RamDomain>(i), level);--            // decrease level counter-            --level;--            // check next node-            std::atomic<Node*>& aNext = node->cell[x].aptr;-            Node* next = aNext;-            if (!next) {-                // create new sub-tree-                Node* newNext = newNode();-                newNext->parent = node;--                // try to update next-                if (!aNext.compare_exchange_strong(next, newNext)) {-                    // some other thread was faster => use updated next-                    delete newNext;-                } else {-                    // the locally created next is the new next-                    next = newNext;--                    // update first-                    if (level == 0) {-                        // compute offset of this node-                        auto off = i & ~INDEX_MASK;--                        // fast over-approximation of whether a update is necessary-                        if (off < unsynced.firstOffset) {-                            // update first reference if this one is the smallest-                            auto first_info = getFirstInfo();-                            while (off < first_info.offset) {-                                first_info.node = next;-                                first_info.offset = off;-                                if (!tryUpdateFirstInfo(first_info)) {-                                    // there was some concurrent update => check again-                                    first_info = getFirstInfo();-                                }-                            }-                        }-                    }-                }--                // now next should be defined-                assert(next);-            }--            // continue one level below-            node = next;-        }--        // update context-        ctxt.lastIndex = (i & ~INDEX_MASK);-        ctxt.lastNode = node;--        // return reference to cell-        return node->cell[i & INDEX_MASK];-    }--public:-    /**-     * Updates the value stored in cell i by the given value.-     */-    void update(index_type i, const value_type& val) {-        op_context ctxt;-        update(i, val, ctxt);-    }--    /**-     * Updates the value stored in cell i by the given value. A operation-     * context can be provided for exploiting temporal locality.-     */-    void update(index_type i, const value_type& val, op_context& ctxt) {-        get(i, ctxt) = val;-    }--    /**-     * Obtains the value associated to index i -- which might be-     * the default value of the covered type if the value hasn't been-     * defined previously.-     */-    value_type operator[](index_type i) const {-        return lookup(i);-    }--    /**-     * Obtains the value associated to index i -- which might be-     * the default value of the covered type if the value hasn't been-     * defined previously.-     */-    value_type lookup(index_type i) const {-        op_context ctxt;-        return lookup(i, ctxt);-    }--    /**-     * Obtains the value associated to index i -- which might be-     * the default value of the covered type if the value hasn't been-     * defined previously. A operation context can be provided for-     * exploiting temporal locality.-     */-    value_type lookup(index_type i, op_context& ctxt) const {-        // check whether it is empty-        if (!unsynced.root) return souffle::detail::default_factory<value_type>()();--        // check boundaries-        if (!inBoundaries(i)) return souffle::detail::default_factory<value_type>()();--        // check context-        if (ctxt.lastNode && ctxt.lastIndex == (i & ~INDEX_MASK)) {-            return ctxt.lastNode->cell[i & INDEX_MASK].value;-        }--        // navigate to value-        Node* node = unsynced.root;-        unsigned level = unsynced.levels;-        while (level != 0) {-            // get X coordinate-            auto x = getIndex(static_cast<RamDomain>(i), level);--            // decrease level counter-            --level;--            // check next node-            Node* next = node->cell[x].ptr;--            // check next step-            if (!next) return souffle::detail::default_factory<value_type>()();--            // continue one level below-            node = next;-        }--        // remember context-        ctxt.lastIndex = (i & ~INDEX_MASK);-        ctxt.lastNode = node;--        // return reference to cell-        return node->cell[i & INDEX_MASK].value;-    }--private:-    /**-     * A static operation utilized internally for merging sub-trees recursively.-     *-     * @param parent the parent node of the current merge operation-     * @param trg a reference to the pointer the cloned node should be stored to-     * @param src the node to be cloned-     * @param levels the height of the cloned node-     */-    static void merge(const Node* parent, Node*& trg, const Node* src, int levels) {-        // if other side is null => done-        if (src == nullptr) {-            return;-        }--        // if the trg sub-tree is empty, clone the corresponding branch-        if (trg == nullptr) {-            trg = clone(src, levels);-            if (trg != nullptr) {-                trg->parent = parent;-            }-            return;  // done-        }--        // otherwise merge recursively--        // the leaf-node step-        if (levels == 0) {-            merge_op merg;-            for (int i = 0; i < NUM_CELLS; ++i) {-                trg->cell[i].value = merg(trg->cell[i].value, src->cell[i].value);-            }-            return;-        }--        // the recursive step-        for (int i = 0; i < NUM_CELLS; ++i) {-            merge(trg, trg->cell[i].ptr, src->cell[i].ptr, levels - 1);-        }-    }--public:-    /**-     * Adds all the values stored in the given array to this array.-     */-    void addAll(const SparseArray& other) {-        // skip if other is empty-        if (other.empty()) {-            return;-        }--        // special case: emptiness-        if (empty()) {-            // use assignment operator-            *this = other;-            return;-        }--        // adjust levels-        while (unsynced.levels < other.unsynced.levels || !inBoundaries(other.unsynced.offset)) {-            raiseLevel();-        }--        // navigate to root node equivalent of the other node in this tree-        auto level = unsynced.levels;-        Node** node = &unsynced.root;-        while (level > other.unsynced.levels) {-            // get X coordinate-            auto x = getIndex(static_cast<RamDomain>(other.unsynced.offset), level);--            // decrease level counter-            --level;--            // check next node-            Node*& next = (*node)->cell[x].ptr;-            if (!next) {-                // create new sub-tree-                next = newNode();-                next->parent = *node;-            }--            // continue one level below-            node = &next;-        }--        // merge sub-branches from here-        merge((*node)->parent, *node, other.unsynced.root, level);--        // update first-        if (unsynced.firstOffset > other.unsynced.firstOffset) {-            unsynced.first = findFirst(*node, level);-            unsynced.firstOffset = other.unsynced.firstOffset;-        }-    }--    // ----------------------------------------------------------------------    //                           Iterator-    // -----------------------------------------------------------------------    /**-     * The iterator type to be utilized to iterate over the non-default elements of this array.-     */-    class iterator {-        using pair_type = std::pair<index_type, value_type>;--        // a pointer to the leaf node currently processed or null (end)-        const Node* node;--        // the value currently pointed to-        pair_type value;--    public:-        // default constructor -- creating an end-iterator-        iterator() : node(nullptr) {}--        iterator(const Node* node, pair_type value) : node(node), value(std::move(value)) {}--        iterator(const Node* first, index_type firstOffset) : node(first), value(firstOffset, 0) {-            // if the start is the end => we are done-            if (!first) return;--            // load the value-            if (first->cell[0].value == value_type()) {-                ++(*this);  // walk to first element-            } else {-                value.second = first->cell[0].value;-            }-        }--        // a copy constructor-        iterator(const iterator& other) = default;--        // an assignment operator-        iterator& operator=(const iterator& other) = default;--        // the equality operator as required by the iterator concept-        bool operator==(const iterator& other) const {-            // only equivalent if pointing to the end-            return (node == nullptr && other.node == nullptr) ||-                   (node == other.node && value.first == other.value.first);-        }--        // the not-equality operator as required by the iterator concept-        bool operator!=(const iterator& other) const {-            return !(*this == other);-        }--        // the deref operator as required by the iterator concept-        const pair_type& operator*() const {-            return value;-        }--        // support for the pointer operator-        const pair_type* operator->() const {-            return &value;-        }--        // the increment operator as required by the iterator concept-        iterator& operator++() {-            // get current offset-            index_type x = value.first & INDEX_MASK;--            // go to next non-empty value in current node-            do {-                x++;-            } while (x < NUM_CELLS && node->cell[x].value == value_type());--            // check whether one has been found-            if (x < NUM_CELLS) {-                // update value and be done-                value.first = (value.first & ~INDEX_MASK) | x;-                value.second = node->cell[x].value;-                return *this;  // done-            }--            // go to parent-            node = node->parent;-            int level = 1;--            // get current index on this level-            x = getIndex(static_cast<RamDomain>(value.first), level);-            x++;--            while (level > 0 && node) {-                // search for next child-                while (x < NUM_CELLS) {-                    if (node->cell[x].ptr != nullptr) {-                        break;-                    }-                    x++;-                }--                // pick next step-                if (x < NUM_CELLS) {-                    // going down-                    node = node->cell[x].ptr;-                    value.first &= getLevelMask(level + 1);-                    value.first |= x << (BIT_PER_STEP * level);-                    level--;-                    x = 0;-                } else {-                    // going up-                    node = node->parent;-                    level++;--                    // get current index on this level-                    x = getIndex(static_cast<RamDomain>(value.first), level);-                    x++;  // go one step further-                }-            }--            // check whether it is the end of range-            if (node == nullptr) {-                return *this;-            }--            // search the first value in this node-            x = 0;-            while (node->cell[x].value == value_type()) {-                x++;-            }--            // update value-            value.first |= x;-            value.second = node->cell[x].value;--            // done-            return *this;-        }--        // True if this iterator is passed the last element.-        bool isEnd() const {-            return node == nullptr;-        }--        // enables this iterator core to be printed (for debugging)-        void print(std::ostream& out) const {-            out << "SparseArrayIter(" << node << " @ " << value << ")";-        }--        friend std::ostream& operator<<(std::ostream& out, const iterator& iter) {-            iter.print(out);-            return out;-        }-    };--    /**-     * Obtains an iterator referencing the first non-default element or end in-     * case there are no such elements.-     */-    iterator begin() const {-        return iterator(unsynced.first, unsynced.firstOffset);-    }--    /**-     * An iterator referencing the position after the last non-default element.-     */-    iterator end() const {-        return iterator();-    }--    /**-     * An operation to obtain an iterator referencing an element addressed by the-     * given index. If the corresponding element is a non-default value, a corresponding-     * iterator will be returned. Otherwise end() will be returned.-     */-    iterator find(index_type i) const {-        op_context ctxt;-        return find(i, ctxt);-    }--    /**-     * An operation to obtain an iterator referencing an element addressed by the-     * given index. If the corresponding element is a non-default value, a corresponding-     * iterator will be returned. Otherwise end() will be returned. A operation context-     * can be provided for exploiting temporal locality.-     */-    iterator find(index_type i, op_context& ctxt) const {-        // check whether it is empty-        if (!unsynced.root) return end();--        // check boundaries-        if (!inBoundaries(i)) return end();--        // check context-        if (ctxt.lastNode && ctxt.lastIndex == (i & ~INDEX_MASK)) {-            Node* node = ctxt.lastNode;--            // check whether there is a proper entry-            value_type value = node->cell[i & INDEX_MASK].value;-            if (value == value_type{}) {-                return end();-            }-            // return iterator pointing to value-            return iterator(node, std::make_pair(i, value));-        }--        // navigate to value-        Node* node = unsynced.root;-        unsigned level = unsynced.levels;-        while (level != 0) {-            // get X coordinate-            auto x = getIndex(i, level);--            // decrease level counter-            --level;--            // check next node-            Node* next = node->cell[x].ptr;--            // check next step-            if (!next) return end();--            // continue one level below-            node = next;-        }--        // register in context-        ctxt.lastNode = node;-        ctxt.lastIndex = (i & ~INDEX_MASK);--        // check whether there is a proper entry-        value_type value = node->cell[i & INDEX_MASK].value;-        if (value == value_type{}) {-            return end();-        }--        // return iterator pointing to cell-        return iterator(node, std::make_pair(i, value));-    }--    /**-     * An operation obtaining the smallest non-default element such that it's index is >=-     * the given index.-     */-    iterator lowerBound(index_type i) const {-        op_context ctxt;-        return lowerBound(i, ctxt);-    }--    /**-     * An operation obtaining the smallest non-default element such that it's index is >=-     * the given index. A operation context can be provided for exploiting temporal locality.-     */-    iterator lowerBound(index_type i, op_context&) const {-        // check whether it is empty-        if (!unsynced.root) return end();--        // check boundaries-        if (!inBoundaries(i)) {-            // if it is on the lower end, return minimum result-            if (i < unsynced.offset) {-                const auto& value = unsynced.first->cell[0].value;-                auto res = iterator(unsynced.first, std::make_pair(unsynced.offset, value));-                if (value == value_type()) {-                    ++res;-                }-                return res;-            }-            // otherwise it is on the high end, return end iterator-            return end();-        }--        // navigate to value-        Node* node = unsynced.root;-        unsigned level = unsynced.levels;-        while (true) {-            // get X coordinate-            auto x = getIndex(static_cast<RamDomain>(i), level);--            // check next node-            Node* next = node->cell[x].ptr;--            // check next step-            if (!next) {-                if (x == NUM_CELLS - 1) {-                    ++level;-                    node = const_cast<Node*>(node->parent);-                    if (!node) return end();-                }--                // continue search-                i = i & getLevelMask(level);--                // find next higher value-                i += 1ull << (BITS * level);--            } else {-                if (level == 0) {-                    // found boundary-                    return iterator(node, std::make_pair(i, node->cell[x].value));-                }--                // decrease level counter-                --level;--                // continue one level below-                node = next;-            }-        }-    }--    /**-     * An operation obtaining the smallest non-default element such that it's index is greater-     * the given index.-     */-    iterator upperBound(index_type i) const {-        op_context ctxt;-        return upperBound(i, ctxt);-    }--    /**-     * An operation obtaining the smallest non-default element such that it's index is greater-     * the given index. A operation context can be provided for exploiting temporal locality.-     */-    iterator upperBound(index_type i, op_context& ctxt) const {-        if (i == std::numeric_limits<index_type>::max()) {-            return end();-        }-        return lowerBound(i + 1, ctxt);-    }--private:-    /**-     * An internal debug utility printing the internal structure of this sparse array to the given output-     * stream.-     */-    void dump(bool detailed, std::ostream& out, const Node& node, int level, index_type offset,-            int indent = 0) const {-        auto x = getIndex(offset, level + 1);-        out << times("\t", indent) << x << ": Node " << &node << " on level " << level-            << " parent: " << node.parent << " -- range: " << offset << " - "-            << (offset + ~getLevelMask(level + 1)) << "\n";--        if (level == 0) {-            for (int i = 0; i < NUM_CELLS; i++) {-                if (detailed || node.cell[i].value != value_type()) {-                    out << times("\t", indent + 1) << i << ": [" << (offset + i) << "] " << node.cell[i].value-                        << "\n";-                }-            }-        } else {-            for (int i = 0; i < NUM_CELLS; i++) {-                if (node.cell[i].ptr) {-                    dump(detailed, out, *node.cell[i].ptr, level - 1,-                            offset + (i * (index_type(1) << (level * BIT_PER_STEP))), indent + 1);-                } else if (detailed) {-                    auto low = offset + (i * (1 << (level * BIT_PER_STEP)));-                    auto hig = low + ~getLevelMask(level);-                    out << times("\t", indent + 1) << i << ": empty range " << low << " - " << hig << "\n";-                }-            }-        }-        out << "\n";-    }--public:-    /**-     * A debug utility printing the internal structure of this sparse array to the given output stream.-     */-    void dump(bool detail = false, std::ostream& out = std::cout) const {-        if (!unsynced.root) {-            out << " - empty - \n";-            return;-        }-        out << "root:  " << unsynced.root << "\n";-        out << "offset: " << unsynced.offset << "\n";-        out << "first: " << unsynced.first << "\n";-        out << "fist offset: " << unsynced.firstOffset << "\n";-        dump(detail, out, *unsynced.root, unsynced.levels, unsynced.offset);-    }--private:-    // ---------------------------------------------------------------------------    //                                 Utilities-    // ----------------------------------------------------------------------------    /**-     * Creates new nodes and initializes them with 0.-     */-    static Node* newNode() {-        return new Node();-    }--    /**-     * Destroys a node and all its sub-nodes recursively.-     */-    static void freeNodes(Node* node, int level) {-        if (!node) return;-        if (level != 0) {-            for (int i = 0; i < NUM_CELLS; i++) {-                freeNodes(node->cell[i].ptr, level - 1);-            }-        }-        delete node;-    }--    /**-     * Conducts a cleanup of the internal tree structure.-     */-    void clean() {-        freeNodes(unsynced.root, unsynced.levels);-        unsynced.root = nullptr;-        unsynced.levels = 0;-    }--    /**-     * Clones the given node and all its sub-nodes.-     */-    static Node* clone(const Node* node, int level) {-        // support null-pointers-        if (node == nullptr) {-            return nullptr;-        }--        // create a clone-        auto* res = new Node();--        // handle leaf level-        if (level == 0) {-            copy_op copy;-            for (int i = 0; i < NUM_CELLS; i++) {-                res->cell[i].value = copy(node->cell[i].value);-            }-            return res;-        }--        // for inner nodes clone each child-        for (int i = 0; i < NUM_CELLS; i++) {-            auto cur = clone(node->cell[i].ptr, level - 1);-            if (cur != nullptr) {-                cur->parent = res;-            }-            res->cell[i].ptr = cur;-        }--        // done-        return res;-    }--    /**-     * Obtains the left-most leaf-node of the tree rooted by the given node-     * with the given level.-     */-    static Node* findFirst(Node* node, int level) {-        while (level > 0) {-            bool found = false;-            for (int i = 0; i < NUM_CELLS; i++) {-                Node* cur = node->cell[i].ptr;-                if (cur) {-                    node = cur;-                    --level;-                    found = true;-                    break;-                }-            }-            assert(found && "No first node!");-        }--        return node;-    }--    /**-     * Raises the level of this tree by one level. It does so by introducing-     * a new root node and inserting the current root node as a child node.-     */-    void raiseLevel() {-        // something went wrong when we pass that line-        assert(unsynced.levels < (sizeof(index_type) * 8 / BITS) + 1);--        // create new root-        Node* node = newNode();-        node->parent = nullptr;--        // insert existing root as child-        auto x = getIndex(static_cast<RamDomain>(unsynced.offset), unsynced.levels + 1);-        node->cell[x].ptr = unsynced.root;--        // swap the root-        unsynced.root->parent = node;--        // update root-        unsynced.root = node;-        ++unsynced.levels;--        // update offset be removing additional bits-        unsynced.offset &= getLevelMask(unsynced.levels + 1);-    }--    /**-     * Attempts to raise the height of this tree based on the given root node-     * information and updates the root-info snapshot correspondingly.-     */-    void raiseLevel(RootInfoSnapshot& info) {-        // something went wrong when we pass that line-        assert(info.levels < (sizeof(index_type) * 8 / BITS) + 1);--        // create new root-        Node* newRoot = newNode();-        newRoot->parent = nullptr;--        // insert existing root as child-        auto x = getIndex(static_cast<RamDomain>(info.offset), info.levels + 1);-        newRoot->cell[x].ptr = info.root;--        // exchange the root in the info struct-        auto oldRoot = info.root;-        info.root = newRoot;--        // update level counter-        ++info.levels;--        // update offset-        info.offset &= getLevelMask(info.levels + 1);--        // try exchanging root info-        if (tryUpdateRootInfo(info)) {-            // success => final step, update parent of old root-            oldRoot->parent = info.root;-        } else {-            // throw away temporary new node-            delete newRoot;-        }-    }--    /**-     * Tests whether the given index is covered by the boundaries defined-     * by the hight and offset of the internally maintained tree.-     */-    bool inBoundaries(index_type a) const {-        return inBoundaries(a, unsynced.levels, unsynced.offset);-    }--    /**-     * Tests whether the given index is within the boundaries defined by the-     * given tree hight and offset.-     */-    static bool inBoundaries(index_type a, uint32_t levels, index_type offset) {-        auto mask = getLevelMask(levels + 1);-        return (a & mask) == offset;-    }--    /**-     * Obtains the index within the arrays of cells of a given index on a given-     * level of the internally maintained tree.-     */-    static index_type getIndex(RamDomain a, unsigned level) {-        return (a & (INDEX_MASK << (level * BIT_PER_STEP))) >> (level * BIT_PER_STEP);-    }--    /**-     * Computes the bit-mask to be applicable to obtain the offset of a node on a-     * given tree level.-     */-    static index_type getLevelMask(unsigned level) {-        if (level > (sizeof(index_type) * 8 / BITS)) return 0;-        return (~(index_type(0)) << (level * BIT_PER_STEP));-    }-};--/**- * A sparse bit-map is a bit map virtually assigning a bit value to every value if the- * uint32_t domain. However, only 1-bits are stored utilizing a nested sparse array- * structure.- *- * @tparam BITS similar to the BITS parameter of the sparse array type- */-template <unsigned BITS = 4>-class SparseBitMap {-    // the element type stored in the nested sparse array-    using value_t = uint64_t;--    // define the bit-level merge operation-    struct merge_op {-        value_t operator()(value_t a, value_t b) const {-            return a | b;  // merging bit masks => bitwise or operation-        }-    };--    // the type of the internal data store-    using data_store_t = SparseArray<value_t, BITS, merge_op>;-    using atomic_value_t = typename data_store_t::atomic_value_type;--    // some constants for manipulating stored values-    static constexpr short BITS_PER_ENTRY = sizeof(value_t) * 8;-    static constexpr short LEAF_INDEX_WIDTH = static_cast<short>(__builtin_ctz(BITS_PER_ENTRY));-    static constexpr uint64_t LEAF_INDEX_MASK = BITS_PER_ENTRY - 1;--public:-    // the type to address individual entries-    using index_type = typename data_store_t::index_type;--private:-    // it utilizes a sparse map to store its data-    data_store_t store;--public:-    // a simple default constructor-    SparseBitMap() = default;--    // a default copy constructor-    SparseBitMap(const SparseBitMap&) = default;--    // a default r-value copy constructor-    SparseBitMap(SparseBitMap&&) = default;--    // a default assignment operator-    SparseBitMap& operator=(const SparseBitMap&) = default;--    // a default r-value assignment operator-    SparseBitMap& operator=(SparseBitMap&&) = default;--    // checks whether this bit-map is empty -- thus it does not have any 1-entries-    bool empty() const {-        return store.empty();-    }--    // the type utilized for recording context information for exploiting temporal locality-    using op_context = typename data_store_t::op_context;--    /**-     * Sets the bit addressed by i to 1.-     */-    bool set(index_type i) {-        op_context ctxt;-        return set(i, ctxt);-    }--    /**-     * Sets the bit addressed by i to 1. A context for exploiting temporal locality-     * can be provided.-     */-    bool set(index_type i, op_context& ctxt) {-        atomic_value_t& val = store.getAtomic(i >> LEAF_INDEX_WIDTH, ctxt);-        value_t bit = (1ull << (i & LEAF_INDEX_MASK));--#ifdef __GNUC__-#if __GNUC__ >= 7-        // In GCC >= 7 the usage of fetch_or causes a bug that needs further investigation-        // For now, this two-instruction based implementation provides a fix that does-        // not sacrifice too much performance.--        while (true) {-            auto order = std::memory_order::memory_order_relaxed;--            // load current value-            value_t old = val.load(order);--            // if bit is already set => we are done-            if (old & bit) return false;--            // set the bit, if failed, repeat-            if (!val.compare_exchange_strong(old, old | bit, order, order)) continue;--            // it worked, new bit added-            return true;-        }--#endif-#endif--        value_t old = val.fetch_or(bit, std::memory_order::memory_order_relaxed);-        return (old & bit) == 0u;-    }--    /**-     * Determines the whether the bit addressed by i is set or not.-     */-    bool test(index_type i) const {-        op_context ctxt;-        return test(i, ctxt);-    }--    /**-     * Determines the whether the bit addressed by i is set or not. A context for-     * exploiting temporal locality can be provided.-     */-    bool test(index_type i, op_context& ctxt) const {-        value_t bit = (1ull << (i & LEAF_INDEX_MASK));-        return store.lookup(i >> LEAF_INDEX_WIDTH, ctxt) & bit;-    }--    /**-     * Determines the whether the bit addressed by i is set or not.-     */-    bool operator[](index_type i) const {-        return test(i);-    }--    /**-     * Resets all contained bits to 0.-     */-    void clear() {-        store.clear();-    }--    /**-     * Determines the number of bits set.-     */-    std::size_t size() const {-        // this is computed on demand to keep the set operation simple.-        std::size_t res = 0;-        for (const auto& cur : store) {-            res += __builtin_popcountll(cur.second);-        }-        return res;-    }--    /**-     * Computes the total memory usage of this data structure.-     */-    std::size_t getMemoryUsage() const {-        // compute the total memory usage-        return sizeof(*this) - sizeof(data_store_t) + store.getMemoryUsage();-    }--    /**-     * Sets all bits set in other to 1 within this bit map.-     */-    void addAll(const SparseBitMap& other) {-        // nothing to do if it is a self-assignment-        if (this == &other) return;--        // merge the sparse store-        store.addAll(other.store);-    }--    // ----------------------------------------------------------------------    //                           Iterator-    // -----------------------------------------------------------------------    /**-     * An iterator iterating over all indices set to 1.-     */-    class iterator {-        using nested_iterator = typename data_store_t::iterator;--        // the iterator through the underlying sparse data structure-        nested_iterator iter;--        // the currently consumed mask-        uint64_t mask = 0;--        // the value currently pointed to-        index_type value{};--    public:-        typedef std::forward_iterator_tag iterator_category;-        typedef index_type value_type;-        typedef ptrdiff_t difference_type;-        typedef value_type* pointer;-        typedef value_type& reference;--        // default constructor -- creating an end-iterator-        iterator() = default;--        iterator(const nested_iterator& iter)-                : iter(iter), mask(toMask(iter->second)), value(iter->first << LEAF_INDEX_WIDTH) {-            moveToNextInMask();-        }--        iterator(const nested_iterator& iter, uint64_t m, index_type value)-                : iter(iter), mask(m), value(value) {}--        // a copy constructor-        iterator(const iterator& other) = default;--        // an assignment operator-        iterator& operator=(const iterator& other) = default;--        // the equality operator as required by the iterator concept-        bool operator==(const iterator& other) const {-            // only equivalent if pointing to the end-            return iter == other.iter && mask == other.mask;-        }--        // the not-equality operator as required by the iterator concept-        bool operator!=(const iterator& other) const {-            return !(*this == other);-        }--        // the deref operator as required by the iterator concept-        const index_type& operator*() const {-            return value;-        }--        // support for the pointer operator-        const index_type* operator->() const {-            return &value;-        }--        // the increment operator as required by the iterator concept-        iterator& operator++() {-            // progress in current mask-            if (moveToNextInMask()) return *this;--            // go to next entry-            ++iter;--            // update value-            if (!iter.isEnd()) {-                value = iter->first << LEAF_INDEX_WIDTH;-                mask = toMask(iter->second);-                moveToNextInMask();-            }--            // done-            return *this;-        }--        bool isEnd() const {-            return iter.isEnd();-        }--        void print(std::ostream& out) const {-            out << "SparseBitMapIter(" << iter << " -> " << std::bitset<64>(mask) << " @ " << value << ")";-        }--        // enables this iterator core to be printed (for debugging)-        friend std::ostream& operator<<(std::ostream& out, const iterator& iter) {-            iter.print(out);-            return out;-        }--        static uint64_t toMask(const value_t& value) {-            static_assert(sizeof(value_t) == sizeof(uint64_t), "Fixed for 64-bit compiler.");-            return reinterpret_cast<const uint64_t&>(value);-        }--    private:-        bool moveToNextInMask() {-            // check if there is something left-            if (mask == 0) return false;--            // get position of leading 1-            auto pos = __builtin_ctzll(mask);--            // consume this bit-            mask &= ~(1llu << pos);--            // update value-            value &= ~LEAF_INDEX_MASK;-            value |= pos;--            // done-            return true;-        }-    };--    /**-     * Obtains an iterator pointing to the first index set to 1. If there-     * is no such bit, end() will be returned.-     */-    iterator begin() const {-        auto it = store.begin();-        if (it.isEnd()) return end();-        return iterator(it);-    }--    /**-     * Returns an iterator referencing the position after the last set bit.-     */-    iterator end() const {-        return iterator();-    }--    /**-     * Obtains an iterator referencing the position i if the corresponding-     * bit is set, end() otherwise.-     */-    iterator find(index_type i) const {-        op_context ctxt;-        return find(i, ctxt);-    }--    /**-     * Obtains an iterator referencing the position i if the corresponding-     * bit is set, end() otherwise. An operation context can be provided-     * to exploit temporal locality.-     */-    iterator find(index_type i, op_context& ctxt) const {-        // check prefix part-        auto it = store.find(i >> LEAF_INDEX_WIDTH, ctxt);-        if (it.isEnd()) return end();--        // check bit-set part-        uint64_t mask = iterator::toMask(it->second);-        if (!(mask & (1llu << (i & LEAF_INDEX_MASK)))) return end();--        // OK, it is there => create iterator-        mask &= ((1ull << (i & LEAF_INDEX_MASK)) - 1);  // remove all bits before pos i-        return iterator(it, mask, i);-    }--    /**-     * Locates an iterator to the first element in this sparse bit map not less-     * than the given index.-     */-    iterator lower_bound(index_type i) const {-        auto it = store.lowerBound(i >> LEAF_INDEX_WIDTH);-        if (it.isEnd()) return end();--        // check bit-set part-        uint64_t mask = iterator::toMask(it->second);--        // if there is no bit remaining in this mask, check next mask.-        if (!(mask & ((~uint64_t(0)) << (i & LEAF_INDEX_MASK)))) {-            index_type next = ((i >> LEAF_INDEX_WIDTH) + 1) << LEAF_INDEX_WIDTH;-            if (next < i) return end();-            return lower_bound(next);-        }--        // there are bits left, use least significant bit of those-        if (it->first == i >> LEAF_INDEX_WIDTH) {-            mask &= ((~uint64_t(0)) << (i & LEAF_INDEX_MASK));  // remove all bits before pos i-        }--        // compute value represented by least significant bit-        index_type pos = __builtin_ctzll(mask);--        // remove this bit as well-        mask = mask & ~(1ull << pos);--        // construct value of this located bit-        index_type val = (it->first << LEAF_INDEX_WIDTH) | pos;-        return iterator(it, mask, val);-    }--    /**-     * Locates an iterator to the first element in this sparse bit map than is greater-     * than the given index.-     */-    iterator upper_bound(index_type i) const {-        if (i == std::numeric_limits<index_type>::max()) {-            return end();-        }-        return lower_bound(i + 1);-    }--    /**-     * A debugging utility printing the internal structure of this map to the-     * given output stream.-     */-    void dump(bool detail = false, std::ostream& out = std::cout) const {-        store.dump(detail, out);-    }--    /**-     * Provides write-protected access to the internal store for running-     * analysis on the data structure.-     */-    const data_store_t& getStore() const {-        return store;-    }-};--// ----------------------------------------------------------------------//                              TRIE-// -----------------------------------------------------------------------namespace detail {--/**- * A base class for the Trie implementation allowing various- * specializations of the Trie template to inherit common functionality.- *- * @tparam Dim the number of dimensions / arity of the stored tuples- * @tparam Derived the type derived from this base class- */-template <unsigned Dim, typename Derived>-class TrieBase {-public:-    /**-     * The type of the stored entries / tuples.-     */-    using entry_type = typename souffle::Tuple<RamDomain, Dim>;--    // -- operation wrappers ----    /**-     * A generic function enabling the insertion of tuple values in a user-friendly way.-     */-    template <typename... Values>-    bool insert(Values... values) {-        return static_cast<Derived&>(*this).insert(entry_type{{RamDomain(values)...}});-    }--    /**-     * A generic function enabling the convenient conduction of a membership check.-     */-    template <typename... Values>-    bool contains(Values... values) const {-        return static_cast<const Derived&>(*this).contains(entry_type{{RamDomain(values)...}});-    }--    // ----------------------------------------------------------------------    //                           Iterator-    // -----------------------------------------------------------------------    /**-     * An iterator over the stored entries.-     *-     * Iterators for tries consist of a top-level iterator maintaining the-     * master copy of a materialized tuple and a recursively nested iterator-     * core -- one for each nested trie level.-     */-    template <template <unsigned D> class IterCore>-    class iterator {-        template <unsigned Len, unsigned Pos, unsigned Dimensions>-        friend struct fix_binding;--        template <unsigned Pos, unsigned Dimensions>-        friend struct fix_lower_bound;--        template <unsigned Pos, unsigned Dimensions>-        friend struct fix_upper_bound;--        template <unsigned Pos, unsigned Dimensions>-        friend struct fix_first;--        // the iterator core of this level-        using iter_core_t = IterCore<0>;--        // the wrapped iterator-        iter_core_t iter_core;--        // the value currently pointed to-        entry_type value;--    public:-        typedef std::forward_iterator_tag iterator_category;-        typedef entry_type value_type;-        typedef ptrdiff_t difference_type;-        typedef value_type* pointer;-        typedef value_type& reference;--        // default constructor -- creating an end-iterator-        iterator() = default;--        // a copy constructor-        iterator(const iterator& other) = default;--        iterator(iterator&& other) = default;--        template <typename Param>-        explicit iterator(const Param& param) : iter_core(param, value) {}--        // an assignment operator-        iterator& operator=(const iterator& other) = default;--        // the equality operator as required by the iterator concept-        bool operator==(const iterator& other) const {-            // equivalent if pointing to the same value-            return iter_core == other.iter_core;-        }--        // the not-equality operator as required by the iterator concept-        bool operator!=(const iterator& other) const {-            return !(*this == other);-        }--        // the deref operator as required by the iterator concept-        const entry_type& operator*() const {-            return value;-        }--        // support for the pointer operator-        const entry_type* operator->() const {-            return &value;-        }--        // the increment operator as required by the iterator concept-        iterator& operator++() {-            iter_core.inc(value);-            return *this;-        }--        // enables this iterator to be printed (for debugging)-        void print(std::ostream& out) const {-            out << "iter(" << iter_core << " -> " << value << ")";-        }--        friend std::ostream& operator<<(std::ostream& out, const iterator& iter) {-            iter.print(out);-            return out;-        }-    };--    /* -------------- operator hint statistics ----------------- */--    // an aggregation of statistical values of the hint utilization-    struct hint_statistics {-        // the counter for insertion operations-        CacheAccessCounter inserts;--        // the counter for contains operations-        CacheAccessCounter contains;--        // the counter for get_boundaries operations-        CacheAccessCounter get_boundaries;-    };--protected:-    // the hint statistic of this b-tree instance-    mutable hint_statistics hint_stats;--public:-    void printStats(std::ostream& out) const {-        out << "---------------------------------\n";-        out << "  insert-hint (hits/misses/total): " << hint_stats.inserts.getHits() << "/"-            << hint_stats.inserts.getMisses() << "/" << hint_stats.inserts.getAccesses() << "\n";-        out << "  contains-hint (hits/misses/total):" << hint_stats.contains.getHits() << "/"-            << hint_stats.contains.getMisses() << "/" << hint_stats.contains.getAccesses() << "\n";-        out << "  get-boundaries-hint (hits/misses/total):" << hint_stats.get_boundaries.getHits() << "/"-            << hint_stats.get_boundaries.getMisses() << "/" << hint_stats.get_boundaries.getAccesses()-            << "\n";-        out << "---------------------------------\n";-    }-};--/**- * A functor extracting a reference to a nested iterator core from an enclosing- * iterator core.- */-template <unsigned Level>-struct get_nested_iter_core {-    template <typename IterCore>-    auto operator()(IterCore& core) -> decltype(get_nested_iter_core<Level - 1>()(core.getNested())) {-        return get_nested_iter_core<Level - 1>()(core.getNested());-    }-};--template <>-struct get_nested_iter_core<0> {-    template <typename IterCore>-    IterCore& operator()(IterCore& core) {-        return core;-    }-};--/**- * A functor initializing an iterator upon creation to reference the first- * element in the associated Trie.- */-template <unsigned Pos, unsigned Dim>-struct fix_first {-    template <unsigned bits, typename iterator>-    void operator()(const SparseBitMap<bits>& store, iterator& iter) const {-        // set iterator to first in store-        auto first = store.begin();-        get_nested_iter_core<Pos>()(iter.iter_core).setIterator(first);-        iter.value[Pos] = *first;-    }--    template <typename Store, typename iterator>-    void operator()(const Store& store, iterator& iter) const {-        // set iterator to first in store-        auto first = store.begin();-        get_nested_iter_core<Pos>()(iter.iter_core).setIterator(first);-        iter.value[Pos] = first->first;-        // and continue recursively-        fix_first<Pos + 1, Dim>()(first->second->getStore(), iter);-    }-};--template <unsigned Dim>-struct fix_first<Dim, Dim> {-    template <typename Store, typename iterator>-    void operator()(const Store&, iterator&) const {-        // terminal case => nothing to do-    }-};--/**- * A functor initializing an iterator upon creation to reference the first element- * exhibiting a given prefix within a given Trie.- */-template <unsigned Len, unsigned Pos, unsigned Dim>-struct fix_binding {-    template <unsigned bits, typename iterator, typename entry_type>-    bool operator()(-            const SparseBitMap<bits>& store, iterator& begin, iterator& end, const entry_type& entry) const {-        // search in current level-        auto cur = store.find(entry[Pos]);--        // if not present => fail-        if (cur == store.end()) return false;--        // take current value-        get_nested_iter_core<Pos>()(begin.iter_core).setIterator(cur);-        ++cur;-        get_nested_iter_core<Pos>()(end.iter_core).setIterator(cur);--        // update iterator value-        begin.value[Pos] = entry[Pos];--        // no more remaining levels to fix-        return true;-    }--    template <typename Store, typename iterator, typename entry_type>-    bool operator()(const Store& store, iterator& begin, iterator& end, const entry_type& entry) const {-        // search in current level-        auto cur = store.find(entry[Pos]);--        // if not present => fail-        if (cur == store.end()) return false;--        // take current value as start-        get_nested_iter_core<Pos>()(begin.iter_core).setIterator(cur);--        // update iterator value-        begin.value[Pos] = entry[Pos];--        // fix remaining nested iterators-        auto res = fix_binding<Len - 1, Pos + 1, Dim>()(cur->second->getStore(), begin, end, entry);--        // update end of iterator-        if (get_nested_iter_core<Pos + 1>()(end.iter_core).getIterator() == cur->second->getStore().end()) {-            ++cur;-            if (cur != store.end()) {-                fix_first<Pos + 1, Dim>()(cur->second->getStore(), end);-            }-        }-        get_nested_iter_core<Pos>()(end.iter_core).setIterator(cur);--        // done-        return res;-    }-};--template <unsigned Pos, unsigned Dim>-struct fix_binding<0, Pos, Dim> {-    template <unsigned bits, typename iterator, typename entry_type>-    bool operator()(const SparseBitMap<bits>& store, iterator& begin, iterator& /* end */,-            const entry_type& /* entry */) const {-        // move begin to begin of store-        auto a = store.begin();-        get_nested_iter_core<Pos>()(begin.iter_core).setIterator(a);-        begin.value[Pos] = *a;--        return true;-    }--    template <typename Store, typename iterator, typename entry_type>-    bool operator()(const Store& store, iterator& begin, iterator& end, const entry_type& entry) const {-        // move begin to begin of store-        auto a = store.begin();-        get_nested_iter_core<Pos>()(begin.iter_core).setIterator(a);-        begin.value[Pos] = a->first;--        // continue recursively-        fix_binding<0, Pos + 1, Dim>()(a->second->getStore(), begin, end, entry);-        return true;-    }-};--template <unsigned Dim>-struct fix_binding<0, Dim, Dim> {-    template <typename Store, typename iterator, typename entry_type>-    bool operator()(const Store& /* store */, iterator& /* begin */, iterator& /* end */,-            const entry_type& /* entry */) const {-        // nothing more to do-        return true;-    }-};--/**- * A functor initializing an iterator upon creation to reference the first element- * within a given Trie being not less than a given value .- */-template <unsigned Pos, unsigned Dim>-struct fix_lower_bound {-    template <unsigned bits, typename iterator, typename entry_type>-    bool operator()(const SparseBitMap<bits>& store, iterator& iter, const entry_type& entry) const {-        // search in current level-        auto cur = store.lower_bound(entry[Pos]);--        if (cur == store.end()) return false;--        get_nested_iter_core<Pos>()(iter.iter_core).setIterator(cur);--        assert(entry[Pos] <= RamDomain(*cur));-        iter.value[Pos] = *cur;--        // no more remaining levels to fix-        return true;-    }--    template <typename Store, typename iterator, typename entry_type>-    bool operator()(const Store& store, iterator& iter, const entry_type& entry) const {-        // search in current level-        auto cur = store.lowerBound(entry[Pos]);--        // if no lower boundary is found, be done-        if (cur == store.end()) return false;-        assert(RamDomain(cur->first) >= entry[Pos]);--        // if the lower bound is higher than the requested value, go to first in subtree-        if (RamDomain(cur->first) > entry[Pos]) {-            get_nested_iter_core<Pos>()(iter.iter_core).setIterator(cur);-            iter.value[Pos] = cur->first;-            fix_first<Pos + 1, Dim>()(cur->second->getStore(), iter);-            return true;-        }--        // attempt to fix the rest-        if (!fix_lower_bound<Pos + 1, Dim>()(cur->second->getStore(), iter, entry)) {-            // if it does not work, since there are no matching elements in this branch, go to next-            entry_type sub = entry;-            sub[Pos] += 1;-            for (size_t i = Pos + 1; i < Dim; ++i) {-                sub[i] = 0;-            }-            return (*this)(store, iter, sub);-        }--        // remember result-        get_nested_iter_core<Pos>()(iter.iter_core).setIterator(cur);--        // update iterator value-        iter.value[Pos] = cur->first;--        // done!-        return true;-    }-};--/**- * A functor initializing an iterator upon creation to reference the first element- * within a given Trie being greater than a given value .- */-template <unsigned Pos, unsigned Dim>-struct fix_upper_bound {-    template <unsigned bits, typename iterator, typename entry_type>-    bool operator()(const SparseBitMap<bits>& store, iterator& iter, const entry_type& entry) const {-        // search in current level-        auto cur = store.upper_bound(entry[Pos]);--        if (cur == store.end()) {-            return false;-        }--        get_nested_iter_core<Pos>()(iter.iter_core).setIterator(cur);--        assert(entry[Pos] <= RamDomain(*cur));-        iter.value[Pos] = *cur;--        // no more remaining levels to fix-        return true;-    }--    template <typename Store, typename iterator, typename entry_type>-    bool operator()(const Store& store, iterator& iter, const entry_type& entry) const {-        // search in current level (if it is not the last level, we need a lower bound)-        auto cur = store.lowerBound(entry[Pos]);--        // if no lower boundary is found, be done-        if (cur == store.end()) {-            return false;-        }-        assert(RamDomain(cur->first) >= entry[Pos]);--        // if the lower bound is higher than the requested value, go to first in subtree-        if (RamDomain(cur->first) > entry[Pos]) {-            get_nested_iter_core<Pos>()(iter.iter_core).setIterator(cur);-            iter.value[Pos] = cur->first;-            fix_first<Pos + 1, Dim>()(cur->second->getStore(), iter);-            return true;-        }--        // attempt to fix the rest-        if (!fix_upper_bound<Pos + 1, Dim>()(cur->second->getStore(), iter, entry)) {-            // if it does not work, since there are no matching elements in this branch, go to next-            entry_type sub = entry;-            sub[Pos] += 1;-            for (size_t i = Pos + 1; i < Dim; ++i) {-                sub[i] = 0;-            }-            return (*this)(store, iter, sub);-        }--        // remember result-        get_nested_iter_core<Pos>()(iter.iter_core).setIterator(cur);--        // update iterator value-        iter.value[Pos] = cur->first;--        // done!-        return true;-    }-};--}  // namespace detail--/**- * The most generic implementation of a Trie forming the top-level of any- * Trie storing tuples of arity > 1.- */-template <unsigned Dim>-class Trie : public souffle::detail::TrieBase<Dim, Trie<Dim>> {-    template <unsigned D>-    friend class Trie;--    template <unsigned D, typename Derived>-    friend class TrieBase;--    // a shortcut for the common base class type-    using base = typename souffle::detail::TrieBase<Dim, Trie<Dim>>;--    // the type of the nested tries (1 dimension less)-    using nested_trie_type = Trie<Dim - 1>;--    // the merge operation capable of merging two nested tries-    struct nested_trie_merger {-        nested_trie_type* operator()(nested_trie_type* a, const nested_trie_type* b) const {-            if (!b) return a;-            if (!a) return new nested_trie_type(*b);-            a->insertAll(*b);-            return a;-        }-    };--    // the operation capable of cloning a nested trie-    struct nested_trie_cloner {-        nested_trie_type* operator()(nested_trie_type* a) const {-            if (!a) return a;-            return new nested_trie_type(*a);-        }-    };--    // the data structure utilized for indexing nested tries-    using store_type = SparseArray<nested_trie_type*,-            6,  // = 2^6 entries per block-            nested_trie_merger, nested_trie_cloner>;--    // the actual data store-    store_type store;--public:-    using entry_type = typename souffle::Tuple<RamDomain, Dim>;-    using element_type = entry_type;--    // ----------------------------------------------------------------------    //                           Iterator-    // -----------------------------------------------------------------------    /**-     * The iterator core for trie iterators involving this level.-     */-    template <unsigned I = 0>-    class iterator_core {-        // the iterator for the current level-        using store_iter_t = typename store_type::iterator;--        // the type of the nested iterator-        using nested_iter_core = typename Trie<Dim - 1>::template iterator_core<I + 1>;--        store_iter_t iter;--        nested_iter_core nested;--    public:-        /** default end-iterator constructor */-        iterator_core() = default;--        template <typename Tuple>-        iterator_core(const store_iter_t& iter, Tuple& entry) : iter(iter) {-            entry[I] = iter->first;-            nested = iter->second->template getBeginCoreIterator<I + 1>(entry);-        }--        void setIterator(const store_iter_t& iter) {-            this->iter = iter;-        }--        store_iter_t& getIterator() {-            return this->iter;-        }--        nested_iter_core& getNested() {-            return nested;-        }--        template <typename Tuple>-        bool inc(Tuple& entry) {-            // increment nested iterator-            if (nested.inc(entry)) return true;--            // increment the iterator on this level-            ++iter;--            // check whether the end has been reached-            if (iter.isEnd()) return false;--            // otherwise update entry value-            entry[I] = iter->first;--            // and restart nested-            nested = iter->second->template getBeginCoreIterator<I + 1>(entry);-            return true;-        }--        bool operator==(const iterator_core& other) const {-            return nested == other.nested && iter == other.iter;-        }--        bool operator!=(const iterator_core& other) const {-            return !(*this == other);-        }--        // enables this iterator core to be printed (for debugging)-        void print(std::ostream& out) const {-            out << iter << " | " << nested;-        }--        friend std::ostream& operator<<(std::ostream& out, const iterator_core& iter) {-            iter.print(out);-            return out;-        }-    };--    // the type of iterator to be utilized when iterating of instances of this trie-    using iterator = typename base::template iterator<iterator_core>;--    // the operation context aggregating all operation contexts of nested structures-    struct op_context {-        using local_ctxt = typename store_type::op_context;-        using nested_ctxt = typename nested_trie_type::op_context;--        // for insert and contain-        local_ctxt local{};-        RamDomain lastQuery{};-        nested_trie_type* lastNested{nullptr};-        nested_ctxt nestedCtxt{};--        // for boundaries-        unsigned lastBoundaryLevels{Dim + 1};-        entry_type lastBoundaryRequest{};-        range<iterator> lastBoundaries{iterator(), iterator()};--        op_context() = default;-    };--    using operation_hints = op_context;--    using base::contains;-    using base::insert;--    /**-     * A simple destructore.-     */-    ~Trie() {-        for (auto& cur : store) {-            delete cur.second;  // clears all nested tries-        }-    }--    /**-     * Determines whether this trie is empty or not.-     */-    bool empty() const {-        return store.empty();-    }--    /**-     * Determines the number of entries in this trie.-     */-    std::size_t size() const {-        // the number of elements is lazy-evaluated-        std::size_t res = 0;-        for (const auto& cur : store) {-            res += cur.second->size();-        }-        return res;-    }--    /**-     * Computes the total memory usage of this data structure.-     */-    std::size_t getMemoryUsage() const {-        // compute the total memory usage of this level-        std::size_t res = sizeof(*this) - sizeof(store) + store.getMemoryUsage();--        // add the memory usage of sub-levels-        for (const auto& cur : store) {-            res += cur.second->getMemoryUsage();-        }--        // done-        return res;-    }--    /**-     * Removes all entries within this trie.-     */-    void clear() {-        // delete lower levels-        for (auto& cur : store) {-            delete cur.second;-        }--        // clear store-        store.clear();-    }--    /**-     * Inserts a new entry.-     *-     * @param tuple the entry to be added-     * @return true if the same tuple hasn't been present before, false otherwise-     */-    bool insert(const entry_type& tuple) {-        op_context ctxt;-        return insert(tuple, ctxt);-    }--    /**-     * Inserts a new entry. A operation context may be provided to exploit temporal-     * locality.-     *-     * @param tuple the entry to be added-     * @param ctxt the operation context to be utilized-     * @return true if the same tuple hasn't been present before, false otherwise-     */-    bool insert(const entry_type& tuple, op_context& ctxt) {-        return insert_internal<0>(tuple, ctxt);-    }--    /**-     * Determines whether a given tuple is present within the set specified-     * by this trie.-     *-     * @param tuple the tuple to be tested-     * @return true if present, false otherwise-     */-    bool contains(const entry_type& tuple) const {-        op_context ctxt;-        return contains(tuple, ctxt);-    }--    /**-     * Determines whether a given tuple is present within the set specified-     * by this trie. A operation context may be provided to exploit temporal-     * locality.-     *-     * @param tuple the entry to be added-     * @param ctxt the operation context to be utilized-     * @return true if the same tuple hasn't been present before, false otherwise-     */-    bool contains(const entry_type& tuple, op_context& ctxt) const {-        return contains_internal<0>(tuple, ctxt);-    }--    /**-     * Inserts all elements stored within the given trie into this trie.-     *-     * @param other the elements to be inserted into this trie-     */-    void insertAll(const Trie& other) {-        store.addAll(other.store);-    }--    /**-     * Obtains an iterator referencing the first element stored within this trie.-     */-    iterator begin() const {-        auto it = store.begin();-        if (it.isEnd()) return end();-        return iterator(it);-    }--    /**-     * Obtains an iterator referencing the position after the last element stored-     * within this trie.-     */-    iterator end() const {-        return iterator();-    }--    iterator find(const entry_type& entry) const {-        op_context ctxt;-        return find(entry, ctxt);-    }--    iterator find(const entry_type& entry, op_context& ctxt) const {-        auto range = getBoundaries<Dim>(entry, ctxt);-        return (!range.empty()) ? range.begin() : end();-    }--    /**-     * Obtains a range of elements matching the prefix of the given entry up to-     * levels elements.-     *-     * @tparam levels the length of the requested matching prefix-     * @param entry the entry to be looking for-     * @return the corresponding range of matching elements-     */-    template <unsigned levels>-    range<iterator> getBoundaries(const entry_type& entry) const {-        op_context ctxt;-        return getBoundaries<levels>(entry, ctxt);-    }--    /**-     * Obtains a range of elements matching the prefix of the given entry up to-     * levels elements. A operation context may be provided to exploit temporal-     * locality.-     *-     * @tparam levels the length of the requested matching prefix-     * @param entry the entry to be looking for-     * @param ctxt the operation context to be utilized-     * @return the corresponding range of matching elements-     */-    template <unsigned levels>-    range<iterator> getBoundaries(const entry_type& entry, op_context& ctxt) const {-        // if nothing is bound => just use begin and end-        if (levels == 0) return make_range(begin(), end());--        // check context-        if (ctxt.lastBoundaryLevels == levels) {-            bool fit = true;-            for (unsigned i = 0; i < levels; ++i) {-                fit = fit && (entry[i] == ctxt.lastBoundaryRequest[i]);-            }--            // if it fits => take it-            if (fit) {-                base::hint_stats.get_boundaries.addHit();-                return ctxt.lastBoundaries;-            }-        }--        // the hint has not been a hit-        base::hint_stats.get_boundaries.addMiss();--        // start with two end iterators-        iterator begin{};-        iterator end{};--        // adapt them level by level-        auto found = souffle::detail::fix_binding<levels, 0, Dim>()(store, begin, end, entry);-        if (!found) return make_range(iterator(), iterator());--        // update context-        ctxt.lastBoundaryLevels = levels;-        ctxt.lastBoundaryRequest = entry;-        ctxt.lastBoundaries = make_range(begin, end);--        // use the result-        return ctxt.lastBoundaries;-    }--    /**-     * Obtains an iterator to the first element not less than the given entry value.-     *-     * @param entry the lower bound for this search-     * @param ctxt the operation context to be utilized-     * @return an iterator addressing the first element in this structure not less than the given value-     */-    iterator lower_bound(const entry_type& entry, op_context& /* ctxt */) const {-        // start with a default-initialized iterator-        iterator res;--        // adapt it level by level-        bool found = detail::fix_lower_bound<0, Dim>()(store, res, entry);--        // use the result-        return found ? res : end();-    }--    /**-     * Obtains an iterator to the first element not less than the given entry value.-     *-     * @param entry the lower bound for this search-     * @return an iterator addressing the first element in this structure not less than the given value-     */-    iterator lower_bound(const entry_type& entry) const {-        op_context ctxt;-        return lower_bound(entry, ctxt);-    }--    /**-     * Obtains an iterator to the first element greater than the given entry value, or end if there is no such-     * element.-     *-     * @param entry the upper bound for this search-     * @param ctxt the operation context to be utilized-     * @return an iterator addressing the first element in this structure greater than the given value-     */-    iterator upper_bound(const entry_type& entry, op_context& /* ctxt */) const {-        // start with a default-initialized iterator-        iterator res;--        // adapt it level by level-        bool found = detail::fix_upper_bound<0, Dim>()(store, res, entry);--        // use the result-        return found ? res : end();-    }--    /**-     * Obtains an iterator to the first element greater than the given entry value, or end if there is no such-     * element.-     *-     * @param entry the upper bound for this search-     * @return an iterator addressing the first element in this structure greater than the given value-     */-    iterator upper_bound(const entry_type& entry) const {-        op_context ctxt;-        return upper_bound(entry, ctxt);-    }--    /**-     * Computes a partition of an approximate number of chunks of the content-     * of this trie. Thus, the union of the resulting set of disjoint ranges is-     * equivalent to the content of this trie.-     *-     * @param chunks the number of chunks requested-     * @return a list of sub-ranges forming a partition of the content of this trie-     */-    std::vector<range<iterator>> partition(unsigned chunks = 500) const {-        std::vector<range<iterator>> res;--        // shortcut for empty trie-        if (this->empty()) return res;--        // use top-level elements for partitioning-        int step = std::max(store.size() / chunks, size_t(1));--        int c = 1;-        auto priv = begin();-        for (auto it = store.begin(); it != store.end(); ++it, c++) {-            if (c % step != 0 || c == 1) {-                continue;-            }-            auto cur = iterator(it);-            res.push_back(make_range(priv, cur));-            priv = cur;-        }-        // add final chunk-        res.push_back(make_range(priv, end()));-        return res;-    }--    /**-     * Provides a protected access to the internally maintained store.-     */-    const store_type& getStore() const {-        return store;-    }--private:-    /**-     * Creates a core iterator for this trie level and updates component-     * I of the given entry to exhibit the corresponding first value.-     *-     * @tparam I the index of the tuple to be processed by the resulting iterator core-     * @tparam Tuple the type of the tuple to be processed by the resulting iterator core-     * @param entry a reference to the tuple to be updated to the first value-     * @return the requested iterator core instance-     */-    template <unsigned I, typename Tuple>-    iterator_core<I> getBeginCoreIterator(Tuple& entry) const {-        return iterator_core<I>(store.begin(), entry);-    }--    /**-     * The internally utilized implementation of the insert operation inserting-     * a given tuple into this sub-trie.-     *-     * @tparam I the component index associated to this level-     * @tparam Tuple the tuple type to be inserted-     * @param tuple the tuple to be inserted-     * @param ctxt a operation context to exploit temporal locality-     * @return true if this tuple wasn't contained before, false otherwise-     */-    template <unsigned I, typename Tuple>-    bool insert_internal(const Tuple& tuple, op_context& ctxt) {-        using value_t = typename store_type::value_type;-        using atomic_value_t = typename store_type::atomic_value_type;--        // check context-        if (ctxt.lastNested && ctxt.lastQuery == tuple[I]) {-            base::hint_stats.inserts.addHit();-            return ctxt.lastNested->template insert_internal<I + 1>(tuple, ctxt.nestedCtxt);-        } else {-            base::hint_stats.inserts.addMiss();-        }--        // lookup nested-        atomic_value_t& next = store.getAtomic(tuple[I], ctxt.local);--        // get pure pointer to next level-        value_t nextPtr = next;--        // conduct a lock-free lazy-creation of nested trees-        if (!nextPtr) {-            // create a new sub-tree-            auto newNested = new nested_trie_type();--            // register new sub-tree atomically-            if (next.compare_exchange_weak(nextPtr, newNested)) {-                nextPtr = newNested;  // worked-            } else {-                delete newNested;  // some other thread was faster => use its version-            }-        }--        // make sure a next has been established-        assert(nextPtr);--        // clear context if necessary-        if (nextPtr != ctxt.lastNested) {-            ctxt.lastQuery = tuple[I];-            ctxt.lastNested = nextPtr;-            ctxt.nestedCtxt = typename op_context::nested_ctxt();-        }--        // conduct recursive step-        return nextPtr->template insert_internal<I + 1>(tuple, ctxt.nestedCtxt);-    }--    /**-     * An internal implementation of the contains member function determining-     * whether a given tuple is present within this sub-trie or not.-     *-     * @tparam I the component index associated to this level-     * @tparam Tuple the tuple type to be checked-     * @param tuple the tuple to be checked-     * @param ctxt a operation context to exploit temporal locality-     * @return true if this tuple is present, false otherwise-     */-    template <unsigned I, typename Tuple>-    bool contains_internal(const Tuple& tuple, op_context& ctxt) const {-        // check context-        if (ctxt.lastNested && ctxt.lastQuery == tuple[I]) {-            base::hint_stats.contains.addHit();-            return ctxt.lastNested->template contains_internal<I + 1>(tuple, ctxt.nestedCtxt);-        } else {-            base::hint_stats.contains.addMiss();-        }--        // lookup next step-        auto next = store.lookup(tuple[I], ctxt.local);--        // clear context if necessary-        if (next != ctxt.lastNested) {-            ctxt.lastQuery = tuple[I];-            ctxt.lastNested = next;-            ctxt.nestedCtxt = typename op_context::nested_ctxt();-        }--        // conduct recursive step-        return next && next->template contains_internal<I + 1>(tuple, ctxt.nestedCtxt);-    }-};--/**- * A template specialization for tries representing a set.- * For improved memory efficiency, this level is the leaf-node level- * of all tries exhibiting an arity >= 1. Internally, values are stored utilizing- * sparse bit maps.- */-template <>-class Trie<1u> : public detail::TrieBase<1u, Trie<1u>> {-    template <unsigned Dim>-    friend class Trie;--    template <unsigned Dim, typename Derived>-    friend class detail::TrieBase;--    // a shortcut for the base type-    using base = typename detail::TrieBase<1u, Trie<1u>>;--    // the map type utilized internally-    using map_type = SparseBitMap<>;--    // the internal data store-    map_type map;--public:-    using element_type = entry_type;-    using op_context = typename map_type::op_context;-    using operation_hints = op_context;--    using base::contains;-    using base::insert;--    /**-     * Determines whether this trie is empty or not.-     */-    bool empty() const {-        return map.empty();-    }--    /**-     * Determines the number of elements stored in this trie.-     */-    std::size_t size() const {-        return map.size();-    }--    /**-     * Computes the total memory usage of this data structure.-     */-    std::size_t getMemoryUsage() const {-        // compute the total memory usage-        return sizeof(*this) - sizeof(map_type) + map.getMemoryUsage();-    }--    /**-     * Removes all elements form this trie.-     */-    void clear() {-        map.clear();-    }--    /**-     * Inserts the given tuple into this trie.-     *-     * @param tuple the tuple to be inserted-     * @return true if the tuple has not been present before, false otherwise-     */-    bool insert(const entry_type& tuple) {-        op_context ctxt;-        return insert(tuple, ctxt);-    }--    /**-     * Inserts the given tuple into this trie.-     * An operation context can be provided to exploit temporal locality.-     *-     * @param tuple the tuple to be inserted-     * @param ctxt an operation context for exploiting temporal locality-     * @return true if the tuple has not been present before, false otherwise-     */-    bool insert(const entry_type& tuple, op_context& ctxt) {-        return insert_internal<0>(tuple, ctxt);-    }--    /**-     * Determines whether the given tuple is present in this trie or not.-     *-     * @param tuple the tuple to be tested-     * @return true if present, false otherwise-     */-    bool contains(const entry_type& tuple) const {-        op_context ctxt;-        return contains(tuple, ctxt);-    }--    /**-     * Determines whether the given tuple is present in this trie or not.-     * An operation context can be provided to exploit temporal locality.-     *-     * @param tuple the tuple to be tested-     * @param ctxt an operation context for exploiting temporal locality-     * @return true if present, false otherwise-     */-    bool contains(const entry_type& tuple, op_context& ctxt) const {-        return contains_internal<0>(tuple, ctxt);-    }--    /**-     * Inserts all tuples stored within the given trie into this trie.-     * This operation is considerably more efficient than the consecutive-     * insertion of the elements in other into this trie.-     */-    void insertAll(const Trie& other) {-        map.addAll(other.map);-    }--    // ----------------------------------------------------------------------    //                           Iterator-    // -----------------------------------------------------------------------    /**-     * The iterator core of this level contributing to the construction of-     * a composed trie iterator.-     */-    template <unsigned I = 0>-    class iterator_core {-        // the iterator for this level-        using iter_type = typename map_type::iterator;--        // the referenced bit-map iterator-        iter_type iter;--    public:-        /** default end-iterator constructor */-        iterator_core() = default;--        template <typename Tuple>-        iterator_core(const iter_type& iter, Tuple& entry) : iter(iter) {-            entry[I] = static_cast<RamDomain>(*iter);-        }--        void setIterator(const iter_type& iter) {-            this->iter = iter;-        }--        iter_type& getIterator() {-            return this->iter;-        }--        template <typename Tuple>-        bool inc(Tuple& entry) {-            // increment the iterator on this level-            ++iter;--            // check whether the end has been reached-            if (iter.isEnd()) return false;--            // otherwise update entry value-            entry[I] = *iter;-            return true;-        }--        bool operator==(const iterator_core& other) const {-            return iter == other.iter;-        }--        bool operator!=(const iterator_core& other) const {-            return !(*this == other);-        }--        // enables this iterator core to be printed (for debugging)-        void print(std::ostream& out) const {-            out << iter;-        }--        friend std::ostream& operator<<(std::ostream& out, const iterator_core& iter) {-            iter.print(out);-            return out;-        }-    };--    // the iterator type utilized by this trie type-    using iterator = typename base::template iterator<iterator_core>;--    /**-     * Obtains an iterator referencing the first element stored within this trie-     * or end() if this trie is empty.-     */-    iterator begin() const {-        if (map.empty()) return end();-        return iterator(map.begin());-    }--    /**-     * Obtains an iterator referencing the first position after the last element-     * within this trie.-     */-    iterator end() const {-        return iterator();-    }--    /**-     * Obtains a partition of this tire such that the resulting list of ranges-     * cover disjoint subsets of the elements stored in this trie. Their union-     * is equivalent to the content of this trie.-     */-    std::vector<range<iterator>> partition(unsigned chunks = 500) const {-        std::vector<range<iterator>> res;--        // shortcut for empty trie-        if (this->empty()) return res;--        // use top-level elements for partitioning-        int step = static_cast<int>(std::max(map.size() / chunks, size_t(1)));--        int c = 1;-        auto priv = begin();-        for (auto it = map.begin(); it != map.end(); ++it, c++) {-            if (c % step != 0 || c == 1) {-                continue;-            }-            auto cur = iterator(it);-            res.push_back(make_range(priv, cur));-            priv = cur;-        }-        // add final chunk-        res.push_back(make_range(priv, end()));-        return res;-    }--    /**-     * Obtains a range of elements matching the prefix of the given entry up to-     * levels elements.-     *-     * @tparam levels the length of the requested matching prefix-     * @param entry the entry to be looking for-     * @return the corresponding range of matching elements-     */-    template <unsigned levels>-    range<iterator> getBoundaries(const entry_type& entry) const {-        op_context ctxt;-        return getBoundaries<levels>(entry, ctxt);-    }--    /**-     * Obtains a range of elements matching the prefix of the given entry up to-     * levels elements. A operation context may be provided to exploit temporal-     * locality.-     *-     * @tparam levels the length of the requested matching prefix-     * @param entry the entry to be looking for-     * @param ctxt the operation context to be utilized-     * @return the corresponding range of matching elements-     */-    template <unsigned levels>-    range<iterator> getBoundaries(const entry_type& entry, op_context& ctxt) const {-        // for levels = 0-        if (levels == 0) return make_range(begin(), end());-        // for levels = 1-        auto pos = map.find(entry[0], ctxt);-        if (pos == map.end()) return make_range(end(), end());-        auto next = pos;-        ++next;-        return make_range(iterator(pos), iterator(next));-    }--    iterator lower_bound(const entry_type& entry, op_context&) const {-        return iterator(map.lower_bound(entry[0]));-    }--    iterator lower_bound(const entry_type& entry) const {-        op_context ctxt;-        return lower_bound(entry, ctxt);-    }--    iterator upper_bound(const entry_type& entry, op_context&) const {-        return iterator(map.upper_bound(entry[0]));-    }--    iterator upper_bound(const entry_type& entry) const {-        op_context ctxt;-        return upper_bound(entry, ctxt);-    }--    /**-     * Provides protected access to the internally maintained store.-     */-    const map_type& getStore() const {-        return map;-    }--private:-    /**-     * Creates a core iterator for this trie level and updates component-     * I of the given entry to exhibit the corresponding first value.-     *-     * @tparam I the index of the tuple to be processed by the resulting iterator core-     * @tparam Tuple the type of the tuple to be processed by the resulting iterator core-     * @param entry a reference to the tuple to be updated to the first value-     * @return the requested iterator core instance-     */-    template <unsigned I, typename Tuple>-    iterator_core<I> getBeginCoreIterator(Tuple& entry) const {-        return iterator_core<I>(map.begin(), entry);-    }--    /**-     * The internally utilized implementation of the insert operation inserting-     * a given tuple into this sub-trie.-     *-     * @tparam I the component index associated to this level-     * @tparam Tuple the tuple type to be inserted-     * @param tuple the tuple to be inserted-     * @param ctxt a operation context to exploit temporal locality-     * @return true if this tuple wasn't contained before, false otherwise-     */-    template <unsigned I, typename Tuple>-    bool insert_internal(const Tuple& tuple, op_context& ctxt) {-        return map.set(tuple[I], ctxt);-    }--    /**-     * An internal implementation of the contains member function determining-     * whether a given tuple is present within this sub-trie or not.-     *-     * @tparam I the component index associated to this level-     * @tparam Tuple the tuple type to be checked-     * @param tuple the tuple to be checked-     * @param ctxt a operation context to exploit temporal locality-     * @return true if this tuple is present, false otherwise-     */-    template <unsigned I, typename Tuple>-    bool contains_internal(const Tuple& tuple, op_context& ctxt) const {-        return map.test(tuple[I], ctxt);-    }-};--}  // end namespace souffle+#include "souffle/RamTypes.h"+#include "souffle/utility/CacheUtil.h"+#include "souffle/utility/ContainerUtil.h"+#include "souffle/utility/MiscUtil.h"+#include "souffle/utility/StreamUtil.h"+#include "souffle/utility/span.h"+#include <algorithm>+#include <atomic>+#include <bitset>+#include <cassert>+#include <climits>+#include <cstdint>+#include <cstring>+#include <iostream>+#include <iterator>+#include <limits>+#include <type_traits>+#include <utility>+#include <vector>++// TODO: replace intrinsics w/ std lib functions?+#ifdef _WIN32+/**+ * When compiling for windows, redefine the gcc builtins which are used to+ * their equivalents on the windows platform.+ */+#define __sync_synchronize MemoryBarrier+#define __sync_bool_compare_and_swap(ptr, oldval, newval) \+    (InterlockedCompareExchangePointer((void* volatile*)ptr, (void*)newval, (void*)oldval) == (void*)oldval)+#endif  // _WIN32++namespace souffle {++template <unsigned Dim>+class Trie;++namespace detail::brie {++// FIXME: These data structs should be parameterised/made agnostic to `RamDomain` type.+using brie_element_type = RamDomain;++using tcb::make_span;++template <typename A>+struct forward_non_output_iterator_traits {+    using value_type = A;+    using difference_type = ptrdiff_t;+    using iterator_category = std::forward_iterator_tag;+    using pointer = const value_type*;+    using reference = const value_type&;+};++template <typename A, std::size_t arity>+auto copy(span<A, arity> s) {+    std::array<std::decay_t<A>, arity> cpy;+    std::copy_n(s.begin(), arity, cpy.begin());+    return cpy;+}++template <std::size_t offset, typename A, std::size_t arity>+auto drop(span<A, arity> s) -> std::enable_if_t<offset <= arity, span<A, arity - offset>> {+    return {s.begin() + offset, s.end()};+}++template <typename C>+auto tail(C& s) {+    return drop<1>(make_span(s));+}++/**+ * A templated functor to obtain default values for+ * unspecified elements of sparse array instances.+ */+template <typename T>+struct default_factory {+    T operator()() const {+        return T();  // just use the default constructor+    }+};++/**+ * A functor representing the identity function.+ */+template <typename T>+struct identity {+    T operator()(T v) const {+        return v;+    }+};++/**+ * A operation to be utilized by the sparse map when merging+ * elements associated to different values.+ */+template <typename T>+struct default_merge {+    /**+     * Merges two values a and b when merging spase maps.+     */+    T operator()(T a, T b) const {+        default_factory<T> def;+        // if a is the default => us b, else stick to a+        return (a != def()) ? a : b;+    }+};++/**+ * Iterator type for `souffle::SparseArray`.+ */+template <typename SparseArray>+struct SparseArrayIter {+    using Node = typename SparseArray::Node;+    using index_type = typename SparseArray::index_type;+    using array_value_type = typename SparseArray::value_type;++    using value_type = std::pair<index_type, array_value_type>;++    SparseArrayIter() = default;  // default constructor -- creating an end-iterator+    SparseArrayIter(const SparseArrayIter&) = default;+    SparseArrayIter& operator=(const SparseArrayIter&) = default;++    SparseArrayIter(const Node* node, value_type value) : node(node), value(std::move(value)) {}++    SparseArrayIter(const Node* first, index_type firstOffset) : node(first), value(firstOffset, 0) {+        // if the start is the end => we are done+        if (!first) return;++        // load the value+        if (first->cell[0].value == array_value_type()) {+            ++(*this);  // walk to first element+        } else {+            value.second = first->cell[0].value;+        }+    }++    // the equality operator as required by the iterator concept+    bool operator==(const SparseArrayIter& other) const {+        // only equivalent if pointing to the end+        return (node == nullptr && other.node == nullptr) ||+               (node == other.node && value.first == other.value.first);+    }++    // the not-equality operator as required by the iterator concept+    bool operator!=(const SparseArrayIter& other) const {+        return !(*this == other);+    }++    // the deref operator as required by the iterator concept+    const value_type& operator*() const {+        return value;+    }++    // support for the pointer operator+    const value_type* operator->() const {+        return &value;+    }++    // the increment operator as required by the iterator concept+    SparseArrayIter& operator++() {+        assert(!isEnd());+        // get current offset+        index_type x = value.first & SparseArray::INDEX_MASK;++        // go to next non-empty value in current node+        do {+            x++;+        } while (x < SparseArray::NUM_CELLS && node->cell[x].value == array_value_type());++        // check whether one has been found+        if (x < SparseArray::NUM_CELLS) {+            // update value and be done+            value.first = (value.first & ~SparseArray::INDEX_MASK) | x;+            value.second = node->cell[x].value;+            return *this;  // done+        }++        // go to parent+        node = node->parent;+        int level = 1;++        // get current index on this level+        x = SparseArray::getIndex(brie_element_type(value.first), level);+        x++;++        while (level > 0 && node) {+            // search for next child+            while (x < SparseArray::NUM_CELLS) {+                if (node->cell[x].ptr != nullptr) {+                    break;+                }+                x++;+            }++            // pick next step+            if (x < SparseArray::NUM_CELLS) {+                // going down+                node = node->cell[x].ptr;+                value.first &= SparseArray::getLevelMask(level + 1);+                value.first |= x << (SparseArray::BIT_PER_STEP * level);+                level--;+                x = 0;+            } else {+                // going up+                node = node->parent;+                level++;++                // get current index on this level+                x = SparseArray::getIndex(brie_element_type(value.first), level);+                x++;  // go one step further+            }+        }++        // check whether it is the end of range+        if (node == nullptr) {+            return *this;+        }++        // search the first value in this node+        x = 0;+        while (node->cell[x].value == array_value_type()) {+            x++;+        }++        // update value+        value.first |= x;+        value.second = node->cell[x].value;++        // done+        return *this;+    }++    SparseArrayIter operator++(int) {+        auto cpy = *this;+        ++(*this);+        return cpy;+    }++    // True if this iterator is passed the last element.+    bool isEnd() const {+        return node == nullptr;+    }++    // enables this iterator core to be printed (for debugging)+    void print(std::ostream& out) const {+        // `StreamUtil.h` defines an overload for `pair`, but we can't rely on it b/c+        // it's disabled if `__EMBEDDED__` is defined.+        out << "SparseArrayIter(" << node << " @ (" << value.first << ", " << value.second << "))";+    }++    friend std::ostream& operator<<(std::ostream& out, const SparseArrayIter& iter) {+        iter.print(out);+        return out;+    }++private:+    // a pointer to the leaf node currently processed or null (end)+    const Node* node{};++    // the value currently pointed to+    value_type value;+};++}  // namespace detail::brie++using namespace detail::brie;++/**+ * A sparse array simulates an array associating to every element+ * of uint32_t an element of a generic type T. Any non-defined element+ * will be default-initialized utilizing the detail::brie::default_factory+ * functor.+ *+ * Internally the array is organized as a balanced tree. The leaf+ * level of the tree corresponds to the elements of the represented+ * array. Inner nodes utilize individual bits of the indices to reference+ * sub-trees. For efficiency reasons, only the minimal sub-tree required+ * to cover all non-null / non-default values stored in the array is+ * maintained. Furthermore, several levels of nodes are aggreated in a+ * B-tree like fashion to inprove cache utilization and reduce the number+ * of steps required for lookup and insert operations.+ *+ * @tparam T the type of the stored elements+ * @tparam BITS the number of bits consumed per node-level+ *              e.g. if it is set to 3, the resulting tree will be of a degree of+ *              2^3=8, and thus 8 child-pointers will be stored in each inner node+ *              and as many values will be stored in each leaf node.+ * @tparam merge_op the functor to be utilized when merging the content of two+ *              instances of this type.+ * @tparam copy_op a functor to be applied to each stored value when copying an+ *              instance of this array. For instance, this is utilized by the+ *              trie implementation to create a clone of each sub-tree instead+ *              of preserving the original pointer.+ */+template <typename T, unsigned BITS = 6, typename merge_op = default_merge<T>, typename copy_op = identity<T>>+class SparseArray {+    template <typename A>+    friend struct detail::brie::SparseArrayIter;++    using this_t = SparseArray<T, BITS, merge_op, copy_op>;+    using key_type = uint64_t;++    // some internal constants+    static constexpr int BIT_PER_STEP = BITS;+    static constexpr int NUM_CELLS = 1 << BIT_PER_STEP;+    static constexpr key_type INDEX_MASK = NUM_CELLS - 1;++public:+    // the type utilized for indexing contained elements+    using index_type = key_type;++    // the type of value stored in this array+    using value_type = T;++    // the atomic view on stored values+    using atomic_value_type = std::atomic<value_type>;++private:+    struct Node;++    /**+     * The value stored in a single cell of a inner+     * or leaf node.+     */+    union Cell {+        // an atomic view on the pointer referencing a nested level+        std::atomic<Node*> aptr;++        // a pointer to the nested level (unsynchronized operations)+        Node* ptr{nullptr};++        // an atomic view on the value stored in this cell (leaf node)+        atomic_value_type avalue;++        // the value stored in this cell (unsynchronized access, leaf node)+        value_type value;+    };++    /**+     * The node type of the internally maintained tree.+     */+    struct Node {+        // a pointer to the parent node (for efficient iteration)+        const Node* parent;+        // the pointers to the child nodes (inner nodes) or the stored values (leaf nodes)+        Cell cell[NUM_CELLS];+    };++    /**+     * A struct describing all the information required by the container+     * class to manage the wrapped up tree.+     */+    struct RootInfo {+        // the root node of the tree+        Node* root;+        // the number of levels of the tree+        uint32_t levels;+        // the absolute offset of the theoretical first element in the tree+        index_type offset;++        // the first leaf node in the tree+        Node* first;+        // the absolute offset of the first element in the first leaf node+        index_type firstOffset;+    };++    union {+        RootInfo unsynced;         // for sequential operations+        volatile RootInfo synced;  // for synchronized operations+    };++public:+    /**+     * A default constructor creating an empty sparse array.+     */+    SparseArray() : unsynced(RootInfo{nullptr, 0, 0, nullptr, std::numeric_limits<index_type>::max()}) {}++    /**+     * A copy constructor for sparse arrays. It creates a deep+     * copy of the data structure maintained by the handed in+     * array instance.+     */+    SparseArray(const SparseArray& other)+            : unsynced(RootInfo{clone(other.unsynced.root, other.unsynced.levels), other.unsynced.levels,+                      other.unsynced.offset, nullptr, other.unsynced.firstOffset}) {+        if (unsynced.root) {+            unsynced.root->parent = nullptr;+            unsynced.first = findFirst(unsynced.root, unsynced.levels);+        }+    }++    /**+     * A r-value based copy constructor for sparse arrays. It+     * takes over ownership of the structure maintained by the+     * handed in array.+     */+    SparseArray(SparseArray&& other)+            : unsynced(RootInfo{other.unsynced.root, other.unsynced.levels, other.unsynced.offset,+                      other.unsynced.first, other.unsynced.firstOffset}) {+        other.unsynced.root = nullptr;+        other.unsynced.levels = 0;+        other.unsynced.first = nullptr;+    }++    /**+     * A destructor for sparse arrays clearing up the internally+     * maintained data structure.+     */+    ~SparseArray() {+        clean();+    }++    /**+     * An assignment creating a deep copy of the handed in+     * array structure (utilizing the copy functor provided+     * as a template parameter).+     */+    SparseArray& operator=(const SparseArray& other) {+        if (this == &other) return *this;++        // clean this one+        clean();++        // copy content+        unsynced.levels = other.unsynced.levels;+        unsynced.root = clone(other.unsynced.root, unsynced.levels);+        if (unsynced.root) {+            unsynced.root->parent = nullptr;+        }+        unsynced.offset = other.unsynced.offset;+        unsynced.first = (unsynced.root) ? findFirst(unsynced.root, unsynced.levels) : nullptr;+        unsynced.firstOffset = other.unsynced.firstOffset;++        // done+        return *this;+    }++    /**+     * An assignment operation taking over ownership+     * from a r-value reference to a sparse array.+     */+    SparseArray& operator=(SparseArray&& other) {+        // clean this one+        clean();++        // harvest content+        unsynced.root = other.unsynced.root;+        unsynced.levels = other.unsynced.levels;+        unsynced.offset = other.unsynced.offset;+        unsynced.first = other.unsynced.first;+        unsynced.firstOffset = other.unsynced.firstOffset;++        // reset other+        other.unsynced.root = nullptr;+        other.unsynced.levels = 0;+        other.unsynced.first = nullptr;++        // done+        return *this;+    }++    /**+     * Tests whether this sparse array is empty, thus it only+     * contains default-values, or not.+     */+    bool empty() const {+        return unsynced.root == nullptr;+    }++    /**+     * Computes the number of non-empty elements within this+     * sparse array.+     */+    std::size_t size() const {+        // quick one for the empty map+        if (empty()) return 0;++        // count elements -- since maintaining is making inserts more expensive+        std::size_t res = 0;+        for (auto it = begin(); it != end(); ++it) {+            ++res;+        }+        return res;+    }++private:+    /**+     * Computes the memory usage of the given sub-tree.+     */+    static std::size_t getMemoryUsage(const Node* node, int level) {+        // support null-nodes+        if (!node) return 0;++        // add size of current node+        std::size_t res = sizeof(Node);++        // sum up memory usage of child nodes+        if (level > 0) {+            for (int i = 0; i < NUM_CELLS; i++) {+                res += getMemoryUsage(node->cell[i].ptr, level - 1);+            }+        }++        // done+        return res;+    }++public:+    /**+     * Computes the total memory usage of this data structure.+     */+    std::size_t getMemoryUsage() const {+        // the memory of the wrapper class+        std::size_t res = sizeof(*this);++        // add nodes+        if (unsynced.root) {+            res += getMemoryUsage(unsynced.root, unsynced.levels);+        }++        // done+        return res;+    }++    /**+     * Resets the content of this array to default values for each contained+     * element.+     */+    void clear() {+        clean();+        unsynced.root = nullptr;+        unsynced.levels = 0;+        unsynced.first = nullptr;+        unsynced.firstOffset = std::numeric_limits<index_type>::max();+    }++    /**+     * A struct to be utilized as a local, temporal context by client code+     * to speed up the execution of various operations (optional parameter).+     */+    struct op_context {+        index_type lastIndex{0};+        Node* lastNode{nullptr};+        op_context() = default;+    };++private:+    // ---------------------------------------------------------------------+    //              Optimistic Locking of Root-Level Infos+    // ---------------------------------------------------------------------++    /**+     * A struct to cover a snapshot of the root node state.+     */+    struct RootInfoSnapshot {+        // the current pointer to a root node+        Node* root;+        // the current number of levels+        uint32_t levels;+        // the current offset of the first theoretical element+        index_type offset;+        // a version number for the optimistic locking+        uintptr_t version;+    };++    /**+     * Obtains the current version of the root.+     */+    uint64_t getRootVersion() const {+        // here it is assumed that the load of a 64-bit word is atomic+        return (uint64_t)synced.root;+    }++    /**+     * Obtains a snapshot of the current root information.+     */+    RootInfoSnapshot getRootInfo() const {+        RootInfoSnapshot res{};+        do {+            // first take the mod counter+            do {+                // if res.mod % 2 == 1 .. there is an update in progress+                res.version = getRootVersion();+            } while (res.version % 2);++            // then the rest+            res.root = synced.root;+            res.levels = synced.levels;+            res.offset = synced.offset;++            // check consistency of obtained data (optimistic locking)+        } while (res.version != getRootVersion());++        // got a consistent snapshot+        return res;+    }++    /**+     * Updates the current root information based on the handed in modified+     * snapshot instance if the version number of the snapshot still corresponds+     * to the current version. Otherwise a concurrent update took place and the+     * operation is aborted.+     *+     * @param info the updated information to be assigned to the active root-info data+     * @return true if successfully updated, false if aborted+     */+    bool tryUpdateRootInfo(const RootInfoSnapshot& info) {+        // check mod counter+        uintptr_t version = info.version;++        // update root to invalid pointer (ending with 1)+        if (!__sync_bool_compare_and_swap(&synced.root, (Node*)version, (Node*)(version + 1))) {+            return false;+        }++        // conduct update+        synced.levels = info.levels;+        synced.offset = info.offset;++        // update root (and thus the version to enable future retrievals)+        __sync_synchronize();+        synced.root = info.root;++        // done+        return true;+    }++    /**+     * A struct summarizing the state of the first node reference.+     */+    struct FirstInfoSnapshot {+        // the pointer to the first node+        Node* node;+        // the offset of the first node+        index_type offset;+        // the version number of the first node (for the optimistic locking)+        uintptr_t version;+    };++    /**+     * Obtains the current version number of the first node information.+     */+    uint64_t getFirstVersion() const {+        // here it is assumed that the load of a 64-bit word is atomic+        return (uint64_t)synced.first;+    }++    /**+     * Obtains a snapshot of the current first-node information.+     */+    FirstInfoSnapshot getFirstInfo() const {+        FirstInfoSnapshot res{};+        do {+            // first take the version+            do {+                res.version = getFirstVersion();+            } while (res.version % 2);++            // collect the values+            res.node = synced.first;+            res.offset = synced.firstOffset;++        } while (res.version != getFirstVersion());++        // we got a consistent snapshot+        return res;+    }++    /**+     * Updates the information stored regarding the first node in a+     * concurrent setting utilizing a optimistic locking approach.+     * This is identical to the approach utilized for the root info.+     */+    bool tryUpdateFirstInfo(const FirstInfoSnapshot& info) {+        // check mod counter+        uintptr_t version = info.version;++        // temporary update first pointer to point to uneven value (lock-out)+        if (!__sync_bool_compare_and_swap(&synced.first, (Node*)version, (Node*)(version + 1))) {+            return false;+        }++        // conduct update+        synced.firstOffset = info.offset;++        // update node pointer (and thus the version number)+        __sync_synchronize();+        synced.first = info.node;  // must be last (and atomic)++        // done+        return true;+    }++public:+    /**+     * Obtains a mutable reference to the value addressed by the given index.+     *+     * @param i the index of the element to be addressed+     * @return a mutable reference to the corresponding element+     */+    value_type& get(index_type i) {+        op_context ctxt;+        return get(i, ctxt);+    }++    /**+     * Obtains a mutable reference to the value addressed by the given index.+     *+     * @param i the index of the element to be addressed+     * @param ctxt a operation context to exploit state-less temporal locality+     * @return a mutable reference to the corresponding element+     */+    value_type& get(index_type i, op_context& ctxt) {+        return getLeaf(i, ctxt).value;+    }++    /**+     * Obtains a mutable reference to the atomic value addressed by the given index.+     *+     * @param i the index of the element to be addressed+     * @return a mutable reference to the corresponding element+     */+    atomic_value_type& getAtomic(index_type i) {+        op_context ctxt;+        return getAtomic(i, ctxt);+    }++    /**+     * Obtains a mutable reference to the atomic value addressed by the given index.+     *+     * @param i the index of the element to be addressed+     * @param ctxt a operation context to exploit state-less temporal locality+     * @return a mutable reference to the corresponding element+     */+    atomic_value_type& getAtomic(index_type i, op_context& ctxt) {+        return getLeaf(i, ctxt).avalue;+    }++private:+    /**+     * An internal function capable of navigating to a given leaf node entry.+     * If the cell does not exist yet it will be created as a side-effect.+     *+     * @param i the index of the requested cell+     * @param ctxt a operation context to exploit state-less temporal locality+     * @return a reference to the requested cell+     */+    inline Cell& getLeaf(index_type i, op_context& ctxt) {+        // check context+        if (ctxt.lastNode && (ctxt.lastIndex == (i & ~INDEX_MASK))) {+            // return reference to referenced+            return ctxt.lastNode->cell[i & INDEX_MASK];+        }++        // get snapshot of root+        auto info = getRootInfo();++        // check for emptiness+        if (info.root == nullptr) {+            // build new root node+            info.root = newNode();++            // initialize the new node+            info.root->parent = nullptr;+            info.offset = i & ~(INDEX_MASK);++            // try updating root information atomically+            if (tryUpdateRootInfo(info)) {+                // success -- finish get call++                // update first+                auto firstInfo = getFirstInfo();+                while (info.offset < firstInfo.offset) {+                    firstInfo.node = info.root;+                    firstInfo.offset = info.offset;+                    if (!tryUpdateFirstInfo(firstInfo)) {+                        // there was some concurrent update => check again+                        firstInfo = getFirstInfo();+                    }+                }++                // return reference to proper cell+                return info.root->cell[i & INDEX_MASK];+            }++            // somebody else was faster => use standard insertion procedure+            delete info.root;++            // retrieve new root info+            info = getRootInfo();++            // make sure there is a root+            assert(info.root);+        }++        // for all other inserts+        //   - check boundary+        //   - navigate to node+        //   - insert value++        // check boundaries+        while (!inBoundaries(i, info.levels, info.offset)) {+            // boundaries need to be expanded by growing upwards+            raiseLevel(info);  // try raising level unless someone else did already+            // update root info+            info = getRootInfo();+        }++        // navigate to node+        Node* node = info.root;+        unsigned level = info.levels;+        while (level != 0) {+            // get X coordinate+            auto x = getIndex(brie_element_type(i), level);++            // decrease level counter+            --level;++            // check next node+            std::atomic<Node*>& aNext = node->cell[x].aptr;+            Node* next = aNext;+            if (!next) {+                // create new sub-tree+                Node* newNext = newNode();+                newNext->parent = node;++                // try to update next+                if (!aNext.compare_exchange_strong(next, newNext)) {+                    // some other thread was faster => use updated next+                    delete newNext;+                } else {+                    // the locally created next is the new next+                    next = newNext;++                    // update first+                    if (level == 0) {+                        // compute offset of this node+                        auto off = i & ~INDEX_MASK;++                        // fast over-approximation of whether a update is necessary+                        if (off < unsynced.firstOffset) {+                            // update first reference if this one is the smallest+                            auto first_info = getFirstInfo();+                            while (off < first_info.offset) {+                                first_info.node = next;+                                first_info.offset = off;+                                if (!tryUpdateFirstInfo(first_info)) {+                                    // there was some concurrent update => check again+                                    first_info = getFirstInfo();+                                }+                            }+                        }+                    }+                }++                // now next should be defined+                assert(next);+            }++            // continue one level below+            node = next;+        }++        // update context+        ctxt.lastIndex = (i & ~INDEX_MASK);+        ctxt.lastNode = node;++        // return reference to cell+        return node->cell[i & INDEX_MASK];+    }++public:+    /**+     * Updates the value stored in cell i by the given value.+     */+    void update(index_type i, const value_type& val) {+        op_context ctxt;+        update(i, val, ctxt);+    }++    /**+     * Updates the value stored in cell i by the given value. A operation+     * context can be provided for exploiting temporal locality.+     */+    void update(index_type i, const value_type& val, op_context& ctxt) {+        get(i, ctxt) = val;+    }++    /**+     * Obtains the value associated to index i -- which might be+     * the default value of the covered type if the value hasn't been+     * defined previously.+     */+    value_type operator[](index_type i) const {+        return lookup(i);+    }++    /**+     * Obtains the value associated to index i -- which might be+     * the default value of the covered type if the value hasn't been+     * defined previously.+     */+    value_type lookup(index_type i) const {+        op_context ctxt;+        return lookup(i, ctxt);+    }++    /**+     * Obtains the value associated to index i -- which might be+     * the default value of the covered type if the value hasn't been+     * defined previously. A operation context can be provided for+     * exploiting temporal locality.+     */+    value_type lookup(index_type i, op_context& ctxt) const {+        // check whether it is empty+        if (!unsynced.root) return default_factory<value_type>()();++        // check boundaries+        if (!inBoundaries(i)) return default_factory<value_type>()();++        // check context+        if (ctxt.lastNode && ctxt.lastIndex == (i & ~INDEX_MASK)) {+            return ctxt.lastNode->cell[i & INDEX_MASK].value;+        }++        // navigate to value+        Node* node = unsynced.root;+        unsigned level = unsynced.levels;+        while (level != 0) {+            // get X coordinate+            auto x = getIndex(brie_element_type(i), level);++            // decrease level counter+            --level;++            // check next node+            Node* next = node->cell[x].ptr;++            // check next step+            if (!next) return default_factory<value_type>()();++            // continue one level below+            node = next;+        }++        // remember context+        ctxt.lastIndex = (i & ~INDEX_MASK);+        ctxt.lastNode = node;++        // return reference to cell+        return node->cell[i & INDEX_MASK].value;+    }++private:+    /**+     * A static operation utilized internally for merging sub-trees recursively.+     *+     * @param parent the parent node of the current merge operation+     * @param trg a reference to the pointer the cloned node should be stored to+     * @param src the node to be cloned+     * @param levels the height of the cloned node+     */+    static void merge(const Node* parent, Node*& trg, const Node* src, int levels) {+        // if other side is null => done+        if (src == nullptr) {+            return;+        }++        // if the trg sub-tree is empty, clone the corresponding branch+        if (trg == nullptr) {+            trg = clone(src, levels);+            if (trg != nullptr) {+                trg->parent = parent;+            }+            return;  // done+        }++        // otherwise merge recursively++        // the leaf-node step+        if (levels == 0) {+            merge_op merg;+            for (int i = 0; i < NUM_CELLS; ++i) {+                trg->cell[i].value = merg(trg->cell[i].value, src->cell[i].value);+            }+            return;+        }++        // the recursive step+        for (int i = 0; i < NUM_CELLS; ++i) {+            merge(trg, trg->cell[i].ptr, src->cell[i].ptr, levels - 1);+        }+    }++public:+    /**+     * Adds all the values stored in the given array to this array.+     */+    void addAll(const SparseArray& other) {+        // skip if other is empty+        if (other.empty()) {+            return;+        }++        // special case: emptiness+        if (empty()) {+            // use assignment operator+            *this = other;+            return;+        }++        // adjust levels+        while (unsynced.levels < other.unsynced.levels || !inBoundaries(other.unsynced.offset)) {+            raiseLevel();+        }++        // navigate to root node equivalent of the other node in this tree+        auto level = unsynced.levels;+        Node** node = &unsynced.root;+        while (level > other.unsynced.levels) {+            // get X coordinate+            auto x = getIndex(brie_element_type(other.unsynced.offset), level);++            // decrease level counter+            --level;++            // check next node+            Node*& next = (*node)->cell[x].ptr;+            if (!next) {+                // create new sub-tree+                next = newNode();+                next->parent = *node;+            }++            // continue one level below+            node = &next;+        }++        // merge sub-branches from here+        merge((*node)->parent, *node, other.unsynced.root, level);++        // update first+        if (unsynced.firstOffset > other.unsynced.firstOffset) {+            unsynced.first = findFirst(*node, level);+            unsynced.firstOffset = other.unsynced.firstOffset;+        }+    }++    // ---------------------------------------------------------------------+    //                           Iterator+    // ---------------------------------------------------------------------++    using iterator = SparseArrayIter<this_t>;++    /**+     * Obtains an iterator referencing the first non-default element or end in+     * case there are no such elements.+     */+    iterator begin() const {+        return iterator(unsynced.first, unsynced.firstOffset);+    }++    /**+     * An iterator referencing the position after the last non-default element.+     */+    iterator end() const {+        return iterator();+    }++    /**+     * An operation to obtain an iterator referencing an element addressed by the+     * given index. If the corresponding element is a non-default value, a corresponding+     * iterator will be returned. Otherwise end() will be returned.+     */+    iterator find(index_type i) const {+        op_context ctxt;+        return find(i, ctxt);+    }++    /**+     * An operation to obtain an iterator referencing an element addressed by the+     * given index. If the corresponding element is a non-default value, a corresponding+     * iterator will be returned. Otherwise end() will be returned. A operation context+     * can be provided for exploiting temporal locality.+     */+    iterator find(index_type i, op_context& ctxt) const {+        // check whether it is empty+        if (!unsynced.root) return end();++        // check boundaries+        if (!inBoundaries(i)) return end();++        // check context+        if (ctxt.lastNode && ctxt.lastIndex == (i & ~INDEX_MASK)) {+            Node* node = ctxt.lastNode;++            // check whether there is a proper entry+            value_type value = node->cell[i & INDEX_MASK].value;+            if (value == value_type{}) {+                return end();+            }+            // return iterator pointing to value+            return iterator(node, std::make_pair(i, value));+        }++        // navigate to value+        Node* node = unsynced.root;+        unsigned level = unsynced.levels;+        while (level != 0) {+            // get X coordinate+            auto x = getIndex(i, level);++            // decrease level counter+            --level;++            // check next node+            Node* next = node->cell[x].ptr;++            // check next step+            if (!next) return end();++            // continue one level below+            node = next;+        }++        // register in context+        ctxt.lastNode = node;+        ctxt.lastIndex = (i & ~INDEX_MASK);++        // check whether there is a proper entry+        value_type value = node->cell[i & INDEX_MASK].value;+        if (value == value_type{}) {+            return end();+        }++        // return iterator pointing to cell+        return iterator(node, std::make_pair(i, value));+    }++    /**+     * An operation obtaining the smallest non-default element such that it's index is >=+     * the given index.+     */+    iterator lowerBound(index_type i) const {+        op_context ctxt;+        return lowerBound(i, ctxt);+    }++    /**+     * An operation obtaining the smallest non-default element such that it's index is >=+     * the given index. A operation context can be provided for exploiting temporal locality.+     */+    iterator lowerBound(index_type i, op_context&) const {+        // check whether it is empty+        if (!unsynced.root) return end();++        // check boundaries+        if (!inBoundaries(i)) {+            // if it is on the lower end, return minimum result+            if (i < unsynced.offset) {+                const auto& value = unsynced.first->cell[0].value;+                auto res = iterator(unsynced.first, std::make_pair(unsynced.offset, value));+                if (value == value_type()) {+                    ++res;+                }+                return res;+            }+            // otherwise it is on the high end, return end iterator+            return end();+        }++        // navigate to value+        Node* node = unsynced.root;+        unsigned level = unsynced.levels;+        while (true) {+            // get X coordinate+            auto x = getIndex(brie_element_type(i), level);++            // check next node+            Node* next = node->cell[x].ptr;++            // check next step+            if (!next) {+                if (x == NUM_CELLS - 1) {+                    ++level;+                    node = const_cast<Node*>(node->parent);+                    if (!node) return end();+                }++                // continue search+                i = i & getLevelMask(level);++                // find next higher value+                i += 1ull << (BITS * level);++            } else {+                if (level == 0) {+                    // found boundary+                    return iterator(node, std::make_pair(i, node->cell[x].value));+                }++                // decrease level counter+                --level;++                // continue one level below+                node = next;+            }+        }+    }++    /**+     * An operation obtaining the smallest non-default element such that it's index is greater+     * the given index.+     */+    iterator upperBound(index_type i) const {+        op_context ctxt;+        return upperBound(i, ctxt);+    }++    /**+     * An operation obtaining the smallest non-default element such that it's index is greater+     * the given index. A operation context can be provided for exploiting temporal locality.+     */+    iterator upperBound(index_type i, op_context& ctxt) const {+        if (i == std::numeric_limits<index_type>::max()) {+            return end();+        }+        return lowerBound(i + 1, ctxt);+    }++private:+    /**+     * An internal debug utility printing the internal structure of this sparse array to the given output+     * stream.+     */+    void dump(bool detailed, std::ostream& out, const Node& node, int level, index_type offset,+            int indent = 0) const {+        auto x = getIndex(offset, level + 1);+        out << times("\t", indent) << x << ": Node " << &node << " on level " << level+            << " parent: " << node.parent << " -- range: " << offset << " - "+            << (offset + ~getLevelMask(level + 1)) << "\n";++        if (level == 0) {+            for (int i = 0; i < NUM_CELLS; i++) {+                if (detailed || node.cell[i].value != value_type()) {+                    out << times("\t", indent + 1) << i << ": [" << (offset + i) << "] " << node.cell[i].value+                        << "\n";+                }+            }+        } else {+            for (int i = 0; i < NUM_CELLS; i++) {+                if (node.cell[i].ptr) {+                    dump(detailed, out, *node.cell[i].ptr, level - 1,+                            offset + (i * (index_type(1) << (level * BIT_PER_STEP))), indent + 1);+                } else if (detailed) {+                    auto low = offset + (i * (1 << (level * BIT_PER_STEP)));+                    auto hig = low + ~getLevelMask(level);+                    out << times("\t", indent + 1) << i << ": empty range " << low << " - " << hig << "\n";+                }+            }+        }+        out << "\n";+    }++public:+    /**+     * A debug utility printing the internal structure of this sparse array to the given output stream.+     */+    void dump(bool detail = false, std::ostream& out = std::cout) const {+        if (!unsynced.root) {+            out << " - empty - \n";+            return;+        }+        out << "root:  " << unsynced.root << "\n";+        out << "offset: " << unsynced.offset << "\n";+        out << "first: " << unsynced.first << "\n";+        out << "fist offset: " << unsynced.firstOffset << "\n";+        dump(detail, out, *unsynced.root, unsynced.levels, unsynced.offset);+    }++private:+    // --------------------------------------------------------------------------+    //                                 Utilities+    // --------------------------------------------------------------------------++    /**+     * Creates new nodes and initializes them with 0.+     */+    static Node* newNode() {+        return new Node();+    }++    /**+     * Destroys a node and all its sub-nodes recursively.+     */+    static void freeNodes(Node* node, int level) {+        if (!node) return;+        if (level != 0) {+            for (int i = 0; i < NUM_CELLS; i++) {+                freeNodes(node->cell[i].ptr, level - 1);+            }+        }+        delete node;+    }++    /**+     * Conducts a cleanup of the internal tree structure.+     */+    void clean() {+        freeNodes(unsynced.root, unsynced.levels);+        unsynced.root = nullptr;+        unsynced.levels = 0;+    }++    /**+     * Clones the given node and all its sub-nodes.+     */+    static Node* clone(const Node* node, int level) {+        // support null-pointers+        if (node == nullptr) {+            return nullptr;+        }++        // create a clone+        auto* res = new Node();++        // handle leaf level+        if (level == 0) {+            copy_op copy;+            for (int i = 0; i < NUM_CELLS; i++) {+                res->cell[i].value = copy(node->cell[i].value);+            }+            return res;+        }++        // for inner nodes clone each child+        for (int i = 0; i < NUM_CELLS; i++) {+            auto cur = clone(node->cell[i].ptr, level - 1);+            if (cur != nullptr) {+                cur->parent = res;+            }+            res->cell[i].ptr = cur;+        }++        // done+        return res;+    }++    /**+     * Obtains the left-most leaf-node of the tree rooted by the given node+     * with the given level.+     */+    static Node* findFirst(Node* node, int level) {+        while (level > 0) {+            [[maybe_unused]] bool found = false;+            for (int i = 0; i < NUM_CELLS; i++) {+                Node* cur = node->cell[i].ptr;+                if (cur) {+                    node = cur;+                    --level;+                    found = true;+                    break;+                }+            }+            assert(found && "No first node!");+        }++        return node;+    }++    /**+     * Raises the level of this tree by one level. It does so by introducing+     * a new root node and inserting the current root node as a child node.+     */+    void raiseLevel() {+        // something went wrong when we pass that line+        assert(unsynced.levels < (sizeof(index_type) * 8 / BITS) + 1);++        // create new root+        Node* node = newNode();+        node->parent = nullptr;++        // insert existing root as child+        auto x = getIndex(brie_element_type(unsynced.offset), unsynced.levels + 1);+        node->cell[x].ptr = unsynced.root;++        // swap the root+        unsynced.root->parent = node;++        // update root+        unsynced.root = node;+        ++unsynced.levels;++        // update offset be removing additional bits+        unsynced.offset &= getLevelMask(unsynced.levels + 1);+    }++    /**+     * Attempts to raise the height of this tree based on the given root node+     * information and updates the root-info snapshot correspondingly.+     */+    void raiseLevel(RootInfoSnapshot& info) {+        // something went wrong when we pass that line+        assert(info.levels < (sizeof(index_type) * 8 / BITS) + 1);++        // create new root+        Node* newRoot = newNode();+        newRoot->parent = nullptr;++        // insert existing root as child+        auto x = getIndex(brie_element_type(info.offset), info.levels + 1);+        newRoot->cell[x].ptr = info.root;++        // exchange the root in the info struct+        auto oldRoot = info.root;+        info.root = newRoot;++        // update level counter+        ++info.levels;++        // update offset+        info.offset &= getLevelMask(info.levels + 1);++        // try exchanging root info+        if (tryUpdateRootInfo(info)) {+            // success => final step, update parent of old root+            oldRoot->parent = info.root;+        } else {+            // throw away temporary new node+            delete newRoot;+        }+    }++    /**+     * Tests whether the given index is covered by the boundaries defined+     * by the hight and offset of the internally maintained tree.+     */+    bool inBoundaries(index_type a) const {+        return inBoundaries(a, unsynced.levels, unsynced.offset);+    }++    /**+     * Tests whether the given index is within the boundaries defined by the+     * given tree hight and offset.+     */+    static bool inBoundaries(index_type a, uint32_t levels, index_type offset) {+        auto mask = getLevelMask(levels + 1);+        return (a & mask) == offset;+    }++    /**+     * Obtains the index within the arrays of cells of a given index on a given+     * level of the internally maintained tree.+     */+    static index_type getIndex(brie_element_type a, unsigned level) {+        return (a & (INDEX_MASK << (level * BIT_PER_STEP))) >> (level * BIT_PER_STEP);+    }++    /**+     * Computes the bit-mask to be applicable to obtain the offset of a node on a+     * given tree level.+     */+    static index_type getLevelMask(unsigned level) {+        if (level > (sizeof(index_type) * 8 / BITS)) return 0;+        return (~(index_type(0)) << (level * BIT_PER_STEP));+    }+};++namespace detail::brie {++/**+ * Iterator type for `souffle::SparseArray`. It enumerates the indices set to 1.+ */+template <typename SparseBitMap>+class SparseBitMapIter {+    using value_t = typename SparseBitMap::value_t;+    using value_type = typename SparseBitMap::index_type;+    using data_store_t = typename SparseBitMap::data_store_t;+    using nested_iterator = typename data_store_t::iterator;++    // the iterator through the underlying sparse data structure+    nested_iterator iter;++    // the currently consumed mask+    uint64_t mask = 0;++    // the value currently pointed to+    value_type value{};++public:+    SparseBitMapIter() = default;  // default constructor -- creating an end-iterator+    SparseBitMapIter(const SparseBitMapIter&) = default;+    SparseBitMapIter& operator=(const SparseBitMapIter&) = default;++    SparseBitMapIter(const nested_iterator& iter)+            : iter(iter), mask(SparseBitMap::toMask(iter->second)),+              value(iter->first << SparseBitMap::LEAF_INDEX_WIDTH) {+        moveToNextInMask();+    }++    SparseBitMapIter(const nested_iterator& iter, uint64_t m, value_type value)+            : iter(iter), mask(m), value(value) {}++    // the equality operator as required by the iterator concept+    bool operator==(const SparseBitMapIter& other) const {+        // only equivalent if pointing to the end+        return iter == other.iter && mask == other.mask;+    }++    // the not-equality operator as required by the iterator concept+    bool operator!=(const SparseBitMapIter& other) const {+        return !(*this == other);+    }++    // the deref operator as required by the iterator concept+    const value_type& operator*() const {+        return value;+    }++    // support for the pointer operator+    const value_type* operator->() const {+        return &value;+    }++    // the increment operator as required by the iterator concept+    SparseBitMapIter& operator++() {+        // progress in current mask+        if (moveToNextInMask()) return *this;++        // go to next entry+        ++iter;++        // update value+        if (!iter.isEnd()) {+            value = iter->first << SparseBitMap::LEAF_INDEX_WIDTH;+            mask = SparseBitMap::toMask(iter->second);+            moveToNextInMask();+        }++        // done+        return *this;+    }++    SparseBitMapIter operator++(int) {+        auto cpy = *this;+        ++(*this);+        return cpy;+    }++    bool isEnd() const {+        return iter.isEnd();+    }++    void print(std::ostream& out) const {+        out << "SparseBitMapIter(" << iter << " -> " << std::bitset<64>(mask) << " @ " << value << ")";+    }++    // enables this iterator core to be printed (for debugging)+    friend std::ostream& operator<<(std::ostream& out, const SparseBitMapIter& iter) {+        iter.print(out);+        return out;+    }++private:+    bool moveToNextInMask() {+        // check if there is something left+        if (mask == 0) return false;++        // get position of leading 1+        auto pos = __builtin_ctzll(mask);++        // consume this bit+        mask &= ~(1llu << pos);++        // update value+        value &= ~SparseBitMap::LEAF_INDEX_MASK;+        value |= pos;++        // done+        return true;+    }+};++}  // namespace detail::brie++/**+ * A sparse bit-map is a bit map virtually assigning a bit value to every value if the+ * uint64_t domain. However, only 1-bits are stored utilizing a nested sparse array+ * structure.+ *+ * @tparam BITS similar to the BITS parameter of the sparse array type+ */+template <unsigned BITS = 4>+class SparseBitMap {+    template <typename A>+    friend class detail::brie::SparseBitMapIter;++    using this_t = SparseBitMap<BITS>;++    // the element type stored in the nested sparse array+    using value_t = uint64_t;++    // define the bit-level merge operation+    struct merge_op {+        value_t operator()(value_t a, value_t b) const {+            return a | b;  // merging bit masks => bitwise or operation+        }+    };++    // the type of the internal data store+    using data_store_t = SparseArray<value_t, BITS, merge_op>;+    using atomic_value_t = typename data_store_t::atomic_value_type;++    // some constants for manipulating stored values+    static constexpr std::size_t BITS_PER_ENTRY = sizeof(value_t) * CHAR_BIT;+    static constexpr std::size_t LEAF_INDEX_WIDTH = __builtin_ctz(static_cast<unsigned long>(BITS_PER_ENTRY));+    static constexpr uint64_t LEAF_INDEX_MASK = BITS_PER_ENTRY - 1;++    static uint64_t toMask(const value_t& value) {+        static_assert(sizeof(value_t) == sizeof(uint64_t), "Fixed for 64-bit compiler.");+        return reinterpret_cast<const uint64_t&>(value);+    }++public:+    // the type to address individual entries+    using index_type = typename data_store_t::index_type;++private:+    // it utilizes a sparse map to store its data+    data_store_t store;++public:+    // a simple default constructor+    SparseBitMap() = default;++    // a default copy constructor+    SparseBitMap(const SparseBitMap&) = default;++    // a default r-value copy constructor+    SparseBitMap(SparseBitMap&&) = default;++    // a default assignment operator+    SparseBitMap& operator=(const SparseBitMap&) = default;++    // a default r-value assignment operator+    SparseBitMap& operator=(SparseBitMap&&) = default;++    // checks whether this bit-map is empty -- thus it does not have any 1-entries+    bool empty() const {+        return store.empty();+    }++    // the type utilized for recording context information for exploiting temporal locality+    using op_context = typename data_store_t::op_context;++    /**+     * Sets the bit addressed by i to 1.+     */+    bool set(index_type i) {+        op_context ctxt;+        return set(i, ctxt);+    }++    /**+     * Sets the bit addressed by i to 1. A context for exploiting temporal locality+     * can be provided.+     */+    bool set(index_type i, op_context& ctxt) {+        atomic_value_t& val = store.getAtomic(i >> LEAF_INDEX_WIDTH, ctxt);+        value_t bit = (1ull << (i & LEAF_INDEX_MASK));++#ifdef __GNUC__+#if __GNUC__ >= 7+        // In GCC >= 7 the usage of fetch_or causes a bug that needs further investigation+        // For now, this two-instruction based implementation provides a fix that does+        // not sacrifice too much performance.++        while (true) {+            auto order = std::memory_order::memory_order_relaxed;++            // load current value+            value_t old = val.load(order);++            // if bit is already set => we are done+            if (old & bit) return false;++            // set the bit, if failed, repeat+            if (!val.compare_exchange_strong(old, old | bit, order, order)) continue;++            // it worked, new bit added+            return true;+        }++#endif+#endif++        value_t old = val.fetch_or(bit, std::memory_order::memory_order_relaxed);+        return (old & bit) == 0u;+    }++    /**+     * Determines the whether the bit addressed by i is set or not.+     */+    bool test(index_type i) const {+        op_context ctxt;+        return test(i, ctxt);+    }++    /**+     * Determines the whether the bit addressed by i is set or not. A context for+     * exploiting temporal locality can be provided.+     */+    bool test(index_type i, op_context& ctxt) const {+        value_t bit = (1ull << (i & LEAF_INDEX_MASK));+        return store.lookup(i >> LEAF_INDEX_WIDTH, ctxt) & bit;+    }++    /**+     * Determines the whether the bit addressed by i is set or not.+     */+    bool operator[](index_type i) const {+        return test(i);+    }++    /**+     * Resets all contained bits to 0.+     */+    void clear() {+        store.clear();+    }++    /**+     * Determines the number of bits set.+     */+    std::size_t size() const {+        // this is computed on demand to keep the set operation simple.+        std::size_t res = 0;+        for (const auto& cur : store) {+            res += __builtin_popcountll(cur.second);+        }+        return res;+    }++    /**+     * Computes the total memory usage of this data structure.+     */+    std::size_t getMemoryUsage() const {+        // compute the total memory usage+        return sizeof(*this) - sizeof(data_store_t) + store.getMemoryUsage();+    }++    /**+     * Sets all bits set in other to 1 within this bit map.+     */+    void addAll(const SparseBitMap& other) {+        // nothing to do if it is a self-assignment+        if (this == &other) return;++        // merge the sparse store+        store.addAll(other.store);+    }++    // ---------------------------------------------------------------------+    //                           Iterator+    // ---------------------------------------------------------------------++    using iterator = SparseBitMapIter<this_t>;++    /**+     * Obtains an iterator pointing to the first index set to 1. If there+     * is no such bit, end() will be returned.+     */+    iterator begin() const {+        auto it = store.begin();+        if (it.isEnd()) return end();+        return iterator(it);+    }++    /**+     * Returns an iterator referencing the position after the last set bit.+     */+    iterator end() const {+        return iterator();+    }++    /**+     * Obtains an iterator referencing the position i if the corresponding+     * bit is set, end() otherwise.+     */+    iterator find(index_type i) const {+        op_context ctxt;+        return find(i, ctxt);+    }++    /**+     * Obtains an iterator referencing the position i if the corresponding+     * bit is set, end() otherwise. An operation context can be provided+     * to exploit temporal locality.+     */+    iterator find(index_type i, op_context& ctxt) const {+        // check prefix part+        auto it = store.find(i >> LEAF_INDEX_WIDTH, ctxt);+        if (it.isEnd()) return end();++        // check bit-set part+        uint64_t mask = toMask(it->second);+        if (!(mask & (1llu << (i & LEAF_INDEX_MASK)))) return end();++        // OK, it is there => create iterator+        mask &= ((1ull << (i & LEAF_INDEX_MASK)) - 1);  // remove all bits before pos i+        return iterator(it, mask, i);+    }++    /**+     * Locates an iterator to the first element in this sparse bit map not less+     * than the given index.+     */+    iterator lower_bound(index_type i) const {+        auto it = store.lowerBound(i >> LEAF_INDEX_WIDTH);+        if (it.isEnd()) return end();++        // check bit-set part+        uint64_t mask = toMask(it->second);++        // if there is no bit remaining in this mask, check next mask.+        if (!(mask & ((~uint64_t(0)) << (i & LEAF_INDEX_MASK)))) {+            index_type next = ((i >> LEAF_INDEX_WIDTH) + 1) << LEAF_INDEX_WIDTH;+            if (next < i) return end();+            return lower_bound(next);+        }++        // there are bits left, use least significant bit of those+        if (it->first == i >> LEAF_INDEX_WIDTH) {+            mask &= ((~uint64_t(0)) << (i & LEAF_INDEX_MASK));  // remove all bits before pos i+        }++        // compute value represented by least significant bit+        index_type pos = __builtin_ctzll(mask);++        // remove this bit as well+        mask = mask & ~(1ull << pos);++        // construct value of this located bit+        index_type val = (it->first << LEAF_INDEX_WIDTH) | pos;+        return iterator(it, mask, val);+    }++    /**+     * Locates an iterator to the first element in this sparse bit map than is greater+     * than the given index.+     */+    iterator upper_bound(index_type i) const {+        if (i == std::numeric_limits<index_type>::max()) {+            return end();+        }+        return lower_bound(i + 1);+    }++    /**+     * A debugging utility printing the internal structure of this map to the+     * given output stream.+     */+    void dump(bool detail = false, std::ostream& out = std::cout) const {+        store.dump(detail, out);+    }++    /**+     * Provides write-protected access to the internal store for running+     * analysis on the data structure.+     */+    const data_store_t& getStore() const {+        return store;+    }+};++// ---------------------------------------------------------------------+//                              TRIE+// ---------------------------------------------------------------------++namespace detail::brie {++/**+ * An iterator over the stored entries.+ *+ * Iterators for tries consist of a top-level iterator maintaining the+ * master copy of a materialized tuple and a recursively nested iterator+ * core -- one for each nested trie level.+ */+template <typename Value, typename IterCore>+class TrieIterator {+    template <unsigned Len, unsigned Pos, unsigned Dimensions>+    friend struct fix_binding;++    template <unsigned Dimensions>+    friend struct fix_lower_bound;++    template <unsigned Dimensions>+    friend struct fix_upper_bound;++    template <unsigned Pos, unsigned Dimensions>+    friend struct fix_first;++    template <unsigned Dimensions>+    friend struct fix_first_nested;++    template <typename A, typename B>+    friend class TrieIterator;++    // remove ref-qual (if any); this can happen if we're a iterator-view+    using iter_core_arg_type = typename std::remove_reference_t<IterCore>::store_iter;++    Value value = {};         // the value currently pointed to+    IterCore iter_core = {};  // the wrapped iterator++    // return an ephemeral nested iterator-view (view -> mutating us mutates our parent)+    // NB: be careful that the lifetime of this iterator-view doesn't exceed that of its parent.+    auto getNestedView() {+        auto& nested_iter_ref = iter_core.getNested();  // by ref (this is critical, we're a view, not a copy)+        auto nested_val = tail(value);+        return TrieIterator<decltype(nested_val), decltype(nested_iter_ref)>(+                std::move(nested_val), nested_iter_ref);+    }++    // special constructor for iterator-views (see `getNestedView`)+    explicit TrieIterator(Value value, IterCore iter_core) : value(std::move(value)), iter_core(iter_core) {}++public:+    TrieIterator() = default;  // default constructor -- creating an end-iterator+    TrieIterator(const TrieIterator&) = default;+    TrieIterator(TrieIterator&&) = default;+    TrieIterator& operator=(const TrieIterator&) = default;+    TrieIterator& operator=(TrieIterator&&) = default;++    explicit TrieIterator(iter_core_arg_type param) : iter_core(std::move(param), value) {}++    // the equality operator as required by the iterator concept+    bool operator==(const TrieIterator& other) const {+        // equivalent if pointing to the same value+        return iter_core == other.iter_core;+    }++    // the not-equality operator as required by the iterator concept+    bool operator!=(const TrieIterator& other) const {+        return !(*this == other);+    }++    const Value& operator*() const {+        return value;+    }++    const Value* operator->() const {+        return &value;+    }++    TrieIterator& operator++() {+        iter_core.inc(value);+        return *this;+    }++    TrieIterator operator++(int) {+        auto cpy = *this;+        ++(*this);+        return cpy;+    }++    // enables this iterator to be printed (for debugging)+    void print(std::ostream& out) const {+        out << "iter(" << iter_core << " -> " << value << ")";+    }++    friend std::ostream& operator<<(std::ostream& out, const TrieIterator& iter) {+        iter.print(out);+        return out;+    }+};++template <unsigned Dim>+struct TrieTypes;++/**+ * A base class for the Trie implementation allowing various+ * specializations of the Trie template to inherit common functionality.+ *+ * @tparam Dim the number of dimensions / arity of the stored tuples+ * @tparam Derived the type derived from this base class+ */+template <unsigned Dim, typename Derived>+class TrieBase {+    Derived& impl() {+        return static_cast<Derived&>(*this);+    }++    const Derived& impl() const {+        return static_cast<const Derived&>(*this);+    }++protected:+    using types = TrieTypes<Dim>;+    using store_type = typename types::store_type;++    store_type store;++public:+    using const_entry_span_type = typename types::const_entry_span_type;+    using entry_span_type = typename types::entry_span_type;+    using entry_type = typename types::entry_type;+    using iterator = typename types::iterator;+    using iterator_core = typename types::iterator_core;+    using op_context = typename types::op_context;++    /**+     * Inserts all tuples stored within the given trie into this trie.+     * This operation is considerably more efficient than the consecutive+     * insertion of the elements in other into this trie.+     *+     * @param other the elements to be inserted into this trie+     */+    void insertAll(const TrieBase& other) {+        store.addAll(other.store);+    }++    /**+     * Provides protected access to the internally maintained store.+     */+    const store_type& getStore() const {+        return store;+    }++    /**+     * Determines whether this trie is empty or not.+     */+    bool empty() const {+        return store.empty();+    }++    /**+     * Obtains an iterator referencing the first element stored within this trie.+     */+    iterator begin() const {+        return empty() ? end() : iterator(store.begin());+    }++    /**+     * Obtains an iterator referencing the position after the last element stored+     * within this trie.+     */+    iterator end() const {+        return iterator();+    }++    iterator find(const_entry_span_type entry, op_context& ctxt) const {+        auto range = impl().template getBoundaries<Dim>(entry, ctxt);+        return range.empty() ? range.end() : range.begin();+    }++    // implemented by `Derived`:+    //      bool insert(const entry_type& tuple, op_context& ctxt);+    //      bool contains(const_entry_span_type tuple, op_context& ctxt) const;+    //      bool lower_bound(const_entry_span_type tuple, op_context& ctxt) const;+    //      bool upper_bound(const_entry_span_type tuple, op_context& ctxt) const;+    //      template <unsigned levels>+    //      range<iterator> getBoundaries(const_entry_span_type, op_context&) const;++    // -- operation wrappers --++    template <unsigned levels>+    range<iterator> getBoundaries(const_entry_span_type entry) const {+        op_context ctxt;+        return impl().template getBoundaries<levels>(entry, ctxt);+    }++    template <unsigned levels>+    range<iterator> getBoundaries(const entry_type& entry, op_context& ctxt) const {+        return impl().template getBoundaries<levels>(const_entry_span_type(entry), ctxt);+    }++    template <unsigned levels>+    range<iterator> getBoundaries(const entry_type& entry) const {+        return impl().template getBoundaries<levels>(const_entry_span_type(entry));+    }++    template <unsigned levels, typename... Values, typename = std::enable_if_t<(isRamType<Values> && ...)>>+    range<iterator> getBoundaries(Values... values) const {+        return impl().template getBoundaries<levels>(entry_type{ramBitCast(values)...});+    }++// declare a initialiser-list compatible overload for a given function+#define BRIE_OVERLOAD_INIT_LIST(fn, constness)                     \+    auto fn(const_entry_span_type entry) constness {               \+        op_context ctxt;                                           \+        return impl().fn(entry, ctxt);                             \+    }                                                              \+    auto fn(const entry_type& entry, op_context& ctxt) constness { \+        return impl().fn(const_entry_span_type(entry), ctxt);      \+    }                                                              \+    auto fn(const entry_type& entry) constness {                   \+        return impl().fn(const_entry_span_type(entry));            \+    }++    BRIE_OVERLOAD_INIT_LIST(insert, )+    BRIE_OVERLOAD_INIT_LIST(find, const)+    BRIE_OVERLOAD_INIT_LIST(contains, const)+    BRIE_OVERLOAD_INIT_LIST(lower_bound, const)+    BRIE_OVERLOAD_INIT_LIST(upper_bound, const)++#undef BRIE_OVERLOAD_INIT_LIST++    /* -------------- operator hint statistics ----------------- */++    // an aggregation of statistical values of the hint utilization+    struct hint_statistics {+        // the counter for insertion operations+        CacheAccessCounter inserts;++        // the counter for contains operations+        CacheAccessCounter contains;++        // the counter for get_boundaries operations+        CacheAccessCounter get_boundaries;+    };++protected:+    // the hint statistic of this b-tree instance+    mutable hint_statistics hint_stats;++public:+    void printStats(std::ostream& out) const {+        out << "---------------------------------\n";+        out << "  insert-hint (hits/misses/total): " << hint_stats.inserts.getHits() << "/"+            << hint_stats.inserts.getMisses() << "/" << hint_stats.inserts.getAccesses() << "\n";+        out << "  contains-hint (hits/misses/total):" << hint_stats.contains.getHits() << "/"+            << hint_stats.contains.getMisses() << "/" << hint_stats.contains.getAccesses() << "\n";+        out << "  get-boundaries-hint (hits/misses/total):" << hint_stats.get_boundaries.getHits() << "/"+            << hint_stats.get_boundaries.getMisses() << "/" << hint_stats.get_boundaries.getAccesses()+            << "\n";+        out << "---------------------------------\n";+    }+};++template <unsigned Dim>+struct TrieTypes;++// FIXME: THIS KILLS COMPILE PERF - O(n^2)+/**+ * A functor extracting a reference to a nested iterator core from an enclosing+ * iterator core.+ */+template <unsigned Level>+struct get_nested_iter_core {+    template <typename IterCore>+    auto operator()(IterCore& core) -> decltype(get_nested_iter_core<Level - 1>()(core.getNested())) {+        return get_nested_iter_core<Level - 1>()(core.getNested());+    }+};++template <>+struct get_nested_iter_core<0> {+    template <typename IterCore>+    IterCore& operator()(IterCore& core) {+        return core;+    }+};++// FIXME: THIS KILLS COMPILE PERF - O(n^2)+/**+ * A functor initializing an iterator upon creation to reference the first+ * element in the associated Trie.+ */+template <unsigned Pos, unsigned Dim>+struct fix_first {+    template <unsigned bits, typename iterator>+    void operator()(const SparseBitMap<bits>& store, iterator& iter) const {+        // set iterator to first in store+        auto first = store.begin();+        get_nested_iter_core<Pos>()(iter.iter_core).setIterator(first);+        iter.value[Pos] = *first;+    }++    template <typename Store, typename iterator>+    void operator()(const Store& store, iterator& iter) const {+        // set iterator to first in store+        auto first = store.begin();+        get_nested_iter_core<Pos>()(iter.iter_core).setIterator(first);+        iter.value[Pos] = first->first;+        // and continue recursively+        fix_first<Pos + 1, Dim>()(first->second->getStore(), iter);+    }+};++template <unsigned Dim>+struct fix_first<Dim, Dim> {+    template <typename Store, typename iterator>+    void operator()(const Store&, iterator&) const {+        // terminal case => nothing to do+    }+};++template <unsigned Dim>+struct fix_first_nested {+    template <unsigned bits, typename iterator>+    void operator()(const SparseBitMap<bits>& store, iterator&& iter) const {+        // set iterator to first in store+        auto first = store.begin();+        iter.value[0] = *first;+        iter.iter_core.setIterator(std::move(first));+    }++    template <typename Store, typename iterator>+    void operator()(const Store& store, iterator&& iter) const {+        // set iterator to first in store+        auto first = store.begin();+        iter.value[0] = first->first;+        iter.iter_core.setIterator(std::move(first));+        // and continue recursively+        fix_first_nested<Dim - 1>()(first->second->getStore(), iter.getNestedView());+    }+};++// TODO: rewrite to erase `Pos` and `Len` arguments. this can cause a template instance explosion+/**+ * A functor initializing an iterator upon creation to reference the first element+ * exhibiting a given prefix within a given Trie.+ */+template <unsigned Len, unsigned Pos, unsigned Dim>+struct fix_binding {+    template <unsigned bits, typename iterator, typename entry_type>+    bool operator()(+            const SparseBitMap<bits>& store, iterator& begin, iterator& end, const entry_type& entry) const {+        // search in current level+        auto cur = store.find(entry[Pos]);++        // if not present => fail+        if (cur == store.end()) return false;++        // take current value+        get_nested_iter_core<Pos>()(begin.iter_core).setIterator(cur);+        ++cur;+        get_nested_iter_core<Pos>()(end.iter_core).setIterator(cur);++        // update iterator value+        begin.value[Pos] = entry[Pos];++        // no more remaining levels to fix+        return true;+    }++    template <typename Store, typename iterator, typename entry_type>+    bool operator()(const Store& store, iterator& begin, iterator& end, const entry_type& entry) const {+        // search in current level+        auto cur = store.find(entry[Pos]);++        // if not present => fail+        if (cur == store.end()) return false;++        // take current value as start+        get_nested_iter_core<Pos>()(begin.iter_core).setIterator(cur);++        // update iterator value+        begin.value[Pos] = entry[Pos];++        // fix remaining nested iterators+        auto res = fix_binding<Len - 1, Pos + 1, Dim>()(cur->second->getStore(), begin, end, entry);++        // update end of iterator+        if (get_nested_iter_core<Pos + 1>()(end.iter_core).getIterator() == cur->second->getStore().end()) {+            ++cur;+            if (cur != store.end()) {+                fix_first<Pos + 1, Dim>()(cur->second->getStore(), end);+            }+        }+        get_nested_iter_core<Pos>()(end.iter_core).setIterator(cur);++        // done+        return res;+    }+};++template <unsigned Pos, unsigned Dim>+struct fix_binding<0, Pos, Dim> {+    template <unsigned bits, typename iterator, typename entry_type>+    bool operator()(const SparseBitMap<bits>& store, iterator& begin, iterator& /* end */,+            const entry_type& /* entry */) const {+        // move begin to begin of store+        auto a = store.begin();+        get_nested_iter_core<Pos>()(begin.iter_core).setIterator(a);+        begin.value[Pos] = *a;++        return true;+    }++    template <typename Store, typename iterator, typename entry_type>+    bool operator()(const Store& store, iterator& begin, iterator& end, const entry_type& entry) const {+        // move begin to begin of store+        auto a = store.begin();+        get_nested_iter_core<Pos>()(begin.iter_core).setIterator(a);+        begin.value[Pos] = a->first;++        // continue recursively+        fix_binding<0, Pos + 1, Dim>()(a->second->getStore(), begin, end, entry);+        return true;+    }+};++template <unsigned Dim>+struct fix_binding<0, Dim, Dim> {+    template <typename Store, typename iterator, typename entry_type>+    bool operator()(const Store& /* store */, iterator& /* begin */, iterator& /* end */,+            const entry_type& /* entry */) const {+        // nothing more to do+        return true;+    }+};++/**+ * A functor initializing an iterator upon creation to reference the first element+ * within a given Trie being not less than a given value .+ */+template <unsigned Dim>+struct fix_lower_bound {+    using types = TrieTypes<Dim>;+    using const_entry_span_type = typename types::const_entry_span_type;++    template <unsigned bits, typename iterator>+    bool operator()(const SparseBitMap<bits>& store, iterator&& iter, const_entry_span_type entry) const {+        auto cur = store.lower_bound(entry[0]);+        if (cur == store.end()) return false;+        assert(entry[0] <= brie_element_type(*cur));++        iter.iter_core.setIterator(cur);+        iter.value[0] = *cur;+        return true;+    }++    template <typename Store, typename iterator>+    bool operator()(const Store& store, iterator&& iter, const_entry_span_type entry) const {+        auto cur = store.lowerBound(entry[0]);  // search in current level+        if (cur == store.end()) return false;   // if no lower boundary is found, be done+        assert(brie_element_type(cur->first) >= entry[0]);++        // if the lower bound is higher than the requested value, go to first in subtree+        if (brie_element_type(cur->first) > entry[0]) {+            iter.iter_core.setIterator(cur);+            iter.value[0] = cur->first;+            fix_first_nested<Dim - 1>()(cur->second->getStore(), iter.getNestedView());+            return true;+        }++        // attempt to fix the rest+        if (!fix_lower_bound<Dim - 1>()(cur->second->getStore(), iter.getNestedView(), tail(entry))) {+            // if it does not work, since there are no matching elements in this branch, go to next+            auto sub = copy(entry);+            sub[0] += 1;+            for (std::size_t i = 1; i < Dim; ++i)+                sub[i] = 0;++            return (*this)(store, iter, sub);+        }++        iter.iter_core.setIterator(cur);  // remember result+        iter.value[0] = cur->first;       // update iterator value+        return true;+    }+};++/**+ * A functor initializing an iterator upon creation to reference the first element+ * within a given Trie being greater than a given value .+ */+template <unsigned Dim>+struct fix_upper_bound {+    using types = TrieTypes<Dim>;+    using const_entry_span_type = typename types::const_entry_span_type;++    template <unsigned bits, typename iterator>+    bool operator()(const SparseBitMap<bits>& store, iterator&& iter, const_entry_span_type entry) const {+        auto cur = store.upper_bound(entry[0]);+        if (cur == store.end()) return false;+        assert(entry[0] <= brie_element_type(*cur));++        iter.iter_core.setIterator(cur);+        iter.value[0] = *cur;+        return true;  // no more remaining levels to fix+    }++    template <typename Store, typename iterator>+    bool operator()(const Store& store, iterator&& iter, const_entry_span_type entry) const {+        auto cur = store.lowerBound(entry[0]);  // search in current level+        if (cur == store.end()) return false;   // if no upper boundary is found, be done+        assert(brie_element_type(cur->first) >= entry[0]);++        // if the lower bound is higher than the requested value, go to first in subtree+        if (brie_element_type(cur->first) > entry[0]) {+            iter.iter_core.setIterator(cur);+            iter.value[0] = cur->first;+            fix_first_nested<Dim - 1>()(cur->second->getStore(), iter.getNestedView());+            return true;+        }++        // attempt to fix the rest+        if (!fix_upper_bound<Dim - 1>()(cur->second->getStore(), iter.getNestedView(), tail(entry))) {+            // if it does not work, since there are no matching elements in this branch, go to next+            auto sub = copy(entry);+            sub[0] += 1;+            for (std::size_t i = 1; i < Dim; ++i)+                sub[i] = 0;++            return (*this)(store, iter, sub);+        }++        iter.iter_core.setIterator(cur);  // remember result+        iter.value[0] = cur->first;       // update iterator value+        return true;+    }+};++template <unsigned Dim>+struct TrieTypes {+    using entry_type = std::array<brie_element_type, Dim>;+    using entry_span_type = span<brie_element_type, Dim>;+    using const_entry_span_type = span<const brie_element_type, Dim>;++    // the type of the nested tries (1 dimension less)+    using nested_trie_type = Trie<Dim - 1>;++    // the merge operation capable of merging two nested tries+    struct nested_trie_merger {+        nested_trie_type* operator()(nested_trie_type* a, const nested_trie_type* b) const {+            if (!b) return a;+            if (!a) return new nested_trie_type(*b);+            a->insertAll(*b);+            return a;+        }+    };++    // the operation capable of cloning a nested trie+    struct nested_trie_cloner {+        nested_trie_type* operator()(nested_trie_type* a) const {+            if (!a) return a;+            return new nested_trie_type(*a);+        }+    };++    // the data structure utilized for indexing nested tries+    using store_type = SparseArray<nested_trie_type*,+            6,  // = 2^6 entries per block+            nested_trie_merger, nested_trie_cloner>;++    // The iterator core for trie iterators involving this level.+    struct iterator_core {+        using store_iter = typename store_type::iterator;  // the iterator for the current level+        using nested_core_iter = typename nested_trie_type::iterator_core;  // the type of the nested iterator++    private:+        store_iter iter;+        nested_core_iter nested;++    public:+        iterator_core() = default;  // default -> end iterator++        iterator_core(store_iter store_iter, entry_span_type entry) : iter(std::move(store_iter)) {+            entry[0] = iter->first;+            nested = {iter->second->getStore().begin(), tail(entry)};+        }++        void setIterator(store_iter store_iter) {+            iter = std::move(store_iter);+        }++        store_iter& getIterator() {+            return iter;+        }++        nested_core_iter& getNested() {+            return nested;+        }++        bool inc(entry_span_type entry) {+            // increment nested iterator+            auto nested_entry = tail(entry);+            if (nested.inc(nested_entry)) return true;++            // increment the iterator on this level+            ++iter;++            // check whether the end has been reached+            if (iter.isEnd()) return false;++            // otherwise update entry value+            entry[0] = iter->first;++            // and restart nested+            nested = {iter->second->getStore().begin(), nested_entry};+            return true;+        }++        bool operator==(const iterator_core& other) const {+            return nested == other.nested && iter == other.iter;+        }++        bool operator!=(const iterator_core& other) const {+            return !(*this == other);+        }++        // enables this iterator core to be printed (for debugging)+        void print(std::ostream& out) const {+            out << iter << " | " << nested;+        }++        friend std::ostream& operator<<(std::ostream& out, const iterator_core& iter) {+            iter.print(out);+            return out;+        }+    };++    using iterator = TrieIterator<entry_type, iterator_core>;++    // the operation context aggregating all operation contexts of nested structures+    struct op_context {+        using local_ctxt = typename store_type::op_context;+        using nested_ctxt = typename nested_trie_type::op_context;++        // for insert and contain+        local_ctxt local{};+        brie_element_type lastQuery{};+        nested_trie_type* lastNested{nullptr};+        nested_ctxt nestedCtxt{};++        // for boundaries+        unsigned lastBoundaryLevels{Dim + 1};+        entry_type lastBoundaryRequest{};+        range<iterator> lastBoundaries{iterator(), iterator()};+    };+};++template <>+struct TrieTypes<1u> {+    using entry_type = std::array<brie_element_type, 1>;+    using entry_span_type = span<brie_element_type, 1>;+    using const_entry_span_type = span<const brie_element_type, 1>;++    // the map type utilized internally+    using store_type = SparseBitMap<>;+    using op_context = store_type::op_context;++    /**+     * The iterator core of this level contributing to the construction of+     * a composed trie iterator.+     */+    struct iterator_core {+        using store_iter = typename store_type::iterator;++    private:+        store_iter iter;++    public:+        iterator_core() = default;  // default end-iterator constructor++        iterator_core(store_iter store_iter, entry_span_type entry)+                : iter(std::move(store_iter))  // NOLINT : mistaken warning -`store_iter` is not const-qual+        {+            entry[0] = brie_element_type(*iter);+        }++        void setIterator(store_iter store_iter) {+            iter = std::move(store_iter);  // NOLINT : mistaken warning - `store_iter` is not const-qual+        }++        store_iter& getIterator() {+            return iter;+        }++        bool inc(entry_span_type entry) {+            // increment the iterator on this level+            ++iter;++            // check whether the end has been reached+            if (iter.isEnd()) return false;++            // otherwise update entry value+            entry[0] = brie_element_type(*iter);+            return true;+        }++        bool operator==(const iterator_core& other) const {+            return iter == other.iter;+        }++        bool operator!=(const iterator_core& other) const {+            return !(*this == other);+        }++        // enables this iterator core to be printed (for debugging)+        void print(std::ostream& out) const {+            out << iter;+        }++        friend std::ostream& operator<<(std::ostream& out, const iterator_core& iter) {+            iter.print(out);+            return out;+        }+    };++    using iterator = TrieIterator<entry_type, iterator_core>;+};++}  // namespace detail::brie++// use an inner class so `TrieN` is fully defined before the recursion, allowing us to use+// `op_context` in `TrieBase`+template <unsigned Dim>+class Trie : public TrieBase<Dim, Trie<Dim>> {+    template <unsigned N>+    friend class Trie;++    // a shortcut for the common base class type+    using base = TrieBase<Dim, Trie<Dim>>;+    using types = TrieTypes<Dim>;+    using nested_trie_type = typename types::nested_trie_type;+    using store_type = typename types::store_type;++    using base::store;++public:+    using const_entry_span_type = typename types::const_entry_span_type;+    using entry_span_type = typename types::entry_span_type;+    using entry_type = typename types::entry_type;+    using iterator = typename types::iterator;+    using iterator_core = typename types::iterator_core;+    using op_context = typename types::op_context;+    // type aliases for compatibility with `BTree` and others+    using operation_hints = op_context;+    using element_type = entry_type;++    using base::begin;+    using base::contains;+    using base::empty;+    using base::end;+    using base::find;+    using base::getBoundaries;+    using base::insert;+    using base::lower_bound;+    using base::upper_bound;++    ~Trie() {+        clear();+    }++    /**+     * Determines the number of entries in this trie.+     */+    std::size_t size() const {+        // the number of elements is lazy-evaluated+        std::size_t res = 0;+        for (auto&& [_, v] : store)+            res += v->size();++        return res;+    }++    /**+     * Computes the total memory usage of this data structure.+     */+    std::size_t getMemoryUsage() const {+        // compute the total memory usage of this level+        auto res = sizeof(*this) - sizeof(store) + store.getMemoryUsage();+        for (auto&& [_, v] : store)+            res += v->getMemoryUsage();  // add the memory usage of sub-levels++        return res;+    }++    /**+     * Removes all entries within this trie.+     */+    void clear() {+        // delete lower levels manually+        // (can't use `Own` b/c we need `atomic` instances and those require trivial assignment)+        for (auto& cur : store)+            delete cur.second;++        // clear store+        store.clear();+    }++    /**+     * Inserts a new entry. A operation context may be provided to exploit temporal+     * locality.+     *+     * @param tuple the entry to be added+     * @param ctxt the operation context to be utilized+     * @return true if the same tuple hasn't been present before, false otherwise+     */+    bool insert(const_entry_span_type tuple, op_context& ctxt) {+        using value_t = typename store_type::value_type;+        using atomic_value_t = typename store_type::atomic_value_type;++        // check context+        if (ctxt.lastNested && ctxt.lastQuery == tuple[0]) {+            base::hint_stats.inserts.addHit();+            return ctxt.lastNested->insert(tail(tuple), ctxt.nestedCtxt);+        }++        base::hint_stats.inserts.addMiss();++        // lookup nested+        atomic_value_t& next = store.getAtomic(tuple[0], ctxt.local);++        // get pure pointer to next level+        value_t nextPtr = next;++        // conduct a lock-free lazy-creation of nested trees+        if (!nextPtr) {+            // create a sub-tree && register it atomically+            auto newNested = mk<nested_trie_type>();+            if (next.compare_exchange_weak(nextPtr, newNested.get())) {+                nextPtr = newNested.release();  // worked, ownership is acquired by `store`+            }+            // otherwise some other thread was faster => use its version+        }++        // make sure a next has been established+        assert(nextPtr);++        // clear context if necessary+        if (nextPtr != ctxt.lastNested) {+            ctxt.lastQuery = tuple[0];+            ctxt.lastNested = nextPtr;+            ctxt.nestedCtxt = {};+        }++        // conduct recursive step+        return nextPtr->insert(tail(tuple), ctxt.nestedCtxt);+    }++    bool contains(const_entry_span_type tuple, op_context& ctxt) const {+        // check context+        if (ctxt.lastNested && ctxt.lastQuery == tuple[0]) {+            base::hint_stats.contains.addHit();+            return ctxt.lastNested->contains(tail(tuple), ctxt.nestedCtxt);+        }++        base::hint_stats.contains.addMiss();++        // lookup next step+        auto next = store.lookup(tuple[0], ctxt.local);++        // clear context if necessary+        if (next != ctxt.lastNested) {+            ctxt.lastQuery = tuple[0];+            ctxt.lastNested = next;+            ctxt.nestedCtxt = {};+        }++        // conduct recursive step+        return next && next->contains(tail(tuple), ctxt.nestedCtxt);+    }++    /**+     * Obtains a range of elements matching the prefix of the given entry up to+     * levels elements. A operation context may be provided to exploit temporal+     * locality.+     *+     * @tparam levels the length of the requested matching prefix+     * @param entry the entry to be looking for+     * @param ctxt the operation context to be utilized+     * @return the corresponding range of matching elements+     */+    template <unsigned levels>+    range<iterator> getBoundaries(const_entry_span_type entry, op_context& ctxt) const {+        // if nothing is bound => just use begin and end+        if constexpr (levels == 0) {+            return make_range(begin(), end());+        } else {  // HACK: explicit `else` branch b/c OSX compiler doesn't do DCE before `0 < limit` warning+            // check context+            if (ctxt.lastBoundaryLevels == levels) {+                bool fit = true;+                for (unsigned i = 0; i < levels; ++i) {+                    fit = fit && (entry[i] == ctxt.lastBoundaryRequest[i]);+                }++                // if it fits => take it+                if (fit) {+                    base::hint_stats.get_boundaries.addHit();+                    return ctxt.lastBoundaries;+                }+            }++            // the hint has not been a hit+            base::hint_stats.get_boundaries.addMiss();++            // start with two end iterators+            iterator begin{};+            iterator end{};++            // adapt them level by level+            auto found = fix_binding<levels, 0, Dim>()(store, begin, end, entry);+            if (!found) return make_range(iterator(), iterator());++            // update context+            static_assert(std::tuple_size_v<decltype(ctxt.lastBoundaryRequest)> == Dim);+            static_assert(std::tuple_size_v<decltype(entry)> == Dim);+            ctxt.lastBoundaryLevels = levels;+            std::copy_n(entry.begin(), Dim, ctxt.lastBoundaryRequest.begin());+            ctxt.lastBoundaries = make_range(begin, end);++            // use the result+            return ctxt.lastBoundaries;+        }+    }++    /**+     * Obtains an iterator to the first element not less than the given entry value.+     *+     * @param entry the lower bound for this search+     * @param ctxt the operation context to be utilized+     * @return an iterator addressing the first element in this structure not less than the given value+     */+    iterator lower_bound(const_entry_span_type entry, op_context& /* ctxt */) const {+        // start with a default-initialized iterator+        iterator res;++        // adapt it level by level+        bool found = fix_lower_bound<Dim>()(store, res, entry);++        // use the result+        return found ? res : end();+    }++    /**+     * Obtains an iterator to the first element greater than the given entry value, or end if there is no+     * such element.+     *+     * @param entry the upper bound for this search+     * @param ctxt the operation context to be utilized+     * @return an iterator addressing the first element in this structure greater than the given value+     */+    iterator upper_bound(const_entry_span_type entry, op_context& /* ctxt */) const {+        // start with a default-initialized iterator+        iterator res;++        // adapt it level by level+        bool found = fix_upper_bound<Dim>()(store, res, entry);++        // use the result+        return found ? res : end();+    }++    /**+     * Computes a partition of an approximate number of chunks of the content+     * of this trie. Thus, the union of the resulting set of disjoint ranges is+     * equivalent to the content of this trie.+     *+     * @param chunks the number of chunks requested+     * @return a list of sub-ranges forming a partition of the content of this trie+     */+    std::vector<range<iterator>> partition(unsigned chunks = 500) const {+        std::vector<range<iterator>> res;++        // shortcut for empty trie+        if (this->empty()) return res;++        // use top-level elements for partitioning+        size_t step = std::max(store.size() / chunks, std::size_t(1));++        size_t c = 1;+        auto priv = begin();+        for (auto it = store.begin(); it != store.end(); ++it, c++) {+            if (c % step != 0 || c == 1) {+                continue;+            }+            auto cur = iterator(it);+            res.push_back(make_range(priv, cur));+            priv = cur;+        }+        // add final chunk+        res.push_back(make_range(priv, end()));+        return res;+    }+};++/**+ * A template specialization for tries representing a set.+ * For improved memory efficiency, this level is the leaf-node level+ * of all tries exhibiting an arity >= 1. Internally, values are stored utilizing+ * sparse bit maps.+ */+template <>+class Trie<1u> : public TrieBase<1u, Trie<1u>> {+    using base = TrieBase<1u, Trie<1u>>;+    using types = TrieTypes<1u>;+    using store_type = typename types::store_type;++    using base::store;++public:+    using const_entry_span_type = typename types::const_entry_span_type;+    using entry_span_type = typename types::entry_span_type;+    using entry_type = typename types::entry_type;+    using iterator = typename types::iterator;+    using iterator_core = typename types::iterator_core;+    using op_context = typename types::op_context;+    // type aliases for compatibility with `BTree` and others+    using operation_hints = op_context;+    using element_type = entry_type;++    using base::begin;+    using base::contains;+    using base::empty;+    using base::end;+    using base::find;+    using base::getBoundaries;+    using base::insert;+    using base::lower_bound;+    using base::upper_bound;++    /**+     * Determines the number of entries in this trie.+     */+    std::size_t size() const {+        return store.size();+    }++    /**+     * Computes the total memory usage of this data structure.+     */+    std::size_t getMemoryUsage() const {+        // compute the total memory usage+        return sizeof(*this) - sizeof(store) + store.getMemoryUsage();+    }++    /**+     * Removes all elements form this trie.+     */+    void clear() {+        store.clear();+    }++    /**+     * Inserts the given tuple into this trie.+     * An operation context can be provided to exploit temporal locality.+     *+     * @param tuple the tuple to be inserted+     * @param ctxt an operation context for exploiting temporal locality+     * @return true if the tuple has not been present before, false otherwise+     */+    bool insert(const_entry_span_type tuple, op_context& ctxt) {+        return store.set(tuple[0], ctxt);+    }++    /**+     * Determines whether the given tuple is present in this trie or not.+     * An operation context can be provided to exploit temporal locality.+     *+     * @param tuple the tuple to be tested+     * @param ctxt an operation context for exploiting temporal locality+     * @return true if present, false otherwise+     */+    bool contains(const_entry_span_type tuple, op_context& ctxt) const {+        return store.test(tuple[0], ctxt);+    }++    // ---------------------------------------------------------------------+    //                           Iterator+    // ---------------------------------------------------------------------++    /**+     * Obtains a partition of this tire such that the resulting list of ranges+     * cover disjoint subsets of the elements stored in this trie. Their union+     * is equivalent to the content of this trie.+     */+    std::vector<range<iterator>> partition(unsigned chunks = 500) const {+        std::vector<range<iterator>> res;++        // shortcut for empty trie+        if (this->empty()) return res;++        // use top-level elements for partitioning+        int step = static_cast<int>(std::max(store.size() / chunks, std::size_t(1)));++        int c = 1;+        auto priv = begin();+        for (auto it = store.begin(); it != store.end(); ++it, c++) {+            if (c % step != 0 || c == 1) {+                continue;+            }+            auto cur = iterator(it);+            res.push_back(make_range(priv, cur));+            priv = cur;+        }+        // add final chunk+        res.push_back(make_range(priv, end()));+        return res;+    }++    /**+     * Obtains a range of elements matching the prefix of the given entry up to+     * levels elements. A operation context may be provided to exploit temporal+     * locality.+     *+     * @tparam levels the length of the requested matching prefix+     * @param entry the entry to be looking for+     * @param ctxt the operation context to be utilized+     * @return the corresponding range of matching elements+     */+    template <unsigned levels>+    range<iterator> getBoundaries(const_entry_span_type entry, op_context& ctxt) const {+        // for levels = 0+        if (levels == 0) return make_range(begin(), end());+        // for levels = 1+        auto pos = store.find(entry[0], ctxt);+        if (pos == store.end()) return make_range(end(), end());+        auto next = pos;+        ++next;+        return make_range(iterator(pos), iterator(next));+    }++    iterator lower_bound(const_entry_span_type entry, op_context&) const {+        return iterator(store.lower_bound(entry[0]));+    }++    iterator upper_bound(const_entry_span_type entry, op_context&) const {+        return iterator(store.upper_bound(entry[0]));+    }+};++}  // end namespace souffle++namespace std {++using namespace ::souffle::detail::brie;++template <typename A>+struct iterator_traits<SparseArrayIter<A>>+        : forward_non_output_iterator_traits<typename SparseArrayIter<A>::value_type> {};++template <typename A>+struct iterator_traits<SparseBitMapIter<A>>+        : forward_non_output_iterator_traits<typename SparseBitMapIter<A>::value_type> {};++template <typename A, typename IterCore>+struct iterator_traits<TrieIterator<A, IterCore>> : forward_non_output_iterator_traits<A> {};++}  // namespace std++#ifdef _WIN32+#undef __sync_synchronize+#undef __sync_bool_compare_and_swap+#endif
+ cbits/souffle/datastructure/ConcurrentFlyweight.h view
@@ -0,0 +1,527 @@+/*+ * Souffle - A Datalog Compiler+ * Copyright (c) 2021, The Souffle Developers. All rights reserved+ * Licensed under the Universal Permissive License v 1.0 as shown at:+ * - https://opensource.org/licenses/UPL+ * - <souffle root>/licenses/SOUFFLE-UPL.txt+ */+#pragma once++#include "ConcurrentInsertOnlyHashMap.h"+#include "souffle/utility/ParallelUtil.h"+#include <cassert>+#include <cstring>++namespace souffle {++/**+ * A concurrent, almost lock-free associative datastructure that implements the+ * Flyweight pattern.  Assigns a unique index to each inserted key. Elements+ * cannot be removed, the datastructure can only grow.+ *+ * The datastructure enables a configurable number of concurrent access lanes.+ * Access to the datastructure is lock-free between different lanes.+ * Concurrent accesses through the same lane is sequential.+ *+ * Growing the datastructure requires to temporarily lock all lanes to let a+ * single lane perform the growing operation. The global lock is amortized+ * thanks to an exponential growth strategy.+ *+ */+template <class LanesPolicy, class Key, class Hash = std::hash<Key>, class KeyEqual = std::equal_to<Key>,+        class KeyFactory = details::Factory<Key>>+class ConcurrentFlyweight {+public:+    using lane_id = typename LanesPolicy::lane_id;+    using index_type = std::size_t;+    using key_type = Key;+    using value_type = std::pair<const Key, const index_type>;+    using pointer = const value_type*;+    using reference = const value_type&;++private:+    // Effectively:+    //  data slot_type = NONE | END | Idx index_type+    //  The last two values in the domain of `index_type` are used to represent cases `NONE` and `END`+    // TODO: strong type-def wrap this to prevent implicit conversions+    using slot_type = index_type;+    static constexpr slot_type NONE = std::numeric_limits<slot_type>::max();  // special case: `std::nullopt`+    static constexpr slot_type END = NONE - 1;                                // special case: end iterator+    static constexpr slot_type SLOT_MAX = END;  // +1 the largest non-special slot value++    static_assert(std::is_same_v<slot_type, index_type>,+            "conversion helpers assume they're the underlying type, "+            "with the last two values reserved for special cases");+    static_assert(std::is_unsigned_v<slot_type>);++    /// Converts from index to slot.+    static slot_type slot(const index_type I) {+        // not expected to happen. you'll run out of memory long before.+        assert(I < SLOT_MAX && "can't represent index in `slot_type` domain");+        return static_cast<slot_type>(I);+    }++    /// Converts from slot to index.+    static index_type index(const slot_type S) {+        assert(S < SLOT_MAX && "slot is sentinal value; can't convert to index !!");+        return static_cast<index_type>(S);+    }++public:+    /// Iterator with concurrent access to the datastructure.+    struct Iterator {+        using iterator_category = std::input_iterator_tag;+        using value_type = ConcurrentFlyweight::value_type;+        using pointer = ConcurrentFlyweight::pointer;+        using reference = ConcurrentFlyweight::reference;++    private:+        const ConcurrentFlyweight* This;++        /// Access lane to the datastructure.+        lane_id Lane;++        /// Current slot.+        slot_type Slot;++        /// Next slot that might be unassigned.+        slot_type NextMaybeUnassignedSlot;++        /// Handle that owns the next slot that might be unassigned.+        slot_type NextMaybeUnassignedHandle = NONE;++    public:+        // The 'begin' iterator+        Iterator(const ConcurrentFlyweight* This, const lane_id H)+                : This(This), Lane(H), Slot(NONE), NextMaybeUnassignedSlot(0) {+            FindNextMaybeUnassignedSlot();+            MoveToNextAssignedSlot();+        }++        // The 'end' iterator+        Iterator(const ConcurrentFlyweight* This)+                : This(This), Lane(0), Slot(END), NextMaybeUnassignedSlot(END) {}++        // The iterator starting at slot I, using access lane H.+        Iterator(const ConcurrentFlyweight* This, const lane_id H, const index_type I)+                : This(This), Lane(H), Slot(slot(I)), NextMaybeUnassignedSlot(slot(I)) {+            FindNextMaybeUnassignedSlot();+            MoveToNextAssignedSlot();+        }++        Iterator(const Iterator& That)+                : This(That.This), Lane(That.Lane), Slot(That.Slot),+                  NextMaybeUnassignedSlot(That.NextMaybeUnassignedSlot),+                  NextMaybeUnassignedHandle(That.NextMaybeUnassignedHandle) {}++        Iterator(Iterator&& That)+                : This(That.This), Lane(That.Lane), Slot(That.Slot),+                  NextMaybeUnassignedSlot(That.NextMaybeUnassignedSlot),+                  NextMaybeUnassignedHandle(That.NextMaybeUnassignedHandle) {}++        Iterator& operator=(const Iterator& That) {+            This = That.This;+            Lane = That.Lane;+            Slot = That.Slot;+            NextMaybeUnassignedSlot = That.NextMaybeUnassignedSlot;+            NextMaybeUnassignedHandle = That.NextMaybeUnassignedHandle;+        }++        Iterator& operator=(Iterator&& That) {+            This = That.This;+            Lane = That.Lane;+            Slot = That.Slot;+            NextMaybeUnassignedSlot = That.NextMaybeUnassignedSlot;+            NextMaybeUnassignedHandle = That.NextMaybeUnassignedHandle;+        }++        reference operator*() const {+            const auto Guard = This->Lanes.guard(Lane);+            return *This->Slots[index(Slot)];+        }++        pointer operator->() const {+            const auto Guard = This->Lanes.guard(Lane);+            return This->Slots[index(Slot)];+        }++        Iterator& operator++() {+            MoveToNextAssignedSlot();+            return *this;+        }++        Iterator operator++(int) {+            Iterator Tmp = *this;+            ++(*this);+            return Tmp;+        }++        bool operator==(const Iterator& That) const {+            return (This == That.This) && (Slot == That.Slot);+        }++        bool operator!=(const Iterator& That) const {+            return (This != That.This) || (Slot != That.Slot);+        }++    private:+        /** Find next slot after Slot that is maybe unassigned. */+        void FindNextMaybeUnassignedSlot() {+            NextMaybeUnassignedSlot = END;+            for (lane_id I = 0; I < This->Lanes.lanes(); ++I) {+                const auto Lane = This->Lanes.guard(I);+                if ((Slot == NONE || This->Handles[I].NextSlot > Slot) &&+                        This->Handles[I].NextSlot < NextMaybeUnassignedSlot) {+                    NextMaybeUnassignedSlot = This->Handles[I].NextSlot;+                    NextMaybeUnassignedHandle = I;+                }+            }+            if (NextMaybeUnassignedSlot == END) {+                NextMaybeUnassignedSlot = This->NextSlot.load(std::memory_order_acquire);+                NextMaybeUnassignedHandle = NONE;+            }+        }++        /**+         * Move Slot to next assigned slot and return true.+         * Otherwise the end is reached and Slot is assigned `END` and return false.+         */+        bool MoveToNextAssignedSlot() {+            static_assert(NONE == std::numeric_limits<slot_type>::max(),+                    "required for wrap around to 0 for begin-iterator-scan");+            static_assert(NONE + 1 == 0, "required for wrap around to 0 for begin-iterator-scan");+            while (Slot != END) {+                assert(Slot + 1 < SLOT_MAX);+                if (Slot + 1 < NextMaybeUnassignedSlot) {  // next unassigned slot not reached+                    Slot = Slot + 1;+                    return true;+                }++                if (NextMaybeUnassignedHandle == NONE) {  // reaching end+                    Slot = END;+                    NextMaybeUnassignedSlot = END;+                    NextMaybeUnassignedHandle = NONE;+                    return false;+                }++                if (NextMaybeUnassignedHandle != NONE) {  // maybe reaching the next unassigned slot+                    This->Lanes.lock(NextMaybeUnassignedHandle);+                    const bool IsAssigned = (Slot + 1 < This->Handles[NextMaybeUnassignedHandle].NextSlot);+                    This->Lanes.unlock(NextMaybeUnassignedHandle);+                    Slot = Slot + 1;+                    FindNextMaybeUnassignedSlot();+                    if (IsAssigned) {+                        return true;+                    }+                }+            }+            return false;+        }+    };++    using iterator = Iterator;++    /// Initialize the datastructure with the given capacity.+    ConcurrentFlyweight(const std::size_t LaneCount, const std::size_t InitialCapacity,+            const bool ReserveFirst, const Hash& hash = Hash(), const KeyEqual& key_equal = KeyEqual(),+            const KeyFactory& key_factory = KeyFactory())+            : Lanes(LaneCount), HandleCount(LaneCount),+              Mapping(LaneCount, InitialCapacity, hash, key_equal, key_factory) {+        Slots = std::make_unique<const value_type*[]>(InitialCapacity);+        Handles = std::make_unique<Handle[]>(HandleCount);+        NextSlot = (ReserveFirst ? 1 : 0);+        SlotCount = InitialCapacity;+    }++    /// Initialize the datastructure with a capacity of 8 elements.+    ConcurrentFlyweight(const std::size_t LaneCount, const bool ReserveFirst, const Hash& hash = Hash(),+            const KeyEqual& key_equal = KeyEqual(), const KeyFactory& key_factory = KeyFactory())++            : ConcurrentFlyweight(LaneCount, 8, ReserveFirst, hash, key_equal, key_factory) {}++    /// Initialize the datastructure with a capacity of 8 elements.+    ConcurrentFlyweight(const std::size_t LaneCount, const Hash& hash = Hash(),+            const KeyEqual& key_equal = KeyEqual(), const KeyFactory& key_factory = KeyFactory())+            : ConcurrentFlyweight(LaneCount, 8, false, hash, key_equal, key_factory) {}++    virtual ~ConcurrentFlyweight() {+        for (lane_id I = 0; I < HandleCount; ++I) {+            if (Handles[I].NextNode) {+                delete Handles[I].NextNode;+            }+        }+    }++    /**+     * Change the number of lanes and possibly grow the number of handles.+     * Do not use while threads are using this datastructure.+     */+    void setNumLanes(const std::size_t NumLanes) {+        if (NumLanes > HandleCount) {+            std::unique_ptr<Handle[]> NextHandles = std::make_unique<Handle[]>(NumLanes);+            std::copy(Handles.get(), Handles.get() + HandleCount, NextHandles.get());+            Handles.swap(NextHandles);+            HandleCount = NumLanes;+        }+        Mapping.setNumLanes(NumLanes);+        Lanes.setNumLanes(NumLanes);+    }++    /** Return a concurrent iterator on the first element. */+    Iterator begin(const lane_id H) const {+        return Iterator(this, H);+    }++    /** Return an iterator past the last element. */+    Iterator end() const {+        return Iterator(this);+    }++    /// Return true if the value is in the map.+    template <typename K>+    bool weakContains(const lane_id H, const K& X) const {+        return Mapping.weakContains(H, X);+    }++    /// Return the value associated with the given index.+    /// Assumption: the index is mapped in the datastructure.+    const Key& fetch(const lane_id H, const index_type Idx) const {+        const auto Lane = Lanes.guard(H);+        assert(Idx < SlotCount.load(std::memory_order_relaxed));+        return Slots[Idx]->first;+    }++    /// Return the pair of the index for the given value and a boolean+    /// indicating if the value was already present (false) or inserted by this handle (true).+    /// Insert the value and return a fresh index if the value is not+    /// yet indexed.+    template <class... Args>+    std::pair<index_type, bool> findOrInsert(const lane_id H, Args&&... Xs) {+        const auto Lane = Lanes.guard(H);+        node_type Node;++        slot_type Slot = Handles[H].NextSlot;++        // Getting the next insertion slot for the current lane may require+        // more than one attempts if the datastructure must grow and other+        // threads are waiting for the same lane @p H.+        while (true) {+            if (Slot == NONE) {+                // Reserve a slot for the lane, the datastructure might need to+                // grow before the slot memory location becomes available.+                Slot = NextSlot++;+                Handles[H].NextSlot = Slot;+                Handles[H].NextNode = Mapping.node(static_cast<index_type>(Slot));+            }++            if (Slot >= SlotCount.load(std::memory_order_relaxed)) {+                // The slot memory location is not yet available, try to+                // grow the datastructure. Other threads in other lanes might+                // be attempting to grow the datastructure concurrently.+                //+                // Anyway when this call returns the Slot memory location is+                // available.+                tryGrow(H);++                // Reload the Slot for the current lane since another thread+                // using the same lane may take-over the lane during tryGrow()+                // and consume the slot before the current thread is+                // rescheduled on the lane.+                Slot = Handles[H].NextSlot;+            } else {+                // From here the slot is known, allocated and available.+                break;+            }+        }++        Node = Handles[H].NextNode;++        // Insert key in the index in advance.+        Slots[Slot] = &Node->value();++        auto Res = Mapping.get(H, Node, std::forward<Args>(Xs)...);+        if (Res.second) {+            // Inserted by self, slot is consumed, clear the lane's state.+            Handles[H].clear();+            return std::make_pair(static_cast<index_type>(Slot), true);+        } else {+            // Inserted concurrently by another thread, clearing the slot is+            // not strictly needed but it avoids leaving a dangling pointer+            // there.+            //+            // The reserved slot and node remains in the lane state so that+            // they can be consumed by the next insertion operation on this+            // lane.+            Slots[Slot] = nullptr;+            return std::make_pair(Res.first->second, false);+        }+    }++private:+    using map_type = ConcurrentInsertOnlyHashMap<LanesPolicy, Key, index_type, Hash, KeyEqual, KeyFactory>;+    using node_type = typename map_type::node_type;++    struct Handle {+        void clear() {+            NextSlot = NONE;+            NextNode = nullptr;+        }++        slot_type NextSlot = NONE;+        node_type NextNode = nullptr;+    };++protected:+    // The concurrency manager.+    LanesPolicy Lanes;++private:+    // Number of handles+    std::size_t HandleCount;++    // Handle for each concurrent lane.+    std::unique_ptr<Handle[]> Handles;++    // Slots[I] points to the value associated with index I.+    std::unique_ptr<const value_type*[]> Slots;++    // The map from keys to index.+    map_type Mapping;++    // Next available slot.+    std::atomic<slot_type> NextSlot;++    // Number of slots.+    std::atomic<slot_type> SlotCount;++    /// Grow the datastructure if needed.+    bool tryGrow(const lane_id H) {+        // This call may release and re-acquire the lane to+        // allow progress of a concurrent growing operation.+        //+        // It is possible that another thread is waiting to+        // enter the same lane, and that other thread might+        // take and leave the lane before the current thread+        // re-acquires it.+        Lanes.beforeLockAllBut(H);++        if (NextSlot < SlotCount) {+            // Current size is fine+            Lanes.beforeUnlockAllBut(H);+            return false;+        }++        Lanes.lockAllBut(H);++        {  // safe section+            const std::size_t CurrentSize = SlotCount;+            std::size_t NewSize = (CurrentSize << 1);  // double size policy+            while (NewSize < NextSlot) {+                NewSize <<= 1;  // double size+            }+            std::unique_ptr<const value_type*[]> NewSlots = std::make_unique<const value_type*[]>(NewSize);+            std::memcpy(NewSlots.get(), Slots.get(), sizeof(const value_type*) * CurrentSize);+            Slots = std::move(NewSlots);+            SlotCount = NewSize;+        }++        Lanes.beforeUnlockAllBut(H);+        Lanes.unlockAllBut(H);++        return true;+    }+};++#ifdef _OPENMP+/** A Flyweight datastructure with concurrent access specialized for OpenMP. */+template <class Key, class Hash = std::hash<Key>, class KeyEqual = std::equal_to<Key>,+        class KeyFactory = details::Factory<Key>>+class OmpFlyweight : protected ConcurrentFlyweight<ConcurrentLanes, Key, Hash, KeyEqual, KeyFactory> {+public:+    using Base = ConcurrentFlyweight<ConcurrentLanes, Key, Hash, KeyEqual, KeyFactory>;+    using index_type = typename Base::index_type;+    using lane_id = typename Base::lane_id;+    using iterator = typename Base::iterator;++    explicit OmpFlyweight(const std::size_t LaneCount, const std::size_t InitialCapacity = 8,+            const bool ReserveFirst = false, const Hash& hash = Hash(),+            const KeyEqual& key_equal = KeyEqual(), const KeyFactory& key_factory = KeyFactory())+            : Base(LaneCount, InitialCapacity, ReserveFirst, hash, key_equal, key_factory) {}++    iterator begin() const {+        return Base::begin(Base::Lanes.threadLane());+    }++    iterator end() const {+        return Base::end();+    }++    template <typename K>+    bool weakContains(const K& X) const {+        return Base::weakContains(Base::Lanes.threadLane(), X);+    }++    const Key& fetch(const index_type Idx) const {+        return Base::fetch(Base::Lanes.threadLane(), Idx);+    }++    template <class... Args>+    std::pair<index_type, bool> findOrInsert(Args&&... Xs) {+        return Base::findOrInsert(Base::Lanes.threadLane(), std::forward<Args>(Xs)...);+    }+};+#endif++/**+ * A Flyweight datastructure with sequential access.+ *+ * Reuse the concurrent flyweight with a single access handle.+ */+template <class Key, class Hash = std::hash<Key>, class KeyEqual = std::equal_to<Key>,+        class KeyFactory = details::Factory<Key>>+class SeqFlyweight : protected ConcurrentFlyweight<SeqConcurrentLanes, Key, Hash, KeyEqual, KeyFactory> {+public:+    using Base = ConcurrentFlyweight<SeqConcurrentLanes, Key, Hash, KeyEqual, KeyFactory>;+    using index_type = typename Base::index_type;+    using lane_id = typename Base::lane_id;+    using iterator = typename Base::iterator;++    explicit SeqFlyweight(const std::size_t NumLanes, const std::size_t InitialCapacity = 8,+            const bool ReserveFirst = false, const Hash& hash = Hash(),+            const KeyEqual& key_equal = KeyEqual(), const KeyFactory& key_factory = KeyFactory())+            : Base(NumLanes, InitialCapacity, ReserveFirst, hash, key_equal, key_factory) {}++    iterator begin() const {+        return Base::begin(0);+    }++    iterator end() const {+        return Base::end();+    }++    template <typename K>+    bool weakContains(const K& X) const {+        return Base::weakContains(0, X);+    }++    const Key& fetch(const index_type Idx) const {+        return Base::fetch(0, Idx);+    }++    template <class... Args>+    std::pair<index_type, bool> findOrInsert(Args&&... Xs) {+        return Base::findOrInsert(0, std::forward<Args>(Xs)...);+    }+};++#ifdef _OPENMP+template <class Key, class Hash = std::hash<Key>, class KeyEqual = std::equal_to<Key>,+        class KeyFactory = details::Factory<Key>>+using FlyweightImpl = OmpFlyweight<Key, Hash, KeyEqual, KeyFactory>;+#else+template <class Key, class Hash = std::hash<Key>, class KeyEqual = std::equal_to<Key>,+        class KeyFactory = details::Factory<Key>>+using FlyweightImpl = SeqFlyweight<Key, Hash, KeyEqual, KeyFactory>;+#endif++}  // namespace souffle
+ cbits/souffle/datastructure/ConcurrentInsertOnlyHashMap.h view
@@ -0,0 +1,466 @@+/*+ * Souffle - A Datalog Compiler+ * Copyright (c) 2021, The Souffle Developers. All rights reserved+ * Licensed under the Universal Permissive License v 1.0 as shown at:+ * - https://opensource.org/licenses/UPL+ * - <souffle root>/licenses/SOUFFLE-UPL.txt+ */+#pragma once++#include "souffle/utility/ParallelUtil.h"++#include <array>+#include <atomic>+#include <cassert>+#include <cmath>+#include <memory>+#include <mutex>+#include <vector>++namespace souffle {+namespace details {++static const std::vector<std::pair<unsigned, unsigned>> ToPrime = {+        // https://primes.utm.edu/lists/2small/0bit.html+        // ((2^n) - k) is prime+        // {n, k}+        {4, 3},  // 2^4 - 3 = 13+        {8, 5},  // 8^5 - 5 = 251+        {9, 3}, {10, 3}, {11, 9}, {12, 3}, {13, 1}, {14, 3}, {15, 19}, {16, 15}, {17, 1}, {18, 5}, {19, 1},+        {20, 3}, {21, 9}, {22, 3}, {23, 15}, {24, 3}, {25, 39}, {26, 5}, {27, 39}, {28, 57}, {29, 3},+        {30, 35}, {31, 1}, {32, 5}, {33, 9}, {34, 41}, {35, 31}, {36, 5}, {37, 25}, {38, 45}, {39, 7},+        {40, 87}, {41, 21}, {42, 11}, {43, 57}, {44, 17}, {45, 55}, {46, 21}, {47, 115}, {48, 59}, {49, 81},+        {50, 27}, {51, 129}, {52, 47}, {53, 111}, {54, 33}, {55, 55}, {56, 5}, {57, 13}, {58, 27}, {59, 55},+        {60, 93}, {61, 1}, {62, 57}, {63, 25}};++// (2^64)-59 is the largest prime that fits in uint64_t+static constexpr uint64_t LargestPrime64 = 18446744073709551557UL;++// Return a prime greater or equal to the lower bound.+// Return 0 if the next prime would not fit in 64 bits.+static uint64_t GreaterOrEqualPrime(const uint64_t LowerBound) {+    if (LowerBound > LargestPrime64) {+        return 0;+    }++    for (std::size_t I = 0; I < ToPrime.size(); ++I) {+        const uint64_t N = ToPrime[I].first;+        const uint64_t K = ToPrime[I].second;+        const uint64_t Prime = (1ULL << N) - K;+        if (Prime >= LowerBound) {+            return Prime;+        }+    }+    return LargestPrime64;+}++template <typename T>+struct Factory {+    template <class... Args>+    T& replace(T& Place, Args&&... Xs) {+        Place = T{std::forward<Args>(Xs)...};+        return Place;+    }+};++}  // namespace details++/**+ * A concurrent, almost lock-free associative hash-map that can only grow.+ * Elements cannot be removed, the hash-map can only grow.+ *+ * The datastructures enables a configurable number of concurrent access lanes.+ * Access to the datastructure is lock-free between different lanes.+ * Concurrent accesses through the same lane is sequential.+ *+ * Growing the datastructure requires to temporarily lock all lanes to let a+ * single lane perform the growing operation. The global lock is amortized+ * thanks to an exponential growth strategy.+ */+template <class LanesPolicy, class Key, class T, class Hash = std::hash<Key>,+        class KeyEqual = std::equal_to<Key>, class KeyFactory = details::Factory<Key>>+class ConcurrentInsertOnlyHashMap {+public:+    class Node;++    using key_type = Key;+    using mapped_type = T;+    using node_type = Node*;+    using value_type = std::pair<const Key, const T>;+    using size_type = std::size_t;+    using hasher = Hash;+    using key_equal = KeyEqual;+    using self_type = ConcurrentInsertOnlyHashMap<Key, T, Hash, KeyEqual, KeyFactory>;+    using lane_id = typename LanesPolicy::lane_id;++    class Node {+    public:+        virtual ~Node() {}+        virtual const value_type& value() const = 0;+        virtual const key_type& key() const = 0;+        virtual const mapped_type& mapped() const = 0;+    };++private:+    // Each bucket of the hash-map is a linked list.+    struct BucketList : Node {+        virtual ~BucketList() {}++        BucketList(const Key& K, const T& V, BucketList* N) : Value(K, V), Next(N) {}++        const value_type& value() const {+            return Value;+        }++        const key_type& key() const {+            return Value.first;+        }++        const mapped_type& mapped() const {+            return Value.second;+        }++        // Stores the couple of a key and its associated value.+        value_type Value;++        // Points to next element of the map that falls into the same bucket.+        BucketList* Next;+    };++public:+    /**+     * @brief Construct a hash-map with at least the given number of buckets.+     *+     * Load-factor is initialized to 1.0.+     */+    ConcurrentInsertOnlyHashMap(const std::size_t LaneCount, const std::size_t Bucket_Count,+            const Hash& hash = Hash(), const KeyEqual& key_equal = KeyEqual(),+            const KeyFactory& key_factory = KeyFactory())+            : Lanes(LaneCount), Hasher(hash), EqualTo(key_equal), Factory(key_factory) {+        Size = 0;+        BucketCount = details::GreaterOrEqualPrime(Bucket_Count);+        if (BucketCount == 0) {+            // Hopefuly this number of buckets is never reached.+            BucketCount = std::numeric_limits<std::size_t>::max();+        }+        LoadFactor = 1.0;+        Buckets = std::make_unique<std::atomic<BucketList*>[]>(BucketCount);+        MaxSizeBeforeGrow = static_cast<std::size_t>(std::ceil(LoadFactor * (double)BucketCount));+    }++    ConcurrentInsertOnlyHashMap(const Hash& hash = Hash(), const KeyEqual& key_equal = KeyEqual(),+            const KeyFactory& key_factory = KeyFactory())+            : ConcurrentInsertOnlyHashMap(8, hash, key_equal, key_factory) {}++    ~ConcurrentInsertOnlyHashMap() {+        for (std::size_t Bucket = 0; Bucket < BucketCount; ++Bucket) {+            BucketList* L = Buckets[Bucket].load(std::memory_order_relaxed);+            while (L != nullptr) {+                BucketList* BL = L;+                L = L->Next;+                delete (BL);+            }+        }+    }++    void setNumLanes(const std::size_t NumLanes) {+        Lanes.setNumLanes(NumLanes);+    }++    /** @brief Create a fresh node initialized with the given value and a+     * default-constructed key.+     *+     * The ownership of the returned node given to the caller.+     */+    node_type node(const T& V) {+        BucketList* BL = new BucketList(Key{}, V, nullptr);+        return static_cast<node_type>(BL);+    }++    /** @brief Checks if the map contains an element with the given key.+     *+     * The search is done concurrently with possible insertion of the+     * searched key. If return true, then there is definitely an element+     * with the specified key, if return false then there was no such+     * element when the search began.+     */+    template <class K>+    bool weakContains(const lane_id H, const K& X) const {+        const size_t HashValue = Hasher(X);+        const auto Guard = Lanes.guard(H);+        const size_t Bucket = HashValue % BucketCount;++        BucketList* L = Buckets[Bucket].load(std::memory_order_acquire);+        while (L != nullptr) {+            if (EqualTo(L->Value.first, X)) {+                // found the key+                return true;+            }+            L = L->Next;+        }+        return false;+    }++    /**+     * @brief Inserts in-place if the key is not mapped, does nothing if the key already exists.+     *+     * @param H is the access lane.+     *+     * @param N is a node initialized with the mapped value to insert.+     *+     * @param Xs are arguments to forward to the hasher, the comparator and and+     * the constructor of the key.+     *+     *+     * Be Careful: the inserted node becomes available to concurrent lanes as+     * soon as it is inserted, thus concurrent lanes may access the inserted+     * value even before the inserting lane returns from this function.+     * This is the reason why the inserting lane must prepare the inserted+     * node's mapped value prior to calling this function.+     *+     * Be Careful: the given node remains the ownership of the caller unless+     * the returned couple second member is true.+     *+     * Be Careful: the given node may not be inserted if the key already+     * exists.  The caller is in charge of handling that case and either+     * dispose of the node or save it for the next insertion operation.+     *+     * Be Careful: Once the given node is actually inserted, its ownership is+     * transfered to the hash-map. However it remains valid.+     *+     * If the key that compares equal to arguments Xs exists, then nothing is+     * inserted. The returned value is the couple of the pointer to the+     * existing value and the false boolean value.+     *+     * If the key that compares equal to arguments Xs does not exist, then the+     * node N is updated with the key constructed from Xs, and inserted in the+     * hash-map. The returned value is the couple of the pointer to the+     * inserted value and the true boolean value.+     *+     */+    template <class... Args>+    std::pair<const value_type*, bool> get(const lane_id H, const node_type N, Args&&... Xs) {+        // At any time a concurrent lane may insert the key before this lane.+        //+        // The synchronisation point is the atomic compare-and-exchange of the+        // head of the bucket list that must contain the inserted node.+        //+        // The insertion algorithm is as follow:+        //+        // 1) Compute the key hash from Xs.+        //+        // 2) Lock the lane, that also prevent concurrent lanes from growing of+        // the datastructure.+        //+        // 3) Determine the bucket where the element must be inserted.+        //+        // 4) Read the "last known head" of the bucket list. Other lanes+        // inserting in the same bucket may update the bucket head+        // concurrently.+        //+        // 5) Search the bucket list for the key by comparing with Xs starting+        // from the last known head. If it is not the first round of search,+        // then stop searching where the previous round of search started.+        //+        // 6) If the key is found return the couple of the value pointer and+        // false (to indicate that this lane did not insert the node N).+        //+        // 7) It the key is not found prepare N for insertion by updating its+        // key with Xs and chaining the last known head.+        //+        // 8) Try to exchange to last known head with N at the bucket head. The+        // atomic compare and exchange operation guarantees that it only+        // succeed if not other node was inserted in the bucket since we+        // searched it, otherwise it fails when another lane has concurrently+        // inserted a node in the same bucket.+        //+        // 9) If the atomic compare and exchange succeeded, the node has just+        // been inserted by this lane. From now-on other lanes can also see+        // the node. Return the couple of a pointer to the inserted value and+        // the true boolean.+        //+        // 10) If the atomic compare and exchange failed, another node has been+        // inserted by a concurrent lane in the same bucket. A new round of+        // search is required -> restart from step 4.+        //+        //+        // The datastructure is optionaly grown after step 9) before returning.++        const value_type* Value = nullptr;+        bool Inserted = false;++        size_t NewSize;++        // 1)+        const size_t HashValue = Hasher(std::forward<Args>(Xs)...);++        // 2)+        Lanes.lock(H);  // prevent the datastructure from growing++        // 3)+        const size_t Bucket = HashValue % BucketCount;++        // 4)+        // the head of the bucket's list last time we checked+        BucketList* LastKnownHead = Buckets[Bucket].load(std::memory_order_acquire);+        // the head of the bucket's list we already searched from+        BucketList* SearchedFrom = nullptr;+        // the node we want to insert+        BucketList* const Node = static_cast<BucketList*>(N);++        // Loop until either the node is inserted or the key is found in the bucket.+        // Assuming bucket collisions are rare this loop is not executed more than once.+        while (true) {+            // 5)+            // search the key in the bucket, stop where we already search at a+            // previous iteration.+            BucketList* L = LastKnownHead;+            while (L != SearchedFrom) {+                if (EqualTo(L->Value.first, std::forward<Args>(Xs)...)) {+                    // 6)+                    // Found the key, no need to insert.+                    // Although it's not strictly necessary, clear the node+                    // chaining to avoid leaving a dangling pointer there.+                    Value = &(L->Value);+                    Node->Next = nullptr;+                    goto Done;+                }+                L = L->Next;+            }+            SearchedFrom = LastKnownHead;++            // 7)+            // Not found in bucket, prepare node chaining.+            Node->Next = LastKnownHead;+            // The factory step could be done only once, but assuming bucket collisions are+            // rare this whole loop is not executed more than once.+            Factory.replace(const_cast<key_type&>(Node->Value.first), std::forward<Args>(Xs)...);++            // 8)+            // Try to insert the key in front of the bucket's list.+            // This operation also performs step 4) because LastKnownHead is+            // updated in the process.+            if (Buckets[Bucket].compare_exchange_strong(+                        LastKnownHead, Node, std::memory_order_release, std::memory_order_relaxed)) {+                // 9)+                Inserted = true;+                NewSize = ++Size;+                Value = &(Node->Value);+                goto AfterInserted;+            }++            // 10) concurrent insertion detected in this bucket, new round required.+        }++    AfterInserted : {+        if (NewSize > MaxSizeBeforeGrow) {+            tryGrow(H);+        }+    }++    Done:++        Lanes.unlock(H);++        // 6,9)+        return std::make_pair(Value, Inserted);+    }++private:+    // The concurrent lanes manager.+    LanesPolicy Lanes;++    /// Hash function.+    Hash Hasher;++    /// Current number of buckets.+    std::size_t BucketCount;++    /// Atomic pointer to head bucket linked-list head.+    std::unique_ptr<std::atomic<BucketList*>[]> Buckets;++    /// The Equal-to function.+    KeyEqual EqualTo;++    KeyFactory Factory;++    /// Current number of elements stored in the map.+    std::atomic<std::size_t> Size;++    /// Maximum size before the map should grow.+    std::size_t MaxSizeBeforeGrow;++    /// The load-factor of the map.+    double LoadFactor;++    // Grow the datastructure.+    // Must be called while owning lane H.+    bool tryGrow(const lane_id H) {+        Lanes.beforeLockAllBut(H);++        if (Size <= MaxSizeBeforeGrow) {+            // Current size is fine+            Lanes.beforeUnlockAllBut(H);+            return false;+        }++        Lanes.lockAllBut(H);++        {  // safe section++            // Compute the new number of buckets:+            // Chose a prime number of buckets that ensures the desired load factor+            // given the current number of elements in the map.+            const std::size_t CurrentSize = Size;+            assert(LoadFactor > 0);+            const std::size_t NeededBucketCount =+                    static_cast<std::size_t>(std::ceil(static_cast<double>(CurrentSize) / LoadFactor));+            std::size_t NewBucketCount = NeededBucketCount;+            for (std::size_t I = 0; I < details::ToPrime.size(); ++I) {+                const uint64_t N = details::ToPrime[I].first;+                const uint64_t K = details::ToPrime[I].second;+                const uint64_t Prime = (1ULL << N) - K;+                if (Prime >= NeededBucketCount) {+                    NewBucketCount = Prime;+                    break;+                }+            }++            std::unique_ptr<std::atomic<BucketList*>[]> NewBuckets =+                    std::make_unique<std::atomic<BucketList*>[]>(NewBucketCount);++            // Rehash, this operation is costly because it requires to scan+            // the existing elements, compute its hash to find its new bucket+            // and insert in the new bucket.+            //+            // Maybe concurrent lanes could help using some job-stealing algorithm.+            //+            // Use relaxed memory ordering since the whole operation takes place+            // in a critical section.+            for (std::size_t B = 0; B < BucketCount; ++B) {+                BucketList* L = Buckets[B].load(std::memory_order_relaxed);+                while (L) {+                    BucketList* const Elem = L;+                    L = L->Next;++                    const auto& Value = Elem->Value;+                    std::size_t NewHash = Hasher(Value.first);+                    const std::size_t NewBucket = NewHash % NewBucketCount;+                    Elem->Next = NewBuckets[NewBucket].load(std::memory_order_relaxed);+                    NewBuckets[NewBucket].store(Elem, std::memory_order_relaxed);+                }+            }++            Buckets = std::move(NewBuckets);+            BucketCount = NewBucketCount;+            MaxSizeBeforeGrow =+                    static_cast<std::size_t>(std::ceil(static_cast<double>(NewBucketCount) * LoadFactor));+        }++        Lanes.beforeUnlockAllBut(H);+        Lanes.unlockAllBut(H);+        return true;+    }+};++}  // namespace souffle
cbits/souffle/datastructure/EquivalenceRelation.h view
@@ -34,6 +34,7 @@ #include <shared_mutex> #include <stdexcept> #include <tuple>+#include <unordered_set> #include <utility> #include <vector> @@ -100,7 +101,7 @@     bool insert(value_type x, value_type y, operation_hints) {         // indicate that iterators will have to generate on request         this->statesMapStale.store(true, std::memory_order_relaxed);-        bool retval = contains(x, y);+        bool retval = !contains(x, y);         sds.unionNodes(x, y);         return retval;     }@@ -113,14 +114,10 @@         other.genAllDisjointSetLists();          // iterate over partitions at a time-        for (typename StatesMap::chunk it : other.equivalencePartition.getChunks(MAX_THREADS)) {-            for (auto& p : it) {-                value_type rep = p.first;-                StatesList& pl = *p.second;-                const size_t ksize = pl.size();-                for (size_t i = 0; i < ksize; ++i) {-                    this->sds.unionNodes(rep, pl.get(i));-                }+        for (auto&& [rep, pl] : other.equivalencePartition) {+            const std::size_t ksize = pl->size();+            for (std::size_t i = 0; i < ksize; ++i) {+                this->sds.unionNodes(rep, pl->get(i));             }         }         // invalidate iterators unconditionally@@ -128,19 +125,27 @@     }      /**-     * Extend this relation with another relation, expanding this equivalence relation-     * The supplied relation is the old knowledge, whilst this relation only contains-     * explicitly new knowledge. After this operation the "implicitly new tuples" are now-     * explicitly inserted this relation.+     * Extend this relation with another relation, expanding this equivalence+     * relation and inserting it into the other relation.+     *+     * The supplied relation is the old knowledge, whilst this relation only+     * contains explicitly new knowledge. After this operation the "implicitly+     * new tuples" are now explicitly inserted this relation, and all of the new+     * tuples in this relation are inserted into the old relation.      */-    void extend(const EquivalenceRelation<TupleType>& other) {-        // nothing to extend if there's no new/original knowledge-        if (other.size() == 0 || this->size() == 0) return;+    void extendAndInsert(EquivalenceRelation<TupleType>& other) {+        if (other.size() == 0 && this->size() == 0) return; -        this->genAllDisjointSetLists();-        other.genAllDisjointSetLists();+        std::unordered_set<value_type> repsCovered; -        std::set<value_type> repsCovered;+        // This vector holds all of the elements of this equivalence relation+        // that aren't yet in other, which get inserted after extending this+        // relation by other. These operations are interleaved for maximum+        // efficiency - either extend or inserting first would make the other+        // operation unnecessarily slow.+        std::vector<std::pair<value_type, value_type>> toInsert;+        auto size = std::distance(this->sds.sparseToDenseMap.begin(), this->sds.sparseToDenseMap.end());+        toInsert.reserve(size);          // find all the disjoint sets that need to be added to this relation         // that exist in other (and exist in this)@@ -156,8 +161,11 @@                         repsCovered.emplace(rep);                     }                 }+                toInsert.emplace_back(el, this->sds.findNode(el));             }         }+        assert(size >= 0);+        assert(toInsert.size() == (std::size_t)size);          // add the intersecting dj sets into this one         {@@ -173,6 +181,16 @@                 }             }         }++        // Insert all new tuples from this relation into the old relation+        {+            value_type el;+            value_type rep;+            for (std::pair<value_type, value_type> p : toInsert) {+                std::tie(el, rep) = p;+                other.insert(el, rep);+            }+        }     }      /**@@ -192,6 +210,10 @@         return contains(tuple[0], tuple[1]);     }; +    bool contains(const TupleType& tuple) const {+        return contains(tuple[0], tuple[1]);+    };+     void emptyPartition() const {         // delete the beautiful values inside (they're raw ptrs, so they need to be.)         for (auto& pair : equivalencePartition) {@@ -219,14 +241,14 @@      * Size of relation      * @return the sum of the number of pairs per disjoint set      */-    size_t size() const {+    std::size_t size() const {         genAllDisjointSetLists();          statesLock.lock_shared(); -        size_t retVal = 0;+        std::size_t retVal = 0;         for (auto& e : this->equivalencePartition) {-            const size_t s = e.second->size();+            const std::size_t s = e.second->size();             retVal += s * s;         } @@ -241,10 +263,10 @@     class iterator {     public:         typedef std::forward_iterator_tag iterator_category;-        typedef TupleType value_type;-        typedef ptrdiff_t difference_type;-        typedef value_type* pointer;-        typedef value_type& reference;+        using value_type = TupleType;+        using difference_type = ptrdiff_t;+        using pointer = value_type*;+        using reference = value_type&;          // one iterator for signalling the end (simplifies)         explicit iterator(const EquivalenceRelation* br, bool /* signalIsEndIterator */)@@ -447,9 +469,9 @@         typename StatesMap::iterator djSetMapListEnd;          // used for ALL, and POSTERIOR (just a current index in the cList)-        size_t cAnteriorIndex = 0;+        std::size_t cAnteriorIndex = 0;         // used for ALL, and ANTERIOR (just a current index in the cList)-        size_t cPosteriorIndex = 0;+        std::size_t cPosteriorIndex = 0;     };  public:@@ -561,6 +583,11 @@         return end();     } +    iterator lower_bound(const TupleType& entry) const {+        operation_hints hints;+        return lower_bound(entry, hints);+    }+     /**      * This function is only here in order to unify interfaces in InterpreterIndex.      * Unlike the name suggestes, it omit the arguments and simply return the end@@ -573,6 +600,11 @@         return end();     } +    iterator upper_bound(const TupleType& entry) const {+        operation_hints hints;+        return upper_bound(entry, hints);+    }+     /**      * Check emptiness.      */@@ -639,11 +671,11 @@      * @param chunks the number of requested partitions      * @return a list of the iterators as ranges      */-    std::vector<souffle::range<iterator>> partition(size_t chunks) const {+    std::vector<souffle::range<iterator>> partition(std::size_t chunks) const {         // generate all reps         genAllDisjointSetLists(); -        size_t numPairs = this->size();+        std::size_t numPairs = this->size();         if (numPairs == 0) return {};         if (numPairs == 1 || chunks <= 1) return {souffle::make_range(begin(), end())}; @@ -659,9 +691,9 @@         // keep it simple stupid         // just go through and if the size of the binrel is > numpairs/chunks, then generate an anteriorIt for         // each-        const size_t perchunk = numPairs / chunks;+        const std::size_t perchunk = numPairs / chunks;         for (const auto& itp : equivalencePartition) {-            const size_t s = itp.second->size();+            const std::size_t s = itp.second->size();             if (s * s > perchunk) {                 for (const auto& i : *itp.second) {                     ret.push_back(souffle::make_range(anteriorIt(i), end()));@@ -717,8 +749,8 @@         // btree version         emptyPartition(); -        size_t dSetSize = this->sds.ds.a_blocks.size();-        for (size_t i = 0; i < dSetSize; ++i) {+        std::size_t dSetSize = this->sds.ds.a_blocks.size();+        for (std::size_t i = 0; i < dSetSize; ++i) {             typename TupleType::value_type sparseVal = this->sds.toSparse(i);             parent_t rep = this->sds.findNode(sparseVal);
cbits/souffle/datastructure/LambdaBTree.h view
@@ -313,8 +313,8 @@                  // split this node                 auto old_root = this->root;-                idx -= cur->rebalance_or_split(-                        const_cast<typename parenttype::node**>(&this->root), this->root_lock, idx, parents);+                idx -= cur->rebalance_or_split(const_cast<typename parenttype::node**>(&this->root),+                        this->root_lock, static_cast<int>(idx), parents);                  // release parent lock                 for (auto it = parents.rbegin(); it != parents.rend(); ++it) {@@ -346,7 +346,7 @@             assert(cur->numElements < parenttype::node::maxKeys && "Split required!");              // move keys-            for (int j = cur->numElements; j > idx; --j) {+            for (int j = static_cast<int>(cur->numElements); j > idx; --j) {                 cur->keys[j] = cur->keys[j - 1];             } @@ -444,8 +444,8 @@              if (cur->numElements >= parenttype::node::maxKeys) {                 // split this node-                idx -= cur->rebalance_or_split(-                        const_cast<typename parenttype::node**>(&this->root), this->root_lock, idx);+                idx -= cur->rebalance_or_split(const_cast<typename parenttype::node**>(&this->root),+                        this->root_lock, static_cast<int>(idx));                  // insert element in right fragment                 if (((typename parenttype::size_type)idx) > cur->numElements) {
cbits/souffle/datastructure/PiggyList.h view
@@ -3,25 +3,22 @@ #include "souffle/utility/ParallelUtil.h" #include <array> #include <atomic>+#include <cassert> #include <cstring> #include <iostream> #include <iterator>  #ifdef _WIN32+#include <intrin.h> /**  * Some versions of MSVC do not provide a builtin for counting leading zeroes  * like gcc, so we have to implement it ourselves.  */-#if _MSC_VER < 1924-unsigned long __inline __builtin_clzll(unsigned long long value) {-    unsigned long msb = 0;--    if (_BitScanReverse64(&msb, value))-        return 63 - msb;-    else-        return 64;+#if defined(_MSC_VER)+int __inline __builtin_clzll(unsigned long long value) {+    return static_cast<int>(__lzcnt64(value)); }-#endif  // _MSC_VER < 1924+#endif  // _MSC_VER #endif  // _WIN32  using std::size_t;@@ -38,17 +35,17 @@     RandomInsertPiggyList() = default;     // an instance where the initial size is not 65k, and instead is user settable (to a power of     // initialbitsize)-    RandomInsertPiggyList(size_t initialbitsize) : BLOCKBITS(initialbitsize) {}+    RandomInsertPiggyList(std::size_t initialbitsize) : BLOCKBITS(initialbitsize) {}      /** copy constructor */     RandomInsertPiggyList(const RandomInsertPiggyList& other) : BLOCKBITS(other.BLOCKBITS) {         this->numElements.store(other.numElements.load());          // copy blocks from the old lookup table to this one-        for (size_t i = 0; i < maxContainers; ++i) {+        for (std::size_t i = 0; i < maxContainers; ++i) {             if (other.blockLookupTable[i].load() != nullptr) {                 // calculate the size of that block-                const size_t blockSize = INITIALBLOCKSIZE << i;+                const std::size_t blockSize = INITIALBLOCKSIZE << i;                  // allocate that in the new container                 this->blockLookupTable[i].store(new T[blockSize]);@@ -71,25 +68,25 @@         freeList();     } -    inline size_t size() const {+    inline std::size_t size() const {         return numElements.load();     } -    inline T* getBlock(size_t blockNum) const {+    inline T* getBlock(std::size_t blockNum) const {         return blockLookupTable[blockNum];     } -    inline T& get(size_t index) const {-        size_t nindex = index + INITIALBLOCKSIZE;-        size_t blockNum = (63 - __builtin_clzll(nindex));-        size_t blockInd = (nindex) & ((1 << blockNum) - 1);+    inline T& get(std::size_t index) const {+        std::size_t nindex = index + INITIALBLOCKSIZE;+        std::size_t blockNum = (63 - __builtin_clzll(nindex));+        std::size_t blockInd = (nindex) & ((1 << blockNum) - 1);         return this->getBlock(blockNum - BLOCKBITS)[blockInd];     } -    void insertAt(size_t index, T value) {+    void insertAt(std::size_t index, T value) {         // starting with an initial blocksize requires some shifting to transform into a nice powers of two         // series-        size_t blockNum = (63 - __builtin_clzll(index + INITIALBLOCKSIZE)) - BLOCKBITS;+        std::size_t blockNum = (63 - __builtin_clzll(index + INITIALBLOCKSIZE)) - BLOCKBITS;          // allocate the block if not allocated         if (blockLookupTable[blockNum].load() == nullptr) {@@ -110,14 +107,14 @@         freeList();         numElements.store(0);     }-    const size_t BLOCKBITS = 16ul;-    const size_t INITIALBLOCKSIZE = (1ul << BLOCKBITS);+    const std::size_t BLOCKBITS = 16ul;+    const std::size_t INITIALBLOCKSIZE = (((std::size_t)1ul) << BLOCKBITS);      // number of elements currently stored within-    std::atomic<size_t> numElements{0};+    std::atomic<std::size_t> numElements{0};      // 2^64 - 1 elements can be stored (default initialised to nullptrs)-    static constexpr size_t maxContainers = 64;+    static constexpr std::size_t maxContainers = 64;     std::array<std::atomic<T*>, maxContainers> blockLookupTable = {};      // for parallel node insertions@@ -129,7 +126,7 @@     void freeList() {         slock.lock();         // delete all - deleting a nullptr is a no-op-        for (size_t i = 0; i < maxContainers; ++i) {+        for (std::size_t i = 0; i < maxContainers; ++i) {             delete[] blockLookupTable[i].load();             // reset the container within to be empty.             blockLookupTable[i].store(nullptr);@@ -142,7 +139,7 @@ class PiggyList { public:     PiggyList() : num_containers(0), container_size(0), m_size(0) {}-    PiggyList(size_t initialbitsize)+    PiggyList(std::size_t initialbitsize)             : BLOCKBITS(initialbitsize), num_containers(0), container_size(0), m_size(0) {}      /** copy constructor */@@ -152,8 +149,8 @@         m_size.store(other.m_size.load());         // copy each chunk from other into this         // the size of the next container to allocate-        size_t cSize = BLOCKSIZE;-        for (size_t i = 0; i < other.num_containers; ++i) {+        std::size_t cSize = BLOCKSIZE;+        for (std::size_t i = 0; i < other.num_containers; ++i) {             this->blockLookupTable[i] = new T[cSize];             std::memcpy(this->blockLookupTable[i], other.blockLookupTable[i], cSize * sizeof(T));             cSize <<= 1;@@ -177,16 +174,16 @@      *  that haven't had time to had containers created and updated      * @return the number of nodes exist within the list + number of nodes queued to be inserted      */-    inline size_t size() const {+    inline std::size_t size() const {         return m_size.load();     }; -    inline T* getBlock(size_t blocknum) const {+    inline T* getBlock(std::size_t blocknum) const {         return this->blockLookupTable[blocknum];     } -    size_t append(T element) {-        size_t new_index = m_size.fetch_add(1, std::memory_order_acquire);+    std::size_t append(T element) {+        std::size_t new_index = m_size.fetch_add(1, std::memory_order_acquire);          // will this not fit?         if (container_size < new_index + 1) {@@ -206,8 +203,8 @@         return new_index;     } -    size_t createNode() {-        size_t new_index = m_size.fetch_add(1, std::memory_order_acquire);+    std::size_t createNode() {+        std::size_t new_index = m_size.fetch_add(1, std::memory_order_acquire);          // will this not fit?         if (container_size < new_index + 1) {@@ -231,11 +228,11 @@      * @param index position to search      * @return the value at index      */-    inline T& get(size_t index) const {+    inline T& get(std::size_t index) const {         // supa fast 2^16 size first block-        size_t nindex = index + BLOCKSIZE;-        size_t blockNum = (63 - __builtin_clzll(nindex));-        size_t blockInd = (nindex) & ((1 << blockNum) - 1);+        std::size_t nindex = index + BLOCKSIZE;+        std::size_t blockNum = (63 - __builtin_clzll(nindex));+        std::size_t blockInd = (nindex) & ((1 << blockNum) - 1);         return this->getBlock(blockNum - BLOCKBITS)[blockInd];     } @@ -251,18 +248,24 @@         container_size = 0;     } -    class iterator : std::iterator<std::forward_iterator_tag, T> {-        size_t cIndex = 0;+    class iterator {+        std::size_t cIndex = 0;         PiggyList* bl;      public:+        using iterator_category = std::forward_iterator_tag;+        using value_type = T;+        using difference_type = void;+        using pointer = T*;+        using reference = T&;+         // default ctor, to silence         iterator() = default;          /* begin iterator for iterating over all elements */         iterator(PiggyList* bl) : bl(bl){};         /* ender iterator for marking the end of the iteration */-        iterator(PiggyList* bl, size_t beginInd) : cIndex(beginInd), bl(bl){};+        iterator(PiggyList* bl, std::size_t beginInd) : cIndex(beginInd), bl(bl){};          T operator*() {             return bl->get(cIndex);@@ -297,17 +300,17 @@     iterator end() {         return iterator(this, size());     }-    const size_t BLOCKBITS = 16ul;-    const size_t BLOCKSIZE = (1ul << BLOCKBITS);+    const std::size_t BLOCKBITS = 16ul;+    const std::size_t BLOCKSIZE = (((std::size_t)1ul) << BLOCKBITS);      // number of inserted-    std::atomic<size_t> num_containers = 0;-    size_t allocsize = BLOCKSIZE;-    std::atomic<size_t> container_size = 0;-    std::atomic<size_t> m_size = 0;+    std::atomic<std::size_t> num_containers = 0;+    std::size_t allocsize = BLOCKSIZE;+    std::atomic<std::size_t> container_size = 0;+    std::atomic<std::size_t> m_size = 0;      // > 2^64 elements can be stored (default initialise to nullptrs)-    static constexpr size_t max_conts = 64;+    static constexpr std::size_t max_conts = 64;     std::array<T*, max_conts> blockLookupTable = {};      // for parallel node insertions@@ -319,7 +322,7 @@     void freeList() {         sl.lock();         // we don't know which ones are taken up!-        for (size_t i = 0; i < num_containers; ++i) {+        for (std::size_t i = 0; i < num_containers; ++i) {             delete[] blockLookupTable[i];         }         sl.unlock();
+ cbits/souffle/datastructure/RecordTableImpl.h view
@@ -0,0 +1,599 @@+/*+ * Souffle - A Datalog Compiler+ * Copyright (c) 2022, The Souffle Developers. All rights reserved+ * Licensed under the Universal Permissive License v 1.0 as shown at:+ * - https://opensource.org/licenses/UPL+ * - <souffle root>/licenses/SOUFFLE-UPL.txt+ */++/**+ * @file RecordTableImpl.h+ *+ * RecordTable definition+ */++#pragma once++#include "souffle/RamTypes.h"+#include "souffle/RecordTable.h"+#include "souffle/datastructure/ConcurrentFlyweight.h"+#include "souffle/utility/span.h"++#include <cassert>+#include <cstddef>+#include <limits>+#include <memory>+#include <utility>+#include <vector>++namespace souffle {++namespace details {++// Helper to unroll for loop+template <auto Start, auto End, auto Inc, class F>+constexpr void constexpr_for(F&& f) {+    if constexpr (Start < End) {+        f(std::integral_constant<decltype(Start), Start>());+        constexpr_for<Start + Inc, End, Inc>(f);+    }+}++/// @brief The data-type of RamDomain records of any size.+using GenericRecord = std::vector<RamDomain>;++/// @brief The data-type of RamDomain records of specialized size.+template <std::size_t Arity>+using SpecializedRecord = std::array<RamDomain, Arity>;++/// @brief A view in a sequence of RamDomain value.+// TODO: use a `span`.+struct GenericRecordView {+    explicit GenericRecordView(const RamDomain* Data, const std::size_t Arity) : Data(Data), Arity(Arity) {}+    GenericRecordView(const GenericRecordView& Other) : Data(Other.Data), Arity(Other.Arity) {}+    GenericRecordView(GenericRecordView&& Other) : Data(Other.Data), Arity(Other.Arity) {}++    const RamDomain* const Data;+    const std::size_t Arity;++    const RamDomain* data() const {+        return Data;+    }++    const RamDomain& operator[](int I) const {+        assert(I >= 0 && static_cast<std::size_t>(I) < Arity);+        return Data[I];+    }+};++template <std::size_t Arity>+struct SpecializedRecordView {+    explicit SpecializedRecordView(const RamDomain* Data) : Data(Data) {}+    SpecializedRecordView(const SpecializedRecordView& Other) : Data(Other.Data) {}+    SpecializedRecordView(SpecializedRecordView&& Other) : Data(Other.Data) {}++    const RamDomain* const Data;++    const RamDomain* data() const {+        return Data;+    }++    const RamDomain& operator[](int I) const {+        assert(I >= 0 && static_cast<std::size_t>(I) < Arity);+        return Data[I];+    }+};++/// @brief Hash function object for a RamDomain record.+struct GenericRecordHash {+    explicit GenericRecordHash(const std::size_t Arity) : Arity(Arity) {}+    GenericRecordHash(const GenericRecordHash& Other) : Arity(Other.Arity) {}+    GenericRecordHash(GenericRecordHash&& Other) : Arity(Other.Arity) {}++    const std::size_t Arity;+    std::hash<RamDomain> domainHash;++    template <typename T>+    std::size_t operator()(const T& Record) const {+        std::size_t Seed = 0;+        for (std::size_t I = 0; I < Arity; ++I) {+            Seed ^= domainHash(Record[(int)I]) + 0x9e3779b9U + (Seed << 6U) + (Seed >> 2U);+        }+        return Seed;+    }+};++template <std::size_t Arity>+struct SpecializedRecordHash {+    explicit SpecializedRecordHash() {}+    SpecializedRecordHash(const SpecializedRecordHash& Other) : DomainHash(Other.DomainHash) {}+    SpecializedRecordHash(SpecializedRecordHash&& Other) : DomainHash(Other.DomainHash) {}++    std::hash<RamDomain> DomainHash;++    template <typename T>+    std::size_t operator()(const T& Record) const {+        std::size_t Seed = 0;+        constexpr_for<0, Arity, 1>([&](auto I) {+            Seed ^= DomainHash(Record[(int)I]) + 0x9e3779b9U + (Seed << 6U) + (Seed >> 2U);+        });+        return Seed;+    }+};++template <>+struct SpecializedRecordHash<0> {+    explicit SpecializedRecordHash() {}+    SpecializedRecordHash(const SpecializedRecordHash&) {}+    SpecializedRecordHash(SpecializedRecordHash&&) {}++    template <typename T>+    std::size_t operator()(const T&) const {+        return 0;+    }+};++/// @brief Equality function object for RamDomain records.+struct GenericRecordEqual {+    explicit GenericRecordEqual(const std::size_t Arity) : Arity(Arity) {}+    GenericRecordEqual(const GenericRecordEqual& Other) : Arity(Other.Arity) {}+    GenericRecordEqual(GenericRecordEqual&& Other) : Arity(Other.Arity) {}++    const std::size_t Arity;++    template <typename T, typename U>+    bool operator()(const T& A, const U& B) const {+        return (std::memcmp(A.data(), B.data(), Arity * sizeof(RamDomain)) == 0);+    }+};++template <std::size_t Arity>+struct SpecializedRecordEqual {+    explicit SpecializedRecordEqual() {}+    SpecializedRecordEqual(const SpecializedRecordEqual&) {}+    SpecializedRecordEqual(SpecializedRecordEqual&&) {}++    template <typename T, typename U>+    bool operator()(const T& A, const U& B) const {+        constexpr std::size_t Len = Arity * sizeof(RamDomain);+        return (std::memcmp(A.data(), B.data(), Len) == 0);+    }+};++template <>+struct SpecializedRecordEqual<0> {+    explicit SpecializedRecordEqual() {}+    SpecializedRecordEqual(const SpecializedRecordEqual&) {}+    SpecializedRecordEqual(SpecializedRecordEqual&&) {}++    template <typename T, typename U>+    bool operator()(const T&, const U&) const {+        return true;+    }+};++/// @brief Less function object for RamDomain records.+struct GenericRecordLess {+    explicit GenericRecordLess(const std::size_t Arity) : Arity(Arity) {}+    GenericRecordLess(const GenericRecordLess& Other) : Arity(Other.Arity) {}+    GenericRecordLess(GenericRecordLess&& Other) : Arity(Other.Arity) {}++    const std::size_t Arity;++    template <typename T, typename U>+    bool operator()(const T& A, const U& B) const {+        return (std::memcmp(A.data(), B.data(), Arity * sizeof(RamDomain)) < 0);+    }+};++template <std::size_t Arity>+struct SpecializedRecordLess {+    explicit SpecializedRecordLess() {}+    SpecializedRecordLess(const SpecializedRecordLess&) {}+    SpecializedRecordLess(SpecializedRecordLess&&) {}++    template <typename T, typename U>+    bool operator()(const T& A, const U& B) const {+        constexpr std::size_t Len = Arity * sizeof(RamDomain);+        return (std::memcmp(A.data(), B.data(), Len) < 0);+    }+};++template <>+struct SpecializedRecordLess<0> {+    explicit SpecializedRecordLess() {}+    SpecializedRecordLess(const SpecializedRecordLess&) {}+    SpecializedRecordLess(SpecializedRecordLess&&) {}++    template <typename T, typename U>+    bool operator()(const T&, const U&) const {+        return false;+    }+};++/// @brief Compare function object for RamDomain records.+struct GenericRecordCmp {+    explicit GenericRecordCmp(const std::size_t Arity) : Arity(Arity) {}+    GenericRecordCmp(const GenericRecordCmp& Other) : Arity(Other.Arity) {}+    GenericRecordCmp(GenericRecordCmp&& Other) : Arity(Other.Arity) {}++    const std::size_t Arity;++    template <typename T, typename U>+    int operator()(const T& A, const U& B) const {+        return std::memcmp(A.data(), B.data(), Arity * sizeof(RamDomain));+    }+};++template <std::size_t Arity>+struct SpecializedRecordCmp {+    explicit SpecializedRecordCmp() {}+    SpecializedRecordCmp(const SpecializedRecordCmp&) {}+    SpecializedRecordCmp(SpecializedRecordCmp&&) {}++    template <typename T, typename U>+    bool operator()(const T& A, const U& B) const {+        constexpr std::size_t Len = Arity * sizeof(RamDomain);+        return std::memcmp(A.data(), B.data(), Len);+    }+};++template <>+struct SpecializedRecordCmp<0> {+    explicit SpecializedRecordCmp() {}+    SpecializedRecordCmp(const SpecializedRecordCmp&) {}+    SpecializedRecordCmp(SpecializedRecordCmp&&) {}++    template <typename T, typename U>+    bool operator()(const T&, const U&) const {+        return 0;+    }+};++/// @brief Factory of RamDomain record.+struct GenericRecordFactory {+    using value_type = GenericRecord;+    using pointer = GenericRecord*;+    using reference = GenericRecord&;++    explicit GenericRecordFactory(const std::size_t Arity) : Arity(Arity) {}+    GenericRecordFactory(const GenericRecordFactory& Other) : Arity(Other.Arity) {}+    GenericRecordFactory(GenericRecordFactory&& Other) : Arity(Other.Arity) {}++    const std::size_t Arity;++    reference replace(reference Place, const std::vector<RamDomain>& V) {+        assert(V.size() == Arity);+        Place = V;+        return Place;+    }++    reference replace(reference Place, const GenericRecordView& V) {+        Place.clear();+        Place.insert(Place.begin(), V.data(), V.data() + Arity);+        return Place;+    }++    reference replace(reference Place, const RamDomain* V) {+        Place.clear();+        Place.insert(Place.begin(), V, V + Arity);+        return Place;+    }+};++template <std::size_t Arity>+struct SpecializedRecordFactory {+    using value_type = SpecializedRecord<Arity>;+    using pointer = SpecializedRecord<Arity>*;+    using reference = SpecializedRecord<Arity>&;++    explicit SpecializedRecordFactory() {}+    SpecializedRecordFactory(const SpecializedRecordFactory&) {}+    SpecializedRecordFactory(SpecializedRecordFactory&&) {}++    reference replace(reference Place, const SpecializedRecord<Arity>& V) {+        assert(V.size() == Arity);+        Place = V;+        return Place;+    }++    reference replace(reference Place, const SpecializedRecordView<Arity>& V) {+        constexpr std::size_t Len = Arity * sizeof(RamDomain);+        std::memcpy(Place.data(), V.data(), Len);+        return Place;+    }++    reference replace(reference Place, const RamDomain* V) {+        constexpr std::size_t Len = Arity * sizeof(RamDomain);+        std::memcpy(Place.data(), V, Len);+        return Place;+    }+};++template <>+struct SpecializedRecordFactory<0> {+    using value_type = SpecializedRecord<0>;+    using pointer = SpecializedRecord<0>*;+    using reference = SpecializedRecord<0>&;++    explicit SpecializedRecordFactory() {}+    SpecializedRecordFactory(const SpecializedRecordFactory&) {}+    SpecializedRecordFactory(SpecializedRecordFactory&&) {}++    reference replace(reference Place, const SpecializedRecord<0>&) {+        return Place;+    }++    reference replace(reference Place, const SpecializedRecordView<0>&) {+        return Place;+    }++    reference replace(reference Place, const RamDomain*) {+        return Place;+    }+};++}  // namespace details++/** @brief Interface of bidirectional mappping between records and record references. */+class RecordMap {+public:+    virtual ~RecordMap() {}+    virtual void setNumLanes(const std::size_t NumLanes) = 0;+    virtual RamDomain pack(const std::vector<RamDomain>& Vector) = 0;+    virtual RamDomain pack(const RamDomain* Tuple) = 0;+    virtual RamDomain pack(const std::initializer_list<RamDomain>& List) = 0;+    virtual const RamDomain* unpack(RamDomain index) const = 0;+};++/** @brief Bidirectional mappping between records and record references, for any record arity. */+class GenericRecordMap : public RecordMap,+                         protected FlyweightImpl<details::GenericRecord, details::GenericRecordHash,+                                 details::GenericRecordEqual, details::GenericRecordFactory> {+    using Base = FlyweightImpl<details::GenericRecord, details::GenericRecordHash,+            details::GenericRecordEqual, details::GenericRecordFactory>;++    const std::size_t Arity;++public:+    explicit GenericRecordMap(const std::size_t lane_count, const std::size_t arity)+            : Base(lane_count, 8, true, details::GenericRecordHash(arity), details::GenericRecordEqual(arity),+                      details::GenericRecordFactory(arity)),+              Arity(arity) {}++    virtual ~GenericRecordMap() {}++    void setNumLanes(const std::size_t NumLanes) override {+        Base::setNumLanes(NumLanes);+    }++    /** @brief converts record to a record reference */+    RamDomain pack(const std::vector<RamDomain>& Vector) override {+        return findOrInsert(Vector).first;+    };++    /** @brief converts record to a record reference */+    RamDomain pack(const RamDomain* Tuple) override {+        details::GenericRecordView View{Tuple, Arity};+        return findOrInsert(View).first;+    }++    /** @brief converts record to a record reference */+    RamDomain pack(const std::initializer_list<RamDomain>& List) override {+        details::GenericRecordView View{std::data(List), Arity};+        return findOrInsert(View).first;+    }++    /** @brief convert record reference to a record pointer */+    const RamDomain* unpack(RamDomain Index) const override {+        return fetch(Index).data();+    }+};++/** @brief Bidirectional mappping between records and record references, specialized for a record arity. */+template <std::size_t Arity>+class SpecializedRecordMap+        : public RecordMap,+          protected FlyweightImpl<details::SpecializedRecord<Arity>, details::SpecializedRecordHash<Arity>,+                  details::SpecializedRecordEqual<Arity>, details::SpecializedRecordFactory<Arity>> {+    using Record = details::SpecializedRecord<Arity>;+    using RecordView = details::SpecializedRecordView<Arity>;+    using RecordHash = details::SpecializedRecordHash<Arity>;+    using RecordEqual = details::SpecializedRecordEqual<Arity>;+    using RecordFactory = details::SpecializedRecordFactory<Arity>;+    using Base = FlyweightImpl<Record, RecordHash, RecordEqual, RecordFactory>;++public:+    SpecializedRecordMap(const std::size_t LaneCount)+            : Base(LaneCount, 8, true, RecordHash(), RecordEqual(), RecordFactory()) {}++    virtual ~SpecializedRecordMap() {}++    void setNumLanes(const std::size_t NumLanes) override {+        Base::setNumLanes(NumLanes);+    }++    /** @brief converts record to a record reference */+    RamDomain pack(const std::vector<RamDomain>& Vector) override {+        assert(Vector.size() == Arity);+        RecordView View{Vector.data()};+        return Base::findOrInsert(View).first;+    };++    /** @brief converts record to a record reference */+    RamDomain pack(const RamDomain* Tuple) override {+        RecordView View{Tuple};+        return Base::findOrInsert(View).first;+    }++    /** @brief converts record to a record reference */+    RamDomain pack(const std::initializer_list<RamDomain>& List) override {+        assert(List.size() == Arity);+        RecordView View{std::data(List)};+        return Base::findOrInsert(View).first;+    }++    /** @brief convert record reference to a record pointer */+    const RamDomain* unpack(RamDomain Index) const override {+        return Base::fetch(Index).data();+    }+};++/** Record map specialized for arity 0 */+template <>+class SpecializedRecordMap<0> : public RecordMap {+    // The empty record always at index 1+    // The index 0 of each map is reserved.+    static constexpr RamDomain EmptyRecordIndex = 1;++    // To comply with previous behavior, the empty record+    // has no data:+    const RamDomain* EmptyRecordData = nullptr;++public:+    SpecializedRecordMap(const std::size_t /* LaneCount */) {}++    virtual ~SpecializedRecordMap() {}++    void setNumLanes(const std::size_t) override {}++    /** @brief converts record to a record reference */+    RamDomain pack([[maybe_unused]] const std::vector<RamDomain>& Vector) override {+        assert(Vector.size() == 0);+        return EmptyRecordIndex;+    };++    /** @brief converts record to a record reference */+    RamDomain pack(const RamDomain*) override {+        return EmptyRecordIndex;+    }++    /** @brief converts record to a record reference */+    RamDomain pack([[maybe_unused]] const std::initializer_list<RamDomain>& List) override {+        assert(List.size() == 0);+        return EmptyRecordIndex;+    }++    /** @brief convert record reference to a record pointer */+    const RamDomain* unpack([[maybe_unused]] RamDomain Index) const override {+        assert(Index == EmptyRecordIndex);+        return EmptyRecordData;+    }+};++/** A concurrent Record Table with some specialized record maps. */+template <std::size_t... SpecializedArities>+class SpecializedRecordTable : public RecordTable {+private:+    // The current size of the Maps vector.+    std::size_t Size;++    // The record maps, indexed by arity.+    std::vector<RecordMap*> Maps;++    // The concurrency manager.+    mutable ConcurrentLanes Lanes;++    template <std::size_t Arity, std::size_t... Arities>+    void CreateSpecializedMaps() {+        if (Arity >= Size) {+            Size = Arity + 1;+            Maps.reserve(Size);+            Maps.resize(Size);+        }+        Maps[Arity] = new SpecializedRecordMap<Arity>(Lanes.lanes());+        if constexpr (sizeof...(Arities) > 0) {+            CreateSpecializedMaps<Arities...>();+        }+    }++public:+    /** @brief Construct a record table with the number of concurrent access lanes. */+    SpecializedRecordTable(const std::size_t LaneCount) : Size(0), Lanes(LaneCount) {+        CreateSpecializedMaps<SpecializedArities...>();+    }++    SpecializedRecordTable() : SpecializedRecordTable(1) {}++    virtual ~SpecializedRecordTable() {+        for (auto Map : Maps) {+            delete Map;+        }+    }++    /**+     * @brief set the number of concurrent access lanes.+     * Not thread-safe, use only when the datastructure is not being used.+     */+    virtual void setNumLanes(const std::size_t NumLanes) override {+        Lanes.setNumLanes(NumLanes);+        for (auto& Map : Maps) {+            if (Map) {+                Map->setNumLanes(NumLanes);+            }+        }+    }++    /** @brief convert tuple to record reference */+    virtual RamDomain pack(const RamDomain* Tuple, const std::size_t Arity) override {+        auto Guard = Lanes.guard();+        return lookupMap(Arity).pack(Tuple);+    }++    /** @brief convert tuple to record reference */+    virtual RamDomain pack(const std::initializer_list<RamDomain>& List) override {+        auto Guard = Lanes.guard();+        return lookupMap(List.size()).pack(std::data(List));+    }++    /** @brief convert record reference to a record */+    virtual const RamDomain* unpack(const RamDomain Ref, const std::size_t Arity) const override {+        auto Guard = Lanes.guard();+        return lookupMap(Arity).unpack(Ref);+    }++private:+    /** @brief lookup RecordMap for a given arity; the map for that arity must exist. */+    RecordMap& lookupMap(const std::size_t Arity) const {+        assert(Arity < Size && "Lookup for an arity while there is no record for that arity.");+        auto* Map = Maps[Arity];+        assert(Map != nullptr && "Lookup for an arity while there is no record for that arity.");+        return *Map;+    }++    /** @brief lookup RecordMap for a given arity; if it does not exist, create new RecordMap */+    RecordMap& lookupMap(const std::size_t Arity) {+        if (Arity < Size) {+            auto* Map = Maps[Arity];+            if (Map) {+                return *Map;+            }+        }++        createMap(Arity);+        return *Maps[Arity];+    }++    /** @brief create the RecordMap for the given arity. */+    void createMap(const std::size_t Arity) {+        Lanes.beforeLockAllBut();+        if (Arity < Size && Maps[Arity] != nullptr) {+            // Map of required arity has been created concurrently+            Lanes.beforeUnlockAllBut();+            return;+        }+        Lanes.lockAllBut();++        if (Arity >= Size) {+            Size = Arity + 1;+            Maps.reserve(Size);+            Maps.resize(Size);+        }+        Maps[Arity] = new GenericRecordMap(Lanes.lanes(), Arity);++        Lanes.beforeUnlockAllBut();+        Lanes.unlockAllBut();+    }+};++}  // namespace souffle
+ cbits/souffle/datastructure/SymbolTableImpl.h view
@@ -0,0 +1,133 @@+/*+ * Souffle - A Datalog Compiler+ * Copyright (c) 2022, The Souffle Developers. All rights reserved+ * Licensed under the Universal Permissive License v 1.0 as shown at:+ * - https://opensource.org/licenses/UPL+ * - <souffle root>/licenses/SOUFFLE-UPL.txt+ */++/**+ * @file SymbolTableImpl.h+ *+ * SymbolTable definition+ */++#pragma once++#include "souffle/SymbolTable.h"+#include "souffle/datastructure/ConcurrentFlyweight.h"+#include "souffle/utility/MiscUtil.h"+#include "souffle/utility/ParallelUtil.h"+#include "souffle/utility/StreamUtil.h"++#include <algorithm>+#include <cstdlib>+#include <deque>+#include <initializer_list>+#include <iostream>+#include <memory>+#include <string>+#include <unordered_map>+#include <utility>+#include <vector>++namespace souffle {++/**+ * @class SymbolTableImpl+ *+ * Implementation of the symbol table.+ */+class SymbolTableImpl : public SymbolTable, protected FlyweightImpl<std::string> {+private:+    using Base = FlyweightImpl<std::string>;++public:+    class IteratorImpl : public SymbolTableIteratorInterface, private Base::iterator {+    public:+        IteratorImpl(Base::iterator&& it) : Base::iterator(it) {}++        IteratorImpl(const Base::iterator& it) : Base::iterator(it) {}++        const std::pair<const std::string, const std::size_t>& get() const {+            return **this;+        }++        bool equals(const SymbolTableIteratorInterface& other) {+            return (*this) == static_cast<const IteratorImpl&>(other);+        }++        SymbolTableIteratorInterface& incr() {+            ++(*this);+            return *this;+        }++        std::unique_ptr<SymbolTableIteratorInterface> copy() const {+            return std::make_unique<IteratorImpl>(*this);+        }+    };++    using iterator = SymbolTable::Iterator;++    /** @brief Construct a symbol table with the given number of concurrent access lanes. */+    SymbolTableImpl(const std::size_t LaneCount = 1) : Base(LaneCount) {}++    /** @brief Construct a symbol table with the given initial symbols. */+    SymbolTableImpl(std::initializer_list<std::string> symbols) : Base(1, symbols.size()) {+        for (const auto& symbol : symbols) {+            findOrInsert(symbol);+        }+    }++    /** @brief Construct a symbol table with the given number of concurrent access lanes and initial symbols.+     */+    SymbolTableImpl(const std::size_t LaneCount, std::initializer_list<std::string> symbols)+            : Base(LaneCount, symbols.size()) {+        for (const auto& symbol : symbols) {+            findOrInsert(symbol);+        }+    }++    /**+     * @brief Set the number of concurrent access lanes.+     * This function is not thread-safe, do not call when other threads are using the datastructure.+     */+    void setNumLanes(const std::size_t NumLanes) {+        Base::setNumLanes(NumLanes);+    }++    iterator begin() const override {+        return SymbolTable::Iterator(std::make_unique<IteratorImpl>(Base::begin()));+    }++    iterator end() const override {+        return SymbolTable::Iterator(std::make_unique<IteratorImpl>(Base::end()));+    }++    bool weakContains(const std::string& symbol) const override {+        return Base::weakContains(symbol);+    }++    RamDomain encode(const std::string& symbol) override {+        return Base::findOrInsert(symbol).first;+    }++    const std::string& decode(const RamDomain index) const override {+        return Base::fetch(index);+    }++    RamDomain unsafeEncode(const std::string& symbol) override {+        return encode(symbol);+    }++    const std::string& unsafeDecode(const RamDomain index) const override {+        return decode(index);+    }++    std::pair<RamDomain, bool> findOrInsert(const std::string& symbol) override {+        auto Res = Base::findOrInsert(symbol);+        return std::make_pair(static_cast<RamDomain>(Res.first), Res.second);+    }+};++}  // namespace souffle
cbits/souffle/datastructure/Table.h view
@@ -49,11 +49,17 @@     std::size_t count = 0;  public:-    class iterator : public std::iterator<std::forward_iterator_tag, T> {+    class iterator {         Block* block;         unsigned pos;      public:+        using iterator_category = std::forward_iterator_tag;+        using value_type = T;+        using difference_type = void;+        using pointer = T*;+        using reference = T&;+         iterator(Block* block = nullptr, unsigned pos = 0) : block(block), pos(pos) {}          iterator(const iterator&) = default;
cbits/souffle/datastructure/UnionFind.h view
@@ -18,6 +18,7 @@  #include "souffle/datastructure/LambdaBTree.h" #include "souffle/datastructure/PiggyList.h"+#include "souffle/utility/MiscUtil.h" #include <atomic> #include <cstddef> #include <cstdint>@@ -67,7 +68,7 @@     /**      * Return the number of elements in this disjoint set (not the number of pairs)      */-    inline size_t size() {+    inline std::size_t size() {         auto sz = a_blocks.size();         return sz;     };@@ -191,7 +192,7 @@      */     inline block_t makeNode() {         // make node and find out where we've added it-        size_t nodeDetails = a_blocks.createNode();+        std::size_t nodeDetails = a_blocks.createNode();          a_blocks.get(nodeDetails).store(pr2b(nodeDetails, 0)); @@ -341,7 +342,7 @@         toDense(val);     }; -    /* whether we the supplied node exists */+    /* whether the supplied node exists */     inline bool nodeExists(const SparseDomain val) const {         return sparseToDenseMap.contains({val, -1});     };
cbits/souffle/io/IOSystem.h view
@@ -1,6 +1,6 @@ /*  * Souffle - A Datalog Compiler- * Copyright (c) 2013, 2014, Oracle and/or its affiliates. All rights reserved+ * Copyright (c) 2021, The Souffle Developers. All rights reserved  * Licensed under the Universal Permissive License v 1.0 as shown at:  * - https://opensource.org/licenses/UPL  * - <souffle root>/licenses/SOUFFLE-UPL.txt@@ -15,6 +15,7 @@ #pragma once  #include "souffle/RamTypes.h"+#include "souffle/RecordTable.h" #include "souffle/SymbolTable.h" #include "souffle/io/ReadStream.h" #include "souffle/io/ReadStreamCSV.h"@@ -34,7 +35,6 @@ #include <string>  namespace souffle {-class RecordTable;  class IOSystem { public:
cbits/souffle/io/ReadStream.h view
@@ -1,6 +1,6 @@ /*  * Souffle - A Datalog Compiler- * Copyright (c) 2013, 2014, Oracle and/or its affiliates. All rights reserved+ * Copyright (c) 2021, The Souffle Developers. All rights reserved  * Licensed under the Universal Permissive License v 1.0 as shown at:  * - https://opensource.org/licenses/UPL  * - <souffle root>/licenses/SOUFFLE-UPL.txt@@ -42,8 +42,6 @@ public:     template <typename T>     void readAll(T& relation) {-        auto lease = symbolTable.acquireLock();-        (void)lease;         while (const auto next = readNextTuple()) {             const RamDomain* ramDomain = next.get();             relation.insert(ramDomain);@@ -60,9 +58,9 @@      * @param consumed - if not nullptr: number of characters read.      *      */-    RamDomain readRecord(const std::string& source, const std::string& recordTypeName, size_t pos = 0,-            size_t* charactersRead = nullptr) {-        const size_t initial_position = pos;+    RamDomain readRecord(const std::string& source, const std::string& recordTypeName, std::size_t pos = 0,+            std::size_t* charactersRead = nullptr) {+        const std::size_t initial_position = pos;          // Check if record type information are present         auto&& recordInfo = types["records"][recordTypeName];@@ -80,15 +78,15 @@         }          auto&& recordTypes = recordInfo["types"];-        const size_t recordArity = recordInfo["arity"].long_value();+        const std::size_t recordArity = recordInfo["arity"].long_value();          std::vector<RamDomain> recordValues(recordArity);          consumeChar(source, '[', pos); -        for (size_t i = 0; i < recordArity; ++i) {+        for (std::size_t i = 0; i < recordArity; ++i) {             const std::string& recordType = recordTypes[i].string_value();-            size_t consumed = 0;+            std::size_t consumed = 0;              if (i > 0) {                 consumeChar(source, ',', pos);@@ -96,7 +94,7 @@             consumeWhiteSpace(source, pos);             switch (recordType[0]) {                 case 's': {-                    recordValues[i] = symbolTable.unsafeLookup(readUntil(source, ",]", pos, &consumed));+                    recordValues[i] = symbolTable.encode(readSymbol(source, ",]", pos, &consumed));                     break;                 }                 case 'i': {@@ -132,11 +130,14 @@         return recordTable.pack(recordValues.data(), recordValues.size());     } -    RamDomain readADT(const std::string& source, const std::string& adtName, size_t pos = 0,-            size_t* charactersRead = nullptr) {-        const size_t initial_position = pos;+    RamDomain readADT(const std::string& source, const std::string& adtName, std::size_t pos = 0,+            std::size_t* charactersRead = nullptr) {+        const std::size_t initial_position = pos; -        // Branch will are encoded as [branchIdx, [branchValues...]].+        // Branch will are encoded as one of the:+        // [branchIdx, [branchValues...]]+        // [branchIdx, branchValue]+        // branchIdx         RamDomain branchIdx = -1;          auto&& adtInfo = types["ADTs"][adtName];@@ -148,11 +149,12 @@          // Consume initial character         consumeChar(source, '$', pos);-        std::string constructor = readAlphanumeric(source, pos);+        std::string constructor = readIdentifier(source, pos);          json11::Json branchInfo = [&]() -> json11::Json {             for (auto branch : branches.array_items()) {                 ++branchIdx;+                 if (branch["name"].string_value() == constructor) {                     return branch;                 }@@ -169,19 +171,25 @@             if (charactersRead != nullptr) {                 *charactersRead = pos - initial_position;             }++            if (adtInfo["enum"].bool_value()) {+                return branchIdx;+            }+             RamDomain emptyArgs = recordTable.pack(toVector<RamDomain>().data(), 0);-            return recordTable.pack(toVector<RamDomain>(branchIdx, emptyArgs).data(), 2);+            const RamDomain record[] = {branchIdx, emptyArgs};+            return recordTable.pack(record, 2);         }          consumeChar(source, '(', pos);          std::vector<RamDomain> branchArgs(branchTypes.size()); -        for (size_t i = 0; i < branchTypes.size(); ++i) {+        for (std::size_t i = 0; i < branchTypes.size(); ++i) {             auto argType = branchTypes[i].string_value();             assert(!argType.empty()); -            size_t consumed = 0;+            std::size_t consumed = 0;              if (i > 0) {                 consumeChar(source, ',', pos);@@ -190,7 +198,7 @@              switch (argType[0]) {                 case 's': {-                    branchArgs[i] = symbolTable.unsafeLookup(readUntil(source, ",)", pos, &consumed));+                    branchArgs[i] = symbolTable.encode(readSymbol(source, ",)", pos, &consumed));                     break;                 }                 case 'i': {@@ -233,31 +241,35 @@             }         }(); -        return recordTable.pack(toVector<RamDomain>(branchIdx, branchValue).data(), 2);+        RamDomain rec[2] = {branchIdx, branchValue};+        return recordTable.pack(rec, 2);     }      /**-     * Read the next alphanumeric sequence (corresponding to IDENT).+     * Read the next alphanumeric + ('_', '?') sequence (corresponding to IDENT).      * Consume preceding whitespace.      * TODO (darth_tytus): use std::string_view?      */-    std::string readAlphanumeric(const std::string& source, size_t& pos) {+    std::string readIdentifier(const std::string& source, std::size_t& pos) {         consumeWhiteSpace(source, pos);         if (pos >= source.length()) {             throw std::invalid_argument("Unexpected end of input");         } -        const size_t bgn = pos;-        while (pos < source.length() && std::isalnum(static_cast<unsigned char>(source[pos]))) {+        const std::size_t bgn = pos;+        while (pos < source.length()) {+            unsigned char ch = static_cast<unsigned char>(source[pos]);+            bool valid = std::isalnum(ch) || ch == '_' || ch == '?';+            if (!valid) break;             ++pos;         }          return source.substr(bgn, pos - bgn);     } -    std::string readUntil(const std::string& source, const std::string stopChars, const size_t pos,-            size_t* charactersRead) {-        size_t endOfSymbol = source.find_first_of(stopChars, pos);+    std::string readUntil(const std::string& source, const std::string& stopChars, const std::size_t pos,+            std::size_t* charactersRead) {+        std::size_t endOfSymbol = source.find_first_of(stopChars, pos);          if (endOfSymbol == std::string::npos) {             throw std::invalid_argument("Unexpected end of input");@@ -268,10 +280,85 @@         return source.substr(pos, *charactersRead);     } +    std::string readQuotedSymbol(const std::string& source, std::size_t pos, std::size_t* charactersRead) {+        const std::size_t start = pos;+        const std::size_t end = source.length();++        const char quoteMark = source[pos];+        ++pos;++        const std::size_t startOfSymbol = pos;+        std::size_t endOfSymbol = std::string::npos;+        bool hasEscaped = false;++        bool escaped = false;+        while (pos < end) {+            if (escaped) {+                hasEscaped = true;+                escaped = false;+                ++pos;+                continue;+            }++            const char c = source[pos];+            if (c == quoteMark) {+                endOfSymbol = pos;+                ++pos;+                break;+            }+            if (c == '\\') {+                escaped = true;+            }+            ++pos;+        }++        if (endOfSymbol == std::string::npos) {+            throw std::invalid_argument("Unexpected end of input");+        }++        *charactersRead = pos - start;++        std::size_t lengthOfSymbol = endOfSymbol - startOfSymbol;++        // fast handling of symbol without escape sequence+        if (!hasEscaped) {+            return source.substr(startOfSymbol, lengthOfSymbol);+        } else {+            // slow handling of symbol with escape sequence+            std::string symbol;+            symbol.reserve(lengthOfSymbol);+            bool escaped = false;+            for (std::size_t pos = startOfSymbol; pos < endOfSymbol; ++pos) {+                char ch = source[pos];+                if (escaped || ch != '\\') {+                    symbol.push_back(ch);+                    escaped = false;+                } else {+                    escaped = true;+                }+            }+            return symbol;+        }+    }+     /**+     * Read the next symbol.+     * It is either a double-quoted symbol with backslash-escaped chars, or the+     * longuest sequence that do not contains any of the given stopChars.+     * */+    std::string readSymbol(const std::string& source, const std::string& stopChars, const std::size_t pos,+            std::size_t* charactersRead) {+        if (source[pos] == '"') {+            return readQuotedSymbol(source, pos, charactersRead);+        } else {+            return readUntil(source, stopChars, pos, charactersRead);+        }+    }++    /**      * Read past given character, consuming any preceding whitespace.      */-    void consumeChar(const std::string& str, char c, size_t& pos) {+    void consumeChar(const std::string& str, char c, std::size_t& pos) {         consumeWhiteSpace(str, pos);         if (pos >= str.length()) {             throw std::invalid_argument("Unexpected end of input");@@ -287,7 +374,7 @@     /**      * Advance position in the string until first non-whitespace character.      */-    void consumeWhiteSpace(const std::string& str, size_t& pos) {+    void consumeWhiteSpace(const std::string& str, std::size_t& pos) {         while (pos < str.length() && std::isspace(static_cast<unsigned char>(str[pos]))) {             ++pos;         }
cbits/souffle/io/ReadStreamCSV.h view
@@ -1,6 +1,6 @@ /*  * Souffle - A Datalog Compiler- * Copyright (c) 2013, 2014, Oracle and/or its affiliates. All rights reserved+ * Copyright (c) 2021, The Souffle Developers. All rights reserved  * Licensed under the Universal Permissive License v 1.0 as shown at:  * - https://opensource.org/licenses/UPL  * - <souffle root>/licenses/SOUFFLE-UPL.txt@@ -15,6 +15,7 @@ #pragma once  #include "souffle/RamTypes.h"+#include "souffle/RecordTable.h" #include "souffle/SymbolTable.h" #include "souffle/io/ReadStream.h" #include "souffle/utility/ContainerUtil.h"@@ -40,15 +41,21 @@ #include <vector>  namespace souffle {-class RecordTable;  class ReadStreamCSV : public ReadStream { public:     ReadStreamCSV(std::istream& file, const std::map<std::string, std::string>& rwOperation,             SymbolTable& symbolTable, RecordTable& recordTable)             : ReadStream(rwOperation, symbolTable, recordTable),-              delimiter(getOr(rwOperation, "delimiter", "\t")), file(file), lineNumber(0),+              rfc4180(getOr(rwOperation, "rfc4180", "false") == std::string("true")),+              delimiter(getOr(rwOperation, "delimiter", (rfc4180 ? "," : "\t"))), file(file), lineNumber(0),               inputMap(getInputColumnMap(rwOperation, static_cast<unsigned int>(arity))) {+        if (rfc4180 && delimiter.find('"') != std::string::npos) {+            std::stringstream errorMessage;+            errorMessage << "CSV delimiter cannot contain '\"' character when rfc4180 is enabled.";+            throw std::invalid_argument(errorMessage.str());+        }+         while (inputMap.size() < arity) {             int size = static_cast<int>(inputMap.size());             inputMap[size] = size;@@ -67,7 +74,7 @@             return nullptr;         }         std::string line;-        Own<RamDomain[]> tuple = std::make_unique<RamDomain[]>(typeAttributes.size());+        Own<RamDomain[]> tuple = mk<RamDomain[]>(typeAttributes.size());          if (!getline(file, line)) {             return nullptr;@@ -78,12 +85,11 @@         }         ++lineNumber; -        size_t start = 0;-        size_t end = 0;-        size_t columnsFilled = 0;+        std::size_t start = 0;+        std::size_t columnsFilled = 0;         for (uint32_t column = 0; columnsFilled < arity; column++) {-            size_t charactersRead = 0;-            std::string element = nextElement(line, start, end);+            std::size_t charactersRead = 0;+            std::string element = nextElement(line, start);             if (inputMap.count(column) == 0) {                 continue;             }@@ -93,7 +99,7 @@                 auto&& ty = typeAttributes.at(inputMap[column]);                 switch (ty[0]) {                     case 's': {-                        tuple[inputMap[column]] = symbolTable.unsafeLookup(element);+                        tuple[inputMap[column]] = symbolTable.encode(element);                         charactersRead = element.size();                         break;                     }@@ -139,7 +145,7 @@      * Read an unsigned element. Possible bases are 2, 10, 16      * Base is indicated by the first two chars.      */-    RamUnsigned readRamUnsigned(const std::string& element, size_t& charactersRead) {+    RamUnsigned readRamUnsigned(const std::string& element, std::size_t& charactersRead) {         // Sanity check         assert(element.size() > 0); @@ -156,13 +162,64 @@         return value;     } -    std::string nextElement(const std::string& line, size_t& start, size_t& end) {+    std::string nextElement(const std::string& line, std::size_t& start) {         std::string element; +        if (rfc4180) {+            if (line[start] == '"') {+                // quoted field+                const std::size_t end = line.length();+                std::size_t pos = start + 1;+                bool foundEndQuote = false;+                while (pos < end) {+                    char c = line[pos++];+                    if (c == '"' && (pos < end) && line[pos] == '"') {+                        // two double-quote => one double-quote+                        element.push_back('"');+                        ++pos;+                    } else if (c == '"') {+                        foundEndQuote = true;+                        break;+                    } else {+                        element.push_back(c);+                    }+                }++                if (!foundEndQuote) {+                    // missing closing quote+                    std::stringstream errorMessage;+                    errorMessage << "Unbalanced field quote in line " << lineNumber << "; ";+                    throw std::invalid_argument(errorMessage.str());+                }++                // field must be immediately followed by delimiter or end of line+                if (pos != line.length()) {+                    std::size_t nextDelimiter = line.find(delimiter, pos);+                    if (nextDelimiter != pos) {+                        std::stringstream errorMessage;+                        errorMessage << "Separator expected immediately after quoted field in line "+                                     << lineNumber << "; ";+                        throw std::invalid_argument(errorMessage.str());+                    }+                }++                start = pos + delimiter.size();+                return element;+            } else {+                // non-quoted field, span until next delimiter or end of line+                const std::size_t end = std::min(line.find(delimiter, start), line.length());+                element = line.substr(start, end - start);+                start = end + delimiter.size();++                return element;+            }+        }++        std::size_t end = start;         // Handle record/tuple delimiter coincidence.         if (delimiter.find(',') != std::string::npos) {             int record_parens = 0;-            size_t next_delimiter = line.find(delimiter, start);+            std::size_t next_delimiter = line.find(delimiter, start);              // Find first delimiter after the record.             while (end < std::min(next_delimiter, line.length()) || record_parens != 0) {@@ -190,7 +247,7 @@             // Handle the end-of-the-line case where parenthesis are unbalanced.             if (record_parens != 0) {                 std::stringstream errorMessage;-                errorMessage << "Unbalanced record parenthesis " << lineNumber << "; ";+                errorMessage << "Unbalanced record parenthesis in line " << lineNumber << "; ";                 throw std::invalid_argument(errorMessage.str());             }         } else {@@ -234,9 +291,10 @@         return inputColumnMap;     } +    const bool rfc4180;     const std::string delimiter;     std::istream& file;-    size_t lineNumber;+    std::size_t lineNumber;     std::map<int, int> inputMap; }; @@ -248,7 +306,10 @@               baseName(souffle::baseName(getFileName(rwOperation))),               fileHandle(getFileName(rwOperation), std::ios::in | std::ios::binary) {         if (!fileHandle.is_open()) {-            throw std::invalid_argument("Cannot open fact file " + baseName + "\n");+            // suppress error message in case file cannot be open when flag -w is set+            if (getOr(rwOperation, "no-warn", "false") != "true") {+                throw std::invalid_argument("Cannot open fact file " + baseName + "\n");+            }         }         // Strip headers if we're using them         if (getOr(rwOperation, "headers", "false") == "true") {@@ -286,8 +347,8 @@      */     static std::string getFileName(const std::map<std::string, std::string>& rwOperation) {         auto name = getOr(rwOperation, "filename", rwOperation.at("name") + ".facts");-        if (name.front() != '/') {-            name = getOr(rwOperation, "fact-dir", ".") + "/" + name;+        if (!isAbsolute(name)) {+            name = getOr(rwOperation, "fact-dir", ".") + pathSeparator + name;         }         return name;     }
cbits/souffle/io/ReadStreamJSON.h view
@@ -15,6 +15,7 @@ #pragma once  #include "souffle/RamTypes.h"+#include "souffle/RecordTable.h" #include "souffle/SymbolTable.h" #include "souffle/io/ReadStream.h" #include "souffle/utility/ContainerUtil.h"@@ -37,8 +38,14 @@ #include <vector>  namespace souffle {-class RecordTable; +template <typename... T>+[[noreturn]] static void throwError(T const&... t) {+    std::ostringstream out;+    (out << ... << t);+    throw std::runtime_error(out.str());+}+ class ReadStreamJSON : public ReadStream { public:     ReadStreamJSON(std::istream& file, const std::map<std::string, std::string>& rwOperation,@@ -47,18 +54,18 @@         std::string err;         params = Json::parse(rwOperation.at("params"), err);         if (err.length() > 0) {-            fatal("cannot get internal params: %s", err);+            throwError("cannot get internal params: ", err);         }     }  protected:     std::istream& file;-    size_t pos;+    std::size_t pos;     Json jsonSource;     Json params;     bool isInitialized;     bool useObjects;-    std::map<const std::string, const size_t> paramIndex;+    std::map<const std::string, const std::size_t> paramIndex;      Own<RamDomain[]> readNextTuple() override {         // for some reasons we cannot initalized our json objects in constructor@@ -71,22 +78,27 @@             jsonSource = Json::parse(source, error);             // it should be wrapped by an extra array             if (error.length() > 0 || !jsonSource.is_array()) {-                fatal("cannot deserialize json because %s:\n%s", error, source);+                throwError("cannot deserialize json because ", error, ":\n", source);             } +            if (jsonSource.array_items().empty()) {+                // No tuples defined+                return nullptr;+            }+             // we only check the first one, since there are extra checks             // in readNextTupleObject/readNextTupleList             if (jsonSource[0].is_array()) {                 useObjects = false;             } else if (jsonSource[0].is_object()) {                 useObjects = true;-                size_t index_pos = 0;+                std::size_t index_pos = 0;                 for (auto param : params["relation"]["params"].array_items()) {                     paramIndex.insert(std::make_pair(param.string_value(), index_pos));                     index_pos++;                 }             } else {-                fatal("the input is neither list nor object format");+                throwError("the input is neither list nor object format");             }         } @@ -102,16 +114,16 @@             return nullptr;         } -        Own<RamDomain[]> tuple = std::make_unique<RamDomain[]>(typeAttributes.size());+        Own<RamDomain[]> tuple = mk<RamDomain[]>(typeAttributes.size());         const Json& jsonObj = jsonSource[pos];         assert(jsonObj.is_array() && "the input is not json array");         pos++;-        for (size_t i = 0; i < typeAttributes.size(); ++i) {+        for (std::size_t i = 0; i < typeAttributes.size(); ++i) {             try {                 auto&& ty = typeAttributes.at(i);                 switch (ty[0]) {                     case 's': {-                        tuple[i] = symbolTable.unsafeLookup(jsonObj[i].string_value());+                        tuple[i] = symbolTable.encode(jsonObj[i].string_value());                         break;                     }                     case 'r': {@@ -130,7 +142,7 @@                         tuple[i] = static_cast<RamDomain>(jsonObj[i].number_value());                         break;                     }-                    default: fatal("invalid type attribute: `%c`", ty[0]);+                    default: throwError("invalid type attribute: '", ty[0], "'");                 }             } catch (...) {                 std::stringstream errorMessage;@@ -160,13 +172,13 @@          assert(source.is_array() && "the input is not json array");         auto&& recordTypes = recordInfo["types"];-        const size_t recordArity = recordInfo["arity"].long_value();+        const std::size_t recordArity = recordInfo["arity"].long_value();         std::vector<RamDomain> recordValues(recordArity);-        for (size_t i = 0; i < recordArity; ++i) {+        for (std::size_t i = 0; i < recordArity; ++i) {             const std::string& recordType = recordTypes[i].string_value();             switch (recordType[0]) {                 case 's': {-                    recordValues[i] = symbolTable.unsafeLookup(source[i].string_value());+                    recordValues[i] = symbolTable.encode(source[i].string_value());                     break;                 }                 case 'r': {@@ -185,7 +197,7 @@                     recordValues[i] = static_cast<RamDomain>(source[i].number_value());                     break;                 }-                default: fatal("invalid type attribute");+                default: throwError("invalid type attribute");             }         } @@ -197,7 +209,7 @@             return nullptr;         } -        Own<RamDomain[]> tuple = std::make_unique<RamDomain[]>(typeAttributes.size());+        Own<RamDomain[]> tuple = mk<RamDomain[]>(typeAttributes.size());         const Json& jsonObj = jsonSource[pos];         assert(jsonObj.is_object() && "the input is not json object");         pos++;@@ -205,13 +217,13 @@             try {                 // get the corresponding position by parameter name                 if (paramIndex.find(p.first) == paramIndex.end()) {-                    fatal("invalid parameter: %s", p.first);+                    throwError("invalid parameter: ", p.first);                 }-                size_t i = paramIndex.at(p.first);+                std::size_t i = paramIndex.at(p.first);                 auto&& ty = typeAttributes.at(i);                 switch (ty[0]) {                     case 's': {-                        tuple[i] = symbolTable.unsafeLookup(p.second.string_value());+                        tuple[i] = symbolTable.encode(p.second.string_value());                         break;                     }                     case 'r': {@@ -230,7 +242,7 @@                         tuple[i] = static_cast<RamDomain>(p.second.number_value());                         break;                     }-                    default: fatal("invalid type attribute: `%c`", ty[0]);+                    default: throwError("invalid type attribute: '", ty[0], "'");                 }             } catch (...) {                 std::stringstream errorMessage;@@ -245,9 +257,9 @@     RamDomain readNextElementObject(const Json& source, const std::string& recordTypeName) {         auto&& recordInfo = types["records"][recordTypeName];         const std::string recordName = recordTypeName.substr(2);-        std::map<const std::string, const size_t> recordIndex;+        std::map<const std::string, const std::size_t> recordIndex; -        size_t index_pos = 0;+        std::size_t index_pos = 0;         for (auto param : params["records"][recordName]["params"].array_items()) {             recordIndex.insert(std::make_pair(param.string_value(), index_pos));             index_pos++;@@ -264,19 +276,19 @@          assert(source.is_object() && "the input is not json object");         auto&& recordTypes = recordInfo["types"];-        const size_t recordArity = recordInfo["arity"].long_value();+        const std::size_t recordArity = recordInfo["arity"].long_value();         std::vector<RamDomain> recordValues(recordArity);         recordValues.reserve(recordIndex.size());         for (auto readParam : source.object_items()) {             // get the corresponding position by parameter name             if (recordIndex.find(readParam.first) == recordIndex.end()) {-                fatal("invalid parameter: %s", readParam.first);+                throwError("invalid parameter: ", readParam.first);             }-            size_t i = recordIndex.at(readParam.first);+            std::size_t i = recordIndex.at(readParam.first);             auto&& type = recordTypes[i].string_value();             switch (type[0]) {                 case 's': {-                    recordValues[i] = symbolTable.unsafeLookup(readParam.second.string_value());+                    recordValues[i] = symbolTable.encode(readParam.second.string_value());                     break;                 }                 case 'r': {@@ -295,7 +307,7 @@                     recordValues[i] = static_cast<RamDomain>(readParam.second.number_value());                     break;                 }-                default: fatal("invalid type attribute: `%c`", type[0]);+                default: throwError("invalid type attribute: '", type[0], "'");             }         } @@ -307,6 +319,8 @@ public:     ReadFileJSON(const std::map<std::string, std::string>& rwOperation, SymbolTable& symbolTable,             RecordTable& recordTable)+            // FIXME: This is bordering on UB - we're passing an unconstructed+            // object (fileHandle) to the base class             : ReadStreamJSON(fileHandle, rwOperation, symbolTable, recordTable),               baseName(souffle::baseName(getFileName(rwOperation))),               fileHandle(getFileName(rwOperation), std::ios::in | std::ios::binary) {
cbits/souffle/io/ReadStreamSQLite.h view
@@ -1,6 +1,6 @@ /*  * Souffle - A Datalog Compiler- * Copyright (c) 2013, 2014, Oracle and/or its affiliates. All rights reserved+ * Copyright (c) 2021, The Souffle Developers. All rights reserved  * Licensed under the Universal Permissive License v 1.0 as shown at:  * - https://opensource.org/licenses/UPL  * - <souffle root>/licenses/SOUFFLE-UPL.txt@@ -15,6 +15,7 @@ #pragma once  #include "souffle/RamTypes.h"+#include "souffle/RecordTable.h" #include "souffle/SymbolTable.h" #include "souffle/io/ReadStream.h" #include "souffle/utility/MiscUtil.h"@@ -30,7 +31,6 @@ #include <sqlite3.h>  namespace souffle {-class RecordTable;  class ReadStreamSQLite : public ReadStream { public:@@ -60,23 +60,28 @@             return nullptr;         } -        Own<RamDomain[]> tuple = std::make_unique<RamDomain[]>(arity + auxiliaryArity);+        Own<RamDomain[]> tuple = mk<RamDomain[]>(arity + auxiliaryArity);          uint32_t column;         for (column = 0; column < arity; column++) {-            std::string element(reinterpret_cast<const char*>(sqlite3_column_text(selectStatement, column)));--            if (element.empty()) {+            std::string element;+            if (0 == sqlite3_column_bytes(selectStatement, column)) {                 element = "n/a";+            } else {+                element = reinterpret_cast<const char*>(sqlite3_column_text(selectStatement, column));++                if (element.empty()) {+                    element = "n/a";+                }             }              try {                 auto&& ty = typeAttributes.at(column);                 switch (ty[0]) {-                    case 's': tuple[column] = symbolTable.unsafeLookup(element); break;+                    case 's': tuple[column] = symbolTable.encode(element); break;+                    case 'f': tuple[column] = ramBitCast(RamFloatFromString(element)); break;                     case 'i':                     case 'u':-                    case 'f':                     case 'r': tuple[column] = RamSignedFromString(element); break;                     default: fatal("invalid type attribute: `%c`", ty[0]);                 }@@ -122,6 +127,7 @@     }      void openDB() {+        sqlite3_config(SQLITE_CONFIG_URI, 1);         if (sqlite3_open(dbFilename.c_str(), &db) != SQLITE_OK) {             throwError("SQLite error in sqlite3_open: ");         }@@ -165,6 +171,10 @@         // convert dbname to filename         auto name = getOr(rwOperation, "dbname", rwOperation.at("name") + ".sqlite");         name = getOr(rwOperation, "filename", name);++        if (name.rfind("file:", 0) == 0 || name.rfind(":memory:", 0) == 0) {+            return name;+        }          if (name.front() != '/') {             name = getOr(rwOperation, "fact-dir", ".") + "/" + name;
cbits/souffle/io/SerialisationStream.h view
@@ -18,6 +18,7 @@  #include "souffle/RamTypes.h" +#include "souffle/utility/StringUtil.h" #include "souffle/utility/json11.h" #include <cassert> #include <cstddef>@@ -43,7 +44,7 @@     using RO = std::conditional_t<readOnlyTables, const A, A>;      SerialisationStream(RO<SymbolTable>& symTab, RO<RecordTable>& recTab, Json types,-            std::vector<std::string> relTypes, size_t auxArity = 0)+            std::vector<std::string> relTypes, std::size_t auxArity = 0)             : symbolTable(symTab), recordTable(recTab), types(std::move(types)),               typeAttributes(std::move(relTypes)), arity(typeAttributes.size() - auxArity),               auxiliaryArity(auxArity) {}@@ -59,31 +60,44 @@         std::string parseErrors;         types = Json::parse(rwOperation.at("types"), parseErrors);         assert(parseErrors.size() == 0 && "Internal JSON parsing failed.");+        if (rwOperation.count("params") > 0) {+            params = Json::parse(rwOperation.at("params"), parseErrors);+            assert(parseErrors.size() == 0 && "Internal JSON parsing failed.");+        } else {+            params = Json::object();+        }++        auxiliaryArity = RamSignedFromString(getOr(rwOperation, "auxArity", "0"));+         setupFromJson();     }      RO<SymbolTable>& symbolTable;     RO<RecordTable>& recordTable;     Json types;+    Json params;     std::vector<std::string> typeAttributes; -    size_t arity = 0;-    size_t auxiliaryArity = 0;+    std::size_t arity = 0;+    std::size_t auxiliaryArity = 0;  private:     void setupFromJson() {         auto&& relInfo = types["relation"];-        arity = static_cast<size_t>(relInfo["arity"].long_value());-        auxiliaryArity = static_cast<size_t>(relInfo["auxArity"].long_value());+        arity = static_cast<std::size_t>(relInfo["arity"].long_value());          assert(relInfo["types"].is_array());         auto&& relTypes = relInfo["types"].array_items();-        assert(relTypes.size() == (arity + auxiliaryArity));+        assert(relTypes.size() == arity); -        for (size_t i = 0; i < arity + auxiliaryArity; ++i) {-            auto&& type = relTypes[i].string_value();-            assert(!type.empty() && "malformed types tag");-            typeAttributes.push_back(type);+        for (const auto& jsonType : relTypes) {+            const auto& typeString = jsonType.string_value();+            assert(!typeString.empty() && "malformed types tag");+            typeAttributes.push_back(typeString);+        }++        for (std::size_t i = 0; i < auxiliaryArity; i++) {+            typeAttributes.push_back("i:number");         }     } };
cbits/souffle/io/WriteStream.h view
@@ -1,6 +1,6 @@ /*  * Souffle - A Datalog Compiler- * Copyright (c) 2013, 2014, Oracle and/or its affiliates. All rights reserved+ * Copyright (c) 2021, The Souffle Developers. All rights reserved  * Licensed under the Universal Permissive License v 1.0 as shown at:  * - https://opensource.org/licenses/UPL  * - <souffle root>/licenses/SOUFFLE-UPL.txt@@ -22,6 +22,7 @@ #include "souffle/utility/json11.h" #include <cassert> #include <cstddef>+#include <iomanip> #include <map> #include <memory> #include <ostream>@@ -43,8 +44,6 @@         if (summary) {             return writeSize(relation.size());         }-        auto lease = symbolTable.acquireLock();-        (void)lease;  // silence "unused variable" warning         if (arity == 0) {             if (relation.begin() != relation.end()) {                 writeNullary();@@ -72,9 +71,14 @@      template <typename Tuple>     void writeNext(const Tuple tuple) {-        writeNextTuple(tuple.data);+        using tcb::make_span;+        writeNextTuple(make_span(tuple).data());     } +    virtual void outputSymbol(std::ostream& destination, const std::string& value) {+        destination << value;+    }+     void outputRecord(std::ostream& destination, const RamDomain value, const std::string& name) {         auto&& recordInfo = types["records"][name]; @@ -88,14 +92,14 @@         }          auto&& recordTypes = recordInfo["types"];-        const size_t recordArity = recordInfo["arity"].long_value();+        const std::size_t recordArity = recordInfo["arity"].long_value();          const RamDomain* tuplePtr = recordTable.unpack(value, recordArity);          destination << "[";          // print record's elements-        for (size_t i = 0; i < recordArity; ++i) {+        for (std::size_t i = 0; i < recordArity; ++i) {             if (i > 0) {                 destination << ", ";             }@@ -107,7 +111,7 @@                 case 'i': destination << recordValue; break;                 case 'f': destination << ramBitCast<RamFloat>(recordValue); break;                 case 'u': destination << ramBitCast<RamUnsigned>(recordValue); break;-                case 's': destination << symbolTable.unsafeResolve(recordValue); break;+                case 's': outputSymbol(destination, symbolTable.decode(recordValue)); break;                 case 'r': outputRecord(destination, recordValue, recordType); break;                 case '+': outputADT(destination, recordValue, recordType); break;                 default: fatal("Unsupported type attribute: `%c`", recordType[0]);@@ -120,28 +124,38 @@         auto&& adtInfo = types["ADTs"][name];          assert(!adtInfo.is_null() && "Missing adt type information");+        assert(adtInfo["arity"].long_value() > 0); -        const size_t numBranches = adtInfo["arity"].long_value();-        assert(numBranches > 0);+        // adt is encoded in one of three possible ways:+        // [branchID, [branch_args]] when |branch_args| != 1+        // [branchID, arg] when a branch takes a single argument.+        // branchID when ADT is an enumeration.+        bool isEnum = adtInfo["enum"].bool_value(); -        // adt is encoded as [branchID, [branch_args]] when |branch_args| != 1-        // and as [branchID, arg] when a branch takes a single argument.-        const RamDomain* tuplePtr = recordTable.unpack(value, 2);+        RamDomain branchId = value;+        const RamDomain* branchArgs = nullptr;+        json11::Json branchInfo;+        json11::Json::array branchTypes; -        const RamDomain branchId = tuplePtr[0];-        const RamDomain rawBranchArgs = tuplePtr[1];+        if (!isEnum) {+            const RamDomain* tuplePtr = recordTable.unpack(value, 2); -        auto branchInfo = adtInfo["branches"][branchId];-        auto branchTypes = branchInfo["types"].array_items();+            branchId = tuplePtr[0];+            branchInfo = adtInfo["branches"][branchId];+            branchTypes = branchInfo["types"].array_items(); -        // Prepare branch's arguments for output.-        const RamDomain* branchArgs = [&]() -> const RamDomain* {-            if (branchTypes.size() > 1) {-                return recordTable.unpack(rawBranchArgs, branchTypes.size());-            } else {-                return &rawBranchArgs;-            }-        }();+            // Prepare branch's arguments for output.+            branchArgs = [&]() -> const RamDomain* {+                if (branchTypes.size() > 1) {+                    return recordTable.unpack(tuplePtr[1], branchTypes.size());+                } else {+                    return &tuplePtr[1];+                }+            }();+        } else {+            branchInfo = adtInfo["branches"][branchId];+            branchTypes = branchInfo["types"].array_items();+        }          destination << "$" << branchInfo["name"].string_value(); @@ -150,7 +164,7 @@         }          // Print arguments-        for (size_t i = 0; i < branchTypes.size(); ++i) {+        for (std::size_t i = 0; i < branchTypes.size(); ++i) {             if (i > 0) {                 destination << ", ";             }@@ -160,7 +174,7 @@                 case 'i': destination << branchArgs[i]; break;                 case 'f': destination << ramBitCast<RamFloat>(branchArgs[i]); break;                 case 'u': destination << ramBitCast<RamUnsigned>(branchArgs[i]); break;-                case 's': destination << symbolTable.unsafeResolve(branchArgs[i]); break;+                case 's': outputSymbol(destination, symbolTable.decode(branchArgs[i])); break;                 case 'r': outputRecord(destination, branchArgs[i], argType); break;                 case '+': outputADT(destination, branchArgs[i], argType); break;                 default: fatal("Unsupported type attribute: `%c`", argType[0]);
cbits/souffle/io/WriteStreamCSV.h view
@@ -1,6 +1,6 @@ /*  * Souffle - A Datalog Compiler- * Copyright (c) 2013, 2014, Oracle and/or its affiliates. All rights reserved+ * Copyright (c) 2021, The Souffle Developers. All rights reserved  * Licensed under the Universal Permissive License v 1.0 as shown at:  * - https://opensource.org/licenses/UPL  * - <souffle root>/licenses/SOUFFLE-UPL.txt@@ -15,6 +15,7 @@ #pragma once  #include "souffle/RamTypes.h"+#include "souffle/RecordTable.h" #include "souffle/SymbolTable.h" #include "souffle/io/WriteStream.h" #include "souffle/utility/ContainerUtil.h"@@ -35,21 +36,28 @@  namespace souffle { -class RecordTable;- class WriteStreamCSV : public WriteStream { protected:     WriteStreamCSV(const std::map<std::string, std::string>& rwOperation, const SymbolTable& symbolTable,             const RecordTable& recordTable)             : WriteStream(rwOperation, symbolTable, recordTable),-              delimiter(getOr(rwOperation, "delimiter", "\t")){};+              rfc4180(getOr(rwOperation, "rfc4180", "false") == std::string("true")),+              delimiter(getOr(rwOperation, "delimiter", (rfc4180 ? "," : "\t"))) {+        if (rfc4180 && delimiter.find('"') != std::string::npos) {+            std::stringstream errorMessage;+            errorMessage << "CSV delimiter cannot contain '\"' character when rfc4180 is enabled.";+            throw std::invalid_argument(errorMessage.str());+        }+    }; +    const bool rfc4180;+     const std::string delimiter;      void writeNextTupleCSV(std::ostream& destination, const RamDomain* tuple) {         writeNextTupleElement(destination, typeAttributes.at(0), tuple[0]); -        for (size_t col = 1; col < arity; ++col) {+        for (std::size_t col = 1; col < arity; ++col) {             destination << delimiter;             writeNextTupleElement(destination, typeAttributes.at(col), tuple[col]);         }@@ -57,14 +65,60 @@         destination << "\n";     } +    virtual void outputSymbol(std::ostream& destination, const std::string& value) {+        outputSymbol(destination, value, false);+    }++    void outputSymbol(std::ostream& destination, const std::string& value, bool fieldValue) {+        if (rfc4180) {+            if (!fieldValue) {+                destination << '"';+            }+            destination << '"';++            const std::size_t end = value.length();+            for (std::size_t pos = 0; pos < end; ++pos) {+                char ch = value[pos];+                if (ch == '"') {+                    destination << '\\';+                    destination << '"';+                }+                destination << ch;+            }++            if (!fieldValue) {+                destination << '"';+            }+            destination << '"';+        } else {+            destination << value;+        }+    }+     void writeNextTupleElement(std::ostream& destination, const std::string& type, RamDomain value) {         switch (type[0]) {-            case 's': destination << symbolTable.unsafeResolve(value); break;+            case 's': outputSymbol(destination, symbolTable.decode(value), true); break;             case 'i': destination << value; break;             case 'u': destination << ramBitCast<RamUnsigned>(value); break;             case 'f': destination << ramBitCast<RamFloat>(value); break;-            case 'r': outputRecord(destination, value, type); break;-            case '+': outputADT(destination, value, type); break;+            case 'r':+                if (rfc4180) {+                    destination << '"';+                }+                outputRecord(destination, value, type);+                if (rfc4180) {+                    destination << '"';+                }+                break;+            case '+':+                if (rfc4180) {+                    destination << '"';+                }+                outputADT(destination, value, type);+                if (rfc4180) {+                    destination << '"';+                }+                break;             default: fatal("unsupported type attribute: `%c`", type[0]);         }     }@@ -183,8 +237,9 @@  class WriteCoutPrintSize : public WriteStream { public:-    explicit WriteCoutPrintSize(const std::map<std::string, std::string>& rwOperation)-            : WriteStream(rwOperation, {}, {}), lease(souffle::getOutputLock().acquire()) {+    WriteCoutPrintSize(const std::map<std::string, std::string>& rwOperation, const SymbolTable& symbolTable,+            const RecordTable& recordTable)+            : WriteStream(rwOperation, symbolTable, recordTable), lease(souffle::getOutputLock().acquire()) {         std::cout << rwOperation.at("name") << "\t";     } @@ -240,9 +295,9 @@  class WriteCoutPrintSizeFactory : public WriteStreamFactory { public:-    Own<WriteStream> getWriter(const std::map<std::string, std::string>& rwOperation, const SymbolTable&,-            const RecordTable&) override {-        return mk<WriteCoutPrintSize>(rwOperation);+    Own<WriteStream> getWriter(const std::map<std::string, std::string>& rwOperation,+            const SymbolTable& symbolTable, const RecordTable& recordTable) override {+        return mk<WriteCoutPrintSize>(rwOperation, symbolTable, recordTable);     }     const std::string& getName() const override {         static const std::string name = "stdoutprintsize";
cbits/souffle/io/WriteStreamJSON.h view
@@ -56,7 +56,7 @@         else             destination << "["; -        for (size_t col = 0; col < arity; ++col) {+        for (std::size_t col = 0; col < arity; ++col) {             if (col > 0) {                 destination << ", ";             }@@ -96,7 +96,7 @@             assert(currType.length() > 2 && "Invalid type length");             switch (currType[0]) {                 // since some strings may need to be escaped, we use dump here-                case 's': destination << Json(symbolTable.unsafeResolve(currValue)).dump(); break;+                case 's': destination << Json(symbolTable.decode(currValue)).dump(); break;                 case 'i': destination << currValue; break;                 case 'u': destination << (int)ramBitCast<RamUnsigned>(currValue); break;                 case 'f': destination << ramBitCast<RamFloat>(currValue); break;@@ -109,7 +109,7 @@                     }                      auto&& recordTypes = recordInfo["types"];-                    const size_t recordArity = recordInfo["arity"].long_value();+                    const std::size_t recordArity = recordInfo["arity"].long_value();                     const RamDomain* tuplePtr = recordTable.unpack(currValue, recordArity);                     worklist.push("]");                     for (auto i = (long long)(recordArity - 1); i >= 0; --i) {@@ -151,7 +151,7 @@             assert(currType.length() > 2 && "Invalid type length");             switch (currType[0]) {                 // since some strings may need to be escaped, we use dump here-                case 's': destination << Json(symbolTable.unsafeResolve(currValue)).dump(); break;+                case 's': destination << Json(symbolTable.decode(currValue)).dump(); break;                 case 'i': destination << currValue; break;                 case 'u': destination << (int)ramBitCast<RamUnsigned>(currValue); break;                 case 'f': destination << ramBitCast<RamFloat>(currValue); break;@@ -164,7 +164,7 @@                     }                      auto&& recordTypes = recordInfo["types"];-                    const size_t recordArity = recordInfo["arity"].long_value();+                    const std::size_t recordArity = recordInfo["arity"].long_value();                     const RamDomain* tuplePtr = recordTable.unpack(currValue, recordArity);                     worklist.push("}");                     for (auto i = (long long)(recordArity - 1); i >= 0; --i) {
cbits/souffle/io/WriteStreamSQLite.h view
@@ -1,6 +1,6 @@ /*  * Souffle - A Datalog Compiler- * Copyright (c) 2013, 2014, Oracle and/or its affiliates. All rights reserved+ * Copyright (c) 2021, The Souffle Developers. All rights reserved  * Licensed under the Universal Permissive License v 1.0 as shown at:  * - https://opensource.org/licenses/UPL  * - <souffle root>/licenses/SOUFFLE-UPL.txt@@ -15,6 +15,7 @@ #pragma once  #include "souffle/RamTypes.h"+#include "souffle/RecordTable.h" #include "souffle/SymbolTable.h" #include "souffle/io/WriteStream.h" #include <cassert>@@ -31,8 +32,6 @@  namespace souffle { -class RecordTable;- class WriteStreamSQLite : public WriteStream { public:     WriteStreamSQLite(const std::map<std::string, std::string>& rwOperation, const SymbolTable& symbolTable,@@ -42,10 +41,11 @@         openDB();         createTables();         prepareStatements();-        //        executeSQL("BEGIN TRANSACTION", db);+        executeSQL("BEGIN TRANSACTION", db);     }      ~WriteStreamSQLite() override {+        executeSQL("COMMIT", db);         sqlite3_finalize(insertStatement);         sqlite3_finalize(symbolInsertStatement);         sqlite3_finalize(symbolSelectStatement);@@ -56,7 +56,7 @@     void writeNullary() override {}      void writeNextTuple(const RamDomain* tuple) override {-        for (size_t i = 0; i < arity; i++) {+        for (std::size_t i = 0; i < arity; i++) {             RamDomain value = 0;  // Silence warning              switch (typeAttributes.at(i)[0]) {@@ -65,9 +65,11 @@             }  #if RAM_DOMAIN_SIZE == 64-            if (sqlite3_bind_int64(insertStatement, i + 1, value) != SQLITE_OK) {+            if (sqlite3_bind_int64(insertStatement, static_cast<int>(i + 1),+                        static_cast<sqlite3_int64>(value)) != SQLITE_OK) { #else-            if (sqlite3_bind_int(insertStatement, i + 1, value) != SQLITE_OK) {+            if (sqlite3_bind_int(insertStatement, static_cast<int>(i + 1), static_cast<int>(value)) !=+                    SQLITE_OK) { #endif                 throwError("SQLite error in sqlite3_bind_text: ");             }@@ -102,8 +104,8 @@         throw std::invalid_argument(error.str());     } -    uint64_t getSymbolTableIDFromDB(int index) {-        if (sqlite3_bind_text(symbolSelectStatement, 1, symbolTable.unsafeResolve(index).c_str(), -1,+    uint64_t getSymbolTableIDFromDB(std::size_t index) {+        if (sqlite3_bind_text(symbolSelectStatement, 1, symbolTable.decode(index).c_str(), -1,                     SQLITE_TRANSIENT) != SQLITE_OK) {             throwError("SQLite error in sqlite3_bind_text: ");         }@@ -115,12 +117,12 @@         sqlite3_reset(symbolSelectStatement);         return rowid;     }-    uint64_t getSymbolTableID(int index) {+    uint64_t getSymbolTableID(std::size_t index) {         if (dbSymbolTable.count(index) != 0) {             return dbSymbolTable[index];         } -        if (sqlite3_bind_text(symbolInsertStatement, 1, symbolTable.unsafeResolve(index).c_str(), -1,+        if (sqlite3_bind_text(symbolInsertStatement, 1, symbolTable.decode(index).c_str(), -1,                     SQLITE_TRANSIENT) != SQLITE_OK) {             throwError("SQLite error in sqlite3_bind_text: ");         }@@ -140,8 +142,9 @@     }      void openDB() {+        sqlite3_config(SQLITE_CONFIG_URI, 1);         if (sqlite3_open(dbFilename.c_str(), &db) != SQLITE_OK) {-            throwError("SQLite error in sqlite3_open");+            throwError("SQLite error in sqlite3_open: ");         }         sqlite3_extended_result_codes(db, 1);         executeSQL("PRAGMA synchronous = OFF", db);@@ -210,6 +213,9 @@      void createRelationView() {         // Create view with symbol strings resolved++        const auto columnNames = params["relation"]["params"].array_items();+         std::stringstream createViewText;         createViewText << "CREATE VIEW IF NOT EXISTS '" << relationName << "' AS ";         std::stringstream projectionClause;@@ -218,22 +224,26 @@         std::stringstream whereClause;         bool firstWhere = true;         for (unsigned int i = 0; i < arity; i++) {-            std::string columnName = std::to_string(i);+            const std::string tableColumnName = std::to_string(i);+            const auto& viewColumnName =+                    (columnNames[i].is_string() ? columnNames[i].string_value() : tableColumnName);             if (i != 0) {                 projectionClause << ",";             }             if (typeAttributes.at(i)[0] == 's') {-                projectionClause << "'_symtab_" << columnName << "'.symbol AS '" << columnName << "'";-                fromClause << ",'" << symbolTableName << "' AS '_symtab_" << columnName << "'";+                projectionClause << "'_symtab_" << tableColumnName << "'.symbol AS '" << viewColumnName+                                 << "'";+                fromClause << ",'" << symbolTableName << "' AS '_symtab_" << tableColumnName << "'";                 if (!firstWhere) {                     whereClause << " AND ";                 } else {                     firstWhere = false;                 }-                whereClause << "'_" << relationName << "'.'" << columnName << "' = "-                            << "'_symtab_" << columnName << "'.id";+                whereClause << "'_" << relationName << "'.'" << tableColumnName << "' = "+                            << "'_symtab_" << tableColumnName << "'.id";             } else {-                projectionClause << "'_" << relationName << "'.'" << columnName << "'";+                projectionClause << "'_" << relationName << "'.'" << tableColumnName << "' AS '"+                                 << viewColumnName << "'";             }         }         createViewText << "SELECT " << projectionClause.str() << " FROM " << fromClause.str();@@ -262,6 +272,10 @@         // convert dbname to filename         auto name = getOr(rwOperation, "dbname", rwOperation.at("name") + ".sqlite");         name = getOr(rwOperation, "filename", name);++        if (name.rfind("file:", 0) == 0 || name.rfind(":memory:", 0) == 0) {+            return name;+        }          if (name.front() != '/') {             name = getOr(rwOperation, "output-dir", ".") + "/" + name;
cbits/souffle/io/gzfstream.h view
@@ -1,6 +1,6 @@ /*  * Souffle - A Datalog Compiler- * Copyright (c) 2013, 2014, Oracle and/or its affiliates. All rights reserved+ * Copyright (c) 2021, The Souffle Developers. All rights reserved  * Licensed under the Universal Permissive License v 1.0 as shown at:  * - https://opensource.org/licenses/UPL  * - <souffle root>/licenses/SOUFFLE-UPL.txt@@ -91,8 +91,8 @@             *pptr() = c;             pbump(1);         }-        int toWrite = pptr() - pbase();-        if (gzwrite(fileHandle, pbase(), toWrite) != toWrite) {+        const int toWrite = static_cast<int>(pptr() - pbase());+        if (gzwrite(fileHandle, pbase(), static_cast<unsigned int>(toWrite)) != toWrite) {             return EOF;         }         pbump(-toWrite);@@ -108,13 +108,14 @@             return traits_type::to_int_type(*gptr());         } -        unsigned charsPutBack = gptr() - eback();+        std::size_t charsPutBack = gptr() - eback();         if (charsPutBack > reserveSize) {             charsPutBack = reserveSize;         }         memcpy(buffer + reserveSize - charsPutBack, gptr() - charsPutBack, charsPutBack); -        int charsRead = gzread(fileHandle, buffer + reserveSize, bufferSize - reserveSize);+        int charsRead =+                gzread(fileHandle, buffer + reserveSize, static_cast<unsigned int>(bufferSize - reserveSize));         if (charsRead <= 0) {             return EOF;         }@@ -126,8 +127,8 @@      int sync() override {         if ((pptr() != nullptr) && pptr() > pbase()) {-            int toWrite = pptr() - pbase();-            if (gzwrite(fileHandle, pbase(), toWrite) != toWrite) {+            const int toWrite = static_cast<int>(pptr() - pbase());+            if (gzwrite(fileHandle, pbase(), static_cast<unsigned int>(toWrite)) != toWrite) {                 return -1;             }             pbump(-toWrite);@@ -136,8 +137,8 @@     }  private:-    static constexpr unsigned int bufferSize = 65536;-    static constexpr unsigned int reserveSize = 16;+    static constexpr std::size_t bufferSize = 65536;+    static constexpr std::size_t reserveSize = 16;      char buffer[bufferSize] = {};     gzFile fileHandle = {};
cbits/souffle/utility/CacheUtil.h view
@@ -1,6 +1,6 @@ /*  * Souffle - A Datalog Compiler- * Copyright (c) 2013, Oracle and/or its affiliates. All rights reserved+ * Copyright (c) 2021, The Souffle Developers. All rights reserved  * Licensed under the Universal Permissive License v 1.0 as shown at:  * - https://opensource.org/licenses/UPL  * - <souffle root>/licenses/SOUFFLE-UPL.txt
cbits/souffle/utility/ContainerUtil.h view
@@ -1,6 +1,6 @@ /*  * Souffle - A Datalog Compiler- * Copyright (c) 2013, Oracle and/or its affiliates. All rights reserved+ * Copyright (c) 2021, The Souffle Developers. All rights reserved  * Licensed under the Universal Permissive License v 1.0 as shown at:  * - https://opensource.org/licenses/UPL  * - <souffle root>/licenses/SOUFFLE-UPL.txt@@ -16,11 +16,15 @@  #pragma once +#include "souffle/utility/DynamicCasting.h"+#include "souffle/utility/Iteration.h"+#include "souffle/utility/MiscUtil.h"+#include "souffle/utility/Types.h"+ #include <algorithm> #include <functional> #include <iterator> #include <map>-#include <memory> #include <set> #include <type_traits> #include <utility>@@ -32,17 +36,6 @@ //                           General Container Utilities // ------------------------------------------------------------------------------- -template <typename A>-using Own = std::unique_ptr<A>;--template <typename A>-using VecOwn = std::vector<Own<A>>;--template <typename A, typename B = A, typename... Args>-Own<A> mk(Args&&... xs) {-    return Own<A>(new B(std::forward<Args>(xs)...));-}- /**  * Use to range-for iterate in reverse.  * Assumes `std::rbegin` and `std::rend` are defined for type `A`.@@ -65,25 +58,17 @@  * A utility to check generically whether a given element is contained in a given  * container.  */-template <typename C>+template <typename C, typename = std::enable_if_t<!is_associative<C>>> bool contains(const C& container, const typename C::value_type& element) {     return std::find(container.begin(), container.end(), element) != container.end(); } -// TODO: Detect and generalise to other set types?-template <typename A>-bool contains(const std::set<A>& container, const A& element) {-    return container.find(element) != container.end();-}- /**- * Version of contains specialised for maps.- *- * This workaround is needed because of set container, for which value_type == key_type,- * which is ambiguous in this context.+ * A utility to check generically whether a given key exists within a given+ * associative container.  */-template <typename C>-bool contains(const C& container, const typename C::value_type::first_type& element) {+template <typename C, typename A, typename = std::enable_if_t<is_associative<C>>>+bool contains(const C& container, A&& element) {     return container.find(element) != container.end(); } @@ -91,19 +76,18 @@  * Returns the first element in a container that satisfies a given predicate,  * nullptr otherwise.  */-template <typename C>-typename C::value_type getIf(const C& container, std::function<bool(const typename C::value_type)> pred) {-    auto res = std::find_if(container.begin(), container.end(),-            [&](const typename C::value_type item) { return pred(item); });-    return res == container.end() ? nullptr : *res;+template <typename C, typename F>+auto getIf(C&& container, F&& pred) {+    auto it = std::find_if(container.begin(), container.end(), std::forward<F>(pred));+    return it == container.end() ? nullptr : *it; }  /**  * Get value for a given key; if not found, return default value.  */-template <typename C>+template <typename C, typename A, typename = std::enable_if_t<is_associative<C>>> typename C::mapped_type const& getOr(-        const C& container, typename C::key_type key, const typename C::mapped_type& defaultValue) {+        const C& container, A&& key, const typename C::mapped_type& defaultValue) {     auto it = container.find(key);      if (it != container.end()) {@@ -114,6 +98,14 @@ }  /**+ * Append elements to a container+ */+template <class C, typename R>+void append(C& container, R&& range) {+    container.insert(container.end(), std::begin(range), std::end(range));+}++/**  * A utility function enabling the creation of a vector with a fixed set of  * elements within a single expression. This is the base case covering empty  * vectors.@@ -129,262 +121,32 @@  * of arbitrary length.  */ template <typename T, typename... R>-std::vector<T> toVector(const T& first, const R&... rest) {-    return {first, rest...};+std::vector<T> toVector(T first, R... rest) {+    // Init-lists are effectively const-arrays. You can't `move` out of them.+    // Combine with `vector`s not having variadic constructors, can't do:+    //   `vector{Own<A>{}, Own<A>{}}`+    // This is inexcusably awful and defeats the purpose of having init-lists.+    std::vector<T> xs;+    T ary[] = {std::move(first), std::move(rest)...};+    for (auto& x : ary) {+        xs.push_back(std::move(x));+    }+    return xs; }  /**  * A utility function enabling the creation of a vector of pointers.  */-template <typename T>-std::vector<T*> toPtrVector(const std::vector<std::unique_ptr<T>>& v) {-    std::vector<T*> res;+template <typename A = void, typename T, typename U = std::conditional_t<std::is_same_v<A, void>, T, A>>+std::vector<U*> toPtrVector(const VecOwn<T>& v) {+    std::vector<U*> res;     for (auto& e : v) {         res.push_back(e.get());     }     return res; } -/**- * Applies a function to each element of a vector and returns the results.- */-template <typename A, typename F /* : A -> B */>-auto map(const std::vector<A>& xs, F&& f) {-    std::vector<decltype(f(xs[0]))> ys;-    ys.reserve(xs.size());-    for (auto&& x : xs) {-        ys.emplace_back(f(x));-    }-    return ys;-}- // --------------------------------------------------------------------------------//                             Cloning Utilities-// ---------------------------------------------------------------------------------template <typename A>-auto clone(const std::vector<A*>& xs) {-    std::vector<std::unique_ptr<A>> ys;-    ys.reserve(xs.size());-    for (auto&& x : xs) {-        ys.emplace_back(x ? std::unique_ptr<A>(x->clone()) : nullptr);-    }-    return ys;-}--template <typename A>-auto clone(const std::vector<std::unique_ptr<A>>& xs) {-    std::vector<std::unique_ptr<A>> ys;-    ys.reserve(xs.size());-    for (auto&& x : xs) {-        ys.emplace_back(x ? std::unique_ptr<A>(x->clone()) : nullptr);-    }-    return ys;-}--// --------------------------------------------------------------//                            Iterators-// ---------------------------------------------------------------/**- * A wrapper for an iterator obtaining pointers of a certain type,- * dereferencing values before forwarding them to the consumer.- *- * @tparam Iter ... the type of wrapped iterator- * @tparam T    ... the value to be accessed by the resulting iterator- */-template <typename Iter, typename T = typename std::remove_pointer<typename Iter::value_type>::type>-struct IterDerefWrapper : public std::iterator<std::forward_iterator_tag, T> {-    /* The nested iterator. */-    Iter iter;--public:-    // some constructors-    IterDerefWrapper() = default;-    IterDerefWrapper(const Iter& iter) : iter(iter) {}--    // defaulted copy and move constructors-    IterDerefWrapper(const IterDerefWrapper&) = default;-    IterDerefWrapper(IterDerefWrapper&&) = default;--    // default assignment operators-    IterDerefWrapper& operator=(const IterDerefWrapper&) = default;-    IterDerefWrapper& operator=(IterDerefWrapper&&) = default;--    /* The equality operator as required by the iterator concept. */-    bool operator==(const IterDerefWrapper& other) const {-        return iter == other.iter;-    }--    /* The not-equality operator as required by the iterator concept. */-    bool operator!=(const IterDerefWrapper& other) const {-        return iter != other.iter;-    }--    /* The deref operator as required by the iterator concept. */-    const T& operator*() const {-        return **iter;-    }--    /* Support for the pointer operator. */-    const T* operator->() const {-        return &(**iter);-    }--    /* The increment operator as required by the iterator concept. */-    IterDerefWrapper& operator++() {-        ++iter;-        return *this;-    }-};--/**- * A factory function enabling the construction of a dereferencing- * iterator utilizing the automated deduction of template parameters.- */-template <typename Iter>-IterDerefWrapper<Iter> derefIter(const Iter& iter) {-    return IterDerefWrapper<Iter>(iter);-}--/**- * An iterator to be utilized if there is only a single element to iterate over.- */-template <typename T>-class SingleValueIterator : public std::iterator<std::forward_iterator_tag, T> {-    T value;--    bool end = true;--public:-    SingleValueIterator() = default;--    SingleValueIterator(const T& value) : value(value), end(false) {}--    // a copy constructor-    SingleValueIterator(const SingleValueIterator& other) = default;--    // an assignment operator-    SingleValueIterator& operator=(const SingleValueIterator& other) = default;--    // the equality operator as required by the iterator concept-    bool operator==(const SingleValueIterator& other) const {-        // only equivalent if pointing to the end-        return end && other.end;-    }--    // the not-equality operator as required by the iterator concept-    bool operator!=(const SingleValueIterator& other) const {-        return !(*this == other);-    }--    // the deref operator as required by the iterator concept-    const T& operator*() const {-        return value;-    }--    // support for the pointer operator-    const T* operator->() const {-        return &value;-    }--    // the increment operator as required by the iterator concept-    SingleValueIterator& operator++() {-        end = true;-        return *this;-    }-};--// --------------------------------------------------------------//                             Ranges-// ---------------------------------------------------------------/**- * A utility class enabling representation of ranges by pairing- * two iterator instances marking lower and upper boundaries.- */-template <typename Iter>-struct range {-    // the lower and upper boundary-    Iter a, b;--    // a constructor accepting a lower and upper boundary-    range(Iter a, Iter b) : a(std::move(a)), b(std::move(b)) {}--    // default copy / move and assignment support-    range(const range&) = default;-    range(range&&) = default;-    range& operator=(const range&) = default;--    // get the lower boundary (for for-all loop)-    Iter& begin() {-        return a;-    }-    const Iter& begin() const {-        return a;-    }--    // get the upper boundary (for for-all loop)-    Iter& end() {-        return b;-    }-    const Iter& end() const {-        return b;-    }--    // emptiness check-    bool empty() const {-        return a == b;-    }--    // splits up this range into the given number of partitions-    std::vector<range> partition(int np = 100) {-        // obtain the size-        int n = 0;-        for (auto i = a; i != b; ++i) {-            n++;-        }--        // split it up-        auto s = n / np;-        auto r = n % np;-        std::vector<range> res;-        res.reserve(np);-        auto cur = a;-        auto last = cur;-        int i = 0;-        int p = 0;-        while (cur != b) {-            ++cur;-            i++;-            if (i >= (s + (p < r ? 1 : 0))) {-                res.push_back({last, cur});-                last = cur;-                p++;-                i = 0;-            }-        }-        if (cur != last) {-            res.push_back({last, cur});-        }-        return res;-    }-};--/**- * A utility function enabling the construction of ranges- * without explicitly specifying the iterator type.- *- * @tparam Iter .. the iterator type- * @param a .. the lower boundary- * @param b .. the upper boundary- */-template <typename Iter>-range<Iter> make_range(const Iter& a, const Iter& b) {-    return range<Iter>(a, b);-}--// ------------------------------------------------------------------------------- //                             Equality Utilities // ------------------------------------------------------------------------------- @@ -396,8 +158,8 @@  */ template <typename toType, typename baseType> bool castEq(const baseType* left, const baseType* right) {-    if (auto castedLeft = dynamic_cast<const toType*>(left)) {-        if (auto castedRight = dynamic_cast<const toType*>(right)) {+    if (auto castedLeft = as<toType>(left)) {+        if (auto castedRight = as<toType>(right)) {             return castedLeft == castedRight;         }     }@@ -453,20 +215,63 @@  * targets.  */ template <typename T, template <typename...> class Container>-bool equal_targets(const Container<std::unique_ptr<T>>& a, const Container<std::unique_ptr<T>>& b) {-    return equal_targets(a, b, comp_deref<std::unique_ptr<T>>());+bool equal_targets(const Container<Own<T>>& a, const Container<Own<T>>& b) {+    return equal_targets(a, b, comp_deref<Own<T>>()); } +#ifdef _MSC_VER+// issue:+// https://developercommunity.visualstudio.com/t/c-template-template-not-recognized-as-class-templa/558979+template <typename T>+bool equal_targets(const std::vector<Own<T>>& a, const std::vector<Own<T>>& b) {+    return equal_targets(a, b, comp_deref<Own<T>>());+}++template <typename T>+bool equal_targets(const std::vector<T>& a, const std::vector<T>& b) {+    return equal_targets(a, b, comp_deref<T>());+}+#endif+ /**  * A function testing whether two maps of unique pointers are referencing to equivalent  * targets.  */ template <typename Key, typename Value>-bool equal_targets(-        const std::map<Key, std::unique_ptr<Value>>& a, const std::map<Key, std::unique_ptr<Value>>& b) {-    auto comp = comp_deref<std::unique_ptr<Value>>();+bool equal_targets(const std::map<Key, Own<Value>>& a, const std::map<Key, Own<Value>>& b) {+    auto comp = comp_deref<Own<Value>>();     return equal_targets(             a, b, [&comp](auto& a, auto& b) { return a.first == b.first && comp(a.second, b.second); }); } +/**+ * A function testing whether two maps are equivalent using projected values.+ */+template <typename Key, typename Value, typename F>+bool equal_targets_map(const std::map<Key, Value>& a, const std::map<Key, Value>& b, F&& comp) {+    return equal_targets(+            a, b, [&](auto& a, auto& b) { return a.first == b.first && comp(a.second, b.second); });+}++// -------------------------------------------------------------------------------+//                             Checking Utilities+// -------------------------------------------------------------------------------+template <typename R>+bool allValidPtrs(R const& range) {+    return std::all_of(range.begin(), range.end(), [](auto&& p) { return (bool)p; });+}+ }  // namespace souffle++namespace std {+template <typename Iter, typename F>+struct iterator_traits<souffle::TransformIterator<Iter, F>> {+    using iter_t = std::iterator_traits<Iter>;+    using iter_tag = typename iter_t::iterator_category;+    using difference_type = typename iter_t::difference_type;+    using reference = decltype(std::declval<F&>()(*std::declval<Iter>()));+    using value_type = std::remove_cv_t<std::remove_reference_t<reference>>;+    using iterator_category = std::conditional_t<std::is_base_of_v<std::random_access_iterator_tag, iter_tag>,+            std::random_access_iterator_tag, iter_tag>;+};+}  // namespace std
+ cbits/souffle/utility/DynamicCasting.h view
@@ -0,0 +1,104 @@++/*+ * Souffle - A Datalog Compiler+ * Copyright (c) 2021, The Souffle Developers. All rights reserved.+ * Licensed under the Universal Permissive License v 1.0 as shown at:+ * - https://opensource.org/licenses/UPL+ * - <souffle root>/licenses/SOUFFLE-UPL.txt+ */++/************************************************************************+ *+ * @file DynamicCasting.h+ *+ * Common utilities for dynamic casting.+ *+ ***********************************************************************/++#pragma once++#include "souffle/utility/Types.h"+#include <cassert>+#include <type_traits>++namespace souffle {++/**+ * This class is used to tell as<> that cross-casting is allowed.+ * I use a named type rather than just a bool to make the code stand out.+ */+class AllowCrossCast {};++namespace detail {+template <typename A>+constexpr bool is_valid_cross_cast_option = std::is_same_v<A, void> || std::is_same_v<A, AllowCrossCast>;+}++/**+ * Helpers for `dynamic_cast`ing without having to specify redundant type qualifiers.+ * e.g. `as<AstLiteral>(p)` instead of `as<AstLiteral>(p)`.+ */+template <typename B, typename CastType = void, typename A,+        typename = std::enable_if_t<detail::is_valid_cross_cast_option<CastType>>>+auto as(A* x) {+    if constexpr (!std::is_same_v<CastType, AllowCrossCast> &&+                  !std::is_base_of_v<std::remove_const_t<B>, std::remove_const_t<A>>) {+        static_assert(std::is_base_of_v<std::remove_const_t<A>, std::remove_const_t<B>>,+                "`as<B, A>` does not allow cross-type dyn casts. "+                "(i.e. `as<B, A>` where `B <: A` is not true.) "+                "Such a cast is likely a mistake or typo.");+    }+    return dynamic_cast<copy_const<A, B>*>(x);+}++template <typename B, typename CastType = void, typename A,+        typename = std::enable_if_t<std::is_same_v<CastType, AllowCrossCast> || std::is_base_of_v<A, B>>,+        typename = std::enable_if_t<std::is_class_v<A> && !is_pointer_like<A>>>+auto as(A& x) {+    return as<B, CastType>(&x);+}++template <typename B, typename CastType = void, typename A, typename = std::enable_if_t<is_pointer_like<A>>>+auto as(const A& x) {+    return as<B, CastType>(x.get());+}++template <typename B, typename CastType = void, typename A>+auto as(const std::reference_wrapper<A>& x) {+    return as<B, CastType>(x.get());+}++/**+ * Down-casts and checks the cast has succeeded+ */+template <typename B, typename CastType = void, typename A>+auto& asAssert(A&& a) {+    auto* cast = as<B, CastType>(std::forward<A>(a));+    assert(cast && "Invalid cast");+    return *cast;+}++template <typename B, typename CastType = void, typename A>+Own<B> UNSAFE_cast(Own<A> x) {+    if constexpr (std::is_assignable_v<Own<B>, Own<A>>) {+        return x;+    } else {+        if (!x) return {};++        auto y = Own<B>(as<B, CastType>(x));+        assert(y && "incorrect typed return");+        x.release();  // release after assert so dbgr can report `x` if it fails+        return y;+    }+}++/**+ * Checks if the object of type Source can be casted to type Destination.+ */+template <typename B, typename CastType = void, typename A>+// [[deprecated("Use `as` and implicit boolean conversion instead.")]]+bool isA(A&& src) {+    return as<B, CastType>(std::forward<A>(src));+}++}  // namespace souffle
cbits/souffle/utility/EvaluatorUtil.h view
@@ -16,14 +16,15 @@  #pragma once -#include "souffle/CompiledTuple.h" #include "souffle/RamTypes.h"-#include "tinyformat.h"+#include "souffle/utility/StringUtil.h"+#include "souffle/utility/tinyformat.h"+#include <csignal>  namespace souffle::evaluator {  template <typename A, typename F /* Tuple<RamDomain,1> -> void */>-void runRange(A from, A to, A step, F&& go) {+void runRange(const A from, const A to, const A step, F&& go) { #define GO(x) go(Tuple<RamDomain, 1>{ramBitCast(x)})     if (0 < step) {         for (auto x = from; x < to; x += step) {@@ -41,10 +42,28 @@ }  template <typename A, typename F /* Tuple<RamDomain,1> -> void */>-void runRange(A from, A to, F&& go) {-    return runRange(from, to, A(from <= to ? 1 : -1), std::forward<F>(go));+void runRangeBackward(const A from, const A to, F&& func) {+    assert(from > to);+    if (from > to) {+        for (auto x = from; x > to; --x) {+            func(Tuple<RamDomain, 1>{ramBitCast(x)});+        }+    } } +template <typename A, typename F /* Tuple<RamDomain,1> -> void */>+void runRange(const A from, const A to, F&& go) {+    if constexpr (std::is_unsigned<A>()) {+        if (from <= to) {+            runRange(from, to, static_cast<A>(1U), std::forward<F>(go));+        } else {+            runRangeBackward(from, to, std::forward<F>(go));+        }+    } else {+        return runRange(from, to, A(from <= to ? 1 : -1), std::forward<F>(go));+    }+}+ template <typename A> A symbol2numeric(const std::string& src) {     try {@@ -54,7 +73,10 @@             return RamSignedFromString(src);         } else if constexpr (std::is_same_v<RamUnsigned, A>) {             return RamUnsignedFromString(src);+        } else {+            static_assert(sizeof(A) == 0, "Invalid type specified for symbol2Numeric");         }+     } catch (...) {         tfm::format(std::cerr, "error: wrong string provided by `to_number(\"%s\")` functor.\n", src);         raise(SIGFPE);
cbits/souffle/utility/FileUtil.h view
@@ -1,6 +1,6 @@ /*  * Souffle - A Datalog Compiler- * Copyright (c) 2013, Oracle and/or its affiliates. All rights reserved+ * Copyright (c) 2021, The Souffle Developers. All rights reserved  * Licensed under the Universal Permissive License v 1.0 as shown at:  * - https://opensource.org/licenses/UPL  * - <souffle root>/licenses/SOUFFLE-UPL.txt@@ -16,36 +16,39 @@  #pragma once +#include <algorithm> #include <climits> #include <cstdio> #include <cstdlib>+#include <filesystem> #include <fstream>+#include <map>+#include <optional> #include <sstream> #include <string> #include <utility> #include <sys/stat.h> +// -------------------------------------------------------------------------------+//                               File Utils+// -------------------------------------------------------------------------------+ #ifndef _WIN32 #include <unistd.h> #else+#define NOMINMAX+#define NOGDI #include <fcntl.h> #include <io.h> #include <stdlib.h> #include <windows.h>  // --------------------------------------------------------------------------------//                               File Utils+//                               Windows // ------------------------------------------------------------------------------- -#define X_OK 1 /* execute permission - unsupported in windows*/- #define PATH_MAX 260 -/**- * access and realpath are missing on windows, we use their windows equivalents- * as work-arounds.- */-#define access _access inline char* realpath(const char* path, char* resolved_path) {     return _fullpath(resolved_path, path, PATH_MAX); }@@ -57,19 +60,52 @@ #define pclose _pclose #endif +// -------------------------------------------------------------------------------+//                               All systems+// -------------------------------------------------------------------------------+ namespace souffle { +// The separator in the PATH variable+#ifdef _MSC_VER+const char PATHdelimiter = ';';+const char pathSeparator = '/';+#else+const char PATHdelimiter = ':';+const char pathSeparator = '/';+#endif++inline std::string& makePreferred(std::string& name) {+    std::replace(name.begin(), name.end(), '\\', '/');+    // std::replace(name.begin(), name.end(), '/', pathSeparator);+    return name;+}++inline bool isAbsolute(const std::string& path) {+    std::filesystem::path P(path);+    return P.is_absolute();+}+ /**  *  Check whether a file exists in the file system  */ inline bool existFile(const std::string& name) {+    static std::map<std::string, bool> existFileCache{};+    auto it = existFileCache.find(name);+    if (it != existFileCache.end()) {+        return it->second;+    }+    std::filesystem::path P(name);+    bool result = std::filesystem::exists(P);+    /*bool result = false;     struct stat buffer = {};-    if (stat(name.c_str(), &buffer) == 0) {+    if (stat(P.native().c_str(), &buffer) == 0) {         if ((buffer.st_mode & S_IFMT) != 0) {-            return true;+            result = true;         }-    }-    return false;+    }*/+    existFileCache[name] = result;+    return result; }  /**@@ -88,16 +124,24 @@ /**  * Check whether a given file exists and it is an executable  */+#ifdef _WIN32 inline bool isExecutable(const std::string& name) {+    return existFile(+            name);  // there is no EXECUTABLE bit on Windows, so theoretically any file may be executable+}+#else+inline bool isExecutable(const std::string& name) {     return existFile(name) && (access(name.c_str(), X_OK) == 0); }+#endif  /**  * Simple implementation of a which tool  */ inline std::string which(const std::string& name) {     // Check if name has path components in it and if so return it immediately-    if (name.find('/') != std::string::npos) {+    std::filesystem::path P(name);+    if (P.has_parent_path()) {         return name;     }     // Get PATH from environment, if it exists.@@ -111,8 +155,8 @@     std::string sub;      // Check for existence of a binary called 'name' in PATH-    while (std::getline(sstr, sub, ':')) {-        std::string path = sub + "/" + name;+    while (std::getline(sstr, sub, PATHdelimiter)) {+        std::string path = sub + pathSeparator + name;         if ((::realpath(path.c_str(), buf) != nullptr) && isExecutable(path) && !existDir(path)) {             return buf;         }@@ -127,19 +171,27 @@     if (name.empty()) {         return ".";     }-    size_t lastNotSlash = name.find_last_not_of('/');++    std::filesystem::path P(name);+    if (P.has_parent_path()) {+        return P.parent_path().string();+    } else {+        return ".";+    }++    std::size_t lastNotSlash = name.find_last_not_of(pathSeparator);     // All '/'     if (lastNotSlash == std::string::npos) {         return "/";     }-    size_t leadingSlash = name.find_last_of('/', lastNotSlash);+    std::size_t leadingSlash = name.find_last_of(pathSeparator, lastNotSlash);     // No '/'     if (leadingSlash == std::string::npos) {         return ".";     }     // dirname is '/'     if (leadingSlash == 0) {-        return "/";+        return std::string(1, pathSeparator);     }     return name.substr(0, leadingSlash); }@@ -157,15 +209,17 @@  *  Join two paths together; note that this does not resolve overlaps or relative paths.  */ inline std::string pathJoin(const std::string& first, const std::string& second) {-    unsigned firstPos = static_cast<unsigned>(first.size()) - 1;-    while (first.at(firstPos) == '/') {+    return (std::filesystem::path(first) / std::filesystem::path(second)).string();++    /*unsigned firstPos = static_cast<unsigned>(first.size()) - 1;+    while (first.at(firstPos) == pathSeparator) {         firstPos--;     }     unsigned secondPos = 0;-    while (second.at(secondPos) == '/') {+    while (second.at(secondPos) == pathSeparator) {         secondPos++;     }-    return first.substr(0, firstPos + 1) + '/' + second.substr(secondPos);+    return first.substr(0, firstPos + 1) + pathSeparator + second.substr(secondPos);*/ }  /*@@ -173,18 +227,19 @@  * relative to the directory given by @ base. A path here refers a  * colon-separated list of directories.  */-inline std::string findTool(const std::string& tool, const std::string& base, const std::string& path) {-    std::string dir = dirName(base);+inline std::optional<std::string> findTool(+        const std::string& tool, const std::string& base, const std::string& path) {+    std::filesystem::path dir(dirName(base));     std::stringstream sstr(path);     std::string sub;      while (std::getline(sstr, sub, ':')) {-        std::string subpath = dir + "/" + sub + '/' + tool;-        if (isExecutable(subpath)) {-            return absPath(subpath);+        auto subpath = (dir / sub / tool);+        if (std::filesystem::exists(subpath)) {+            return absPath(subpath.string());         }     }-    return "";+    return {}; }  /*@@ -195,14 +250,14 @@         return ".";     } -    size_t lastNotSlash = filename.find_last_not_of('/');+    std::size_t lastNotSlash = filename.find_last_not_of(pathSeparator);     if (lastNotSlash == std::string::npos) {-        return "/";+        return std::string(1, pathSeparator);     } -    size_t lastSlashBeforeBasename = filename.find_last_of('/', lastNotSlash - 1);+    std::size_t lastSlashBeforeBasename = filename.find_last_of(pathSeparator, lastNotSlash - 1);     if (lastSlashBeforeBasename == std::string::npos) {-        lastSlashBeforeBasename = static_cast<size_t>(-1);+        lastSlashBeforeBasename = static_cast<std::size_t>(-1);     }     return filename.substr(lastSlashBeforeBasename + 1, lastNotSlash - lastSlashBeforeBasename); }@@ -212,12 +267,12 @@  */ inline std::string simpleName(const std::string& path) {     std::string name = baseName(path);-    const size_t lastDot = name.find_last_of('.');+    const std::size_t lastDot = name.find_last_of('.');     // file has no extension     if (lastDot == std::string::npos) {         return name;     }-    const size_t lastSlash = name.find_last_of('/');+    const std::size_t lastSlash = name.find_last_of(pathSeparator);     // last slash occurs after last dot, so no extension     if (lastSlash != std::string::npos && lastSlash > lastDot) {         return name;@@ -231,12 +286,12 @@  */ inline std::string fileExtension(const std::string& path) {     std::string name = path;-    const size_t lastDot = name.find_last_of('.');+    const std::size_t lastDot = name.find_last_of('.');     // file has no extension     if (lastDot == std::string::npos) {         return std::string();     }-    const size_t lastSlash = name.find_last_of('/');+    const std::size_t lastSlash = name.find_last_of(pathSeparator);     // last slash occurs after last dot, so no extension     if (lastSlash != std::string::npos && lastSlash > lastDot) {         return std::string();@@ -250,10 +305,11 @@  */ inline std::string tempFile() { #ifdef _WIN32+    char ctempl[L_tmpnam];     std::string templ;     std::FILE* f = nullptr;     while (f == nullptr) {-        templ = std::tmpnam(nullptr);+        templ = std::tmpnam(ctempl);         f = fopen(templ.c_str(), "wx");     }     fclose(f);@@ -268,13 +324,16 @@ inline std::stringstream execStdOut(char const* cmd) {     FILE* in = popen(cmd, "r");     std::stringstream data;-    while (in != nullptr) {++    if (in == nullptr) {+        return data;+    }++    while (!feof(in)) {         int c = fgetc(in);-        if (feof(in) != 0) {-            break;-        }         data << static_cast<char>(c);     }+     pclose(in);     return data; }
− cbits/souffle/utility/FunctionalUtil.h 
@@ -1,153 +0,0 @@-/*- * Souffle - A Datalog Compiler- * Copyright (c) 2013, Oracle and/or its affiliates. All rights reserved- * Licensed under the Universal Permissive License v 1.0 as shown at:- * - https://opensource.org/licenses/UPL- * - <souffle root>/licenses/SOUFFLE-UPL.txt- */--/************************************************************************- *- * @file FunctionalUtil.h- *- * @brief Datalog project utilities- *- ***********************************************************************/--#pragma once--#include <algorithm>-#include <functional>-#include <utility>-#include <vector>--namespace souffle {--// --------------------------------------------------------------------------------//                              Functional Utils-// ---------------------------------------------------------------------------------/**- * A functor comparing the dereferenced value of a pointer type utilizing a- * given comparator. Its main use case are sets of non-null pointers which should- * be ordered according to the value addressed by the pointer.- */-template <typename T, typename C = std::less<T>>-struct deref_less {-    bool operator()(const T* a, const T* b) const {-        return C()(*a, *b);-    }-};--// --------------------------------------------------------------------------------//                               Lambda Utils-// ---------------------------------------------------------------------------------namespace detail {--template <typename T>-struct lambda_traits_helper;--template <typename R>-struct lambda_traits_helper<R()> {-    using result_type = R;-};--template <typename R, typename A0>-struct lambda_traits_helper<R(A0)> {-    using result_type = R;-    using arg0_type = A0;-};--template <typename R, typename A0, typename A1>-struct lambda_traits_helper<R(A0, A1)> {-    using result_type = R;-    using arg0_type = A0;-    using arg1_type = A1;-};--template <typename R, typename... Args>-struct lambda_traits_helper<R(Args...)> {-    using result_type = R;-};--template <typename R, typename C, typename... Args>-struct lambda_traits_helper<R (C::*)(Args...)> : public lambda_traits_helper<R(Args...)> {};--template <typename R, typename C, typename... Args>-struct lambda_traits_helper<R (C::*)(Args...) const> : public lambda_traits_helper<R (C::*)(Args...)> {};-}  // namespace detail--/**- * A type trait enabling the deduction of type properties of lambdas.- * Those include so far:- *      - the result type (result_type)- *      - the first argument type (arg0_type)- */-template <typename Lambda>-struct lambda_traits : public detail::lambda_traits_helper<decltype(&Lambda::operator())> {};--// --------------------------------------------------------------------------------//                              General Algorithms-// ---------------------------------------------------------------------------------/**- * A generic test checking whether all elements within a container satisfy a- * certain predicate.- *- * @param c the container- * @param p the predicate- * @return true if for all elements x in c the predicate p(x) is true, false- *          otherwise; for empty containers the result is always true- */-template <typename Container, typename UnaryPredicate>-bool all_of(const Container& c, UnaryPredicate p) {-    return std::all_of(c.begin(), c.end(), p);-}--/**- * A generic test checking whether any elements within a container satisfy a- * certain predicate.- *- * @param c the container- * @param p the predicate- * @return true if there is an element x in c such that predicate p(x) is true, false- *          otherwise; for empty containers the result is always false- */-template <typename Container, typename UnaryPredicate>-bool any_of(const Container& c, UnaryPredicate p) {-    return std::any_of(c.begin(), c.end(), p);-}--/**- * A generic test checking whether all elements within a container satisfy a- * certain predicate.- *- * @param c the container- * @param p the predicate- * @return true if for all elements x in c the predicate p(x) is true, false- *          otherwise; for empty containers the result is always true- */-template <typename Container, typename UnaryPredicate>-bool none_of(const Container& c, UnaryPredicate p) {-    return std::none_of(c.begin(), c.end(), p);-}--/**- * Filter a vector to exclude certain elements.- */-template <typename A, typename F>-std::vector<A> filterNot(std::vector<A> xs, F&& f) {-    xs.erase(std::remove_if(xs.begin(), xs.end(), std::forward<F>(f)), xs.end());-    return xs;-}--/**- * Filter a vector to include certain elements.- */-template <typename A, typename F>-std::vector<A> filter(std::vector<A> xs, F&& f) {-    return filterNot(std::move(xs), [&](auto&& x) { return !f(x); });-}--}  // namespace souffle
+ cbits/souffle/utility/General.h view
@@ -0,0 +1,27 @@+/*+ * Souffle - A Datalog Compiler+ * Copyright (c) 2021, The Souffle Developers. All rights reserved+ * Licensed under the Universal Permissive License v 1.0 as shown at:+ * - https://opensource.org/licenses/UPL+ * - <souffle root>/licenses/SOUFFLE-UPL.txt+ */++/************************************************************************+ *+ * @file General.h+ *+ * @brief Lightweight / cheap header for misc utilities.+ *+ * Misc utilities that require non-trivial headers should go in `MiscUtil.h`+ *+ ***********************************************************************/++#if defined(_MSC_VER)+#define SOUFFLE_ALWAYS_INLINE /* TODO: MSVC equiv */+#else+// clang / gcc recognize this attribute+// NB: GCC will only inline when optimisation is on, and will warn about it.+//     Adding `inline` (even though the KW nominally has nothing to do with+//     inlining) will force it to inline in all cases. Lovely.+#define SOUFFLE_ALWAYS_INLINE [[gnu::always_inline]] inline+#endif
+ cbits/souffle/utility/Iteration.h view
@@ -0,0 +1,405 @@+/*+ * Souffle - A Datalog Compiler+ * Copyright (c) 2020, The Souffle Developers. All rights reserved+ * Licensed under the Universal Permissive License v 1.0 as shown at:+ * - https://opensource.org/licenses/UPL+ * - <souffle root>/licenses/SOUFFLE-UPL.txt+ */++/************************************************************************+ *+ * @file Iteration.h+ *+ * @brief Utilities for iterators and ranges+ *+ ***********************************************************************/++#pragma once++#include "souffle/utility/Types.h"++#include <iterator>+#include <type_traits>+#include <utility>+#include <vector>++namespace souffle {++namespace detail {+#ifdef _MSC_VER+#pragma warning(push)+#pragma warning(disable : 4172)+#elif defined(__GNUC__) && (__GNUC__ >= 7)+#pragma GCC diagnostic push+#pragma GCC diagnostic ignored "-Wreturn-local-addr"+#elif defined(__has_warning)+#pragma clang diagnostic push+#if __has_warning("-Wreturn-stack-address")+#pragma clang diagnostic ignored "-Wreturn-stack-address"+#endif+#endif+// This is a helper in the cases when the lambda is stateless+template <typename F>+F makeFun() {+    static_assert(std::is_empty_v<F>);+    // Even thought the lambda is stateless, it has no default ctor+    // Is this gross?  Yes, yes it is.+    // FIXME: Remove after C++20+    typename std::aligned_storage<sizeof(F)>::type fakeLam{};+    return reinterpret_cast<F const&>(fakeLam);+}+#ifdef _MSC_VER+#pragma warning(pop)+#elif defined(__GNUC__) && (__GNUC__ >= 7)+#pragma GCC diagnostic pop+#elif defined(__has_warning)+#pragma clang diagnostic pop+#endif+}  // namespace detail++// -------------------------------------------------------------+//                            Iterators+// -------------------------------------------------------------+/**+ * A wrapper for an iterator that transforms values returned by+ * the underlying iter.+ *+ * @tparam Iter ... the type of wrapped iterator+ * @tparam F    ... the function to apply+ *+ */+template <typename Iter, typename F>+class TransformIterator {+    using iter_t = std::iterator_traits<Iter>;++public:+    using difference_type = typename iter_t::difference_type;+    // TODO: The iterator concept doesn't map correctly to ephemeral views.+    //       e.g. there is no l-value store for a deref.+    //       Figure out what these should be set to.+    using value_type = decltype(std::declval<F>()(*std::declval<Iter>()));+    using pointer = std::remove_reference_t<value_type>*;+    using reference = value_type;+    static_assert(std::is_empty_v<F>, "Function object must be stateless");++    // some constructors+    template <typename = std::enable_if_t<std::is_empty_v<F>>>+    TransformIterator(Iter iter) : TransformIterator(std::move(iter), detail::makeFun<F>()) {}+    TransformIterator(Iter iter, F f) : iter(std::move(iter)), fun(std::move(f)) {}++    /* The equality operator as required by the iterator concept. */+    bool operator==(const TransformIterator& other) const {+        return iter == other.iter;+    }++    /* The not-equality operator as required by the iterator concept. */+    bool operator!=(const TransformIterator& other) const {+        return iter != other.iter;+    }++    bool operator<(TransformIterator const& other) const {+        return iter < other.iter;+    }++    bool operator<=(TransformIterator const& other) const {+        return iter <= other.iter;+    }++    bool operator>(TransformIterator const& other) const {+        return iter > other.iter;+    }++    bool operator>=(TransformIterator const& other) const {+        return iter >= other.iter;+    }++    /* The deref operator as required by the iterator concept. */+    auto operator*() const -> reference {+        return fun(*iter);+    }++    /* Support for the pointer operator. */+    auto operator->() const {+        return &**this;+    }++    /* The increment operator as required by the iterator concept. */+    TransformIterator& operator++() {+        ++iter;+        return *this;+    }++    TransformIterator operator++(int) {+        auto res = *this;+        ++iter;+        return res;+    }++    TransformIterator& operator--() {+        --iter;+        return *this;+    }++    TransformIterator operator--(int) {+        auto res = *this;+        --iter;+        return res;+    }++    TransformIterator& operator+=(difference_type n) {+        iter += n;+        return *this;+    }++    TransformIterator operator+(difference_type n) {+        auto res = *this;+        res += n;+        return res;+    }++    TransformIterator& operator-=(difference_type n) {+        iter -= n;+        return *this;+    }++    TransformIterator operator-(difference_type n) {+        auto res = *this;+        res -= n;+        return res;+    }++    difference_type operator-(TransformIterator const& other) {+        return iter - other.iter;+    }++    auto operator[](difference_type ii) const -> reference {+        return f(iter[ii]);+    }++private:+    /* The nested iterator. */+    Iter iter;+    F fun;+};++template <typename Iter, typename F>+auto operator+(+        typename TransformIterator<Iter, F>::difference_type n, TransformIterator<Iter, F> const& iter) {+    return iter + n;+}++template <typename Iter, typename F>+auto transformIter(Iter&& iter, F&& f) {+    return TransformIterator<remove_cvref_t<Iter>, std::remove_reference_t<F>>(+            std::forward<Iter>(iter), std::forward<F>(f));+}++/**+ * A wrapper for an iterator obtaining pointers of a certain type,+ * dereferencing values before forwarding them to the consumer.+ */+namespace detail {+// HACK: Use explicit structure w/ `operator()` b/c pre-C++20 lambdas do not have copy-assign operators+struct IterTransformDeref {+    template <typename A>+    auto operator()(A&& x) const -> decltype(*x) {+        return *x;+    }+};++// HACK: Use explicit structure w/ `operator()` b/c pre-C++20 lambdas do not have copy-assign operators+struct IterTransformToPtr {+    template <typename A>+    A* operator()(Own<A> const& x) const {+        return x.get();+    }+};++}  // namespace detail++template <typename Iter>+using IterDerefWrapper = TransformIterator<Iter, detail::IterTransformDeref>;++/**+ * A factory function enabling the construction of a dereferencing+ * iterator utilizing the automated deduction of template parameters.+ */+template <typename Iter>+auto derefIter(Iter&& iter) {+    return transformIter(std::forward<Iter>(iter), detail::IterTransformDeref{});+}++/**+ * A factory function that transforms an smart-ptr iter to dumb-ptr iter.+ */+template <typename Iter>+auto ptrIter(Iter&& iter) {+    return transformIter(std::forward<Iter>(iter), detail::IterTransformToPtr{});+}++// -------------------------------------------------------------+//                             Ranges+// -------------------------------------------------------------++/**+ * A utility class enabling representation of ranges by pairing+ * two iterator instances marking lower and upper boundaries.+ */+template <typename Iter>+struct range {+    using iterator = Iter;+    using const_iterator = Iter;++    // the lower and upper boundary+    Iter a, b;++    // a constructor accepting a lower and upper boundary+    range(Iter a, Iter b) : a(std::move(a)), b(std::move(b)) {}++    // default copy / move and assignment support+    range(const range&) = default;+    range(range&&) = default;+    range& operator=(const range&) = default;++    // get the lower boundary (for for-all loop)+    Iter& begin() {+        return a;+    }+    const Iter& begin() const {+        return a;+    }++    // get the upper boundary (for for-all loop)+    Iter& end() {+        return b;+    }+    const Iter& end() const {+        return b;+    }++    // emptiness check+    bool empty() const {+        return a == b;+    }++    // splits up this range into the given number of partitions+    std::vector<range> partition(std::size_t np = 100) {+        // obtain the size+        std::size_t n = 0;+        for (auto i = a; i != b; ++i) {+            n++;+        }++        // split it up+        auto s = n / np;+        auto r = n % np;+        std::vector<range> res;+        res.reserve(np);+        auto cur = a;+        auto last = cur;+        std::size_t i = 0;+        std::size_t p = 0;+        while (cur != b) {+            ++cur;+            i++;+            if (i >= (s + (p < r ? 1 : 0))) {+                res.push_back({last, cur});+                last = cur;+                p++;+                i = 0;+            }+        }+        if (cur != last) {+            res.push_back({last, cur});+        }+        return res;+    }+};++/**+ * A utility function enabling the construction of ranges+ * without explicitly specifying the iterator type.+ *+ * @tparam Iter .. the iterator type+ * @param a .. the lower boundary+ * @param b .. the upper boundary+ */+template <typename Iter>+range<Iter> make_range(const Iter& a, const Iter& b) {+    return range<Iter>(a, b);+}++template <typename Iter, typename F>+auto makeTransformRange(Iter&& begin, Iter&& end, F const& f) {+    return make_range(transformIter(std::forward<Iter>(begin), f), transformIter(std::forward<Iter>(end), f));+}++template <typename R, typename F>+auto makeTransformRange(R&& range, F const& f) {+    return makeTransformRange(range.begin(), range.end(), f);+}++template <typename Iter>+auto makeDerefRange(Iter&& begin, Iter&& end) {+    return make_range(derefIter(std::forward<Iter>(begin)), derefIter(std::forward<Iter>(end)));+}++template <typename R>+auto makePtrRange(R const& xs) {+    return make_range(ptrIter(std::begin(xs)), ptrIter(std::end(xs)));+}++/**+ * This wraps the Range container, and const_casts in place.+ */+template <typename Range, typename F>+class OwningTransformRange {+public:+    using iterator = decltype(transformIter(std::begin(std::declval<Range>()), std::declval<F>()));+    using const_iterator =+            decltype(transformIter(std::begin(std::declval<const Range>()), std::declval<const F>()));++    OwningTransformRange(Range&& range, F f) : range(std::move(range)), f(std::move(f)) {}++    auto begin() {+        return transformIter(std::begin(range), f);+    }++    auto begin() const {+        return transformIter(std::begin(range), f);+    }++    auto cbegin() const {+        return transformIter(std::cbegin(range), f);+    }++    auto end() {+        return transformIter(std::end(range), f);+    }++    auto end() const {+        return transformIter(std::end(range), f);+    }++    auto cend() const {+        return transformIter(std::cend(range), f);+    }++    auto size() const {+        return range.size();+    }++    auto& operator[](std::size_t ii) {+        return begin()[ii];+    }++    auto& operator[](std::size_t ii) const {+        return cbegin()[ii];+    }++private:+    Range range;+    F f;+};++}  // namespace souffle
cbits/souffle/utility/MiscUtil.h view
@@ -1,6 +1,6 @@ /*  * Souffle - A Datalog Compiler- * Copyright (c) 2013, Oracle and/or its affiliates. All rights reserved+ * Copyright (c) 2021, The Souffle Developers. All rights reserved  * Licensed under the Universal Permissive License v 1.0 as shown at:  * - https://opensource.org/licenses/UPL  * - <souffle root>/licenses/SOUFFLE-UPL.txt@@ -16,28 +16,28 @@  #pragma once +#include "souffle/utility/General.h"+#include "souffle/utility/Iteration.h"+#include "souffle/utility/Types.h" #include "tinyformat.h" #include <cassert> #include <chrono>-#include <cstdlib> #include <iostream>+#include <map> #include <memory>+#include <optional>+#include <type_traits> #include <utility>  #ifdef _WIN32+#define NOMINMAX+#define NOGDI #include <fcntl.h> #include <io.h> #include <stdlib.h> #include <windows.h>  /**- * Windows headers define these and they interfere with the standard library- * functions.- */-#undef min-#undef max--/**  * On windows, the following gcc builtins are missing.  *  * In the case of popcountll, __popcnt64 is the windows equivalent.@@ -48,26 +48,39 @@  */ #define __builtin_popcountll __popcnt64 -#if _MSC_VER < 1924-constexpr unsigned long __builtin_ctz(unsigned long value) {+#if defined(_MSC_VER)+// return the number of trailing zeroes in value, or 32 if value is zero.+inline constexpr unsigned long __builtin_ctz(unsigned long value) {     unsigned long trailing_zeroes = 0;+    if (value == 0) return 32;     while ((value = value >> 1) ^ 1) {         ++trailing_zeroes;     }     return trailing_zeroes; } -inline unsigned long __builtin_ctzll(unsigned long long value) {-    unsigned long trailing_zero = 0;+// return the number of trailing zeroes in value, or 64 if value is zero.+inline constexpr int __builtin_ctzll_constexpr(unsigned long long value) {+    int trailing_zeroes = 0; -    if (_BitScanForward64(&trailing_zero, value)) {-        return trailing_zero;+    if (value == 0) return 64;+    while ((value = value >> 1) ^ 1) {+        ++trailing_zeroes;+    }+    return trailing_zeroes;+}++inline int __builtin_ctzll(unsigned long long value) {+    unsigned long trailing_zeroes = 0;++    if (_BitScanForward64(&trailing_zeroes, value)) {+        return static_cast<int>(trailing_zeroes);     } else {-        return 64;+        return 64;  // return 64 like GCC would when value == 0     } }-#endif  // _MSC_VER < 1924-#endif+#endif  // _MSC_VER+#endif  // _WIN32  // ------------------------------------------------------------------------------- //                               Timing Utils@@ -98,16 +111,69 @@ //                             Cloning Utilities // ------------------------------------------------------------------------------- +namespace detail {+// TODO: This function is still used by ram::Node::clone() because it hasn't been+// converted to return Own<>.  Once converted, remove this.+template <typename D, typename B>+Own<D> downCast(B* ptr) {+    // ensure the clone operation casts to appropriate pointer+    static_assert(std::is_base_of_v<std::remove_const_t<B>, std::remove_const_t<D>>,+            "Needs to be able to downcast");+    return Own<D>(ptr);+}++template <typename D, typename B>+Own<D> downCast(Own<B> ptr) {+    // ensure the clone operation casts to appropriate pointer+    static_assert(std::is_base_of_v<std::remove_const_t<B>, std::remove_const_t<D>>,+            "Needs to be able to downcast");+    return Own<D>(static_cast<D*>(ptr.release()));+}++}  // namespace detail+ template <typename A>-std::unique_ptr<A> clone(const A* node) {-    return node ? std::unique_ptr<A>(node->clone()) : nullptr;+std::enable_if_t<!std::is_pointer_v<A> && !is_range_v<A>, Own<A>> clone(const A& node) {+    return detail::downCast<A>(node.cloneImpl()); }  template <typename A>-std::unique_ptr<A> clone(const std::unique_ptr<A>& node) {-    return node ? std::unique_ptr<A>(node->clone()) : nullptr;+Own<A> clone(const A* node) {+    return node ? clone(*node) : nullptr; } +template <typename A>+Own<A> clone(const Own<A>& node) {+    return clone(node.get());+}++template <typename K, typename V>+auto clone(const std::map<K, V>& xs) {+    std::map<K, decltype(clone(std::declval<const V&>()))> ys;+    for (auto&& [k, v] : xs)+        ys.insert({k, clone(v)});+    return ys;+}++/**+ * Clone a range+ */+template <typename R>+auto cloneRange(R const& range) {+    return makeTransformRange(std::begin(range), std::end(range), [](auto const& x) { return clone(x); });+}++/**+ * Clone a range, optionally allowing up-casting the result to D+ */+template <typename D = void, typename R, std::enable_if_t<is_range_v<R>, void*> = nullptr>+auto clone(R const& range) {+    auto rn = cloneRange(range);+    using ValueType = remove_cvref_t<decltype(**std::begin(range))>;+    using ResType = std::conditional_t<std::is_same_v<D, void>, ValueType, D>;+    return VecOwn<ResType>(rn.begin(), rn.end());+}+ template <typename A, typename B> auto clone(const std::pair<A, B>& p) {     return std::make_pair(clone(p.first), clone(p.second));@@ -136,54 +202,10 @@  * pointers are null is also considered equivalent.  */ template <typename T>-bool equal_ptr(const std::unique_ptr<T>& a, const std::unique_ptr<T>& b) {+bool equal_ptr(const Own<T>& a, const Own<T>& b) {     return equal_ptr(a.get(), b.get()); } -template <typename A, typename B>-using copy_const_t = std::conditional_t<std::is_const_v<A>, const B, B>;--/**- * Helpers for `dynamic_cast`ing without having to specify redundant type qualifiers.- * e.g. `as<AstLiteral>(p)` instead of `dynamic_cast<const AstLiteral*>(p.get())`.- */-template <typename B, typename A>-auto as(A* x) {-    static_assert(std::is_base_of_v<A, B>,-            "`as<B, A>` does not allow cross-type dyn casts. "-            "(i.e. `as<B, A>` where `B <: A` is not true.) "-            "Such a cast is likely a mistake or typo.");-    return dynamic_cast<copy_const_t<A, B>*>(x);-}--template <typename B, typename A>-std::enable_if_t<std::is_base_of_v<A, B>, copy_const_t<A, B>*> as(A& x) {-    return as<B>(&x);-}--template <typename B, typename A>-B* as(const std::unique_ptr<A>& x) {-    return as<B>(x.get());-}--/**- * Checks if the object of type Source can be casted to type Destination.- */-template <typename B, typename A>-bool isA(A* x) {-    return dynamic_cast<copy_const_t<A, B>*>(x) != nullptr;-}--template <typename B, typename A>-std::enable_if_t<std::is_base_of_v<A, B>, bool> isA(A& x) {-    return isA<B>(&x);-}--template <typename B, typename A>-bool isA(const std::unique_ptr<A>& x) {-    return isA<B>(x.get());-}- // ------------------------------------------------------------------------------- //                               Error Utilities // -------------------------------------------------------------------------------@@ -198,4 +220,18 @@  // HACK:  Workaround to suppress spurious reachability warnings. #define UNREACHABLE_BAD_CASE_ANALYSIS fatal("unhandled switch branch");++// -------------------------------------------------------------------------------+//                               Other Utilities+// -------------------------------------------------------------------------------++template <typename F>+auto lazy(F f) {+    using A = decltype(f());+    return [cache = std::optional<A>{}, f = std::move(f)]() mutable -> A& {+        if (!cache) cache = f();+        return *cache;+    };+}+ }  // namespace souffle
cbits/souffle/utility/ParallelUtil.h view
@@ -18,7 +18,22 @@ #pragma once  #include <atomic>+#include <cassert>+#include <cstddef>+#include <memory>+#include <new> +// https://bugs.llvm.org/show_bug.cgi?id=41423+#if defined(__cpp_lib_hardware_interference_size) && (__cpp_lib_hardware_interference_size != 201703L)+using std::hardware_constructive_interference_size;+using std::hardware_destructive_interference_size;+#else+// 64 bytes on x86-64 │ L1_CACHE_BYTES │ L1_CACHE_SHIFT │ __cacheline_aligned │+// ...+constexpr std::size_t hardware_constructive_interference_size = 2 * sizeof(max_align_t);+constexpr std::size_t hardware_destructive_interference_size = 2 * sizeof(max_align_t);+#endif+ #ifdef _OPENMP  /**@@ -29,14 +44,32 @@  #ifdef __APPLE__ #define pthread_yield pthread_yield_np+#elif !defined(_MSC_VER)+#include <sched.h>+// pthread_yield is deprecated and should be replaced by sched_yield+#define pthread_yield sched_yield+#elif defined _MSC_VER+#include <thread>+#define NOMINMAX+#include <windows.h>+#define pthread_yield std::this_thread::yield #endif +#ifdef _MSC_VER // support for a parallel region+#define PARALLEL_START __pragma(omp parallel) {+#define PARALLEL_END }++// support for parallel loops+#define pfor __pragma(omp for schedule(dynamic)) for+#else+// support for a parallel region #define PARALLEL_START _Pragma("omp parallel") { #define PARALLEL_END }  // support for parallel loops #define pfor _Pragma("omp for schedule(dynamic)") for+#endif  // spawn and sync are processed sequentially (overhead to expensive) #define task_spawn@@ -55,7 +88,7 @@ #define SECTION_END }  // a macro to create an operation context-#define CREATE_OP_CONTEXT(NAME, INIT) auto NAME = INIT;+#define CREATE_OP_CONTEXT(NAME, INIT) [[maybe_unused]] auto NAME = INIT; #define READ_OP_CONTEXT(NAME) NAME  #else@@ -80,7 +113,7 @@ #define SECTION_END }  // a macro to create an operation context-#define CREATE_OP_CONTEXT(NAME, INIT) auto NAME = INIT;+#define CREATE_OP_CONTEXT(NAME, INIT) [[maybe_unused]] auto NAME = INIT; #define READ_OP_CONTEXT(NAME) NAME  // mark es sequential@@ -93,17 +126,66 @@ #endif  #ifdef IS_PARALLEL+#include <mutex>+#include <vector> #define MAX_THREADS (omp_get_max_threads()) #else #define MAX_THREADS (1) #endif -#ifdef IS_PARALLEL+namespace souffle { -#include <mutex>+struct SeqConcurrentLanes {+    struct TrivialLock {+        ~TrivialLock() {}+    }; -namespace souffle {+    using lane_id = std::size_t;+    using unique_lock_type = TrivialLock; +    explicit SeqConcurrentLanes(std::size_t = 1) {}+    SeqConcurrentLanes(const SeqConcurrentLanes&) = delete;+    SeqConcurrentLanes(SeqConcurrentLanes&&) = delete;++    virtual ~SeqConcurrentLanes() {}++    std::size_t lanes() const {+        return 1;+    }++    void setNumLanes(const std::size_t) {}++    unique_lock_type guard(const lane_id) const {+        return TrivialLock();+    }++    void lock(const lane_id) const {+        return;+    }++    void unlock(const lane_id) const {+        return;+    }++    void beforeLockAllBut(const lane_id) const {+        return;+    }++    void beforeUnlockAllBut(const lane_id) const {+        return;+    }++    void lockAllBut(const lane_id) const {+        return;+    }++    void unlockAllBut(const lane_id) const {+        return;+    }+};++#ifdef IS_PARALLEL+ /**  * A small utility class for implementing simple locks.  */@@ -153,11 +235,15 @@ namespace detail {  /* Pause instruction to prevent excess processor bus usage */+#if defined _MSC_VER+#define cpu_relax() YieldProcessor()+#else #ifdef __x86_64__ #define cpu_relax() asm volatile("pause\n" : : : "memory") #else #define cpu_relax() asm volatile("" : : : "memory") #endif+#endif  /**  * A utility class managing waiting operations for spin locks.@@ -457,10 +543,188 @@     } }; -#else+/** Concurrent lanes locking mechanism. */+struct MutexConcurrentLanes {+    using lane_id = std::size_t;+    using unique_lock_type = std::unique_lock<std::mutex>; -namespace souffle {+    explicit MutexConcurrentLanes(const std::size_t Sz) : Size(Sz), Attribution(attribution(Sz)) {+        Lanes = std::make_unique<Lane[]>(Sz);+    }+    MutexConcurrentLanes(const MutexConcurrentLanes&) = delete;+    MutexConcurrentLanes(MutexConcurrentLanes&&) = delete; +    virtual ~MutexConcurrentLanes() {}++    // Return the number of lanes.+    std::size_t lanes() const {+        return Size;+    }++    // Select a lane+    lane_id getLane(std::size_t I) const {+        if (Attribution == lane_attribution::mod_power_of_2) {+            return I & (Size - 1);+        } else {+            return I % Size;+        }+    }++    /** Change the number of lanes.+     * DO not use while threads are using this object.+     */+    void setNumLanes(const std::size_t NumLanes) {+        Size = (NumLanes == 0 ? 1 : NumLanes);+        Attribution = attribution(Size);+        Lanes = std::make_unique<Lane[]>(Size);+    }++    unique_lock_type guard(const lane_id Lane) const {+        return unique_lock_type(Lanes[Lane].Access);+    }++    // Lock the given lane.+    // Must eventually be followed by unlock(Lane).+    void lock(const lane_id Lane) const {+        Lanes[Lane].Access.lock();+    }++    // Unlock the given lane.+    // Must already be the owner of the lane's lock.+    void unlock(const lane_id Lane) const {+        Lanes[Lane].Access.unlock();+    }++    // Acquire the capability to lock all other lanes than the given one.+    //+    // Must eventually be followed by beforeUnlockAllBut(Lane).+    void beforeLockAllBut(const lane_id Lane) const {+        if (!BeforeLockAll.try_lock()) {+            // If we cannot get the lock immediately, it means it was acquired+            // concurrently by another lane that will also try to acquire our+            // lane lock.+            // So we release our lane lock to let the concurrent operation+            // progress.+            unlock(Lane);+            BeforeLockAll.lock();+            lock(Lane);+        }+    }++    // Release the capability to lock all other lanes than the given one.+    //+    // Must already be the owner of that capability.+    void beforeUnlockAllBut(const lane_id) const {+        BeforeLockAll.unlock();+    }++    // Lock all lanes but the given one.+    //+    // Must already have acquired the capability to lock all other lanes+    // by calling beforeLockAllBut(Lane).+    //+    // Must eventually be followed by unlockAllBut(Lane).+    void lockAllBut(const lane_id Lane) const {+        for (std::size_t I = 0; I < Size; ++I) {+            if (I != Lane) {+                Lanes[I].Access.lock();+            }+        }+    }++    // Unlock all lanes but the given one.+    // Must already be the owner of all the lanes' locks.+    void unlockAllBut(const lane_id Lane) const {+        for (std::size_t I = 0; I < Size; ++I) {+            if (I != Lane) {+                Lanes[I].Access.unlock();+            }+        }+    }++private:+    enum lane_attribution { mod_power_of_2, mod_other };++    struct Lane {+        alignas(hardware_destructive_interference_size) std::mutex Access;+    };++    static constexpr lane_attribution attribution(const std::size_t Sz) {+        assert(Sz > 0);+        if ((Sz & (Sz - 1)) == 0) {+            // Sz is a power of 2+            return lane_attribution::mod_power_of_2;+        } else {+            return lane_attribution::mod_other;+        }+    }++protected:+    std::size_t Size;+    lane_attribution Attribution;++private:+    mutable std::unique_ptr<Lane[]> Lanes;++    alignas(hardware_destructive_interference_size) mutable std::mutex BeforeLockAll;+};++class ConcurrentLanes : public MutexConcurrentLanes {+    using Base = MutexConcurrentLanes;++public:+    using lane_id = Base::lane_id;+    using Base::beforeLockAllBut;+    using Base::beforeUnlockAllBut;+    using Base::guard;+    using Base::lock;+    using Base::lockAllBut;+    using Base::unlock;+    using Base::unlockAllBut;++    explicit ConcurrentLanes(const std::size_t Sz) : MutexConcurrentLanes(Sz) {}+    ConcurrentLanes(const ConcurrentLanes&) = delete;+    ConcurrentLanes(ConcurrentLanes&&) = delete;++    lane_id threadLane() const {+        return getLane(static_cast<std::size_t>(omp_get_thread_num()));+    }++    void setNumLanes(const std::size_t NumLanes) {+        Base::setNumLanes(NumLanes == 0 ? omp_get_max_threads() : NumLanes);+    }++    unique_lock_type guard() const {+        return Base::guard(threadLane());+    }++    void lock() const {+        return Base::lock(threadLane());+    }++    void unlock() const {+        return Base::unlock(threadLane());+    }++    void beforeLockAllBut() const {+        return Base::beforeLockAllBut(threadLane());+    }++    void beforeUnlockAllBut() const {+        return Base::beforeUnlockAllBut(threadLane());+    }++    void lockAllBut() const {+        return Base::lockAllBut(threadLane());+    }++    void unlockAllBut() const {+        return Base::unlockAllBut(threadLane());+    }+};++#else+ /**  * A small utility class for implementing simple locks.  */@@ -560,6 +824,53 @@     } }; +struct ConcurrentLanes : protected SeqConcurrentLanes {+    using Base = SeqConcurrentLanes;+    using lane_id = SeqConcurrentLanes::lane_id;+    using unique_lock_type = SeqConcurrentLanes::unique_lock_type;++    using Base::lanes;+    using Base::setNumLanes;++    explicit ConcurrentLanes(std::size_t Sz = MAX_THREADS) : Base(Sz) {}+    ConcurrentLanes(const ConcurrentLanes&) = delete;+    ConcurrentLanes(ConcurrentLanes&&) = delete;++    virtual ~ConcurrentLanes() {}++    lane_id threadLane() const {+        return 0;+    }++    unique_lock_type guard() const {+        return Base::guard(threadLane());+    }++    void lock() const {+        return Base::lock(threadLane());+    }++    void unlock() const {+        return Base::unlock(threadLane());+    }++    void beforeLockAllBut() const {+        return Base::beforeLockAllBut(threadLane());+    }++    void beforeUnlockAllBut() const {+        return Base::beforeUnlockAllBut(threadLane());+    }++    void lockAllBut() const {+        return Base::lockAllBut(threadLane());+    }++    void unlockAllBut() const {+        return Base::unlockAllBut(threadLane());+    }+};+ #endif  /**@@ -570,4 +881,4 @@     return outputLock; } -}  // end of namespace souffle+}  // namespace souffle
cbits/souffle/utility/StreamUtil.h view
@@ -1,6 +1,6 @@ /*  * Souffle - A Datalog Compiler- * Copyright (c) 2013, Oracle and/or its affiliates. All rights reserved+ * Copyright (c) 2021, The Souffle Developers. All rights reserved  * Licensed under the Universal Permissive License v 1.0 as shown at:  * - https://opensource.org/licenses/UPL  * - <souffle root>/licenses/SOUFFLE-UPL.txt@@ -26,6 +26,7 @@ #include <vector>  #include "souffle/utility/ContainerUtil.h"+#include "souffle/utility/span.h"  // ------------------------------------------------------------------------------- //                           General Print Utilities@@ -33,6 +34,25 @@  namespace souffle { +// Usage:       `using namespace stream_write_qualified_char_as_number;`+//              NB: `using` must appear in the same namespace as the `<<` callers.+//                  Putting the `using` in a parent namespace will have no effect.+// Motivation:  Octet sized numeric types are often defined as aliases of a qualified+//              `char`. e.g. `using uint8_t = unsigned char'`+//              `std::ostream` has an overload which converts qualified `char`s to plain `char`.+//              You don't usually want to print a `uint8_t` as an ASCII character.+//+// NOTE:        `char`, `signed char`, and `unsigned char` are distinct types.+namespace stream_write_qualified_char_as_number {+inline std::ostream& operator<<(std::ostream& os, signed char c) {+    return os << int(c);+}++inline std::ostream& operator<<(std::ostream& os, unsigned char c) {+    return os << unsigned(c);+}+}  // namespace stream_write_qualified_char_as_number+ template <typename A> struct IsPtrLike : std::is_pointer<A> {}; template <typename A>@@ -146,9 +166,8 @@  * For use cases see the test case {util_test.cpp}.  */ template <typename Iter, typename Printer>-detail::joined_sequence<Iter, Printer> join(-        const Iter& a, const Iter& b, const std::string& sep, const Printer& p) {-    return souffle::detail::joined_sequence<Iter, Printer>(a, b, sep, p);+detail::joined_sequence<Iter, Printer> join(const Iter& a, const Iter& b, std::string sep, const Printer& p) {+    return souffle::detail::joined_sequence<Iter, Printer>(a, b, std::move(sep), p); }  /**@@ -170,8 +189,8 @@  * For use cases see the test case {util_test.cpp}.  */ template <typename Container, typename Printer, typename Iter = typename Container::const_iterator>-detail::joined_sequence<Iter, Printer> join(const Container& c, const std::string& sep, const Printer& p) {-    return join(c.begin(), c.end(), sep, p);+detail::joined_sequence<Iter, Printer> join(const Container& c, std::string sep, const Printer& p) {+    return join(c.begin(), c.end(), std::move(sep), p); }  // Decide if the sane default is to deref-then-print or just print.@@ -186,19 +205,29 @@  * For use cases see the test case {util_test.cpp}.  */ template <typename Container, typename Iter = typename Container::const_iterator,-        typename T = typename Iter::value_type>+        typename T = typename std::iterator_traits<Iter>::value_type> std::enable_if_t<!JoinShouldDeref<T>, detail::joined_sequence<Iter, detail::print<id<T>>>> join(-        const Container& c, const std::string& sep = ",") {-    return join(c.begin(), c.end(), sep, detail::print<id<T>>());+        const Container& c, std::string sep = ",") {+    return join(c.begin(), c.end(), std::move(sep), detail::print<id<T>>()); }  template <typename Container, typename Iter = typename Container::const_iterator,-        typename T = typename Iter::value_type>+        typename T = typename std::iterator_traits<Iter>::value_type> std::enable_if_t<JoinShouldDeref<T>, detail::joined_sequence<Iter, detail::print<deref<T>>>> join(-        const Container& c, const std::string& sep = ",") {-    return join(c.begin(), c.end(), sep, detail::print<deref<T>>());+        const Container& c, std::string sep = ",") {+    return join(c.begin(), c.end(), std::move(sep), detail::print<deref<T>>()); } +template <typename C, typename F>+auto joinMap(const C& c, F&& map) {+    return join(c.begin(), c.end(), ",", [&](auto&& os, auto&& x) { return os << map(x); });+}++template <typename C, typename F>+auto joinMap(const C& c, std::string sep, F&& map) {+    return join(c.begin(), c.end(), std::move(sep), [&](auto&& os, auto&& x) { return os << map(x); });+}+ }  // end namespace souffle  #ifndef __EMBEDDED_SOUFFLE__@@ -206,6 +235,15 @@ namespace std {  /**+ * Enables the generic printing of `array`s assuming their element types+ * are printable.+ */+template <typename T, std::size_t E>+ostream& operator<<(ostream& out, const array<T, E>& v) {+    return out << "[" << souffle::join(v) << "]";+}++/**  * Introduces support for printing pairs as long as their components can be printed.  */ template <typename A, typename B>@@ -219,6 +257,15 @@  */ template <typename T, typename A> ostream& operator<<(ostream& out, const vector<T, A>& v) {+    return out << "[" << souffle::join(v) << "]";+}++/**+ * Enables the generic printing of `span`s assuming their element types+ * are printable.+ */+template <typename T, std::size_t E>+ostream& operator<<(ostream& out, const souffle::span<T, E>& v) {     return out << "[" << souffle::join(v) << "]"; }
cbits/souffle/utility/StringUtil.h view
@@ -1,6 +1,6 @@ /*  * Souffle - A Datalog Compiler- * Copyright (c) 2013, Oracle and/or its affiliates. All rights reserved+ * Copyright (c) 2021, The Souffle Developers. All rights reserved  * Licensed under the Universal Permissive License v 1.0 as shown at:  * - https://opensource.org/licenses/UPL  * - <souffle root>/licenses/SOUFFLE-UPL.txt@@ -22,6 +22,7 @@ #include <cstdlib> #include <fstream> #include <limits>+#include <set> #include <sstream> #include <stdexcept> #include <string>@@ -154,7 +155,7 @@  * starts with minus (c++ default semantics).  */ inline bool canBeParsedAsRamSigned(const std::string& string) {-    size_t charactersRead = 0;+    std::size_t charactersRead = 0;      try {         RamSignedFromString(string, &charactersRead, 0);@@ -171,7 +172,7 @@  * Souffle accepts: hex, binary and base 10.  */ inline bool canBeParsedAsRamUnsigned(const std::string& string) {-    size_t charactersRead = 0;+    std::size_t charactersRead = 0;     try {         RamUnsignedFromString(string, &charactersRead, 0);     } catch (...) {@@ -184,7 +185,7 @@  * Can a string be parsed as RamFloat.  */ inline bool canBeParsedAsRamFloat(const std::string& string) {-    size_t charactersRead = 0;+    std::size_t charactersRead = 0;     try {         RamFloatFromString(string, &charactersRead);     } catch (...) {@@ -312,24 +313,48 @@ /**  * Splits a string given a delimiter  */-inline std::vector<std::string> splitString(const std::string& str, char delimiter) {-    std::vector<std::string> parts;-    std::stringstream strstr(str);-    std::string token;-    while (std::getline(strstr, token, delimiter)) {-        parts.push_back(token);+inline std::vector<std::string_view> splitView(std::string_view toSplit, std::string_view delimiter) {+    if (toSplit.empty()) return {toSplit};++    auto delimLen = std::max<size_t>(1, delimiter.size());  // ensure we advance even w/ an empty needle++    std::vector<std::string_view> parts;+    for (auto tail = toSplit;;) {+        auto pos = tail.find(delimiter);+        parts.push_back(tail.substr(0, pos));+        if (pos == tail.npos) break;++        tail = tail.substr(pos + delimLen);     }+     return parts; }  /**+ * Splits a string given a delimiter+ */+inline std::vector<std::string> splitString(std::string_view str, char delimiter) {+    std::vector<std::string> xs;+    for (auto&& x : splitView(str, std::string_view{&delimiter, 1}))+        xs.push_back(std::string(x));+    return xs;+}++/**+ * Strips the prefix of a given string if it exists. No change otherwise.+ */+inline std::string stripPrefix(const std::string& prefix, const std::string& element) {+    return isPrefix(prefix, element) ? element.substr(prefix.length()) : element;+}++/**  * Stringify a string using escapes for escape, newline, tab, double-quotes and semicolons  */ inline std::string stringify(const std::string& input) {     std::string str(input);      // replace escapes with double escape sequence-    size_t start_pos = 0;+    std::size_t start_pos = 0;     while ((start_pos = str.find('\\', start_pos)) != std::string::npos) {         str.replace(start_pos, 1, "\\\\");         start_pos += 2;@@ -379,7 +404,7 @@  /** Valid C++ identifier, note that this does not ensure the uniqueness of identifiers returned. */ inline std::string identifier(std::string id) {-    for (size_t i = 0; i < id.length(); i++) {+    for (std::size_t i = 0; i < id.length(); i++) {         if (((isalpha(id[i]) == 0) && i == 0) || ((isalnum(id[i]) == 0) && id[i] != '_')) {             id[i] = '_';         }@@ -392,7 +417,7 @@ inline std::string unescape(         const std::string& inputString, const std::string& needle, const std::string& replacement) {     std::string result = inputString;-    size_t pos = 0;+    std::size_t pos = 0;     while ((pos = result.find(needle, pos)) != std::string::npos) {         result = result.replace(pos, needle.length(), replacement);         pos += replacement.length();@@ -411,7 +436,7 @@ inline std::string escape(         const std::string& inputString, const std::string& needle, const std::string& replacement) {     std::string result = inputString;-    size_t pos = 0;+    std::size_t pos = 0;     while ((pos = result.find(needle, pos)) != std::string::npos) {         result = result.replace(pos, needle.length(), replacement);         pos += replacement.length();@@ -425,6 +450,22 @@     escaped = escape(escaped, "\r", "\\r");     escaped = escape(escaped, "\n", "\\n");     return escaped;+}++template <typename C>+auto escape(C&& os, std::string_view str, std::set<char> const& needs_escape, std::string_view esc) {+    for (auto&& x : str) {+        if (needs_escape.find(x) != needs_escape.end()) {+            os << esc;+        }+        os << x;+    }++    return std::forward<C>(os);+}++inline std::string escape(std::string_view str, std::set<char> const& needs_escape, std::string_view esc) {+    return escape(std::stringstream{}, str, needs_escape, esc).str(); }  }  // end namespace souffle
+ cbits/souffle/utility/Types.h view
@@ -0,0 +1,155 @@+/*+ * Souffle - A Datalog Compiler+ * Copyright (c) 2020, The Souffle Developers. All rights reserved+ * Licensed under the Universal Permissive License v 1.0 as shown at:+ * - https://opensource.org/licenses/UPL+ * - <souffle root>/licenses/SOUFFLE-UPL.txt+ */++/************************************************************************+ *+ * @file Types.h+ *+ * @brief Shared type definitions+ *+ ***********************************************************************/++#pragma once++#include <iterator>+#include <memory>+#include <type_traits>+#include <vector>++namespace souffle {++// TODO: replace with C++20 concepts+template <typename CC, typename A>+constexpr bool is_iterable_of = std::is_constructible_v<A&,+        typename std::iterator_traits<decltype(std::begin(std::declval<CC>()))>::value_type&>;++// basically std::monostate, but doesn't require importing all of `<variant>`+struct Unit {};++constexpr bool operator==(Unit, Unit) noexcept {+    return true;+}+constexpr bool operator<=(Unit, Unit) noexcept {+    return true;+}+constexpr bool operator>=(Unit, Unit) noexcept {+    return true;+}+constexpr bool operator!=(Unit, Unit) noexcept {+    return false;+}+constexpr bool operator<(Unit, Unit) noexcept {+    return false;+}+constexpr bool operator>(Unit, Unit) noexcept {+    return false;+}++template <typename A>+using Own = std::unique_ptr<A>;++template <typename A, typename B = A, typename... Args>+Own<A> mk(Args&&... xs) {+    return std::make_unique<B>(std::forward<Args>(xs)...);+}++template <typename A>+using VecOwn = std::vector<Own<A>>;++/**+ * Copy the const qualifier of type T onto type U+ */+template <typename A, typename B>+using copy_const = std::conditional_t<std::is_const_v<A>, const B, B>;++namespace detail {++template <typename A>+struct is_own_ptr_t : std::false_type {};++template <typename A>+struct is_own_ptr_t<Own<A>> : std::true_type {};++template <typename T, typename U = void>+struct is_range_impl : std::false_type {};++template <typename T>+struct is_range_impl<T, std::void_t<decltype(*std::begin(std::declval<T&>()))>> : std::true_type {};++template <typename A, typename = void>+struct is_associative : std::false_type {};++template <typename A>+struct is_associative<A, std::void_t<typename A::key_type>> : std::true_type {};++template <typename A, typename = void, typename = void>+struct is_set : std::false_type {};++template <typename A>+struct is_set<A, std::void_t<typename A::key_type>, std::void_t<typename A::value_type>>+        : std::is_same<typename A::key_type, typename A::value_type> {};++}  // namespace detail++template <typename A>+constexpr bool is_own_ptr = detail::is_own_ptr_t<std::decay_t<A>>::value;++/**+ * A simple test to check if T is a range (i.e. has std::begin())+ */+template <typename T>+struct is_range : detail::is_range_impl<T> {};++template <typename A>+constexpr bool is_range_v = is_range<A>::value;++template <typename A>+constexpr bool is_remove_ref_const = std::is_const_v<std::remove_reference_t<A>>;++/**+ * Type identity, remove once we have C++20+ */+template <typename T>+struct type_identity {+    using type = T;+};++/**+ * Remove cv ref, remove once we have C++ 20+ */+template <typename T>+using remove_cvref = std::remove_cv<std::remove_reference_t<T>>;++template <class T>+using remove_cvref_t = typename remove_cvref<T>::type;++namespace detail {+template <typename T>+struct is_pointer_like : std::is_pointer<T> {};++template <typename T>+struct is_pointer_like<Own<T>> : std::true_type {};++}  // namespace detail++template <typename T>+constexpr bool is_pointer_like = detail::is_pointer_like<remove_cvref_t<T>>::value;++// TODO: complete these or move to C++20+template <typename A>+constexpr bool is_associative = detail::is_associative<A>::value;++template <typename A>+constexpr bool is_set = detail::is_set<A>::value;++// Useful for `static_assert`ing in unhandled cases with `constexpr` static dispatching+// Gives nicer error messages. (e.g. "failed due to req' unhandled_dispatch_type<...>")+template <typename A>+constexpr bool unhandled_dispatch_type = !std::is_same_v<A, A>;++}  // namespace souffle
cbits/souffle/utility/json11.h view
@@ -168,7 +168,7 @@     const object& object_items() const;      // Return a reference to arr[i] if this is an array, Json() otherwise.-    const Json& operator[](size_t i) const;+    const Json& operator[](std::size_t i) const;     // Return a reference to obj[key] if this is an object, Json() otherwise.     const Json& operator[](const std::string& key) const; @@ -257,7 +257,7 @@     virtual bool bool_value() const;     virtual const std::string& string_value() const;     virtual const Json::array& array_items() const;-    virtual const Json& operator[](size_t i) const;+    virtual const Json& operator[](std::size_t i) const;     virtual const Json::object& object_items() const;     virtual const Json& operator[](const std::string& key) const;     virtual ~JsonValue() = default;@@ -308,7 +308,7 @@  static void dump(const std::string& value, std::string& out) {     out += '"';-    for (size_t i = 0; i < value.length(); i++) {+    for (std::size_t i = 0; i < value.length(); i++) {         const char ch = value[i];         if (ch == '\\') {             out += "\\\\";@@ -465,7 +465,7 @@     const Json::array& array_items() const override {         return m_value;     }-    const Json& operator[](size_t i) const override;+    const Json& operator[](std::size_t i) const override;  public:     explicit JsonArray(const Json::array& value) : Value(value) {}@@ -557,7 +557,7 @@ inline const std::map<std::string, Json>& Json::object_items() const {     return m_ptr->object_items(); }-inline const Json& Json::operator[](size_t i) const {+inline const Json& Json::operator[](std::size_t i) const {     return (*m_ptr)[i]; } inline const Json& Json::operator[](const std::string& key) const {@@ -585,7 +585,7 @@ inline const std::map<std::string, Json>& JsonValue::object_items() const {     return statics().empty_map; }-inline const Json& JsonValue::operator[](size_t) const {+inline const Json& JsonValue::operator[](std::size_t) const {     return static_null(); } inline const Json& JsonValue::operator[](const std::string&) const {@@ -596,7 +596,7 @@     auto iter = m_value.find(key);     return (iter == m_value.end()) ? static_null() : iter->second; }-inline const Json& JsonArray::operator[](size_t i) const {+inline const Json& JsonArray::operator[](std::size_t i) const {     if (i >= m_value.size()) {         return static_null();     }@@ -660,7 +660,7 @@     /* State      */     const std::string& str;-    size_t i;+    std::size_t i;     std::string& err;     bool failed;     const JsonParse strategy;@@ -835,7 +835,7 @@                 if (esc.length() < 4) {                     return fail("bad \\u escape: " + esc, "");                 }-                for (size_t j = 0; j < 4; j++) {+                for (std::size_t j = 0; j < 4; j++) {                     if (!in_range(esc[j], 'a', 'f') && !in_range(esc[j], 'A', 'F') &&                             !in_range(esc[j], '0', '9'))                         return fail("bad \\u escape: " + esc, "");@@ -888,7 +888,7 @@      * Parse a double.      */     Json parse_number() {-        size_t start_pos = i;+        std::size_t start_pos = i;          if (str[i] == '-') {             i++;@@ -910,7 +910,7 @@         }          if (str[i] != '.' && str[i] != 'e' && str[i] != 'E' &&-                (i - start_pos) <= static_cast<size_t>(std::numeric_limits<int>::digits10)) {+                (i - start_pos) <= static_cast<std::size_t>(std::numeric_limits<int>::digits10)) {             return std::atoll(str.c_str() + start_pos);         }
+ cbits/souffle/utility/span.h view
@@ -0,0 +1,666 @@+#pragma once++#if __cplusplus >= 202000L++#include <span>  // use std lib impl++namespace souffle {+constexpr auto dynamic_extent = std::dynamic_extent;++template <typename A, std::size_t E = std::dynamic_extent>+using span = std::span<A, E>;+}  // namespace souffle++#else++// clang-format off+/*+This is an implementation of C++20's std::span+http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2019/n4820.pdf+*/++//          Copyright Tristan Brindle 2018.+// Distributed under the Boost Software License, Version 1.0.+//    (See accompanying file ../../LICENSE_1_0.txt or copy at+//          https://www.boost.org/LICENSE_1_0.txt)++#ifndef TCB_SPAN_HPP_INCLUDED+#define TCB_SPAN_HPP_INCLUDED++#include <array>+#include <cstddef>+#include <cstdint>+#include <type_traits>++#ifndef TCB_SPAN_NO_EXCEPTIONS+// Attempt to discover whether we're being compiled with exception support+#if !(defined(__cpp_exceptions) || defined(__EXCEPTIONS) || defined(_CPPUNWIND))+#define TCB_SPAN_NO_EXCEPTIONS+#endif+#endif++#ifndef TCB_SPAN_NO_EXCEPTIONS+#include <cstdio>+#include <stdexcept>+#endif++// Various feature test macros++#ifndef TCB_SPAN_NAMESPACE_NAME+#define TCB_SPAN_NAMESPACE_NAME tcb+#endif++#if __cplusplus >= 201703L || (defined(_MSVC_LANG) && _MSVC_LANG >= 201703L)+#define TCB_SPAN_HAVE_CPP17+#endif++#if __cplusplus >= 201402L || (defined(_MSVC_LANG) && _MSVC_LANG >= 201402L)+#define TCB_SPAN_HAVE_CPP14+#endif++namespace TCB_SPAN_NAMESPACE_NAME {++// Establish default contract checking behavior+#if !defined(TCB_SPAN_THROW_ON_CONTRACT_VIOLATION) &&                          \+    !defined(TCB_SPAN_TERMINATE_ON_CONTRACT_VIOLATION) &&                      \+    !defined(TCB_SPAN_NO_CONTRACT_CHECKING)+#if defined(NDEBUG) || !defined(TCB_SPAN_HAVE_CPP14)+#define TCB_SPAN_NO_CONTRACT_CHECKING+#else+#define TCB_SPAN_TERMINATE_ON_CONTRACT_VIOLATION+#endif+#endif++#if defined(TCB_SPAN_THROW_ON_CONTRACT_VIOLATION)+struct contract_violation_error : std::logic_error {+    explicit contract_violation_error(const char* msg) : std::logic_error(msg)+    {}+};++inline void contract_violation(const char* msg)+{+    throw contract_violation_error(msg);+}++#elif defined(TCB_SPAN_TERMINATE_ON_CONTRACT_VIOLATION)+[[noreturn]] inline void contract_violation(const char* /*unused*/)+{+    std::terminate();+}+#endif++#if !defined(TCB_SPAN_NO_CONTRACT_CHECKING)+#define TCB_SPAN_STRINGIFY(cond) #cond+#define TCB_SPAN_EXPECT(cond)                                                  \+    cond ? (void) 0 : contract_violation("Expected " TCB_SPAN_STRINGIFY(cond))+#else+#define TCB_SPAN_EXPECT(cond)+#endif++#if defined(TCB_SPAN_HAVE_CPP17) || defined(__cpp_inline_variables)+#define TCB_SPAN_INLINE_VAR inline+#else+#define TCB_SPAN_INLINE_VAR+#endif++#if defined(TCB_SPAN_HAVE_CPP14) ||                                            \+    (defined(__cpp_constexpr) && __cpp_constexpr >= 201304)+#define TCB_SPAN_HAVE_CPP14_CONSTEXPR+#endif++#if defined(TCB_SPAN_HAVE_CPP14_CONSTEXPR)+#define TCB_SPAN_CONSTEXPR14 constexpr+#else+#define TCB_SPAN_CONSTEXPR14+#endif++#if defined(TCB_SPAN_HAVE_CPP14_CONSTEXPR) &&                                  \+    (!defined(_MSC_VER) || _MSC_VER > 1900)+#define TCB_SPAN_CONSTEXPR_ASSIGN constexpr+#else+#define TCB_SPAN_CONSTEXPR_ASSIGN+#endif++#if defined(TCB_SPAN_NO_CONTRACT_CHECKING)+#define TCB_SPAN_CONSTEXPR11 constexpr+#else+#define TCB_SPAN_CONSTEXPR11 TCB_SPAN_CONSTEXPR14+#endif++#if defined(TCB_SPAN_HAVE_CPP17) || defined(__cpp_deduction_guides)+#define TCB_SPAN_HAVE_DEDUCTION_GUIDES+#endif++#if defined(TCB_SPAN_HAVE_CPP17) || defined(__cpp_lib_byte)+#define TCB_SPAN_HAVE_STD_BYTE+#endif++#if defined(TCB_SPAN_HAVE_CPP17) || defined(__cpp_lib_array_constexpr)+#define TCB_SPAN_HAVE_CONSTEXPR_STD_ARRAY_ETC+#endif++#if defined(TCB_SPAN_HAVE_CONSTEXPR_STD_ARRAY_ETC)+#define TCB_SPAN_ARRAY_CONSTEXPR constexpr+#else+#define TCB_SPAN_ARRAY_CONSTEXPR+#endif++#ifdef TCB_SPAN_HAVE_STD_BYTE+using byte = std::byte;+#else+using byte = unsigned char;+#endif++#if defined(TCB_SPAN_HAVE_CPP17)+#define TCB_SPAN_NODISCARD [[nodiscard]]+#else+#define TCB_SPAN_NODISCARD+#endif++TCB_SPAN_INLINE_VAR constexpr std::size_t dynamic_extent = SIZE_MAX;++template <typename ElementType, std::size_t Extent = dynamic_extent>+class span;++namespace detail {++template <typename E, std::size_t S>+struct span_storage {+    constexpr span_storage() noexcept = default;++    constexpr span_storage(E* p_ptr, std::size_t /*unused*/) noexcept+       : ptr(p_ptr)+    {}++    E* ptr = nullptr;+    static constexpr std::size_t size = S;+};++template <typename E>+struct span_storage<E, dynamic_extent> {+    constexpr span_storage() noexcept = default;++    constexpr span_storage(E* p_ptr, std::size_t p_size) noexcept+        : ptr(p_ptr), size(p_size)+    {}++    E* ptr = nullptr;+    std::size_t size = 0;+};++// Reimplementation of C++17 std::size() and std::data()+#if defined(TCB_SPAN_HAVE_CPP17) ||                                            \+    defined(__cpp_lib_nonmember_container_access)+using std::data;+using std::size;+#else+template <class C>+constexpr auto size(const C& c) -> decltype(c.size())+{+    return c.size();+}++template <class T, std::size_t N>+constexpr std::size_t size(const T (&)[N]) noexcept+{+    return N;+}++template <class C>+constexpr auto data(C& c) -> decltype(c.data())+{+    return c.data();+}++template <class C>+constexpr auto data(const C& c) -> decltype(c.data())+{+    return c.data();+}++template <class T, std::size_t N>+constexpr T* data(T (&array)[N]) noexcept+{+    return array;+}++template <class E>+constexpr const E* data(std::initializer_list<E> il) noexcept+{+    return il.begin();+}+#endif // TCB_SPAN_HAVE_CPP17++#if defined(TCB_SPAN_HAVE_CPP17) || defined(__cpp_lib_void_t)+using std::void_t;+#else+template <typename...>+using void_t = void;+#endif++template <typename T>+using uncvref_t =+    typename std::remove_cv<typename std::remove_reference<T>::type>::type;++template <typename>+struct is_span : std::false_type {};++template <typename T, std::size_t S>+struct is_span<span<T, S>> : std::true_type {};++template <typename>+struct is_std_array : std::false_type {};++template <typename T, std::size_t N>+struct is_std_array<std::array<T, N>> : std::true_type {};++template <typename, typename = void>+struct has_size_and_data : std::false_type {};++template <typename T>+struct has_size_and_data<T, void_t<decltype(detail::size(std::declval<T>())),+                                   decltype(detail::data(std::declval<T>()))>>+    : std::true_type {};++template <typename C, typename U = uncvref_t<C>>+struct is_container {+    static constexpr bool value =+        !is_span<U>::value && !is_std_array<U>::value &&+        !std::is_array<U>::value && has_size_and_data<C>::value;+};++template <typename T>+using remove_pointer_t = typename std::remove_pointer<T>::type;++template <typename, typename, typename = void>+struct is_container_element_type_compatible : std::false_type {};++template <typename T, typename E>+struct is_container_element_type_compatible<+    T, E,+    typename std::enable_if<+        !std::is_same<typename std::remove_cv<decltype(+                          detail::data(std::declval<T>()))>::type,+                      void>::value>::type>+    // HACK: WORKAROUND - GCC 9.2.1 claims `A* (*)[]` is not compatible w/ `A const* (*)[]`.+    //       This seems BS and Clang 10.0 is perfectly happy.+    //       GCC 9.2.1 does, however, agree that `A**` is compatible w/ `A const**`.+    //       Use the `*` test instead of `(*)[]`.+    : std::is_convertible<+          remove_pointer_t<decltype(detail::data(std::declval<T>()))>*,+          E*> {};++template <typename, typename = std::size_t>+struct is_complete : std::false_type {};++template <typename T>+struct is_complete<T, decltype(sizeof(T))> : std::true_type {};++} // namespace detail++template <typename ElementType, std::size_t Extent>+class span {+    static_assert(std::is_object<ElementType>::value,+                  "A span's ElementType must be an object type (not a "+                  "reference type or void)");+    static_assert(detail::is_complete<ElementType>::value,+                  "A span's ElementType must be a complete type (not a forward "+                  "declaration)");+    static_assert(!std::is_abstract<ElementType>::value,+                  "A span's ElementType cannot be an abstract class type");++    using storage_type = detail::span_storage<ElementType, Extent>;++public:+    // constants and types+    using element_type = ElementType;+    using value_type = typename std::remove_cv<ElementType>::type;+    using size_type = std::size_t;+    using difference_type = std::ptrdiff_t;+    using pointer = element_type*;+    using const_pointer = const element_type*;+    using reference = element_type&;+    using const_reference = const element_type&;+    using iterator = pointer;+    using reverse_iterator = std::reverse_iterator<iterator>;++    static constexpr size_type extent = Extent;++    // [span.cons], span constructors, copy, assignment, and destructor+    template <+        std::size_t E = Extent,+        typename std::enable_if<(E == dynamic_extent || E <= 0), int>::type = 0>+    constexpr span() noexcept // NOLINT : clang-tidy is mistaken. one cannot `default` a template ctor+    {}++    TCB_SPAN_CONSTEXPR11 span(pointer ptr, size_type count)+        : storage_(ptr, count)+    {+        TCB_SPAN_EXPECT(extent == dynamic_extent || count == extent);+    }++    TCB_SPAN_CONSTEXPR11 span(pointer first_elem, pointer last_elem)+        : storage_(first_elem, last_elem - first_elem)+    {+        TCB_SPAN_EXPECT(extent == dynamic_extent ||+                        last_elem - first_elem ==+                            static_cast<std::ptrdiff_t>(extent));+    }++    template <std::size_t N, std::size_t E = Extent,+              typename std::enable_if<+                  (E == dynamic_extent || N == E) &&+                      detail::is_container_element_type_compatible<+                          element_type (&)[N], ElementType>::value,+                  int>::type = 0>+    constexpr span(element_type (&arr)[N]) noexcept : storage_(arr, N)+    {}++    template <std::size_t N, std::size_t E = Extent,+              typename std::enable_if<+                  (E == dynamic_extent || N == E) &&+                      detail::is_container_element_type_compatible<+                          std::array<value_type, N>&, ElementType>::value,+                  int>::type = 0>+    TCB_SPAN_ARRAY_CONSTEXPR span(std::array<value_type, N>& arr) noexcept+        : storage_(arr.data(), N)+    {}++    template <std::size_t N, std::size_t E = Extent,+              typename std::enable_if<+                  (E == dynamic_extent || N == E) &&+                      detail::is_container_element_type_compatible<+                          const std::array<value_type, N>&, ElementType>::value,+                  int>::type = 0>+    TCB_SPAN_ARRAY_CONSTEXPR span(const std::array<value_type, N>& arr) noexcept+        : storage_(arr.data(), N)+    {}++    template <+        typename Container, std::size_t E = Extent,+        typename std::enable_if<+            E == dynamic_extent && detail::is_container<Container>::value &&+                detail::is_container_element_type_compatible<+                    Container&, ElementType>::value,+            int>::type = 0>+    constexpr span(Container& cont)+        : storage_(detail::data(cont), detail::size(cont))+    {}++    template <+        typename Container, std::size_t E = Extent,+        typename std::enable_if<+            E == dynamic_extent && detail::is_container<Container>::value &&+                detail::is_container_element_type_compatible<+                    const Container&, ElementType>::value,+            int>::type = 0>+    constexpr span(const Container& cont)+        : storage_(detail::data(cont), detail::size(cont))+    {}++    constexpr span(const span& other) noexcept = default;++    template <typename OtherElementType, std::size_t OtherExtent,+              typename std::enable_if<+                  (Extent == OtherExtent || Extent == dynamic_extent) &&+                      std::is_convertible<OtherElementType (*)[],+                                          ElementType (*)[]>::value,+                  int>::type = 0>+    constexpr span(const span<OtherElementType, OtherExtent>& other) noexcept+        : storage_(other.data(), other.size())+    {}++    ~span() noexcept = default;++    TCB_SPAN_CONSTEXPR_ASSIGN span&+    operator=(const span& other) noexcept = default;++    // [span.sub], span subviews+    template <std::size_t Count>+    TCB_SPAN_CONSTEXPR11 span<element_type, Count> first() const+    {+        TCB_SPAN_EXPECT(Count <= size());+        return {data(), Count};+    }++    template <std::size_t Count>+    TCB_SPAN_CONSTEXPR11 span<element_type, Count> last() const+    {+        TCB_SPAN_EXPECT(Count <= size());+        return {data() + (size() - Count), Count};+    }++    template <std::size_t Offset, std::size_t Count = dynamic_extent>+    using subspan_return_t =+        span<ElementType, Count != dynamic_extent+                              ? Count+                              : (Extent != dynamic_extent ? Extent - Offset+                                                          : dynamic_extent)>;++    template <std::size_t Offset, std::size_t Count = dynamic_extent>+    TCB_SPAN_CONSTEXPR11 subspan_return_t<Offset, Count> subspan() const+    {+        TCB_SPAN_EXPECT(Offset <= size() &&+                        (Count == dynamic_extent || Offset + Count <= size()));+        return {data() + Offset,+                Count != dynamic_extent ? Count : size() - Offset};+    }++    TCB_SPAN_CONSTEXPR11 span<element_type, dynamic_extent>+    first(size_type count) const+    {+        TCB_SPAN_EXPECT(count <= size());+        return {data(), count};+    }++    TCB_SPAN_CONSTEXPR11 span<element_type, dynamic_extent>+    last(size_type count) const+    {+        TCB_SPAN_EXPECT(count <= size());+        return {data() + (size() - count), count};+    }++    TCB_SPAN_CONSTEXPR11 span<element_type, dynamic_extent>+    subspan(size_type offset, size_type count = dynamic_extent) const+    {+        TCB_SPAN_EXPECT(offset <= size() &&+                        (count == dynamic_extent || offset + count <= size()));+        return {data() + offset,+                count == dynamic_extent ? size() - offset : count};+    }++    // [span.obs], span observers+    constexpr size_type size() const noexcept { return storage_.size; }++    constexpr size_type size_bytes() const noexcept+    {+        return size() * sizeof(element_type);+    }++    TCB_SPAN_NODISCARD constexpr bool empty() const noexcept+    {+        return size() == 0;+    }++    // [span.elem], span element access+    TCB_SPAN_CONSTEXPR11 reference operator[](size_type idx) const+    {+        TCB_SPAN_EXPECT(idx < size());+        return *(data() + idx);+    }++    TCB_SPAN_CONSTEXPR11 reference front() const+    {+        TCB_SPAN_EXPECT(!empty());+        return *data();+    }++    TCB_SPAN_CONSTEXPR11 reference back() const+    {+        TCB_SPAN_EXPECT(!empty());+        return *(data() + (size() - 1));+    }++    constexpr pointer data() const noexcept { return storage_.ptr; }++    // [span.iterators], span iterator support+    constexpr iterator begin() const noexcept { return data(); }++    constexpr iterator end() const noexcept { return data() + size(); }++    TCB_SPAN_ARRAY_CONSTEXPR reverse_iterator rbegin() const noexcept+    {+        return reverse_iterator(end());+    }++    TCB_SPAN_ARRAY_CONSTEXPR reverse_iterator rend() const noexcept+    {+        return reverse_iterator(begin());+    }++private:+    storage_type storage_{};+};++#ifdef TCB_SPAN_HAVE_DEDUCTION_GUIDES++/* Deduction Guides */+template <class T, std::size_t N>+span(T (&)[N])->span<T, N>;++template <class T, std::size_t N>+span(std::array<T, N>&)->span<T, N>;++template <class T, std::size_t N>+span(const std::array<T, N>&)->span<const T, N>;++template <class Container>+span(Container&)->span<typename Container::value_type>;++template <class Container>+span(const Container&)->span<const typename Container::value_type>;++#endif // TCB_HAVE_DEDUCTION_GUIDES++template <typename ElementType, std::size_t Extent>+constexpr span<ElementType, Extent>+make_span(span<ElementType, Extent> s) noexcept+{+    return s;+}++template <typename T, std::size_t N>+constexpr span<T, N> make_span(T (&arr)[N]) noexcept+{+    return {arr};+}++template <typename T, std::size_t N>+TCB_SPAN_ARRAY_CONSTEXPR span<T, N> make_span(std::array<T, N>& arr) noexcept+{+    return {arr};+}++template <typename T, std::size_t N>+TCB_SPAN_ARRAY_CONSTEXPR span<const T, N>+make_span(const std::array<T, N>& arr) noexcept+{+    return {arr};+}++template <typename Container>+constexpr span<typename Container::value_type> make_span(Container& cont)+{+    return {cont};+}++template <typename Container>+constexpr span<const typename Container::value_type>+make_span(const Container& cont)+{+    return {cont};+}++template <typename ElementType, std::size_t Extent>+span<const byte, ((Extent == dynamic_extent) ? dynamic_extent+                                             : sizeof(ElementType) * Extent)>+as_bytes(span<ElementType, Extent> s) noexcept+{+    return {reinterpret_cast<const byte*>(s.data()), s.size_bytes()};+}++template <+    class ElementType, std::size_t Extent,+    typename std::enable_if<!std::is_const<ElementType>::value, int>::type = 0>+span<byte, ((Extent == dynamic_extent) ? dynamic_extent+                                       : sizeof(ElementType) * Extent)>+as_writable_bytes(span<ElementType, Extent> s) noexcept+{+    return {reinterpret_cast<byte*>(s.data()), s.size_bytes()};+}++template <std::size_t N, typename E, std::size_t S>+constexpr auto get(span<E, S> s) -> decltype(s[N])+{+    return s[N];+}++} // namespace TCB_SPAN_NAMESPACE_NAME++namespace std {++// see:     https://gcc.gnu.org/bugzilla/show_bug.cgi?id=82716+// libc++:  https://reviews.llvm.org/D55466#1325498+//          `libc++` changed to use `struct`+// spec:    http://eel.is/c++draft/tuple.helper+//          Spec says to use `struct`.+// MSVC:    Has different ABI for `class`/`struct`.+//          Defined `tuple_size` as `class`.+#if defined(_MSC_VER)+    #define TCB_SPAN_TUPLE_SIZE_KIND class+#else+    #define TCB_SPAN_TUPLE_SIZE_KIND struct+#endif++#if defined(__clang__)+    #pragma clang diagnostic push+    #pragma clang diagnostic ignored "-Wmismatched-tags"+#endif++template <typename ElementType, std::size_t Extent>+TCB_SPAN_TUPLE_SIZE_KIND tuple_size<TCB_SPAN_NAMESPACE_NAME::span<ElementType, Extent>>+    : public integral_constant<std::size_t, Extent> {};++template <typename ElementType>+TCB_SPAN_TUPLE_SIZE_KIND tuple_size<TCB_SPAN_NAMESPACE_NAME::span<+    ElementType, TCB_SPAN_NAMESPACE_NAME::dynamic_extent>>; // not defined++template <std::size_t I, typename ElementType, std::size_t Extent>+TCB_SPAN_TUPLE_SIZE_KIND tuple_element<I, TCB_SPAN_NAMESPACE_NAME::span<ElementType, Extent>> {+public:+    static_assert(Extent != TCB_SPAN_NAMESPACE_NAME::dynamic_extent &&+                      I < Extent,+                  "");+    using type = ElementType;+};++#if defined(__clang__)+    #pragma clang diagnostic pop+#endif++#undef TCB_SPAN_TUPLE_SIZE_KIND++} // end namespace std++#endif // TCB_SPAN_HPP_INCLUDED++// clang-format on++namespace souffle {+constexpr auto dynamic_extent = tcb::dynamic_extent;++template <typename A, std::size_t E = tcb::dynamic_extent>+using span = tcb::span<A, E>;+}  // namespace souffle++#endif
cbits/souffle/utility/tinyformat.h view
@@ -50,7 +50,7 @@ // //   std::string weekday = "Wednesday"; //   const char* month = "July";-//   size_t day = 27;+//   std::size_t day = 27; //   long hour = 14; //   int min = 44; //@@ -66,7 +66,7 @@ // // The strange types here emphasize the type safety of the interface; it is // possible to print a std::string using the "%s" conversion, and a-// size_t using the "%d" conversion.  A similar result could be achieved+// std::size_t using the "%d" conversion.  A similar result could be achieved // using either of the tfm::format() functions.  One prints on a user provided // stream: //@@ -792,27 +792,29 @@             break;         case 'X':             out.setf(std::ios::uppercase);-            // Falls through-        case 'x': case 'p':+            [[fallthrough]];+        case 'x':+            [[fallthrough]]; +        case 'p':             out.setf(std::ios::hex, std::ios::basefield);             intConversion = true;             break;         case 'E':             out.setf(std::ios::uppercase);-            // Falls through+            [[fallthrough]];         case 'e':             out.setf(std::ios::scientific, std::ios::floatfield);             out.setf(std::ios::dec, std::ios::basefield);             break;         case 'F':             out.setf(std::ios::uppercase);-            // Falls through+            [[fallthrough]];         case 'f':             out.setf(std::ios::fixed, std::ios::floatfield);             break;         case 'A':             out.setf(std::ios::uppercase);-            // Falls through+            [[fallthrough]];         case 'a': #           ifdef _MSC_VER             // Workaround https://developercommunity.visualstudio.com/content/problem/520472/hexfloat-stream-output-does-not-ignore-precision-a.html@@ -824,7 +826,7 @@             break;         case 'G':             out.setf(std::ios::uppercase);-            // Falls through+            [[fallthrough]];         case 'g':             out.setf(std::ios::dec, std::ios::basefield);             // As in boost::format, let stream decide float format.
+ lib/Language/Souffle/Analysis.hs view
@@ -0,0 +1,160 @@+{-# LANGUAGE UndecidableInstances, TupleSections #-}++{- | This module provides an 'Analysis' type for combining multiple Datalog+     analyses together. Composition of analyses is done via the various+     type-classes that are implemented for this type. For a longer explanation+     of how the 'Analysis' type works, see this+     <https://luctielen.com/posts/analyses_are_arrows/ blogpost>.++     If you are just starting out using this library, you are probably better+     of taking a look at the "Language.Souffle.Interpreted" module instead to+     start interacting with a single Datalog program.+-}+module Language.Souffle.Analysis+  ( Analysis+  , mkAnalysis+  , execAnalysis+  ) where++import Prelude hiding (id, (.))+import Data.Kind (Type)+import Control.Category+import Control.Monad+import Control.Arrow+import Data.Profunctor++-- | Data type used to compose multiple Datalog programs. Composition is mainly+--   done via the various type-classes implemented for this type.+--   Values of this type can be created using 'mkAnalysis'.+--+--   The @m@ type-variable represents the monad the analysis will run in. In+--   most cases, this will be the @SouffleM@ monad from either+--   "Language.Souffle.Compiled" or "Language.Souffle.Interpreted".+--   The @a@ and @b@ type-variables represent respectively the input and output+--   types of the analysis.+type Analysis :: (Type -> Type) -> Type -> Type -> Type+data Analysis m a b+  = Analysis (a -> m ()) (m ()) (a -> m b)++-- | Creates an 'Analysis' value.+mkAnalysis :: (a -> m ()) -- ^ Function for finding facts used by the 'Analysis'.+           -> m ()        -- ^ Function for actually running the 'Analysis'.+           -> m b         -- ^ Function for retrieving the 'Analysis' results from Souffle.+           -> Analysis m a b+mkAnalysis f r g = Analysis f r (const g)+{-# INLINABLE mkAnalysis #-}++-- | Converts an 'Analysis' into an effectful function, so it can be executed.+execAnalysis :: Applicative m => Analysis m a b -> (a -> m b)+execAnalysis (Analysis f r g) a = f a *> r *> g a+{-# INLINABLE execAnalysis #-}++instance Functor m => Functor (Analysis m a) where+  fmap func (Analysis f r g) =+    Analysis f r (fmap func <$> g)+  {-# INLINABLE fmap #-}++instance Functor m => Profunctor (Analysis m) where+  lmap fn (Analysis f r g) =+    Analysis (lmap fn f) r (lmap fn g)+  {-# INLINABLE lmap #-}++  rmap = fmap+  {-# INLINABLE rmap #-}++instance (Monoid (m ()), Applicative m) => Applicative (Analysis m a) where+  pure a = Analysis mempty mempty (const $ pure a)+  {-# INLINABLE pure #-}++  Analysis f1 r1 g1 <*> Analysis f2 r2 g2 =+    Analysis (f1 <> f2) (r1 <> r2) (\a -> g1 a <*> g2 a)+  {-# INLINABLE (<*>) #-}++instance (Semigroup (m ()), Semigroup (m b)) => Semigroup (Analysis m a b) where+  Analysis f1 r1 g1 <> Analysis f2 r2 g2 =+    Analysis (f1 <> f2) (r1 <> r2) (g1 <> g2)+  {-# INLINABLE (<>) #-}++instance (Monoid (m ()), Monoid (m b)) => Monoid (Analysis m a b) where+  mempty = Analysis mempty mempty mempty+  {-# INLINABLE mempty #-}++instance (Monoid (m ()), Monad m) => Category (Analysis m) where+  id = Analysis mempty mempty pure+  {-# INLINABLE id #-}++  Analysis f1 r1 g1 . Analysis f2 r2 g2 = Analysis f r1 g+    where+      f = execAnalysis (Analysis f2 r2 g2) >=> f1+      -- NOTE: lazyness avoids work here in g2 in cases where "const" is used+      g = g2 >=> g1+  {-# INLINABLE (.) #-}++instance Functor m => Strong (Analysis m) where+  first' (Analysis f r g) =+    Analysis (f . fst) r $ \(b, d) -> (,d) <$> g b+  {-# INLINABLE first' #-}++  second' (Analysis f r g) =+    Analysis (f . snd) r $ \(d, b) -> (d,) <$> g b+  {-# INLINABLE second' #-}++instance Applicative m => Choice (Analysis m) where+  left' (Analysis f r g) = Analysis f' r g'+    where+      f' = \case+        Left b -> f b+        Right _ -> pure ()+      g' = \case+        Left b -> Left <$> g b+        Right d -> pure $ Right d+  {-# INLINABLE left' #-}++  right' (Analysis f r g) = Analysis f' r g'+    where+      f' = \case+        Left _ -> pure ()+        Right b -> f b+      g' = \case+        Left d -> pure $ Left d+        Right b -> Right <$> g b+  {-# INLINABLE right' #-}++instance (Monad m, Monoid (m ()), Category (Analysis m)) => Arrow (Analysis m) where+  arr f = Analysis mempty mempty (pure . f)+  {-# INLINABLE arr #-}++  first = first'+  {-# INLINABLE first #-}++  second = second'+  {-# INLINABLE second #-}++  Analysis f1 r1 g1 *** Analysis f2 r2 g2 =+    Analysis (\(b, b') -> f1 b *> f2 b') (r1 <> r2) $ \(b, b') -> do+      c <- g1 b+      c' <- g2 b'+      pure (c, c')+  {-# INLINABLE (***) #-}++  Analysis f1 r1 g1 &&& Analysis f2 r2 g2 =+    Analysis (f1 <> f2) (r1 <> r2) $ \b -> (,) <$> g1 b <*> g2 b+  {-# INLINABLE (&&&) #-}++instance (Monad m, Monoid (m ())) => ArrowChoice (Analysis m) where+  left = left'+  {-# INLINABLE left #-}++  right = right'+  {-# INLINABLE right #-}++  Analysis f1 r1 g1 +++ Analysis f2 r2 g2 = Analysis f' (r1 <> r2) g'+    where+      f' = \case+        Left b -> f1 b+        Right b' -> f2 b'+      g' = \case+        Left b -> Left <$> g1 b+        Right b' -> Right <$> g2 b'+  {-# INLINABLE (+++) #-}+
lib/Language/Souffle/Class.hs view
@@ -1,5 +1,5 @@ {-# LANGUAGE DataKinds, UndecidableInstances, FlexibleContexts #-}-{-# LANGUAGE TypeFamilies, TypeOperators #-}+{-# LANGUAGE TypeFamilies, TypeOperators, TypeApplications #-}  -- | This module provides the top level API for Souffle related operations. --   It makes use of Haskell's powerful typesystem to make certain invalid states@@ -15,7 +15,9 @@ --   type safety and user-friendly error messages. module Language.Souffle.Class   ( Program(..)+  , ProgramOptions(..)   , Fact(..)+  , FactOptions(..)   , Marshal.Marshal(..)   , Direction(..)   , ContainsInputFact@@ -28,50 +30,57 @@ import Prelude hiding ( init )  import Control.Monad.Except-import Control.Monad.RWS.Strict import Control.Monad.Reader-import Control.Monad.State import Control.Monad.Writer+import qualified Control.Monad.RWS.Strict as StrictRWS+import qualified Control.Monad.RWS.Lazy as LazyRWS+import qualified Control.Monad.State.Strict as StrictState+import qualified Control.Monad.State.Lazy as LazyState import Data.Proxy import Data.Kind import Data.Word+import GHC.TypeLits import qualified Language.Souffle.Marshal as Marshal-import Type.Errors.Pretty   -- | A helper type family for checking if a specific Souffle `Program` contains --   a certain `Fact`. Additionally, it also checks if the fact is marked as --   either `Input` or `InputOutput`. This constraint will generate a --   user-friendly type error if these conditions are not met.-type family ContainsInputFact prog fact :: Constraint where+type ContainsInputFact :: Type -> Type -> Constraint+type family ContainsInputFact prog fact where   ContainsInputFact prog fact = (ContainsFact prog fact, IsInput fact (FactDirection fact))  -- | A helper type family for checking if a specific Souffle `Program` contains --   a certain `Fact`. Additionally, it also checks if the fact is marked as --   either `Output` or `InputOutput`. This constraint will generate a --   user-friendly type error if these conditions are not met.-type family ContainsOutputFact prog fact :: Constraint where+type ContainsOutputFact :: Type -> Type -> Constraint+type family ContainsOutputFact prog fact where   ContainsOutputFact prog fact = (ContainsFact prog fact, IsOutput fact (FactDirection fact)) -type family IsInput (fact :: Type) (dir :: Direction) :: Constraint where+type IsInput :: Type -> Direction -> Constraint+type family IsInput fact dir where   IsInput _ 'Input = ()   IsInput _ 'InputOutput = ()   IsInput fact dir = TypeError-    ( "You tried to use an " <> FormatDirection dir <> " fact of type " <> fact <> " as an input."-    % "Possible solution: change the FactDirection of " <> fact-      <> " to either 'Input' or 'InputOutput'."+    ( 'Text "You tried to use an " ':<>: 'ShowType (FormatDirection dir) ':<>: 'Text " fact of type " ':<>: 'ShowType fact ':<>: 'Text " as an input."+    ':$$: 'Text "Possible solution: change the FactDirection of " ':<>: 'ShowType fact+      ':<>: 'Text " to either 'Input' or 'InputOutput'."     ) -type family IsOutput (fact :: Type) (dir :: Direction) :: Constraint where+type IsOutput :: Type -> Direction -> Constraint+type family IsOutput fact dir where   IsOutput _ 'Output = ()   IsOutput _ 'InputOutput = ()   IsOutput fact dir = TypeError-    ( "You tried to use an " <> FormatDirection dir <> " fact of type " <> fact <> " as an output."-    % "Possible solution: change the FactDirection of " <> fact-      <> " to either 'Output' or 'InputOutput'."+    ( 'Text "You tried to use an " ':<>: 'ShowType (FormatDirection dir) ':<>: 'Text " fact of type " ':<>: 'ShowType fact ':<>: 'Text " as an output."+    ':$$: 'Text "Possible solution: change the FactDirection of " ':<>: 'ShowType fact+      ':<>: 'Text " to either 'Output' or 'InputOutput'."     ) -type family FormatDirection (dir :: Direction) where+type FormatDirection :: Direction -> Symbol+type family FormatDirection dir where   FormatDirection 'Output = "output"   FormatDirection 'Input = "input"   FormatDirection 'Internal = "internal"@@ -79,18 +88,20 @@ -- | A helper type family for checking if a specific Souffle `Program` contains --   a certain `Fact`. This constraint will generate a user-friendly type error --   if this is not the case.-type family ContainsFact prog fact :: Constraint where+type ContainsFact :: Type -> Type -> Constraint+type family ContainsFact prog fact where   ContainsFact prog fact =     CheckContains prog (ProgramFacts prog) fact +type CheckContains :: Type -> [Type] -> Type -> Constraint type family CheckContains prog facts fact :: Constraint where   CheckContains prog '[] fact =-    TypeError ("You tried to perform an action with a fact of type '" <> fact-    <> "' for program '" <> prog <> "'."-    % "The program contains the following facts: " <> ProgramFacts prog <> "."-    % "It does not contain fact: " <> fact <> "."-    % "You can fix this error by adding the type '" <> fact-    <> "' to the ProgramFacts type in the Program instance for " <> prog <> ".")+    TypeError ('Text "You tried to perform an action with a fact of type '" ':<>: 'ShowType fact+    ':<>: 'Text "' for program '" ':<>: 'ShowType prog ':<>: 'Text "'."+    ':$$: 'Text "The program contains the following facts: " ':<>: 'ShowType (ProgramFacts prog) ':<>: 'Text "."+    ':$$: 'Text "It does not contain fact: " ':<>: 'ShowType fact ':<>: 'Text "."+    ':$$: 'Text "You can fix this error by adding the type '" ':<>: 'ShowType fact+    ':<>: 'Text "' to the ProgramFacts type in the Program instance for " ':<>: 'ShowType prog ':<>: 'Text ".")   CheckContains _ (a ': _) a = ()   CheckContains prog (_ ': as) b = CheckContains prog as b @@ -106,6 +117,7 @@ --   type ProgramFacts Path = '[Edge, Reachable] --   programName = const "path" -- @+type Program :: Type -> Constraint class Program a where   -- | A type level list of facts that belong to this program.   --   This list is used to check that only known facts are added to a program.@@ -115,6 +127,30 @@   --   This has to be the same as the name of the .dl file (minus the extension).   programName :: a -> String +-- | A helper data type, used in combination with the DerivingVia extension to+--   automatically generate code to bind Haskell to a Souffle Datalog program.+--+-- The following is an example how to bind to a Datalog program "path"+-- (saved as path.dl / path.cpp), that uses two facts called "edge" and+-- "reachable" (represented with the Edge and Reachable types):+--+-- @+-- data Path = Path+--   deriving Souffle.Program+--   via Souffle.ProgramOptions Path "path" '[Edge, Reachable]+-- @+--+-- See also: 'FactOptions'.+type ProgramOptions :: Type -> Symbol -> [Type] -> Type+newtype ProgramOptions prog progName facts+  = ProgramOptions prog++instance KnownSymbol progName => Program (ProgramOptions prog progName facts) where+  type ProgramFacts (ProgramOptions _ _ facts) = facts++  programName = const $ symbolVal (Proxy @progName)+  {-# INLINABLE programName #-}+ -- | A typeclass for data types representing a fact in datalog. -- -- Example usage:@@ -124,6 +160,7 @@ --   type FactDirection Edge = 'Input --   factName = const "edge" -- @+type Fact :: Type -> Constraint class Marshal.Marshal a => Fact a where   -- | The direction or "mode" a fact can be used in.   --   This is used to perform compile-time checks that a fact is only used@@ -136,9 +173,46 @@   -- It uses a 'Proxy' to select the correct instance.   factName :: Proxy a -> String +-- | A helper data type, used in combination with the DerivingVia extension to+--   automatically generate code to bind a Haskell datatype to a Souffle+--   Datalog fact.+--+-- The following is an example how to bind to a Datalog fact "edge"+-- that contains two symbols (strings in Haskell) that is an input (from the+-- Datalog point of view):+--+-- @+-- data Edge = Edge String String+--   deriving (Eq, Show, Generic)+--   deriving anyclass Souffle.Marshal+--   deriving Souffle.Fact+--   via Souffle.FactOptions Edge "edge" 'Souffle.Input+-- @+--+-- See also: 'ProgramOptions'.+type FactOptions :: Type -> Symbol -> Direction -> Type+newtype FactOptions fact factName dir+  = FactOptions fact++instance Marshal.Marshal fact => Marshal.Marshal (FactOptions fact name dir) where+  push (FactOptions fact) = Marshal.push fact+  {-# INLINABLE push #-}+  pop = FactOptions <$> Marshal.pop+  {-# INLINABLE pop #-}++instance ( Marshal.Marshal fact+         , KnownSymbol factName+         ) => Fact (FactOptions fact factName dir) where+  type FactDirection (FactOptions _ _ dir) = dir++  factName = const $ symbolVal (Proxy @factName)+  {-# INLINABLE factName #-}++ -- | A datatype describing which operations a certain fact supports. --   The direction is from the datalog perspective, so that it --   aligns with ".decl" statements in Souffle.+type Direction :: Type data Direction   = Input   -- ^ Fact can only be stored in Datalog (using `addFact`/`addFacts`).@@ -151,6 +225,7 @@   --   facts that are only visible inside Datalog itself.  -- | A mtl-style typeclass for Souffle-related actions.+type MonadSouffle :: (Type -> Type) -> Constraint class Monad m => MonadSouffle m where   -- | Represents a handle for interacting with a Souffle program.   --@@ -161,6 +236,9 @@   -- | Helper associated type constraint that allows collecting facts from   --   Souffle in a list or vector. Only used internally.   type CollectFacts m (c :: Type -> Type) :: Constraint+  -- | Helper associated type constraint that allows submitting facts to+  --   Souffle. Only used internally.+  type SubmitFacts m (a :: Type) :: Constraint    -- | Runs the Souffle program.   run :: Handler m prog -> m ()@@ -181,21 +259,22 @@   --   --   Conceptually equivalent to @List.find (== fact) \<$\> getFacts prog@,   --   but this operation can be implemented much faster.-  findFact :: (Fact a, ContainsOutputFact prog a, Eq a)+  findFact :: (Fact a, ContainsOutputFact prog a, Eq a, SubmitFacts m a)            => Handler m prog -> a -> m (Maybe a)    -- | Adds a fact to the program.-  addFact :: (Fact a, ContainsInputFact prog a)+  addFact :: (Fact a, ContainsInputFact prog a, SubmitFacts m a)           => Handler m prog -> a -> m ()    -- | Adds multiple facts to the program. This function could be implemented   --   in terms of 'addFact', but this is done as a minor optimization.-  addFacts :: (Foldable t, Fact a, ContainsInputFact prog a)+  addFacts :: (Foldable t, Fact a, ContainsInputFact prog a, SubmitFacts m a)            => Handler m prog -> t a -> m ()  instance MonadSouffle m => MonadSouffle (ReaderT r m) where   type Handler (ReaderT r m) = Handler m   type CollectFacts (ReaderT r m) c = CollectFacts m c+  type SubmitFacts (ReaderT r m) a = SubmitFacts m a    run = lift . run   {-# INLINABLE run #-}@@ -215,6 +294,7 @@ instance (Monoid w, MonadSouffle m) => MonadSouffle (WriterT w m) where   type Handler (WriterT w m) = Handler m   type CollectFacts (WriterT w m) c = CollectFacts m c+  type SubmitFacts (WriterT w m) a = SubmitFacts m a    run = lift . run   {-# INLINABLE run #-}@@ -231,9 +311,10 @@   addFacts facts = lift . addFacts facts   {-# INLINABLE addFacts #-} -instance MonadSouffle m => MonadSouffle (StateT s m) where-  type Handler (StateT s m) = Handler m-  type CollectFacts (StateT s m) c = CollectFacts m c+instance MonadSouffle m => MonadSouffle (LazyState.StateT s m) where+  type Handler (LazyState.StateT s m) = Handler m+  type CollectFacts (LazyState.StateT s m) c = CollectFacts m c+  type SubmitFacts (LazyState.StateT s m) a = SubmitFacts m a    run = lift . run   {-# INLINABLE run #-}@@ -250,9 +331,10 @@   addFacts facts = lift . addFacts facts   {-# INLINABLE addFacts #-} -instance (MonadSouffle m, Monoid w) => MonadSouffle (RWST r w s m) where-  type Handler (RWST r w s m) = Handler m-  type CollectFacts (RWST r w s m) c = CollectFacts m c+instance MonadSouffle m => MonadSouffle (StrictState.StateT s m) where+  type Handler (StrictState.StateT s m) = Handler m+  type CollectFacts (StrictState.StateT s m) c = CollectFacts m c+  type SubmitFacts (StrictState.StateT s m) a = SubmitFacts m a    run = lift . run   {-# INLINABLE run #-}@@ -269,9 +351,50 @@   addFacts facts = lift . addFacts facts   {-# INLINABLE addFacts #-} +instance (MonadSouffle m, Monoid w) => MonadSouffle (LazyRWS.RWST r w s m) where+  type Handler (LazyRWS.RWST r w s m) = Handler m+  type CollectFacts (LazyRWS.RWST r w s m) c = CollectFacts m c+  type SubmitFacts (LazyRWS.RWST r w s m) a = SubmitFacts m a++  run = lift . run+  {-# INLINABLE run #-}+  setNumThreads prog = lift . setNumThreads prog+  {-# INLINABLE setNumThreads #-}+  getNumThreads = lift . getNumThreads+  {-# INLINABLE getNumThreads #-}+  getFacts = lift . getFacts+  {-# INLINABLE getFacts #-}+  findFact prog = lift . findFact prog+  {-# INLINABLE findFact #-}+  addFact fact = lift . addFact fact+  {-# INLINABLE addFact #-}+  addFacts facts = lift . addFacts facts+  {-# INLINABLE addFacts #-}++instance (MonadSouffle m, Monoid w) => MonadSouffle (StrictRWS.RWST r w s m) where+  type Handler (StrictRWS.RWST r w s m) = Handler m+  type CollectFacts (StrictRWS.RWST r w s m) c = CollectFacts m c+  type SubmitFacts (StrictRWS.RWST r w s m) a = SubmitFacts m a++  run = lift . run+  {-# INLINABLE run #-}+  setNumThreads prog = lift . setNumThreads prog+  {-# INLINABLE setNumThreads #-}+  getNumThreads = lift . getNumThreads+  {-# INLINABLE getNumThreads #-}+  getFacts = lift . getFacts+  {-# INLINABLE getFacts #-}+  findFact prog = lift . findFact prog+  {-# INLINABLE findFact #-}+  addFact fact = lift . addFact fact+  {-# INLINABLE addFact #-}+  addFacts facts = lift . addFacts facts+  {-# INLINABLE addFacts #-}+ instance MonadSouffle m => MonadSouffle (ExceptT e m) where   type Handler (ExceptT e m) = Handler m   type CollectFacts (ExceptT e m) c = CollectFacts m c+  type SubmitFacts (ExceptT e m) a = SubmitFacts m a    run = lift . run   {-# INLINABLE run #-}@@ -288,8 +411,8 @@   addFacts facts = lift . addFacts facts   {-# INLINABLE addFacts #-} - -- | A mtl-style typeclass for Souffle-related actions that involve file IO.+type MonadSouffleFileIO :: (Type -> Type) -> Constraint class MonadSouffle m => MonadSouffleFileIO m where   -- | Load all facts from files in a certain directory.   loadFiles :: Handler m prog -> FilePath -> m ()@@ -310,20 +433,33 @@   writeFiles prog = lift . writeFiles prog   {-# INLINABLE writeFiles #-} -instance MonadSouffleFileIO m => MonadSouffleFileIO (StateT s m) where+instance MonadSouffleFileIO m => MonadSouffleFileIO (StrictState.StateT s m) where   loadFiles prog = lift . loadFiles prog   {-# INLINABLE loadFiles #-}   writeFiles prog = lift . writeFiles prog   {-# INLINABLE writeFiles #-} -instance (MonadSouffleFileIO m, Monoid w) => MonadSouffleFileIO (RWST r w s m) where+instance MonadSouffleFileIO m => MonadSouffleFileIO (LazyState.StateT s m) where   loadFiles prog = lift . loadFiles prog   {-# INLINABLE loadFiles #-}   writeFiles prog = lift . writeFiles prog   {-# INLINABLE writeFiles #-} +instance (MonadSouffleFileIO m, Monoid w) => MonadSouffleFileIO (LazyRWS.RWST r w s m) where+  loadFiles prog = lift . loadFiles prog+  {-# INLINABLE loadFiles #-}+  writeFiles prog = lift . writeFiles prog+  {-# INLINABLE writeFiles #-}++instance (MonadSouffleFileIO m, Monoid w) => MonadSouffleFileIO (StrictRWS.RWST r w s m) where+  loadFiles prog = lift . loadFiles prog+  {-# INLINABLE loadFiles #-}+  writeFiles prog = lift . writeFiles prog+  {-# INLINABLE writeFiles #-}+ instance MonadSouffleFileIO m => MonadSouffleFileIO (ExceptT s m) where   loadFiles prog = lift . loadFiles prog   {-# INLINABLE loadFiles #-}   writeFiles prog = lift . writeFiles prog   {-# INLINABLE writeFiles #-}+
lib/Language/Souffle/Compiled.hs view
@@ -1,5 +1,8 @@ {-# OPTIONS_GHC -Wno-redundant-constraints #-}-{-# LANGUAGE FlexibleInstances, TypeFamilies, DerivingVia, InstanceSigs, BangPatterns #-}+{-# LANGUAGE FlexibleInstances, FlexibleContexts, TypeFamilies, DerivingVia #-}+{-# LANGUAGE BangPatterns, RoleAnnotations, MultiParamTypeClasses #-}+{-# LANGUAGE InstanceSigs, DataKinds, TypeApplications, TypeOperators #-}+{-# LANGUAGE ConstraintKinds, PolyKinds, UndecidableInstances #-}  -- | This module provides an implementation for the typeclasses defined in --   "Language.Souffle.Class".@@ -12,11 +15,14 @@ --   C++ compilation times. module Language.Souffle.Compiled   ( Program(..)+  , ProgramOptions(..)   , Fact(..)+  , FactOptions(..)   , Marshal(..)   , Direction(..)   , ContainsInputFact   , ContainsOutputFact+  , Submit   , Handle   , SouffleM   , MonadSouffle(..)@@ -25,30 +31,59 @@   ) where  import Prelude hiding ( init )--import Control.Monad.Except-import Control.Monad.RWS.Strict-import Control.Monad.Reader+import Control.Monad.State.Strict import Data.Foldable ( traverse_ )+import Data.Functor.Identity import Data.Proxy+import Data.Kind import qualified Data.Array as A import qualified Data.Array.IO as A import qualified Data.Array.Unsafe as A+import qualified Data.ByteString as BS+import qualified Data.ByteString.Unsafe as BSU+import qualified Data.Text as T+import qualified Data.Text.Encoding as TE+import qualified Data.Text.Internal.StrictBuilder as TB+import qualified Data.Text.Lazy as TL import qualified Data.Vector as V import qualified Data.Vector.Mutable as MV+import Data.Int+import Data.Word import Foreign.ForeignPtr+import Foreign.ForeignPtr.Unsafe+import Foreign (copyBytes) import Foreign.Ptr+import qualified Foreign.Storable as S+import GHC.Generics import Language.Souffle.Class import qualified Language.Souffle.Internal as Internal import Language.Souffle.Marshal+import Control.Concurrent  +type ByteCount :: Type+type ByteCount = Int+type ByteBuf :: Type+type ByteBuf = Internal.ByteBuf++type BufData :: Type+data BufData+  = BufData+  { bufPtr :: {-# UNPACK #-} !(ForeignPtr ByteBuf)+  , bufSize :: {-# UNPACK #-} !ByteCount+  }+ -- | A datatype representing a handle to a datalog program. --   The type parameter is used for keeping track of which program --   type the handle belongs to for additional type safety.-newtype Handle prog = Handle (ForeignPtr Internal.Souffle)+type Handle :: Type -> Type+data Handle prog+  = Handle {-# UNPACK #-} !(ForeignPtr Internal.Souffle)+           {-# UNPACK #-} !(MVar BufData)+type role Handle nominal  -- | A monad for executing Souffle-related actions in.+type SouffleM :: Type -> Type newtype SouffleM a = SouffleM (IO a)   deriving (Functor, Applicative, Monad, MonadIO) via IO   deriving (Semigroup, Monoid) via (IO a)@@ -66,172 +101,398 @@ runSouffle prog action =   let progName = programName prog       (SouffleM result) = do-        handle <- fmap Handle <$> liftIO (Internal.init progName)-        action handle+        maybeHandle <- liftIO (Internal.init progName) >>= \case+          Nothing -> pure Nothing+          Just souffleHandle -> do+            bufData <- liftIO $ do+              ptr <- newForeignPtr_ nullPtr+              newMVar $ BufData ptr 0+            pure $ Just $ Handle souffleHandle bufData+        action maybeHandle    in result--type Tuple = Ptr Internal.Tuple+{-# INLINABLE runSouffle #-}  -- | A monad used solely for marshalling and unmarshalling---   between Haskell and Souffle Datalog.-newtype CMarshal a = CMarshal (ReaderT Tuple IO a)-  deriving (Functor, Applicative, Monad, MonadIO, MonadReader Tuple)-  via ( ReaderT Tuple IO )+--   between Haskell and Souffle Datalog. This fast variant is used when the+--   marshalling from Haskell to C++ and the exact size of a datastructure+--   is statically known (read: data type contains no string-like types),+--   or when marshalling from C++ to Haskell (pointer is then managed by C++).+type CMarshalFast :: Type -> Type+newtype CMarshalFast a = CMarshalFast (StateT (Ptr ByteBuf) IO a)+  deriving (Functor, Applicative, Monad, MonadIO, MonadState (Ptr ByteBuf))+  via (StateT (Ptr ByteBuf) IO) -runM :: CMarshal a -> Tuple -> IO a-runM (CMarshal m) = runReaderT m-{-# INLINABLE runM #-}+runMarshalFastM :: CMarshalFast a -> Ptr ByteBuf -> IO a+runMarshalFastM (CMarshalFast m) = evalStateT m+{-# INLINABLE runMarshalFastM #-} -instance MonadPush CMarshal where-  pushInt32 int = do-    tuple <- ask-    liftIO $ Internal.tuplePushInt32 tuple int-  {-# INLINABLE pushInt32 #-}+-- NOTE: assumes Souffle is compiled with 32-bit RAM domain.+ramDomainSize :: Int+ramDomainSize = 4 -  pushUInt32 int = do-    tuple <- ask-    liftIO $ Internal.tuplePushUInt32 tuple int-  {-# INLINABLE pushUInt32 #-}+writeAsBytes :: (S.Storable a, Marshal a) => a -> CMarshalFast ()+writeAsBytes a = do+  ptr <- gets castPtr+  liftIO $ S.poke ptr a+  put $ ptr `plusPtr` ramDomainSize+{-# INLINABLE writeAsBytes #-} -  pushFloat float = do-    tuple <- ask-    liftIO $ Internal.tuplePushFloat tuple float-  {-# INLINABLE pushFloat #-}+readAsBytes :: (S.Storable a, Marshal a) => CMarshalFast a+readAsBytes = do+  ptr <- gets castPtr+  a <- liftIO $ S.peek ptr+  put $ ptr `plusPtr` ramDomainSize+  pure a+{-# INLINABLE readAsBytes #-} -  pushString str = do-    tuple <- ask-    liftIO $ Internal.tuplePushString tuple str+instance MonadPush CMarshalFast where+  pushInt32 = writeAsBytes+  {-# INLINABLE pushInt32 #-}+  pushUInt32 = writeAsBytes+  {-# INLINABLE pushUInt32 #-}+  pushFloat = writeAsBytes+  {-# INLINABLE pushFloat #-}+  pushString str = pushText $ T.pack str   {-# INLINABLE pushString #-}+  pushText _ =+    error "Fast marshalling does not support serializing string-like values."+  {-# INLINABLE pushText #-} -instance MonadPop CMarshal where-  popInt32 = do-    tuple <- ask-    liftIO $ Internal.tuplePopInt32 tuple+instance MonadPop CMarshalFast where+  popInt32 = readAsBytes   {-# INLINABLE popInt32 #-}--  popUInt32 = do-    tuple <- ask-    liftIO $ Internal.tuplePopUInt32 tuple+  popUInt32 = readAsBytes   {-# INLINABLE popUInt32 #-}--  popFloat = do-    tuple <- ask-    liftIO $ Internal.tuplePopFloat tuple+  popFloat = readAsBytes   {-# INLINABLE popFloat #-}--  popString = do-    tuple <- ask-    liftIO $ Internal.tuplePopString tuple+  popString = T.unpack <$> popText   {-# INLINABLE popString #-}+  popText = do+    byteCount <- popUInt32+    if byteCount == 0+      then pure T.empty+      else do+        ptr <- gets castPtr+        bs <- liftIO $ BSU.unsafePackCStringLen (ptr, fromIntegral byteCount)+        put $ ptr `plusPtr` fromIntegral byteCount+        -- NOTE: $! is needed here to force the text value. A copy needs to+        -- be made, before the bytearray is overwritten.+        pure $! TB.toText $ TB.unsafeFromByteString bs+  {-# INLINABLE popText #-} ++type MarshalState :: Type+data MarshalState+  = MarshalState+  { _buf :: {-# UNPACK #-} !BufData+  , _ptr :: {-# UNPACK #-} !(Ptr ByteBuf)+  , _ptrOffset :: {-# UNPACK #-} !Int+  }++-- | A monad used solely for marshalling from Haskell to Souffle Datalog (C++).+--   This slow variant is used when the exact size of a datastructure is *not*+--   statically known (read: data type contains string-like types).+type CMarshalSlow :: Type -> Type+newtype CMarshalSlow a = CMarshalSlow (StateT MarshalState IO a)+  deriving (Functor, Applicative, Monad, MonadIO, MonadState MarshalState)+  via (StateT MarshalState IO)++runMarshalSlowM :: BufData -> Int -> CMarshalSlow a -> IO a+runMarshalSlowM bufData byteCount (CMarshalSlow m) = do+  bufData' <- if bufSize bufData > byteCount+    then pure bufData+    else flip BufData byteCount <$> allocateBuf byteCount+  let ptr = unsafeForeignPtrToPtr (bufPtr bufData')+  evalStateT m $ MarshalState bufData' ptr 0+{-# INLINABLE runMarshalSlowM #-}++resizeBufWhenNeeded :: ByteCount -> CMarshalSlow ()+resizeBufWhenNeeded byteCount = do+  MarshalState bufData _ offset <- get+  let totalByteCount = bufSize bufData+  when (byteCount + offset > totalByteCount) $ do+    let newTotalByteCount = getNewTotalByteCount byteCount offset totalByteCount+    newBuf <- allocateBuf newTotalByteCount+    copyBuf newBuf (bufPtr bufData) totalByteCount+    let newPtr = unsafeForeignPtrToPtr newBuf+        bufData' = BufData newBuf newTotalByteCount+    put $ MarshalState bufData' (newPtr `plusPtr` offset) offset+{-# INLINABLE resizeBufWhenNeeded #-}++allocateBuf :: MonadIO m => ByteCount -> m (ForeignPtr ByteBuf)+allocateBuf byteCount = liftIO $+  mallocForeignPtrBytes byteCount+{-# INLINABLE allocateBuf #-}++copyBuf :: ForeignPtr ByteBuf -> ForeignPtr ByteBuf -> Int -> CMarshalSlow ()+copyBuf dst src byteCount = liftIO $+  withForeignPtr src $ \srcPtr ->+  withForeignPtr dst $ \dstPtr ->+    copyBytes dstPtr srcPtr byteCount+{-# INLINABLE copyBuf #-}++getNewTotalByteCount :: ByteCount -> Int -> ByteCount -> ByteCount+getNewTotalByteCount byteCount offset = go where+  go totalByteCount+    | byteCount + offset > totalByteCount = go (totalByteCount * 2)+    | otherwise = totalByteCount+{-# INLINABLE getNewTotalByteCount #-}++incrementPtr :: ByteCount -> CMarshalSlow ()+incrementPtr byteCount =+  modify $ \(MarshalState buf ptr offset) ->+    MarshalState buf (ptr `plusPtr` byteCount) (offset + byteCount)+{-# INLINABLE incrementPtr #-}++instance MonadPush CMarshalSlow where+  pushInt32 = writeAsBytesSlow+  {-# INLINABLE pushInt32 #-}+  pushUInt32 = writeAsBytesSlow+  {-# INLINABLE pushUInt32 #-}+  pushFloat = writeAsBytesSlow+  {-# INLINABLE pushFloat #-}+  pushString str = pushText $ T.pack str+  {-# INLINABLE pushString #-}+  pushText txt = do+    let bs = TE.encodeUtf8 txt  -- TODO: is it possible to get rid of this copy?+        len = BS.length bs+    resizeBufWhenNeeded (ramDomainSize + len)+    pushUInt32 (fromIntegral len)+    if len == 0+      then pure ()+      else do+        ptr <- gets (castPtr . _ptr)+        liftIO $ BSU.unsafeUseAsCString bs $ flip (copyBytes ptr) len+        incrementPtr len+  {-# INLINABLE pushText #-}++writeAsBytesSlow :: (S.Storable a, Marshal a) => a -> CMarshalSlow ()+writeAsBytesSlow a = do+  resizeBufWhenNeeded ramDomainSize+  ptr <- gets (castPtr . _ptr)+  liftIO $ S.poke ptr a+  incrementPtr ramDomainSize+{-# INLINABLE writeAsBytesSlow #-}+++type Collect :: (Type -> Type) -> Constraint class Collect c where-  collect :: Marshal a => Int -> ForeignPtr Internal.RelationIterator -> IO (c a)+  collect :: Marshal a => Word32 -> CMarshalFast (c a)  instance Collect [] where-  collect factCount = go 0 factCount []-    where-      go idx count acc _ | idx == count = pure acc-      go idx count !acc !it = do-        tuple <- Internal.relationIteratorNext it-        result <- runM pop tuple-        go (idx + 1) count (result : acc) it+  collect objCount = go objCount [] where+    go count acc+      | count == 0 = pure acc+      | otherwise = do+        !x <- pop+        go (count - 1) (x:acc)   {-# INLINABLE collect #-}  instance Collect V.Vector where-  collect factCount iterator = do-    vec <- MV.unsafeNew factCount-    go vec 0 factCount iterator+  collect objCount = do+    vm <- liftIO $ MV.unsafeNew objCount'+    collect' vm 0     where-      go vec idx count _ | idx == count = V.unsafeFreeze vec-      go vec idx count it = do-        tuple <- Internal.relationIteratorNext it-        result <- runM pop tuple-        MV.unsafeWrite vec idx result-        go vec (idx + 1) count it+      objCount' = fromIntegral objCount+      collect' vec idx+        | idx == objCount' = liftIO $ V.unsafeFreeze vec+        | otherwise = do+          !obj <- pop+          liftIO $ MV.write vec idx obj+          collect' vec (idx + 1)   {-# INLINABLE collect #-}  instance Collect (A.Array Int) where-  collect factCount iterator = do-    array <- A.newArray_ (0, factCount - 1)-    go array 0 factCount iterator+  collect objCount = do+    ma <- liftIO $ A.newArray_ (0, objCount' - 1)+    collect' ma 0     where-      go :: Marshal a-         => A.IOArray Int a-         -> Int-         -> Int-         -> ForeignPtr Internal.RelationIterator-         -> IO (A.Array Int a)-      go array idx count _ | idx == count = A.unsafeFreeze array-      go array idx count it = do-        tuple <- Internal.relationIteratorNext it-        result <- runM pop tuple-        A.writeArray array idx result-        go array (idx + 1) count it+      objCount' = fromIntegral objCount+      collect' :: Marshal a => A.IOArray Int a -> Int -> CMarshalFast (A.Array Int a)+      collect' array idx+        | idx == objCount' = liftIO $ A.unsafeFreeze array+        | otherwise = do+          !obj <- pop+          liftIO $ A.writeArray array idx obj+          collect' array (idx + 1)   {-# INLINABLE collect #-} +-- | A helper typeclass constraint, needed to serialize Datalog facts from+--   Haskell to C++.+type Submit :: Type -> Constraint+type Submit a = ToByteSize (GetFields (Rep a))+ instance MonadSouffle SouffleM where   type Handler SouffleM = Handle   type CollectFacts SouffleM c = Collect c+  type SubmitFacts SouffleM a = Submit a -  run (Handle prog) = SouffleM $ Internal.run prog+  run (Handle prog _) = SouffleM $ Internal.run prog   {-# INLINABLE run #-} -  setNumThreads (Handle prog) numCores =+  setNumThreads (Handle prog _) numCores =     SouffleM $ Internal.setNumThreads prog numCores   {-# INLINABLE setNumThreads #-} -  getNumThreads (Handle prog) =+  getNumThreads (Handle prog _) =     SouffleM $ Internal.getNumThreads prog   {-# INLINABLE getNumThreads #-} -  addFact :: forall a prog. (Fact a, ContainsInputFact prog a)+  addFact :: forall a prog. (Fact a, ContainsInputFact prog a, Submit a)           => Handle prog -> a -> SouffleM ()-  addFact (Handle prog) fact = liftIO $ do+  addFact (Handle prog bufVar) fact = liftIO $ do     let relationName = factName (Proxy :: Proxy a)     relation <- Internal.getRelation prog relationName-    addFact' relation fact+    writeBytes bufVar relation (Identity fact)   {-# INLINABLE addFact #-} -  addFacts :: forall t a prog . (Foldable t, Fact a, ContainsInputFact prog a)+  addFacts :: forall t a prog. (Foldable t, Fact a, ContainsInputFact prog a, Submit a)            => Handle prog -> t a -> SouffleM ()-  addFacts (Handle prog) facts = liftIO $ do+  addFacts (Handle prog bufVar) facts = liftIO $ do     let relationName = factName (Proxy :: Proxy a)     relation <- Internal.getRelation prog relationName-    traverse_ (addFact' relation) facts+    writeBytes bufVar relation facts   {-# INLINABLE addFacts #-}    getFacts :: forall a c prog. (Fact a, ContainsOutputFact prog a, Collect c)            => Handle prog -> SouffleM (c a)-  getFacts (Handle prog) = SouffleM $ do+  getFacts (Handle prog _) = SouffleM $ do     let relationName = factName (Proxy :: Proxy a)     relation <- Internal.getRelation prog relationName-    factCount <- Internal.countFacts relation-    Internal.getRelationIterator relation >>= collect factCount+    buf <- withForeignPtr prog $ flip Internal.popFacts relation+    flip runMarshalFastM buf $ collect =<< popUInt32   {-# INLINABLE getFacts #-} -  findFact :: forall a prog. (Fact a, ContainsOutputFact prog a)+  findFact :: forall a prog. (Fact a, ContainsOutputFact prog a, Submit a)            => Handle prog -> a -> SouffleM (Maybe a)-  findFact (Handle prog) fact = SouffleM $ do+  findFact (Handle prog bufVar) fact = SouffleM $ do     let relationName = factName (Proxy :: Proxy a)     relation <- Internal.getRelation prog relationName-    tuple <- Internal.allocTuple relation-    withForeignPtr tuple $ runM (push fact)-    found <- Internal.containsTuple relation tuple+    found <- case estimateNumBytes (Proxy @a) of+      Exact numBytes -> do+        modifyMVarMasked bufVar $ \bufData -> do+          bufData' <- if bufSize bufData > numBytes+            then pure bufData+            else flip BufData numBytes <$> allocateBuf numBytes+          found <- withForeignPtr (bufPtr bufData') $ \ptr -> do+            runMarshalFastM (push fact) ptr+            Internal.containsFact relation ptr+          pure (bufData', found)+      Estimated numBytes -> modifyMVarMasked bufVar $ \bufData ->+        runMarshalSlowM bufData numBytes $ do+          push fact+          bufData' <- gets _buf+          liftIO $ withForeignPtr (bufPtr bufData') $ \ptr -> do+            found <- Internal.containsFact relation ptr+            pure (bufData', found)     pure $ if found then Just fact else Nothing   {-# INLINABLE findFact #-} -addFact' :: Fact a => Ptr Internal.Relation -> a -> IO ()-addFact' relation fact = do-  tuple <- Internal.allocTuple relation-  withForeignPtr tuple $ runM (push fact)-  Internal.addTuple relation tuple-{-# INLINABLE addFact' #-}-- instance MonadSouffleFileIO SouffleM where-  loadFiles (Handle prog) = SouffleM . Internal.loadAll prog+  loadFiles (Handle prog _) = SouffleM . Internal.loadAll prog   {-# INLINABLE loadFiles #-} -  writeFiles (Handle prog) = SouffleM . Internal.printAll prog+  writeFiles (Handle prog _) = SouffleM . Internal.printAll prog   {-# INLINABLE writeFiles #-} ++type ByteSize :: Type+data ByteSize+  = Exact {-# UNPACK #-} !ByteCount+  | Estimated {-# UNPACK #-} !ByteCount++instance Semigroup ByteSize where+  Exact s1 <> Exact s2 = Exact (s1 + s2)+  Exact s1 <> Estimated s2 = Estimated (s1 + s2)+  Estimated s1 <> Exact s2 = Estimated (s1 + s2)+  Estimated s1 <> Estimated s2 = Estimated (s1 + s2)+  {-# INLINABLE (<>) #-}++type ToByteSize :: k -> Constraint+class ToByteSize a where+  toByteSize :: Proxy a -> ByteSize++instance ToByteSize Int32 where+  toByteSize = const $ Exact 4+  {-# INLINABLE toByteSize #-}++instance ToByteSize Word32 where+  toByteSize = const $ Exact 4+  {-# INLINABLE toByteSize #-}++instance ToByteSize Float where+  toByteSize = const $ Exact 4+  {-# INLINABLE toByteSize #-}++instance ToByteSize String where+  -- 4 for length prefix + 32 for actual string+  toByteSize = const $ Estimated 36+  {-# INLINABLE toByteSize #-}++instance ToByteSize T.Text where+  -- 4 for length prefix + 32 for actual string+  toByteSize = const $ Estimated 36+  {-# INLINABLE toByteSize #-}++instance ToByteSize TL.Text where+  -- 4 for length prefix + 32 for actual string+  toByteSize = const $ Estimated 36+  {-# INLINABLE toByteSize #-}++instance ToByteSize '[] where+  toByteSize = const $ Exact 0+  {-# INLINABLE toByteSize #-}++instance (ToByteSize a, ToByteSize as) => ToByteSize (a ': as) where+  toByteSize =+    const $ toByteSize (Proxy @a) <> toByteSize (Proxy @as)+  {-# INLINABLE toByteSize #-}++-- | A helper type family, for getting all directly marshallable fields of a type.+type GetFields :: k -> [Type]+type family GetFields a where+  GetFields (K1 _ a) = DoGetFields a+  GetFields (M1 _ _ a) = GetFields a+  GetFields (f :*: g) = GetFields f ++ GetFields g++type DoGetFields :: Type -> [Type]+type family DoGetFields a where+  DoGetFields Int32 = '[Int32]+  DoGetFields Word32 = '[Word32]+  DoGetFields Float = '[Float]+  DoGetFields String = '[String]+  DoGetFields T.Text = '[T.Text]+  DoGetFields TL.Text = '[TL.Text]+  DoGetFields a = GetFields (Rep a)++type (++) :: [Type] -> [Type] -> [Type]+type family a ++ b where+  '[] ++ b = b+  (a ': as) ++ bs = a ': as ++ bs++estimateNumBytes :: forall a. Submit a => Proxy a -> ByteSize+estimateNumBytes _ = toByteSize (Proxy @(GetFields (Rep a)))+{-# INLINABLE estimateNumBytes #-}++writeBytes :: forall f a. (Foldable f, Marshal a, Submit a)+           => MVar BufData -> Ptr Internal.Relation -> f a -> IO ()+writeBytes bufVar relation fa = case estimateNumBytes (Proxy @a) of+  Exact numBytes -> modifyMVarMasked_ bufVar $ \bufData -> do+    let totalByteCount = numBytes * objCount+    bufData' <- if bufSize bufData > totalByteCount+      then pure bufData+      else flip BufData totalByteCount <$> allocateBuf totalByteCount+    withForeignPtr (bufPtr bufData') $ \ptr -> do+      runMarshalFastM (traverse_ push fa) ptr+      Internal.pushFacts relation ptr (fromIntegral objCount)+    pure bufData'++  Estimated numBytes -> modifyMVarMasked_ bufVar $ \bufData ->+    runMarshalSlowM bufData (numBytes * objCount) $ do+      traverse_ push fa+      bufData' <- gets _buf+      liftIO $ withForeignPtr (bufPtr bufData') $ \ptr -> do+        Internal.pushFacts relation ptr (fromIntegral objCount)+        pure bufData'+  where objCount = length fa+{-# INLINABLE writeBytes #-}
− lib/Language/Souffle/Experimental.hs 
@@ -1,1104 +0,0 @@-{-# LANGUAGE GADTs, RankNTypes, DataKinds, TypeOperators, ConstraintKinds #-}-{-# LANGUAGE UndecidableInstances, UndecidableSuperClasses, FlexibleContexts #-}-{-# LANGUAGE MultiParamTypeClasses, FlexibleInstances, DerivingVia, ScopedTypeVariables #-}-{-# LANGUAGE PolyKinds, TypeFamilyDependencies #-}-{-# OPTIONS_GHC -Wno-redundant-constraints #-}--{-| This module provides an experimental DSL for generating Souffle Datalog code,-    directly from Haskell.--    The module is meant to be imported unqualified, unlike the rest of this-    library. This allows for a syntax that is very close to the corresponding-    Datalog syntax you would normally write.--    The functions and operators provided by this module follow a naming scheme:--    - If there is no clash with something imported via Prelude, the-      function or operator is named exactly the same as in Souffle.-    - If there is a clash for functions, an apostrophe is appended-      (e.g. "max" in Datalog is 'max'' in Haskell).-    - Most operators (besides those from the Num typeclass) start with a "."-      (e.g. '.^' is the "^"  operator in Datalog)--    The DSL makes heavy use of Haskell's typesystem to avoid-    many kinds of errors. This being said, not everything can be checked at-    compile-time (for example performing comparisons on ungrounded variables-    can't be checked). For this reason you should regularly write the-    Datalog code to a file while prototyping your algorithm and check it using-    the Souffle executable for errors.--    A large subset of the Souffle language is covered, with some exceptions-    such as "$", aggregates, ... There are no special functions for supporting-    components either, but this is automatically possible by making use of-    polymorphism in Haskell.--    Here's an example snippet of Haskell code that can generate Datalog code:--    @-    -- Assuming we have 2 types of facts named Edge and Reachable:-    data Edge = Edge String String-    data Reachable = Reachable String String--    program = do-      Predicate edge <- predicateFor \@Edge-      Predicate reachable <- predicateFor \@Reachable-      a <- var "a"-      b <- var "b"-      c <- var "c"-      reachable(a, b) |- edge(a, b)-      reachable(a, b) |- do-        edge(a, c)-        reachable(c, b)-    @--    When rendered to a file (using 'renderIO'), this generates the following-    Souffle code:--    @-    .decl edge(t1: symbol, t2: symbol)-    .input edge-    .decl reachable(t1: symbol, t2: symbol)-    .output reachable-    reachable(a, b) :--      edge(a, b)-    reachable(a, b) :- do-      edge(a, c)-      reachable(c, b)-    @--    For more examples, take a look at the <https://github.com/luc-tielen/souffle-haskell/blob/2c24e1e169da269c45fc192ab5efd4ff2196114b/tests/Test/Language/Souffle/ExperimentalSpec.hs tests>.--}-module Language.Souffle.Experimental-  ( -- * DSL-related types and functions-    -- ** Types-    Predicate(..)-  , Fragment-  , Tuple-  , DSL-  , Head-  , Body-  , Term-  , VarName-  , UsageContext(..)-  , Direction(..)-  , ToPredicate-  , FactMetadata(..)-  , Metadata(..)-  , StructureOpt(..)-  , InlineOpt(..)-  -- ** Basic building blocks-  , predicateFor-  , var-  , __-  , underscore-  , (|-)-  , (\/)-  , not'-  -- ** Souffle operators-  , (.<)-  , (.<=)-  , (.>)-  , (.>=)-  , (.=)-  , (.!=)-  , (.^)-  , (.%)-  , band-  , bor-  , bxor-  , lor-  , land-  -- ** Souffle functions-  , max'-  , min'-  , cat-  , contains-  , match-  , ord-  , strlen-  , substr-  , to_number-  , to_string-  -- * Functions for running a Datalog DSL fragment / AST directly.-  , runSouffleInterpretedWith-  , runSouffleInterpreted-  , embedProgram-  -- * Rendering functions-  , render-  , renderIO-  -- * Helper type families useful in some situations-  , Structure-  , NoVarsInAtom-  , SupportsArithmetic-  ) where--import Control.Monad.Reader-import Control.Monad.State-import Control.Monad.Writer-import Data.Int-import Data.Kind-import Data.List.NonEmpty (NonEmpty(..), toList)-import Data.Map ( Map )-import qualified Data.Map as Map-import Data.Maybe (fromMaybe, catMaybes, mapMaybe)-import Data.Proxy-import Data.String-import qualified Data.Text as T-import qualified Data.Text.IO as TIO-import qualified Data.Text.Lazy as TL-import Data.Word-import GHC.Generics-import GHC.TypeLits-import Language.Haskell.TH.Syntax (qRunIO, qAddForeignFilePath, Q, Dec, ForeignSrcLang(..))-import Language.Souffle.Class ( Program(..), Fact(..), ContainsFact, Direction(..) )-import Language.Souffle.Internal.Constraints (SimpleProduct)-import qualified Language.Souffle.Interpreted as I-import System.Directory-import System.FilePath-import System.IO.Temp-import System.Process-import Text.Printf (printf)-import Type.Errors.Pretty----- | A datatype that contains a function for generating Datalog AST fragments---   that can be glued together using other functions in this module.------   The rank-N type allows using the inner function in multiple places to---   generate different parts of the AST. This is one of the key things---   that allows writing Haskell code in a very smilar way to the Datalog code.------   The inner function uses the 'Structure' of a type to compute what the---   shape of the input tuple for the predicate should be. For example, if a---   fact has a data constructor containing a Float and a String,---   the resulting tuple will be of type ('Term' ctx Float, 'Term' ctx String).------   Currently, only facts with up to 10 fields are supported. If you need more---   fields, please file an issue on---   <https://github.com/luc-tielen/souffle-haskell/issues Github>.-newtype Predicate a-  = Predicate (forall f ctx. Fragment f ctx => Tuple ctx (Structure a) -> f ctx ())--type VarMap = Map VarName Int---- | The main monad in which Datalog AST fragments are combined together---   using other functions in this module.------   - The "prog" type variable is used for performing many compile time checks.---     This variable is filled (automatically) with a type that implements the---     'Program' typeclass.---   - The "ctx" type variable is the context in which a DSL fragment is used.---     For more information, see 'UsageContext'.---   - The "a" type variable is the value contained inside---     (just like other monads).-newtype DSL prog ctx a = DSL (StateT VarMap (Writer [AST]) a)-  deriving (Functor, Applicative, Monad, MonadWriter [AST], MonadState VarMap)-  via (StateT VarMap (Writer [AST]))--addDefinition :: AST -> DSL prog 'Definition ()-addDefinition dl = tell [dl]---- | This function runs the DSL fragment directly using the souffle interpreter---   executable.------   It does this by saving the fragment to a temporary file right before---   running the souffle interpreter. All created files are automatically---   cleaned up after the souffle related actions have been executed. If this is---   not your intended behavior, see 'runSouffleInterpretedWith' which allows---   passing in different interpreter settings.-runSouffleInterpreted-  :: (MonadIO m, Program prog)-  => prog-  -> DSL prog 'Definition ()-  -> (Maybe (I.Handle prog) -> I.SouffleM a)-  -> m a-runSouffleInterpreted program dsl f = liftIO $ do-  tmpDir <- getCanonicalTemporaryDirectory-  souffleHsDir <- createTempDirectory tmpDir "souffle-haskell"-  defaultCfg <- I.defaultConfig-  let cfg = defaultCfg { I.cfgDatalogDir = souffleHsDir-                       , I.cfgFactDir = Just souffleHsDir-                       , I.cfgOutputDir = Just souffleHsDir-                       }-  runSouffleInterpretedWith cfg program dsl f <* removeDirectoryRecursive souffleHsDir---- | This function runs the DSL fragment directly using the souffle interpreter---   executable.------   It does this by saving the fragment to a file in the directory specified by---   the 'I.cfgDatalogDir' field in the interpreter settings. Depending on the---   chosen settings, the fact and output files may not be automatically cleaned---   up after running the souffle interpreter. See 'I.runSouffleWith' for more---   information on automatic cleanup.-runSouffleInterpretedWith-  :: (MonadIO m, Program prog)-  => I.Config-  -> prog-  -> DSL prog 'Definition ()-  -> (Maybe (I.Handle prog) -> I.SouffleM a)-  -> m a-runSouffleInterpretedWith config program dsl f = liftIO $ do-  let progName = programName program-      datalogFile = I.cfgDatalogDir config </> progName <.> "dl"-  renderIO program datalogFile dsl-  I.runSouffleWith config program f---- | Embeds a Datalog program from a DSL fragment directly in a Haskell file.------   Note that due to TemplateHaskell staging restrictions, this function must---   be used in a different module than the module where 'Program' and 'Fact'---   instances are defined.------   In order to use this function correctly, you have to add the following---   line to the top of the module where 'embedProgram' is used in order---   for the embedded C++ code to be compiled correctly:------   > {-# OPTIONS_GHC -optc-std=c++17 -D__EMBEDDED_SOUFFLE__ #-}-embedProgram :: Program prog => prog -> DSL prog 'Definition () -> Q [Dec]-embedProgram program dsl = do-  cppFile <- qRunIO $ do-    tmpDir <- getCanonicalTemporaryDirectory-    souffleHsDir <- createTempDirectory tmpDir "souffle-haskell"-    let progName = programName program-        datalogFile = souffleHsDir </> progName <.> "dl"-        cppFile = souffleHsDir </> progName <.> "cpp"-    renderIO program datalogFile dsl-    callCommand $ printf "souffle -g %s %s" cppFile datalogFile-    pure cppFile-  qAddForeignFilePath LangCxx cppFile-  pure []--runDSL :: Program prog => prog -> DSL prog 'Definition a -> DL-runDSL _ (DSL a) = Statements $ mapMaybe simplify $ execWriter (evalStateT a mempty) where-  simplify = \case-    Declare' name dir fields opts -> pure $ Declare name dir fields opts-    Rule' name terms body -> Rule name terms <$> simplify body-    Atom' name terms -> pure $ Atom name terms-    And' exprs -> case mapMaybe simplify exprs of-      [] -> Nothing-      exprs' -> pure $ foldl1 And exprs'-    Or' exprs -> case mapMaybe simplify exprs of-      [] -> Nothing-      exprs' -> pure $ foldl1 Or exprs'-    Not' expr -> Not <$> simplify expr-    Constrain' e -> pure $ Constrain e---- | Generates a unique variable, using the name argument as a hint.------   The type of the variable is determined the first predicate it is used in.---   The 'NoVarsInAtom' constraint generates a user-friendly type error if the---   generated variable is used inside a relation (which is not valid in---   Datalog).------   Note: If a variable is created but not used using this function, you will---   get a compile-time error because it can't deduce the constraint.-var :: NoVarsInAtom ctx => VarName -> DSL prog ctx' (Term ctx ty)-var name = do-  count <- fromMaybe 0 <$> gets (Map.lookup name)-  modify $ Map.insert name (count + 1)-  let varName = if count == 0 then name else name <> "_" <> T.pack (show count)-  pure $ VarTerm varName---- | Data type representing the head of a relation---   (the part before ":-" in a Datalog relation).------   - The "ctx" type variable is the context in which this type is used.---     For this type, this will always be 'Relation'. The variable is there to---     perform some compile-time checks.---   - The "unused" type variable is unused and only there so the type has the---     same kind as 'Body' and 'DSL'.------   See also '|-'.-data Head ctx unused-  = Head Name (NonEmpty SimpleTerm)---- | Data type representing the body of a relation---   (what follows after ":-" in a Datalog relation).------   By being a monad, it supports do-notation which allows for a syntax---   that is quite close to Datalog.------   - The "ctx" type variable is the context in which this type is used.---     For this type, this will always be 'Relation'. The variable is there to---     perform some compile-time checks.---   - The "a" type variable is the value contained inside---     (just like other monads).------   See also '|-'.-newtype Body ctx a = Body (Writer [AST] a)-  deriving (Functor, Applicative, Monad, MonadWriter [AST])-  via (Writer [AST])---- | Creates a fragment that is the logical disjunction (OR) of 2 sub-fragments.---   This corresponds with ";" in Datalog.-(\/) :: Body ctx () -> Body ctx () -> Body ctx ()-body1 \/ body2 = do-  let rules1 = And' $ runBody body1-      rules2 = And' $ runBody body2-  tell [Or' [rules1, rules2]]---- | Creates a fragment that is the logical negation of a sub-fragment.---   This is equivalent to "!" in Datalog. (But this operator can't be used---   in Haskell since it only allows unary negation as a prefix operator.)-not' :: Body ctx a -> Body ctx ()-not' body = do-  let rules = And' $ runBody body-  tell [Not' rules]--runBody :: Body ctx a -> [AST]-runBody (Body m) = execWriter m--data TypeInfo (a :: k) (ts :: [Type]) = TypeInfo---- | Constraint that makes sure a type can be converted to a predicate function.---   It gives a user-friendly error in case any of the sub-constraints---   are not met.-type ToPredicate prog a =-  ( Fact a-  , FactMetadata a-  , ContainsFact prog a-  , SimpleProduct a-  , Assert (Length (Structure a) <=? 10) BigTupleError-  , KnownDLTypes (Structure a)-  , KnownDirection (FactDirection a)-  , KnownSymbols (AccessorNames a)-  , ToTerms (Structure a)-  )---- | A typeclass for optionally configuring extra settings---   (for performance reasons).----   Since it contains no required functions, it is possible to derive this---   typeclass automatically (this gives you the default behavior):------   @---   data Edge = Edge String String---     deriving (Generic, Marshal, FactMetadata)---   @-class (Fact a, SimpleProduct a) => FactMetadata a where-  -- | An optional function for configuring fact metadata.-  ---  --   By default no extra options are configured.-  --   For more information, see the 'Metadata' type.-  factOpts :: Proxy a -> Metadata a-  factOpts = const $ Metadata Automatic NoInline---- | A data type that allows for finetuning of fact settings---   (for performance reasons).-data Metadata a-  = Metadata (StructureOpt a) (InlineOpt (FactDirection a))---- | Datatype describing the way a fact is stored inside Datalog.---   A different choice of storage type can lead to an improvement in---   performance (potentially).------   For more information, see this---   <https://souffle-lang.github.io/tuning#datastructure link> and this---   <https://souffle-lang.github.io/relations link>.-data StructureOpt (a :: Type) where-  -- | Automatically choose the underlying storage for a relation.-  --   This is the storage type that is used by default.-  ---  --   For Souffle, it will choose a direct btree for facts with arity <= 6.-  --   For larger facts, it will use an indirect btree.-  Automatic :: StructureOpt a-  -- | Uses a direct btree structure.-  BTree :: StructureOpt a-  -- | Uses a brie structure. This can improve performance in some cases and is-  --   more memory efficient for particularly large relations.-  Brie :: StructureOpt a-  -- | A high performance datastructure optimised specifically for equivalence-  --   relations. This is only valid for binary facts with 2 fields of the-  --   same type.-  EqRel :: (IsBinaryRelation a, Structure a ~ '[t, t]) => StructureOpt a--type IsBinaryRelation a =-  Assert (Length (Structure a) == 2)-         ("Equivalence relations are only allowed with binary relations" <> ".")---- | Datatype indicating if we should inline a fact or not.-data InlineOpt (d :: Direction) where-  -- | Inlines the fact, only possible for internal facts.-  Inline :: InlineOpt 'Internal-  -- | Does not inline the fact.-  NoInline :: InlineOpt d---- | Generates a function for a type that implements 'Fact' and is a---   'SimpleProduct'. The predicate function takes the same amount of arguments---   as the original fact type. Calling the function with a tuple of arguments,---   creates fragments of datalog code that can be glued together using other---   functions in this module.------   Note: You need to specify for which fact you want to return a predicate---   for using TypeApplications.-predicateFor :: forall a prog. ToPredicate prog a => DSL prog 'Definition (Predicate a)-predicateFor = do-  let typeInfo = TypeInfo :: TypeInfo a (Structure a)-      p = Proxy :: Proxy a-      name = T.pack $ factName p-      accNames = fromMaybe genericNames $ accessorNames p-      opts = toSimpleMetadata $ factOpts p-      genericNames = map (("t" <>) . T.pack . show) [1..]-      tys = getTypes (Proxy :: Proxy (Structure a))-      direction = getDirection (Proxy :: Proxy (FactDirection a))-      fields = zipWith FieldData tys accNames-      definition = Declare' name direction fields opts-  addDefinition definition-  pure $ Predicate $ toFragment typeInfo name--toSimpleMetadata :: Metadata a -> SimpleMetadata-toSimpleMetadata (Metadata struct inline) =-  let structOpt = case struct of-        Automatic -> AutomaticLayout-        BTree -> BTreeLayout-        Brie -> BrieLayout-        EqRel -> EqRelLayout-      inlineOpt = case inline of-        Inline -> DoInline-        NoInline -> DoNotInline-  in SimpleMetadata structOpt inlineOpt--class KnownDirection a where-  getDirection :: Proxy a -> Direction-instance KnownDirection 'Input where getDirection = const Input-instance KnownDirection 'Output where getDirection = const Output-instance KnownDirection 'InputOutput where getDirection = const InputOutput-instance KnownDirection 'Internal where getDirection = const Internal---- | Turnstile operator from Datalog, used in relations.------   This is used for creating a DSL fragment that contains a relation.---   NOTE: |- is used instead of :- due to limitations of the Haskell syntax.-(|-) :: Head 'Relation a -> Body 'Relation () -> DSL prog 'Definition ()-Head name terms |- body =-  let rules = runBody body-      relation = Rule' name terms (And' rules)-  in addDefinition relation--infixl 0 |----- | A typeclass used for generating AST fragments of Datalog code.---   The generated fragments can be further glued together using the---   various functions in this module.-class Fragment f ctx where-  toFragment :: ToTerms ts => TypeInfo a ts -> Name -> Tuple ctx ts -> f ctx ()--instance Fragment Head 'Relation where-  toFragment typeInfo name terms =-    let terms' = toTerms (Proxy :: Proxy 'Relation) typeInfo terms-     in Head name terms'--instance Fragment Body 'Relation where-  toFragment typeInfo name terms =-    let terms' = toTerms (Proxy :: Proxy 'Relation) typeInfo terms-    in tell [Atom' name terms']--instance Fragment (DSL prog) 'Definition where-  toFragment typeInfo name terms =-    let terms' = toTerms (Proxy :: Proxy 'Definition) typeInfo terms-     in addDefinition $ Atom' name terms'---data RenderMode = Nested | TopLevel---- | Renders a DSL fragment to the corresponding Datalog code and writes it to---   a file.-renderIO :: Program prog => prog -> FilePath -> DSL prog 'Definition () -> IO ()-renderIO prog path = TIO.writeFile path . render prog---- | Renders a DSL fragment to the corresponding Datalog code.-render :: Program prog => prog -> DSL prog 'Definition () -> T.Text-render prog = flip runReader TopLevel . f . runDSL prog where-  f = \case-    Statements stmts ->-      T.unlines <$> traverse f stmts-    Declare name dir fields metadata ->-      let fieldPairs = map renderField fields-          renderedFactOpts = renderMetadata metadata-          renderedOpts = if T.null renderedFactOpts then "" else " " <> renderedFactOpts-       in pure $ T.intercalate "\n" $ catMaybes-        [ Just $ ".decl " <> name <> "(" <> T.intercalate ", " fieldPairs <> ")" <> renderedOpts-        , renderDir name dir-        ]-    Atom name terms -> do-      let rendered = name <> "(" <> renderTerms (toList terms) <> ")"-      end <- maybeDot-      pure $ rendered <> end-    Rule name terms body -> do-      body' <- f body-      let rendered =-            name <> "(" <> renderTerms (toList terms) <> ") :-\n" <>-            T.intercalate "\n" (map indent $ T.lines body')-      pure rendered-    And e1 e2 -> do-      txt <- nested $ do-        txt1 <- f e1-        txt2 <- f e2-        pure $ txt1 <> ",\n" <> txt2-      end <- maybeDot-      pure $ txt <> end-    Or e1 e2 -> do-      txt <- nested $ do-        txt1 <- f e1-        txt2 <- f e2-        pure $ txt1 <> ";\n" <> txt2-      end <- maybeDot-      case end of-        "." -> pure $ txt <> end-        _ -> pure $ "(" <> txt <> ")"-    Not e -> do-      let maybeAddParens txt = case e of-            And _ _ -> "(" <> txt <> ")"-            _ -> txt-      txt <- maybeAddParens <$> nested (f e)-      end <- maybeDot-      case end of-        "." -> pure $ "!" <> txt <> end-        _ -> pure $ "!" <> txt-    Constrain t -> do-      let t' = renderTerm t-      end <- maybeDot-      case end of-        "." -> pure $ t' <> "."-        _ -> pure t'-  indent = ("  " <>)-  nested = local (const Nested)-  maybeDot = ask >>= \case-    TopLevel -> pure "."-    Nested -> pure mempty--renderDir :: VarName -> Direction -> Maybe T.Text-renderDir name = \case-  Input -> Just $ ".input " <> name-  Output -> Just $ ".output " <> name-  InputOutput -> Just $ T.intercalate "\n"-                      $ catMaybes [renderDir name Input, renderDir name Output]-  Internal -> Nothing--renderField :: FieldData -> T.Text-renderField (FieldData ty accName) =-  let txt = case ty of-        DLNumber -> ": number"-        DLUnsigned -> ": unsigned"-        DLFloat -> ": float"-        DLString -> ": symbol"-   in accName <> txt--renderMetadata :: SimpleMetadata -> T.Text-renderMetadata (SimpleMetadata struct inline) =-  let structTxt = case struct of-        AutomaticLayout -> Nothing-        BTreeLayout -> Just "btree"-        BrieLayout -> Just "brie"-        EqRelLayout -> Just "eqrel"-      inlineTxt = case inline of-        DoInline -> Just "inline"-        DoNotInline -> Nothing-  in T.intercalate " " $ catMaybes [structTxt, inlineTxt]--renderTerms :: [SimpleTerm] -> T.Text-renderTerms = T.intercalate ", " . map renderTerm--renderTerm :: SimpleTerm -> T.Text-renderTerm = \case-  I x -> T.pack $ show x-  U x -> T.pack $ show x-  F x -> T.pack $ printf "%f" x-  S s -> "\"" <> T.pack s <> "\""-  V v -> v-  Underscore -> "_"--  BinOp' op t1 t2 -> renderTerm t1 <> " " <> renderBinOp op <> " " <> renderTerm t2-  UnaryOp' op t1 -> renderUnaryOp op <> renderTerm t1-  Func' name ts -> renderFunc name <> "(" <> renderTerms (toList ts) <> ")"-  where-    renderFunc = \case-      Max -> "max"-      Min -> "min"-      Cat -> "cat"-      Contains -> "contains"-      Match -> "match"-      Ord -> "ord"-      StrLen -> "strlen"-      Substr -> "substr"-      ToNumber -> "to_number"-      ToString -> "to_string"-    renderBinOp = \case-      Plus -> "+"-      Mul -> "*"-      Subtract -> "-"-      Div -> "/"-      Pow -> "^"-      Rem -> "%"-      BinaryAnd -> "band"-      BinaryOr -> "bor"-      BinaryXor -> "bxor"-      LogicalAnd -> "land"-      LogicalOr -> "lor"-      LessThan -> "<"-      LessThanOrEqual -> "<="-      GreaterThan -> ">"-      GreaterThanOrEqual -> ">="-      IsEqual -> "="-      IsNotEqual -> "!="-    renderUnaryOp Negate = "-"---type Name = T.Text---- | Type representing a variable name in Datalog.-type VarName = T.Text--type AccessorName = T.Text--data DLType-  = DLNumber-  | DLUnsigned-  | DLFloat-  | DLString--data FieldData = FieldData DLType AccessorName---- | A type level tag describing in which context a DSL fragment is used.---   This is only used on the type level and helps catch some semantic errors---   at compile time.-data UsageContext-  = Definition-  -- ^ A DSL fragment is used in a top level definition.-  | Relation-  -- ^ A DSL fragment is used inside a relation (either head or body of a relation).---- | A type family used for generating a user-friendly type error in case---   you use a variable in a DSL fragment where it is not allowed---   (outside of relations).-type family NoVarsInAtom (ctx :: UsageContext) :: Constraint where-  NoVarsInAtom ctx = Assert (ctx == 'Relation) NoVarsInAtomError--type NoVarsInAtomError =-  ( "You tried to use a variable in a top level fact, which is not supported in Souffle."-  % "Possible solutions:"-  % "  - Move the fact inside a rule body."-  % "  - Replace the variable in the fact with a string, number, unsigned or float constant."-  )---- | Data type for representing Datalog terms.------   All constructors are hidden, but with the `Num`, 'Fractional' and---   `IsString` instances it is possible to create terms using Haskell syntax---   for literals. For non-literal values, smart constructors are provided.---   (See for example 'underscore' / '__'.)-data Term ctx ty where-  -- NOTE: type family is used here instead of "Term 'Relation ty";-  -- this allows giving a better type error in some situations.-  VarTerm :: NoVarsInAtom ctx => VarName -> Term ctx ty-  UnderscoreTerm :: Term ctx ty-  NumberTerm :: Int32 -> Term ctx Int32-  UnsignedTerm :: Word32 -> Term ctx Word32-  FloatTerm :: Float -> Term ctx Float-  StringTerm :: ToString ty => ty -> Term ctx ty--  UnaryOp :: Num ty => Op1 -> Term ctx ty -> Term ctx ty-  BinOp :: Num ty => Op2 -> Term ctx ty -> Term ctx ty -> Term ctx ty-  Func :: FuncName -> NonEmpty SimpleTerm -> Term ctx ty2--data Op2-  = Plus-  | Mul-  | Subtract-  | Div-  | Pow-  | Rem-  | BinaryAnd-  | BinaryOr-  | BinaryXor-  | LogicalAnd-  | LogicalOr-  | LessThan-  | LessThanOrEqual-  | GreaterThan-  | GreaterThanOrEqual-  | IsEqual-  | IsNotEqual--data Op1 = Negate--data FuncName-  = Max-  | Min-  | Cat-  | Contains-  | Match-  | Ord-  | StrLen-  | Substr-  | ToNumber-  | ToString----- | Term representing a wildcard ("_") in Datalog.-underscore :: Term ctx ty-underscore = UnderscoreTerm---- | Term representing a wildcard ("_") in Datalog. Note that in the DSL this---   is with 2 underscores. (Single underscore is reserved for typed holes!)-__ :: Term ctx ty-__ = underscore--class ToString a where-  toString :: a -> String--instance ToString String where toString = id-instance ToString T.Text where toString = T.unpack-instance ToString TL.Text where toString = TL.unpack--instance IsString (Term ctx String) where fromString = StringTerm-instance IsString (Term ctx T.Text) where fromString = StringTerm . T.pack-instance IsString (Term ctx TL.Text) where fromString = StringTerm . TL.pack---- | A helper typeclass, mainly used for avoiding a lot of boilerplate---   in the 'Num' instance for 'Term'.-class Num ty => SupportsArithmetic ty where-  fromInteger' :: Integer -> Term ctx ty--instance SupportsArithmetic Int32 where-  fromInteger' = NumberTerm . fromInteger-instance SupportsArithmetic Word32 where-  fromInteger' = UnsignedTerm . fromInteger-instance SupportsArithmetic Float where-  fromInteger' = FloatTerm . fromInteger--instance (SupportsArithmetic ty, Num ty) => Num (Term ctx ty) where-  fromInteger = fromInteger'-  (+) = BinOp Plus-  (*) = BinOp Mul-  (-) = BinOp Subtract-  negate = UnaryOp Negate-  abs = error "'abs' is not supported for Souffle terms"-  signum = error "'signum' is not supported for Souffle terms"--instance Fractional (Term ctx Float) where-  fromRational = FloatTerm . fromRational-  (/) = BinOp Div---- | Exponentiation operator ("^" in Datalog).-(.^) :: Num ty => Term ctx ty -> Term ctx ty -> Term ctx ty-(.^) = BinOp Pow---- | Remainder operator ("%" in Datalog).-(.%) :: (Num ty, Integral ty) => Term ctx ty -> Term ctx ty -> Term ctx ty-(.%) = BinOp Rem---- | Creates a less than constraint (a < b), for use in the body of a relation.-(.<) :: Num ty => Term ctx ty -> Term ctx ty -> Body ctx ()-(.<) = addConstraint LessThan-infix 1 .<---- | Creates a less than or equal constraint (a <= b), for use in the body of---   a relation.-(.<=) :: Num ty => Term ctx ty -> Term ctx ty -> Body ctx ()-(.<=) = addConstraint LessThanOrEqual-infix 1 .<=---- | Creates a greater than constraint (a > b), for use in the body of a relation.-(.>) :: Num ty => Term ctx ty -> Term ctx ty -> Body ctx ()-(.>) = addConstraint GreaterThan-infix 1 .>---- | Creates a greater than or equal constraint (a >= b), for use in the body of---   a relation.-(.>=) :: Num ty => Term ctx ty -> Term ctx ty -> Body ctx ()-(.>=) = addConstraint GreaterThanOrEqual-infix 1 .>=---- | Creates a constraint that 2 terms should be equal to each other (a = b),---   for use in the body of a relation.-(.=) :: Term ctx ty -> Term ctx ty -> Body ctx ()-(.=) = addConstraint IsEqual-infix 1 .=---- | Creates a constraint that 2 terms should not be equal to each other---   (a != b), for use in the body of a relation.-(.!=) :: Term ctx ty -> Term ctx ty -> Body ctx ()-(.!=) = addConstraint IsNotEqual-infix 1 .!=--addConstraint :: Op2 -> Term ctx ty -> Term ctx ty -> Body ctx ()-addConstraint op e1 e2 =-  let expr = BinOp' op (toTerm e1) (toTerm e2)-   in tell [Constrain' expr]---- | Binary AND operator.-band :: (Num ty, Integral ty) => Term ctx ty -> Term ctx ty -> Term ctx ty-band = BinOp BinaryAnd---- | Binary OR operator.-bor :: (Num ty, Integral ty) => Term ctx ty -> Term ctx ty -> Term ctx ty-bor = BinOp BinaryOr---- | Binary XOR operator.-bxor :: (Num ty, Integral ty) => Term ctx ty -> Term ctx ty -> Term ctx ty-bxor = BinOp BinaryXor---- | Logical AND operator.-land :: (Num ty, Integral ty) => Term ctx ty -> Term ctx ty -> Term ctx ty-land = BinOp LogicalAnd---- | Logical OR operator.-lor :: (Num ty, Integral ty) => Term ctx ty -> Term ctx ty -> Term ctx ty-lor = BinOp LogicalOr---- | "max" function.-max' :: Num ty => Term ctx ty -> Term ctx ty -> Term ctx ty-max' = func2 Max---- | "min" function.-min' :: Num ty => Term ctx ty -> Term ctx ty -> Term ctx ty-min' = func2 Min---- | "cat" function (string concatenation).-cat :: ToString ty => Term ctx ty -> Term ctx ty -> Term ctx ty-cat = func2 Cat---- | "contains" predicate, checks if 2nd string contains the first.-contains :: ToString ty => Term ctx ty -> Term ctx ty -> Body ctx ()-contains a b =-  let expr = toTerm $ func2 Contains a b-   in tell [Constrain' expr]---- | "match" predicate, checks if a wildcard string matches a given string.-match :: ToString ty => Term ctx ty -> Term ctx ty -> Body ctx ()-match p s =-  let expr = toTerm $ func2 Match p s-  in tell [Constrain' expr]---- | "ord" function.-ord :: ToString ty => Term ctx ty -> Term ctx Int32-ord = func1 Ord---- | "strlen" function.-strlen :: ToString ty => Term ctx ty -> Term ctx Int32-strlen = func1 StrLen---- | "substr" function.-substr :: ToString ty => Term ctx ty -> Term ctx Int32 -> Term ctx Int32 -> Term ctx ty-substr a b c = Func Substr $ toTerm a :| [toTerm b, toTerm c]---- | "to_number" function.-to_number :: ToString ty => Term ctx ty -> Term ctx Int32-to_number = func1 ToNumber---- | "to_string" function.-to_string :: ToString ty => Term ctx Int32 -> Term ctx ty-to_string = func1 ToString--func1 :: FuncName -> Term ctx ty -> Term ctx ty2-func1 name a = Func name $ toTerm a :| []--func2 :: FuncName -> Term ctx ty -> Term ctx ty -> Term ctx ty2-func2 name a b = Func name $ toTerm a :| [toTerm b]--data SimpleTerm-  = V VarName-  | I Int32-  | U Word32-  | F Float-  | S String-  | Underscore--  | BinOp' Op2 SimpleTerm SimpleTerm-  | UnaryOp' Op1 SimpleTerm-  | Func' FuncName (NonEmpty SimpleTerm)--data SimpleMetadata = SimpleMetadata StructureOption InlineOption--data StructureOption-  = AutomaticLayout-  | BTreeLayout-  | BrieLayout-  | EqRelLayout--data InlineOption-  = DoInline-  | DoNotInline--data AST-  = Declare' VarName Direction [FieldData] SimpleMetadata-  | Rule' Name (NonEmpty SimpleTerm) AST-  | Atom' Name (NonEmpty SimpleTerm)-  | And' [AST]-  | Or' [AST]-  | Not' AST-  | Constrain' SimpleTerm--data DL-  = Statements [DL]-  | Declare VarName Direction [FieldData] SimpleMetadata-  | Rule Name (NonEmpty SimpleTerm) DL-  | Atom Name (NonEmpty SimpleTerm)-  | And DL DL-  | Or DL DL-  | Not DL-  | Constrain SimpleTerm---class KnownDLTypes (ts :: [Type]) where-  getTypes :: Proxy ts -> [DLType]--instance KnownDLTypes '[] where-  getTypes _ = []--instance (KnownDLType t, KnownDLTypes ts) => KnownDLTypes (t ': ts) where-  getTypes _ = getType (Proxy :: Proxy t) : getTypes (Proxy :: Proxy ts)--class KnownDLType t where-  getType :: Proxy t -> DLType--instance KnownDLType Int32 where getType = const DLNumber-instance KnownDLType Word32 where getType = const DLUnsigned-instance KnownDLType Float where getType = const DLFloat-instance KnownDLType String where getType = const DLString-instance KnownDLType T.Text where getType = const DLString-instance KnownDLType TL.Text where getType = const DLString--type family AccessorNames a :: [Symbol] where-  AccessorNames a = GetAccessorNames (Rep a)--type family GetAccessorNames (f :: Type -> Type) :: [Symbol] where-  GetAccessorNames (a :*: b) = GetAccessorNames a ++ GetAccessorNames b-  GetAccessorNames (C1 ('MetaCons _ _ 'False) _) = '[]-  GetAccessorNames (S1 ('MetaSel ('Just name) _ _ _) a) = '[name] ++ GetAccessorNames a-  GetAccessorNames (M1 _ _ a) = GetAccessorNames a-  GetAccessorNames (K1 _ _) = '[]--class KnownSymbols (symbols :: [Symbol]) where-  toStrings :: Proxy symbols -> [String]--instance KnownSymbols '[] where-  toStrings = const []--instance (KnownSymbol s, KnownSymbols symbols) => KnownSymbols (s ': symbols) where-  toStrings _ =-    let sym = symbolVal (Proxy :: Proxy s)-        symbols =  toStrings (Proxy :: Proxy symbols)-     in sym : symbols--accessorNames :: forall a. KnownSymbols (AccessorNames a) => Proxy a -> Maybe [T.Text]-accessorNames _ = case toStrings (Proxy :: Proxy (AccessorNames a)) of-  [] -> Nothing-  names -> Just $ T.pack <$> names---- | A type synonym for a tuple consisting of Datalog 'Term's.---   Only tuples containing up to 10 elements are currently supported.-type Tuple ctx ts = TupleOf (MapType (Term ctx) ts)--class ToTerms (ts :: [Type]) where-  toTerms :: Proxy ctx -> TypeInfo a ts -> Tuple ctx ts -> NonEmpty SimpleTerm--instance ToTerms '[t] where-  toTerms _ _ a =-    toTerm a :| []--instance ToTerms '[t1, t2] where-  toTerms _ _ (a, b) =-    toTerm a :| [toTerm b]--instance ToTerms '[t1, t2, t3] where-  toTerms _ _ (a, b, c) =-    toTerm a :| [toTerm b, toTerm c]--instance ToTerms '[t1, t2, t3, t4] where-  toTerms _ _ (a, b, c, d) =-    toTerm a :| [toTerm b, toTerm c, toTerm d]--instance ToTerms '[t1, t2, t3, t4, t5] where-  toTerms _ _ (a, b, c, d, e) =-    toTerm a :| [toTerm b, toTerm c, toTerm d, toTerm e]--instance ToTerms '[t1, t2, t3, t4, t5, t6] where-  toTerms _ _ (a, b, c, d, e, f) =-    toTerm a :| [toTerm b, toTerm c, toTerm d, toTerm e, toTerm f]--instance ToTerms '[t1, t2, t3, t4, t5, t6, t7] where-  toTerms _ _ (a, b, c, d, e, f, g) =-    toTerm a :| [toTerm b, toTerm c, toTerm d, toTerm e, toTerm f, toTerm g]--instance ToTerms '[t1, t2, t3, t4, t5, t6, t7, t8] where-  toTerms _ _ (a, b, c, d, e, f, g, h) =-    toTerm a :| [toTerm b, toTerm c, toTerm d, toTerm e, toTerm f, toTerm g, toTerm h]--instance ToTerms '[t1, t2, t3, t4, t5, t6, t7, t8, t9] where-  toTerms _ _ (a, b, c, d, e, f, g, h, i) =-    toTerm a :| [toTerm b, toTerm c, toTerm d, toTerm e, toTerm f, toTerm g, toTerm h, toTerm i]--instance ToTerms '[t1, t2, t3, t4, t5, t6, t7, t8, t9, t10] where-  toTerms _ _ (a, b, c, d, e, f, g, h, i, j) =-    toTerm a :| [ toTerm b, toTerm c, toTerm d, toTerm e, toTerm f-                , toTerm g, toTerm h, toTerm i, toTerm j-                ]--toTerm :: Term ctx t -> SimpleTerm-toTerm = \case-  VarTerm v -> V v-  StringTerm s -> S $ toString s-  NumberTerm x -> I x-  UnsignedTerm x -> U x-  FloatTerm x -> F x-  UnderscoreTerm -> Underscore--  BinOp op t1 t2 -> BinOp' op (toTerm t1) (toTerm t2)-  UnaryOp op t1 -> UnaryOp' op (toTerm t1)-  Func name ts -> Func' name ts----- Helper functions / type families / ...--type family MapType (f :: Type -> Type) (ts :: [Type]) :: [Type] where-  MapType _ '[] = '[]-  MapType f (t ': ts) = f t ': MapType f ts--type family Assert (c :: Bool) (msg :: ErrorMessage) :: Constraint where-  Assert 'True _ = ()-  Assert 'False msg = TypeError msg--type family (a :: k) == (b :: k) :: Bool where-  a == a = 'True-  _ == _ = 'False--type family Length (xs :: [Type]) :: Nat where-  Length '[] = 0-  Length (_ ': xs) = 1 + Length xs---- | A helper type family for computing the list of types used in a data type.---   (The type family assumes a data type with a single data constructor.)-type family Structure a :: [Type] where-  Structure a = Collect (Rep a)--type family Collect (a :: Type -> Type) where-  Collect (a :*: b) = Collect a ++ Collect b-  Collect (M1 _ _ a) = Collect a-  Collect (K1 _ ty) = '[ty]--type family a ++ b = c where-  '[] ++ b = b-  a ++ '[] = a-  (a ': b) ++ c = a ': (b ++ c)--type family TupleOf (ts :: [Type]) = t where-  TupleOf '[t] = t-  TupleOf '[t1, t2] = (t1, t2)-  TupleOf '[t1, t2, t3] = (t1, t2, t3)-  TupleOf '[t1, t2, t3, t4] = (t1, t2, t3, t4)-  TupleOf '[t1, t2, t3, t4, t5] = (t1, t2, t3, t4, t5)-  TupleOf '[t1, t2, t3, t4, t5, t6] = (t1, t2, t3, t4, t5, t6)-  TupleOf '[t1, t2, t3, t4, t5, t6, t7] = (t1, t2, t3, t4, t5, t6, t7)-  TupleOf '[t1, t2, t3, t4, t5, t6, t7, t8] = (t1, t2, t3, t4, t5, t6, t7, t8)-  TupleOf '[t1, t2, t3, t4, t5, t6, t7, t8, t9] = (t1, t2, t3, t4, t5, t6, t7, t8, t9)-  TupleOf '[t1, t2, t3, t4, t5, t6, t7, t8, t9, t10] = (t1, t2, t3, t4, t5, t6, t7, t8, t9, t10)-  TupleOf _ = TypeError BigTupleError--type BigTupleError =-  ( "The DSL only supports facts/tuples consisting of up to 10 elements."-  % "If you need more arguments, please submit an issue on Github "-  <> "(https://github.com/luc-tielen/souffle-haskell/issues)"-  )-
lib/Language/Souffle/Internal.hs view
@@ -10,8 +10,7 @@ module Language.Souffle.Internal   ( Souffle   , Relation-  , RelationIterator-  , Tuple+  , ByteBuf   , init   , setNumThreads   , getNumThreads@@ -19,35 +18,22 @@   , loadAll   , printAll   , getRelation-  , countFacts-  , getRelationIterator-  , relationIteratorNext-  , allocTuple-  , addTuple-  , containsTuple-  , tuplePushInt32-  , tuplePushUInt32-  , tuplePushFloat-  , tuplePushString-  , tuplePopInt32-  , tuplePopUInt32-  , tuplePopFloat-  , tuplePopString+  , pushFacts+  , popFacts+  , containsFact   ) where  import Prelude hiding ( init ) import Data.Functor ( (<&>) ) import Data.Word-import Data.Int-import Foreign.Marshal.Alloc-import Foreign.Storable import Foreign.C.String import Foreign.C.Types import Foreign.ForeignPtr import Foreign.Ptr import qualified Language.Souffle.Internal.Bindings as Bindings import Language.Souffle.Internal.Bindings-  ( Souffle, Relation, RelationIterator, Tuple )+  ( Souffle, Relation, ByteBuf )+import Control.Exception (mask_)   {- | Initializes a Souffle program.@@ -60,7 +46,7 @@      in this module. -} init :: String -> IO (Maybe (ForeignPtr Souffle))-init prog = do+init prog = mask_ $ do   ptr <- withCString prog Bindings.init   if ptr == nullPtr     then pure Nothing@@ -106,103 +92,37 @@   withCString relation $ Bindings.getRelation ptr {-# INLINABLE getRelation #-} --- | Returns the amount of facts found in a relation.-countFacts :: Ptr Relation -> IO Int-countFacts relation =-  Bindings.getTupleCount relation >>= \(CSize count) ->-    -- TODO: check what happens for really large sizes?-    pure (fromIntegral count)---- | Create an iterator for iterating over the facts of a relation.-getRelationIterator :: Ptr Relation -> IO (ForeignPtr RelationIterator)-getRelationIterator relation =-  Bindings.getRelationIterator relation >>= newForeignPtr Bindings.freeRelationIterator-{-# INLINABLE getRelationIterator #-}--{-| Advances the relation iterator by 1 position.+{-| Serializes many facts from Datalog to Haskell. -    Calling this function when there are no more results to be returned-    will result in a crash.+    You need to check if the passed pointers are non-NULL before passing it+    to this function. Not doing so results in undefined behavior.+    Passing in a different count of objects to what is actually inside the+    byte buffer will crash. -}-relationIteratorNext :: ForeignPtr RelationIterator -> IO (Ptr Tuple)-relationIteratorNext iter = withForeignPtr iter Bindings.relationIteratorNext-{-# INLINABLE relationIteratorNext #-}+pushFacts :: Ptr Relation -> Ptr ByteBuf -> Word64 -> IO ()+pushFacts relation buf x =+  Bindings.pushByteBuf relation buf (CSize x)+{-# INLINABLE pushFacts #-} --- | Allocates memory for a tuple (fact) to be added to a relation.-allocTuple :: Ptr Relation -> IO (ForeignPtr Tuple)-allocTuple relation =-  Bindings.allocTuple relation >>= newForeignPtr Bindings.freeTuple-{-# INLINABLE allocTuple #-}+{-| Serializes many facts from Haskell to Datalog. --- | Adds a tuple (fact) to a relation.-addTuple :: Ptr Relation -> ForeignPtr Tuple -> IO ()-addTuple relation tuple =-  withForeignPtr tuple $ Bindings.addTuple relation-{-# INLINABLE addTuple #-}+    You need to check if the passed pointer is non-NULL before passing it+    to this function. Not doing so results in undefined behavior. +    Returns a pointer to a byte buffer that contains the serialized Datalog facts.+-}+popFacts :: Ptr Souffle -> Ptr Relation -> IO (Ptr ByteBuf)+popFacts = Bindings.popByteBuf+{-# INLINABLE popFacts #-}+ {- | Checks if a relation contains a certain tuple.       Returns True if the tuple was found in the relation; otherwise False. -}-containsTuple :: Ptr Relation -> ForeignPtr Tuple -> IO Bool-containsTuple relation tuple = withForeignPtr tuple $ \ptr ->-  Bindings.containsTuple relation ptr <&> \case+containsFact :: Ptr Relation -> Ptr ByteBuf -> IO Bool+containsFact relation buf =+  Bindings.containsTuple relation buf <&> \case     CBool 0 -> False     CBool _ -> True-{-# INLINABLE containsTuple #-}---- | Pushes an integer value into a tuple.-tuplePushInt32 :: Ptr Tuple -> Int32 -> IO ()-tuplePushInt32 tuple i = Bindings.tuplePushInt32 tuple (CInt i)-{-# INLINABLE tuplePushInt32 #-}---- | Pushes an unsigned integer value into a tuple.-tuplePushUInt32 :: Ptr Tuple -> Word32 -> IO ()-tuplePushUInt32 tuple i = Bindings.tuplePushUInt32 tuple (CUInt i)-{-# INLINABLE tuplePushUInt32 #-}---- | Pushes a float value into a tuple.-tuplePushFloat :: Ptr Tuple -> Float -> IO ()-tuplePushFloat tuple f = Bindings.tuplePushFloat tuple (CFloat f)-{-# INLINABLE tuplePushFloat #-}---- | Pushes a string value into a tuple.-tuplePushString :: Ptr Tuple -> String -> IO ()-tuplePushString tuple str =-  withCString str $ Bindings.tuplePushString tuple-{-# INLINABLE tuplePushString #-}---- | Extracts a 32 bit signed integer value from a tuple.-tuplePopInt32 :: Ptr Tuple -> IO Int32-tuplePopInt32 tuple = alloca $ \ptr -> do-  Bindings.tuplePopInt32 tuple ptr-  (CInt res) <- peek ptr-  pure res-{-# INLINABLE tuplePopInt32 #-}---- | Extracts a 32 bit unsigned integer value from a tuple.-tuplePopUInt32 :: Ptr Tuple -> IO Word32-tuplePopUInt32 tuple = alloca $ \ptr -> do-  Bindings.tuplePopUInt32 tuple ptr-  (CUInt res) <- peek ptr-  pure res-{-# INLINABLE tuplePopUInt32 #-}---- | Extracts a float value from a tuple.-tuplePopFloat :: Ptr Tuple -> IO Float-tuplePopFloat tuple = alloca $ \ptr -> do-  Bindings.tuplePopFloat tuple ptr-  (CFloat res) <- peek ptr-  pure res-{-# INLINABLE tuplePopFloat #-}---- | Extracts a string value from a tuple.-tuplePopString :: Ptr Tuple -> IO String-tuplePopString tuple = alloca $ \ptr -> do-  Bindings.tuplePopString tuple ptr-  cstr <- peek ptr-  str <- peekCString cstr-  free cstr-  pure str-{-# INLINABLE tuplePopString #-}+{-# INLINABLE containsFact #-}
lib/Language/Souffle/Internal/Bindings.hs view
@@ -5,8 +5,7 @@ module Language.Souffle.Internal.Bindings   ( Souffle   , Relation-  , RelationIterator-  , Tuple+  , ByteBuf   , init   , free   , setNumThreads@@ -15,25 +14,13 @@   , loadAll   , printAll   , getRelation-  , getTupleCount-  , getRelationIterator-  , freeRelationIterator-  , relationIteratorNext-  , allocTuple-  , freeTuple-  , addTuple+  , pushByteBuf+  , popByteBuf   , containsTuple-  , tuplePushInt32-  , tuplePushUInt32-  , tuplePushString-  , tuplePushFloat-  , tuplePopInt32-  , tuplePopUInt32-  , tuplePopString-  , tuplePopFloat   ) where  import Prelude hiding ( init )+import Data.Kind (Type) import Foreign.C.String import Foreign.C.Types import Foreign.Ptr@@ -41,18 +28,16 @@  -- | A void type, used for tagging a pointer that points to an embedded --   Souffle program.+type Souffle :: Type data Souffle  -- | A void type, used for tagging a pointer that points to a relation.+type Relation :: Type data Relation --- | A void type, used for tagging a pointer that points to an---   iterator used for iterating over a relation.-data RelationIterator---- | A void type, used for tagging a pointer that points to a tuple---   (term used in the Souffle compiler for a fact).-data Tuple+-- | A void type, used for tagging a pointer that points to a raw bytearray.+type ByteBuf :: Type+data ByteBuf   {- | Initializes a Souffle program.@@ -128,187 +113,33 @@ foreign import ccall unsafe "souffle_relation" getRelation   :: Ptr Souffle -> CString -> IO (Ptr Relation) -{-| Gets the amount of tuples found in a relation.--    You need to check if both passed pointers are not equal to 'nullPtr' before-    passing it to this function. Not doing so results in undefined behavior (in C++).--    Returns the amount of tuples found in a relation.--}-foreign import ccall unsafe "souffle_relation_tuple_count" getTupleCount-  :: Ptr Relation -> IO CSize--{-| Create an iterator for iterating over the facts of a relation.--    You need to check if the passed pointer is not equal to 'nullPtr' before-    passing it to this function. Not doing so results in undefined behavior (in C++).--    The returned pointer needs to be freed with 'freeRelationIterator'-    after it is no longer needed.--}-foreign import ccall unsafe "souffle_relation_iterator" getRelationIterator-  :: Ptr Relation -> IO (Ptr RelationIterator)--{-| Frees a pointer previously allocated with 'getRelationIterator'.--    You need to check if the passed pointer is not equal to 'nullPtr' before-    passing it to this function. Not doing so results in undefined behavior (in C++).--}-foreign import ccall unsafe "&souffle_relation_iterator_free" freeRelationIterator-  :: FunPtr (Ptr RelationIterator -> IO ())--{-| Advances the relation iterator by 1 position.--    You need to check if the passed pointer is not equal to 'nullPtr' before-    passing it to this function. Not doing so results in undefined behavior (in C++).--    Calling this function when there are no more tuples to be returned-    will result in a crash.--    Returns a pointer to the next tuple. This pointer is not allowed to be freed-    as it is managed by the Souffle program already.--}-foreign import ccall unsafe "souffle_relation_iterator_next" relationIteratorNext-  :: Ptr RelationIterator -> IO (Ptr Tuple)--{-| Allocates memory for a tuple (fact) to be added to a relation.--    You need to check if the passed pointer is not equal to 'nullPtr' before-    passing it to this function. Not doing so results in undefined behavior (in C++).--    Returns a pointer to a new tuple. Use 'freeTuple' when the tuple-    is no longer required.--}-foreign import ccall unsafe "souffle_tuple_alloc" allocTuple-  :: Ptr Relation -> IO (Ptr Tuple)--{-| Frees memory of a tuple that was previously allocated (in Haskell).--    You need to check if the passed pointer is not equal to 'nullPtr' before-    passing it to this function. Not doing so results in undefined behavior (in C++).--}-foreign import ccall unsafe "&souffle_tuple_free" freeTuple-  :: FunPtr (Ptr Tuple -> IO ())--{-| Adds a tuple to a relation.--    You need to check if both passed pointers are not equal to 'nullPtr' before-    passing it to this function. Not doing so results in undefined behavior (in C++).--}-foreign import ccall unsafe "souffle_tuple_add" addTuple-  :: Ptr Relation -> Ptr Tuple -> IO ()--{- | Checks if a relation contains a certain tuple.--     You need to check if the passed pointers are non-NULL before passing it-     to this function. Not doing so results in undefined behavior.--     Returns True if the tuple was found in the relation; otherwise False.--}-foreign import ccall unsafe "souffle_contains_tuple" containsTuple-  :: Ptr Relation -> Ptr Tuple -> IO CBool--{-| Pushes a 32 bit signed integer value into a tuple.--    You need to check if the passed pointer is not equal to 'nullPtr' before-    passing it to this function. Not doing so results in undefined behavior (in C++).--    Pushing an integer value onto a tuple that expects another type results-    in a crash. Pushing a value into a tuple when it already is "full"-    also results in a crash.--}-foreign import ccall unsafe "souffle_tuple_push_int32" tuplePushInt32-  :: Ptr Tuple -> CInt -> IO ()--{-| Pushes a 32 bit unsigned integer value into a tuple.--    You need to check if the passed pointer is not equal to 'nullPtr' before-    passing it to this function. Not doing so results in undefined behavior (in C++).--    Pushing an integer value onto a tuple that expects another type results-    in a crash. Pushing a value into a tuple when it already is "full"-    also results in a crash.--}-foreign import ccall unsafe "souffle_tuple_push_uint32" tuplePushUInt32-  :: Ptr Tuple -> CUInt -> IO ()--{-| Pushes a float value into a tuple.--    You need to check if the passed pointer is not equal to 'nullPtr' before-    passing it to this function. Not doing so results in undefined behavior (in C++).--    Pushing a float value onto a tuple that expects another type results-    in a crash. Pushing a value into a tuple when it already is "full"-    also results in a crash.--}-foreign import ccall unsafe "souffle_tuple_push_float" tuplePushFloat-  :: Ptr Tuple -> CFloat -> IO ()--{-| Pushes a string value into a tuple.--    You need to check if the passed pointer is not equal to 'nullPtr' before-    passing it to this function. Not doing so results in undefined behavior (in C++).--    Pushing a string value onto a tuple that expects another type results-    in a crash. Pushing a value into a tuple when it already is "full"-    also results in a crash.--}-foreign import ccall unsafe "souffle_tuple_push_string" tuplePushString-  :: Ptr Tuple -> CString -> IO ()--{-| Extracts a 32 bit signed integer value from a tuple.--    You need to check if the passed pointer is not equal to 'nullPtr' before passing it-    to this function. Not doing so results in undefined behavior.--    Extracting an integer value from a tuple that expects another type results-    in a crash. Extracting a value from a tuple when it is already "empty"-    also results in a crash.--    The popped integer will be stored in the pointer that is passed in.--}-foreign import ccall unsafe "souffle_tuple_pop_int32" tuplePopInt32-  :: Ptr Tuple -> Ptr CInt -> IO ()--{-| Extracts a 32 bit unsigned integer value from a tuple.+{-| Checks if a relation contains a certain tuple. -    You need to check if the passed pointer is not equal to 'nullPtr' before passing it+    You need to check if the passed pointers are non-NULL before passing it     to this function. Not doing so results in undefined behavior. -    Extracting an integer value from a tuple that expects another type results-    in a crash. Extracting a value from a tuple when it is already "empty"-    also results in a crash.--    The popped integer will be stored in the pointer that is passed in.+    Returns True if the tuple was found in the relation; otherwise False. -}-foreign import ccall unsafe "souffle_tuple_pop_uint32" tuplePopUInt32-  :: Ptr Tuple -> Ptr CUInt -> IO ()+foreign import ccall unsafe "souffle_contains_tuple" containsTuple+  :: Ptr Relation -> Ptr ByteBuf -> IO CBool -{-| Extracts a float value from a tuple.+{-| Serializes many Datalog facts from Haskell to C++. -    You need to check if the passed pointer is not equal to 'nullPtr' before passing it+    You need to check if the passed pointers are non-NULL before passing it     to this function. Not doing so results in undefined behavior.--    Extracting a float value from a tuple that expects another type results-    in a crash. Extracting a value from a tuple when it is already "empty"-    also results in a crash.--    The popped float will be stored in the pointer that is passed in.+    Passing in a different count of objects to what is actually inside the+    byte buffer will crash. -}-foreign import ccall unsafe "souffle_tuple_pop_float" tuplePopFloat-  :: Ptr Tuple -> Ptr CFloat -> IO ()+foreign import ccall unsafe "souffle_tuple_push_many" pushByteBuf+  :: Ptr Relation -> Ptr ByteBuf -> CSize -> IO () -{-| Extracts a string value from a tuple.+{-| Serializes many Datalog facts from Datalog to Haskell -    You need to check if the passed pointer is not equal to 'nullPtr' before passing it+    You need to check if the passed pointers are non-NULL before passing it     to this function. Not doing so results in undefined behavior. -    Extracting a string value from a tuple that expects another type results-    in a crash. Extracting a value from a tuple when it is already "empty"-    also results in a crash.--    The popped string will be stored in the result pointer.+    Returns a pointer to a byte buffer that contains the serialized Datalog facts. -}-foreign import ccall unsafe "souffle_tuple_pop_string" tuplePopString-  :: Ptr Tuple -> Ptr CString -> IO ()+foreign import ccall unsafe "souffle_tuple_pop_many" popByteBuf+  :: Ptr Souffle -> Ptr Relation -> IO (Ptr ByteBuf)
− lib/Language/Souffle/Internal/Constraints.hs 
@@ -1,67 +0,0 @@-{-# LANGUAGE TypeFamilies, DataKinds, TypeOperators, UndecidableInstances #-}---- | A helper module for generating more user friendly type errors in the form---   of custom constraints.---   This is an internal module, not meant to be used directly.-module Language.Souffle.Internal.Constraints-  ( SimpleProduct-  ) where--import Type.Errors.Pretty-import GHC.Generics-import Data.Kind-import Data.Int-import Data.Word-import qualified Data.Text as T-import qualified Data.Text.Lazy as TL----- | A helper type family used for generating a more user-friendly type error---   for incompatible types when generically deriving marshalling code for---   the 'Language.Souffle.Marshal.Marshal' typeclass.------   The __a__ type parameter is the original type, used when displaying the type error.------   A type error is returned if the passed in type is not a simple product type---   consisting of only "simple" types like Int32, Word32, Float, String and Text.-type family SimpleProduct (a :: Type) :: Constraint where-  SimpleProduct a = (ProductLike a (Rep a), OnlySimpleFields a (Rep a))--type family ProductLike (t :: Type) (f :: Type -> Type) :: Constraint where-  ProductLike t (_ :*: b) = ProductLike t b-  ProductLike t (M1 _ _ a) = ProductLike t a-  ProductLike _ (K1 _ _) = ()-  ProductLike t (_ :+: _) =-    TypeError ( "Error while deriving marshalling code for type " <> t <> ":"-              % "Cannot derive sum type, only product types are supported.")-  ProductLike t U1 =-    TypeError ( "Error while deriving marshalling code for type " <> t <> ":"-              % "Cannot automatically derive code for 0 argument constructor.")-  ProductLike t V1 =-    TypeError ( "Error while deriving marshalling code for type " <> t <> ":"-              % "Cannot derive void type.")--type family OnlySimpleFields (t :: Type) (f :: Type -> Type) :: Constraint where-  OnlySimpleFields t (a :*: b) = (OnlySimpleFields t a, OnlySimpleFields t b)-  OnlySimpleFields t (a :+: b) = (OnlySimpleFields t a, OnlySimpleFields t b)-  OnlySimpleFields t (M1 _ _ a) = OnlySimpleFields t a-  OnlySimpleFields _ U1 = ()-  OnlySimpleFields _ V1 = ()-  OnlySimpleFields t k = OnlySimpleField t k--type family OnlySimpleField (a :: Type) (f :: Type -> Type) :: Constraint where-  OnlySimpleField t (M1 _ _ a) = OnlySimpleField t a-  OnlySimpleField t (K1 _ a) = DirectlyMarshallable t a--type family DirectlyMarshallable (a :: Type) (b :: Type) :: Constraint where-  DirectlyMarshallable _ T.Text = ()-  DirectlyMarshallable _ TL.Text = ()-  DirectlyMarshallable _ Int32 = ()-  DirectlyMarshallable _ Word32 = ()-  DirectlyMarshallable _ Float = ()-  DirectlyMarshallable _ String = ()-  DirectlyMarshallable t a =-    TypeError ( "Error while generating marshalling code for " <> t <> ":"-              % "Can only marshal values of Int32, Word32, Float, String and Text directly"-             <> ", but found " <> a <> " type instead.")-
lib/Language/Souffle/Interpreted.hs view
@@ -1,5 +1,6 @@ {-# OPTIONS_GHC -Wno-redundant-constraints #-}-{-# LANGUAGE FlexibleInstances, TypeFamilies, DerivingVia, InstanceSigs, UndecidableInstances #-}+{-# LANGUAGE FlexibleInstances, TypeFamilies, DerivingVia, InstanceSigs #-}+{-# LANGUAGE UndecidableInstances, RoleAnnotations #-}  -- | This module provides an implementation for the `MonadSouffle` typeclass --   defined in "Language.Souffle.Class".@@ -11,7 +12,9 @@ --   the compiled alternative once the prototyping phase is finished. module Language.Souffle.Interpreted   ( Program(..)+  , ProgramOptions(..)   , Fact(..)+  , FactOptions(..)   , Marshal(..)   , Direction(..)   , ContainsInputFact@@ -28,13 +31,14 @@   ) where  import Prelude hiding (init)+import Data.Kind (Type, Constraint)  import Control.DeepSeq (deepseq) import Control.Exception (ErrorCall(..), throwIO, bracket) import Control.Monad.State.Strict import Data.IORef import Data.Foldable (traverse_)-import Data.List hiding (init)+import qualified Data.List as List hiding (init) import Data.Semigroup (Last(..)) import Data.Maybe (fromMaybe) import Data.Proxy@@ -55,6 +59,7 @@   -- | A monad for executing Souffle-related actions in.+type SouffleM :: Type -> Type newtype SouffleM a = SouffleM (IO a)   deriving (Functor, Applicative, Monad, MonadIO) via IO   deriving (Semigroup, Monoid) via (IO a)@@ -71,13 +76,14 @@ --   souffle session. --   - __cfgOutputDir__: The directory where the output fact file(s) are created. --   If Nothing, it will be part of the temporary directory.+type Config :: Type data Config   = Config   { cfgDatalogDir   :: FilePath   , cfgSouffleBin   :: Maybe FilePath   , cfgFactDir      :: Maybe FilePath   , cfgOutputDir    :: Maybe FilePath-  } deriving Show+  } deriving stock Show  -- | Retrieves the default config for the interpreter. These settings can --   be overridden using record update syntax if needed.@@ -164,15 +170,18 @@ -- | A datatype representing a handle to a datalog program. --   The type parameter is used for keeping track of which program --   type the handle belongs to for additional type safety.+type Handle :: Type -> Type data Handle prog = Handle   { handleData   :: IORef HandleData   , stdoutResult :: IORef (Maybe T.Text)   , stderrResult :: IORef (Maybe T.Text)   }+type role Handle nominal  -- | The data needed for the interpreter is the path where the souffle --   executable can be found, and a template directory where the program --   is stored.+type HandleData :: Type data HandleData = HandleData   { soufflePath :: FilePath   , tmpDirPath  :: FilePath@@ -182,6 +191,7 @@   , noOfThreads :: Word64   } +type IMarshal :: Type -> Type newtype IMarshal a = IMarshal (State [String] a)   deriving (Functor, Applicative, Monad, MonadState [String])   via (State [String])@@ -199,6 +209,9 @@   pushString str = modify (str:)   {-# INLINABLE pushString #-} +  pushText txt = pushString (T.unpack txt)+  {-# INLINABLE pushText #-}+ instance MonadPop IMarshal where   popInt32 = state $ \case     [] -> error "Empty fact stack"@@ -220,6 +233,13 @@     (h:t) -> (h, t)   {-# INLINABLE popString #-} +  popText = do+    str <- state $ \case+      [] -> error "Empty fact stack"+      (h:t) -> (h, t)+    pure $ T.pack str+  {-# INLINABLE popText #-}+ popMarshalT :: IMarshal a -> [String] -> a popMarshalT (IMarshal m) = evalState m {-# INLINABLE popMarshalT #-}@@ -228,6 +248,7 @@ pushMarshalT (IMarshal m) = reverse $ execState m [] {-# INLINABLE pushMarshalT #-} +type Collect :: (Type -> Type) -> Constraint class Collect c where   collect :: Marshal a => FilePath -> IO (c a) @@ -252,6 +273,7 @@ instance MonadSouffle SouffleM where   type Handler SouffleM = Handle   type CollectFacts SouffleM c = Collect c+  type SubmitFacts SouffleM _ = ()    run (Handle refHandleData refHandleStdOut refHandleStdErr) = liftIO $ do     handle <- readIORef refHandleData@@ -310,7 +332,7 @@            => Handle prog -> a -> SouffleM (Maybe a)   findFact prog fact = do     facts :: [a] <- getFacts prog-    pure $ find (== fact) facts+    pure $ List.find (== fact) facts   {-# INLINABLE findFact #-}    addFact :: forall a prog. (Fact a, ContainsInputFact prog a, Marshal a)@@ -320,7 +342,7 @@     let relationName = factName (Proxy :: Proxy a)     let factFile = factPath handle </> relationName <.> "facts"     let line = pushMarshalT (push fact)-    appendFile factFile $ intercalate "\t" line ++ "\n"+    appendFile factFile $ List.intercalate "\t" line ++ "\n"   {-# INLINABLE addFact #-}    addFacts :: forall a prog f. (Fact a, ContainsInputFact prog a, Marshal a, Foldable f)@@ -330,7 +352,7 @@     let relationName = factName (Proxy :: Proxy a)     let factFile = factPath handle </> relationName <.> "facts"     let factLines = map (pushMarshalT . push) (foldMap pure facts)-    traverse_ (\line -> appendFile factFile (intercalate "\t" line ++ "\n")) factLines+    traverse_ (\line -> appendFile factFile (List.intercalate "\t" line ++ "\n")) factLines   {-# INLINABLE addFacts #-}  datalogProgramFile :: forall prog. Program prog => prog -> FilePath -> IO (Maybe FilePath)@@ -347,8 +369,9 @@   (_, Just hout, _, locateCmdHandle) <- createProcess locateCmd   waitForProcess locateCmdHandle >>= \case     ExitFailure _ -> pure Nothing-    ExitSuccess ->-      words <$> hGetContents hout >>= \case+    ExitSuccess -> do+      contents <- hGetContents hout+      case words contents of         [souffleBin] -> pure $ Just souffleBin         _ -> pure Nothing {-# INLINABLE locateSouffle #-}
lib/Language/Souffle/Marshal.hs view
@@ -1,6 +1,7 @@ {-# OPTIONS_GHC -Wno-redundant-constraints #-} {-# LANGUAGE FlexibleInstances, FlexibleContexts #-} {-# LANGUAGE DefaultSignatures, TypeOperators #-}+{-# LANGUAGE TypeFamilies, DataKinds, UndecidableInstances #-}  -- | This module exposes a uniform interface to marshal values --   to and from Souffle Datalog. This is done via the 'Marshal' typeclass.@@ -10,14 +11,16 @@   ( Marshal(..)   , MonadPush(..)   , MonadPop(..)+  , SimpleProduct   ) where +import GHC.TypeLits import GHC.Generics import Data.Int import Data.Word+import Data.Kind import qualified Data.Text as T import qualified Data.Text.Lazy as TL-import qualified Language.Souffle.Internal.Constraints as C  {- | A typeclass for serializing primitive values from Haskell to Datalog. @@ -25,6 +28,7 @@  See also: 'MonadPop', 'Marshal'. -}+type MonadPush :: (Type -> Type) -> Constraint class Monad m => MonadPush m where   -- | Marshals a signed 32 bit integer to the datalog side.   pushInt32 :: Int32 -> m ()@@ -34,6 +38,8 @@   pushFloat :: Float -> m ()   -- | Marshals a string to the datalog side.   pushString :: String -> m ()+  -- | Marshals a UTF8-encoded Text string to the datalog side.+  pushText :: T.Text -> m ()  {- | A typeclass for serializing primitive values from Datalog to Haskell. @@ -41,6 +47,7 @@  See also: 'MonadPush', 'Marshal'. -}+type MonadPop :: (Type -> Type) -> Constraint class Monad m => MonadPop m where   -- | Unmarshals a signed 32 bit integer from the datalog side.   popInt32 :: m Int32@@ -50,6 +57,8 @@   popFloat :: m Float   -- | Unmarshals a string from the datalog side.   popString :: m String+  -- | Unmarshals a UTF8-encoded Text string from the datalog side.+  popText :: m T.Text  {- | A typeclass for providing a uniform API to marshal/unmarshal values      between Haskell and Souffle datalog.@@ -58,7 +67,9 @@ pushed/popped one by one. You need to make sure that the marshalling of values happens in the correct order or unexpected things might happen (including crashes). Pushing and popping of fields should happen in the-same order (from left to right, as defined in Datalog).+same order (from left to right, as defined in Datalog). The ordering of how+nested products are serialized is the same as when the fields of the nested+product types are inlined into the parent type.  Generic implementations for 'push' and 'pop' that perform the previously described behavior are available. This makes it possible to@@ -70,6 +81,7 @@ instance Marshal Edge @ -}+type Marshal :: Type -> Constraint class Marshal a where   -- | Marshals a value to the datalog side.   push :: MonadPush m => a -> m ()@@ -77,10 +89,10 @@   pop :: MonadPop m => m a    default push-    :: (Generic a, C.SimpleProduct a, GMarshal (Rep a), MonadPush m)+    :: (Generic a, SimpleProduct a, GMarshal (Rep a), MonadPush m)     => a -> m ()   default pop-    :: (Generic a, C.SimpleProduct a, GMarshal (Rep a), MonadPop m)+    :: (Generic a, SimpleProduct a, GMarshal (Rep a), MonadPop m)     => m a   push a = gpush (from a)   {-# INLINABLE push #-}@@ -112,17 +124,18 @@   {-# INLINABLE pop #-}  instance Marshal T.Text where-  push = push . T.unpack+  push = pushText   {-# INLINABLE push #-}-  pop = T.pack <$> pop+  pop = popText   {-# INLINABLE pop #-}  instance Marshal TL.Text where-  push = push . TL.unpack+  push = push . TL.toStrict   {-# INLINABLE push #-}-  pop = TL.pack <$> pop+  pop = TL.fromStrict <$> pop   {-# INLINABLE pop #-} +type GMarshal :: (Type -> Type) -> Constraint class GMarshal f where   gpush :: MonadPush m => f a -> m ()   gpop  :: MonadPop m => m (f a)@@ -146,3 +159,45 @@   {-# INLINABLE gpush #-}   gpop = M1 <$> gpop   {-# INLINABLE gpop #-}+++-- | A helper type family used for generating a more user-friendly type error+--   for incompatible types when generically deriving marshalling code for+--   the 'Language.Souffle.Marshal.Marshal' typeclass.+--+--   The __a__ type parameter is the original type, used when displaying the type error.+--+--   A type error is returned if the passed in type is not a simple product type+--   consisting of only types that implement 'Marshal'.+type SimpleProduct :: Type -> Constraint+type family SimpleProduct a where+  SimpleProduct a = (ProductLike a (Rep a), OnlyMarshallableFields (Rep a))++type ProductLike :: Type -> (Type -> Type) -> Constraint+type family ProductLike t f where+  ProductLike t (a :*: b) = (ProductLike t a, ProductLike t b)+  ProductLike t (M1 _ _ a) = ProductLike t a+  ProductLike _ (K1 _ _) = ()+  ProductLike t (_ :+: _) =+    TypeError ( 'Text "Error while deriving marshalling code for type " ':<>: 'ShowType t ':<>: 'Text ":"+              ':$$: 'Text "Cannot derive sum type, only product types are supported.")+  ProductLike t U1 =+    TypeError ( 'Text "Error while deriving marshalling code for type " ':<>: 'ShowType t ':<>: 'Text ":"+              ':$$: 'Text "Cannot automatically derive code for 0 argument constructor.")+  ProductLike t V1 =+    TypeError ( 'Text "Error while deriving marshalling code for type " ':<>: 'ShowType t ':<>: 'Text ":"+              ':$$: 'Text "Cannot derive void type.")++type OnlyMarshallableFields :: (Type -> Type) -> Constraint+type family OnlyMarshallableFields f where+  OnlyMarshallableFields (a :*: b) = (OnlyMarshallableFields a, OnlyMarshallableFields b)+  OnlyMarshallableFields (a :+: b) = (OnlyMarshallableFields a, OnlyMarshallableFields b)+  OnlyMarshallableFields (M1 _ _ a) = OnlyMarshallableFields a+  OnlyMarshallableFields U1 = ()+  OnlyMarshallableFields V1 = ()+  OnlyMarshallableFields k = OnlyMarshallableField k++type OnlyMarshallableField :: (Type -> Type) -> Constraint+type family OnlyMarshallableField f where+  OnlyMarshallableField (M1 _ _ a) = OnlyMarshallableField a+  OnlyMarshallableField (K1 _ a) = Marshal a
souffle-haskell.cabal view
@@ -1,13 +1,11 @@ cabal-version: 2.2 --- This file has been generated from package.yaml by hpack version 0.33.0.+-- This file has been generated from package.yaml by hpack version 0.35.2. -- -- see: https://github.com/sol/hpack------ hash: 282b7f644a46aaacc10fd325f80d88a1529d0ea5737403640996967d6b523a4c  name:           souffle-haskell-version:        2.1.0+version:        4.0.0 synopsis:       Souffle Datalog bindings for Haskell description:    Souffle Datalog bindings for Haskell. category:       Logic Programming, Foreign Binding, Bindings@@ -15,22 +13,24 @@ bug-reports:    https://github.com/luc-tielen/souffle-haskell/issues author:         Luc Tielen maintainer:     luc.tielen@gmail.com-copyright:      2020 Luc Tielen+copyright:      2022 Luc Tielen license:        MIT license-file:   LICENSE build-type:     Simple extra-source-files:-    README.md-    CHANGELOG.md-    LICENSE     cbits/souffle.h     cbits/souffle/CompiledSouffle.h-    cbits/souffle/CompiledTuple.h     cbits/souffle/datastructure/Brie.h     cbits/souffle/datastructure/BTree.h+    cbits/souffle/datastructure/BTreeDelete.h+    cbits/souffle/datastructure/BTreeUtil.h+    cbits/souffle/datastructure/ConcurrentFlyweight.h+    cbits/souffle/datastructure/ConcurrentInsertOnlyHashMap.h     cbits/souffle/datastructure/EquivalenceRelation.h     cbits/souffle/datastructure/LambdaBTree.h     cbits/souffle/datastructure/PiggyList.h+    cbits/souffle/datastructure/RecordTableImpl.h+    cbits/souffle/datastructure/SymbolTableImpl.h     cbits/souffle/datastructure/Table.h     cbits/souffle/datastructure/UnionFind.h     cbits/souffle/io/gzfstream.h@@ -51,16 +51,25 @@     cbits/souffle/SymbolTable.h     cbits/souffle/utility/CacheUtil.h     cbits/souffle/utility/ContainerUtil.h+    cbits/souffle/utility/DynamicCasting.h     cbits/souffle/utility/EvaluatorUtil.h     cbits/souffle/utility/FileUtil.h-    cbits/souffle/utility/FunctionalUtil.h+    cbits/souffle/utility/General.h+    cbits/souffle/utility/Iteration.h     cbits/souffle/utility/json11.h     cbits/souffle/utility/MiscUtil.h     cbits/souffle/utility/ParallelUtil.h+    cbits/souffle/utility/span.h     cbits/souffle/utility/StreamUtil.h     cbits/souffle/utility/StringUtil.h     cbits/souffle/utility/tinyformat.h+    cbits/souffle/utility/Types.h+    cbits/souffle.cpp     cbits/souffle/LICENSE+extra-doc-files:+    README.md+    CHANGELOG.md+    LICENSE  source-repository head   type: git@@ -68,12 +77,11 @@  library   exposed-modules:+      Language.Souffle.Analysis       Language.Souffle.Class       Language.Souffle.Compiled-      Language.Souffle.Experimental       Language.Souffle.Internal       Language.Souffle.Internal.Bindings-      Language.Souffle.Internal.Constraints       Language.Souffle.Interpreted       Language.Souffle.Marshal   other-modules:@@ -82,38 +90,54 @@       Paths_souffle_haskell   hs-source-dirs:       lib-  default-extensions: OverloadedStrings LambdaCase ScopedTypeVariables-  ghc-options: -Wall -Weverything -Wno-safe -Wno-unsafe -Wno-implicit-prelude -Wno-missed-specializations -Wno-all-missed-specializations -Wno-missing-import-lists -Wno-type-defaults -Wno-missing-local-signatures -Wno-monomorphism-restriction -Wno-missing-deriving-strategies -Wno-prepositive-qualified-module -Wno-missing-safe-haskell-mode -optP-Wno-nonportable-include-path -fhide-source-paths -fno-show-valid-hole-fits -fno-sort-valid-hole-fits+  default-extensions:+      DerivingStrategies+      FlexibleContexts+      LambdaCase+      OverloadedStrings+      ScopedTypeVariables+      StandaloneKindSignatures+  ghc-options: -Wall -Weverything -Wno-safe -Wno-unsafe -Wno-implicit-prelude -Wno-missed-specializations -Wno-all-missed-specializations -Wno-missing-import-lists -Wno-type-defaults -Wno-missing-local-signatures -Wno-monomorphism-restriction -Wno-prepositive-qualified-module -Wno-missing-safe-haskell-mode -Wno-operator-whitespace -optP-Wno-nonportable-include-path -fhide-source-paths -fno-show-valid-hole-fits -fno-sort-valid-hole-fits   cxx-options: -std=c++17 -Wall   include-dirs:       cbits       cbits/souffle   install-includes:       souffle/CompiledSouffle.h-      souffle/CompiledTuple.h       souffle/RamTypes.h       souffle/RecordTable.h+      souffle/utility/span.h       souffle/SignalHandler.h       souffle/SouffleInterface.h       souffle/SymbolTable.h       souffle/utility/MiscUtil.h+      souffle/utility/General.h+      souffle/utility/Iteration.h+      souffle/utility/Types.h       souffle/utility/tinyformat.h-      souffle/utility/ParallelUtil.h-      souffle/utility/StreamUtil.h+      souffle/datastructure/BTreeDelete.h+      souffle/datastructure/BTreeUtil.h+      souffle/utility/CacheUtil.h       souffle/utility/ContainerUtil.h+      souffle/utility/DynamicCasting.h+      souffle/utility/ParallelUtil.h       souffle/datastructure/Brie.h-      souffle/utility/CacheUtil.h+      souffle/utility/StreamUtil.h       souffle/datastructure/EquivalenceRelation.h       souffle/datastructure/LambdaBTree.h       souffle/datastructure/BTree.h       souffle/datastructure/PiggyList.h       souffle/datastructure/UnionFind.h+      souffle/datastructure/RecordTableImpl.h+      souffle/datastructure/ConcurrentFlyweight.h+      souffle/datastructure/ConcurrentInsertOnlyHashMap.h+      souffle/datastructure/SymbolTableImpl.h       souffle/datastructure/Table.h       souffle/io/IOSystem.h       souffle/io/ReadStream.h       souffle/io/SerialisationStream.h-      souffle/utility/json11.h       souffle/utility/StringUtil.h+      souffle/utility/json11.h       souffle/io/ReadStreamCSV.h       souffle/utility/FileUtil.h       souffle/io/gzfstream.h@@ -124,73 +148,89 @@       souffle/io/ReadStreamSQLite.h       souffle/io/WriteStreamSQLite.h       souffle/utility/EvaluatorUtil.h-      souffle/utility/FunctionalUtil.h+      souffle/utility/EvaluatorUtil.h   cxx-sources:       cbits/souffle.cpp   build-depends:       array <=1.0     , base >=4.12 && <5-    , containers >=0.6.2.1 && <1+    , bytestring >=0.10.10 && <1     , deepseq >=1.4.4 && <2     , directory >=1.3.3 && <2     , filepath >=1.4.2 && <2     , mtl >=2.0 && <3     , process >=1.6 && <2-    , template-haskell >=2 && <3+    , profunctors >=5.6.2 && <6     , temporary >=1.3 && <2-    , text >=1.0 && <2-    , type-errors-pretty >=0.0.1.0 && <1+    , text >=2.0.2 && <3     , vector <=1.0+  default-language: Haskell2010   if os(linux)     extra-libraries:         stdc++-  default-language: Haskell2010  test-suite souffle-haskell-test   type: exitcode-stdio-1.0   main-is: test.hs   other-modules:+      Test.Language.Souffle.AnalysisSpec       Test.Language.Souffle.CompiledSpec-      Test.Language.Souffle.Experimental.Fixtures-      Test.Language.Souffle.Experimental.FixturesCompiled-      Test.Language.Souffle.ExperimentalSpec+      Test.Language.Souffle.DerivingViaSpec       Test.Language.Souffle.InterpretedSpec       Test.Language.Souffle.MarshalSpec       Paths_souffle_haskell+  autogen-modules:+      Paths_souffle_haskell   hs-source-dirs:       tests-  default-extensions: OverloadedStrings LambdaCase ScopedTypeVariables-  ghc-options: -Wall -Weverything -Wno-safe -Wno-unsafe -Wno-implicit-prelude -Wno-missed-specializations -Wno-all-missed-specializations -Wno-missing-import-lists -Wno-type-defaults -Wno-missing-local-signatures -Wno-monomorphism-restriction -Wno-missing-deriving-strategies -Wno-prepositive-qualified-module -Wno-missing-safe-haskell-mode -optP-Wno-nonportable-include-path -fhide-source-paths -fno-show-valid-hole-fits -fno-sort-valid-hole-fits+  default-extensions:+      DerivingStrategies+      FlexibleContexts+      LambdaCase+      OverloadedStrings+      ScopedTypeVariables+      StandaloneKindSignatures+  ghc-options: -Wall -Weverything -Wno-safe -Wno-unsafe -Wno-implicit-prelude -Wno-missed-specializations -Wno-all-missed-specializations -Wno-missing-import-lists -Wno-type-defaults -Wno-missing-local-signatures -Wno-monomorphism-restriction -Wno-prepositive-qualified-module -Wno-missing-safe-haskell-mode -Wno-operator-whitespace -optP-Wno-nonportable-include-path -fhide-source-paths -fno-show-valid-hole-fits -fno-sort-valid-hole-fits -Wno-missing-kind-signatures -Wno-operator-whitespace   cxx-options: -std=c++17 -D__EMBEDDED_SOUFFLE__   include-dirs:       cbits       cbits/souffle   install-includes:       souffle/CompiledSouffle.h-      souffle/CompiledTuple.h       souffle/RamTypes.h       souffle/RecordTable.h+      souffle/utility/span.h       souffle/SignalHandler.h       souffle/SouffleInterface.h       souffle/SymbolTable.h       souffle/utility/MiscUtil.h+      souffle/utility/General.h+      souffle/utility/Iteration.h+      souffle/utility/Types.h       souffle/utility/tinyformat.h-      souffle/utility/ParallelUtil.h-      souffle/utility/StreamUtil.h+      souffle/datastructure/BTreeDelete.h+      souffle/datastructure/BTreeUtil.h+      souffle/utility/CacheUtil.h       souffle/utility/ContainerUtil.h+      souffle/utility/DynamicCasting.h+      souffle/utility/ParallelUtil.h       souffle/datastructure/Brie.h-      souffle/utility/CacheUtil.h+      souffle/utility/StreamUtil.h       souffle/datastructure/EquivalenceRelation.h       souffle/datastructure/LambdaBTree.h       souffle/datastructure/BTree.h       souffle/datastructure/PiggyList.h       souffle/datastructure/UnionFind.h+      souffle/datastructure/RecordTableImpl.h+      souffle/datastructure/ConcurrentFlyweight.h+      souffle/datastructure/ConcurrentInsertOnlyHashMap.h+      souffle/datastructure/SymbolTableImpl.h       souffle/datastructure/Table.h       souffle/io/IOSystem.h       souffle/io/ReadStream.h       souffle/io/SerialisationStream.h-      souffle/utility/json11.h       souffle/utility/StringUtil.h+      souffle/utility/json11.h       souffle/io/ReadStreamCSV.h       souffle/utility/FileUtil.h       souffle/io/gzfstream.h@@ -201,30 +241,106 @@       souffle/io/ReadStreamSQLite.h       souffle/io/WriteStreamSQLite.h       souffle/utility/EvaluatorUtil.h-      souffle/utility/FunctionalUtil.h+      souffle/utility/EvaluatorUtil.h   cxx-sources:+      tests/fixtures/edge_cases.cpp       tests/fixtures/path.cpp       tests/fixtures/round_trip.cpp   build-depends:       array <=1.0     , base >=4.12 && <5-    , containers >=0.6.2.1 && <1-    , deepseq >=1.4.4 && <2     , directory >=1.3.3 && <2-    , filepath >=1.4.2 && <2     , hedgehog ==1.*     , hspec >=2.6.1 && <3.0.0     , hspec-hedgehog ==0.*-    , mtl >=2.0 && <3-    , neat-interpolation ==0.*-    , process >=1.6 && <2+    , profunctors >=5.6.2 && <6     , souffle-haskell-    , template-haskell >=2 && <3     , temporary >=1.3 && <2-    , text >=1.0 && <2-    , type-errors-pretty >=0.0.1.0 && <1+    , text >=2.0.2 && <3     , vector <=1.0+  default-language: Haskell2010   if os(darwin)     extra-libraries:         c++++benchmark souffle-haskell-benchmarks+  type: exitcode-stdio-1.0+  main-is: bench.hs+  other-modules:+      Paths_souffle_haskell+  autogen-modules:+      Paths_souffle_haskell+  hs-source-dirs:+      benchmarks+  default-extensions:+      DerivingStrategies+      FlexibleContexts+      LambdaCase+      OverloadedStrings+      ScopedTypeVariables+      StandaloneKindSignatures+  ghc-options: -Wall -Weverything -Wno-safe -Wno-unsafe -Wno-implicit-prelude -Wno-missed-specializations -Wno-all-missed-specializations -Wno-missing-import-lists -Wno-type-defaults -Wno-missing-local-signatures -Wno-monomorphism-restriction -Wno-prepositive-qualified-module -Wno-missing-safe-haskell-mode -Wno-operator-whitespace -optP-Wno-nonportable-include-path -fhide-source-paths -fno-show-valid-hole-fits -fno-sort-valid-hole-fits +RTS -N1 -RTS+  cxx-options: -std=c++17 -D__EMBEDDED_SOUFFLE__ -std=c++17 -march=native+  include-dirs:+      cbits+      cbits/souffle+  install-includes:+      souffle/CompiledSouffle.h+      souffle/RamTypes.h+      souffle/RecordTable.h+      souffle/utility/span.h+      souffle/SignalHandler.h+      souffle/SouffleInterface.h+      souffle/SymbolTable.h+      souffle/utility/MiscUtil.h+      souffle/utility/General.h+      souffle/utility/Iteration.h+      souffle/utility/Types.h+      souffle/utility/tinyformat.h+      souffle/datastructure/BTreeDelete.h+      souffle/datastructure/BTreeUtil.h+      souffle/utility/CacheUtil.h+      souffle/utility/ContainerUtil.h+      souffle/utility/DynamicCasting.h+      souffle/utility/ParallelUtil.h+      souffle/datastructure/Brie.h+      souffle/utility/StreamUtil.h+      souffle/datastructure/EquivalenceRelation.h+      souffle/datastructure/LambdaBTree.h+      souffle/datastructure/BTree.h+      souffle/datastructure/PiggyList.h+      souffle/datastructure/UnionFind.h+      souffle/datastructure/RecordTableImpl.h+      souffle/datastructure/ConcurrentFlyweight.h+      souffle/datastructure/ConcurrentInsertOnlyHashMap.h+      souffle/datastructure/SymbolTableImpl.h+      souffle/datastructure/Table.h+      souffle/io/IOSystem.h+      souffle/io/ReadStream.h+      souffle/io/SerialisationStream.h+      souffle/utility/StringUtil.h+      souffle/utility/json11.h+      souffle/io/ReadStreamCSV.h+      souffle/utility/FileUtil.h+      souffle/io/gzfstream.h+      souffle/io/ReadStreamJSON.h+      souffle/io/WriteStream.h+      souffle/io/WriteStreamCSV.h+      souffle/io/WriteStreamJSON.h+      souffle/io/ReadStreamSQLite.h+      souffle/io/WriteStreamSQLite.h+      souffle/utility/EvaluatorUtil.h+      souffle/utility/EvaluatorUtil.h+  cxx-sources:+      benchmarks/fixtures/bench.cpp+  build-depends:+      base >=4.12 && <5+    , criterion ==1.*+    , deepseq >=1.4.4 && <2+    , souffle-haskell+    , text >=2.0.2 && <3+    , vector <=1.0   default-language: Haskell2010+  if os(darwin)+    extra-libraries:+        c++
+ tests/Test/Language/Souffle/AnalysisSpec.hs view
@@ -0,0 +1,249 @@+{-# LANGUAGE DataKinds, TypeFamilies, DeriveGeneric, Arrows #-}++module Test.Language.Souffle.AnalysisSpec+  ( module Test.Language.Souffle.AnalysisSpec+  ) where++import Prelude hiding ((.), id)+import Control.Arrow+import Control.Category+import Test.Hspec+import Data.Profunctor+import GHC.Generics+import Control.Monad.IO.Class+import Language.Souffle.Analysis+import qualified Language.Souffle.Interpreted as Souffle++data Path = Path++data Edge = Edge String String+  deriving stock (Eq, Show, Generic)++data Reachable = Reachable String String+  deriving stock (Eq, Show, Generic)++instance Souffle.Program Path where+  type ProgramFacts Path = '[Edge, Reachable]+  programName = const "path"++instance Souffle.Fact Edge where+  type FactDirection Edge = 'Souffle.InputOutput+  factName = const "edge"++instance Souffle.Fact Reachable where+  type FactDirection Reachable = 'Souffle.Output+  factName = const "reachable"++instance Souffle.Marshal Edge+instance Souffle.Marshal Reachable++data Results = Results [Reachable] [Edge]+  deriving stock (Eq, Show)++pathAnalysis :: Souffle.Handle Path+             -> Analysis Souffle.SouffleM [Edge] [Reachable]+pathAnalysis h =+  mkAnalysis (Souffle.addFacts h) (Souffle.run h) (Souffle.getFacts h)++-- A little bit silly, but good enough to test different forms of application with+pathAnalysis' :: Souffle.Handle Path+             -> Analysis Souffle.SouffleM [Edge] [Edge]+pathAnalysis' h =+  mkAnalysis (Souffle.addFacts h) (Souffle.run h) (Souffle.getFacts h)++data RoundTrip = RoundTrip++newtype StringFact = StringFact String+  deriving stock (Eq, Show, Generic)++instance Souffle.Program RoundTrip where+  type ProgramFacts RoundTrip = '[StringFact]++  programName = const "round_trip"++instance Souffle.Fact StringFact where+  type FactDirection StringFact = 'Souffle.InputOutput++  factName = const "string_fact"++instance Souffle.Marshal StringFact+++roundTripAnalysis :: Souffle.Handle RoundTrip+                  -> Analysis Souffle.SouffleM [Reachable] [StringFact]+roundTripAnalysis h =+  mkAnalysis addFacts (Souffle.run h) (Souffle.getFacts h)+  where+    addFacts rs = do+      Souffle.addFacts h $ map (\(Reachable a _) -> StringFact a) rs++withSouffle :: Souffle.Program a => a -> (Souffle.Handle a -> Souffle.SouffleM ()) -> IO ()+withSouffle prog f = Souffle.runSouffle prog $ \case+  Nothing -> error "Failed to load program"+  Just h -> f h++edges :: [Edge]+edges = [Edge "a" "b", Edge "b" "c", Edge "b" "d", Edge "d" "e"]++spec :: Spec+spec = describe "composing analyses" $ parallel $ do+  it "supports fmap" $ do+    withSouffle Path $ \h -> do+      let analysis = pathAnalysis h+          analysis' = fmap length analysis+      count <- execAnalysis analysis' edges+      liftIO $ count `shouldBe` 8++  describe "analysis used as a profunctor" $ parallel $ do+    it "supports lmap" $ do+      withSouffle Path $ \h -> do+        let inputs = [("a", "b"), ("b", "c")]+            analysis = pathAnalysis h+            analysis' = lmap (map (uncurry Edge)) analysis+        rs <- execAnalysis analysis' inputs+        liftIO $ rs `shouldBe` [ Reachable "a" "b"+                               , Reachable "a" "c"+                               , Reachable "b" "c"+                               ]++    it "supports rmap" $ do+      withSouffle Path $ \h -> do+        let analysis = pathAnalysis h+            analysis' = rmap length analysis+        count <- execAnalysis analysis' edges+        liftIO $ count `shouldBe` 8++  it "supports applicative composition" $+    withSouffle Path $ \hPath -> do+      let analysis1 = pathAnalysis hPath+          analysis2 = pathAnalysis' hPath+          analysis = Results <$> analysis1 <*> analysis2+          inputs = [Edge "a" "b", Edge "b" "c"]+          reachables = [ Reachable "a" "b"+                       , Reachable "a" "c"+                       , Reachable "b" "c"+                       ]+      results <- execAnalysis analysis inputs+      liftIO $ results `shouldBe` Results reachables inputs++  it "supports semigroupal composition" $ do+    withSouffle Path $ \h -> do+      let analysis = pathAnalysis h+          analysis' = analysis <> analysis+      rs <- execAnalysis analysis' [Edge "a" "b", Edge "b" "c"]+      let results = [ Reachable "a" "b"+                    , Reachable "a" "c"+                    , Reachable "b" "c"+                    ]+          results' = mconcat $ replicate 2 results+      liftIO $ rs `shouldBe` results'++  it "supports mempty" $ do+    withSouffle Path $ \_ -> do+      let analysis :: Analysis Souffle.SouffleM [Edge] [Reachable]+          analysis = mempty+      rs <- execAnalysis analysis [Edge "a" "b", Edge "b" "c"]+      liftIO $ rs `shouldBe` []++  it "supports converting an analysis to a monadic function" $ do+    withSouffle Path $ \h -> do+      let analysis = pathAnalysis h+      rs <- execAnalysis analysis [Edge "a" "b", Edge "b" "c"]+      let results = [ Reachable "a" "b"+                    , Reachable "a" "c"+                    , Reachable "b" "c"+                    ]+      liftIO $ rs `shouldBe` results++  describe "analysis used as a category" $ parallel $ do+    it "supports 'id'" $ do+      withSouffle Path $ \_ -> do+        let analysis :: Analysis Souffle.SouffleM [Edge] [Edge]+            analysis = id+        edges' <- execAnalysis analysis edges+        liftIO $ edges' `shouldBe` edges++    it "supports sequential composition using (.)" $ do+      withSouffle Path $ \h -> do+        let reachableToFlippedEdge (Reachable a b) = Edge b a+            analysis1 = pathAnalysis h+            analysis2 = lmap (map reachableToFlippedEdge) $ pathAnalysis h+        rs <- execAnalysis (analysis2 . analysis1) [Edge "a" "b", Edge "b" "c"]+        let results = [ Reachable "a" "a"+                      , Reachable "a" "b"+                      , Reachable "a" "c"+                      , Reachable "b" "a"+                      , Reachable "b" "b"+                      , Reachable "b" "c"+                      , Reachable "c" "a"+                      , Reachable "c" "b"+                      , Reachable "c" "c"+                      ]+        liftIO $ rs `shouldBe` results++  describe "analysis used as an arrow" $ parallel $ do+    it "supports 'arr'" $ do+      withSouffle Path $ \_ -> do+        let analysis :: Analysis Souffle.SouffleM Int Int+            analysis = arr (+1)+        result1 <- execAnalysis analysis 41+        result2 <- execAnalysis (arr id) 41+        liftIO $ result1 `shouldBe` 42+        liftIO $ result2 `shouldBe` 41++    it "supports 'first'" $ do+      withSouffle Path $ \_ -> do+        let analysis :: Analysis Souffle.SouffleM (Int, Bool) (Int, Bool)+            analysis = first (arr (+1))+            input = (41, True)+        result <- execAnalysis analysis input+        liftIO $ result `shouldBe` (42, True)++    it "supports 'second'" $ do+      withSouffle Path $ \_ -> do+        let analysis :: Analysis Souffle.SouffleM (Bool, Int) (Bool, Int)+            analysis = second (arr (+1))+            input = (True, 41)+        result <- execAnalysis analysis input+        liftIO $ result `shouldBe` (True, 42)++    it "supports (***)" $ do+      withSouffle Path $ \_ -> do+        let analysis :: Analysis Souffle.SouffleM (Bool, Int) (Bool, Int)+            analysis = arr not *** arr (+1)+            input = (True, 41)+        result <- execAnalysis analysis input+        liftIO $ result `shouldBe` (False, 42)++    it "supports (&&&)" $ do+      withSouffle Path $ \_ -> do+        let analysis :: Analysis Souffle.SouffleM Int (Bool, Int)+            analysis = arr (== 1000) &&& arr (+1)+            input = 41+        result <- execAnalysis analysis input+        liftIO $ result `shouldBe` (False, 42)++    it "supports arrow notation" $ do+      withSouffle Path $ \h -> do+        liftIO $ withSouffle RoundTrip $ \h' -> do+          let arrowAnalysis = proc es -> do+                rs <- pathAnalysis h -< es+                strs <- roundTripAnalysis h' -< rs+                returnA -< strs+          result <- execAnalysis arrowAnalysis edges+          liftIO $ result `shouldBe` [ StringFact "a"+                                     , StringFact "b"+                                     , StringFact "d"+                                     ]++    it "supports case expressions in arrow notation" $ do+      withSouffle Path $ \h -> do+        let analysis =  proc es -> do+              rs <- pathAnalysis h -< es+              case rs of+                [] -> returnA -< []+                rs' -> returnA -< take 2 rs'+        result <- execAnalysis analysis edges+        let expected = [ Reachable "a" "b", Reachable "a" "c" ]+        liftIO $ result `shouldBe` expected+
tests/Test/Language/Souffle/CompiledSpec.hs view
@@ -14,10 +14,10 @@ data Path = Path  data Edge = Edge String String-  deriving (Eq, Show, Generic)+  deriving stock (Eq, Show, Generic)  data Reachable = Reachable String String-  deriving (Eq, Show, Generic)+  deriving stock (Eq, Show, Generic)  instance Souffle.Program Path where   type ProgramFacts Path = '[Edge, Reachable]@@ -38,7 +38,7 @@ data BadPath = BadPath  instance Souffle.Program BadPath where-  type ProgramFacts BadPath = [Edge, Reachable]+  type ProgramFacts BadPath = '[Edge, Reachable]   programName = const "bad_path"  @@ -54,6 +54,27 @@       isJust prog `shouldBe` True    describe "getFacts" $ parallel $ do+    it "doesn't crash if used as last action (lists)" $ do+      edges <- Souffle.runSouffle Path $ \handle -> do+        let prog = fromJust handle+        Souffle.run prog+        Souffle.getFacts prog+      edges `shouldBe` [Edge "b" "c", Edge "a" "b"]++    it "doesn't crash if used as last action (vectors)" $ do+      edges <- Souffle.runSouffle Path $ \handle -> do+        let prog = fromJust handle+        Souffle.run prog+        Souffle.getFacts prog+      edges `shouldBe` V.fromList [Edge "a" "b", Edge "b" "c"]++    it "doesn't crash if used as last action (arrays)" $ do+      edges <- Souffle.runSouffle Path $ \handle -> do+        let prog = fromJust handle+        Souffle.run prog+        Souffle.getFacts prog+      edges `shouldBe` A.listArray (0 :: Int, 1) [Edge "a" "b", Edge "b" "c"]+     it "can retrieve facts as a list" $ do       (edges, reachables) <- Souffle.runSouffle Path $ \handle -> do         let prog = fromJust handle@@ -170,6 +191,19 @@         pure (e, r)       edge `shouldBe` Just (Edge "a" "b")       reachable `shouldBe` Just (Reachable "a" "c")++    it "can handle unicode characters" $ do+      let fact = Edge "∀∀" "bla"+          fact2 = Edge "∃∃" "bla"+          fact3 = Edge "℀℀" "bla"+      results <- Souffle.runSouffle Path $ \handle -> do+        let prog = fromJust handle+        Souffle.addFact prog fact+        Souffle.run prog+        (,,) <$> Souffle.findFact prog fact+             <*> Souffle.findFact prog fact2+             <*> Souffle.findFact prog fact3+      results `shouldBe` (Just fact, Nothing, Nothing)    -- TODO writeFiles / loadFiles
+ tests/Test/Language/Souffle/DerivingViaSpec.hs view
@@ -0,0 +1,40 @@+{-# LANGUAGE UndecidableInstances, DataKinds, DeriveGeneric, DeriveAnyClass, DerivingVia #-}++module Test.Language.Souffle.DerivingViaSpec+  ( module Test.Language.Souffle.DerivingViaSpec+  ) where++import Test.Hspec+import GHC.Generics+import Data.Maybe+import qualified Language.Souffle.Interpreted as Souffle+++data Path = Path+  deriving Souffle.Program+  via Souffle.ProgramOptions Path "path" '[Edge, Reachable]++data Edge = Edge String String+  deriving stock (Eq, Show, Generic)+  deriving anyclass Souffle.Marshal+  deriving Souffle.Fact+  via Souffle.FactOptions Edge "edge" 'Souffle.InputOutput++data Reachable = Reachable String String+  deriving stock (Eq, Show, Generic)+  deriving anyclass Souffle.Marshal+  deriving Souffle.Fact+  via Souffle.FactOptions Reachable "reachable" 'Souffle.Output+++spec :: Spec+spec = describe "Souffle DerivingVia-style API" $ do+  it "can get and put facts from souffle" $ do++    edges <- Souffle.runSouffle Path $ \handle -> do+      let prog = fromJust handle+      Souffle.addFacts prog [Edge "e" "f", Edge "f" "g"]+      Souffle.run prog+      Souffle.getFacts prog+    edges `shouldBe` [Edge "a" "b", Edge "b" "c", Edge "e" "f", Edge "f" "g"]+
− tests/Test/Language/Souffle/Experimental/Fixtures.hs 
@@ -1,31 +0,0 @@--{-# LANGUAGE DataKinds, TypeFamilies, DeriveGeneric, DeriveAnyClass #-}--module Test.Language.Souffle.Experimental.Fixtures-  ( module Test.Language.Souffle.Experimental.Fixtures-  ) where--import GHC.Generics-import Language.Souffle.Class-import Language.Souffle.Experimental--data CompiledProgram = CompiledProgram--instance Program CompiledProgram where-  type ProgramFacts CompiledProgram = [Edge, Reachable]-  programName = const "compiledprogram"--data Edge = Edge String String-  deriving (Generic, Marshal, FactMetadata)--data Reachable = Reachable String String-  deriving (Eq, Show, Generic, Marshal, FactMetadata)--instance Fact Edge where-  type FactDirection Edge = 'Input-  factName = const "edge"--instance Fact Reachable where-  type FactDirection Reachable = 'Output-  factName = const "reachable"-
− tests/Test/Language/Souffle/Experimental/FixturesCompiled.hs 
@@ -1,22 +0,0 @@--{-# OPTIONS_GHC -optc-std=c++17 -D__EMBEDDED_SOUFFLE__ #-}-{-# LANGUAGE TypeApplications, TemplateHaskell #-}-module Test.Language.Souffle.Experimental.FixturesCompiled () where---- NOTE: this module can't be grouped together with "Fixtures"--- due to TemplateHaskell staging restriction.--import Test.Language.Souffle.Experimental.Fixtures-import Language.Souffle.Experimental--$(embedProgram CompiledProgram $ do-  Predicate edge <- predicateFor @Edge-  Predicate reachable <- predicateFor @Reachable-  a <- var "a"-  b <- var "b"-  c <- var "c"-  reachable(a, b) |- edge(a, b)-  reachable(a, b) |- do-    edge(a, c)-    reachable(c, b)- )
− tests/Test/Language/Souffle/ExperimentalSpec.hs 
@@ -1,1117 +0,0 @@--{-# LANGUAGE DeriveGeneric, DeriveAnyClass, TypeApplications, QuasiQuotes, TypeOperators #-}-{-# LANGUAGE DataKinds, TypeFamilies #-}--module Test.Language.Souffle.ExperimentalSpec-  ( module Test.Language.Souffle.ExperimentalSpec-  ) where--import qualified Test.Language.Souffle.Experimental.Fixtures as F-import Test.Hspec-import GHC.Generics-import Data.Int-import Data.Word-import Data.Maybe (fromJust)-import qualified Data.Text as T-import qualified Data.Text.Lazy as TL-import System.IO.Temp-import Language.Souffle.Experimental-import Language.Souffle.Class-import Language.Souffle.Interpreted as I-import Language.Souffle.Compiled as C-import NeatInterpolation---data Point = Point { x :: Int32, y :: Int32 }-  deriving (Generic, Marshal, FactMetadata)--newtype IntFact = IntFact Int32-  deriving (Generic, Marshal, FactMetadata)--newtype UnsignedFact = UnsignedFact Word32-  deriving (Generic, Marshal, FactMetadata)--newtype FloatFact = FloatFact Float-  deriving (Generic, Marshal, FactMetadata)--data TextFact = TextFact T.Text TL.Text-  deriving (Generic, Marshal, FactMetadata)--data Triple = Triple String Int32 String-  deriving (Generic, Marshal, FactMetadata)--newtype Vertex = Vertex String-  deriving (Generic, Marshal, FactMetadata)--data Edge = Edge String String-  deriving (Generic, Marshal, FactMetadata)--data Reachable = Reachable String String-  deriving (Eq, Show, Generic, Marshal, FactMetadata)--newtype BTreeFact = BTreeFact Int32-  deriving (Generic, Marshal)--newtype BrieFact = BrieFact Int32-  deriving (Generic, Marshal)--data EqRelFact = EqRelFact Int32 Int32-  deriving (Generic, Marshal)--data DSLProgram = DSLProgram--instance Program DSLProgram where-  type ProgramFacts DSLProgram =-    [ Point-    , IntFact-    , FloatFact-    , UnsignedFact-    , TextFact-    , BTreeFact-    , BrieFact-    , EqRelFact-    , Triple-    , Vertex-    , Edge-    , Reachable-    ]-  programName = const "dslprogram"--instance Fact Point where-  type FactDirection Point = 'Input-  factName = const "point"-instance Fact IntFact where-  type FactDirection IntFact = 'Input-  factName = const "intfact"-instance Fact FloatFact where-  type FactDirection FloatFact = 'Input-  factName = const "floatfact"-instance Fact UnsignedFact where-  type FactDirection UnsignedFact = 'Input-  factName = const "unsignedfact"-instance Fact TextFact where-  type FactDirection TextFact = 'InputOutput-  factName = const "textfact"-instance Fact Triple where-  type FactDirection Triple = 'Internal-  factName = const "triple"-instance Fact Vertex where-  type FactDirection Vertex = 'Output-  factName = const "vertex"-instance Fact Edge where-  type FactDirection Edge = 'Input-  factName = const "edge"-instance Fact Reachable where-  type FactDirection Reachable = 'Output-  factName = const "reachable"-instance Fact BTreeFact where-  type FactDirection BTreeFact = 'Internal-  factName = const "btreefact"-instance Fact BrieFact where-  type FactDirection BrieFact = 'Internal-  factName = const "briefact"-instance Fact EqRelFact where-  type FactDirection EqRelFact = 'Input-  factName = const "eqrelfact"-instance FactMetadata BTreeFact where-  factOpts = const $ Metadata BTree NoInline-instance FactMetadata BrieFact where-  factOpts = const $ Metadata Brie Inline-instance FactMetadata EqRelFact where-  factOpts = const $ Metadata EqRel NoInline--spec :: Spec-spec = describe "Souffle DSL" $ parallel $ do-  describe "code generation" $ parallel $ do-    let prog ==> txt =-          let rendered = T.strip $ render DSLProgram prog-              expected = T.strip txt-          in rendered `shouldBe` expected--    it "can render an empty program" $-      render DSLProgram (pure ()) `shouldBe` ""--    it "can render a program with an input type definition" $ do-      let prog = do-            Predicate _ <- predicateFor @Edge-            pure ()-      prog ==> [text|-        .decl edge(t1: symbol, t2: symbol)-        .input edge-        |]--    it "can render a program with an output type definition" $ do-      let prog = do-            Predicate _ <- predicateFor @Reachable-            pure ()-      prog ==> [text|-        .decl reachable(t1: symbol, t2: symbol)-        .output reachable-        |]--    it "can render a program with type declared both as in- and output" $ do-      let prog = do-            Predicate _ <- predicateFor @TextFact-            pure ()-      prog ==> [text|-        .decl textfact(t1: symbol, t2: symbol)-        .input textfact-        .output textfact-        |]--    it "can render a program with type declared and only used internally" $ do-      let prog = do-            Predicate _ <- predicateFor @Triple-            pure ()-      prog ==> [text|-        .decl triple(t1: symbol, t2: number, t3: symbol)-        |]--    it "renders type declaration based on type info" $ do-      let prog = do-            Predicate _ <- predicateFor @IntFact-            Predicate _ <- predicateFor @UnsignedFact-            Predicate _ <- predicateFor @FloatFact-            Predicate _ <- predicateFor @Triple-            Predicate _ <- predicateFor @BTreeFact-            Predicate _ <- predicateFor @BrieFact-            Predicate _ <- predicateFor @EqRelFact-            pure ()-      prog ==> [text|-        .decl intfact(t1: number)-        .input intfact-        .decl unsignedfact(t1: unsigned)-        .input unsignedfact-        .decl floatfact(t1: float)-        .input floatfact-        .decl triple(t1: symbol, t2: number, t3: symbol)-        .decl btreefact(t1: number) btree-        .decl briefact(t1: number) brie inline-        .decl eqrelfact(t1: number, t2: number) eqrel-        .input eqrelfact-        |]--    it "uses record accessors as attribute names in type declaration if provided" $ do-      let prog = do-            Predicate _ <- predicateFor @Point-            pure ()-      prog ==> [text|-        .decl point(x: number, y: number)-        .input point-        |]--    it "can render facts" $ do-      let prog = do-            Predicate edge <- predicateFor @Edge-            Predicate triple <- predicateFor @Triple-            Predicate txt <- predicateFor @TextFact-            Predicate unsigned <- predicateFor @UnsignedFact-            Predicate float <- predicateFor @FloatFact-            edge("a", "b")-            triple("cde", 1000, "fgh")-            txt("ijk", "lmn")-            unsigned(42)-            float(42.42)-            float(0.01)-      prog ==> [text|-        .decl edge(t1: symbol, t2: symbol)-        .input edge-        .decl triple(t1: symbol, t2: number, t3: symbol)-        .decl textfact(t1: symbol, t2: symbol)-        .input textfact-        .output textfact-        .decl unsignedfact(t1: unsigned)-        .input unsignedfact-        .decl floatfact(t1: float)-        .input floatfact-        edge("a", "b").-        triple("cde", 1000, "fgh").-        textfact("ijk", "lmn").-        unsignedfact(42).-        floatfact(42.42).-        floatfact(0.01).-        |]--    it "can render a relation with a single rule" $ do-      let prog = do-            Predicate edge <- predicateFor @Edge-            Predicate reachable <- predicateFor @Reachable-            a <- var "a"-            b <- var "b"-            reachable(a, b) |- edge(a, b)-      prog ==> [text|-        .decl edge(t1: symbol, t2: symbol)-        .input edge-        .decl reachable(t1: symbol, t2: symbol)-        .output reachable-        reachable(a, b) :--          edge(a, b).-        |]--    it "can render a relation with multiple rules" $ do-      let prog = do-            Predicate edge <- predicateFor @Edge-            Predicate reachable <- predicateFor @Reachable-            a <- var "a"-            b <- var "b"-            reachable(a, b) |- do-              edge(a, a)-              edge(b, b)-              edge(a, b)-      prog ==> [text|-        .decl edge(t1: symbol, t2: symbol)-        .input edge-        .decl reachable(t1: symbol, t2: symbol)-        .output reachable-        reachable(a, b) :--          edge(a, a),-          edge(b, b),-          edge(a, b).-        |]--    it "can render a relation containing a wildcard" $ do-      let prog = do-            Predicate edge <- predicateFor @Edge-            Predicate vertex <- predicateFor @Vertex-            a <- var "a"-            vertex(a) |- do-              edge(a, __)-              edge(__, a)-      prog ==> [text|-        .decl edge(t1: symbol, t2: symbol)-        .input edge-        .decl vertex(t1: symbol)-        .output vertex-        vertex(a) :--          edge(a, _),-          edge(_, a).-        |]--    it "can render a relation with a logical or in the rule block" $ do-      let prog = do-            Predicate edge <- predicateFor @Edge-            Predicate reachable <- predicateFor @Reachable-            a <- var "a"-            b <- var "b"-            reachable(a, b) |- do-              let rules1 = do-                    edge(a, a)-                    edge(b, b)-                  rules2 = do-                    edge(a, b)-                    edge(b, a)-              rules1 \/ rules2-      prog ==> [text|-        .decl edge(t1: symbol, t2: symbol)-        .input edge-        .decl reachable(t1: symbol, t2: symbol)-        .output reachable-        reachable(a, b) :--          edge(a, a),-          edge(b, b);-          edge(a, b),-          edge(b, a).-        |]--    it "can render a relation with multiple clauses" $ do-      let prog = do-            Predicate edge <- predicateFor @Edge-            Predicate reachable <- predicateFor @Reachable-            a <- var "a"-            b <- var "b"-            c <- var "c"-            reachable(a, b) |- edge(a, b)-            reachable(a, b) |- do-              edge(a, c)-              reachable(c, b)-      prog ==> [text|-        .decl edge(t1: symbol, t2: symbol)-        .input edge-        .decl reachable(t1: symbol, t2: symbol)-        .output reachable-        reachable(a, b) :--          edge(a, b).-        reachable(a, b) :--          edge(a, c),-          reachable(c, b).-        |]--    it "can render a mix of and- and or- clauses correctly" $ do-      let prog = do-            Predicate edge <- predicateFor @Edge-            Predicate reachable <- predicateFor @Reachable-            a <- var "a"-            b <- var "b"-            c <- var "c"-            reachable(a, b) |- do-              edge(a, c) \/ edge(a, b)-              reachable(c, b)-      prog ==> [text|-        .decl edge(t1: symbol, t2: symbol)-        .input edge-        .decl reachable(t1: symbol, t2: symbol)-        .output reachable-        reachable(a, b) :--          (edge(a, c);-          edge(a, b)),-          reachable(c, b).-        |]--    it "discards empty alternative blocks" $ do-      let prog = do-            Predicate edge <- predicateFor @Edge-            Predicate reachable <- predicateFor @Reachable-            a <- var "a"-            b <- var "b"-            c <- var "c"-            reachable(a, b) |- do-              pure () \/ edge(a, c)-              reachable(c, b)-            reachable(a, b) |- do-              edge(a,c) \/ pure ()-              reachable(c, b)-            reachable(a, b) |- do-              pure () \/ do-                edge(a, c)-                reachable(c, b)-      prog ==> [text|-        .decl edge(t1: symbol, t2: symbol)-        .input edge-        .decl reachable(t1: symbol, t2: symbol)-        .output reachable-        reachable(a, b) :--          edge(a, c),-          reachable(c, b).-        reachable(a, b) :--          edge(a, c),-          reachable(c, b).-        reachable(a, b) :--          edge(a, c),-          reachable(c, b).-        |]--    it "discards rules with empty rule blocks completely" $ do-      let prog = do-            Predicate reachable <- predicateFor @Reachable-            a <- var "a"-            b <- var "b"-            reachable(a, b) |--              pure ()-            reachable(a, b) |- do-              pure () \/ pure ()-              pure ()-      prog ==> [text|-        .decl reachable(t1: symbol, t2: symbol)-        .output reachable-        |]--    it "discards empty negations" $ do-      let prog = do-            Predicate reachable <- predicateFor @Reachable-            a <- var "a"-            b <- var "b"-            reachable(a, b) |- do-              not' $ pure ()-      prog ==> [text|-        .decl reachable(t1: symbol, t2: symbol)-        .output reachable-        |]--    it "allows generically describing predicate relations" $ do-      -- NOTE: type signature not required, but it results in more clear type errors-      -- and can serve as documentation.-      let transitive :: forall prog p1 p2 t. Structure p1 ~ Structure p2-                      => Structure p1 ~ '[t, t]-                      => Predicate p1 -> Predicate p2 -> DSL prog 'Definition ()-          transitive (Predicate p1) (Predicate p2) = do-            a <- var "a"-            b <- var "b"-            c <- var "c"-            p1(a, b) |- p2(a, b)-            p1(a, b) |- do-              p2(a, c)-              p1(c, b)-          prog = do-            edge <- predicateFor @Edge-            reachable <- predicateFor @Reachable-            transitive reachable edge-      prog ==> [text|-        .decl edge(t1: symbol, t2: symbol)-        .input edge-        .decl reachable(t1: symbol, t2: symbol)-        .output reachable-        reachable(a, b) :--          edge(a, b).-        reachable(a, b) :--          edge(a, c),-          reachable(c, b).-        |]--    it "can render logical negation in rule block" $ do-      let prog = do-            Predicate edge <- predicateFor @Edge-            Predicate triple <- predicateFor @Triple-            a <- var "a"-            b <- var "b"-            c <- var "c"-            triple(a, b, c) |- do-              not' $ edge(a,c)-            triple(a, b, c) |- do-              not' $ do-                edge(a,a)-                edge(c,c)-            triple(a, b, c) |- do-              not' $ edge(a,a) \/ edge(c,c)-            triple(a, b, c) |- do-              not' $ not' $ edge(a,a)-      prog ==> [text|-        .decl edge(t1: symbol, t2: symbol)-        .input edge-        .decl triple(t1: symbol, t2: number, t3: symbol)-        triple(a, b, c) :--          !edge(a, c).-        triple(a, b, c) :--          !(edge(a, a),-          edge(c, c)).-        triple(a, b, c) :--          !(edge(a, a);-          edge(c, c)).-        triple(a, b, c) :--          !!edge(a, a).-        |]--    it "generates unique var names to avoid name collisions" $ do-      let prog = do-            Predicate edge <- predicateFor @Edge-            Predicate reachable <- predicateFor @Reachable-            a <- var "a"-            a' <- var "a"-            reachable(a, a') |- edge(a, a')-      prog ==> [text|-        .decl edge(t1: symbol, t2: symbol)-        .input edge-        .decl reachable(t1: symbol, t2: symbol)-        .output reachable-        reachable(a, a_1) :--          edge(a, a_1).-        |]--    describe "operators" $ parallel $ do-      -- TODO: check for number, unsigned, float; check underscore is not allowed-      describe "arithmetic" $ parallel $ do-        it "supports +" $ do-          let prog = do-                Predicate int <- predicateFor @IntFact-                Predicate unsigned <- predicateFor @UnsignedFact-                Predicate float <- predicateFor @FloatFact-                int(10 + 32)-                int(10 + 80 + 10)-                unsigned(10 + 32)-                float(10.12 + 31.88)-          prog ==> [text|-            .decl intfact(t1: number)-            .input intfact-            .decl unsignedfact(t1: unsigned)-            .input unsignedfact-            .decl floatfact(t1: float)-            .input floatfact-            intfact(10 + 32).-            intfact(10 + 80 + 10).-            unsignedfact(10 + 32).-            floatfact(10.12 + 31.88).-            |]--        it "supports *" $ do-          let prog = do-                Predicate int <- predicateFor @IntFact-                Predicate unsigned <- predicateFor @UnsignedFact-                Predicate float <- predicateFor @FloatFact-                int(10 * 32)-                int(10 * 80 * 10)-                unsigned(10 * 32)-                float(10.12 * 31.88)-          prog ==> [text|-            .decl intfact(t1: number)-            .input intfact-            .decl unsignedfact(t1: unsigned)-            .input unsignedfact-            .decl floatfact(t1: float)-            .input floatfact-            intfact(10 * 32).-            intfact(10 * 80 * 10).-            unsignedfact(10 * 32).-            floatfact(10.12 * 31.88).-            |]--        it "supports binary -" $ do-          let prog = do-                Predicate int <- predicateFor @IntFact-                Predicate unsigned <- predicateFor @UnsignedFact-                Predicate float <- predicateFor @FloatFact-                int(10 - 32)-                int(10 - 80 - 10)-                unsigned(10 - 32)-                float(10.12 - 31.88)-          prog ==> [text|-            .decl intfact(t1: number)-            .input intfact-            .decl unsignedfact(t1: unsigned)-            .input unsignedfact-            .decl floatfact(t1: float)-            .input floatfact-            intfact(10 - 32).-            intfact(10 - 80 - 10).-            unsignedfact(10 - 32).-            floatfact(10.12 - 31.88).-            |]--        it "supports unary -" $ do-          let prog = do-                Predicate int <- predicateFor @IntFact-                Predicate float <- predicateFor @FloatFact-                int(-42)-                int(-100)-                float(-13.37)-          prog ==> [text|-            .decl intfact(t1: number)-            .input intfact-            .decl floatfact(t1: float)-            .input floatfact-            intfact(-42).-            intfact(-100).-            floatfact(-13.37).-            |]--        it "supports /" $ do-          let prog = do-                Predicate float <- predicateFor @FloatFact-                float(13.37 / 0.01)-          prog ==> [text|-            .decl floatfact(t1: float)-            .input floatfact-            floatfact(13.37 / 0.01).-            |]--        it "supports ^" $ do-          let prog = do-                Predicate int <- predicateFor @IntFact-                Predicate unsigned <- predicateFor @UnsignedFact-                Predicate float <- predicateFor @FloatFact-                int(10 .^ 32)-                int(10 .^ 80 .^ 10)-                unsigned(10 .^ 32)-                float(42.42 .^ 2)-          prog ==> [text|-            .decl intfact(t1: number)-            .input intfact-            .decl unsignedfact(t1: unsigned)-            .input unsignedfact-            .decl floatfact(t1: float)-            .input floatfact-            intfact(10 ^ 32).-            intfact(10 ^ 80 ^ 10).-            unsignedfact(10 ^ 32).-            floatfact(42.42 ^ 2.0).-            |]--        it "supports %" $ do-          let prog = do-                Predicate int <- predicateFor @IntFact-                Predicate unsigned <- predicateFor @UnsignedFact-                int(10 .% 32)-                int(10 .% 80 .% 10)-                unsigned(10 .% 32)-          prog ==> [text|-            .decl intfact(t1: number)-            .input intfact-            .decl unsignedfact(t1: unsigned)-            .input unsignedfact-            intfact(10 % 32).-            intfact(10 % 80 % 10).-            unsignedfact(10 % 32).-            |]--      describe "logical operators" $ parallel $ do-        it "supports band" $ do-          let prog = do-                Predicate int <- predicateFor @IntFact-                Predicate unsigned <- predicateFor @UnsignedFact-                int(10 `band` 32)-                int(10 `band` 80 `band` 10)-                unsigned(10 `band` 32)-          prog ==> [text|-            .decl intfact(t1: number)-            .input intfact-            .decl unsignedfact(t1: unsigned)-            .input unsignedfact-            intfact(10 band 32).-            intfact(10 band 80 band 10).-            unsignedfact(10 band 32).-            |]--        it "supports bor" $ do-          let prog = do-                Predicate int <- predicateFor @IntFact-                Predicate unsigned <- predicateFor @UnsignedFact-                int(10 `bor` 32)-                int(10 `bor` 80 `bor` 10)-                unsigned(10 `bor` 32)-          prog ==> [text|-            .decl intfact(t1: number)-            .input intfact-            .decl unsignedfact(t1: unsigned)-            .input unsignedfact-            intfact(10 bor 32).-            intfact(10 bor 80 bor 10).-            unsignedfact(10 bor 32).-            |]--        it "supports bxor" $ do-          let prog = do-                Predicate int <- predicateFor @IntFact-                Predicate unsigned <- predicateFor @UnsignedFact-                int(10 `bxor` 32)-                int(10 `bxor` 80 `bxor` 10)-                unsigned(10 `bxor` 32)-          prog ==> [text|-            .decl intfact(t1: number)-            .input intfact-            .decl unsignedfact(t1: unsigned)-            .input unsignedfact-            intfact(10 bxor 32).-            intfact(10 bxor 80 bxor 10).-            unsignedfact(10 bxor 32).-            |]--        it "supports land" $ do-          let prog = do-                Predicate int <- predicateFor @IntFact-                Predicate unsigned <- predicateFor @UnsignedFact-                int(10 `land` 32)-                int(10 `land` 80 `land` 10)-                unsigned(10 `land` 32)-          prog ==> [text|-            .decl intfact(t1: number)-            .input intfact-            .decl unsignedfact(t1: unsigned)-            .input unsignedfact-            intfact(10 land 32).-            intfact(10 land 80 land 10).-            unsignedfact(10 land 32).-            |]--        it "supports lor" $ do-          let prog = do-                Predicate int <- predicateFor @IntFact-                Predicate unsigned <- predicateFor @UnsignedFact-                int(10 `lor` 32)-                int(10 `lor` 80 `lor` 10)-                unsigned(10 `lor` 32)-          prog ==> [text|-            .decl intfact(t1: number)-            .input intfact-            .decl unsignedfact(t1: unsigned)-            .input unsignedfact-            intfact(10 lor 32).-            intfact(10 lor 80 lor 10).-            unsignedfact(10 lor 32).-            |]--      describe "comparisons and equality, inequality" $ parallel $ do-        -- NOTE: the following generated programs are not correct-        -- since vars are not grounded (but is done to keep tests succinct)-        it "supports <" $ do-          let prog = do-                Predicate int <- predicateFor @IntFact-                Predicate unsigned <- predicateFor @UnsignedFact-                Predicate float <- predicateFor @FloatFact-                a <- var "a"-                b <- var "b"-                c <- var "c"-                int(a) |- a .< 10-                unsigned(b) |- b .< 10-                float(c) |- c .< 10.1-          prog ==> [text|-            .decl intfact(t1: number)-            .input intfact-            .decl unsignedfact(t1: unsigned)-            .input unsignedfact-            .decl floatfact(t1: float)-            .input floatfact-            intfact(a) :--              a < 10.-            unsignedfact(b) :--              b < 10.-            floatfact(c) :--              c < 10.1.-            |]--        it "supports <=" $ do-          let prog = do-                Predicate int <- predicateFor @IntFact-                Predicate unsigned <- predicateFor @UnsignedFact-                Predicate float <- predicateFor @FloatFact-                a <- var "a"-                b <- var "b"-                c <- var "c"-                int(a) |- a .<= 10-                unsigned(b) |- b .<= 10-                float(c) |- c .<= 10.1-          prog ==> [text|-            .decl intfact(t1: number)-            .input intfact-            .decl unsignedfact(t1: unsigned)-            .input unsignedfact-            .decl floatfact(t1: float)-            .input floatfact-            intfact(a) :--              a <= 10.-            unsignedfact(b) :--              b <= 10.-            floatfact(c) :--              c <= 10.1.-            |]--        it "supports >" $ do-          let prog = do-                Predicate int <- predicateFor @IntFact-                Predicate unsigned <- predicateFor @UnsignedFact-                Predicate float <- predicateFor @FloatFact-                a <- var "a"-                b <- var "b"-                c <- var "c"-                int(a) |- a .> 10-                unsigned(b) |- b .> 10-                float(c) |- c .> 10.1-          prog ==> [text|-            .decl intfact(t1: number)-            .input intfact-            .decl unsignedfact(t1: unsigned)-            .input unsignedfact-            .decl floatfact(t1: float)-            .input floatfact-            intfact(a) :--              a > 10.-            unsignedfact(b) :--              b > 10.-            floatfact(c) :--              c > 10.1.-            |]--        it "supports >=" $ do-          let prog = do-                Predicate int <- predicateFor @IntFact-                Predicate unsigned <- predicateFor @UnsignedFact-                Predicate float <- predicateFor @FloatFact-                a <- var "a"-                b <- var "b"-                c <- var "c"-                int(a) |- a .>= 10-                unsigned(b) |- b .>= 10-                float(c) |- c .>= 10.1-          prog ==> [text|-            .decl intfact(t1: number)-            .input intfact-            .decl unsignedfact(t1: unsigned)-            .input unsignedfact-            .decl floatfact(t1: float)-            .input floatfact-            intfact(a) :--              a >= 10.-            unsignedfact(b) :--              b >= 10.-            floatfact(c) :--              c >= 10.1.-            |]--        it "supports =" $ do-          let prog = do-                Predicate int <- predicateFor @IntFact-                Predicate unsigned <- predicateFor @UnsignedFact-                Predicate float <- predicateFor @FloatFact-                Predicate vertex <- predicateFor @Vertex-                a <- var "a"-                b <- var "b"-                c <- var "c"-                d <- var "d"-                int(a) |- a .= 10-                unsigned(b) |- b .= 10-                float(c) |- c .= 10.1-                vertex(d) |- d .= "abc"-          prog ==> [text|-            .decl intfact(t1: number)-            .input intfact-            .decl unsignedfact(t1: unsigned)-            .input unsignedfact-            .decl floatfact(t1: float)-            .input floatfact-            .decl vertex(t1: symbol)-            .output vertex-            intfact(a) :--              a = 10.-            unsignedfact(b) :--              b = 10.-            floatfact(c) :--              c = 10.1.-            vertex(d) :--              d = "abc".-            |]--        it "supports !=" $ do-          let prog = do-                Predicate int <- predicateFor @IntFact-                Predicate unsigned <- predicateFor @UnsignedFact-                Predicate float <- predicateFor @FloatFact-                Predicate vertex <- predicateFor @Vertex-                a <- var "a"-                b <- var "b"-                c <- var "c"-                d <- var "d"-                int(a) |- a .!= 10-                unsigned(b) |- b .!= 10-                float(c) |- c .!= 10.1-                vertex(d) |- d .!= "abc"-          prog ==> [text|-            .decl intfact(t1: number)-            .input intfact-            .decl unsignedfact(t1: unsigned)-            .input unsignedfact-            .decl floatfact(t1: float)-            .input floatfact-            .decl vertex(t1: symbol)-            .output vertex-            intfact(a) :--              a != 10.-            unsignedfact(b) :--              b != 10.-            floatfact(c) :--              c != 10.1.-            vertex(d) :--              d != "abc".-            |]--    describe "functors" $ parallel $ do-      it "supports max" $ do-        let prog = do-              Predicate int <- predicateFor @IntFact-              Predicate unsigned <- predicateFor @UnsignedFact-              Predicate float <- predicateFor @FloatFact-              int(max' 10 32)-              int(max' (max' 10 80) 10)-              unsigned(max' 10 32)-              float(max' 42.42 2)-        prog ==> [text|-          .decl intfact(t1: number)-          .input intfact-          .decl unsignedfact(t1: unsigned)-          .input unsignedfact-          .decl floatfact(t1: float)-          .input floatfact-          intfact(max(10, 32)).-          intfact(max(max(10, 80), 10)).-          unsignedfact(max(10, 32)).-          floatfact(max(42.42, 2.0)).-          |]--      it "supports min" $ do-        let prog = do-              Predicate int <- predicateFor @IntFact-              Predicate unsigned <- predicateFor @UnsignedFact-              Predicate float <- predicateFor @FloatFact-              int(min' 10 32)-              int(min' (min' 10 80) 10)-              unsigned(min' 10 32)-              float(min' 42.42 2)-        prog ==> [text|-          .decl intfact(t1: number)-          .input intfact-          .decl unsignedfact(t1: unsigned)-          .input unsignedfact-          .decl floatfact(t1: float)-          .input floatfact-          intfact(min(10, 32)).-          intfact(min(min(10, 80), 10)).-          unsignedfact(min(10, 32)).-          floatfact(min(42.42, 2.0)).-          |]--      it "supports cat" $ do-        let prog = do-              Predicate vertex <- predicateFor @Vertex-              vertex(cat "abc" "def")-              vertex(cat "abc" $ cat "def" "ghi")-        prog ==> [text|-          .decl vertex(t1: symbol)-          .output vertex-          vertex(cat("abc", "def")).-          vertex(cat("abc", cat("def", "ghi"))).-          |]--      it "supports contains" $ do-        let prog = do-              Predicate vertex <- predicateFor @Vertex-              Predicate int <- predicateFor @IntFact-              a <- var "a"-              b <- var "b"-              int(0) |- do-                vertex(a)-                vertex(b)-                contains a b-        prog ==> [text|-          .decl vertex(t1: symbol)-          .output vertex-          .decl intfact(t1: number)-          .input intfact-          intfact(0) :--            vertex(a),-            vertex(b),-            contains(a, b).-          |]--      it "supports match" $ do-        let prog = do-              Predicate vertex <- predicateFor @Vertex-              Predicate int <- predicateFor @IntFact-              a <- var "a"-              int(0) |- do-                vertex(a)-                match "*.a" a-        prog ==> [text|-          .decl vertex(t1: symbol)-          .output vertex-          .decl intfact(t1: number)-          .input intfact-          intfact(0) :--            vertex(a),-            match("*.a", a).-          |]--      it "supports ord" $ do-        let prog = do-              Predicate vertex <- predicateFor @Vertex-              Predicate int <- predicateFor @IntFact-              a <- var "a"-              int(ord a) |--                vertex(a)-        prog ==> [text|-          .decl vertex(t1: symbol)-          .output vertex-          .decl intfact(t1: number)-          .input intfact-          intfact(ord(a)) :--            vertex(a).-          |]--      it "supports strlen" $ do-        let prog = do-              Predicate vertex <- predicateFor @Vertex-              Predicate int <- predicateFor @IntFact-              a <- var "a"-              int(strlen a) |--                vertex(a)-        prog ==> [text|-          .decl vertex(t1: symbol)-          .output vertex-          .decl intfact(t1: number)-          .input intfact-          intfact(strlen(a)) :--            vertex(a).-          |]--      it "supports substr" $ do-        let prog = do-              Predicate vertex <- predicateFor @Vertex-              a <- var "a"-              vertex(a) |--                a .= substr "Hello" 1 3-        prog ==> [text|-          .decl vertex(t1: symbol)-          .output vertex-          vertex(a) :--            a = substr("Hello", 1, 3).-          |]---      it "supports to_number" $ do-        let prog = do-              Predicate vertex <- predicateFor @Vertex-              Predicate int <- predicateFor @IntFact-              a <- var "a"-              int(to_number a) |--                vertex(a)-        prog ==> [text|-          .decl vertex(t1: symbol)-          .output vertex-          .decl intfact(t1: number)-          .input intfact-          intfact(to_number(a)) :--            vertex(a).-          |]--      it "supports to_string" $ do-        let prog = do-              Predicate vertex <- predicateFor @Vertex-              Predicate int <- predicateFor @IntFact-              a <- var "a"-              vertex(to_string a) |--                int(a)-        prog ==> [text|-          .decl vertex(t1: symbol)-          .output vertex-          .decl intfact(t1: number)-          .input intfact-          vertex(to_string(a)) :--            intfact(a).-          |]--  describe "running DSL code directly " $ parallel $ do-    it "can run DSL with default config in interpreted mode" $ do-      let ast = do-            Predicate edge <- predicateFor @Edge-            Predicate reachable <- predicateFor @Reachable-            a <- var "a"-            b <- var "b"-            c <- var "c"-            reachable(a, b) |- edge(a, b)-            reachable(a, b) |- do-              edge(a, c)-              reachable(c, b)-          action handle = do-            let prog = fromJust handle-            I.addFacts prog [Edge "a" "b", Edge "b" "c"]-            I.run prog-            I.getFacts prog-      rs <- runSouffleInterpreted DSLProgram ast action-      rs `shouldBe` [Reachable "a" "b", Reachable "a" "c", Reachable "b" "c"]--    it "can run DSL with modified config in interpreted mode" $ do-      tmpDir <- getCanonicalTemporaryDirectory-      souffleHsDir <- createTempDirectory tmpDir "souffle-haskell"-      cfg <- I.defaultConfig-      let config = cfg { I.cfgDatalogDir = souffleHsDir }-          ast = do-            Predicate edge <- predicateFor @Edge-            Predicate reachable <- predicateFor @Reachable-            a <- var "a"-            b <- var "b"-            c <- var "c"-            reachable(a, b) |- edge(a, b)-            reachable(a, b) |- do-              edge(a, c)-              reachable(c, b)-          action handle = do-            let prog = fromJust handle-            I.addFacts prog [Edge "a" "b", Edge "b" "c"]-            I.run prog-            I.getFacts prog-      rs <- runSouffleInterpretedWith config DSLProgram ast action-      rs `shouldBe` [Reachable "a" "b", Reachable "a" "c", Reachable "b" "c"]--    it "can run DSL in compiled mode" $ do-      rs <- C.runSouffle F.CompiledProgram $ \handle -> do-        let prog = fromJust handle-        C.addFacts prog [F.Edge "a" "b", F.Edge "b" "c"]-        C.run prog-        C.getFacts prog-      rs `shouldBe` [F.Reachable "b" "c", F.Reachable "a" "c", F.Reachable "a" "b"]-
tests/Test/Language/Souffle/InterpretedSpec.hs view
@@ -23,10 +23,10 @@ data BadPath = BadPath  data Edge = Edge String String-  deriving (Eq, Show, Generic)+  deriving stock (Eq, Show, Generic)  data Reachable = Reachable String String-  deriving (Eq, Show, Generic)+  deriving stock (Eq, Show, Generic)  instance Souffle.Fact Edge where   type FactDirection Edge = 'Souffle.InputOutput@@ -40,15 +40,15 @@ instance Souffle.Marshal Reachable  instance Souffle.Program Path where-  type ProgramFacts Path = [Edge, Reachable]+  type ProgramFacts Path = '[Edge, Reachable]   programName = const "path"  instance Souffle.Program PathNoInput where-  type ProgramFacts PathNoInput = [Edge, Reachable]+  type ProgramFacts PathNoInput = '[Edge, Reachable]   programName = const "path_no_input"  instance Souffle.Program BadPath where-  type ProgramFacts BadPath = [Edge, Reachable]+  type ProgramFacts BadPath = '[Edge, Reachable]   programName = const "bad_path"  getTestTemporaryDirectory :: IO FilePath
tests/Test/Language/Souffle/MarshalSpec.hs view
@@ -1,6 +1,5 @@- {-# LANGUAGE DeriveGeneric, TypeFamilies, DataKinds, RankNTypes #-}-+{-# LANGUAGE FlexibleInstances, FlexibleContexts #-} module Test.Language.Souffle.MarshalSpec   ( module Test.Language.Souffle.MarshalSpec   ) where@@ -17,55 +16,66 @@ import Data.Word import Data.Maybe ( fromJust ) import Control.Monad.IO.Class ( liftIO )+import Control.Monad import Language.Souffle.Marshal import qualified Language.Souffle.Marshal as Souffle import qualified Language.Souffle.Class as Souffle import qualified Language.Souffle.Compiled as Compiled import qualified Language.Souffle.Interpreted as Interpreted+import Data.String (IsString)+import Data.Void (Void)   data Edge = Edge String String-  deriving (Eq, Show, Generic)+  deriving stock (Eq, Show, Generic)  newtype EdgeUInt = EdgeUInt Word32-  deriving (Eq, Show, Generic)+  deriving stock (Eq, Show, Generic)  newtype FloatValue = FloatValue Float-  deriving (Eq, Show, Generic)+  deriving stock (Eq, Show, Generic)  data EdgeStrict = EdgeStrict !String !String-  deriving (Eq, Show, Generic)+  deriving stock (Eq, Show, Generic)  data EdgeUnpacked   = EdgeUnpacked {-# UNPACK #-} !Int32 {-# UNPACK #-} !Int32-  deriving (Eq, Show, Generic)+  deriving stock (Eq, Show, Generic)  type Vertex = Text type Vertex' = Text  data EdgeSynonyms = EdgeSynonyms Vertex Vertex-  deriving (Eq, Show, Generic)+  deriving stock (Eq, Show, Generic)  data EdgeMultipleSynonyms = EdgeMultipleSynonyms Vertex Vertex'-  deriving (Eq, Show, Generic)+  deriving stock (Eq, Show, Generic)  data EdgeMixed = EdgeMixed Text Vertex-  deriving (Eq, Show, Generic)+  deriving stock (Eq, Show, Generic)  data EdgeRecord   = EdgeRecord   { fromNode :: Text   , toNode :: Text-  } deriving (Eq, Show, Generic)+  } deriving stock (Eq, Show, Generic)  data IntsAndStrings = IntsAndStrings Text Int32 Text-  deriving (Eq, Show, Generic)+  deriving stock (Eq, Show, Generic)  data LargeRecord   = LargeRecord Int32 Int32 Int32 Int32-  deriving (Eq, Show, Generic)+  deriving stock (Eq, Show, Generic) +newtype NestedNewtype = NestedNewtype LargeRecord+  deriving stock (Eq, Show, Generic) +data Pair = Pair Int32 Int32+  deriving stock (Eq, Show, Generic)++data NestedRecord = NestedRecord Pair Pair+  deriving stock (Eq, Show, Generic)+ instance Marshal Edge instance Marshal EdgeUInt instance Marshal FloatValue@@ -77,27 +87,29 @@ instance Marshal EdgeRecord instance Marshal IntsAndStrings instance Marshal LargeRecord-+instance Marshal Pair+instance Marshal NestedNewtype+instance Marshal NestedRecord  data RoundTrip = RoundTrip  newtype StringFact = StringFact String-  deriving (Eq, Show, Generic)+  deriving stock (Eq, Show, Generic)  newtype TextFact = TextFact T.Text-  deriving (Eq, Show, Generic)+  deriving stock (Eq, Show, Generic)  newtype LazyTextFact = LazyTextFact TL.Text-  deriving (Eq, Show, Generic)+  deriving stock (Eq, Show, Generic)  newtype Int32Fact = Int32Fact Int32-  deriving (Eq, Show, Generic)+  deriving stock (Eq, Show, Generic)  newtype Word32Fact = Word32Fact Word32-  deriving (Eq, Show, Generic)+  deriving stock (Eq, Show, Generic)  newtype FloatFact = FloatFact Float-  deriving (Eq, Show, Generic)+  deriving stock (Eq, Show, Generic)  instance Souffle.Fact StringFact where   type FactDirection StringFact = 'Souffle.InputOutput@@ -123,6 +135,14 @@   type FactDirection FloatFact = 'Souffle.InputOutput   factName = const "float_fact" +instance Souffle.Fact NestedNewtype where+  type FactDirection NestedNewtype = 'Souffle.InputOutput+  factName = const "large_record"++instance Souffle.Fact NestedRecord where+  type FactDirection NestedRecord = 'Souffle.InputOutput+  factName = const "large_record"+ instance Souffle.Marshal StringFact instance Souffle.Marshal TextFact instance Souffle.Marshal LazyTextFact@@ -132,15 +152,89 @@  instance Souffle.Program RoundTrip where   type ProgramFacts RoundTrip =-    [StringFact, TextFact, LazyTextFact, Int32Fact, Word32Fact, FloatFact]+    '[StringFact, TextFact, LazyTextFact, Int32Fact, Word32Fact, FloatFact, NestedNewtype, NestedRecord]   programName = const "round_trip"  type RoundTripAction   = forall a. Souffle.Fact a   => Souffle.ContainsInputFact RoundTrip a   => Souffle.ContainsOutputFact RoundTrip a+  => Compiled.Submit a   => a -> PropertyT IO a ++data EdgeCases = EdgeCases++data EmptyStrings a+  = EmptyStrings a a Int32+  deriving stock (Eq, Show, Generic)++newtype LongStrings a+  = LongStrings a+  deriving stock (Eq, Show, Generic)++newtype Unicode a+  = Unicode a+  deriving stock (Eq, Show, Generic)++data NoStrings a = NoStrings Word32 Int32 Float+  deriving stock (Eq, Show, Generic)++instance Souffle.Program EdgeCases where+  type ProgramFacts EdgeCases =+    '[ EmptyStrings String, EmptyStrings T.Text, EmptyStrings TL.Text+    , LongStrings String, LongStrings T.Text, LongStrings TL.Text+    , Unicode String, Unicode T.Text, Unicode TL.Text+    , NoStrings Void+    ]+  programName = const "edge_cases"++instance Souffle.Fact (EmptyStrings String) where+  type FactDirection (EmptyStrings String) = 'Souffle.InputOutput+  factName = const "empty_strings"+instance Souffle.Fact (EmptyStrings T.Text) where+  type FactDirection (EmptyStrings T.Text) = 'Souffle.InputOutput+  factName = const "empty_strings"+instance Souffle.Fact (EmptyStrings TL.Text) where+  type FactDirection (EmptyStrings TL.Text) = 'Souffle.InputOutput+  factName = const "empty_strings"++instance Souffle.Fact (LongStrings String) where+  type FactDirection (LongStrings String) = 'Souffle.InputOutput+  factName = const "long_strings"+instance Souffle.Fact (LongStrings T.Text) where+  type FactDirection (LongStrings T.Text) = 'Souffle.InputOutput+  factName = const "long_strings"+instance Souffle.Fact (LongStrings TL.Text) where+  type FactDirection (LongStrings TL.Text) = 'Souffle.InputOutput+  factName = const "long_strings"++instance Souffle.Fact (Unicode String) where+  type FactDirection (Unicode String) = 'Souffle.InputOutput+  factName = const "unicode"+instance Souffle.Fact (Unicode T.Text) where+  type FactDirection (Unicode T.Text) = 'Souffle.InputOutput+  factName = const "unicode"+instance Souffle.Fact (Unicode TL.Text) where+  type FactDirection (Unicode TL.Text) = 'Souffle.InputOutput+  factName = const "unicode"++instance Souffle.Fact (NoStrings a) where+  type FactDirection (NoStrings _) = 'Souffle.InputOutput+  factName = const "no_strings"++instance Marshal (EmptyStrings String)+instance Marshal (EmptyStrings T.Text)+instance Marshal (EmptyStrings TL.Text)+instance Marshal (LongStrings String)+instance Marshal (LongStrings T.Text)+instance Marshal (LongStrings TL.Text)+instance Marshal (Unicode String)+instance Marshal (Unicode T.Text)+instance Marshal (Unicode TL.Text)+instance Marshal (NoStrings a)++ spec :: Spec spec = describe "Marshalling" $ parallel $ do   describe "Auto-deriving marshalling code" $@@ -148,58 +242,283 @@       -- If this file compiles, then the test has already passed       42 `shouldBe` 42 -  describe "data transfer between Haskell and Souffle" $ parallel $ do-    let roundTripTests :: RoundTripAction -> Spec-        roundTripTests run = do-          it "can serialize and deserialize String values" $ hedgehog $ do-            str <- forAll $ Gen.string (Range.linear 0 10) Gen.unicode-            let fact = StringFact str-            fact' <- run fact-            fact === fact'+  roundTripSpecs+  edgeCaseSpecs -          it "can serialize and deserialize lazy Text" $ hedgehog $ do-            str <- forAll $ Gen.string (Range.linear 0 10) Gen.unicode-            let fact = LazyTextFact (TL.pack str)-            fact' <- run fact-            fact === fact'+roundTripSpecs :: Spec+roundTripSpecs = describe "data transfer between Haskell and Souffle" $ parallel $ do+  let roundTripTests :: RoundTripAction -> Spec+      roundTripTests run = do+        it "can serialize and deserialize String values" $ hedgehog $ do+          str <- forAll $ Gen.string (Range.linear 0 10) Gen.unicode+          let fact = StringFact str+          fact' <- run fact+          fact === fact' -          it "can serialize and deserialize strict Text values" $ hedgehog $ do-            str <- forAll $ Gen.text (Range.linear 0 10) Gen.unicode-            let fact = TextFact str-            fact' <- run fact-            fact === fact'+        it "can serialize and deserialize lazy Text values" $ hedgehog $ do+          str <- forAll $ Gen.string (Range.linear 0 10) Gen.unicode+          let fact = LazyTextFact (TL.pack str)+          fact' <- run fact+          fact === fact' -          it "can serialize and deserialize Int32 values" $ hedgehog $ do-            x <- forAll $ Gen.int32 (Range.linear minBound maxBound)-            let fact = Int32Fact x-            fact' <- run fact-            fact === fact'+        it "can serialize and deserialize strict Text values" $ hedgehog $ do+          str <- forAll $ Gen.text (Range.linear 0 10) Gen.unicode+          let fact = TextFact str+          fact' <- run fact+          fact === fact' -          it "can serialize and deserialize Word32 values" $ hedgehog $ do-            x <- forAll $ Gen.word32 (Range.linear minBound maxBound)-            let fact = Word32Fact x-            fact' <- run fact-            fact === fact'+        it "can serialize and deserialize Int32 values" $ hedgehog $ do+          x <- forAll $ Gen.int32 (Range.linear minBound maxBound)+          let fact = Int32Fact x+          fact' <- run fact+          fact === fact' -          it "can serialize and deserialize Float values" $ hedgehog $ do-            let epsilon = 1e-6-                fmin = -1e9-                fmax =  1e9-            x <- forAll $ Gen.float (Range.exponentialFloat fmin fmax)-            let fact = FloatFact x-            FloatFact x' <- run fact-            (abs (x' - x) < epsilon) === True+        it "can serialize and deserialize Word32 values" $ hedgehog $ do+          x <- forAll $ Gen.word32 (Range.linear minBound maxBound)+          let fact = Word32Fact x+          fact' <- run fact+          fact === fact' -    describe "interpreted mode" $ parallel $-      roundTripTests $ \fact -> liftIO $ Interpreted.runSouffle RoundTrip $ \handle -> do+        it "can serialize and deserialize Float values" $ hedgehog $ do+          let epsilon = 1e-6+              fmin = -1e9+              fmax =  1e9+          x <- forAll $ Gen.float (Range.exponentialFloat fmin fmax)+          let fact = FloatFact x+          FloatFact x' <- run fact+          (abs (x' - x) < epsilon) === True++        it "can serialize and deserialize newtypes" $ hedgehog $ do+          a <- forAll $ Gen.int32 (Range.linear minBound maxBound)+          b <- forAll $ Gen.int32 (Range.linear minBound maxBound)+          c <- forAll $ Gen.int32 (Range.linear minBound maxBound)+          d <- forAll $ Gen.int32 (Range.linear minBound maxBound)+          let fact = NestedNewtype $ LargeRecord a b c d+          fact' <- run fact+          fact === fact'++        it "can serialize and deserialize nested product types" $ hedgehog $ do+          a <- forAll $ Gen.int32 (Range.linear minBound maxBound)+          b <- forAll $ Gen.int32 (Range.linear minBound maxBound)+          c <- forAll $ Gen.int32 (Range.linear minBound maxBound)+          d <- forAll $ Gen.int32 (Range.linear minBound maxBound)+          let fact = NestedRecord (Pair a b) (Pair c d)+          fact' <- run fact+          fact === fact'+++  describe "interpreted mode" $ parallel $+    roundTripTests $ \fact -> liftIO $ Interpreted.runSouffle RoundTrip $ \handle -> do+      let prog = fromJust handle+      Interpreted.addFact prog fact+      Interpreted.run prog+      Prelude.head <$> Interpreted.getFacts prog++  describe "compiled mode" $ parallel $+    roundTripTests $ \fact -> liftIO $ Compiled.runSouffle RoundTrip $ \handle -> do+      let prog = fromJust handle+      Compiled.addFact prog fact+      Compiled.run prog+      Prelude.head <$> Compiled.getFacts prog++edgeCaseSpecs :: Spec+edgeCaseSpecs = describe "edge cases" $ parallel $ do+  let longString :: IsString a => a+      longString = "long_string_from_DL:...............................................................................................................................................................................................................................................................................................end"++      getFactsI :: forall f a. (Souffle.Fact (f a), Souffle.ContainsOutputFact EdgeCases (f a)) => IO [f a]+      getFactsI = Interpreted.runSouffle EdgeCases $ \handle -> do         let prog = fromJust handle-        Interpreted.addFact prog fact         Interpreted.run prog-        Prelude.head <$> Interpreted.getFacts prog+        Interpreted.getFacts prog+      getFactsC :: forall f a. (Souffle.Fact (f a), Souffle.ContainsOutputFact EdgeCases (f a)) => IO [f a]+      getFactsC = Compiled.runSouffle EdgeCases $ \handle -> do+        let prog = fromJust handle+        Compiled.run prog+        Prelude.reverse <$> Compiled.getFacts prog -    describe "compiled mode" $ parallel $-      roundTripTests $ \fact -> liftIO $ Compiled.runSouffle RoundTrip $ \handle -> do+      getUnicodeFactsI :: forall a. (IsString a, Eq a, Souffle.Fact (Unicode a), Souffle.ContainsOutputFact EdgeCases (Unicode a))+                        => IO ([Unicode a], Maybe (Unicode a), Maybe (Unicode a))+      getUnicodeFactsI = Interpreted.runSouffle EdgeCases $ \handle -> do         let prog = fromJust handle-        Compiled.addFact prog fact+        Interpreted.run prog+        (,,) <$> Interpreted.getFacts prog+              <*> Interpreted.findFact prog (Unicode "⌀")  -- \x2300 iso \x2200+              <*> Interpreted.findFact prog (Unicode "≂")  -- \x2242 iso \x2200++      getUnicodeFactsC :: forall a. (IsString a, Eq a, Souffle.Fact (Unicode a), Souffle.ContainsOutputFact EdgeCases (Unicode a), Compiled.Submit (Unicode a))+                        => IO ([Unicode a], Maybe (Unicode a), Maybe (Unicode a))+      getUnicodeFactsC = Compiled.runSouffle EdgeCases $ \handle -> do+        let prog = fromJust handle         Compiled.run prog-        Prelude.head <$> Compiled.getFacts prog+        (,,) <$> (Prelude.reverse <$> Compiled.getFacts prog)+              <*> Compiled.findFact prog (Unicode "⌀")  -- \x2300 iso \x2200+              <*> Compiled.findFact prog (Unicode "≂")  -- \x2242 iso \x2200++      addAndGetFactsI :: Souffle.Fact (f a)+                      => Souffle.ContainsInputFact EdgeCases (f a)+                      => Souffle.ContainsOutputFact EdgeCases (f a)+                      => [f a] -> IO [f a]+      addAndGetFactsI fs = Interpreted.runSouffle EdgeCases $ \handle -> do+        let prog = fromJust handle+        Interpreted.addFacts prog fs+        Interpreted.run prog+        Interpreted.getFacts prog+      addAndGetFactsC :: Souffle.Fact (f a)+                      => Souffle.ContainsInputFact EdgeCases (f a)+                      => Souffle.ContainsOutputFact EdgeCases (f a)+                      => Compiled.Submit (f a)+                      => [f a] -> IO [f a]+      addAndGetFactsC fs = Compiled.runSouffle EdgeCases $ \handle -> do+        let prog = fromJust handle+        Compiled.addFacts prog fs+        Compiled.run prog+        Prelude.reverse <$> Compiled.getFacts prog+++      runTests :: (forall f a. (Souffle.Fact (f a), Souffle.ContainsOutputFact EdgeCases (f a)) => IO [f a])+                -> (forall a. (IsString a, Eq a, Souffle.Fact (Unicode a), Souffle.ContainsOutputFact EdgeCases (Unicode a), Compiled.Submit (Unicode a))+                    => IO ([Unicode a], Maybe (Unicode a), Maybe (Unicode a)))+                -> (forall f a. Souffle.Fact (f a)+                    => Souffle.ContainsInputFact EdgeCases (f a)+                    => Souffle.ContainsOutputFact EdgeCases (f a)+                    => Compiled.Submit (f a)+                    => [f a] -> IO [f a])+                -> Spec+      runTests getFacts getUnicodeFacts addAndGetFacts = do+        it "correctly marshals facts with number-like types" $ do+          facts <- getFacts+          (facts :: [NoStrings Void])+            `shouldBe` [ NoStrings 42 (-100) 1.5+                       , NoStrings 123 (-456) 3.14+                       ]++        it "correctly marshals facts with empty Strings" $ do+          facts <- getFacts+          (facts :: [EmptyStrings String])+            `shouldBe` [ EmptyStrings "" "" 42+                       , EmptyStrings "" "abc" 42+                       , EmptyStrings "abc" "" 42+                       ]++        it "correctly marshals facts with empty Texts" $ do+          facts <- getFacts+          (facts :: [EmptyStrings T.Text])+            `shouldBe` [ EmptyStrings "" "" 42+                        , EmptyStrings "" "abc" 42+                        , EmptyStrings "abc" "" 42+                        ]++        it "correctly marshals facts with empty lazy Texts" $ do+          facts <- getFacts+          (facts :: [EmptyStrings TL.Text])+            `shouldBe` [ EmptyStrings "" "" 42+                        , EmptyStrings "" "abc" 42+                        , EmptyStrings "abc" "" 42+                        ]++        it "correctly marshals facts really with long (>255 chars) String" $ do+          facts <- getFacts+          (facts :: [LongStrings String]) `shouldBe` [ LongStrings longString ]++        it "correctly marshals facts really with long (>255 chars) Text" $ do+          facts <- getFacts+          (facts :: [LongStrings T.Text]) `shouldBe` [ LongStrings longString ]++        it "correctly marshals facts really with long (>255 chars) lazy Text" $ do+          facts <- getFacts+          (facts :: [LongStrings TL.Text]) `shouldBe` [ LongStrings longString ]++        it "correctly marshals facts containing unicode characters (String)" $ do+          results <- getUnicodeFacts+          results `shouldBe`+            ( [ Unicode ("∀" :: String), Unicode "∀∀" ]+            , Nothing :: Maybe (Unicode String)+            , Nothing :: Maybe (Unicode String)+            )++        it "correctly marshals facts containing unicode characters (Text)" $ do+          results <- getUnicodeFacts+          results `shouldBe`+            ( [ Unicode ("∀" :: T.Text), Unicode "∀∀" ]+            , Nothing :: Maybe (Unicode T.Text)+            , Nothing :: Maybe (Unicode T.Text)+            )++        it "correctly marshals facts containing unicode characters (lazy Text)" $ do+          results <- getUnicodeFacts+          results `shouldBe`+            ( [ Unicode ("∀" :: TL.Text), Unicode "∀∀" ]+            , Nothing :: Maybe (Unicode TL.Text)+            , Nothing :: Maybe (Unicode TL.Text)+            )++        it "correctly marshals empty strings back and forth (Strings)" $ do+          let facts :: [EmptyStrings String]+              facts = [EmptyStrings "" "" 1, EmptyStrings "" "" 42, EmptyStrings "" "abc" 2, EmptyStrings "" "abc" 42, EmptyStrings "abc" "" 3, EmptyStrings "abc" "" 42]+          facts' <- addAndGetFacts facts+          facts' `shouldBe` facts++        it "correctly marshals empty strings back and forth (Text)" $ do+          let facts :: [EmptyStrings T.Text]+              facts = [EmptyStrings "" "" 1, EmptyStrings "" "" 42, EmptyStrings "" "abc" 2, EmptyStrings "" "abc" 42, EmptyStrings "abc" "" 3, EmptyStrings "abc" "" 42]+          facts' <- addAndGetFacts facts+          facts' `shouldBe` facts++        it "correctly marshals empty strings back and forth (lazy Text)" $ do+          let facts :: [EmptyStrings TL.Text]+              facts = [EmptyStrings "" "" 1, EmptyStrings "" "" 42, EmptyStrings "" "abc" 2, EmptyStrings "" "abc" 42, EmptyStrings "abc" "" 3, EmptyStrings "abc" "" 42]+          facts' <- addAndGetFacts facts+          facts' `shouldBe` facts++        it "correctly marshals unicode back and forth (Strings)" $ do+          let facts :: [Unicode String]+              facts = [Unicode "∀", Unicode "∀∀", Unicode "≂", Unicode "⌀", Unicode "⌀⌀"]+          facts' <- addAndGetFacts facts+          facts' `shouldBe` facts++        it "correctly marshals unicode back and forth (Text)" $ do+          let facts :: [Unicode T.Text]+              facts = [Unicode "∀", Unicode "∀∀", Unicode "≂", Unicode "⌀", Unicode "⌀⌀"]+          facts' <- addAndGetFacts facts+          facts' `shouldBe` facts++        it "correctly marshals unicode back and forth (lazy Text)" $ do+          let facts :: [Unicode TL.Text]+              facts = [Unicode "∀", Unicode "∀∀", Unicode "≂", Unicode "⌀", Unicode "⌀⌀"]+          facts' <- addAndGetFacts facts+          facts' `shouldBe` facts++        it "correctly marshals really long strings back and forth (Strings)" $ do+          let facts :: [LongStrings String]+              facts = [LongStrings longString, LongStrings $ join $ Prelude.replicate 10000 "abc"]+          facts' <- addAndGetFacts facts+          facts' `shouldBe` facts++        it "correctly marshals really long strings back and forth (Text)" $ do+          let facts :: [LongStrings T.Text]+              facts = [LongStrings longString, LongStrings $ T.pack $ join $ Prelude.replicate 10000 "abc"]+          facts' <- addAndGetFacts facts+          facts' `shouldBe` facts++        it "correctly marshals really long strings back and forth (lazy Text)" $ do+          let facts :: [LongStrings TL.Text]+              facts = [LongStrings longString, LongStrings $ TL.pack $ join $ Prelude.replicate 10000 "abc"]+          facts' <- addAndGetFacts facts+          facts' `shouldBe` facts++        it "correctly marshals facts with number-like types" $ do+          let facts :: [NoStrings Void]+              facts = [ NoStrings 42 (-100) 1.5+                      , NoStrings 123 (-456) 3.14+                      , NoStrings 789 (-789) 1000.123+                      , NoStrings 0x12345678 (-1000) 1234.56789+                      ]+          facts' <- addAndGetFacts facts+          facts' `shouldBe` facts++  describe "interpreted mode" $ parallel $ do+    runTests getFactsI getUnicodeFactsI addAndGetFactsI++  describe "compiled mode" $ parallel $ do+    runTests getFactsC getUnicodeFactsC addAndGetFactsC
+ tests/fixtures/edge_cases.cpp view
@@ -0,0 +1,709 @@++#include "souffle/CompiledSouffle.h"++namespace functors {+ extern "C" {+}+}++namespace souffle {+static const RamDomain RAM_BIT_SHIFT_MASK = RAM_DOMAIN_SIZE - 1;+struct t_btree_iii__0_1_2__111 {+static constexpr Relation::arity_type Arity = 3;+using t_tuple = Tuple<RamDomain, 3>;+struct t_comparator_0{+ int operator()(const t_tuple& a, const t_tuple& b) const {+  return (ramBitCast<RamSigned>(a[0]) < ramBitCast<RamSigned>(b[0])) ? -1 : (ramBitCast<RamSigned>(a[0]) > ramBitCast<RamSigned>(b[0])) ? 1 :((ramBitCast<RamSigned>(a[1]) < ramBitCast<RamSigned>(b[1])) ? -1 : (ramBitCast<RamSigned>(a[1]) > ramBitCast<RamSigned>(b[1])) ? 1 :((ramBitCast<RamSigned>(a[2]) < ramBitCast<RamSigned>(b[2])) ? -1 : (ramBitCast<RamSigned>(a[2]) > ramBitCast<RamSigned>(b[2])) ? 1 :(0)));+ }+bool less(const t_tuple& a, const t_tuple& b) const {+  return (ramBitCast<RamSigned>(a[0]) < ramBitCast<RamSigned>(b[0]))|| ((ramBitCast<RamSigned>(a[0]) == ramBitCast<RamSigned>(b[0])) && ((ramBitCast<RamSigned>(a[1]) < ramBitCast<RamSigned>(b[1]))|| ((ramBitCast<RamSigned>(a[1]) == ramBitCast<RamSigned>(b[1])) && ((ramBitCast<RamSigned>(a[2]) < ramBitCast<RamSigned>(b[2]))))));+ }+bool equal(const t_tuple& a, const t_tuple& b) const {+return (ramBitCast<RamSigned>(a[0]) == ramBitCast<RamSigned>(b[0]))&&(ramBitCast<RamSigned>(a[1]) == ramBitCast<RamSigned>(b[1]))&&(ramBitCast<RamSigned>(a[2]) == ramBitCast<RamSigned>(b[2]));+ }+};+using t_ind_0 = btree_set<t_tuple,t_comparator_0>;+t_ind_0 ind_0;+using iterator = t_ind_0::iterator;+struct context {+t_ind_0::operation_hints hints_0_lower;+t_ind_0::operation_hints hints_0_upper;+};+context createContext() { return context(); }+bool insert(const t_tuple& t) {+context h;+return insert(t, h);+}+bool insert(const t_tuple& t, context& h) {+if (ind_0.insert(t, h.hints_0_lower)) {+return true;+} else return false;+}+bool insert(const RamDomain* ramDomain) {+RamDomain data[3];+std::copy(ramDomain, ramDomain + 3, data);+const t_tuple& tuple = reinterpret_cast<const t_tuple&>(data);+context h;+return insert(tuple, h);+}+bool insert(RamDomain a0,RamDomain a1,RamDomain a2) {+RamDomain data[3] = {a0,a1,a2};+return insert(data);+}+bool contains(const t_tuple& t, context& h) const {+return ind_0.contains(t, h.hints_0_lower);+}+bool contains(const t_tuple& t) const {+context h;+return contains(t, h);+}+std::size_t size() const {+return ind_0.size();+}+iterator find(const t_tuple& t, context& h) const {+return ind_0.find(t, h.hints_0_lower);+}+iterator find(const t_tuple& t) const {+context h;+return find(t, h);+}+range<iterator> lowerUpperRange_000(const t_tuple& /* lower */, const t_tuple& /* upper */, context& /* h */) const {+return range<iterator>(ind_0.begin(),ind_0.end());+}+range<iterator> lowerUpperRange_000(const t_tuple& /* lower */, const t_tuple& /* upper */) const {+return range<iterator>(ind_0.begin(),ind_0.end());+}+range<t_ind_0::iterator> lowerUpperRange_111(const t_tuple& lower, const t_tuple& upper, context& h) const {+t_comparator_0 comparator;+int cmp = comparator(lower, upper);+if (cmp == 0) {+    auto pos = ind_0.find(lower, h.hints_0_lower);+    auto fin = ind_0.end();+    if (pos != fin) {fin = pos; ++fin;}+    return make_range(pos, fin);+}+if (cmp > 0) {+    return make_range(ind_0.end(), ind_0.end());+}+return make_range(ind_0.lower_bound(lower, h.hints_0_lower), ind_0.upper_bound(upper, h.hints_0_upper));+}+range<t_ind_0::iterator> lowerUpperRange_111(const t_tuple& lower, const t_tuple& upper) const {+context h;+return lowerUpperRange_111(lower,upper,h);+}+bool empty() const {+return ind_0.empty();+}+std::vector<range<iterator>> partition() const {+return ind_0.getChunks(400);+}+void purge() {+ind_0.clear();+}+iterator begin() const {+return ind_0.begin();+}+iterator end() const {+return ind_0.end();+}+void printStatistics(std::ostream& o) const {+o << " arity 3 direct b-tree index 0 lex-order [0,1,2]\n";+ind_0.printStats(o);+}+};+struct t_btree_i__0__1 {+static constexpr Relation::arity_type Arity = 1;+using t_tuple = Tuple<RamDomain, 1>;+struct t_comparator_0{+ int operator()(const t_tuple& a, const t_tuple& b) const {+  return (ramBitCast<RamSigned>(a[0]) < ramBitCast<RamSigned>(b[0])) ? -1 : (ramBitCast<RamSigned>(a[0]) > ramBitCast<RamSigned>(b[0])) ? 1 :(0);+ }+bool less(const t_tuple& a, const t_tuple& b) const {+  return (ramBitCast<RamSigned>(a[0]) < ramBitCast<RamSigned>(b[0]));+ }+bool equal(const t_tuple& a, const t_tuple& b) const {+return (ramBitCast<RamSigned>(a[0]) == ramBitCast<RamSigned>(b[0]));+ }+};+using t_ind_0 = btree_set<t_tuple,t_comparator_0>;+t_ind_0 ind_0;+using iterator = t_ind_0::iterator;+struct context {+t_ind_0::operation_hints hints_0_lower;+t_ind_0::operation_hints hints_0_upper;+};+context createContext() { return context(); }+bool insert(const t_tuple& t) {+context h;+return insert(t, h);+}+bool insert(const t_tuple& t, context& h) {+if (ind_0.insert(t, h.hints_0_lower)) {+return true;+} else return false;+}+bool insert(const RamDomain* ramDomain) {+RamDomain data[1];+std::copy(ramDomain, ramDomain + 1, data);+const t_tuple& tuple = reinterpret_cast<const t_tuple&>(data);+context h;+return insert(tuple, h);+}+bool insert(RamDomain a0) {+RamDomain data[1] = {a0};+return insert(data);+}+bool contains(const t_tuple& t, context& h) const {+return ind_0.contains(t, h.hints_0_lower);+}+bool contains(const t_tuple& t) const {+context h;+return contains(t, h);+}+std::size_t size() const {+return ind_0.size();+}+iterator find(const t_tuple& t, context& h) const {+return ind_0.find(t, h.hints_0_lower);+}+iterator find(const t_tuple& t) const {+context h;+return find(t, h);+}+range<iterator> lowerUpperRange_0(const t_tuple& /* lower */, const t_tuple& /* upper */, context& /* h */) const {+return range<iterator>(ind_0.begin(),ind_0.end());+}+range<iterator> lowerUpperRange_0(const t_tuple& /* lower */, const t_tuple& /* upper */) const {+return range<iterator>(ind_0.begin(),ind_0.end());+}+range<t_ind_0::iterator> lowerUpperRange_1(const t_tuple& lower, const t_tuple& upper, context& h) const {+t_comparator_0 comparator;+int cmp = comparator(lower, upper);+if (cmp == 0) {+    auto pos = ind_0.find(lower, h.hints_0_lower);+    auto fin = ind_0.end();+    if (pos != fin) {fin = pos; ++fin;}+    return make_range(pos, fin);+}+if (cmp > 0) {+    return make_range(ind_0.end(), ind_0.end());+}+return make_range(ind_0.lower_bound(lower, h.hints_0_lower), ind_0.upper_bound(upper, h.hints_0_upper));+}+range<t_ind_0::iterator> lowerUpperRange_1(const t_tuple& lower, const t_tuple& upper) const {+context h;+return lowerUpperRange_1(lower,upper,h);+}+bool empty() const {+return ind_0.empty();+}+std::vector<range<iterator>> partition() const {+return ind_0.getChunks(400);+}+void purge() {+ind_0.clear();+}+iterator begin() const {+return ind_0.begin();+}+iterator end() const {+return ind_0.end();+}+void printStatistics(std::ostream& o) const {+o << " arity 1 direct b-tree index 0 lex-order [0]\n";+ind_0.printStats(o);+}+};+struct t_btree_uif__0_1_2__111 {+static constexpr Relation::arity_type Arity = 3;+using t_tuple = Tuple<RamDomain, 3>;+struct t_comparator_0{+ int operator()(const t_tuple& a, const t_tuple& b) const {+  return (ramBitCast<RamUnsigned>(a[0]) < ramBitCast<RamUnsigned>(b[0])) ? -1 : (ramBitCast<RamUnsigned>(a[0]) > ramBitCast<RamUnsigned>(b[0])) ? 1 :((ramBitCast<RamSigned>(a[1]) < ramBitCast<RamSigned>(b[1])) ? -1 : (ramBitCast<RamSigned>(a[1]) > ramBitCast<RamSigned>(b[1])) ? 1 :((ramBitCast<RamFloat>(a[2]) < ramBitCast<RamFloat>(b[2])) ? -1 : (ramBitCast<RamFloat>(a[2]) > ramBitCast<RamFloat>(b[2])) ? 1 :(0)));+ }+bool less(const t_tuple& a, const t_tuple& b) const {+  return (ramBitCast<RamUnsigned>(a[0]) < ramBitCast<RamUnsigned>(b[0]))|| ((ramBitCast<RamUnsigned>(a[0]) == ramBitCast<RamUnsigned>(b[0])) && ((ramBitCast<RamSigned>(a[1]) < ramBitCast<RamSigned>(b[1]))|| ((ramBitCast<RamSigned>(a[1]) == ramBitCast<RamSigned>(b[1])) && ((ramBitCast<RamFloat>(a[2]) < ramBitCast<RamFloat>(b[2]))))));+ }+bool equal(const t_tuple& a, const t_tuple& b) const {+return (ramBitCast<RamUnsigned>(a[0]) == ramBitCast<RamUnsigned>(b[0]))&&(ramBitCast<RamSigned>(a[1]) == ramBitCast<RamSigned>(b[1]))&&(ramBitCast<RamFloat>(a[2]) == ramBitCast<RamFloat>(b[2]));+ }+};+using t_ind_0 = btree_set<t_tuple,t_comparator_0>;+t_ind_0 ind_0;+using iterator = t_ind_0::iterator;+struct context {+t_ind_0::operation_hints hints_0_lower;+t_ind_0::operation_hints hints_0_upper;+};+context createContext() { return context(); }+bool insert(const t_tuple& t) {+context h;+return insert(t, h);+}+bool insert(const t_tuple& t, context& h) {+if (ind_0.insert(t, h.hints_0_lower)) {+return true;+} else return false;+}+bool insert(const RamDomain* ramDomain) {+RamDomain data[3];+std::copy(ramDomain, ramDomain + 3, data);+const t_tuple& tuple = reinterpret_cast<const t_tuple&>(data);+context h;+return insert(tuple, h);+}+bool insert(RamDomain a0,RamDomain a1,RamDomain a2) {+RamDomain data[3] = {a0,a1,a2};+return insert(data);+}+bool contains(const t_tuple& t, context& h) const {+return ind_0.contains(t, h.hints_0_lower);+}+bool contains(const t_tuple& t) const {+context h;+return contains(t, h);+}+std::size_t size() const {+return ind_0.size();+}+iterator find(const t_tuple& t, context& h) const {+return ind_0.find(t, h.hints_0_lower);+}+iterator find(const t_tuple& t) const {+context h;+return find(t, h);+}+range<iterator> lowerUpperRange_000(const t_tuple& /* lower */, const t_tuple& /* upper */, context& /* h */) const {+return range<iterator>(ind_0.begin(),ind_0.end());+}+range<iterator> lowerUpperRange_000(const t_tuple& /* lower */, const t_tuple& /* upper */) const {+return range<iterator>(ind_0.begin(),ind_0.end());+}+range<t_ind_0::iterator> lowerUpperRange_111(const t_tuple& lower, const t_tuple& upper, context& h) const {+t_comparator_0 comparator;+int cmp = comparator(lower, upper);+if (cmp == 0) {+    auto pos = ind_0.find(lower, h.hints_0_lower);+    auto fin = ind_0.end();+    if (pos != fin) {fin = pos; ++fin;}+    return make_range(pos, fin);+}+if (cmp > 0) {+    return make_range(ind_0.end(), ind_0.end());+}+return make_range(ind_0.lower_bound(lower, h.hints_0_lower), ind_0.upper_bound(upper, h.hints_0_upper));+}+range<t_ind_0::iterator> lowerUpperRange_111(const t_tuple& lower, const t_tuple& upper) const {+context h;+return lowerUpperRange_111(lower,upper,h);+}+bool empty() const {+return ind_0.empty();+}+std::vector<range<iterator>> partition() const {+return ind_0.getChunks(400);+}+void purge() {+ind_0.clear();+}+iterator begin() const {+return ind_0.begin();+}+iterator end() const {+return ind_0.end();+}+void printStatistics(std::ostream& o) const {+o << " arity 3 direct b-tree index 0 lex-order [0,1,2]\n";+ind_0.printStats(o);+}+};++class Sf_edge_cases : public SouffleProgram {+private:+static inline std::string substr_wrapper(const std::string& str, std::size_t idx, std::size_t len) {+   std::string result; +   try { result = str.substr(idx,len); } catch(...) { +     std::cerr << "warning: wrong index position provided by substr(\"";+     std::cerr << str << "\"," << (int32_t)idx << "," << (int32_t)len << ") functor.\n";+   } return result;+}+public:+// -- initialize symbol table --+SymbolTableImpl symTable{+	R"_()_",+	R"_(abc)_",+	R"_(long_string_from_DL:...............................................................................................................................................................................................................................................................................................end)_",+	R"_(∀)_",+	R"_(∀∀)_",+};// -- initialize record table --+SpecializedRecordTable<0> recordTable{};+// -- Table: empty_strings+Own<t_btree_iii__0_1_2__111> rel_1_empty_strings = mk<t_btree_iii__0_1_2__111>();+souffle::RelationWrapper<t_btree_iii__0_1_2__111> wrapper_rel_1_empty_strings;+// -- Table: long_strings+Own<t_btree_i__0__1> rel_2_long_strings = mk<t_btree_i__0__1>();+souffle::RelationWrapper<t_btree_i__0__1> wrapper_rel_2_long_strings;+// -- Table: no_strings+Own<t_btree_uif__0_1_2__111> rel_3_no_strings = mk<t_btree_uif__0_1_2__111>();+souffle::RelationWrapper<t_btree_uif__0_1_2__111> wrapper_rel_3_no_strings;+// -- Table: unicode+Own<t_btree_i__0__1> rel_4_unicode = mk<t_btree_i__0__1>();+souffle::RelationWrapper<t_btree_i__0__1> wrapper_rel_4_unicode;+public:+Sf_edge_cases()+: wrapper_rel_1_empty_strings(0, *rel_1_empty_strings, *this, "empty_strings", std::array<const char *,3>{{"s:symbol","s:symbol","i:number"}}, std::array<const char *,3>{{"s","s2","n"}}, 0)+, wrapper_rel_2_long_strings(1, *rel_2_long_strings, *this, "long_strings", std::array<const char *,1>{{"s:symbol"}}, std::array<const char *,1>{{"s"}}, 0)+, wrapper_rel_3_no_strings(2, *rel_3_no_strings, *this, "no_strings", std::array<const char *,3>{{"u:unsigned","i:number","f:float"}}, std::array<const char *,3>{{"u","n","f"}}, 0)+, wrapper_rel_4_unicode(3, *rel_4_unicode, *this, "unicode", std::array<const char *,1>{{"s:symbol"}}, std::array<const char *,1>{{"s"}}, 0)+{+addRelation("empty_strings", wrapper_rel_1_empty_strings, true, true);+addRelation("long_strings", wrapper_rel_2_long_strings, true, true);+addRelation("no_strings", wrapper_rel_3_no_strings, true, true);+addRelation("unicode", wrapper_rel_4_unicode, true, true);+}+~Sf_edge_cases() {+}++private:+std::string             inputDirectory;+std::string             outputDirectory;+SignalHandler*          signalHandler {SignalHandler::instance()};+std::atomic<RamDomain>  ctr {};+std::atomic<std::size_t>     iter {};++void runFunction(std::string  inputDirectoryArg,+                 std::string  outputDirectoryArg,+                 bool         performIOArg,+                 bool         pruneImdtRelsArg) {+    this->inputDirectory  = std::move(inputDirectoryArg);+    this->outputDirectory = std::move(outputDirectoryArg);+    this->performIO       = performIOArg;+    this->pruneImdtRels   = pruneImdtRelsArg; ++    // set default threads (in embedded mode)+    // if this is not set, and omp is used, the default omp setting of number of cores is used.+#if defined(_OPENMP)+    if (0 < getNumThreads()) { omp_set_num_threads(static_cast<int>(getNumThreads())); }+#endif++    signalHandler->set();+// -- query evaluation --+{+ std::vector<RamDomain> args, ret;+subroutine_0(args, ret);+}+{+ std::vector<RamDomain> args, ret;+subroutine_1(args, ret);+}+{+ std::vector<RamDomain> args, ret;+subroutine_2(args, ret);+}+{+ std::vector<RamDomain> args, ret;+subroutine_3(args, ret);+}++// -- relation hint statistics --+signalHandler->reset();+}+public:+void run() override { runFunction("", "", false, false); }+public:+void runAll(std::string inputDirectoryArg = "", std::string outputDirectoryArg = "", bool performIOArg=true, bool pruneImdtRelsArg=true) override { runFunction(inputDirectoryArg, outputDirectoryArg, performIOArg, pruneImdtRelsArg);+}+public:+void printAll(std::string outputDirectoryArg = "") override {+try {std::map<std::string, std::string> directiveMap({{"IO","file"},{"attributeNames","s"},{"auxArity","0"},{"name","long_strings"},{"operation","output"},{"output-dir","."},{"params","{\"records\": {}, \"relation\": {\"arity\": 1, \"params\": [\"s\"]}}"},{"types","{\"ADTs\": {}, \"records\": {}, \"relation\": {\"arity\": 1, \"types\": [\"s:symbol\"]}}"}});+if (!outputDirectoryArg.empty()) {directiveMap["output-dir"] = outputDirectoryArg;}+IOSystem::getInstance().getWriter(directiveMap, symTable, recordTable)->writeAll(*rel_2_long_strings);+} catch (std::exception& e) {std::cerr << e.what();exit(1);}+try {std::map<std::string, std::string> directiveMap({{"IO","file"},{"attributeNames","s"},{"auxArity","0"},{"name","unicode"},{"operation","output"},{"output-dir","."},{"params","{\"records\": {}, \"relation\": {\"arity\": 1, \"params\": [\"s\"]}}"},{"types","{\"ADTs\": {}, \"records\": {}, \"relation\": {\"arity\": 1, \"types\": [\"s:symbol\"]}}"}});+if (!outputDirectoryArg.empty()) {directiveMap["output-dir"] = outputDirectoryArg;}+IOSystem::getInstance().getWriter(directiveMap, symTable, recordTable)->writeAll(*rel_4_unicode);+} catch (std::exception& e) {std::cerr << e.what();exit(1);}+try {std::map<std::string, std::string> directiveMap({{"IO","file"},{"attributeNames","s\ts2\tn"},{"auxArity","0"},{"name","empty_strings"},{"operation","output"},{"output-dir","."},{"params","{\"records\": {}, \"relation\": {\"arity\": 3, \"params\": [\"s\", \"s2\", \"n\"]}}"},{"types","{\"ADTs\": {}, \"records\": {}, \"relation\": {\"arity\": 3, \"types\": [\"s:symbol\", \"s:symbol\", \"i:number\"]}}"}});+if (!outputDirectoryArg.empty()) {directiveMap["output-dir"] = outputDirectoryArg;}+IOSystem::getInstance().getWriter(directiveMap, symTable, recordTable)->writeAll(*rel_1_empty_strings);+} catch (std::exception& e) {std::cerr << e.what();exit(1);}+try {std::map<std::string, std::string> directiveMap({{"IO","file"},{"attributeNames","u\tn\tf"},{"auxArity","0"},{"name","no_strings"},{"operation","output"},{"output-dir","."},{"params","{\"records\": {}, \"relation\": {\"arity\": 3, \"params\": [\"u\", \"n\", \"f\"]}}"},{"types","{\"ADTs\": {}, \"records\": {}, \"relation\": {\"arity\": 3, \"types\": [\"u:unsigned\", \"i:number\", \"f:float\"]}}"}});+if (!outputDirectoryArg.empty()) {directiveMap["output-dir"] = outputDirectoryArg;}+IOSystem::getInstance().getWriter(directiveMap, symTable, recordTable)->writeAll(*rel_3_no_strings);+} catch (std::exception& e) {std::cerr << e.what();exit(1);}+}+public:+void loadAll(std::string inputDirectoryArg = "") override {+try {std::map<std::string, std::string> directiveMap({{"IO","file"},{"attributeNames","u\tn\tf"},{"auxArity","0"},{"fact-dir","."},{"name","no_strings"},{"operation","input"},{"params","{\"records\": {}, \"relation\": {\"arity\": 3, \"params\": [\"u\", \"n\", \"f\"]}}"},{"types","{\"ADTs\": {}, \"records\": {}, \"relation\": {\"arity\": 3, \"types\": [\"u:unsigned\", \"i:number\", \"f:float\"]}}"}});+if (!inputDirectoryArg.empty()) {directiveMap["fact-dir"] = inputDirectoryArg;}+IOSystem::getInstance().getReader(directiveMap, symTable, recordTable)->readAll(*rel_3_no_strings);+} catch (std::exception& e) {std::cerr << "Error loading no_strings data: " << e.what() << '\n';}+try {std::map<std::string, std::string> directiveMap({{"IO","file"},{"attributeNames","s\ts2\tn"},{"auxArity","0"},{"fact-dir","."},{"name","empty_strings"},{"operation","input"},{"params","{\"records\": {}, \"relation\": {\"arity\": 3, \"params\": [\"s\", \"s2\", \"n\"]}}"},{"types","{\"ADTs\": {}, \"records\": {}, \"relation\": {\"arity\": 3, \"types\": [\"s:symbol\", \"s:symbol\", \"i:number\"]}}"}});+if (!inputDirectoryArg.empty()) {directiveMap["fact-dir"] = inputDirectoryArg;}+IOSystem::getInstance().getReader(directiveMap, symTable, recordTable)->readAll(*rel_1_empty_strings);+} catch (std::exception& e) {std::cerr << "Error loading empty_strings data: " << e.what() << '\n';}+try {std::map<std::string, std::string> directiveMap({{"IO","file"},{"attributeNames","s"},{"auxArity","0"},{"fact-dir","."},{"name","unicode"},{"operation","input"},{"params","{\"records\": {}, \"relation\": {\"arity\": 1, \"params\": [\"s\"]}}"},{"types","{\"ADTs\": {}, \"records\": {}, \"relation\": {\"arity\": 1, \"types\": [\"s:symbol\"]}}"}});+if (!inputDirectoryArg.empty()) {directiveMap["fact-dir"] = inputDirectoryArg;}+IOSystem::getInstance().getReader(directiveMap, symTable, recordTable)->readAll(*rel_4_unicode);+} catch (std::exception& e) {std::cerr << "Error loading unicode data: " << e.what() << '\n';}+try {std::map<std::string, std::string> directiveMap({{"IO","file"},{"attributeNames","s"},{"auxArity","0"},{"fact-dir","."},{"name","long_strings"},{"operation","input"},{"params","{\"records\": {}, \"relation\": {\"arity\": 1, \"params\": [\"s\"]}}"},{"types","{\"ADTs\": {}, \"records\": {}, \"relation\": {\"arity\": 1, \"types\": [\"s:symbol\"]}}"}});+if (!inputDirectoryArg.empty()) {directiveMap["fact-dir"] = inputDirectoryArg;}+IOSystem::getInstance().getReader(directiveMap, symTable, recordTable)->readAll(*rel_2_long_strings);+} catch (std::exception& e) {std::cerr << "Error loading long_strings data: " << e.what() << '\n';}+}+public:+void dumpInputs() override {+try {std::map<std::string, std::string> rwOperation;+rwOperation["IO"] = "stdout";+rwOperation["name"] = "no_strings";+rwOperation["types"] = "{\"relation\": {\"arity\": 3, \"auxArity\": 0, \"types\": [\"u:unsigned\", \"i:number\", \"f:float\"]}}";+IOSystem::getInstance().getWriter(rwOperation, symTable, recordTable)->writeAll(*rel_3_no_strings);+} catch (std::exception& e) {std::cerr << e.what();exit(1);}+try {std::map<std::string, std::string> rwOperation;+rwOperation["IO"] = "stdout";+rwOperation["name"] = "empty_strings";+rwOperation["types"] = "{\"relation\": {\"arity\": 3, \"auxArity\": 0, \"types\": [\"s:symbol\", \"s:symbol\", \"i:number\"]}}";+IOSystem::getInstance().getWriter(rwOperation, symTable, recordTable)->writeAll(*rel_1_empty_strings);+} catch (std::exception& e) {std::cerr << e.what();exit(1);}+try {std::map<std::string, std::string> rwOperation;+rwOperation["IO"] = "stdout";+rwOperation["name"] = "unicode";+rwOperation["types"] = "{\"relation\": {\"arity\": 1, \"auxArity\": 0, \"types\": [\"s:symbol\"]}}";+IOSystem::getInstance().getWriter(rwOperation, symTable, recordTable)->writeAll(*rel_4_unicode);+} catch (std::exception& e) {std::cerr << e.what();exit(1);}+try {std::map<std::string, std::string> rwOperation;+rwOperation["IO"] = "stdout";+rwOperation["name"] = "long_strings";+rwOperation["types"] = "{\"relation\": {\"arity\": 1, \"auxArity\": 0, \"types\": [\"s:symbol\"]}}";+IOSystem::getInstance().getWriter(rwOperation, symTable, recordTable)->writeAll(*rel_2_long_strings);+} catch (std::exception& e) {std::cerr << e.what();exit(1);}+}+public:+void dumpOutputs() override {+try {std::map<std::string, std::string> rwOperation;+rwOperation["IO"] = "stdout";+rwOperation["name"] = "long_strings";+rwOperation["types"] = "{\"relation\": {\"arity\": 1, \"auxArity\": 0, \"types\": [\"s:symbol\"]}}";+IOSystem::getInstance().getWriter(rwOperation, symTable, recordTable)->writeAll(*rel_2_long_strings);+} catch (std::exception& e) {std::cerr << e.what();exit(1);}+try {std::map<std::string, std::string> rwOperation;+rwOperation["IO"] = "stdout";+rwOperation["name"] = "unicode";+rwOperation["types"] = "{\"relation\": {\"arity\": 1, \"auxArity\": 0, \"types\": [\"s:symbol\"]}}";+IOSystem::getInstance().getWriter(rwOperation, symTable, recordTable)->writeAll(*rel_4_unicode);+} catch (std::exception& e) {std::cerr << e.what();exit(1);}+try {std::map<std::string, std::string> rwOperation;+rwOperation["IO"] = "stdout";+rwOperation["name"] = "empty_strings";+rwOperation["types"] = "{\"relation\": {\"arity\": 3, \"auxArity\": 0, \"types\": [\"s:symbol\", \"s:symbol\", \"i:number\"]}}";+IOSystem::getInstance().getWriter(rwOperation, symTable, recordTable)->writeAll(*rel_1_empty_strings);+} catch (std::exception& e) {std::cerr << e.what();exit(1);}+try {std::map<std::string, std::string> rwOperation;+rwOperation["IO"] = "stdout";+rwOperation["name"] = "no_strings";+rwOperation["types"] = "{\"relation\": {\"arity\": 3, \"auxArity\": 0, \"types\": [\"u:unsigned\", \"i:number\", \"f:float\"]}}";+IOSystem::getInstance().getWriter(rwOperation, symTable, recordTable)->writeAll(*rel_3_no_strings);+} catch (std::exception& e) {std::cerr << e.what();exit(1);}+}+public:+SymbolTable& getSymbolTable() override {+return symTable;+}+RecordTable& getRecordTable() override {+return recordTable;+}+void setNumThreads(std::size_t numThreadsValue) override {+SouffleProgram::setNumThreads(numThreadsValue);+symTable.setNumLanes(getNumThreads());+recordTable.setNumLanes(getNumThreads());+}+void executeSubroutine(std::string name, const std::vector<RamDomain>& args, std::vector<RamDomain>& ret) override {+if (name == "stratum_0") {+subroutine_0(args, ret);+return;}+if (name == "stratum_1") {+subroutine_1(args, ret);+return;}+if (name == "stratum_2") {+subroutine_2(args, ret);+return;}+if (name == "stratum_3") {+subroutine_3(args, ret);+return;}+fatal("unknown subroutine");+}+#ifdef _MSC_VER+#pragma warning(disable: 4100)+#endif // _MSC_VER+void subroutine_0(const std::vector<RamDomain>& args, std::vector<RamDomain>& ret) {+if (performIO) {+try {std::map<std::string, std::string> directiveMap({{"IO","file"},{"attributeNames","s\ts2\tn"},{"auxArity","0"},{"fact-dir","."},{"name","empty_strings"},{"operation","input"},{"params","{\"records\": {}, \"relation\": {\"arity\": 3, \"params\": [\"s\", \"s2\", \"n\"]}}"},{"types","{\"ADTs\": {}, \"records\": {}, \"relation\": {\"arity\": 3, \"types\": [\"s:symbol\", \"s:symbol\", \"i:number\"]}}"}});+if (!inputDirectory.empty()) {directiveMap["fact-dir"] = inputDirectory;}+IOSystem::getInstance().getReader(directiveMap, symTable, recordTable)->readAll(*rel_1_empty_strings);+} catch (std::exception& e) {std::cerr << "Error loading empty_strings data: " << e.what() << '\n';}+}+signalHandler->setMsg(R"_(empty_strings("","",42).+in file edge_cases.dl [20:1-20:27])_");+[&](){+CREATE_OP_CONTEXT(rel_1_empty_strings_op_ctxt,rel_1_empty_strings->createContext());+Tuple<RamDomain,3> tuple{{ramBitCast(RamSigned(0)),ramBitCast(RamSigned(0)),ramBitCast(RamSigned(42))}};+rel_1_empty_strings->insert(tuple,READ_OP_CONTEXT(rel_1_empty_strings_op_ctxt));+}+();signalHandler->setMsg(R"_(empty_strings("","abc",42).+in file edge_cases.dl [21:1-21:30])_");+[&](){+CREATE_OP_CONTEXT(rel_1_empty_strings_op_ctxt,rel_1_empty_strings->createContext());+Tuple<RamDomain,3> tuple{{ramBitCast(RamSigned(0)),ramBitCast(RamSigned(1)),ramBitCast(RamSigned(42))}};+rel_1_empty_strings->insert(tuple,READ_OP_CONTEXT(rel_1_empty_strings_op_ctxt));+}+();signalHandler->setMsg(R"_(empty_strings("abc","",42).+in file edge_cases.dl [22:1-22:30])_");+[&](){+CREATE_OP_CONTEXT(rel_1_empty_strings_op_ctxt,rel_1_empty_strings->createContext());+Tuple<RamDomain,3> tuple{{ramBitCast(RamSigned(1)),ramBitCast(RamSigned(0)),ramBitCast(RamSigned(42))}};+rel_1_empty_strings->insert(tuple,READ_OP_CONTEXT(rel_1_empty_strings_op_ctxt));+}+();if (performIO) {+try {std::map<std::string, std::string> directiveMap({{"IO","file"},{"attributeNames","s\ts2\tn"},{"auxArity","0"},{"name","empty_strings"},{"operation","output"},{"output-dir","."},{"params","{\"records\": {}, \"relation\": {\"arity\": 3, \"params\": [\"s\", \"s2\", \"n\"]}}"},{"types","{\"ADTs\": {}, \"records\": {}, \"relation\": {\"arity\": 3, \"types\": [\"s:symbol\", \"s:symbol\", \"i:number\"]}}"}});+if (!outputDirectory.empty()) {directiveMap["output-dir"] = outputDirectory;}+IOSystem::getInstance().getWriter(directiveMap, symTable, recordTable)->writeAll(*rel_1_empty_strings);+} catch (std::exception& e) {std::cerr << e.what();exit(1);}+}+}+#ifdef _MSC_VER+#pragma warning(default: 4100)+#endif // _MSC_VER+#ifdef _MSC_VER+#pragma warning(disable: 4100)+#endif // _MSC_VER+void subroutine_1(const std::vector<RamDomain>& args, std::vector<RamDomain>& ret) {+if (performIO) {+try {std::map<std::string, std::string> directiveMap({{"IO","file"},{"attributeNames","s"},{"auxArity","0"},{"fact-dir","."},{"name","long_strings"},{"operation","input"},{"params","{\"records\": {}, \"relation\": {\"arity\": 1, \"params\": [\"s\"]}}"},{"types","{\"ADTs\": {}, \"records\": {}, \"relation\": {\"arity\": 1, \"types\": [\"s:symbol\"]}}"}});+if (!inputDirectory.empty()) {directiveMap["fact-dir"] = inputDirectory;}+IOSystem::getInstance().getReader(directiveMap, symTable, recordTable)->readAll(*rel_2_long_strings);+} catch (std::exception& e) {std::cerr << "Error loading long_strings data: " << e.what() << '\n';}+}+signalHandler->setMsg(R"_(long_strings("long_string_from_DL:...............................................................................................................................................................................................................................................................................................end").+in file edge_cases.dl [25:1-25:328])_");+[&](){+CREATE_OP_CONTEXT(rel_2_long_strings_op_ctxt,rel_2_long_strings->createContext());+Tuple<RamDomain,1> tuple{{ramBitCast(RamSigned(2))}};+rel_2_long_strings->insert(tuple,READ_OP_CONTEXT(rel_2_long_strings_op_ctxt));+}+();if (performIO) {+try {std::map<std::string, std::string> directiveMap({{"IO","file"},{"attributeNames","s"},{"auxArity","0"},{"name","long_strings"},{"operation","output"},{"output-dir","."},{"params","{\"records\": {}, \"relation\": {\"arity\": 1, \"params\": [\"s\"]}}"},{"types","{\"ADTs\": {}, \"records\": {}, \"relation\": {\"arity\": 1, \"types\": [\"s:symbol\"]}}"}});+if (!outputDirectory.empty()) {directiveMap["output-dir"] = outputDirectory;}+IOSystem::getInstance().getWriter(directiveMap, symTable, recordTable)->writeAll(*rel_2_long_strings);+} catch (std::exception& e) {std::cerr << e.what();exit(1);}+}+}+#ifdef _MSC_VER+#pragma warning(default: 4100)+#endif // _MSC_VER+#ifdef _MSC_VER+#pragma warning(disable: 4100)+#endif // _MSC_VER+void subroutine_2(const std::vector<RamDomain>& args, std::vector<RamDomain>& ret) {+if (performIO) {+try {std::map<std::string, std::string> directiveMap({{"IO","file"},{"attributeNames","u\tn\tf"},{"auxArity","0"},{"fact-dir","."},{"name","no_strings"},{"operation","input"},{"params","{\"records\": {}, \"relation\": {\"arity\": 3, \"params\": [\"u\", \"n\", \"f\"]}}"},{"types","{\"ADTs\": {}, \"records\": {}, \"relation\": {\"arity\": 3, \"types\": [\"u:unsigned\", \"i:number\", \"f:float\"]}}"}});+if (!inputDirectory.empty()) {directiveMap["fact-dir"] = inputDirectory;}+IOSystem::getInstance().getReader(directiveMap, symTable, recordTable)->readAll(*rel_3_no_strings);+} catch (std::exception& e) {std::cerr << "Error loading no_strings data: " << e.what() << '\n';}+}+signalHandler->setMsg(R"_(no_strings(42,-100,1.5).+in file edge_cases.dl [33:1-33:27])_");+[&](){+CREATE_OP_CONTEXT(rel_3_no_strings_op_ctxt,rel_3_no_strings->createContext());+Tuple<RamDomain,3> tuple{{ramBitCast(RamUnsigned(42)),ramBitCast(RamSigned(-100)),ramBitCast(RamFloat(1.5))}};+rel_3_no_strings->insert(tuple,READ_OP_CONTEXT(rel_3_no_strings_op_ctxt));+}+();signalHandler->setMsg(R"_(no_strings(123,-456,3.14).+in file edge_cases.dl [34:1-34:29])_");+[&](){+CREATE_OP_CONTEXT(rel_3_no_strings_op_ctxt,rel_3_no_strings->createContext());+Tuple<RamDomain,3> tuple{{ramBitCast(RamUnsigned(123)),ramBitCast(RamSigned(-456)),ramBitCast(RamFloat(3.1400001))}};+rel_3_no_strings->insert(tuple,READ_OP_CONTEXT(rel_3_no_strings_op_ctxt));+}+();if (performIO) {+try {std::map<std::string, std::string> directiveMap({{"IO","file"},{"attributeNames","u\tn\tf"},{"auxArity","0"},{"name","no_strings"},{"operation","output"},{"output-dir","."},{"params","{\"records\": {}, \"relation\": {\"arity\": 3, \"params\": [\"u\", \"n\", \"f\"]}}"},{"types","{\"ADTs\": {}, \"records\": {}, \"relation\": {\"arity\": 3, \"types\": [\"u:unsigned\", \"i:number\", \"f:float\"]}}"}});+if (!outputDirectory.empty()) {directiveMap["output-dir"] = outputDirectory;}+IOSystem::getInstance().getWriter(directiveMap, symTable, recordTable)->writeAll(*rel_3_no_strings);+} catch (std::exception& e) {std::cerr << e.what();exit(1);}+}+}+#ifdef _MSC_VER+#pragma warning(default: 4100)+#endif // _MSC_VER+#ifdef _MSC_VER+#pragma warning(disable: 4100)+#endif // _MSC_VER+void subroutine_3(const std::vector<RamDomain>& args, std::vector<RamDomain>& ret) {+if (performIO) {+try {std::map<std::string, std::string> directiveMap({{"IO","file"},{"attributeNames","s"},{"auxArity","0"},{"fact-dir","."},{"name","unicode"},{"operation","input"},{"params","{\"records\": {}, \"relation\": {\"arity\": 1, \"params\": [\"s\"]}}"},{"types","{\"ADTs\": {}, \"records\": {}, \"relation\": {\"arity\": 1, \"types\": [\"s:symbol\"]}}"}});+if (!inputDirectory.empty()) {directiveMap["fact-dir"] = inputDirectory;}+IOSystem::getInstance().getReader(directiveMap, symTable, recordTable)->readAll(*rel_4_unicode);+} catch (std::exception& e) {std::cerr << "Error loading unicode data: " << e.what() << '\n';}+}+signalHandler->setMsg(R"_(unicode("∀").+in file edge_cases.dl [30:1-30:16])_");+[&](){+CREATE_OP_CONTEXT(rel_4_unicode_op_ctxt,rel_4_unicode->createContext());+Tuple<RamDomain,1> tuple{{ramBitCast(RamSigned(3))}};+rel_4_unicode->insert(tuple,READ_OP_CONTEXT(rel_4_unicode_op_ctxt));+}+();signalHandler->setMsg(R"_(unicode("∀∀").+in file edge_cases.dl [31:1-31:19])_");+[&](){+CREATE_OP_CONTEXT(rel_4_unicode_op_ctxt,rel_4_unicode->createContext());+Tuple<RamDomain,1> tuple{{ramBitCast(RamSigned(4))}};+rel_4_unicode->insert(tuple,READ_OP_CONTEXT(rel_4_unicode_op_ctxt));+}+();if (performIO) {+try {std::map<std::string, std::string> directiveMap({{"IO","file"},{"attributeNames","s"},{"auxArity","0"},{"name","unicode"},{"operation","output"},{"output-dir","."},{"params","{\"records\": {}, \"relation\": {\"arity\": 1, \"params\": [\"s\"]}}"},{"types","{\"ADTs\": {}, \"records\": {}, \"relation\": {\"arity\": 1, \"types\": [\"s:symbol\"]}}"}});+if (!outputDirectory.empty()) {directiveMap["output-dir"] = outputDirectory;}+IOSystem::getInstance().getWriter(directiveMap, symTable, recordTable)->writeAll(*rel_4_unicode);+} catch (std::exception& e) {std::cerr << e.what();exit(1);}+}+}+#ifdef _MSC_VER+#pragma warning(default: 4100)+#endif // _MSC_VER+};+SouffleProgram *newInstance_edge_cases(){return new Sf_edge_cases;}+SymbolTable *getST_edge_cases(SouffleProgram *p){return &reinterpret_cast<Sf_edge_cases*>(p)->getSymbolTable();}++#ifdef __EMBEDDED_SOUFFLE__+class factory_Sf_edge_cases: public souffle::ProgramFactory {+SouffleProgram *newInstance() {+return new Sf_edge_cases();+};+public:+factory_Sf_edge_cases() : ProgramFactory("edge_cases"){}+};+extern "C" {+factory_Sf_edge_cases __factory_Sf_edge_cases_instance;+}+}+#else+}+int main(int argc, char** argv)+{+try{+souffle::CmdOptions opt(R"(edge_cases.dl)",+R"()",+R"()",+false,+R"()",+1);+if (!opt.parse(argc,argv)) return 1;+souffle::Sf_edge_cases obj;+#if defined(_OPENMP) +obj.setNumThreads(opt.getNumJobs());++#endif+obj.runAll(opt.getInputFileDir(), opt.getOutputFileDir());+return 0;+} catch(std::exception &e) { souffle::SignalHandler::instance()->error(e.what());}+}++#endif
tests/fixtures/path.cpp view
@@ -1,19 +1,22 @@  #include "souffle/CompiledSouffle.h" -extern "C" {+namespace functors {+ extern "C" { }+}  namespace souffle { static const RamDomain RAM_BIT_SHIFT_MASK = RAM_DOMAIN_SIZE - 1;-struct t_btree_ii__0_1__11__10 {+struct t_btree_ii__0_1__11 {+static constexpr Relation::arity_type Arity = 2; using t_tuple = Tuple<RamDomain, 2>; struct t_comparator_0{  int operator()(const t_tuple& a, const t_tuple& b) const {   return (ramBitCast<RamSigned>(a[0]) < ramBitCast<RamSigned>(b[0])) ? -1 : (ramBitCast<RamSigned>(a[0]) > ramBitCast<RamSigned>(b[0])) ? 1 :((ramBitCast<RamSigned>(a[1]) < ramBitCast<RamSigned>(b[1])) ? -1 : (ramBitCast<RamSigned>(a[1]) > ramBitCast<RamSigned>(b[1])) ? 1 :(0));  } bool less(const t_tuple& a, const t_tuple& b) const {-  return (ramBitCast<RamSigned>(a[0]) < ramBitCast<RamSigned>(b[0]))|| (ramBitCast<RamSigned>(a[0]) == ramBitCast<RamSigned>(b[0])) && ((ramBitCast<RamSigned>(a[1]) < ramBitCast<RamSigned>(b[1])));+  return (ramBitCast<RamSigned>(a[0]) < ramBitCast<RamSigned>(b[0]))|| ((ramBitCast<RamSigned>(a[0]) == ramBitCast<RamSigned>(b[0])) && ((ramBitCast<RamSigned>(a[1]) < ramBitCast<RamSigned>(b[1]))));  } bool equal(const t_tuple& a, const t_tuple& b) const { return (ramBitCast<RamSigned>(a[0]) == ramBitCast<RamSigned>(b[0]))&&(ramBitCast<RamSigned>(a[1]) == ramBitCast<RamSigned>(b[1]));@@ -88,18 +91,6 @@ context h; return lowerUpperRange_11(lower,upper,h); }-range<t_ind_0::iterator> lowerUpperRange_10(const t_tuple& lower, const t_tuple& upper, context& h) const {-t_comparator_0 comparator;-int cmp = comparator(lower, upper);-if (cmp > 0) {-    return make_range(ind_0.end(), ind_0.end());-}-return make_range(ind_0.lower_bound(lower, h.hints_0_lower), ind_0.upper_bound(upper, h.hints_0_upper));-}-range<t_ind_0::iterator> lowerUpperRange_10(const t_tuple& lower, const t_tuple& upper) const {-context h;-return lowerUpperRange_10(lower,upper,h);-} bool empty() const { return ind_0.empty(); }@@ -120,14 +111,15 @@ ind_0.printStats(o); } };-struct t_btree_ii__0_1__11 {+struct t_btree_ii__0_1__11__10 {+static constexpr Relation::arity_type Arity = 2; using t_tuple = Tuple<RamDomain, 2>; struct t_comparator_0{  int operator()(const t_tuple& a, const t_tuple& b) const {   return (ramBitCast<RamSigned>(a[0]) < ramBitCast<RamSigned>(b[0])) ? -1 : (ramBitCast<RamSigned>(a[0]) > ramBitCast<RamSigned>(b[0])) ? 1 :((ramBitCast<RamSigned>(a[1]) < ramBitCast<RamSigned>(b[1])) ? -1 : (ramBitCast<RamSigned>(a[1]) > ramBitCast<RamSigned>(b[1])) ? 1 :(0));  } bool less(const t_tuple& a, const t_tuple& b) const {-  return (ramBitCast<RamSigned>(a[0]) < ramBitCast<RamSigned>(b[0]))|| (ramBitCast<RamSigned>(a[0]) == ramBitCast<RamSigned>(b[0])) && ((ramBitCast<RamSigned>(a[1]) < ramBitCast<RamSigned>(b[1])));+  return (ramBitCast<RamSigned>(a[0]) < ramBitCast<RamSigned>(b[0]))|| ((ramBitCast<RamSigned>(a[0]) == ramBitCast<RamSigned>(b[0])) && ((ramBitCast<RamSigned>(a[1]) < ramBitCast<RamSigned>(b[1]))));  } bool equal(const t_tuple& a, const t_tuple& b) const { return (ramBitCast<RamSigned>(a[0]) == ramBitCast<RamSigned>(b[0]))&&(ramBitCast<RamSigned>(a[1]) == ramBitCast<RamSigned>(b[1]));@@ -202,6 +194,18 @@ context h; return lowerUpperRange_11(lower,upper,h); }+range<t_ind_0::iterator> lowerUpperRange_10(const t_tuple& lower, const t_tuple& upper, context& h) const {+t_comparator_0 comparator;+int cmp = comparator(lower, upper);+if (cmp > 0) {+    return make_range(ind_0.end(), ind_0.end());+}+return make_range(ind_0.lower_bound(lower, h.hints_0_lower), ind_0.upper_bound(upper, h.hints_0_upper));+}+range<t_ind_0::iterator> lowerUpperRange_10(const t_tuple& lower, const t_tuple& upper) const {+context h;+return lowerUpperRange_10(lower,upper,h);+} bool empty() const { return ind_0.empty(); }@@ -225,15 +229,7 @@  class Sf_path : public SouffleProgram { private:-static inline bool regex_wrapper(const std::string& pattern, const std::string& text) {-   bool result = false; -   try { result = std::regex_match(text, std::regex(pattern)); } catch(...) { -     std::cerr << "warning: wrong pattern provided for match(\"" << pattern << "\",\"" << text << "\").\n";-}-   return result;-}-private:-static inline std::string substr_wrapper(const std::string& str, size_t idx, size_t len) {+static inline std::string substr_wrapper(const std::string& str, std::size_t idx, std::size_t len) {    std::string result;     try { result = str.substr(idx,len); } catch(...) {       std::cerr << "warning: wrong index position provided by substr(\"";@@ -242,48 +238,56 @@ } public: // -- initialize symbol table ---SymbolTable symTable{+SymbolTableImpl symTable{ 	R"_(a)_", 	R"_(b)_", 	R"_(c)_", };// -- initialize record table ---RecordTable recordTable;-// -- Table: @delta_reachable-Own<t_btree_ii__0_1__11__10> rel_1_delta_reachable = mk<t_btree_ii__0_1__11__10>();-// -- Table: @new_reachable-Own<t_btree_ii__0_1__11__10> rel_2_new_reachable = mk<t_btree_ii__0_1__11__10>();+SpecializedRecordTable<0> recordTable{}; // -- Table: edge-Own<t_btree_ii__0_1__11> rel_3_edge = mk<t_btree_ii__0_1__11>();-souffle::RelationWrapper<0,t_btree_ii__0_1__11,Tuple<RamDomain,2>,2,0> wrapper_rel_3_edge;+Own<t_btree_ii__0_1__11> rel_1_edge = mk<t_btree_ii__0_1__11>();+souffle::RelationWrapper<t_btree_ii__0_1__11> wrapper_rel_1_edge; // -- Table: reachable-Own<t_btree_ii__0_1__11> rel_4_reachable = mk<t_btree_ii__0_1__11>();-souffle::RelationWrapper<1,t_btree_ii__0_1__11,Tuple<RamDomain,2>,2,0> wrapper_rel_4_reachable;+Own<t_btree_ii__0_1__11> rel_2_reachable = mk<t_btree_ii__0_1__11>();+souffle::RelationWrapper<t_btree_ii__0_1__11> wrapper_rel_2_reachable;+// -- Table: @delta_reachable+Own<t_btree_ii__0_1__11__10> rel_3_delta_reachable = mk<t_btree_ii__0_1__11__10>();+// -- Table: @new_reachable+Own<t_btree_ii__0_1__11__10> rel_4_new_reachable = mk<t_btree_ii__0_1__11__10>(); public:-Sf_path() : -wrapper_rel_3_edge(*rel_3_edge,symTable,"edge",std::array<const char *,2>{{"s:symbol","s:symbol"}},std::array<const char *,2>{{"n","m"}}),--wrapper_rel_4_reachable(*rel_4_reachable,symTable,"reachable",std::array<const char *,2>{{"s:symbol","s:symbol"}},std::array<const char *,2>{{"n","m"}}){-addRelation("edge",&wrapper_rel_3_edge,true,true);-addRelation("reachable",&wrapper_rel_4_reachable,false,true);+Sf_path()+: wrapper_rel_1_edge(0, *rel_1_edge, *this, "edge", std::array<const char *,2>{{"s:symbol","s:symbol"}}, std::array<const char *,2>{{"n","m"}}, 0)+, wrapper_rel_2_reachable(1, *rel_2_reachable, *this, "reachable", std::array<const char *,2>{{"s:symbol","s:symbol"}}, std::array<const char *,2>{{"n","m"}}, 0)+{+addRelation("edge", wrapper_rel_1_edge, true, true);+addRelation("reachable", wrapper_rel_2_reachable, false, true); } ~Sf_path() { }+ private:-std::string inputDirectory;-std::string outputDirectory;-bool performIO;-std::atomic<RamDomain> ctr{};+std::string             inputDirectory;+std::string             outputDirectory;+SignalHandler*          signalHandler {SignalHandler::instance()};+std::atomic<RamDomain>  ctr {};+std::atomic<std::size_t>     iter {}; -std::atomic<size_t> iter{};-void runFunction(std::string inputDirectoryArg = "", std::string outputDirectoryArg = "", bool performIOArg = false) {-this->inputDirectory = inputDirectoryArg;-this->outputDirectory = outputDirectoryArg;-this->performIO = performIOArg;-SignalHandler::instance()->set();+void runFunction(std::string  inputDirectoryArg,+                 std::string  outputDirectoryArg,+                 bool         performIOArg,+                 bool         pruneImdtRelsArg) {+    this->inputDirectory  = std::move(inputDirectoryArg);+    this->outputDirectory = std::move(outputDirectoryArg);+    this->performIO       = performIOArg;+    this->pruneImdtRels   = pruneImdtRelsArg; ++    // set default threads (in embedded mode)+    // if this is not set, and omp is used, the default omp setting of number of cores is used. #if defined(_OPENMP)-if (getNumThreads() > 0) {omp_set_num_threads(getNumThreads());}+    if (0 < getNumThreads()) { omp_set_num_threads(static_cast<int>(getNumThreads())); } #endif +    signalHandler->set(); // -- query evaluation -- {  std::vector<RamDomain> args, ret;@@ -295,30 +299,30 @@ }  // -- relation hint statistics ---SignalHandler::instance()->reset();+signalHandler->reset(); } public:-void run() override { runFunction("", "", false); }+void run() override { runFunction("", "", false, false); } public:-void runAll(std::string inputDirectoryArg = "", std::string outputDirectoryArg = "") override { runFunction(inputDirectoryArg, outputDirectoryArg, true);+void runAll(std::string inputDirectoryArg = "", std::string outputDirectoryArg = "", bool performIOArg=true, bool pruneImdtRelsArg=true) override { runFunction(inputDirectoryArg, outputDirectoryArg, performIOArg, pruneImdtRelsArg); } public: void printAll(std::string outputDirectoryArg = "") override {-try {std::map<std::string, std::string> directiveMap({{"IO","file"},{"attributeNames","n\tm"},{"name","edge"},{"operation","output"},{"output-dir","."},{"params","{\"records\": {}, \"relation\": {\"arity\": 2, \"auxArity\": 0, \"params\": [\"n\", \"m\"]}}"},{"types","{\"ADTs\": {}, \"records\": {}, \"relation\": {\"arity\": 2, \"auxArity\": 0, \"types\": [\"s:symbol\", \"s:symbol\"]}}"}});+try {std::map<std::string, std::string> directiveMap({{"IO","file"},{"attributeNames","n\tm"},{"auxArity","0"},{"name","reachable"},{"operation","output"},{"output-dir","."},{"params","{\"records\": {}, \"relation\": {\"arity\": 2, \"params\": [\"n\", \"m\"]}}"},{"types","{\"ADTs\": {}, \"records\": {}, \"relation\": {\"arity\": 2, \"types\": [\"s:symbol\", \"s:symbol\"]}}"}}); if (!outputDirectoryArg.empty()) {directiveMap["output-dir"] = outputDirectoryArg;}-IOSystem::getInstance().getWriter(directiveMap, symTable, recordTable)->writeAll(*rel_3_edge);+IOSystem::getInstance().getWriter(directiveMap, symTable, recordTable)->writeAll(*rel_2_reachable); } catch (std::exception& e) {std::cerr << e.what();exit(1);}-try {std::map<std::string, std::string> directiveMap({{"IO","file"},{"attributeNames","n\tm"},{"name","reachable"},{"operation","output"},{"output-dir","."},{"params","{\"records\": {}, \"relation\": {\"arity\": 2, \"auxArity\": 0, \"params\": [\"n\", \"m\"]}}"},{"types","{\"ADTs\": {}, \"records\": {}, \"relation\": {\"arity\": 2, \"auxArity\": 0, \"types\": [\"s:symbol\", \"s:symbol\"]}}"}});+try {std::map<std::string, std::string> directiveMap({{"IO","file"},{"attributeNames","n\tm"},{"auxArity","0"},{"name","edge"},{"operation","output"},{"output-dir","."},{"params","{\"records\": {}, \"relation\": {\"arity\": 2, \"params\": [\"n\", \"m\"]}}"},{"types","{\"ADTs\": {}, \"records\": {}, \"relation\": {\"arity\": 2, \"types\": [\"s:symbol\", \"s:symbol\"]}}"}}); if (!outputDirectoryArg.empty()) {directiveMap["output-dir"] = outputDirectoryArg;}-IOSystem::getInstance().getWriter(directiveMap, symTable, recordTable)->writeAll(*rel_4_reachable);+IOSystem::getInstance().getWriter(directiveMap, symTable, recordTable)->writeAll(*rel_1_edge); } catch (std::exception& e) {std::cerr << e.what();exit(1);} } public: void loadAll(std::string inputDirectoryArg = "") override {-try {std::map<std::string, std::string> directiveMap({{"IO","file"},{"attributeNames","n\tm"},{"fact-dir","."},{"name","edge"},{"operation","input"},{"params","{\"records\": {}, \"relation\": {\"arity\": 2, \"auxArity\": 0, \"params\": [\"n\", \"m\"]}}"},{"types","{\"ADTs\": {}, \"records\": {}, \"relation\": {\"arity\": 2, \"auxArity\": 0, \"types\": [\"s:symbol\", \"s:symbol\"]}}"}});+try {std::map<std::string, std::string> directiveMap({{"IO","file"},{"attributeNames","n\tm"},{"auxArity","0"},{"fact-dir","."},{"name","edge"},{"operation","input"},{"params","{\"records\": {}, \"relation\": {\"arity\": 2, \"params\": [\"n\", \"m\"]}}"},{"types","{\"ADTs\": {}, \"records\": {}, \"relation\": {\"arity\": 2, \"types\": [\"s:symbol\", \"s:symbol\"]}}"}}); if (!inputDirectoryArg.empty()) {directiveMap["fact-dir"] = inputDirectoryArg;}-IOSystem::getInstance().getReader(directiveMap, symTable, recordTable)->readAll(*rel_3_edge);-} catch (std::exception& e) {std::cerr << "Error loading data: " << e.what() << '\n';}+IOSystem::getInstance().getReader(directiveMap, symTable, recordTable)->readAll(*rel_1_edge);+} catch (std::exception& e) {std::cerr << "Error loading edge data: " << e.what() << '\n';} } public: void dumpInputs() override {@@ -326,28 +330,36 @@ rwOperation["IO"] = "stdout"; rwOperation["name"] = "edge"; rwOperation["types"] = "{\"relation\": {\"arity\": 2, \"auxArity\": 0, \"types\": [\"s:symbol\", \"s:symbol\"]}}";-IOSystem::getInstance().getWriter(rwOperation, symTable, recordTable)->writeAll(*rel_3_edge);+IOSystem::getInstance().getWriter(rwOperation, symTable, recordTable)->writeAll(*rel_1_edge); } catch (std::exception& e) {std::cerr << e.what();exit(1);} } public: void dumpOutputs() override { try {std::map<std::string, std::string> rwOperation; rwOperation["IO"] = "stdout";-rwOperation["name"] = "edge";+rwOperation["name"] = "reachable"; rwOperation["types"] = "{\"relation\": {\"arity\": 2, \"auxArity\": 0, \"types\": [\"s:symbol\", \"s:symbol\"]}}";-IOSystem::getInstance().getWriter(rwOperation, symTable, recordTable)->writeAll(*rel_3_edge);+IOSystem::getInstance().getWriter(rwOperation, symTable, recordTable)->writeAll(*rel_2_reachable); } catch (std::exception& e) {std::cerr << e.what();exit(1);} try {std::map<std::string, std::string> rwOperation; rwOperation["IO"] = "stdout";-rwOperation["name"] = "reachable";+rwOperation["name"] = "edge"; rwOperation["types"] = "{\"relation\": {\"arity\": 2, \"auxArity\": 0, \"types\": [\"s:symbol\", \"s:symbol\"]}}";-IOSystem::getInstance().getWriter(rwOperation, symTable, recordTable)->writeAll(*rel_4_reachable);+IOSystem::getInstance().getWriter(rwOperation, symTable, recordTable)->writeAll(*rel_1_edge); } catch (std::exception& e) {std::cerr << e.what();exit(1);} } public: SymbolTable& getSymbolTable() override { return symTable; }+RecordTable& getRecordTable() override {+return recordTable;+}+void setNumThreads(std::size_t numThreadsValue) override {+SouffleProgram::setNumThreads(numThreadsValue);+symTable.setNumLanes(getNumThreads());+recordTable.setNumLanes(getNumThreads());+} void executeSubroutine(std::string name, const std::vector<RamDomain>& args, std::vector<RamDomain>& ret) override { if (name == "stratum_0") { subroutine_0(args, ret);@@ -362,29 +374,29 @@ #endif // _MSC_VER void subroutine_0(const std::vector<RamDomain>& args, std::vector<RamDomain>& ret) { if (performIO) {-try {std::map<std::string, std::string> directiveMap({{"IO","file"},{"attributeNames","n\tm"},{"fact-dir","."},{"name","edge"},{"operation","input"},{"params","{\"records\": {}, \"relation\": {\"arity\": 2, \"auxArity\": 0, \"params\": [\"n\", \"m\"]}}"},{"types","{\"ADTs\": {}, \"records\": {}, \"relation\": {\"arity\": 2, \"auxArity\": 0, \"types\": [\"s:symbol\", \"s:symbol\"]}}"}});+try {std::map<std::string, std::string> directiveMap({{"IO","file"},{"attributeNames","n\tm"},{"auxArity","0"},{"fact-dir","."},{"name","edge"},{"operation","input"},{"params","{\"records\": {}, \"relation\": {\"arity\": 2, \"params\": [\"n\", \"m\"]}}"},{"types","{\"ADTs\": {}, \"records\": {}, \"relation\": {\"arity\": 2, \"types\": [\"s:symbol\", \"s:symbol\"]}}"}}); if (!inputDirectory.empty()) {directiveMap["fact-dir"] = inputDirectory;}-IOSystem::getInstance().getReader(directiveMap, symTable, recordTable)->readAll(*rel_3_edge);-} catch (std::exception& e) {std::cerr << "Error loading data: " << e.what() << '\n';}+IOSystem::getInstance().getReader(directiveMap, symTable, recordTable)->readAll(*rel_1_edge);+} catch (std::exception& e) {std::cerr << "Error loading edge data: " << e.what() << '\n';} }-SignalHandler::instance()->setMsg(R"_(edge("a","b").-in file /Users/luc/personal/souffle-hs/tests/fixtures/path.dl [11:1-11:16])_");+signalHandler->setMsg(R"_(edge("a","b").+in file path.dl [11:1-11:16])_"); [&](){-CREATE_OP_CONTEXT(rel_3_edge_op_ctxt,rel_3_edge->createContext());+CREATE_OP_CONTEXT(rel_1_edge_op_ctxt,rel_1_edge->createContext()); Tuple<RamDomain,2> tuple{{ramBitCast(RamSigned(0)),ramBitCast(RamSigned(1))}};-rel_3_edge->insert(tuple,READ_OP_CONTEXT(rel_3_edge_op_ctxt));+rel_1_edge->insert(tuple,READ_OP_CONTEXT(rel_1_edge_op_ctxt)); }-();SignalHandler::instance()->setMsg(R"_(edge("b","c").-in file /Users/luc/personal/souffle-hs/tests/fixtures/path.dl [12:1-12:16])_");+();signalHandler->setMsg(R"_(edge("b","c").+in file path.dl [12:1-12:16])_"); [&](){-CREATE_OP_CONTEXT(rel_3_edge_op_ctxt,rel_3_edge->createContext());+CREATE_OP_CONTEXT(rel_1_edge_op_ctxt,rel_1_edge->createContext()); Tuple<RamDomain,2> tuple{{ramBitCast(RamSigned(1)),ramBitCast(RamSigned(2))}};-rel_3_edge->insert(tuple,READ_OP_CONTEXT(rel_3_edge_op_ctxt));+rel_1_edge->insert(tuple,READ_OP_CONTEXT(rel_1_edge_op_ctxt)); } ();if (performIO) {-try {std::map<std::string, std::string> directiveMap({{"IO","file"},{"attributeNames","n\tm"},{"name","edge"},{"operation","output"},{"output-dir","."},{"params","{\"records\": {}, \"relation\": {\"arity\": 2, \"auxArity\": 0, \"params\": [\"n\", \"m\"]}}"},{"types","{\"ADTs\": {}, \"records\": {}, \"relation\": {\"arity\": 2, \"auxArity\": 0, \"types\": [\"s:symbol\", \"s:symbol\"]}}"}});+try {std::map<std::string, std::string> directiveMap({{"IO","file"},{"attributeNames","n\tm"},{"auxArity","0"},{"name","edge"},{"operation","output"},{"output-dir","."},{"params","{\"records\": {}, \"relation\": {\"arity\": 2, \"params\": [\"n\", \"m\"]}}"},{"types","{\"ADTs\": {}, \"records\": {}, \"relation\": {\"arity\": 2, \"types\": [\"s:symbol\", \"s:symbol\"]}}"}}); if (!outputDirectory.empty()) {directiveMap["output-dir"] = outputDirectory;}-IOSystem::getInstance().getWriter(directiveMap, symTable, recordTable)->writeAll(*rel_3_edge);+IOSystem::getInstance().getWriter(directiveMap, symTable, recordTable)->writeAll(*rel_1_edge); } catch (std::exception& e) {std::cerr << e.what();exit(1);} } }@@ -395,81 +407,81 @@ #pragma warning(disable: 4100) #endif // _MSC_VER void subroutine_1(const std::vector<RamDomain>& args, std::vector<RamDomain>& ret) {-SignalHandler::instance()->setMsg(R"_(reachable(x,y) :- +signalHandler->setMsg(R"_(reachable(x,y) :-     edge(x,y).-in file /Users/luc/personal/souffle-hs/tests/fixtures/path.dl [14:1-14:31])_");-if(!(rel_3_edge->empty())) {+in file path.dl [14:1-14:31])_");+if(!(rel_1_edge->empty())) { [&](){-CREATE_OP_CONTEXT(rel_3_edge_op_ctxt,rel_3_edge->createContext());-CREATE_OP_CONTEXT(rel_4_reachable_op_ctxt,rel_4_reachable->createContext());-for(const auto& env0 : *rel_3_edge) {+CREATE_OP_CONTEXT(rel_1_edge_op_ctxt,rel_1_edge->createContext());+CREATE_OP_CONTEXT(rel_2_reachable_op_ctxt,rel_2_reachable->createContext());+for(const auto& env0 : *rel_1_edge) { Tuple<RamDomain,2> tuple{{ramBitCast(env0[0]),ramBitCast(env0[1])}};-rel_4_reachable->insert(tuple,READ_OP_CONTEXT(rel_4_reachable_op_ctxt));+rel_2_reachable->insert(tuple,READ_OP_CONTEXT(rel_2_reachable_op_ctxt)); } } ();} [&](){-CREATE_OP_CONTEXT(rel_4_reachable_op_ctxt,rel_4_reachable->createContext());-CREATE_OP_CONTEXT(rel_1_delta_reachable_op_ctxt,rel_1_delta_reachable->createContext());-for(const auto& env0 : *rel_4_reachable) {+CREATE_OP_CONTEXT(rel_2_reachable_op_ctxt,rel_2_reachable->createContext());+CREATE_OP_CONTEXT(rel_3_delta_reachable_op_ctxt,rel_3_delta_reachable->createContext());+for(const auto& env0 : *rel_2_reachable) { Tuple<RamDomain,2> tuple{{ramBitCast(env0[0]),ramBitCast(env0[1])}};-rel_1_delta_reachable->insert(tuple,READ_OP_CONTEXT(rel_1_delta_reachable_op_ctxt));+rel_3_delta_reachable->insert(tuple,READ_OP_CONTEXT(rel_3_delta_reachable_op_ctxt)); } } ();iter = 0; for(;;) {-SignalHandler::instance()->setMsg(R"_(reachable(x,z) :- +signalHandler->setMsg(R"_(reachable(x,z) :-     edge(x,y),    reachable(y,z).-in file /Users/luc/personal/souffle-hs/tests/fixtures/path.dl [15:1-15:48])_");-if(!(rel_3_edge->empty()) && !(rel_1_delta_reachable->empty())) {+in file path.dl [15:1-15:48])_");+if(!(rel_1_edge->empty()) && !(rel_3_delta_reachable->empty())) { [&](){-CREATE_OP_CONTEXT(rel_3_edge_op_ctxt,rel_3_edge->createContext());-CREATE_OP_CONTEXT(rel_4_reachable_op_ctxt,rel_4_reachable->createContext());-CREATE_OP_CONTEXT(rel_1_delta_reachable_op_ctxt,rel_1_delta_reachable->createContext());-CREATE_OP_CONTEXT(rel_2_new_reachable_op_ctxt,rel_2_new_reachable->createContext());-for(const auto& env0 : *rel_3_edge) {-auto range = rel_1_delta_reachable->lowerUpperRange_10(Tuple<RamDomain,2>{{ramBitCast(env0[1]), ramBitCast<RamDomain>(MIN_RAM_SIGNED)}},Tuple<RamDomain,2>{{ramBitCast(env0[1]), ramBitCast<RamDomain>(MAX_RAM_SIGNED)}},READ_OP_CONTEXT(rel_1_delta_reachable_op_ctxt));+CREATE_OP_CONTEXT(rel_1_edge_op_ctxt,rel_1_edge->createContext());+CREATE_OP_CONTEXT(rel_4_new_reachable_op_ctxt,rel_4_new_reachable->createContext());+CREATE_OP_CONTEXT(rel_2_reachable_op_ctxt,rel_2_reachable->createContext());+CREATE_OP_CONTEXT(rel_3_delta_reachable_op_ctxt,rel_3_delta_reachable->createContext());+for(const auto& env0 : *rel_1_edge) {+auto range = rel_3_delta_reachable->lowerUpperRange_10(Tuple<RamDomain,2>{{ramBitCast(env0[1]), ramBitCast<RamDomain>(MIN_RAM_SIGNED)}},Tuple<RamDomain,2>{{ramBitCast(env0[1]), ramBitCast<RamDomain>(MAX_RAM_SIGNED)}},READ_OP_CONTEXT(rel_3_delta_reachable_op_ctxt)); for(const auto& env1 : range) {-if( !(rel_4_reachable->contains(Tuple<RamDomain,2>{{ramBitCast(env0[0]),ramBitCast(env1[1])}},READ_OP_CONTEXT(rel_4_reachable_op_ctxt)))) {+if( !(rel_2_reachable->contains(Tuple<RamDomain,2>{{ramBitCast(env0[0]),ramBitCast(env1[1])}},READ_OP_CONTEXT(rel_2_reachable_op_ctxt)))) { Tuple<RamDomain,2> tuple{{ramBitCast(env0[0]),ramBitCast(env1[1])}};-rel_2_new_reachable->insert(tuple,READ_OP_CONTEXT(rel_2_new_reachable_op_ctxt));+rel_4_new_reachable->insert(tuple,READ_OP_CONTEXT(rel_4_new_reachable_op_ctxt)); } } } } ();}-if(rel_2_new_reachable->empty()) break;+if(rel_4_new_reachable->empty()) break; [&](){-CREATE_OP_CONTEXT(rel_4_reachable_op_ctxt,rel_4_reachable->createContext());-CREATE_OP_CONTEXT(rel_2_new_reachable_op_ctxt,rel_2_new_reachable->createContext());-for(const auto& env0 : *rel_2_new_reachable) {+CREATE_OP_CONTEXT(rel_4_new_reachable_op_ctxt,rel_4_new_reachable->createContext());+CREATE_OP_CONTEXT(rel_2_reachable_op_ctxt,rel_2_reachable->createContext());+for(const auto& env0 : *rel_4_new_reachable) { Tuple<RamDomain,2> tuple{{ramBitCast(env0[0]),ramBitCast(env0[1])}};-rel_4_reachable->insert(tuple,READ_OP_CONTEXT(rel_4_reachable_op_ctxt));+rel_2_reachable->insert(tuple,READ_OP_CONTEXT(rel_2_reachable_op_ctxt)); } }-();std::swap(rel_1_delta_reachable, rel_2_new_reachable);-rel_2_new_reachable->purge();+();std::swap(rel_3_delta_reachable, rel_4_new_reachable);+rel_4_new_reachable->purge(); iter++; } iter = 0;-rel_1_delta_reachable->purge();-rel_2_new_reachable->purge();+rel_3_delta_reachable->purge();+rel_4_new_reachable->purge(); if (performIO) {-try {std::map<std::string, std::string> directiveMap({{"IO","file"},{"attributeNames","n\tm"},{"name","reachable"},{"operation","output"},{"output-dir","."},{"params","{\"records\": {}, \"relation\": {\"arity\": 2, \"auxArity\": 0, \"params\": [\"n\", \"m\"]}}"},{"types","{\"ADTs\": {}, \"records\": {}, \"relation\": {\"arity\": 2, \"auxArity\": 0, \"types\": [\"s:symbol\", \"s:symbol\"]}}"}});+try {std::map<std::string, std::string> directiveMap({{"IO","file"},{"attributeNames","n\tm"},{"auxArity","0"},{"name","reachable"},{"operation","output"},{"output-dir","."},{"params","{\"records\": {}, \"relation\": {\"arity\": 2, \"params\": [\"n\", \"m\"]}}"},{"types","{\"ADTs\": {}, \"records\": {}, \"relation\": {\"arity\": 2, \"types\": [\"s:symbol\", \"s:symbol\"]}}"}}); if (!outputDirectory.empty()) {directiveMap["output-dir"] = outputDirectory;}-IOSystem::getInstance().getWriter(directiveMap, symTable, recordTable)->writeAll(*rel_4_reachable);+IOSystem::getInstance().getWriter(directiveMap, symTable, recordTable)->writeAll(*rel_2_reachable); } catch (std::exception& e) {std::cerr << e.what();exit(1);} }-if (performIO) rel_4_reachable->purge();-if (performIO) rel_3_edge->purge();+if (pruneImdtRels) rel_1_edge->purge();+if (pruneImdtRels) rel_2_reachable->purge(); } #ifdef _MSC_VER #pragma warning(default: 4100) #endif // _MSC_VER }; SouffleProgram *newInstance_path(){return new Sf_path;}-SymbolTable *getST_path(SouffleProgram *p){return &reinterpret_cast<Sf_path*>(p)->symTable;}+SymbolTable *getST_path(SouffleProgram *p){return &reinterpret_cast<Sf_path*>(p)->getSymbolTable();}  #ifdef __EMBEDDED_SOUFFLE__ class factory_Sf_path: public souffle::ProgramFactory {
tests/fixtures/round_trip.cpp view
@@ -1,12 +1,15 @@  #include "souffle/CompiledSouffle.h" -extern "C" {+namespace functors {+ extern "C" { }+}  namespace souffle { static const RamDomain RAM_BIT_SHIFT_MASK = RAM_DOMAIN_SIZE - 1; struct t_btree_f__0__1 {+static constexpr Relation::arity_type Arity = 1; using t_tuple = Tuple<RamDomain, 1>; struct t_comparator_0{  int operator()(const t_tuple& a, const t_tuple& b) const {@@ -108,7 +111,111 @@ ind_0.printStats(o); } };+struct t_btree_iiii__0_1_2_3__1111 {+static constexpr Relation::arity_type Arity = 4;+using t_tuple = Tuple<RamDomain, 4>;+struct t_comparator_0{+ int operator()(const t_tuple& a, const t_tuple& b) const {+  return (ramBitCast<RamSigned>(a[0]) < ramBitCast<RamSigned>(b[0])) ? -1 : (ramBitCast<RamSigned>(a[0]) > ramBitCast<RamSigned>(b[0])) ? 1 :((ramBitCast<RamSigned>(a[1]) < ramBitCast<RamSigned>(b[1])) ? -1 : (ramBitCast<RamSigned>(a[1]) > ramBitCast<RamSigned>(b[1])) ? 1 :((ramBitCast<RamSigned>(a[2]) < ramBitCast<RamSigned>(b[2])) ? -1 : (ramBitCast<RamSigned>(a[2]) > ramBitCast<RamSigned>(b[2])) ? 1 :((ramBitCast<RamSigned>(a[3]) < ramBitCast<RamSigned>(b[3])) ? -1 : (ramBitCast<RamSigned>(a[3]) > ramBitCast<RamSigned>(b[3])) ? 1 :(0))));+ }+bool less(const t_tuple& a, const t_tuple& b) const {+  return (ramBitCast<RamSigned>(a[0]) < ramBitCast<RamSigned>(b[0]))|| ((ramBitCast<RamSigned>(a[0]) == ramBitCast<RamSigned>(b[0])) && ((ramBitCast<RamSigned>(a[1]) < ramBitCast<RamSigned>(b[1]))|| ((ramBitCast<RamSigned>(a[1]) == ramBitCast<RamSigned>(b[1])) && ((ramBitCast<RamSigned>(a[2]) < ramBitCast<RamSigned>(b[2]))|| ((ramBitCast<RamSigned>(a[2]) == ramBitCast<RamSigned>(b[2])) && ((ramBitCast<RamSigned>(a[3]) < ramBitCast<RamSigned>(b[3]))))))));+ }+bool equal(const t_tuple& a, const t_tuple& b) const {+return (ramBitCast<RamSigned>(a[0]) == ramBitCast<RamSigned>(b[0]))&&(ramBitCast<RamSigned>(a[1]) == ramBitCast<RamSigned>(b[1]))&&(ramBitCast<RamSigned>(a[2]) == ramBitCast<RamSigned>(b[2]))&&(ramBitCast<RamSigned>(a[3]) == ramBitCast<RamSigned>(b[3]));+ }+};+using t_ind_0 = btree_set<t_tuple,t_comparator_0>;+t_ind_0 ind_0;+using iterator = t_ind_0::iterator;+struct context {+t_ind_0::operation_hints hints_0_lower;+t_ind_0::operation_hints hints_0_upper;+};+context createContext() { return context(); }+bool insert(const t_tuple& t) {+context h;+return insert(t, h);+}+bool insert(const t_tuple& t, context& h) {+if (ind_0.insert(t, h.hints_0_lower)) {+return true;+} else return false;+}+bool insert(const RamDomain* ramDomain) {+RamDomain data[4];+std::copy(ramDomain, ramDomain + 4, data);+const t_tuple& tuple = reinterpret_cast<const t_tuple&>(data);+context h;+return insert(tuple, h);+}+bool insert(RamDomain a0,RamDomain a1,RamDomain a2,RamDomain a3) {+RamDomain data[4] = {a0,a1,a2,a3};+return insert(data);+}+bool contains(const t_tuple& t, context& h) const {+return ind_0.contains(t, h.hints_0_lower);+}+bool contains(const t_tuple& t) const {+context h;+return contains(t, h);+}+std::size_t size() const {+return ind_0.size();+}+iterator find(const t_tuple& t, context& h) const {+return ind_0.find(t, h.hints_0_lower);+}+iterator find(const t_tuple& t) const {+context h;+return find(t, h);+}+range<iterator> lowerUpperRange_0000(const t_tuple& /* lower */, const t_tuple& /* upper */, context& /* h */) const {+return range<iterator>(ind_0.begin(),ind_0.end());+}+range<iterator> lowerUpperRange_0000(const t_tuple& /* lower */, const t_tuple& /* upper */) const {+return range<iterator>(ind_0.begin(),ind_0.end());+}+range<t_ind_0::iterator> lowerUpperRange_1111(const t_tuple& lower, const t_tuple& upper, context& h) const {+t_comparator_0 comparator;+int cmp = comparator(lower, upper);+if (cmp == 0) {+    auto pos = ind_0.find(lower, h.hints_0_lower);+    auto fin = ind_0.end();+    if (pos != fin) {fin = pos; ++fin;}+    return make_range(pos, fin);+}+if (cmp > 0) {+    return make_range(ind_0.end(), ind_0.end());+}+return make_range(ind_0.lower_bound(lower, h.hints_0_lower), ind_0.upper_bound(upper, h.hints_0_upper));+}+range<t_ind_0::iterator> lowerUpperRange_1111(const t_tuple& lower, const t_tuple& upper) const {+context h;+return lowerUpperRange_1111(lower,upper,h);+}+bool empty() const {+return ind_0.empty();+}+std::vector<range<iterator>> partition() const {+return ind_0.getChunks(400);+}+void purge() {+ind_0.clear();+}+iterator begin() const {+return ind_0.begin();+}+iterator end() const {+return ind_0.end();+}+void printStatistics(std::ostream& o) const {+o << " arity 4 direct b-tree index 0 lex-order [0,1,2,3]\n";+ind_0.printStats(o);+}+}; struct t_btree_i__0__1 {+static constexpr Relation::arity_type Arity = 1; using t_tuple = Tuple<RamDomain, 1>; struct t_comparator_0{  int operator()(const t_tuple& a, const t_tuple& b) const {@@ -211,6 +318,7 @@ } }; struct t_btree_u__0__1 {+static constexpr Relation::arity_type Arity = 1; using t_tuple = Tuple<RamDomain, 1>; struct t_comparator_0{  int operator()(const t_tuple& a, const t_tuple& b) const {@@ -315,15 +423,7 @@  class Sf_round_trip : public SouffleProgram { private:-static inline bool regex_wrapper(const std::string& pattern, const std::string& text) {-   bool result = false; -   try { result = std::regex_match(text, std::regex(pattern)); } catch(...) { -     std::cerr << "warning: wrong pattern provided for match(\"" << pattern << "\",\"" << text << "\").\n";-}-   return result;-}-private:-static inline std::string substr_wrapper(const std::string& str, size_t idx, size_t len) {+static inline std::string substr_wrapper(const std::string& str, std::size_t idx, std::size_t len) {    std::string result;     try { result = str.substr(idx,len); } catch(...) {       std::cerr << "warning: wrong index position provided by substr(\"";@@ -332,52 +432,63 @@ } public: // -- initialize symbol table ---SymbolTable symTable;// -- initialize record table ---RecordTable recordTable;+SymbolTableImpl symTable;// -- initialize record table --+SpecializedRecordTable<0> recordTable{}; // -- Table: float_fact Own<t_btree_f__0__1> rel_1_float_fact = mk<t_btree_f__0__1>();-souffle::RelationWrapper<0,t_btree_f__0__1,Tuple<RamDomain,1>,1,0> wrapper_rel_1_float_fact;+souffle::RelationWrapper<t_btree_f__0__1> wrapper_rel_1_float_fact;+// -- Table: large_record+Own<t_btree_iiii__0_1_2_3__1111> rel_2_large_record = mk<t_btree_iiii__0_1_2_3__1111>();+souffle::RelationWrapper<t_btree_iiii__0_1_2_3__1111> wrapper_rel_2_large_record; // -- Table: number_fact-Own<t_btree_i__0__1> rel_2_number_fact = mk<t_btree_i__0__1>();-souffle::RelationWrapper<1,t_btree_i__0__1,Tuple<RamDomain,1>,1,0> wrapper_rel_2_number_fact;+Own<t_btree_i__0__1> rel_3_number_fact = mk<t_btree_i__0__1>();+souffle::RelationWrapper<t_btree_i__0__1> wrapper_rel_3_number_fact; // -- Table: string_fact-Own<t_btree_i__0__1> rel_3_string_fact = mk<t_btree_i__0__1>();-souffle::RelationWrapper<2,t_btree_i__0__1,Tuple<RamDomain,1>,1,0> wrapper_rel_3_string_fact;+Own<t_btree_i__0__1> rel_4_string_fact = mk<t_btree_i__0__1>();+souffle::RelationWrapper<t_btree_i__0__1> wrapper_rel_4_string_fact; // -- Table: unsigned_fact-Own<t_btree_u__0__1> rel_4_unsigned_fact = mk<t_btree_u__0__1>();-souffle::RelationWrapper<3,t_btree_u__0__1,Tuple<RamDomain,1>,1,0> wrapper_rel_4_unsigned_fact;+Own<t_btree_u__0__1> rel_5_unsigned_fact = mk<t_btree_u__0__1>();+souffle::RelationWrapper<t_btree_u__0__1> wrapper_rel_5_unsigned_fact; public:-Sf_round_trip() : -wrapper_rel_1_float_fact(*rel_1_float_fact,symTable,"float_fact",std::array<const char *,1>{{"f:float"}},std::array<const char *,1>{{"x"}}),--wrapper_rel_2_number_fact(*rel_2_number_fact,symTable,"number_fact",std::array<const char *,1>{{"i:number"}},std::array<const char *,1>{{"x"}}),--wrapper_rel_3_string_fact(*rel_3_string_fact,symTable,"string_fact",std::array<const char *,1>{{"s:symbol"}},std::array<const char *,1>{{"x"}}),--wrapper_rel_4_unsigned_fact(*rel_4_unsigned_fact,symTable,"unsigned_fact",std::array<const char *,1>{{"u:unsigned"}},std::array<const char *,1>{{"x"}}){-addRelation("float_fact",&wrapper_rel_1_float_fact,true,true);-addRelation("number_fact",&wrapper_rel_2_number_fact,true,true);-addRelation("string_fact",&wrapper_rel_3_string_fact,true,true);-addRelation("unsigned_fact",&wrapper_rel_4_unsigned_fact,true,true);+Sf_round_trip()+: wrapper_rel_1_float_fact(0, *rel_1_float_fact, *this, "float_fact", std::array<const char *,1>{{"f:float"}}, std::array<const char *,1>{{"x"}}, 0)+, wrapper_rel_2_large_record(1, *rel_2_large_record, *this, "large_record", std::array<const char *,4>{{"i:number","i:number","i:number","i:number"}}, std::array<const char *,4>{{"a","b","c","d"}}, 0)+, wrapper_rel_3_number_fact(2, *rel_3_number_fact, *this, "number_fact", std::array<const char *,1>{{"i:number"}}, std::array<const char *,1>{{"x"}}, 0)+, wrapper_rel_4_string_fact(3, *rel_4_string_fact, *this, "string_fact", std::array<const char *,1>{{"s:symbol"}}, std::array<const char *,1>{{"x"}}, 0)+, wrapper_rel_5_unsigned_fact(4, *rel_5_unsigned_fact, *this, "unsigned_fact", std::array<const char *,1>{{"u:unsigned"}}, std::array<const char *,1>{{"x"}}, 0)+{+addRelation("float_fact", wrapper_rel_1_float_fact, true, true);+addRelation("large_record", wrapper_rel_2_large_record, true, true);+addRelation("number_fact", wrapper_rel_3_number_fact, true, true);+addRelation("string_fact", wrapper_rel_4_string_fact, true, true);+addRelation("unsigned_fact", wrapper_rel_5_unsigned_fact, true, true); } ~Sf_round_trip() { }+ private:-std::string inputDirectory;-std::string outputDirectory;-bool performIO;-std::atomic<RamDomain> ctr{};+std::string             inputDirectory;+std::string             outputDirectory;+SignalHandler*          signalHandler {SignalHandler::instance()};+std::atomic<RamDomain>  ctr {};+std::atomic<std::size_t>     iter {}; -std::atomic<size_t> iter{};-void runFunction(std::string inputDirectoryArg = "", std::string outputDirectoryArg = "", bool performIOArg = false) {-this->inputDirectory = inputDirectoryArg;-this->outputDirectory = outputDirectoryArg;-this->performIO = performIOArg;-SignalHandler::instance()->set();+void runFunction(std::string  inputDirectoryArg,+                 std::string  outputDirectoryArg,+                 bool         performIOArg,+                 bool         pruneImdtRelsArg) {+    this->inputDirectory  = std::move(inputDirectoryArg);+    this->outputDirectory = std::move(outputDirectoryArg);+    this->performIO       = performIOArg;+    this->pruneImdtRels   = pruneImdtRelsArg; ++    // set default threads (in embedded mode)+    // if this is not set, and omp is used, the default omp setting of number of cores is used. #if defined(_OPENMP)-if (getNumThreads() > 0) {omp_set_num_threads(getNumThreads());}+    if (0 < getNumThreads()) { omp_set_num_threads(static_cast<int>(getNumThreads())); } #endif +    signalHandler->set(); // -- query evaluation -- {  std::vector<RamDomain> args, ret;@@ -395,52 +506,64 @@  std::vector<RamDomain> args, ret; subroutine_3(args, ret); }+{+ std::vector<RamDomain> args, ret;+subroutine_4(args, ret);+}  // -- relation hint statistics ---SignalHandler::instance()->reset();+signalHandler->reset(); } public:-void run() override { runFunction("", "", false); }+void run() override { runFunction("", "", false, false); } public:-void runAll(std::string inputDirectoryArg = "", std::string outputDirectoryArg = "") override { runFunction(inputDirectoryArg, outputDirectoryArg, true);+void runAll(std::string inputDirectoryArg = "", std::string outputDirectoryArg = "", bool performIOArg=true, bool pruneImdtRelsArg=true) override { runFunction(inputDirectoryArg, outputDirectoryArg, performIOArg, pruneImdtRelsArg); } public: void printAll(std::string outputDirectoryArg = "") override {-try {std::map<std::string, std::string> directiveMap({{"IO","file"},{"attributeNames","x"},{"name","number_fact"},{"operation","output"},{"output-dir","."},{"params","{\"records\": {}, \"relation\": {\"arity\": 1, \"auxArity\": 0, \"params\": [\"x\"]}}"},{"types","{\"ADTs\": {}, \"records\": {}, \"relation\": {\"arity\": 1, \"auxArity\": 0, \"types\": [\"i:number\"]}}"}});+try {std::map<std::string, std::string> directiveMap({{"IO","file"},{"attributeNames","a\tb\tc\td"},{"auxArity","0"},{"name","large_record"},{"operation","output"},{"output-dir","."},{"params","{\"records\": {}, \"relation\": {\"arity\": 4, \"params\": [\"a\", \"b\", \"c\", \"d\"]}}"},{"types","{\"ADTs\": {}, \"records\": {}, \"relation\": {\"arity\": 4, \"types\": [\"i:number\", \"i:number\", \"i:number\", \"i:number\"]}}"}}); if (!outputDirectoryArg.empty()) {directiveMap["output-dir"] = outputDirectoryArg;}-IOSystem::getInstance().getWriter(directiveMap, symTable, recordTable)->writeAll(*rel_2_number_fact);+IOSystem::getInstance().getWriter(directiveMap, symTable, recordTable)->writeAll(*rel_2_large_record); } catch (std::exception& e) {std::cerr << e.what();exit(1);}-try {std::map<std::string, std::string> directiveMap({{"IO","file"},{"attributeNames","x"},{"name","unsigned_fact"},{"operation","output"},{"output-dir","."},{"params","{\"records\": {}, \"relation\": {\"arity\": 1, \"auxArity\": 0, \"params\": [\"x\"]}}"},{"types","{\"ADTs\": {}, \"records\": {}, \"relation\": {\"arity\": 1, \"auxArity\": 0, \"types\": [\"u:unsigned\"]}}"}});+try {std::map<std::string, std::string> directiveMap({{"IO","file"},{"attributeNames","x"},{"auxArity","0"},{"name","number_fact"},{"operation","output"},{"output-dir","."},{"params","{\"records\": {}, \"relation\": {\"arity\": 1, \"params\": [\"x\"]}}"},{"types","{\"ADTs\": {}, \"records\": {}, \"relation\": {\"arity\": 1, \"types\": [\"i:number\"]}}"}}); if (!outputDirectoryArg.empty()) {directiveMap["output-dir"] = outputDirectoryArg;}-IOSystem::getInstance().getWriter(directiveMap, symTable, recordTable)->writeAll(*rel_4_unsigned_fact);+IOSystem::getInstance().getWriter(directiveMap, symTable, recordTable)->writeAll(*rel_3_number_fact); } catch (std::exception& e) {std::cerr << e.what();exit(1);}-try {std::map<std::string, std::string> directiveMap({{"IO","file"},{"attributeNames","x"},{"name","string_fact"},{"operation","output"},{"output-dir","."},{"params","{\"records\": {}, \"relation\": {\"arity\": 1, \"auxArity\": 0, \"params\": [\"x\"]}}"},{"types","{\"ADTs\": {}, \"records\": {}, \"relation\": {\"arity\": 1, \"auxArity\": 0, \"types\": [\"s:symbol\"]}}"}});+try {std::map<std::string, std::string> directiveMap({{"IO","file"},{"attributeNames","x"},{"auxArity","0"},{"name","string_fact"},{"operation","output"},{"output-dir","."},{"params","{\"records\": {}, \"relation\": {\"arity\": 1, \"params\": [\"x\"]}}"},{"types","{\"ADTs\": {}, \"records\": {}, \"relation\": {\"arity\": 1, \"types\": [\"s:symbol\"]}}"}}); if (!outputDirectoryArg.empty()) {directiveMap["output-dir"] = outputDirectoryArg;}-IOSystem::getInstance().getWriter(directiveMap, symTable, recordTable)->writeAll(*rel_3_string_fact);+IOSystem::getInstance().getWriter(directiveMap, symTable, recordTable)->writeAll(*rel_4_string_fact); } catch (std::exception& e) {std::cerr << e.what();exit(1);}-try {std::map<std::string, std::string> directiveMap({{"IO","file"},{"attributeNames","x"},{"name","float_fact"},{"operation","output"},{"output-dir","."},{"params","{\"records\": {}, \"relation\": {\"arity\": 1, \"auxArity\": 0, \"params\": [\"x\"]}}"},{"types","{\"ADTs\": {}, \"records\": {}, \"relation\": {\"arity\": 1, \"auxArity\": 0, \"types\": [\"f:float\"]}}"}});+try {std::map<std::string, std::string> directiveMap({{"IO","file"},{"attributeNames","x"},{"auxArity","0"},{"name","unsigned_fact"},{"operation","output"},{"output-dir","."},{"params","{\"records\": {}, \"relation\": {\"arity\": 1, \"params\": [\"x\"]}}"},{"types","{\"ADTs\": {}, \"records\": {}, \"relation\": {\"arity\": 1, \"types\": [\"u:unsigned\"]}}"}}); if (!outputDirectoryArg.empty()) {directiveMap["output-dir"] = outputDirectoryArg;}+IOSystem::getInstance().getWriter(directiveMap, symTable, recordTable)->writeAll(*rel_5_unsigned_fact);+} catch (std::exception& e) {std::cerr << e.what();exit(1);}+try {std::map<std::string, std::string> directiveMap({{"IO","file"},{"attributeNames","x"},{"auxArity","0"},{"name","float_fact"},{"operation","output"},{"output-dir","."},{"params","{\"records\": {}, \"relation\": {\"arity\": 1, \"params\": [\"x\"]}}"},{"types","{\"ADTs\": {}, \"records\": {}, \"relation\": {\"arity\": 1, \"types\": [\"f:float\"]}}"}});+if (!outputDirectoryArg.empty()) {directiveMap["output-dir"] = outputDirectoryArg;} IOSystem::getInstance().getWriter(directiveMap, symTable, recordTable)->writeAll(*rel_1_float_fact); } catch (std::exception& e) {std::cerr << e.what();exit(1);} } public: void loadAll(std::string inputDirectoryArg = "") override {-try {std::map<std::string, std::string> directiveMap({{"IO","file"},{"attributeNames","x"},{"fact-dir","."},{"name","float_fact"},{"operation","input"},{"params","{\"records\": {}, \"relation\": {\"arity\": 1, \"auxArity\": 0, \"params\": [\"x\"]}}"},{"types","{\"ADTs\": {}, \"records\": {}, \"relation\": {\"arity\": 1, \"auxArity\": 0, \"types\": [\"f:float\"]}}"}});+try {std::map<std::string, std::string> directiveMap({{"IO","file"},{"attributeNames","x"},{"auxArity","0"},{"fact-dir","."},{"name","float_fact"},{"operation","input"},{"params","{\"records\": {}, \"relation\": {\"arity\": 1, \"params\": [\"x\"]}}"},{"types","{\"ADTs\": {}, \"records\": {}, \"relation\": {\"arity\": 1, \"types\": [\"f:float\"]}}"}}); if (!inputDirectoryArg.empty()) {directiveMap["fact-dir"] = inputDirectoryArg;} IOSystem::getInstance().getReader(directiveMap, symTable, recordTable)->readAll(*rel_1_float_fact);-} catch (std::exception& e) {std::cerr << "Error loading data: " << e.what() << '\n';}-try {std::map<std::string, std::string> directiveMap({{"IO","file"},{"attributeNames","x"},{"fact-dir","."},{"name","number_fact"},{"operation","input"},{"params","{\"records\": {}, \"relation\": {\"arity\": 1, \"auxArity\": 0, \"params\": [\"x\"]}}"},{"types","{\"ADTs\": {}, \"records\": {}, \"relation\": {\"arity\": 1, \"auxArity\": 0, \"types\": [\"i:number\"]}}"}});+} catch (std::exception& e) {std::cerr << "Error loading float_fact data: " << e.what() << '\n';}+try {std::map<std::string, std::string> directiveMap({{"IO","file"},{"attributeNames","a\tb\tc\td"},{"auxArity","0"},{"fact-dir","."},{"name","large_record"},{"operation","input"},{"params","{\"records\": {}, \"relation\": {\"arity\": 4, \"params\": [\"a\", \"b\", \"c\", \"d\"]}}"},{"types","{\"ADTs\": {}, \"records\": {}, \"relation\": {\"arity\": 4, \"types\": [\"i:number\", \"i:number\", \"i:number\", \"i:number\"]}}"}}); if (!inputDirectoryArg.empty()) {directiveMap["fact-dir"] = inputDirectoryArg;}-IOSystem::getInstance().getReader(directiveMap, symTable, recordTable)->readAll(*rel_2_number_fact);-} catch (std::exception& e) {std::cerr << "Error loading data: " << e.what() << '\n';}-try {std::map<std::string, std::string> directiveMap({{"IO","file"},{"attributeNames","x"},{"fact-dir","."},{"name","string_fact"},{"operation","input"},{"params","{\"records\": {}, \"relation\": {\"arity\": 1, \"auxArity\": 0, \"params\": [\"x\"]}}"},{"types","{\"ADTs\": {}, \"records\": {}, \"relation\": {\"arity\": 1, \"auxArity\": 0, \"types\": [\"s:symbol\"]}}"}});+IOSystem::getInstance().getReader(directiveMap, symTable, recordTable)->readAll(*rel_2_large_record);+} catch (std::exception& e) {std::cerr << "Error loading large_record data: " << e.what() << '\n';}+try {std::map<std::string, std::string> directiveMap({{"IO","file"},{"attributeNames","x"},{"auxArity","0"},{"fact-dir","."},{"name","number_fact"},{"operation","input"},{"params","{\"records\": {}, \"relation\": {\"arity\": 1, \"params\": [\"x\"]}}"},{"types","{\"ADTs\": {}, \"records\": {}, \"relation\": {\"arity\": 1, \"types\": [\"i:number\"]}}"}}); if (!inputDirectoryArg.empty()) {directiveMap["fact-dir"] = inputDirectoryArg;}-IOSystem::getInstance().getReader(directiveMap, symTable, recordTable)->readAll(*rel_3_string_fact);-} catch (std::exception& e) {std::cerr << "Error loading data: " << e.what() << '\n';}-try {std::map<std::string, std::string> directiveMap({{"IO","file"},{"attributeNames","x"},{"fact-dir","."},{"name","unsigned_fact"},{"operation","input"},{"params","{\"records\": {}, \"relation\": {\"arity\": 1, \"auxArity\": 0, \"params\": [\"x\"]}}"},{"types","{\"ADTs\": {}, \"records\": {}, \"relation\": {\"arity\": 1, \"auxArity\": 0, \"types\": [\"u:unsigned\"]}}"}});+IOSystem::getInstance().getReader(directiveMap, symTable, recordTable)->readAll(*rel_3_number_fact);+} catch (std::exception& e) {std::cerr << "Error loading number_fact data: " << e.what() << '\n';}+try {std::map<std::string, std::string> directiveMap({{"IO","file"},{"attributeNames","x"},{"auxArity","0"},{"fact-dir","."},{"name","string_fact"},{"operation","input"},{"params","{\"records\": {}, \"relation\": {\"arity\": 1, \"params\": [\"x\"]}}"},{"types","{\"ADTs\": {}, \"records\": {}, \"relation\": {\"arity\": 1, \"types\": [\"s:symbol\"]}}"}}); if (!inputDirectoryArg.empty()) {directiveMap["fact-dir"] = inputDirectoryArg;}-IOSystem::getInstance().getReader(directiveMap, symTable, recordTable)->readAll(*rel_4_unsigned_fact);-} catch (std::exception& e) {std::cerr << "Error loading data: " << e.what() << '\n';}+IOSystem::getInstance().getReader(directiveMap, symTable, recordTable)->readAll(*rel_4_string_fact);+} catch (std::exception& e) {std::cerr << "Error loading string_fact data: " << e.what() << '\n';}+try {std::map<std::string, std::string> directiveMap({{"IO","file"},{"attributeNames","x"},{"auxArity","0"},{"fact-dir","."},{"name","unsigned_fact"},{"operation","input"},{"params","{\"records\": {}, \"relation\": {\"arity\": 1, \"params\": [\"x\"]}}"},{"types","{\"ADTs\": {}, \"records\": {}, \"relation\": {\"arity\": 1, \"types\": [\"u:unsigned\"]}}"}});+if (!inputDirectoryArg.empty()) {directiveMap["fact-dir"] = inputDirectoryArg;}+IOSystem::getInstance().getReader(directiveMap, symTable, recordTable)->readAll(*rel_5_unsigned_fact);+} catch (std::exception& e) {std::cerr << "Error loading unsigned_fact data: " << e.what() << '\n';} } public: void dumpInputs() override {@@ -452,45 +575,57 @@ } catch (std::exception& e) {std::cerr << e.what();exit(1);} try {std::map<std::string, std::string> rwOperation; rwOperation["IO"] = "stdout";+rwOperation["name"] = "large_record";+rwOperation["types"] = "{\"relation\": {\"arity\": 4, \"auxArity\": 0, \"types\": [\"i:number\", \"i:number\", \"i:number\", \"i:number\"]}}";+IOSystem::getInstance().getWriter(rwOperation, symTable, recordTable)->writeAll(*rel_2_large_record);+} catch (std::exception& e) {std::cerr << e.what();exit(1);}+try {std::map<std::string, std::string> rwOperation;+rwOperation["IO"] = "stdout"; rwOperation["name"] = "number_fact"; rwOperation["types"] = "{\"relation\": {\"arity\": 1, \"auxArity\": 0, \"types\": [\"i:number\"]}}";-IOSystem::getInstance().getWriter(rwOperation, symTable, recordTable)->writeAll(*rel_2_number_fact);+IOSystem::getInstance().getWriter(rwOperation, symTable, recordTable)->writeAll(*rel_3_number_fact); } catch (std::exception& e) {std::cerr << e.what();exit(1);} try {std::map<std::string, std::string> rwOperation; rwOperation["IO"] = "stdout"; rwOperation["name"] = "string_fact"; rwOperation["types"] = "{\"relation\": {\"arity\": 1, \"auxArity\": 0, \"types\": [\"s:symbol\"]}}";-IOSystem::getInstance().getWriter(rwOperation, symTable, recordTable)->writeAll(*rel_3_string_fact);+IOSystem::getInstance().getWriter(rwOperation, symTable, recordTable)->writeAll(*rel_4_string_fact); } catch (std::exception& e) {std::cerr << e.what();exit(1);} try {std::map<std::string, std::string> rwOperation; rwOperation["IO"] = "stdout"; rwOperation["name"] = "unsigned_fact"; rwOperation["types"] = "{\"relation\": {\"arity\": 1, \"auxArity\": 0, \"types\": [\"u:unsigned\"]}}";-IOSystem::getInstance().getWriter(rwOperation, symTable, recordTable)->writeAll(*rel_4_unsigned_fact);+IOSystem::getInstance().getWriter(rwOperation, symTable, recordTable)->writeAll(*rel_5_unsigned_fact); } catch (std::exception& e) {std::cerr << e.what();exit(1);} } public: void dumpOutputs() override { try {std::map<std::string, std::string> rwOperation; rwOperation["IO"] = "stdout";-rwOperation["name"] = "number_fact";-rwOperation["types"] = "{\"relation\": {\"arity\": 1, \"auxArity\": 0, \"types\": [\"i:number\"]}}";-IOSystem::getInstance().getWriter(rwOperation, symTable, recordTable)->writeAll(*rel_2_number_fact);+rwOperation["name"] = "large_record";+rwOperation["types"] = "{\"relation\": {\"arity\": 4, \"auxArity\": 0, \"types\": [\"i:number\", \"i:number\", \"i:number\", \"i:number\"]}}";+IOSystem::getInstance().getWriter(rwOperation, symTable, recordTable)->writeAll(*rel_2_large_record); } catch (std::exception& e) {std::cerr << e.what();exit(1);} try {std::map<std::string, std::string> rwOperation; rwOperation["IO"] = "stdout";-rwOperation["name"] = "unsigned_fact";-rwOperation["types"] = "{\"relation\": {\"arity\": 1, \"auxArity\": 0, \"types\": [\"u:unsigned\"]}}";-IOSystem::getInstance().getWriter(rwOperation, symTable, recordTable)->writeAll(*rel_4_unsigned_fact);+rwOperation["name"] = "number_fact";+rwOperation["types"] = "{\"relation\": {\"arity\": 1, \"auxArity\": 0, \"types\": [\"i:number\"]}}";+IOSystem::getInstance().getWriter(rwOperation, symTable, recordTable)->writeAll(*rel_3_number_fact); } catch (std::exception& e) {std::cerr << e.what();exit(1);} try {std::map<std::string, std::string> rwOperation; rwOperation["IO"] = "stdout"; rwOperation["name"] = "string_fact"; rwOperation["types"] = "{\"relation\": {\"arity\": 1, \"auxArity\": 0, \"types\": [\"s:symbol\"]}}";-IOSystem::getInstance().getWriter(rwOperation, symTable, recordTable)->writeAll(*rel_3_string_fact);+IOSystem::getInstance().getWriter(rwOperation, symTable, recordTable)->writeAll(*rel_4_string_fact); } catch (std::exception& e) {std::cerr << e.what();exit(1);} try {std::map<std::string, std::string> rwOperation; rwOperation["IO"] = "stdout";+rwOperation["name"] = "unsigned_fact";+rwOperation["types"] = "{\"relation\": {\"arity\": 1, \"auxArity\": 0, \"types\": [\"u:unsigned\"]}}";+IOSystem::getInstance().getWriter(rwOperation, symTable, recordTable)->writeAll(*rel_5_unsigned_fact);+} catch (std::exception& e) {std::cerr << e.what();exit(1);}+try {std::map<std::string, std::string> rwOperation;+rwOperation["IO"] = "stdout"; rwOperation["name"] = "float_fact"; rwOperation["types"] = "{\"relation\": {\"arity\": 1, \"auxArity\": 0, \"types\": [\"f:float\"]}}"; IOSystem::getInstance().getWriter(rwOperation, symTable, recordTable)->writeAll(*rel_1_float_fact);@@ -500,6 +635,14 @@ SymbolTable& getSymbolTable() override { return symTable; }+RecordTable& getRecordTable() override {+return recordTable;+}+void setNumThreads(std::size_t numThreadsValue) override {+SouffleProgram::setNumThreads(numThreadsValue);+symTable.setNumLanes(getNumThreads());+recordTable.setNumLanes(getNumThreads());+} void executeSubroutine(std::string name, const std::vector<RamDomain>& args, std::vector<RamDomain>& ret) override { if (name == "stratum_0") { subroutine_0(args, ret);@@ -513,6 +656,9 @@ if (name == "stratum_3") { subroutine_3(args, ret); return;}+if (name == "stratum_4") {+subroutine_4(args, ret);+return;} fatal("unknown subroutine"); } #ifdef _MSC_VER@@ -520,15 +666,15 @@ #endif // _MSC_VER void subroutine_0(const std::vector<RamDomain>& args, std::vector<RamDomain>& ret) { if (performIO) {-try {std::map<std::string, std::string> directiveMap({{"IO","file"},{"attributeNames","x"},{"fact-dir","."},{"name","string_fact"},{"operation","input"},{"params","{\"records\": {}, \"relation\": {\"arity\": 1, \"auxArity\": 0, \"params\": [\"x\"]}}"},{"types","{\"ADTs\": {}, \"records\": {}, \"relation\": {\"arity\": 1, \"auxArity\": 0, \"types\": [\"s:symbol\"]}}"}});+try {std::map<std::string, std::string> directiveMap({{"IO","file"},{"attributeNames","x"},{"auxArity","0"},{"fact-dir","."},{"name","float_fact"},{"operation","input"},{"params","{\"records\": {}, \"relation\": {\"arity\": 1, \"params\": [\"x\"]}}"},{"types","{\"ADTs\": {}, \"records\": {}, \"relation\": {\"arity\": 1, \"types\": [\"f:float\"]}}"}}); if (!inputDirectory.empty()) {directiveMap["fact-dir"] = inputDirectory;}-IOSystem::getInstance().getReader(directiveMap, symTable, recordTable)->readAll(*rel_3_string_fact);-} catch (std::exception& e) {std::cerr << "Error loading data: " << e.what() << '\n';}+IOSystem::getInstance().getReader(directiveMap, symTable, recordTable)->readAll(*rel_1_float_fact);+} catch (std::exception& e) {std::cerr << "Error loading float_fact data: " << e.what() << '\n';} } if (performIO) {-try {std::map<std::string, std::string> directiveMap({{"IO","file"},{"attributeNames","x"},{"name","string_fact"},{"operation","output"},{"output-dir","."},{"params","{\"records\": {}, \"relation\": {\"arity\": 1, \"auxArity\": 0, \"params\": [\"x\"]}}"},{"types","{\"ADTs\": {}, \"records\": {}, \"relation\": {\"arity\": 1, \"auxArity\": 0, \"types\": [\"s:symbol\"]}}"}});+try {std::map<std::string, std::string> directiveMap({{"IO","file"},{"attributeNames","x"},{"auxArity","0"},{"name","float_fact"},{"operation","output"},{"output-dir","."},{"params","{\"records\": {}, \"relation\": {\"arity\": 1, \"params\": [\"x\"]}}"},{"types","{\"ADTs\": {}, \"records\": {}, \"relation\": {\"arity\": 1, \"types\": [\"f:float\"]}}"}}); if (!outputDirectory.empty()) {directiveMap["output-dir"] = outputDirectory;}-IOSystem::getInstance().getWriter(directiveMap, symTable, recordTable)->writeAll(*rel_3_string_fact);+IOSystem::getInstance().getWriter(directiveMap, symTable, recordTable)->writeAll(*rel_1_float_fact); } catch (std::exception& e) {std::cerr << e.what();exit(1);} } }@@ -540,15 +686,15 @@ #endif // _MSC_VER void subroutine_1(const std::vector<RamDomain>& args, std::vector<RamDomain>& ret) { if (performIO) {-try {std::map<std::string, std::string> directiveMap({{"IO","file"},{"attributeNames","x"},{"fact-dir","."},{"name","number_fact"},{"operation","input"},{"params","{\"records\": {}, \"relation\": {\"arity\": 1, \"auxArity\": 0, \"params\": [\"x\"]}}"},{"types","{\"ADTs\": {}, \"records\": {}, \"relation\": {\"arity\": 1, \"auxArity\": 0, \"types\": [\"i:number\"]}}"}});+try {std::map<std::string, std::string> directiveMap({{"IO","file"},{"attributeNames","a\tb\tc\td"},{"auxArity","0"},{"fact-dir","."},{"name","large_record"},{"operation","input"},{"params","{\"records\": {}, \"relation\": {\"arity\": 4, \"params\": [\"a\", \"b\", \"c\", \"d\"]}}"},{"types","{\"ADTs\": {}, \"records\": {}, \"relation\": {\"arity\": 4, \"types\": [\"i:number\", \"i:number\", \"i:number\", \"i:number\"]}}"}}); if (!inputDirectory.empty()) {directiveMap["fact-dir"] = inputDirectory;}-IOSystem::getInstance().getReader(directiveMap, symTable, recordTable)->readAll(*rel_2_number_fact);-} catch (std::exception& e) {std::cerr << "Error loading data: " << e.what() << '\n';}+IOSystem::getInstance().getReader(directiveMap, symTable, recordTable)->readAll(*rel_2_large_record);+} catch (std::exception& e) {std::cerr << "Error loading large_record data: " << e.what() << '\n';} } if (performIO) {-try {std::map<std::string, std::string> directiveMap({{"IO","file"},{"attributeNames","x"},{"name","number_fact"},{"operation","output"},{"output-dir","."},{"params","{\"records\": {}, \"relation\": {\"arity\": 1, \"auxArity\": 0, \"params\": [\"x\"]}}"},{"types","{\"ADTs\": {}, \"records\": {}, \"relation\": {\"arity\": 1, \"auxArity\": 0, \"types\": [\"i:number\"]}}"}});+try {std::map<std::string, std::string> directiveMap({{"IO","file"},{"attributeNames","a\tb\tc\td"},{"auxArity","0"},{"name","large_record"},{"operation","output"},{"output-dir","."},{"params","{\"records\": {}, \"relation\": {\"arity\": 4, \"params\": [\"a\", \"b\", \"c\", \"d\"]}}"},{"types","{\"ADTs\": {}, \"records\": {}, \"relation\": {\"arity\": 4, \"types\": [\"i:number\", \"i:number\", \"i:number\", \"i:number\"]}}"}}); if (!outputDirectory.empty()) {directiveMap["output-dir"] = outputDirectory;}-IOSystem::getInstance().getWriter(directiveMap, symTable, recordTable)->writeAll(*rel_2_number_fact);+IOSystem::getInstance().getWriter(directiveMap, symTable, recordTable)->writeAll(*rel_2_large_record); } catch (std::exception& e) {std::cerr << e.what();exit(1);} } }@@ -560,15 +706,15 @@ #endif // _MSC_VER void subroutine_2(const std::vector<RamDomain>& args, std::vector<RamDomain>& ret) { if (performIO) {-try {std::map<std::string, std::string> directiveMap({{"IO","file"},{"attributeNames","x"},{"fact-dir","."},{"name","unsigned_fact"},{"operation","input"},{"params","{\"records\": {}, \"relation\": {\"arity\": 1, \"auxArity\": 0, \"params\": [\"x\"]}}"},{"types","{\"ADTs\": {}, \"records\": {}, \"relation\": {\"arity\": 1, \"auxArity\": 0, \"types\": [\"u:unsigned\"]}}"}});+try {std::map<std::string, std::string> directiveMap({{"IO","file"},{"attributeNames","x"},{"auxArity","0"},{"fact-dir","."},{"name","number_fact"},{"operation","input"},{"params","{\"records\": {}, \"relation\": {\"arity\": 1, \"params\": [\"x\"]}}"},{"types","{\"ADTs\": {}, \"records\": {}, \"relation\": {\"arity\": 1, \"types\": [\"i:number\"]}}"}}); if (!inputDirectory.empty()) {directiveMap["fact-dir"] = inputDirectory;}-IOSystem::getInstance().getReader(directiveMap, symTable, recordTable)->readAll(*rel_4_unsigned_fact);-} catch (std::exception& e) {std::cerr << "Error loading data: " << e.what() << '\n';}+IOSystem::getInstance().getReader(directiveMap, symTable, recordTable)->readAll(*rel_3_number_fact);+} catch (std::exception& e) {std::cerr << "Error loading number_fact data: " << e.what() << '\n';} } if (performIO) {-try {std::map<std::string, std::string> directiveMap({{"IO","file"},{"attributeNames","x"},{"name","unsigned_fact"},{"operation","output"},{"output-dir","."},{"params","{\"records\": {}, \"relation\": {\"arity\": 1, \"auxArity\": 0, \"params\": [\"x\"]}}"},{"types","{\"ADTs\": {}, \"records\": {}, \"relation\": {\"arity\": 1, \"auxArity\": 0, \"types\": [\"u:unsigned\"]}}"}});+try {std::map<std::string, std::string> directiveMap({{"IO","file"},{"attributeNames","x"},{"auxArity","0"},{"name","number_fact"},{"operation","output"},{"output-dir","."},{"params","{\"records\": {}, \"relation\": {\"arity\": 1, \"params\": [\"x\"]}}"},{"types","{\"ADTs\": {}, \"records\": {}, \"relation\": {\"arity\": 1, \"types\": [\"i:number\"]}}"}}); if (!outputDirectory.empty()) {directiveMap["output-dir"] = outputDirectory;}-IOSystem::getInstance().getWriter(directiveMap, symTable, recordTable)->writeAll(*rel_4_unsigned_fact);+IOSystem::getInstance().getWriter(directiveMap, symTable, recordTable)->writeAll(*rel_3_number_fact); } catch (std::exception& e) {std::cerr << e.what();exit(1);} } }@@ -580,24 +726,44 @@ #endif // _MSC_VER void subroutine_3(const std::vector<RamDomain>& args, std::vector<RamDomain>& ret) { if (performIO) {-try {std::map<std::string, std::string> directiveMap({{"IO","file"},{"attributeNames","x"},{"fact-dir","."},{"name","float_fact"},{"operation","input"},{"params","{\"records\": {}, \"relation\": {\"arity\": 1, \"auxArity\": 0, \"params\": [\"x\"]}}"},{"types","{\"ADTs\": {}, \"records\": {}, \"relation\": {\"arity\": 1, \"auxArity\": 0, \"types\": [\"f:float\"]}}"}});+try {std::map<std::string, std::string> directiveMap({{"IO","file"},{"attributeNames","x"},{"auxArity","0"},{"fact-dir","."},{"name","string_fact"},{"operation","input"},{"params","{\"records\": {}, \"relation\": {\"arity\": 1, \"params\": [\"x\"]}}"},{"types","{\"ADTs\": {}, \"records\": {}, \"relation\": {\"arity\": 1, \"types\": [\"s:symbol\"]}}"}}); if (!inputDirectory.empty()) {directiveMap["fact-dir"] = inputDirectory;}-IOSystem::getInstance().getReader(directiveMap, symTable, recordTable)->readAll(*rel_1_float_fact);-} catch (std::exception& e) {std::cerr << "Error loading data: " << e.what() << '\n';}+IOSystem::getInstance().getReader(directiveMap, symTable, recordTable)->readAll(*rel_4_string_fact);+} catch (std::exception& e) {std::cerr << "Error loading string_fact data: " << e.what() << '\n';} } if (performIO) {-try {std::map<std::string, std::string> directiveMap({{"IO","file"},{"attributeNames","x"},{"name","float_fact"},{"operation","output"},{"output-dir","."},{"params","{\"records\": {}, \"relation\": {\"arity\": 1, \"auxArity\": 0, \"params\": [\"x\"]}}"},{"types","{\"ADTs\": {}, \"records\": {}, \"relation\": {\"arity\": 1, \"auxArity\": 0, \"types\": [\"f:float\"]}}"}});+try {std::map<std::string, std::string> directiveMap({{"IO","file"},{"attributeNames","x"},{"auxArity","0"},{"name","string_fact"},{"operation","output"},{"output-dir","."},{"params","{\"records\": {}, \"relation\": {\"arity\": 1, \"params\": [\"x\"]}}"},{"types","{\"ADTs\": {}, \"records\": {}, \"relation\": {\"arity\": 1, \"types\": [\"s:symbol\"]}}"}}); if (!outputDirectory.empty()) {directiveMap["output-dir"] = outputDirectory;}-IOSystem::getInstance().getWriter(directiveMap, symTable, recordTable)->writeAll(*rel_1_float_fact);+IOSystem::getInstance().getWriter(directiveMap, symTable, recordTable)->writeAll(*rel_4_string_fact); } catch (std::exception& e) {std::cerr << e.what();exit(1);} } } #ifdef _MSC_VER #pragma warning(default: 4100) #endif // _MSC_VER+#ifdef _MSC_VER+#pragma warning(disable: 4100)+#endif // _MSC_VER+void subroutine_4(const std::vector<RamDomain>& args, std::vector<RamDomain>& ret) {+if (performIO) {+try {std::map<std::string, std::string> directiveMap({{"IO","file"},{"attributeNames","x"},{"auxArity","0"},{"fact-dir","."},{"name","unsigned_fact"},{"operation","input"},{"params","{\"records\": {}, \"relation\": {\"arity\": 1, \"params\": [\"x\"]}}"},{"types","{\"ADTs\": {}, \"records\": {}, \"relation\": {\"arity\": 1, \"types\": [\"u:unsigned\"]}}"}});+if (!inputDirectory.empty()) {directiveMap["fact-dir"] = inputDirectory;}+IOSystem::getInstance().getReader(directiveMap, symTable, recordTable)->readAll(*rel_5_unsigned_fact);+} catch (std::exception& e) {std::cerr << "Error loading unsigned_fact data: " << e.what() << '\n';}+}+if (performIO) {+try {std::map<std::string, std::string> directiveMap({{"IO","file"},{"attributeNames","x"},{"auxArity","0"},{"name","unsigned_fact"},{"operation","output"},{"output-dir","."},{"params","{\"records\": {}, \"relation\": {\"arity\": 1, \"params\": [\"x\"]}}"},{"types","{\"ADTs\": {}, \"records\": {}, \"relation\": {\"arity\": 1, \"types\": [\"u:unsigned\"]}}"}});+if (!outputDirectory.empty()) {directiveMap["output-dir"] = outputDirectory;}+IOSystem::getInstance().getWriter(directiveMap, symTable, recordTable)->writeAll(*rel_5_unsigned_fact);+} catch (std::exception& e) {std::cerr << e.what();exit(1);}+}+}+#ifdef _MSC_VER+#pragma warning(default: 4100)+#endif // _MSC_VER }; SouffleProgram *newInstance_round_trip(){return new Sf_round_trip;}-SymbolTable *getST_round_trip(SouffleProgram *p){return &reinterpret_cast<Sf_round_trip*>(p)->symTable;}+SymbolTable *getST_round_trip(SouffleProgram *p){return &reinterpret_cast<Sf_round_trip*>(p)->getSymbolTable();}  #ifdef __EMBEDDED_SOUFFLE__ class factory_Sf_round_trip: public souffle::ProgramFactory {