packages feed

haskoin-store-1.16.0: test/Haskoin/Store/Database/WriterSpec.hs

module Haskoin.Store.Database.WriterSpec (spec) where

import Control.Monad (forM_)
import Data.ByteString (ByteString)
import Data.ByteString qualified as B
import Data.Default (def)
import Data.IntMap.Strict qualified as IntMap
import Data.Maybe (fromMaybe)
import Data.Serialize (encode)
import Data.Word (Word32, Word64)
import Database.RocksDB (columnFamilies)
import Haskoin
  ( Address,
    OutPoint (..),
    Tx (..),
    TxHash (..),
    TxIn (..),
    TxOut (..),
    btcRegTest,
    eitherToMaybe,
    scriptToAddressBS,
    sha256,
    txHash,
  )
import Haskoin.Node (LogLevel (..), Logger (..))
import Haskoin.Store.Data
  ( Balance (..),
    BlockRef (..),
    Spender (..),
    TxData (..),
    TxRef (..),
    Unspent (..),
  )
import Haskoin.Store.Database.Query qualified as Q
import Haskoin.Store.Database.Reader (DatabaseReader (..))
import Haskoin.Store.Database.Types (BalVal (..))
import Haskoin.Store.Database.Writer
import Haskoin.Store.Histories (dump, openLayered)
import Haskoin.Store.Integration.Temp (withSystemTempDirectory)
import Test.Hspec

spec :: Spec
spec = describe "a batch of writes" $ do
  describe "its read cache" $ do
    it "never hides a change of the batch, whether the key was read ahead or not" $
      withDB $ \db -> do
        batch db $ \w -> insertUnspent w (coin 1) >> setBalance w (balanceOf 1 10)
        batch db $ \w -> do
          -- one that the database has, one that it has not
          readAheadUnspents w [(coin 1).outpoint, (coin 2).outpoint]
          readAheadBalances w [address 1, address 2]
          getUnspent w (coin 1).outpoint `shouldReturn` Just (coin 1)
          getUnspent w (coin 2).outpoint `shouldReturn` Nothing
          getBalance w (address 1) `shouldReturn` Just (balanceOf 1 10)
          getBalance w (address 2) `shouldReturn` Nothing
          deleteUnspent w (coin 1).outpoint
          insertUnspent w (coin 2)
          setBalance w (balanceOf 1 20)
          setBalance w (balanceOf 2 30)
          -- reading ahead again changes nothing either
          readAheadUnspents w [(coin 1).outpoint, (coin 2).outpoint]
          readAheadBalances w [address 1, address 2]
          getUnspent w (coin 1).outpoint `shouldReturn` Nothing
          getUnspent w (coin 2).outpoint `shouldReturn` Just (coin 2)
          getBalance w (address 1) `shouldReturn` Just (balanceOf 1 20)
          getBalance w (address 2) `shouldReturn` Just (balanceOf 2 30)
          -- a key that was not read ahead is read when asked for
          getUnspent w (coin 3).outpoint `shouldReturn` Nothing
          insertUnspent w (coin 3)
          getUnspent w (coin 3).outpoint `shouldReturn` Just (coin 3)
        Q.getUnspent db (coin 1).outpoint `shouldReturn` Nothing
        Q.getUnspent db (coin 2).outpoint `shouldReturn` Just (coin 2)
        Q.getUnspent db (coin 3).outpoint `shouldReturn` Just (coin 3)
        Q.getBalance db (address 1) `shouldReturn` Just (balanceOf 1 20)

    it "reads the groups of spenders ahead, and adds to them" $
      withDB $ \db -> do
        let made = TxHash (sha256 ("made" :: ByteString))
            spender i = Spender (TxHash (sha256 (encode i))) i
        batch db $ \w -> insertSpender w (OutPoint made 1) (spender 1)
        batch db $ \w -> do
          readAheadSpenders w [OutPoint made 2, OutPoint made 40]
          insertSpender w (OutPoint made 2) (spender 2)
          insertSpender w (OutPoint made 40) (spender 40)
          getChainSpender w (OutPoint made 1) `shouldReturn` Just (spender 1)
        Q.getChainSpenders db made
          `shouldReturn` IntMap.fromList [(1, spender 1), (2, spender 2), (40, spender 40)]

    it "gives the same answers without the cache" $
      withDB $ \db -> do
        batch db $ \w -> insertUnspent w (coin 1)
        batch db $ \w -> do
          let plain = withoutReadCache w
          readsAhead plain `shouldBe` False
          readAheadUnspents plain [(coin 1).outpoint]
          getUnspent plain (coin 1).outpoint `shouldReturn` Just (coin 1)
          -- a change made through one is seen through the other
          deleteUnspent plain (coin 1).outpoint
          getUnspent w (coin 1).outpoint `shouldReturn` Nothing

  describe "its merges: changes to values that it has not read" $ do
    it "adds differences to a balance that it has not read, and shows the sum to its own reads" $
      withDB $ \db -> do
        batch db $ \w -> setBalance w (balanceOf 1 10)
        batch db $ \w -> do
          mergeBalance w (address 1) (def :: BalVal) {confirmed = 5, txs = 1}
          mergeBalance w (address 1) (def :: BalVal) {confirmed = negate 3, utxo = 2}
          mergeBalance w (address 2) (def :: BalVal) {received = 7, txs = 1}
          -- the database is read for the value, and the differences added
          getBalance w (address 1) `shouldReturn` Just (numbers 1 12 3 2 10)
          getBalance w (address 2) `shouldReturn` Just (numbers 2 0 0 1 7)
          getBalance w (address 3) `shouldReturn` Nothing
          -- and the same through a writer that remembers no read
          getBalance (withoutReadCache w) (address 1) `shouldReturn` Just (numbers 1 12 3 2 10)
        Q.getBalance db (address 1) `shouldReturn` Just (numbers 1 12 3 2 10)
        Q.getBalance db (address 2) `shouldReturn` Just (numbers 2 0 0 1 7)
        Q.getBalance db (address 3) `shouldReturn` Nothing
        -- one batch after another, each without a read
        batch db $ \w -> mergeBalance w (address 2) (def :: BalVal) {received = 1}
        batch db $ \w -> mergeBalance w (address 2) (def :: BalVal) {received = 1}
        Q.getBalance db (address 2) `shouldReturn` Just (numbers 2 0 0 1 9)

    it "applies differences to a balance that it has whole, set or deleted" $
      withDB $ \db -> do
        batch db $ \w -> setBalance w (balanceOf 1 10) >> setBalance w (balanceOf 2 20)
        batch db $ \w -> do
          setBalance w (numbers 1 100 1 1 100)
          mergeBalance w (address 1) (def :: BalVal) {confirmed = 1}
          getBalance w (address 1) `shouldReturn` Just (numbers 1 101 1 1 100)
          deleteBalance w (address 2)
          mergeBalance w (address 2) (def :: BalVal) {confirmed = 4, txs = 1}
          getBalance w (address 2) `shouldReturn` Just (numbers 2 4 0 1 0)
        Q.getBalance db (address 1) `shouldReturn` Just (numbers 1 101 1 1 100)
        -- from zero, not from the 20 that the batch deleted
        Q.getBalance db (address 2) `shouldReturn` Just (numbers 2 4 0 1 0)

    it "lets a whole balance take the place of the differences, which whoever set it had read" $
      withDB $ \db -> do
        batch db $ \w -> setBalance w (balanceOf 1 10) >> setBalance w (balanceOf 2 20)
        batch db $ \w -> do
          mergeBalance w (address 1) (def :: BalVal) {confirmed = 5}
          mergeBalance w (address 2) (def :: BalVal) {confirmed = 5}
          -- as the revert of a block does: read, change, write whole
          Just b <- getBalance w (address 1)
          b.confirmed `shouldBe` 15
          setBalance w (b :: Balance) {confirmed = b.confirmed - 1}
          getBalance w (address 1) `shouldReturn` Just (numbers 1 14 1 1 10)
          deleteBalance w (address 2)
          getBalance w (address 2) `shouldReturn` Nothing
          -- and differences after that are applied to the whole value
          mergeBalance w (address 1) (def :: BalVal) {confirmed = 1}
          getBalance w (address 1) `shouldReturn` Just (numbers 1 15 1 1 10)
        Q.getBalance db (address 1) `shouldReturn` Just (numbers 1 15 1 1 10)
        Q.getBalance db (address 2) `shouldReturn` Nothing

    it "adds spenders to groups that it has not read, and shows them to each of its reads" $
      withDB $ \db -> do
        let made = TxHash (sha256 ("made" :: ByteString))
            other = TxHash (sha256 ("other" :: ByteString))
            spender i = Spender (TxHash (sha256 (encode i))) i
            out = OutPoint made
        batch db $ \w -> insertSpender w (out 1) (spender 1) >> insertSpender w (out 40) (spender 40)
        batch db $ \w -> do
          mergeSpender w (out 2) (spender 2)
          mergeSpender w (out 3) (spender 3)
          mergeSpender w (out 17) (spender 17)
          mergeSpender w (OutPoint other 0) (spender 9)
          -- one output at a time: what the database has, and what is added
          getChainSpender w (out 1) `shouldReturn` Just (spender 1)
          getChainSpender w (out 2) `shouldReturn` Just (spender 2)
          getChainSpender w (out 17) `shouldReturn` Just (spender 17)
          getChainSpender w (out 4) `shouldReturn` Nothing
          getChainSpender w (OutPoint other 0) `shouldReturn` Just (spender 9)
          getChainSpender (withoutReadCache w) (out 3) `shouldReturn` Just (spender 3)
          -- and all those of a transaction, over its three groups
          getChainSpenders w made 48
            `shouldReturn` IntMap.fromList [(i, spender (fromIntegral i)) | i <- [1, 2, 3, 17, 40]]
          getChainSpenders w other 1 `shouldReturn` IntMap.fromList [(0, spender 9)]
        Q.getChainSpenders db made
          `shouldReturn` IntMap.fromList [(i, spender (fromIntegral i)) | i <- [1, 2, 3, 17, 40]]
        Q.getChainSpender db (OutPoint other 0) `shouldReturn` Just (spender 9)
        -- the newer spender wins for an output that already has one
        batch db $ \w -> mergeSpender w (out 2) (spender 22)
        Q.getChainSpender db (out 2) `shouldReturn` Just (spender 22)

    it "puts a spender in a group that it has whole, and lets a whole group take the place of what it added" $
      withDB $ \db -> do
        let made = TxHash (sha256 ("made" :: ByteString))
            spender i = Spender (TxHash (sha256 (encode i))) i
            out = OutPoint made
            group is = IntMap.fromList [(i, spender (fromIntegral i)) | i <- is]
        batch db $ \w -> mapM_ (\i -> insertSpender w (out i) (spender i)) [1, 20, 35]
        batch db $ \w -> do
          -- added to, then read and written whole, as a revert does
          mergeSpender w (out 2) (spender 2)
          deleteSpender w (out 1)
          getChainSpenders w made 48 `shouldReturn` group [2, 20, 35]
          -- whole, then added to
          insertSpender w (out 3) (spender 3)
          mergeSpender w (out 4) (spender 4)
          getChainSpenders w made 48 `shouldReturn` group [2, 3, 4, 20, 35]
          -- a group that the batch empties is deleted, and what is added
          -- afterwards is all that it has
          deleteSpender w (out 20)
          getChainSpender w (out 20) `shouldReturn` Nothing
          mergeSpender w (out 21) (spender 21)
          getChainSpenders w made 48 `shouldReturn` group [2, 3, 4, 21, 35]
          -- added to, then a spender put in by reading the group: what
          -- is written whole has what the database had and what was added
          mergeSpender w (out 33) (spender 33)
          insertSpender w (out 34) (spender 34)
          getChainSpender w (out 33) `shouldReturn` Just (spender 33)
          getChainSpenders w made 48 `shouldReturn` group [2, 3, 4, 21, 33, 34, 35]
        Q.getChainSpenders db made `shouldReturn` group [2, 3, 4, 21, 33, 34, 35]

  it "writes a large batch, packed on all cores, as it writes the same changes a few at a time" $
    withDB $ \large ->
      withDB $ \small -> do
        -- Two thousand unspent outputs and balances and as many
        -- transactions, then a thousand transactions of three kilobytes,
        -- which fill the buffer of each core more than once. Among them,
        -- so that the records around them have to keep their place, one
        -- with a script too long to be copied into a shared buffer and one
        -- larger than such a buffer.
        let changes =
              concat
                [ [ \w -> insertUnspent w (coin i),
                    \w -> setBalance w (balanceOf i i),
                    \w -> insertAddrTx w (address i) (TxRef (BlockRef 7 i) (txHash (payment i))),
                    \w -> insertAddrUnspent w (address i) (coin i),
                    \w -> insertTx w (txData i (payment i))
                  ]
                | i <- [1 .. 2000]
                ]
                <> [\w -> insertTx w (txData 1 long), \w -> insertTx w (txData 2 big)]
                <> [\w -> insertTx w (txData i (wide i)) | i <- [1 .. 1000]]
                -- and merge records, for balances and for groups of
                -- spenders that have no value yet: two for each key, which
                -- one batch makes one record of and forty at a time two
                <> concat
                  [ [ \w -> mergeBalance w (address i) (def :: BalVal) {confirmed = fromIntegral i, txs = 1},
                      \w -> mergeSpender w (OutPoint (txHash (payment i)) (i `mod` 40)) (Spender (txHash (wide i)) i)
                    ]
                  | i <- [5001 .. 5600] <> [5001 .. 5600]
                  ]
        report <- snd <$> runWriterReport btcRegTest large quiet (\w -> mapM_ ($ w) changes)
        report.operations `shouldBe` 12202
        -- a few at a time, a batch is not packed: each operation is handed
        -- to RocksDB by itself, as all of them were before
        forM_ (pieces 40 changes) $ \some -> batch small (\w -> mapM_ ($ w) some)
        forM_ (zip (columnFamilies large.db) (columnFamilies small.db)) $ \(a, b) -> do
          written <- dump large a
          expected <- dump small b
          length written `shouldBe` length expected
          written `shouldBe` expected
        entries <- concat <$> mapM (dump large) (columnFamilies large.db)
        length entries `shouldBe` 12202
        -- every change set a key of its own, or merged into one that had
        -- no value, where the one merge record that the batch makes of
        -- the two is the value. So the bytes of the batch are those of the
        -- keys and values that the database now has
        report.bytes `shouldBe` sum [B.length k + B.length v | (k, v) <- entries]
        report.bytes `shouldSatisfy` (> 3600000)
        Q.getBalance large (address 5001) `shouldReturn` Just (numbers 5001 10002 0 2 0)

quiet :: Logger
quiet = Logger {level = LevelError, source = "", push = \_ -> return ()}

withDB :: (DatabaseReader -> IO a) -> IO a
withDB action = withSystemTempDirectory "haskoin-store-writer-" $ \dir -> openLayered btcRegTest dir action

batch :: DatabaseReader -> (Writer -> IO a) -> IO a
batch db = runWriter btcRegTest db quiet

address :: Word32 -> Address
address i = fromMaybe (error "no address") (eitherToMaybe (scriptToAddressBS btcRegTest (scriptOf i)))

scriptOf :: Word32 -> ByteString
scriptOf i = "\x76\xa9\x14" <> B.take 20 (encode (sha256 (encode i))) <> "\x88\xac"

-- | An unspent output of a number, paid to the address of that number.
coin :: Word32 -> Unspent
coin i =
  Unspent
    { block = BlockRef 7 i,
      outpoint = OutPoint (TxHash (sha256 (encode i))) (i `mod` 3),
      value = 1000 + fromIntegral i,
      script = scriptOf i,
      address = Just (address i)
    }

-- | A balance of an address by its numbers: confirmed, unspent outputs,
-- transactions and received.
numbers :: Word32 -> Word64 -> Word64 -> Word64 -> Word64 -> Balance
numbers i confirmed utxo txs received =
  Balance {address = address i, confirmed, unconfirmed = 0, utxo, txs, received}

balanceOf :: Word32 -> Word32 -> Balance
balanceOf i n =
  Balance
    { address = address i,
      confirmed = fromIntegral n,
      unconfirmed = 0,
      utxo = 1,
      txs = 1,
      received = fromIntegral n
    }

payment :: Word32 -> Tx
payment i = Tx 2 [TxIn (coin i).outpoint "" maxBound] [TxOut 900 (scriptOf (i + 1))] [] 0

-- | A transaction with an output script of twenty kilobytes.
long :: Tx
long = Tx 2 [TxIn (coin 1).outpoint "" maxBound] [TxOut 1 (B.replicate 20000 0x6a)] [] 0

-- | A transaction of three kilobytes with a number: short enough to be
-- copied into a shared buffer, and a few dozen of them fill one.
wide :: Word32 -> Tx
wide i = Tx 2 [TxIn (OutPoint (TxHash (sha256 (encode i))) 7) "" maxBound] [TxOut 1 (B.replicate 3000 0x6a)] [] 0

-- | A transaction of four hundred kilobytes, in outputs of a kilobyte.
big :: Tx
big = Tx 2 [TxIn (coin 2).outpoint "" maxBound] (replicate 400 (TxOut 1 (B.replicate 1000 0x6a))) [] 0

txData :: Word32 -> Tx -> TxData
txData i tx =
  TxData
    { block = BlockRef 7 i,
      tx = tx,
      prevs = mempty,
      deleted = False,
      rbf = False,
      timestamp = 1700000000,
      spenders = mempty
    }

pieces :: Int -> [a] -> [[a]]
pieces _ [] = []
pieces n xs = let (a, b) = splitAt n xs in a : pieces n b