packages feed

haskoin-store-1.16.0: src/Haskoin/Store/XPub.hs

{-# LANGUAGE MultiWayIf #-}
{-# LANGUAGE TemplateHaskell #-}

-- |
-- Module      : Haskoin.Store.XPub
-- Copyright   : (c) Haskoin Developers
-- License     : MIT
-- Maintainer  : jprupp@protonmail.ch
-- Stability   : experimental
-- Portability : POSIX
--
-- Extended public keys (xpubs) answered from a cache and indexed in the
-- background: what the parts of that design share. @docs\/xpub-indexing.md@
-- explains the design; this module holds its vocabulary.
--
-- An xpub with many used addresses is slow to read from the database: each
-- address is derived and read, then its transactions or outputs. So no
-- request reads an xpub from the database itself. It reads the xpub from a
-- cache, and on a miss it hands the xpub to the indexer
-- ("Haskoin.Store.XPub.Indexer"), which reads it in the background and
-- writes it to the cache, while the request waits or is told to come back.
--
-- The cache is the fleet's, in Redis ("Haskoin.Store.Fleet.XPub"), shared
-- by every store, and it cannot be turned off: its size is all there is to
-- set. It gives the reads of a t'XPubReads', each of one version of an
-- entry. It keeps the xpubs that cost most to index again and were used
-- most recently (GreedyDual, 'greedyLua'), and counts that cost in the
-- units of the prices below: they rank entries, and charge nobody. A job
-- settles and tells its size through a t'Budget'.
module Haskoin.Store.XPub
  ( -- * Reading a cache
    XPubReads (..),
    EntryMoved (..),

    -- * Paying for a job
    Budget (..),
    noBudget,

    -- * Prices
    addressPrice,
    entryPrice,
    bodyPrice,
    indexCost,

    -- * Keeping up with the store
    inSync,
    txAddresses,

    -- * Eviction in Redis
    greedyLua,
  )
where

import Control.Exception (Exception)
import Data.ByteString (ByteString)
import Data.Either (rights)
import Data.FileEmbed (embedFile, makeRelativeToProject)
import Data.IntMap.Strict qualified as IntMap
import Data.List (nub)
import Data.Word (Word32)
import Haskoin
  ( Address,
    BlockNode (..),
    Network,
    Tx (..),
    TxHash,
    TxOut (..),
    headerHash,
    scriptToAddressBS,
  )
import Haskoin.Node (Chain, chainGetBest)
import Haskoin.Store.Common (Limits (..))
import Haskoin.Store.Data
  ( Prev (..),
    TxData (..),
    TxRef (..),
    XPubBal (..),
    XPubSpec,
    XPubUnspent,
  )
import Haskoin.Store.Database.Query qualified as Q
import Haskoin.Store.Database.Reader (DatabaseReader (..))

-- | What the web server reads from the cache of xpubs. Each
-- read is 'Nothing' when the cache does not have the xpub, and marks it as
-- used when it does.
data XPubReads = XPubReads
  { -- | balances of the addresses up to the gap
    balances :: XPubSpec -> IO (Maybe [XPubBal]),
    -- | a page of transactions, newest first, as the database gives it
    transactions :: XPubSpec -> Limits -> IO (Maybe [TxRef]),
    -- | number of transactions
    count :: XPubSpec -> IO (Maybe Word32),
    -- | a page of unspent outputs, newest first, given the balances, which
    -- the Redis cache needs for the paths of the addresses
    unspents :: XPubSpec -> [XPubBal] -> Limits -> IO (Maybe [XPubUnspent]),
    -- | bodies of transactions, with their spenders, in the order of the
    -- hashes; 'Nothing' for those the cache does not have
    bodies :: [TxHash] -> IO [Maybe TxData],
    -- | the bodies of transactions as the entries that the reads read have
    -- them, wherever the bodies come from: the confirmation of each
    -- transaction of an entry, and who spends each output of its wallet, at
    -- the version that the request read; the rest as the bodies have it
    -- (@docs\/xpub-indexing.md@, Exact reads)
    wallet :: [Maybe TxData] -> IO [Maybe TxData]
  }

-- | A request read an entry of a cache that changed before the request was
-- done: a write landed between two of its reads, or the entry went. The
-- reads of one request are of one version of each xpub, so the web server
-- answers it again from the start, before anything of it was sent
-- ('Haskoin.Store.Fleet.Member.fleetSession', @docs\/xpub-indexing.md@,
-- Exact reads).
data EntryMoved = EntryMoved
  deriving (Eq, Show)

instance Exception EntryMoved

-- | What a job of the indexer calls as it goes. 'settle' is called at the
-- safe points of a job, where it holds no iterator of the database: between
-- batches of addresses, between calls into the database. 'finish' is
-- called when the job is done, however it ends. Every job of the store gets
-- 'noBudget', in which only 'sized', set by the indexer, does anything:
-- the jobs are not paced. The seam is kept so that a job could be paced
-- or measured without changing the indexer.
data Budget = Budget
  { settle :: IO (),
    finish :: IO (),
    -- | the job tells how many transactions it has to read, once it knows:
    -- the indexer tells from it when the job should be done
    -- ('Haskoin.Store.XPub.Indexer.joinEstimate')
    sized :: Int -> IO ()
  }

-- | A budget that measures nothing and never waits.
noBudget :: Budget
noBudget = Budget {settle = return (), finish = return (), sized = \_ -> return ()}

-- | The cost of an address of an xpub: derived, 25, and its balance read
-- by key, 10. First guesses all, in the same arbitrary units.
addressPrice :: Int
addressPrice = 35

-- | The cost of an entry of a scan: a transaction of an address's history,
-- or one of its unspent outputs.
entryPrice :: Int
entryPrice = 2

-- | The cost of the body of a transaction: read by key, 10, and decoded, 5.
bodyPrice :: Int
bodyPrice = 15

-- | What it costs to index an xpub from nothing: its addresses,
-- its transactions and its unspent outputs, and the bodies of the
-- transactions if the cache keeps them. It is the cost that eviction weighs
-- ('greedyLua'): what the cache would pay to read the
-- xpub again.
--
-- It is worked out from what was read rather than from the time the job
-- took, which would count the load of the store with the size of the xpub.
indexCost ::
  -- | addresses
  Int ->
  -- | transactions
  Int ->
  -- | unspent outputs
  Int ->
  -- | bodies
  Int ->
  Int
indexCost addrs txs utxos bods =
  addressPrice * addrs + entryPrice * (txs + utxos) + bodyPrice * bods

-- | Whether the database has the best header of the node. A store that is
-- not in sync takes no job of the indexer: it would write what knows less
-- than the database of a store that is, and its upkeep would chase every
-- block that it imports while it catches up.
inSync :: DatabaseReader -> Chain -> IO Bool
inSync db chain =
  Q.getBestBlock db >>= \case
    Nothing -> return False
    Just bb -> do
      cb <- chainGetBest chain
      return (cb.height > 0 && headerHash cb.header == bb)

-- | Addresses that a transaction spends from or pays to: those whose cached
-- xpubs it changes.
txAddresses :: Network -> TxData -> [Address]
txAddresses net td =
  nub . rights $
    map (scriptToAddressBS net . (.script)) (IntMap.elems td.prevs)
      <> map (scriptToAddressBS net . (.script)) td.tx.outputs

-- | GreedyDual for the caches of xpubs in Redis, as Lua functions that the
-- scripts which evict or mark a use start with: @greedy_use@, which gives
-- an entry the level plus its cost as its priority, @greedy_evict@, which
-- evicts the lowest priorities past a number and raises the level to each,
-- and @greedy_migrate@, which gives the level to the entries of a sorted
-- set that held times of last use, so that they go first. The cache of
-- the fleet ("Haskoin.Store.Fleet.XPub") runs it in front of its scripts.
-- Between
-- equal priorities Redis goes by the id of the xpub. The file is
-- @src\/Haskoin\/Store\/XPub\/lua\/greedy.lua@; a script that uses it is
-- run with it in front, as @cat greedy.lua script.lua@ for @redis-cli@.
greedyLua :: ByteString
greedyLua = $(makeRelativeToProject "src/Haskoin/Store/XPub/lua/greedy.lua" >>= embedFile)