packages feed

haskoin-store-1.16.1: src/Haskoin/Store/Manager.hs

-- |
-- Module      : Haskoin.Store.Manager
-- Copyright   : (c) Haskoin Developers
-- License     : MIT
-- Maintainer  : jprupp@protonmail.ch
-- Stability   : experimental
-- Portability : POSIX
--
-- Wires the components of a store together: the RocksDB database, the
-- peer-to-peer node, the block store that imports data, and the optional Redis
-- cache. Node events are forwarded to the block store and turned into
-- 'StoreEvent's, which the cache follows.
--
-- Keeping the wiring in one place means each component only knows about the
-- interfaces it uses: the node does not know about the block store, and the
-- block store does not know about the cache. 'withStore' starts them in dependency order
-- and stops them all together.
module Haskoin.Store.Manager
  ( StoreConfig (..),
    cachePrefix,
    Store (..),
    withStore,
  )
where

import Control.Concurrent.Async (link, withAsync)
import Control.Concurrent.STM (STM, atomically)
import Control.Monad (forM_, unless, when)
import Data.ByteString (ByteString)
import Data.ByteString.Char8 qualified as C
import Data.Serialize (Serialize, decode)
import Data.String.Conversions (cs)
import Data.Time.Clock (NominalDiffTime)
import Data.Word (Word32, Word64)
import Database.Redis qualified as Redis
import Haskoin
  ( BlockHash (..),
    Headers (..),
    Inv (..),
    InvType (..),
    InvVector (..),
    Message (..),
    MessageCommand (..),
    Network (..),
    NetworkAddress (..),
    NotFound (..),
    Pong (..),
    Reject (..),
    TxHash (..),
    VarString (..),
    headerHash,
    sockToHostAddress,
  )
import Haskoin.Node
  ( Chain,
    ChainEvent (..),
    LogLevel (..),
    Logger,
    Node (..),
    NodeConfig (..),
    NodeEvent (..),
    PeerEvent (..),
    PeerMgr,
    WithConnection,
    logAt,
    withNode,
    withSource,
  )
import Haskoin.Store.BlockStore
  ( BlockStore,
    BlockStoreConfig (..),
    blockStoreAnnouncedSTM,
    blockStoreBlockSTM,
    blockStoreHeadSTM,
    blockStoreNotFoundSTM,
    blockStorePeerConnectSTM,
    blockStorePeerDisconnectSTM,
    blockStorePongSTM,
    blockStoreTxHashSTM,
    blockStoreTxSTM,
    withBlockStore,
  )
import Haskoin.Store.Common
  ( StoreEvent (..),
    loopForever,
  )
import Haskoin.Store.Database.Migrate (Migration (..), upgradeDatabase)
import Haskoin.Store.Database.Reader
  ( DatabaseReader (..),
    createDataMetrics,
    withLayeredReader,
  )
import Haskoin.Store.Fleet.Member
  ( Member (..),
    MemberConfig (..),
    memberCacheLoop,
    memberEvents,
    memberForget,
    memberLoop,
    memberReceived,
    memberSeen,
    newMember,
    noteBest,
  )
import Haskoin.Store.Fleet.Redis (FleetEvent (..))
import Haskoin.Store.XPub.Service (XPubConfig (..))
import NQE
  ( Inbox,
    Publisher,
    publish,
    publishSTM,
    receive,
    withPublisher,
    withSubscription,
  )
import Network.Socket (SockAddr (..))
import System.Metrics.StatsD
import System.Random (randomIO)

-- | Store mailboxes. Handles to the running components of a store, given by
-- 'withStore'.
data Store = Store
  { -- | peer manager of the node
    peerMgr :: !PeerMgr,
    -- | block header chain of the node
    chain :: !Chain,
    -- | process that imports blocks and transactions
    block :: !BlockStore,
    -- | database reader
    db :: !DatabaseReader,
    -- | the store as a member of its fleet, with the door to the fleet's
    -- cache of xpubs that the web server answers them from; 'Nothing'
    -- without Redis, when the store is unhealthy and answers 503
    fleet :: !(Maybe Member),
    -- | what the store tells its clients: new best blocks and changes of
    -- the mempool. For a store alone they are its own, those that the
    -- block store publishes.
    -- For a member of a fleet they are those of the fleet, read from the
    -- Redis that it shares, so that every server tells the same.
    events :: !(Publisher StoreEvent),
    -- | store events are published here
    pub :: !(Publisher StoreEvent),
    -- | network constants
    net :: !Network,
    -- | how the store opens a connection to a node, for those it makes
    -- besides the ones of the peer manager
    -- ("Haskoin.Store.Web.Publish")
    connect :: !(SockAddr -> WithConnection)
  }

-- | Configuration for a 'Store'.
data StoreConfig = StoreConfig
  { -- | max peers to connect to
    maxPeers :: !Int,
    -- | static set of peers to connect to
    initPeers :: ![String],
    -- | discover new peers
    discover :: !Bool,
    -- | RocksDB database path
    db :: !FilePath,
    -- | network constants
    net :: !Network,
    -- | Redis URL of the fleet: its tip, pool and cache of extended public
    -- keys ("Haskoin.Store.Fleet.Member"); an empty URL for the local Redis.
    -- A store always has one: 'Nothing' is for tests of the block store
    -- alone, which then syncs blocks but answers 503.
    redis :: !(Maybe String),
    -- | gap on extended public key with no transactions
    initGap :: !Word32,
    -- | gap for extended public keys
    gap :: !Word32,
    -- | most xpubs in the fleet's cache, at least 1, by GreedyDual: those
    -- that cost least to index again go first, of those not used for a
    -- while
    redisMaxXPubs :: !Int,
    -- | the indexers of xpubs ("Haskoin.Store.XPub.Service");
    -- 'Haskoin.Store.XPub.Service.defaultXPubConfig' for one thread a core
    xpubs :: !XPubConfig,
    -- | blocks to request from the syncing peer in one batch
    blockWindow :: !Int,
    -- | delete mempool transactions older than this number of days
    mempoolTimeout :: !Int,
    -- | disconnect peer if message not received for this many seconds
    peerTimeout :: !NominalDiffTime,
    -- | the longest that the syncing peer may deliver nothing before another
    -- peer is asked for its blocks; a quick peer is given less
    blockStall :: !NominalDiffTime,
    -- | a peer is asked for blocks if it answers a ping within this long,
    -- or if none does
    blockPing :: !NominalDiffTime,
    -- | disconnect peer if it has been connected this long
    maxPeerLife :: !NominalDiffTime,
    -- | connect to peers using the function 'Haskoin.Node.withConnection'
    connect :: !(SockAddr -> WithConnection),
    -- | StatsD
    stats :: !(Maybe Stats),
    -- | Bloom filters on the column families that are read by key
    bloom :: !Bool,
    -- | RocksDB max_open_files ('Nothing' = unlimited, which is what a
    -- store with a block cache wants: its index and filter blocks are in
    -- the cache, an open file costs a few kilobytes, and a file that is
    -- not open has to be opened for every lookup that lands in it. Without
    -- a block cache every open file keeps its index and filter in memory,
    -- and the limit is what bounds them)
    maxOpenFiles :: !(Maybe Int),
    -- | bytes of RocksDB memtables across all column families ('Nothing' =
    -- unlimited: each column family keeps up to two 64 MiB memtables)
    writeBufferSize :: !(Maybe Int),
    -- | bytes of write-ahead log after which the column families holding the
    -- oldest log file are flushed ('Nothing' = RocksDB default)
    maxWalSize :: !(Maybe Int),
    -- | threads that RocksDB may use at once for flushes and compactions
    -- together ('Nothing' = RocksDB default: two, one of each). More of
    -- them shorten a backlog of compactions where the disk and the cores
    -- have room, and take both from the requests where they have not
    backgroundJobs :: !(Maybe Int),
    -- | bytes of RocksDB block cache shared by all column families, which
    -- also holds their index and filter blocks ('Nothing' = RocksDB default:
    -- 32 MiB per column family, index and filter blocks outside it)
    blockCache :: !(Maybe Int),
    -- | log messages here
    logger :: !Logger
  }

-- | Run a store and pass its handles to the given action. Everything is stopped
-- when the action returns, and an exception in any component is re-thrown in
-- the calling thread.
--
-- The database is opened first, then the node, the block store and the cache
-- (if 'redis' is set).
withStore ::
  StoreConfig ->
  (Store -> IO a) ->
  IO a
withStore cfg action =
  connectDB cfg $ \db ->
    withPublisher $ \pub ->
      withPublisher $ \node_pub ->
        withSubscription node_pub $ \node_sub ->
          withNode (nodeCfg cfg db node_pub) $ \node -> do
            member <- storeMember cfg db
            withBlockStore (blockStoreCfg cfg node pub db member) $ \block_store ->
              withAsync (nodeForwarder block_store pub node_sub) $ \fwd -> do
                link fwd
                withStoreFleet cfg pub member $
                  withFleetEvents cfg pub member $ \events ->
                    action $
                      Store
                        { peerMgr = node.peerMgr,
                          chain = node.chain,
                          block = block_store,
                          db = db,
                          fleet = member,
                          events = events,
                          pub = pub,
                          net = cfg.net,
                          connect = cfg.connect
                        }

-- | What every key of the fleet in Redis starts with.
cachePrefix :: Network -> ByteString
cachePrefix net = "hs2:" <> C.pack net.name <> ":"

-- | The store as a member of a fleet, if the configuration says that it is
-- one and has a Redis to share: under a name of its own, with a lease of
-- ten seconds, views of its last six blocks, and the store's limit on the
-- age of unconfirmed transactions.
storeMember :: StoreConfig -> DatabaseReader -> IO (Maybe Member)
storeMember cfg db = case cfg.redis of
  Just url -> do
    info <-
      if null url
        then return Redis.defaultConnectInfo
        else either fail return (Redis.parseConnectInfo url)
    conn <- Redis.checkedConnect info
    name <- C.pack . show <$> (randomIO :: IO Word64)
    Just
      <$> newMember
        MemberConfig
          { prefix = cachePrefix cfg.net,
            name,
            lease = 10000,
            depth = 6,
            expiry = fromIntegral cfg.mempoolTimeout * 24 * 3600,
            cacheMax = max 1 cfg.redisMaxXPubs,
            indexer = cfg.xpubs.indexer,
            stats = cfg.stats,
            note = \level -> logAt level (withSource "Fleet" cfg.logger)
          }
        conn
        db
  -- without Redis the store syncs blocks, but is unhealthy and answers 503
  Nothing -> return Nothing

-- | Take the store's turns as a member of its fleet while an action runs,
-- if it is one: every two seconds, and whenever the best block changes.
-- Beside them, in a thread of its own, the member builds the entries of
-- the fleet's cache of extended public keys that requests asked for.
withStoreFleet :: StoreConfig -> Publisher StoreEvent -> Maybe Member -> IO a -> IO a
withStoreFleet _ _ Nothing action = action
withStoreFleet _ pub (Just member) action =
  withSubscription pub $ \sub ->
    withAsync (memberLoop member (nextBlock sub)) $ \a ->
      withAsync (memberCacheLoop member) $ \b -> link a >> link b >> action
  where
    -- until the next block, telling the member what the store deletes
    nextBlock sub =
      receive sub >>= \case
        StoreBestBlock _ -> return ()
        StoreMempoolDelete h -> memberForget member [h] >> nextBlock sub
        _ -> nextBlock sub

-- | Run an action with the events that the store tells its clients. A
-- store alone tells its own. A member of a fleet tells those of the
-- fleet, which a thread reads from Redis for as long as the action runs.
withFleetEvents ::
  StoreConfig ->
  Publisher StoreEvent ->
  Maybe Member ->
  (Publisher StoreEvent -> IO a) ->
  IO a
withFleetEvents _ pub Nothing action = action pub
withFleetEvents _ _ (Just member) action =
  withPublisher $ \shared ->
    withAsync (memberEvents member False (tell shared)) $ \a -> link a >> action shared
  where
    tell shared event = forM_ (told event) (`publish` shared)
    told = \case
      FleetBlock h -> StoreBestBlock <$> hash h
      FleetEntered h -> StoreMempoolNew <$> hash h
      FleetLeft h -> StoreMempoolDelete <$> hash h
    hash :: (Serialize x) => ByteString -> Maybe x
    hash = either (const Nothing) Just . decode

-- | Open the database described by the configuration, bring it to the
-- layout that the store runs on ('upgradeDatabase'), and run an action with
-- its reader. A database of version 20 has its mempool moved to an area of
-- its own here, once, before anything reads it; older stores refuse the
-- database from then on.
connectDB ::
  StoreConfig ->
  (DatabaseReader -> IO a) ->
  IO a
connectDB cfg f = do
  stats <- mapM createDataMetrics cfg.stats
  withLayeredReader
    cfg.net
    cfg.initGap
    cfg.gap
    cfg.db
    cfg.bloom
    cfg.maxOpenFiles
    cfg.writeBufferSize
    cfg.maxWalSize
    cfg.backgroundJobs
    cfg.blockCache
    stats
    $ \db -> do
      done <- upgradeDatabase (logAt LevelInfo (withSource "Database" cfg.logger)) db
      when (done.moved > 0) $
        logAt LevelInfo (withSource "Database" cfg.logger) $
          "Moved " <> cs (show done.moved) <> " mempool transactions to their own area"
      when (done.rewritten > 0) $
        logAt LevelInfo (withSource "Database" cfg.logger) $
          "Wrote the records of " <> cs (show done.rewritten) <> " confirmed transactions again, to be written once"
      f db

-- | Configuration of the block store, from the store configuration.
blockStoreCfg ::
  StoreConfig ->
  Node ->
  Publisher StoreEvent ->
  DatabaseReader ->
  Maybe Member ->
  BlockStoreConfig
blockStoreCfg cfg node pub db member =
  BlockStoreConfig
    { chain = node.chain,
      peerMgr = node.peerMgr,
      pub = pub,
      db = db,
      net = cfg.net,
      onBest = maybe (\_ -> return ()) noteBest member,
      onTx = maybe (\_ _ -> return ()) memberReceived member,
      seen = maybe (\_ -> return mempty) memberSeen member,
      blockWindow = cfg.blockWindow,
      mempoolTimeout = cfg.mempoolTimeout,
      peerTimeout = cfg.peerTimeout,
      blockStall = cfg.blockStall,
      blockPing = cfg.blockPing,
      stats = cfg.stats,
      logger = withSource "BlockStore" cfg.logger
    }

-- | Configuration of the node, from the store configuration. The node keeps its
-- headers in the default column family of the store database.
nodeCfg ::
  StoreConfig ->
  DatabaseReader ->
  Publisher NodeEvent ->
  NodeConfig
nodeCfg cfg db pub =
  NodeConfig
    { maxPeers = cfg.maxPeers,
      db = db.db,
      cf = Nothing,
      peers = cfg.initPeers,
      discover = cfg.discover,
      pub = pub,
      address =
        NetworkAddress
          0
          (sockToHostAddress (SockAddrInet 0 0)),
      net = cfg.net,
      timeout = cfg.peerTimeout,
      maxPeerLife = cfg.maxPeerLife,
      connect = cfg.connect,
      logger = cfg.logger
    }

-- | Forward every node event to the block store and the store publisher, forever.
nodeForwarder ::
  BlockStore ->
  Publisher StoreEvent ->
  Inbox NodeEvent ->
  IO ()
nodeForwarder b pub sub =
  loopForever $ receive sub >>= atomically . storeDispatch b pub

-- | Dispatcher of node events. Peer connections, blocks, transactions,
-- inventories and not-found messages go to the block store; peer events,
-- pongs, transaction announcements and rejections are published as store
-- events; new best headers go to the block store.
storeDispatch ::
  BlockStore ->
  Publisher StoreEvent ->
  NodeEvent ->
  STM ()
storeDispatch b pub (PeerEvent pe) =
  case pe of
    PeerConnected p -> do
      publishSTM (StorePeerConnected p) pub
      blockStorePeerConnectSTM p b
    PeerDisconnected p -> do
      publishSTM (StorePeerDisconnected p) pub
      blockStorePeerDisconnectSTM p b
    PeerMessage p msg ->
      case msg of
        MPong (Pong n) -> do
          publishSTM (StorePeerPong p n) pub
          blockStorePongSTM p n b
        -- the headers of a message are a chain: the last says what the
        -- peer has
        MHeaders (Headers hs)
          | (h, _) : _ <- reverse hs -> blockStoreAnnouncedSTM p [headerHash h] b
        MBlock block ->
          blockStoreBlockSTM p block b
        MTx tx ->
          blockStoreTxSTM p tx b
        MNotFound (NotFound is) -> do
          let blocks =
                [ BlockHash h
                | InvVector t h <- is,
                  t == InvBlock || t == InvWitnessBlock
                ]
          unless (null blocks) $ blockStoreNotFoundSTM p blocks b
        MInv (Inv is) -> do
          let txs =
                [ TxHash h
                | InvVector t h <- is,
                  t == InvTx || t == InvWitnessTx
                ]
              blocks =
                [ BlockHash h
                | InvVector t h <- is,
                  t == InvBlock || t == InvWitnessBlock
                ]
          publishSTM (StoreTxAnnounce p txs) pub
          unless (null blocks) $ blockStoreAnnouncedSTM p blocks b
          unless (null txs) $ blockStoreTxHashSTM p txs b
        -- A reject that names no transaction hash cannot be attributed to a
        -- transaction and falls through to the catch-all below.
        MReject Reject {message = MCTx, extra = Just h, code, reason} ->
          publishSTM (StoreTxReject p (TxHash h) code reason.get) pub
        _ -> return ()
storeDispatch b _ (ChainEvent (ChainBestBlock bn)) =
  blockStoreHeadSTM bn b
storeDispatch _ _ (ChainEvent _) = return ()