packages feed

haskoin-store 1.16.1 → 1.16.2

raw patch · 10 files changed

+401/−40 lines, 10 filesPVP: major bump suggested

API removals or changes: PVP suggests a major version bump

API changes (from Hackage documentation)

+ Haskoin.Store: [writes] :: Store -> !TVar Stall
+ Haskoin.Store.Database.Stalls: Delayed :: Writes
+ Haskoin.Store.Database.Stalls: Flowing :: Writes
+ Haskoin.Store.Database.Stalls: Stall :: !Writes -> !Integer -> !Integer -> Stall
+ Haskoin.Store.Database.Stalls: Stopped :: Writes
+ Haskoin.Store.Database.Stalls: [pending] :: Stall -> !Integer
+ Haskoin.Store.Database.Stalls: [rate] :: Stall -> !Integer
+ Haskoin.Store.Database.Stalls: [writes] :: Stall -> !Writes
+ Haskoin.Store.Database.Stalls: data Stall
+ Haskoin.Store.Database.Stalls: data Writes
+ Haskoin.Store.Database.Stalls: defaultBackgroundJobs :: Int -> Int
+ Haskoin.Store.Database.Stalls: instance GHC.Classes.Eq Haskoin.Store.Database.Stalls.Stall
+ Haskoin.Store.Database.Stalls: instance GHC.Classes.Eq Haskoin.Store.Database.Stalls.Writes
+ Haskoin.Store.Database.Stalls: instance GHC.Classes.Ord Haskoin.Store.Database.Stalls.Writes
+ Haskoin.Store.Database.Stalls: instance GHC.Internal.Enum.Bounded Haskoin.Store.Database.Stalls.Writes
+ Haskoin.Store.Database.Stalls: instance GHC.Internal.Enum.Enum Haskoin.Store.Database.Stalls.Writes
+ Haskoin.Store.Database.Stalls: instance GHC.Internal.Show.Show Haskoin.Store.Database.Stalls.Stall
+ Haskoin.Store.Database.Stalls: instance GHC.Internal.Show.Show Haskoin.Store.Database.Stalls.Writes
+ Haskoin.Store.Database.Stalls: noStall :: Stall
+ Haskoin.Store.Database.Stalls: readStall :: DB -> IO Stall
+ Haskoin.Store.Database.Stalls: stallNote :: Writes -> Stall -> Maybe (LogLevel, Text)
+ Haskoin.Store.Database.Stalls: watchStalls :: (LogLevel -> Text -> IO ()) -> Maybe Stats -> DB -> TVar Stall -> IO ()
+ Haskoin.Store.Manager: [writes] :: Store -> !TVar Stall
+ Haskoin.Store.Web: withDatabase :: Stall -> HealthCheck -> Value
- Haskoin.Store: Store :: !PeerMgr -> !Chain -> !BlockStore -> !DatabaseReader -> !Maybe Member -> !Publisher StoreEvent -> !Publisher StoreEvent -> !Network -> !SockAddr -> WithConnection -> Store
+ Haskoin.Store: Store :: !PeerMgr -> !Chain -> !BlockStore -> !DatabaseReader -> !Maybe Member -> !Publisher StoreEvent -> !Publisher StoreEvent -> !Network -> !SockAddr -> WithConnection -> !TVar Stall -> Store
- Haskoin.Store.Manager: Store :: !PeerMgr -> !Chain -> !BlockStore -> !DatabaseReader -> !Maybe Member -> !Publisher StoreEvent -> !Publisher StoreEvent -> !Network -> !SockAddr -> WithConnection -> Store
+ Haskoin.Store.Manager: Store :: !PeerMgr -> !Chain -> !BlockStore -> !DatabaseReader -> !Maybe Member -> !Publisher StoreEvent -> !Publisher StoreEvent -> !Network -> !SockAddr -> WithConnection -> !TVar Stall -> Store

Files

CHANGELOG.md view
@@ -4,6 +4,24 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html) +## [1.16.2] - 2026-11-18++### Changed+- **RocksDB gets a background job a core**, from 2 to 16, instead of its+  default of two. `--background-jobs` (`BACKGROUND_JOBS`) still overrides it,+  and 0 still means RocksDB's default. A fresh sync of Bitcoin Cash froze at+  height 436,814 because compactions fell behind and RocksDB stopped writes.+  `max_subcompactions` stays at 1: `rocksdb-haskell-jprupp` 2.8.0 cannot set+  it.++### Added+- **Write stalls are visible.** The new `Haskoin.Store.Database.Stalls`+  polls RocksDB every five seconds and logs a warning when it delays or stops+  writes, with the bytes of compaction pending, and a line when they recover.+  `/health` reports it under `"database"`; stopped writes make the store+  unhealthy, delayed ones do not. StatsD gauges `db.pending_compaction_bytes`+  and `db.write_delayed`.+ ## [1.16.1] - 2026-11-18  ### Changed
README.md view
@@ -424,7 +424,7 @@ | `--max-open-files` | `MAX_OPEN_FILES` | `0` | Table files kept open (0 = all of them) | | `--write-buffer-mb` | `WRITE_BUFFER_MB` | `256` | Memtables across all column families (0 = unlimited) | | `--max-wal-mb` | `MAX_WAL_MB` | `1024` | Write-ahead log size that forces flushes (0 = RocksDB default) |-| `--background-jobs` | `BACKGROUND_JOBS` | `0` | Threads for flushes and compactions together (0 = RocksDB default of two) |+| `--background-jobs` | `BACKGROUND_JOBS` | one a core, 2 to 16 | Threads for flushes and compactions together (0 = RocksDB default of two) | | `--block-cache-mb` | `BLOCK_CACHE_MB` | `1024` | Shared block cache for data blocks; indexes and filters stay in memory outside it (0 = RocksDB default) |  Memory used by RocksDB is roughly the block cache plus the memtables, on top of@@ -451,10 +451,24 @@   longer on a large database. The block cache (`--block-cache-mb`) holds   data blocks only. -RocksDB flushes and compacts in the background, with two threads by-default: one for each. `--background-jobs` gives it more, a quarter of them-for flushes and the rest for compactions.+RocksDB flushes and compacts in the background. The store gives it one+thread a core, at least 2 and at most 16, a quarter of them for flushes+and the rest for compactions. `--background-jobs` sets another number, and+0 leaves RocksDB's own default of two: one for each. +- **Why not RocksDB's two:** a fresh sync writes faster than one+  compaction thread keeps up with. RocksDB then delays writes once 64 GiB+  of compaction is pending in a column family, and stops them at 256 GiB.+  A sync of Bitcoin Cash on 24 cores froze at height 436,814 that way, with+  209 GB pending, and moved again at once with 12 jobs. The store now logs+  when RocksDB delays or stops writes, and `/health` reports it (see+  [Health](#health)).+- **Why not more than 16:** past that the disk is the limit, and more+  threads only make reads wait longer for it.+- **Subcompactions** would let one large compaction use several threads,+  but `rocksdb-haskell-jprupp` 2.8.0 cannot set `max_subcompactions`, so+  each compaction has one thread.+ - **When it helps:** a backlog of compactions on a machine whose disk and   cores have room: the first sync, or the hours after a migration (see   [Upgrading an existing database](#upgrading-an-existing-database)). One@@ -465,8 +479,7 @@   75% I/O wait. A second thread there would have made every request wait   longer for the same disk. - **What it costs:** cores and disk that the requests would otherwise have.-  Leave it at the default on a machine that serves traffic with all its-  cores.+  On a slow disk that already limits a store serving traffic, set fewer.  Bloom filters are on by default. They let RocksDB skip the table files that cannot have a key, which is most of what a lookup of a transaction, an@@ -863,6 +876,7 @@ | `Haskoin.Store.Database.Views` | Views of the database at recent blocks, one of them the fleet tip | | `Haskoin.Store.Database.Slices` | Stored blocks and transactions read in part, for what an answer needs | | `Haskoin.Store.Database.Migrate` | The migrations from versions 20, 21 and 22 |+| `Haskoin.Store.Database.Stalls` | The default number of RocksDB background jobs, and the watch on write stalls | | `Haskoin.Store.XPub` | The vocabulary of the xpub cache: its reads and the budget of a job | | `Haskoin.Store.XPub.Indexer` | The threads that read xpubs into the cache, and the queue of their jobs | | `Haskoin.Store.XPub.Lock` | The lock in Redis that keeps two stores from reading the same xpub |@@ -907,6 +921,19 @@ last block and of the last transaction, the transactions waiting to be imported, the connected peers, and Redis (`"redis"` in the JSON). +**Writes to the database** are in `"database"`: `"writes"` is `flowing`,+`delayed` or `stopped`, with `"pending-compaction-bytes"` and+`"delayed-write-rate"` (bytes a second, 0 when not delayed). RocksDB+delays writes and then stops them when its compactions fall behind+([RocksDB](#rocksdb)); the store reads this every five seconds, and logs a+warning when writes are delayed or stopped and a line when they recover.++- **Stopped writes make the store unhealthy:** it imports nothing until+  they flow again.+- **Delayed writes do not.** RocksDB slows writes for seconds at a time+  during a sync, and the store answers requests at full speed meanwhile.+  If the delay makes it fall behind, the check of the blocks says so.+ **In a fleet**, a server that does not have the block that the fleet answers from can answer any request with 503 and `{"error": "behind-fleet"}`. Have the load balancer try another server, and@@ -996,6 +1023,8 @@ | `store.block_apply` | timing | The rules, in memory | | `store.block_write` | timing | Writing the batch to RocksDB | | `store.last_block_ms`, `store.last_block_prepare_ms`, `store.last_block_prefetch_ms`, `store.last_block_apply_ms`, `store.last_block_write_ms` | gauge | The same times of the latest block, kept until the next one, so that a chart averaging seconds shows them right at any zoom |+| `db.pending_compaction_bytes` | gauge | Bytes RocksDB has to compact, over every column family, every five seconds |+| `db.write_delayed` | gauge | Writes to RocksDB: 0 flowing, 1 delayed, 2 stopped |  - **The four stages add up to nearly all of `store.block`.** A slow   `store.block_prefetch` is the disk or RocksDB's cache, and the four
app/Main.hs view
@@ -13,6 +13,7 @@ import Data.Maybe (fromMaybe) import Data.String.Conversions (cs) import Data.Text qualified as T+import GHC.Conc (getNumProcessors) import Haskoin   ( Network (..),     allNets,@@ -36,6 +37,7 @@     withStore,   ) import Haskoin.Store.Common (loopForever)+import Haskoin.Store.Database.Stalls (defaultBackgroundJobs) import Haskoin.Store.Runtime (runtimeMetrics) import Haskoin.Store.XPub.Indexer (Settings (..)) import NQE (receive, withSubscription)@@ -238,8 +240,11 @@     env "WRITE_BUFFER_MB" 256 readMaybe   maxWalMB <-     env "MAX_WAL_MB" 1024 readMaybe+  -- one RocksDB background job a core, from 2 to 16: RocksDB's own two+  -- let compactions fall behind a fresh sync until it stopped writes+  processors <- getNumProcessors   backgroundJobs <--    env "BACKGROUND_JOBS" 0 readMaybe+    env "BACKGROUND_JOBS" (defaultBackgroundJobs processors) readMaybe   blockCacheMB <-     env "BLOCK_CACHE_MB" 1024 readMaybe   return Config {version = False, ..}@@ -734,7 +739,7 @@           "BACKGROUND_JOBS"           ( unwords               [ "RocksDB threads for flushes and compactions together",-                "(0 = RocksDB default of two)"+                "(default: one a core, from 2 to 16; 0 = RocksDB default of two)"               ]           )         <> showDefault
haskoin-store.cabal view
@@ -5,7 +5,7 @@ -- see: https://github.com/sol/hpack  name:           haskoin-store-version:        1.16.1+version:        1.16.2 synopsis:       Storage and index for Bitcoin and Bitcoin Cash description:    An indexer for Bitcoin and Bitcoin Cash. It downloads blocks and mempool                 transactions from the peer-to-peer network, indexes them in RocksDB, and@@ -60,6 +60,7 @@       Haskoin.Store.Database.Query.Pooled       Haskoin.Store.Database.Reader       Haskoin.Store.Database.Slices+      Haskoin.Store.Database.Stalls       Haskoin.Store.Database.Table       Haskoin.Store.Database.Types       Haskoin.Store.Database.Views@@ -296,6 +297,7 @@       Haskoin.Store.Database.MergeSpec       Haskoin.Store.Database.MigrateSpec       Haskoin.Store.Database.SlicesSpec+      Haskoin.Store.Database.StallsSpec       Haskoin.Store.Database.ViewsSpec       Haskoin.Store.Database.WriterSpec       Haskoin.Store.ExceptionSpec
+ src/Haskoin/Store/Database/Stalls.hs view
@@ -0,0 +1,165 @@+{-# LANGUAGE OverloadedRecordDot #-}+{-# LANGUAGE OverloadedStrings #-}++-- |+-- Module      : Haskoin.Store.Database.Stalls+-- Copyright   : (c) Haskoin Developers+-- License     : MIT+-- Maintainer  : jprupp@protonmail.ch+-- Stability   : experimental+-- Portability : POSIX+--+-- RocksDB's background work, and the write stalls it causes when it falls+-- behind.+--+-- RocksDB writes new data to memtables, flushes them to table files, and+-- compacts the files into larger ones in background threads. When the+-- compactions fall behind, RocksDB protects reads by slowing every write,+-- and in the end by stopping them: past 64 GiB of compaction pending in a+-- column family it delays writes, past 256 GiB it stops them. It says so+-- only in its own @LOG@ file, so the store looked frozen while its own log+-- and @\/health@ said nothing. A fresh sync of Bitcoin Cash froze that way+-- at height 436,814, with 209 GB of compaction pending in the transactions+-- and RocksDB's default of two background jobs on a machine of 24 cores.+--+-- This module does two things about it:+--+-- * 'defaultBackgroundJobs' gives RocksDB a background job for each core,+--   within bounds, instead of RocksDB's two;+--+-- * 'watchStalls' reads RocksDB's properties every few seconds, logs when+--   writes start to be delayed or stopped and when they recover, keeps the+--   state for @\/health@, and sends it to StatsD.+--+-- The properties are polled rather than heard from an event listener of+-- RocksDB, which the C API of the binding does not have: a stall lasts+-- minutes to hours, so a few seconds late is soon enough, and each poll is+-- three reads of counters that RocksDB keeps anyway.+module Haskoin.Store.Database.Stalls+  ( -- * Background jobs+    defaultBackgroundJobs,++    -- * Write stalls+    Writes (..),+    Stall (..),+    noStall,+    readStall,+    stallNote,+    watchStalls,+  )+where++import Control.Concurrent (threadDelay)+import Control.Concurrent.STM (TVar, atomically, readTVarIO, writeTVar)+import Control.Monad (forM, forM_, forever)+import Data.ByteString.Char8 qualified as BS8+import Data.Maybe (fromMaybe)+import Data.Text (Text)+import Data.Text qualified as T+import Database.RocksDB (DB, Property (..), columnFamilies, getProperty, getPropertyCF)+import Haskoin.Node (LogLevel (..))+import System.Metrics.StatsD (Stats, newStatGauge, setGauge)+import Text.Read (readMaybe)++-- | Background jobs for RocksDB on a machine with this many cores: one a+-- core, at least 2 and at most 16.+--+-- RocksDB's own default is two jobs, one flush and one compaction, and a+-- single compaction thread does about 90 MiB a second. A store that syncs+-- writes faster than that, and its pending compactions grow until RocksDB+-- stops writes. One job a core lets the compactions of the different+-- column families and levels run side by side.+--+-- * __Why not more than 16:__ past that, the disk is the limit on any+--   machine the store runs on, and more threads make every read wait+--   longer for it.+-- * __Why one a core and not fewer:__ compaction threads wait for the disk+--   much of the time, and RocksDB runs only the compactions that are due,+--   so idle jobs cost nothing. A machine that serves traffic with all its+--   cores and has a slow disk can set fewer with @--background-jobs@.+-- * __Why at least 2:__ RocksDB splits the jobs into a quarter for flushes+--   and the rest for compactions, at least one of each.+defaultBackgroundJobs :: Int -> Int+defaultBackgroundJobs cores = max 2 (min 16 cores)++-- | Whether RocksDB lets writes through.+data Writes+  = -- | at full speed+    Flowing+  | -- | slowed down to let compactions catch up+    Delayed+  | -- | stopped until compactions or flushes catch up+    Stopped+  deriving (Eq, Ord, Show, Enum, Bounded)++-- | What RocksDB said at a poll.+data Stall = Stall+  { -- | whether writes go through+    writes :: !Writes,+    -- | bytes that RocksDB estimates it has to compact, summed over every+    -- column family (@rocksdb.estimate-pending-compaction-bytes@)+    pending :: !Integer,+    -- | bytes a second that writes are slowed to, 0 when they are not+    -- (@rocksdb.actual-delayed-write-rate@)+    rate :: !Integer+  }+  deriving (Eq, Show)++-- | The state before the first poll: writes flowing, nothing pending.+noStall :: Stall+noStall = Stall {writes = Flowing, pending = 0, rate = 0}++-- | Read the state of writes from RocksDB.+readStall :: DB -> IO Stall+readStall db = do+  pendings <- forM (Nothing : map Just (columnFamilies db)) $ \cf ->+    number <$> maybe (getProperty db pendingP) (\c -> getPropertyCF db c pendingP) cf+  stopped <- number <$> getProperty db (PropertyName "rocksdb.is-write-stopped")+  rate <- number <$> getProperty db (PropertyName "rocksdb.actual-delayed-write-rate")+  let writes+        | stopped /= 0 = Stopped+        | rate /= 0 = Delayed+        | otherwise = Flowing+  return Stall {writes, pending = sum pendings, rate}+  where+    pendingP = PropertyName "rocksdb.estimate-pending-compaction-bytes"+    number = fromMaybe 0 . (>>= readMaybe . BS8.unpack)++-- | What to log when the state of writes was the first and is now the+-- second: nothing while it stays the same, a warning when it gets worse,+-- and an info line when it gets better. A stall lasts long, and a line at+-- every poll would bury the rest of the log.+stallNote :: Writes -> Stall -> Maybe (LogLevel, Text)+stallNote before now+  | now.writes == before = Nothing+  | now.writes > before = Just (LevelWarn, worse)+  | otherwise = Just (LevelInfo, better)+  where+    worse = case now.writes of+      Stopped -> "RocksDB stopped writes" <> behind <> hint+      _ -> "RocksDB delays writes to " <> T.pack (show now.rate) <> " bytes/s" <> behind <> hint+    better = case now.writes of+      Flowing -> "RocksDB writes at full speed again" <> behind+      _ -> "RocksDB no longer stops writes, but delays them to " <> T.pack (show now.rate) <> " bytes/s" <> behind+    behind = ": " <> T.pack (show now.pending) <> " bytes of compaction pending"+    hint = "; more --background-jobs may help if the cores and disk have room"++-- | Poll RocksDB every five seconds, forever: run it in a thread of its own.+-- It keeps the latest state in the variable, logs the changes+-- ('stallNote'), and sends the gauges @db.pending_compaction_bytes@ and+-- @db.write_delayed@ (0 flowing, 1 delayed, 2 stopped) to StatsD.+watchStalls :: (LogLevel -> Text -> IO ()) -> Maybe Stats -> DB -> TVar Stall -> IO ()+watchStalls note stats db var = do+  gauges <- forM stats $ \s ->+    (,)+      <$> newStatGauge s "db.pending_compaction_bytes" 0+      <*> newStatGauge s "db.write_delayed" 0+  forever $ do+    before <- readTVarIO var+    now <- readStall db+    atomically (writeTVar var now)+    forM_ (stallNote before.writes now) (uncurry note)+    forM_ gauges $ \(pendingG, writesG) -> do+      setGauge pendingG (fromIntegral now.pending)+      setGauge writesG (fromEnum now.writes)+    threadDelay 5000000
src/Haskoin/Store/Manager.hs view
@@ -24,7 +24,7 @@ where  import Control.Concurrent.Async (link, withAsync)-import Control.Concurrent.STM (STM, atomically)+import Control.Concurrent.STM (STM, TVar, atomically, newTVarIO) import Control.Monad (forM_, unless, when) import Data.ByteString (ByteString) import Data.ByteString.Char8 qualified as C@@ -90,6 +90,7 @@     createDataMetrics,     withLayeredReader,   )+import Haskoin.Store.Database.Stalls (Stall, noStall, watchStalls) import Haskoin.Store.Fleet.Member   ( Member (..),     MemberConfig (..),@@ -145,7 +146,10 @@     -- | how the store opens a connection to a node, for those it makes     -- besides the ones of the peer manager     -- ("Haskoin.Store.Web.Publish")-    connect :: !(SockAddr -> WithConnection)+    connect :: !(SockAddr -> WithConnection),+    -- | whether RocksDB lets writes through, polled every five seconds+    -- ("Haskoin.Store.Database.Stalls"); @\/health@ reports it+    writes :: !(TVar Stall)   }  -- | Configuration for a 'Store'.@@ -226,36 +230,48 @@ -- when the action returns, and an exception in any component is re-thrown in -- the calling thread. ----- The database is opened first, then the node, the block store and the cache--- (if 'redis' is set).+-- The database is opened first, with a thread that watches it for write+-- stalls (@withStalls@), then the node, the block store and the cache (if+-- 'redis' is set). withStore ::   StoreConfig ->   (Store -> IO a) ->   IO a withStore cfg action =   connectDB cfg $ \db ->-    withPublisher $ \pub ->-      withPublisher $ \node_pub ->-        withSubscription node_pub $ \node_sub ->-          withNode (nodeCfg cfg db node_pub) $ \node -> do-            member <- storeMember cfg db-            withBlockStore (blockStoreCfg cfg node pub db member) $ \block_store ->-              withAsync (nodeForwarder block_store pub node_sub) $ \fwd -> do-                link fwd-                withStoreFleet cfg pub member $-                  withFleetEvents cfg pub member $ \events ->-                    action $-                      Store-                        { peerMgr = node.peerMgr,-                          chain = node.chain,-                          block = block_store,-                          db = db,-                          fleet = member,-                          events = events,-                          pub = pub,-                          net = cfg.net,-                          connect = cfg.connect-                        }+    withStalls cfg db $ \writes ->+      withPublisher $ \pub ->+        withPublisher $ \node_pub ->+          withSubscription node_pub $ \node_sub ->+            withNode (nodeCfg cfg db node_pub) $ \node -> do+              member <- storeMember cfg db+              withBlockStore (blockStoreCfg cfg node pub db member) $ \block_store ->+                withAsync (nodeForwarder block_store pub node_sub) $ \fwd -> do+                  link fwd+                  withStoreFleet cfg pub member $+                    withFleetEvents cfg pub member $ \events ->+                      action $+                        Store+                          { peerMgr = node.peerMgr,+                            chain = node.chain,+                            block = block_store,+                            db = db,+                            fleet = member,+                            events = events,+                            pub = pub,+                            net = cfg.net,+                            connect = cfg.connect,+                            writes = writes+                          }++-- | Watch RocksDB for write stalls while an action runs+-- ("Haskoin.Store.Database.Stalls"), and give the action the latest state.+withStalls :: StoreConfig -> DatabaseReader -> (TVar Stall -> IO a) -> IO a+withStalls cfg db action = do+  writes <- newTVarIO noStall+  withAsync (watchStalls note cfg.stats db.db writes) $ \a -> link a >> action writes+  where+    note level = logAt level (withSource "Database" cfg.logger)  -- | What every key of the fleet in Redis starts with. cachePrefix :: Network -> ByteString
src/Haskoin/Store/Web.hs view
@@ -84,6 +84,9 @@     logIt,     ailments, +    -- * Health+    withDatabase,+     -- * Xpubs     Unindexed (..),     xpubQuery,@@ -238,6 +241,7 @@ import Haskoin.Store.Database.Query qualified as Q import Haskoin.Store.Database.Query.Pooled (chainRead) import Haskoin.Store.Database.Reader+import Haskoin.Store.Database.Stalls (Stall (..), Writes (..)) import Haskoin.Store.Exception (bracket, bracket_, catchAny, finally, handleAny, onException, tryAny) import Haskoin.Store.Fleet.Member   ( Member (..),@@ -3846,14 +3850,48 @@ pathHealth st =   route (renderStdMethod (resourceMethod proxy)) (capturePath proxy) [] $ \_ -> do     h <- handleHealth st GetHealth-    let (status, extra)-          | not (isOK h) = (status503, [("Retry-After", "5")])+    stall <- readTVarIO st.config.store.writes+    let stopped = stall.writes == Stopped+        (status, extra)+          | not (isOK h) || stopped = (status503, [("Retry-After", "5")])           | h.redis == Just False = (status503, [("Retry-After", "5"), ("Haskoin-Degraded", "redis")])           | otherwise = (status200, [])     return $-      responseLBS status ((hContentType, "application/json") : extra) (encodingToLazyByteString (toEncoding h))+      responseLBS status ((hContentType, "application/json") : extra) (A.encode (withDatabase stall h))   where     proxy = Proxy :: Proxy GetHealth++-- | The JSON of a health check with the state of RocksDB's writes added+-- ("Haskoin.Store.Database.Stalls"), under @\"database\"@:+--+-- > "database": {"ok": true, "writes": "delayed",+-- >              "pending-compaction-bytes": 70000000000, "delayed-write-rate": 16777216}+--+-- Writes that RocksDB stopped make the store unhealthy: it imports nothing+-- until they flow again. Delayed writes do not. RocksDB slows writes for+-- seconds at a time while a store syncs, and a store that serves requests+-- answers them at full speed meanwhile; if the delay makes it fall behind,+-- the check of the blocks says so.+--+-- The field is added here and not to 'HealthCheck', which is in+-- haskoin-store-data: clients that read it ignore the field, and see+-- @\"ok\": false@ when writes stop.+withDatabase :: Stall -> HealthCheck -> A.Value+withDatabase stall h = case toJSON h of+  A.Object o ->+    A.Object $+      KeyMap.insert "ok" (A.Bool (isOK h && not stopped)) $+        KeyMap.insert "database" database o+  v -> v+  where+    stopped = stall.writes == Stopped+    database =+      A.object+        [ "ok" .= not stopped,+          "writes" .= T.toLower (T.pack (show stall.writes)),+          "pending-compaction-bytes" .= stall.pending,+          "delayed-write-rate" .= stall.rate+        ]  -- | Handler for @GET \/health@: health of the store. handleHealth ::
swagger.yaml view
@@ -6523,6 +6523,34 @@           properties:             ok:               type: boolean+        database:+          type: object+          description: |+            Whether RocksDB lets writes through. RocksDB delays writes, and then stops+            them, when its compactions fall behind. Stopped writes make `ok` false and+            the status 503; delayed writes do not, as the server still answers at full+            speed, and `blocks` says so if it falls behind. Read every five seconds.+          required:+            - ok+            - writes+            - pending-compaction-bytes+            - delayed-write-rate+          properties:+            ok:+              type: boolean+              description: |+                False when writes are stopped.+            writes:+              type: string+              enum: [flowing, delayed, stopped]+            pending-compaction-bytes:+              type: integer+              description: |+                Bytes RocksDB estimates it has to compact, over every column family.+            delayed-write-rate:+              type: integer+              description: |+                Bytes a second that writes are slowed to; 0 when they are not.         net:           type: string           description: |@@ -6534,7 +6562,7 @@         ok:           type: boolean           description: |-            Whether every check passed.+            Whether every check passed, writes to the database included.         time:           type: string           description: |
+ test/Haskoin/Store/Database/StallsSpec.hs view
@@ -0,0 +1,44 @@+{-# LANGUAGE OverloadedRecordDot #-}++module Haskoin.Store.Database.StallsSpec (spec) where++import Data.Maybe (isNothing)+import Haskoin (btcRegTest)+import Haskoin.Node (LogLevel (..))+import Haskoin.Store.Database.Reader (DatabaseReader (..))+import Haskoin.Store.Database.Stalls+import Haskoin.Store.Histories (openLayered)+import Haskoin.Store.Integration.Temp (withSystemTempDirectory)+import Test.Hspec++spec :: Spec+spec = do+  describe "the default number of background jobs" $ do+    it "is one a core between 2 and 16" $+      map defaultBackgroundJobs [1, 2, 4, 12, 16, 24] `shouldBe` [2, 2, 4, 12, 16, 16]+    it "is at least 2 even for a nonsense core count" $+      map defaultBackgroundJobs [-1, 0] `shouldBe` [2, 2]++  describe "the log of write stalls" $ do+    let at w = noStall {writes = w, pending = 70000000000, rate = if w == Delayed then 16777216 else 0}+        level before now = fst <$> stallNote before (at now)+    it "says nothing while the state stays the same" $+      mapM_ (\w -> stallNote w (at w) `shouldSatisfy` isNothing) [minBound .. maxBound]+    it "warns when writes get worse" $ do+      level Flowing Delayed `shouldBe` Just LevelWarn+      level Flowing Stopped `shouldBe` Just LevelWarn+      level Delayed Stopped `shouldBe` Just LevelWarn+    it "says so at info when they get better" $ do+      level Stopped Delayed `shouldBe` Just LevelInfo+      level Stopped Flowing `shouldBe` Just LevelInfo+      level Delayed Flowing `shouldBe` Just LevelInfo+    it "logs once for a run of polls in the same state" $ do+      let states = [Flowing, Flowing, Delayed, Delayed, Delayed, Stopped, Stopped, Delayed, Flowing, Flowing]+          notes = [l | (b, n) <- zip states (drop 1 states), Just (l, _) <- [stallNote b (at n)]]+      notes `shouldBe` [LevelWarn, LevelWarn, LevelInfo, LevelInfo]++  describe "reading the state from RocksDB" $+    it "finds writes flowing on a new database" $+      withSystemTempDirectory "haskoin-store-stalls-" $ \dir ->+        openLayered btcRegTest dir $ \db ->+          (.writes) <$> readStall db.db `shouldReturn` Flowing
test/Haskoin/Store/WebSpec.hs view
@@ -2,6 +2,8 @@  import Control.Concurrent (threadDelay) import Control.Exception (ErrorCall (..), throwIO, try)+import Data.Aeson qualified as A+import Data.Aeson.KeyMap qualified as KeyMap import Data.ByteString (ByteString) import Data.Either (isLeft) import Data.IORef (modifyIORef', newIORef, readIORef)@@ -17,7 +19,8 @@     MaxHealth (..),     TimeHealth (..),   )-import Haskoin.Store.Web (ailments, decodeTxBody, logIt)+import Haskoin.Store.Database.Stalls (Stall (..), Writes (..), noStall)+import Haskoin.Store.Web (ailments, decodeTxBody, logIt, withDatabase) import Network.HTTP.Types (status200, status404, status500, status503) import Network.Wai (Request (..), defaultRequest, responseLBS) import Network.Wai.Internal (ResponseReceived (..))@@ -129,6 +132,19 @@           alone = behind {peers = CountHealth {count = 0, min = 1}} :: HealthCheck       ailments behind `shouldBe` "5 blocks behind the headers"       ailments alone `shouldBe` "5 blocks behind the headers, 0 peers"+  describe "the health check with the database" $ do+    let field k v = case v of+          A.Object o -> KeyMap.lookup k o+          _ -> Nothing+        db w = withDatabase noStall {writes = w, pending = 7, rate = 0} well+    it "is ok while writes flow or are only delayed" $ do+      field "ok" (db Flowing) `shouldBe` Just (A.Bool True)+      field "ok" (db Delayed) `shouldBe` Just (A.Bool True)+      (field "database" (db Delayed) >>= field "writes") `shouldBe` Just (A.String "delayed")+      (field "database" (db Delayed) >>= field "pending-compaction-bytes") `shouldBe` Just (A.Number 7)+    it "is not ok when writes are stopped" $ do+      field "ok" (db Stopped) `shouldBe` Just (A.Bool False)+      (field "database" (db Stopped) >>= field "ok") `shouldBe` Just (A.Bool False)   describe "decodeTxBody" $ do     -- The id is the double SHA-256 of the bytes of plainTxHex, reversed. It     -- was computed outside this library. The witness is not part of the id,