haskoin-store 1.16.1 → 1.16.2
raw patch · 10 files changed
+401/−40 lines, 10 filesPVP: major bump suggested
API removals or changes: PVP suggests a major version bump
API changes (from Hackage documentation)
+ Haskoin.Store: [writes] :: Store -> !TVar Stall
+ Haskoin.Store.Database.Stalls: Delayed :: Writes
+ Haskoin.Store.Database.Stalls: Flowing :: Writes
+ Haskoin.Store.Database.Stalls: Stall :: !Writes -> !Integer -> !Integer -> Stall
+ Haskoin.Store.Database.Stalls: Stopped :: Writes
+ Haskoin.Store.Database.Stalls: [pending] :: Stall -> !Integer
+ Haskoin.Store.Database.Stalls: [rate] :: Stall -> !Integer
+ Haskoin.Store.Database.Stalls: [writes] :: Stall -> !Writes
+ Haskoin.Store.Database.Stalls: data Stall
+ Haskoin.Store.Database.Stalls: data Writes
+ Haskoin.Store.Database.Stalls: defaultBackgroundJobs :: Int -> Int
+ Haskoin.Store.Database.Stalls: instance GHC.Classes.Eq Haskoin.Store.Database.Stalls.Stall
+ Haskoin.Store.Database.Stalls: instance GHC.Classes.Eq Haskoin.Store.Database.Stalls.Writes
+ Haskoin.Store.Database.Stalls: instance GHC.Classes.Ord Haskoin.Store.Database.Stalls.Writes
+ Haskoin.Store.Database.Stalls: instance GHC.Internal.Enum.Bounded Haskoin.Store.Database.Stalls.Writes
+ Haskoin.Store.Database.Stalls: instance GHC.Internal.Enum.Enum Haskoin.Store.Database.Stalls.Writes
+ Haskoin.Store.Database.Stalls: instance GHC.Internal.Show.Show Haskoin.Store.Database.Stalls.Stall
+ Haskoin.Store.Database.Stalls: instance GHC.Internal.Show.Show Haskoin.Store.Database.Stalls.Writes
+ Haskoin.Store.Database.Stalls: noStall :: Stall
+ Haskoin.Store.Database.Stalls: readStall :: DB -> IO Stall
+ Haskoin.Store.Database.Stalls: stallNote :: Writes -> Stall -> Maybe (LogLevel, Text)
+ Haskoin.Store.Database.Stalls: watchStalls :: (LogLevel -> Text -> IO ()) -> Maybe Stats -> DB -> TVar Stall -> IO ()
+ Haskoin.Store.Manager: [writes] :: Store -> !TVar Stall
+ Haskoin.Store.Web: withDatabase :: Stall -> HealthCheck -> Value
- Haskoin.Store: Store :: !PeerMgr -> !Chain -> !BlockStore -> !DatabaseReader -> !Maybe Member -> !Publisher StoreEvent -> !Publisher StoreEvent -> !Network -> !SockAddr -> WithConnection -> Store
+ Haskoin.Store: Store :: !PeerMgr -> !Chain -> !BlockStore -> !DatabaseReader -> !Maybe Member -> !Publisher StoreEvent -> !Publisher StoreEvent -> !Network -> !SockAddr -> WithConnection -> !TVar Stall -> Store
- Haskoin.Store.Manager: Store :: !PeerMgr -> !Chain -> !BlockStore -> !DatabaseReader -> !Maybe Member -> !Publisher StoreEvent -> !Publisher StoreEvent -> !Network -> !SockAddr -> WithConnection -> Store
+ Haskoin.Store.Manager: Store :: !PeerMgr -> !Chain -> !BlockStore -> !DatabaseReader -> !Maybe Member -> !Publisher StoreEvent -> !Publisher StoreEvent -> !Network -> !SockAddr -> WithConnection -> !TVar Stall -> Store
Files
- CHANGELOG.md +18/−0
- README.md +35/−6
- app/Main.hs +7/−2
- haskoin-store.cabal +3/−1
- src/Haskoin/Store/Database/Stalls.hs +165/−0
- src/Haskoin/Store/Manager.hs +42/−26
- src/Haskoin/Store/Web.hs +41/−3
- swagger.yaml +29/−1
- test/Haskoin/Store/Database/StallsSpec.hs +44/−0
- test/Haskoin/Store/WebSpec.hs +17/−1
CHANGELOG.md view
@@ -4,6 +4,24 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html) +## [1.16.2] - 2026-11-18++### Changed+- **RocksDB gets a background job a core**, from 2 to 16, instead of its+ default of two. `--background-jobs` (`BACKGROUND_JOBS`) still overrides it,+ and 0 still means RocksDB's default. A fresh sync of Bitcoin Cash froze at+ height 436,814 because compactions fell behind and RocksDB stopped writes.+ `max_subcompactions` stays at 1: `rocksdb-haskell-jprupp` 2.8.0 cannot set+ it.++### Added+- **Write stalls are visible.** The new `Haskoin.Store.Database.Stalls`+ polls RocksDB every five seconds and logs a warning when it delays or stops+ writes, with the bytes of compaction pending, and a line when they recover.+ `/health` reports it under `"database"`; stopped writes make the store+ unhealthy, delayed ones do not. StatsD gauges `db.pending_compaction_bytes`+ and `db.write_delayed`.+ ## [1.16.1] - 2026-11-18 ### Changed
README.md view
@@ -424,7 +424,7 @@ | `--max-open-files` | `MAX_OPEN_FILES` | `0` | Table files kept open (0 = all of them) | | `--write-buffer-mb` | `WRITE_BUFFER_MB` | `256` | Memtables across all column families (0 = unlimited) | | `--max-wal-mb` | `MAX_WAL_MB` | `1024` | Write-ahead log size that forces flushes (0 = RocksDB default) |-| `--background-jobs` | `BACKGROUND_JOBS` | `0` | Threads for flushes and compactions together (0 = RocksDB default of two) |+| `--background-jobs` | `BACKGROUND_JOBS` | one a core, 2 to 16 | Threads for flushes and compactions together (0 = RocksDB default of two) | | `--block-cache-mb` | `BLOCK_CACHE_MB` | `1024` | Shared block cache for data blocks; indexes and filters stay in memory outside it (0 = RocksDB default) | Memory used by RocksDB is roughly the block cache plus the memtables, on top of@@ -451,10 +451,24 @@ longer on a large database. The block cache (`--block-cache-mb`) holds data blocks only. -RocksDB flushes and compacts in the background, with two threads by-default: one for each. `--background-jobs` gives it more, a quarter of them-for flushes and the rest for compactions.+RocksDB flushes and compacts in the background. The store gives it one+thread a core, at least 2 and at most 16, a quarter of them for flushes+and the rest for compactions. `--background-jobs` sets another number, and+0 leaves RocksDB's own default of two: one for each. +- **Why not RocksDB's two:** a fresh sync writes faster than one+ compaction thread keeps up with. RocksDB then delays writes once 64 GiB+ of compaction is pending in a column family, and stops them at 256 GiB.+ A sync of Bitcoin Cash on 24 cores froze at height 436,814 that way, with+ 209 GB pending, and moved again at once with 12 jobs. The store now logs+ when RocksDB delays or stops writes, and `/health` reports it (see+ [Health](#health)).+- **Why not more than 16:** past that the disk is the limit, and more+ threads only make reads wait longer for it.+- **Subcompactions** would let one large compaction use several threads,+ but `rocksdb-haskell-jprupp` 2.8.0 cannot set `max_subcompactions`, so+ each compaction has one thread.+ - **When it helps:** a backlog of compactions on a machine whose disk and cores have room: the first sync, or the hours after a migration (see [Upgrading an existing database](#upgrading-an-existing-database)). One@@ -465,8 +479,7 @@ 75% I/O wait. A second thread there would have made every request wait longer for the same disk. - **What it costs:** cores and disk that the requests would otherwise have.- Leave it at the default on a machine that serves traffic with all its- cores.+ On a slow disk that already limits a store serving traffic, set fewer. Bloom filters are on by default. They let RocksDB skip the table files that cannot have a key, which is most of what a lookup of a transaction, an@@ -863,6 +876,7 @@ | `Haskoin.Store.Database.Views` | Views of the database at recent blocks, one of them the fleet tip | | `Haskoin.Store.Database.Slices` | Stored blocks and transactions read in part, for what an answer needs | | `Haskoin.Store.Database.Migrate` | The migrations from versions 20, 21 and 22 |+| `Haskoin.Store.Database.Stalls` | The default number of RocksDB background jobs, and the watch on write stalls | | `Haskoin.Store.XPub` | The vocabulary of the xpub cache: its reads and the budget of a job | | `Haskoin.Store.XPub.Indexer` | The threads that read xpubs into the cache, and the queue of their jobs | | `Haskoin.Store.XPub.Lock` | The lock in Redis that keeps two stores from reading the same xpub |@@ -907,6 +921,19 @@ last block and of the last transaction, the transactions waiting to be imported, the connected peers, and Redis (`"redis"` in the JSON). +**Writes to the database** are in `"database"`: `"writes"` is `flowing`,+`delayed` or `stopped`, with `"pending-compaction-bytes"` and+`"delayed-write-rate"` (bytes a second, 0 when not delayed). RocksDB+delays writes and then stops them when its compactions fall behind+([RocksDB](#rocksdb)); the store reads this every five seconds, and logs a+warning when writes are delayed or stopped and a line when they recover.++- **Stopped writes make the store unhealthy:** it imports nothing until+ they flow again.+- **Delayed writes do not.** RocksDB slows writes for seconds at a time+ during a sync, and the store answers requests at full speed meanwhile.+ If the delay makes it fall behind, the check of the blocks says so.+ **In a fleet**, a server that does not have the block that the fleet answers from can answer any request with 503 and `{"error": "behind-fleet"}`. Have the load balancer try another server, and@@ -996,6 +1023,8 @@ | `store.block_apply` | timing | The rules, in memory | | `store.block_write` | timing | Writing the batch to RocksDB | | `store.last_block_ms`, `store.last_block_prepare_ms`, `store.last_block_prefetch_ms`, `store.last_block_apply_ms`, `store.last_block_write_ms` | gauge | The same times of the latest block, kept until the next one, so that a chart averaging seconds shows them right at any zoom |+| `db.pending_compaction_bytes` | gauge | Bytes RocksDB has to compact, over every column family, every five seconds |+| `db.write_delayed` | gauge | Writes to RocksDB: 0 flowing, 1 delayed, 2 stopped | - **The four stages add up to nearly all of `store.block`.** A slow `store.block_prefetch` is the disk or RocksDB's cache, and the four
app/Main.hs view
@@ -13,6 +13,7 @@ import Data.Maybe (fromMaybe) import Data.String.Conversions (cs) import Data.Text qualified as T+import GHC.Conc (getNumProcessors) import Haskoin ( Network (..), allNets,@@ -36,6 +37,7 @@ withStore, ) import Haskoin.Store.Common (loopForever)+import Haskoin.Store.Database.Stalls (defaultBackgroundJobs) import Haskoin.Store.Runtime (runtimeMetrics) import Haskoin.Store.XPub.Indexer (Settings (..)) import NQE (receive, withSubscription)@@ -238,8 +240,11 @@ env "WRITE_BUFFER_MB" 256 readMaybe maxWalMB <- env "MAX_WAL_MB" 1024 readMaybe+ -- one RocksDB background job a core, from 2 to 16: RocksDB's own two+ -- let compactions fall behind a fresh sync until it stopped writes+ processors <- getNumProcessors backgroundJobs <-- env "BACKGROUND_JOBS" 0 readMaybe+ env "BACKGROUND_JOBS" (defaultBackgroundJobs processors) readMaybe blockCacheMB <- env "BLOCK_CACHE_MB" 1024 readMaybe return Config {version = False, ..}@@ -734,7 +739,7 @@ "BACKGROUND_JOBS" ( unwords [ "RocksDB threads for flushes and compactions together",- "(0 = RocksDB default of two)"+ "(default: one a core, from 2 to 16; 0 = RocksDB default of two)" ] ) <> showDefault
haskoin-store.cabal view
@@ -5,7 +5,7 @@ -- see: https://github.com/sol/hpack name: haskoin-store-version: 1.16.1+version: 1.16.2 synopsis: Storage and index for Bitcoin and Bitcoin Cash description: An indexer for Bitcoin and Bitcoin Cash. It downloads blocks and mempool transactions from the peer-to-peer network, indexes them in RocksDB, and@@ -60,6 +60,7 @@ Haskoin.Store.Database.Query.Pooled Haskoin.Store.Database.Reader Haskoin.Store.Database.Slices+ Haskoin.Store.Database.Stalls Haskoin.Store.Database.Table Haskoin.Store.Database.Types Haskoin.Store.Database.Views@@ -296,6 +297,7 @@ Haskoin.Store.Database.MergeSpec Haskoin.Store.Database.MigrateSpec Haskoin.Store.Database.SlicesSpec+ Haskoin.Store.Database.StallsSpec Haskoin.Store.Database.ViewsSpec Haskoin.Store.Database.WriterSpec Haskoin.Store.ExceptionSpec
+ src/Haskoin/Store/Database/Stalls.hs view
@@ -0,0 +1,165 @@+{-# LANGUAGE OverloadedRecordDot #-}+{-# LANGUAGE OverloadedStrings #-}++-- |+-- Module : Haskoin.Store.Database.Stalls+-- Copyright : (c) Haskoin Developers+-- License : MIT+-- Maintainer : jprupp@protonmail.ch+-- Stability : experimental+-- Portability : POSIX+--+-- RocksDB's background work, and the write stalls it causes when it falls+-- behind.+--+-- RocksDB writes new data to memtables, flushes them to table files, and+-- compacts the files into larger ones in background threads. When the+-- compactions fall behind, RocksDB protects reads by slowing every write,+-- and in the end by stopping them: past 64 GiB of compaction pending in a+-- column family it delays writes, past 256 GiB it stops them. It says so+-- only in its own @LOG@ file, so the store looked frozen while its own log+-- and @\/health@ said nothing. A fresh sync of Bitcoin Cash froze that way+-- at height 436,814, with 209 GB of compaction pending in the transactions+-- and RocksDB's default of two background jobs on a machine of 24 cores.+--+-- This module does two things about it:+--+-- * 'defaultBackgroundJobs' gives RocksDB a background job for each core,+-- within bounds, instead of RocksDB's two;+--+-- * 'watchStalls' reads RocksDB's properties every few seconds, logs when+-- writes start to be delayed or stopped and when they recover, keeps the+-- state for @\/health@, and sends it to StatsD.+--+-- The properties are polled rather than heard from an event listener of+-- RocksDB, which the C API of the binding does not have: a stall lasts+-- minutes to hours, so a few seconds late is soon enough, and each poll is+-- three reads of counters that RocksDB keeps anyway.+module Haskoin.Store.Database.Stalls+ ( -- * Background jobs+ defaultBackgroundJobs,++ -- * Write stalls+ Writes (..),+ Stall (..),+ noStall,+ readStall,+ stallNote,+ watchStalls,+ )+where++import Control.Concurrent (threadDelay)+import Control.Concurrent.STM (TVar, atomically, readTVarIO, writeTVar)+import Control.Monad (forM, forM_, forever)+import Data.ByteString.Char8 qualified as BS8+import Data.Maybe (fromMaybe)+import Data.Text (Text)+import Data.Text qualified as T+import Database.RocksDB (DB, Property (..), columnFamilies, getProperty, getPropertyCF)+import Haskoin.Node (LogLevel (..))+import System.Metrics.StatsD (Stats, newStatGauge, setGauge)+import Text.Read (readMaybe)++-- | Background jobs for RocksDB on a machine with this many cores: one a+-- core, at least 2 and at most 16.+--+-- RocksDB's own default is two jobs, one flush and one compaction, and a+-- single compaction thread does about 90 MiB a second. A store that syncs+-- writes faster than that, and its pending compactions grow until RocksDB+-- stops writes. One job a core lets the compactions of the different+-- column families and levels run side by side.+--+-- * __Why not more than 16:__ past that, the disk is the limit on any+-- machine the store runs on, and more threads make every read wait+-- longer for it.+-- * __Why one a core and not fewer:__ compaction threads wait for the disk+-- much of the time, and RocksDB runs only the compactions that are due,+-- so idle jobs cost nothing. A machine that serves traffic with all its+-- cores and has a slow disk can set fewer with @--background-jobs@.+-- * __Why at least 2:__ RocksDB splits the jobs into a quarter for flushes+-- and the rest for compactions, at least one of each.+defaultBackgroundJobs :: Int -> Int+defaultBackgroundJobs cores = max 2 (min 16 cores)++-- | Whether RocksDB lets writes through.+data Writes+ = -- | at full speed+ Flowing+ | -- | slowed down to let compactions catch up+ Delayed+ | -- | stopped until compactions or flushes catch up+ Stopped+ deriving (Eq, Ord, Show, Enum, Bounded)++-- | What RocksDB said at a poll.+data Stall = Stall+ { -- | whether writes go through+ writes :: !Writes,+ -- | bytes that RocksDB estimates it has to compact, summed over every+ -- column family (@rocksdb.estimate-pending-compaction-bytes@)+ pending :: !Integer,+ -- | bytes a second that writes are slowed to, 0 when they are not+ -- (@rocksdb.actual-delayed-write-rate@)+ rate :: !Integer+ }+ deriving (Eq, Show)++-- | The state before the first poll: writes flowing, nothing pending.+noStall :: Stall+noStall = Stall {writes = Flowing, pending = 0, rate = 0}++-- | Read the state of writes from RocksDB.+readStall :: DB -> IO Stall+readStall db = do+ pendings <- forM (Nothing : map Just (columnFamilies db)) $ \cf ->+ number <$> maybe (getProperty db pendingP) (\c -> getPropertyCF db c pendingP) cf+ stopped <- number <$> getProperty db (PropertyName "rocksdb.is-write-stopped")+ rate <- number <$> getProperty db (PropertyName "rocksdb.actual-delayed-write-rate")+ let writes+ | stopped /= 0 = Stopped+ | rate /= 0 = Delayed+ | otherwise = Flowing+ return Stall {writes, pending = sum pendings, rate}+ where+ pendingP = PropertyName "rocksdb.estimate-pending-compaction-bytes"+ number = fromMaybe 0 . (>>= readMaybe . BS8.unpack)++-- | What to log when the state of writes was the first and is now the+-- second: nothing while it stays the same, a warning when it gets worse,+-- and an info line when it gets better. A stall lasts long, and a line at+-- every poll would bury the rest of the log.+stallNote :: Writes -> Stall -> Maybe (LogLevel, Text)+stallNote before now+ | now.writes == before = Nothing+ | now.writes > before = Just (LevelWarn, worse)+ | otherwise = Just (LevelInfo, better)+ where+ worse = case now.writes of+ Stopped -> "RocksDB stopped writes" <> behind <> hint+ _ -> "RocksDB delays writes to " <> T.pack (show now.rate) <> " bytes/s" <> behind <> hint+ better = case now.writes of+ Flowing -> "RocksDB writes at full speed again" <> behind+ _ -> "RocksDB no longer stops writes, but delays them to " <> T.pack (show now.rate) <> " bytes/s" <> behind+ behind = ": " <> T.pack (show now.pending) <> " bytes of compaction pending"+ hint = "; more --background-jobs may help if the cores and disk have room"++-- | Poll RocksDB every five seconds, forever: run it in a thread of its own.+-- It keeps the latest state in the variable, logs the changes+-- ('stallNote'), and sends the gauges @db.pending_compaction_bytes@ and+-- @db.write_delayed@ (0 flowing, 1 delayed, 2 stopped) to StatsD.+watchStalls :: (LogLevel -> Text -> IO ()) -> Maybe Stats -> DB -> TVar Stall -> IO ()+watchStalls note stats db var = do+ gauges <- forM stats $ \s ->+ (,)+ <$> newStatGauge s "db.pending_compaction_bytes" 0+ <*> newStatGauge s "db.write_delayed" 0+ forever $ do+ before <- readTVarIO var+ now <- readStall db+ atomically (writeTVar var now)+ forM_ (stallNote before.writes now) (uncurry note)+ forM_ gauges $ \(pendingG, writesG) -> do+ setGauge pendingG (fromIntegral now.pending)+ setGauge writesG (fromEnum now.writes)+ threadDelay 5000000
src/Haskoin/Store/Manager.hs view
@@ -24,7 +24,7 @@ where import Control.Concurrent.Async (link, withAsync)-import Control.Concurrent.STM (STM, atomically)+import Control.Concurrent.STM (STM, TVar, atomically, newTVarIO) import Control.Monad (forM_, unless, when) import Data.ByteString (ByteString) import Data.ByteString.Char8 qualified as C@@ -90,6 +90,7 @@ createDataMetrics, withLayeredReader, )+import Haskoin.Store.Database.Stalls (Stall, noStall, watchStalls) import Haskoin.Store.Fleet.Member ( Member (..), MemberConfig (..),@@ -145,7 +146,10 @@ -- | how the store opens a connection to a node, for those it makes -- besides the ones of the peer manager -- ("Haskoin.Store.Web.Publish")- connect :: !(SockAddr -> WithConnection)+ connect :: !(SockAddr -> WithConnection),+ -- | whether RocksDB lets writes through, polled every five seconds+ -- ("Haskoin.Store.Database.Stalls"); @\/health@ reports it+ writes :: !(TVar Stall) } -- | Configuration for a 'Store'.@@ -226,36 +230,48 @@ -- when the action returns, and an exception in any component is re-thrown in -- the calling thread. ----- The database is opened first, then the node, the block store and the cache--- (if 'redis' is set).+-- The database is opened first, with a thread that watches it for write+-- stalls (@withStalls@), then the node, the block store and the cache (if+-- 'redis' is set). withStore :: StoreConfig -> (Store -> IO a) -> IO a withStore cfg action = connectDB cfg $ \db ->- withPublisher $ \pub ->- withPublisher $ \node_pub ->- withSubscription node_pub $ \node_sub ->- withNode (nodeCfg cfg db node_pub) $ \node -> do- member <- storeMember cfg db- withBlockStore (blockStoreCfg cfg node pub db member) $ \block_store ->- withAsync (nodeForwarder block_store pub node_sub) $ \fwd -> do- link fwd- withStoreFleet cfg pub member $- withFleetEvents cfg pub member $ \events ->- action $- Store- { peerMgr = node.peerMgr,- chain = node.chain,- block = block_store,- db = db,- fleet = member,- events = events,- pub = pub,- net = cfg.net,- connect = cfg.connect- }+ withStalls cfg db $ \writes ->+ withPublisher $ \pub ->+ withPublisher $ \node_pub ->+ withSubscription node_pub $ \node_sub ->+ withNode (nodeCfg cfg db node_pub) $ \node -> do+ member <- storeMember cfg db+ withBlockStore (blockStoreCfg cfg node pub db member) $ \block_store ->+ withAsync (nodeForwarder block_store pub node_sub) $ \fwd -> do+ link fwd+ withStoreFleet cfg pub member $+ withFleetEvents cfg pub member $ \events ->+ action $+ Store+ { peerMgr = node.peerMgr,+ chain = node.chain,+ block = block_store,+ db = db,+ fleet = member,+ events = events,+ pub = pub,+ net = cfg.net,+ connect = cfg.connect,+ writes = writes+ }++-- | Watch RocksDB for write stalls while an action runs+-- ("Haskoin.Store.Database.Stalls"), and give the action the latest state.+withStalls :: StoreConfig -> DatabaseReader -> (TVar Stall -> IO a) -> IO a+withStalls cfg db action = do+ writes <- newTVarIO noStall+ withAsync (watchStalls note cfg.stats db.db writes) $ \a -> link a >> action writes+ where+ note level = logAt level (withSource "Database" cfg.logger) -- | What every key of the fleet in Redis starts with. cachePrefix :: Network -> ByteString
src/Haskoin/Store/Web.hs view
@@ -84,6 +84,9 @@ logIt, ailments, + -- * Health+ withDatabase,+ -- * Xpubs Unindexed (..), xpubQuery,@@ -238,6 +241,7 @@ import Haskoin.Store.Database.Query qualified as Q import Haskoin.Store.Database.Query.Pooled (chainRead) import Haskoin.Store.Database.Reader+import Haskoin.Store.Database.Stalls (Stall (..), Writes (..)) import Haskoin.Store.Exception (bracket, bracket_, catchAny, finally, handleAny, onException, tryAny) import Haskoin.Store.Fleet.Member ( Member (..),@@ -3846,14 +3850,48 @@ pathHealth st = route (renderStdMethod (resourceMethod proxy)) (capturePath proxy) [] $ \_ -> do h <- handleHealth st GetHealth- let (status, extra)- | not (isOK h) = (status503, [("Retry-After", "5")])+ stall <- readTVarIO st.config.store.writes+ let stopped = stall.writes == Stopped+ (status, extra)+ | not (isOK h) || stopped = (status503, [("Retry-After", "5")]) | h.redis == Just False = (status503, [("Retry-After", "5"), ("Haskoin-Degraded", "redis")]) | otherwise = (status200, []) return $- responseLBS status ((hContentType, "application/json") : extra) (encodingToLazyByteString (toEncoding h))+ responseLBS status ((hContentType, "application/json") : extra) (A.encode (withDatabase stall h)) where proxy = Proxy :: Proxy GetHealth++-- | The JSON of a health check with the state of RocksDB's writes added+-- ("Haskoin.Store.Database.Stalls"), under @\"database\"@:+--+-- > "database": {"ok": true, "writes": "delayed",+-- > "pending-compaction-bytes": 70000000000, "delayed-write-rate": 16777216}+--+-- Writes that RocksDB stopped make the store unhealthy: it imports nothing+-- until they flow again. Delayed writes do not. RocksDB slows writes for+-- seconds at a time while a store syncs, and a store that serves requests+-- answers them at full speed meanwhile; if the delay makes it fall behind,+-- the check of the blocks says so.+--+-- The field is added here and not to 'HealthCheck', which is in+-- haskoin-store-data: clients that read it ignore the field, and see+-- @\"ok\": false@ when writes stop.+withDatabase :: Stall -> HealthCheck -> A.Value+withDatabase stall h = case toJSON h of+ A.Object o ->+ A.Object $+ KeyMap.insert "ok" (A.Bool (isOK h && not stopped)) $+ KeyMap.insert "database" database o+ v -> v+ where+ stopped = stall.writes == Stopped+ database =+ A.object+ [ "ok" .= not stopped,+ "writes" .= T.toLower (T.pack (show stall.writes)),+ "pending-compaction-bytes" .= stall.pending,+ "delayed-write-rate" .= stall.rate+ ] -- | Handler for @GET \/health@: health of the store. handleHealth ::
swagger.yaml view
@@ -6523,6 +6523,34 @@ properties: ok: type: boolean+ database:+ type: object+ description: |+ Whether RocksDB lets writes through. RocksDB delays writes, and then stops+ them, when its compactions fall behind. Stopped writes make `ok` false and+ the status 503; delayed writes do not, as the server still answers at full+ speed, and `blocks` says so if it falls behind. Read every five seconds.+ required:+ - ok+ - writes+ - pending-compaction-bytes+ - delayed-write-rate+ properties:+ ok:+ type: boolean+ description: |+ False when writes are stopped.+ writes:+ type: string+ enum: [flowing, delayed, stopped]+ pending-compaction-bytes:+ type: integer+ description: |+ Bytes RocksDB estimates it has to compact, over every column family.+ delayed-write-rate:+ type: integer+ description: |+ Bytes a second that writes are slowed to; 0 when they are not. net: type: string description: |@@ -6534,7 +6562,7 @@ ok: type: boolean description: |- Whether every check passed.+ Whether every check passed, writes to the database included. time: type: string description: |
+ test/Haskoin/Store/Database/StallsSpec.hs view
@@ -0,0 +1,44 @@+{-# LANGUAGE OverloadedRecordDot #-}++module Haskoin.Store.Database.StallsSpec (spec) where++import Data.Maybe (isNothing)+import Haskoin (btcRegTest)+import Haskoin.Node (LogLevel (..))+import Haskoin.Store.Database.Reader (DatabaseReader (..))+import Haskoin.Store.Database.Stalls+import Haskoin.Store.Histories (openLayered)+import Haskoin.Store.Integration.Temp (withSystemTempDirectory)+import Test.Hspec++spec :: Spec+spec = do+ describe "the default number of background jobs" $ do+ it "is one a core between 2 and 16" $+ map defaultBackgroundJobs [1, 2, 4, 12, 16, 24] `shouldBe` [2, 2, 4, 12, 16, 16]+ it "is at least 2 even for a nonsense core count" $+ map defaultBackgroundJobs [-1, 0] `shouldBe` [2, 2]++ describe "the log of write stalls" $ do+ let at w = noStall {writes = w, pending = 70000000000, rate = if w == Delayed then 16777216 else 0}+ level before now = fst <$> stallNote before (at now)+ it "says nothing while the state stays the same" $+ mapM_ (\w -> stallNote w (at w) `shouldSatisfy` isNothing) [minBound .. maxBound]+ it "warns when writes get worse" $ do+ level Flowing Delayed `shouldBe` Just LevelWarn+ level Flowing Stopped `shouldBe` Just LevelWarn+ level Delayed Stopped `shouldBe` Just LevelWarn+ it "says so at info when they get better" $ do+ level Stopped Delayed `shouldBe` Just LevelInfo+ level Stopped Flowing `shouldBe` Just LevelInfo+ level Delayed Flowing `shouldBe` Just LevelInfo+ it "logs once for a run of polls in the same state" $ do+ let states = [Flowing, Flowing, Delayed, Delayed, Delayed, Stopped, Stopped, Delayed, Flowing, Flowing]+ notes = [l | (b, n) <- zip states (drop 1 states), Just (l, _) <- [stallNote b (at n)]]+ notes `shouldBe` [LevelWarn, LevelWarn, LevelInfo, LevelInfo]++ describe "reading the state from RocksDB" $+ it "finds writes flowing on a new database" $+ withSystemTempDirectory "haskoin-store-stalls-" $ \dir ->+ openLayered btcRegTest dir $ \db ->+ (.writes) <$> readStall db.db `shouldReturn` Flowing
test/Haskoin/Store/WebSpec.hs view
@@ -2,6 +2,8 @@ import Control.Concurrent (threadDelay) import Control.Exception (ErrorCall (..), throwIO, try)+import Data.Aeson qualified as A+import Data.Aeson.KeyMap qualified as KeyMap import Data.ByteString (ByteString) import Data.Either (isLeft) import Data.IORef (modifyIORef', newIORef, readIORef)@@ -17,7 +19,8 @@ MaxHealth (..), TimeHealth (..), )-import Haskoin.Store.Web (ailments, decodeTxBody, logIt)+import Haskoin.Store.Database.Stalls (Stall (..), Writes (..), noStall)+import Haskoin.Store.Web (ailments, decodeTxBody, logIt, withDatabase) import Network.HTTP.Types (status200, status404, status500, status503) import Network.Wai (Request (..), defaultRequest, responseLBS) import Network.Wai.Internal (ResponseReceived (..))@@ -129,6 +132,19 @@ alone = behind {peers = CountHealth {count = 0, min = 1}} :: HealthCheck ailments behind `shouldBe` "5 blocks behind the headers" ailments alone `shouldBe` "5 blocks behind the headers, 0 peers"+ describe "the health check with the database" $ do+ let field k v = case v of+ A.Object o -> KeyMap.lookup k o+ _ -> Nothing+ db w = withDatabase noStall {writes = w, pending = 7, rate = 0} well+ it "is ok while writes flow or are only delayed" $ do+ field "ok" (db Flowing) `shouldBe` Just (A.Bool True)+ field "ok" (db Delayed) `shouldBe` Just (A.Bool True)+ (field "database" (db Delayed) >>= field "writes") `shouldBe` Just (A.String "delayed")+ (field "database" (db Delayed) >>= field "pending-compaction-bytes") `shouldBe` Just (A.Number 7)+ it "is not ok when writes are stopped" $ do+ field "ok" (db Stopped) `shouldBe` Just (A.Bool False)+ (field "database" (db Stopped) >>= field "ok") `shouldBe` Just (A.Bool False) describe "decodeTxBody" $ do -- The id is the double SHA-256 of the bytes of plainTxHex, reversed. It -- was computed outside this library. The witness is not part of the id,