haskoin-store 1.16.5 → 1.16.6
raw patch · 37 files changed
+635/−1694 lines, 37 filesPVP: major bump suggested
API removals or changes: PVP suggests a major version bump
API changes (from Hackage documentation)
- Haskoin.Store.BlockStore: [mempoolTimeout] :: BlockStoreConfig -> !Int
- Haskoin.Store.BlockStore.Pending: PendingTx :: !UTCTime -> !Tx -> !HashSet TxHash -> PendingTx
- Haskoin.Store.BlockStore.Pending: [deps] :: PendingTx -> !HashSet TxHash
- Haskoin.Store.BlockStore.Pending: [time] :: PendingTx -> !UTCTime
- Haskoin.Store.BlockStore.Pending: [tx] :: PendingTx -> !Tx
- Haskoin.Store.BlockStore.Pending: data PendingTx
- Haskoin.Store.BlockStore.Pending: imported :: TxHash -> Pending -> Pending
- Haskoin.Store.BlockStore.Pending: instance GHC.Classes.Eq Haskoin.Store.BlockStore.Pending.PendingTx
- Haskoin.Store.BlockStore.Pending: instance GHC.Classes.Ord Haskoin.Store.BlockStore.Pending.PendingTx
- Haskoin.Store.BlockStore.Pending: instance GHC.Internal.Show.Show Haskoin.Store.BlockStore.Pending.PendingTx
- Haskoin.Store.BlockStore.Pending: orphanTimeout :: NominalDiffTime
- Haskoin.Store.BlockStore.Pending: queued :: Pending -> Int
- Haskoin.Store.BlockStore.Pending: received :: UTCTime -> Tx -> HashSet TxHash -> Pending -> Pending
- Haskoin.Store.BlockStore.Pending: refresh :: HashMap TxHash (Maybe (HashSet TxHash)) -> Pending -> Pending
- Haskoin.Store.BlockStore.Pending: takeReady :: Int -> Pending -> ([PendingTx], Pending)
- Haskoin.Store.BlockStore.Pending: waiting :: Pending -> [PendingTx]
- Haskoin.Store.Database.Reader: derivedAddrsCached :: DatabaseReader -> XPubSpec -> IO (Derived, Bool)
- Haskoin.Store.Database.Writer: insertReplaced :: Writer -> TxHash -> Replaced -> IO ()
- Haskoin.Store.Fleet.Member: [changed] :: Member -> !TVar Bool
- Haskoin.Store.Fleet.Member: [gone] :: Member -> !TVar (HashSet TxHash)
- Haskoin.Store.Fleet.Member: [offeredAt] :: Member -> !TVar (Maybe BlockHash)
- Haskoin.Store.Fleet.Member: [offered] :: Member -> !TVar (HashSet TxHash)
- Haskoin.Store.Fleet.Member: [rounds] :: Member -> !TVar Int
- Haskoin.Store.Fleet.Member: memberForget :: Member -> [TxHash] -> IO ()
- Haskoin.Store.BlockStore: BlockStoreConfig :: !PeerMgr -> !Chain -> !Publisher StoreEvent -> !DatabaseReader -> !Network -> !BlockHash -> IO () -> !UnixTime -> Tx -> IO () -> ![TxHash] -> IO (HashSet TxHash) -> !Int -> !Int -> !NominalDiffTime -> !NominalDiffTime -> !NominalDiffTime -> !Maybe Stats -> !Logger -> BlockStoreConfig
+ Haskoin.Store.BlockStore: BlockStoreConfig :: !PeerMgr -> !Chain -> !Publisher StoreEvent -> !DatabaseReader -> !Network -> !BlockHash -> IO () -> !UnixTime -> Tx -> IO () -> ![TxHash] -> IO (HashSet TxHash) -> !Int -> !NominalDiffTime -> !NominalDiffTime -> !NominalDiffTime -> !Maybe Stats -> !Logger -> BlockStoreConfig
- Haskoin.Store.Fleet.Member: Member :: !MemberConfig -> !Connection -> !FleetKeys -> !DatabaseReader -> !Views BlockHash -> !TVar [BlockHash] -> !TVar [BlockHash] -> !TVar (Maybe BlockHash) -> !TVar (HashSet TxHash) -> !TVar (HashMap TxHash (UnixTime, Tx)) -> !TVar Int -> !TVar (Maybe BlockHash) -> !TVar Bool -> !TVar Bool -> !Indexer XPubSpec () -> !Lock XPubSpec -> !TVar (HashMap ByteString Int) -> !XPubs -> !TVar (Maybe Integer) -> !Maybe CacheMetrics -> !TVar (HashSet TxHash) -> !Outage -> !TVar (Maybe ByteString) -> !TVar Int -> !IORef (ByteString, Word64) -> Member
+ Haskoin.Store.Fleet.Member: Member :: !MemberConfig -> !Connection -> !FleetKeys -> !DatabaseReader -> !Views BlockHash -> !TVar [BlockHash] -> !TVar [BlockHash] -> !TVar (Maybe BlockHash) -> !TVar (HashMap TxHash (UnixTime, Tx)) -> !TVar Bool -> !Indexer XPubSpec () -> !Lock XPubSpec -> !TVar (HashMap ByteString Int) -> !XPubs -> !Maybe CacheMetrics -> !Outage -> !TVar (Maybe ByteString) -> !TVar (Maybe Integer) -> !TVar Int -> !IORef (ByteString, Word64) -> Member
Files
- CHANGELOG.md +81/−0
- README.md +64/−77
- app/Main.hs +7/−5
- cbits/haskoin_store_merge.h +1/−1
- haskoin-store.cabal +7/−76
- integration/Haskoin/Store/Integration/Image.hs +3/−13
- integration/Haskoin/Store/Integration/Temp.hs +1/−1
- integration/Haskoin/Store/Integration/Valkey.hs +1/−1
- src/Haskoin/Store.hs +17/−14
- src/Haskoin/Store/BlockStore.hs +51/−88
- src/Haskoin/Store/BlockStore/Pending.hs +29/−117
- src/Haskoin/Store/Common.hs +9/−8
- src/Haskoin/Store/Database/Merge.hs +2/−2
- src/Haskoin/Store/Database/Query/Pooled.hs +5/−4
- src/Haskoin/Store/Database/Reader.hs +6/−14
- src/Haskoin/Store/Database/Types.hs +3/−2
- src/Haskoin/Store/Database/Views.hs +3/−3
- src/Haskoin/Store/Database/Writer.hs +2/−7
- src/Haskoin/Store/Fleet.hs +3/−3
- src/Haskoin/Store/Fleet/Member.hs +88/−118
- src/Haskoin/Store/Fleet/Pool.hs +11/−11
- src/Haskoin/Store/Fleet/Redis.hs +15/−10
- src/Haskoin/Store/Fleet/XPub.hs +1/−1
- src/Haskoin/Store/Fleet/lua/pool.lua +16/−5
- src/Haskoin/Store/Manager.hs +31/−33
- src/Haskoin/Store/Outage.hs +2/−2
- src/Haskoin/Store/Web.hs +28/−23
- src/Haskoin/Store/Web/Publish.hs +4/−4
- src/Haskoin/Store/XPub/Lock.hs +3/−3
- src/Haskoin/Store/XPub/Service.hs +1/−2
- stress/Main.hs +0/−911
- stress/wrk.lua +0/−32
- swagger.yaml +34/−11
- test/Haskoin/Store/BlockStore/PendingSpec.hs +21/−74
- test/Haskoin/Store/Fleet/CacheSpec.hs +59/−0
- test/Haskoin/Store/Fleet/RedisSpec.hs +14/−6
- test/Haskoin/Store/IntegrationSpec.hs +12/−12
CHANGELOG.md view
@@ -4,6 +4,87 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html) +## [1.16.6] - 2026-11-19++### Fixed+- **The pool no longer stops at a transaction that pays an output of value+ 0.** Taking such an output out of an address's summary added `-0`, which+ Redis does not read as an integer. The script failed at every turn of the+ fleet, and the pool took in nothing more until the next restart of the+ pool. Pay-to-anchor outputs, which Bitcoin Core relays since version 29,+ have value 0. Since Redis keeps what a failed script wrote before its+ error, the summaries of the addresses that the stuck turn touched could+ drift; emptying the pool clears them.+- **A store no longer keeps every transaction it ever offered in memory.**+ The fleet member kept a set of what it had offered to the pool, emptied+ only when the pool was started again or Redis lost its data: about half+ a million transactions a day on Bitcoin. It also made the store refuse,+ silently, a transaction that left the pool and was sent again, such as+ one that a replacement pushed out. The set is gone: the fleet's seen set+ in Redis already keeps stores from fetching what the pool decided on.+- **Orphans no longer stop the store from offering transactions.** A+ transaction whose parent never came waited in memory for ever, and a+ round offers the 2,000 oldest that wait: once that many orphans had piled+ up, nothing else was offered. An orphan is now let go 20 minutes after+ the store heard of it, as Bitcoin Core does.+- **A restart of the pool no longer drops what the store was about to+ offer.**+- **A restart of the pool fills it again.** After a reorganisation deeper+ than 50 blocks, the pool was emptied and its transactions came back only+ in a block, since the fleet had seen them and no store fetched them+ again. The restart now empties the seen set too, and every store reads+ its node's mempool over RPC (`--node-rpc`) and asks its peers for theirs,+ as after a loss of Redis.+- **A store no longer loses its Bitcoin Core peers after Redis loses its+ data.** From the first refill on, the store asked every peer that+ connected for its mempool. Bitcoin Core disconnects a peer that asks+ unless it whitelists it or serves bloom filters, which it does not by+ default; so the store lost its public Core peers one after the other.+ It now asks only the peers connected at the refill, once.+- **README: memory for indexes and filters.** They come to about 0.9% of the+ database (16 GB for Bitcoin, 4 GB for Bitcoin Cash, measured on+ api.haskoin.com), not 0.34%; and they, Bloom filters included, are held+ outside the block cache, not in it. The `--block-cache-mb` help said the+ same wrong thing and is fixed too.+- **Documentation brought up to date with Redis being required.** The+ package description, the README's Docker quick start (it started the+ store without Redis, which cannot work), the help of `/health` in+ `swagger.yaml` (the `redis` and `database` fields are always there; a+ store without Redis answers 503), and comments throughout the code that+ still described an optional Redis, a store's own mempool, the token+ design or functions that no longer exist.+- **`swagger.yaml`**: version 1.16.5; `request-too-large` (413) is sent;+ the health check runs every second and at each block, not every 30+ seconds; the 202 body of `POST /transactions` has a `message`.+- **README**: `nocache` is accepted and ignored on xpub routes; `stack.yaml`+ may pin git commits; the integration image is under `integration/`; the+ code map lists every module; links point at the document that has what+ they promise.+- **Messages about pending transactions** (the `/health` log, the help of+ `--max-pending-txs`, the reference): they count transactions requested+ from peers and not received yet, not transactions waiting to be imported.+ The log line of the migration from version 20 says the transactions were+ taken out of the chain, not moved.++### Removed+- **The stress test of the cache** (`stress/`, `scripts/stress-cache.sh`,+ `scripts/stress-report.py`, the `haskoin-store-stress` benchmark, and+ wrk in the test image). It no longer ran against the current server and+ measured a cache that no longer exists. `scripts/heap-profile-summary.py`+ stays; the README says how to use it on its own.+- **The StatsD gauge `store.queued_txs`**: it counted transactions waiting+ to be imported, which the block store no longer holds, and was always 0.+- **Dead code**: what the block store kept of transactions received from+ peers, which nothing put there (`Haskoin.Store.BlockStore.Pending` now+ holds only the requests); `insertReplaced`; `BlockStoreConfig`'s+ `mempoolTimeout`; and unused fields of the fleet member.++### Changed+- **`hsq --help` fits the terminal.** The usage text is laid out for the+ terminal's width (at most 100 columns): prose reflows, the command and+ option lists stay aligned when there is room and stack when there is+ not, and examples and the sample configuration are kept as written.+ ## [1.16.5] - 2026-11-19 ### Added
README.md view
@@ -105,15 +105,18 @@ git clone https://github.com/jprupp/haskoin-store.git cd haskoin-store docker build -t haskoin-store .-docker run -d --name haskoin-store -p 3000:3000 \- -e NET=btc -e DISCOVER=true \+docker network create haskoin+docker run -d --name valkey --network haskoin valkey/valkey+docker run -d --name haskoin-store --network haskoin -p 3000:3000 \+ -e NET=btc -e DISCOVER=true -e REDIS_URL=redis://valkey:6379 \ -v haskoin-store-data:/home/haskoin/.haskoin-store \ haskoin-store curl -s 'http://localhost:3000/block/best' | jq ``` -The first sync downloads and indexes the whole chain, which takes a long-time and a lot of disk. The API answers while it syncs; `/health` reports+The store needs a Redis or Valkey server, here in a container of its own:+without one it does not start. The first sync downloads and indexes the+whole chain, which takes a long time and a lot of disk. The API answers while it syncs; `/health` reports whether the node is in sync. ## Building from source@@ -161,6 +164,10 @@ discovery on by itself. Data goes in a directory named after the network inside `--dir`, so one data directory can hold several networks. +It needs a Redis or Valkey server, at `redis://localhost:6379` unless+`--redis-url` names another, and stops at start with a message if there is+none ([Redis: the fleet](#redis-the-fleet)).+ For a big public deployment, also set `MALLOC_ARENA_MAX=2` (the Docker image does) to keep glibc from fragmenting RocksDB's memory. @@ -300,7 +307,8 @@ answers 429 with `Retry-After`, the same on every store: ask again then. - **The cache keeps the costly ones longest**, up to `--redis-max-xpubs`. The cache cannot be turned off; that option sets only its size, at least- 1. There is no `nocache`: an answer from the cache is always exact.+ 1. `nocache` is accepted and ignored: an answer from the cache is always+ exact. `docs/xpub-indexing.md` has the details. @@ -350,7 +358,7 @@ | `--target-delay` | `TARGET_DELAY` | `25` | Milliseconds that a thread ready to run may wait for a core before the limit comes down | | `--request-queue` | `REQUEST_QUEUE` | `64` | Long requests that may wait for a slot; more are refused with 429 | | `--request-wait` | `REQUEST_WAIT` | `1` | Seconds that a long request may wait for a slot; then it is refused with 429 |-| `--node-rpc` | `NODE_RPC` | none | JSON-RPC URL of a full node, such as `http://user:password@127.0.0.1:8332/`, to publish transactions through, and to fill the shared mempool from again after Redis loses its data; without it, or when it does not answer, transactions go to the peers |+| `--node-rpc` | `NODE_RPC` | none | JSON-RPC URL of a full node, such as `http://user:password@127.0.0.1:8332/`, to publish transactions through, and to fill the shared mempool from again after it is emptied; without it, or when it does not answer, transactions go to the peers | | `--def-item-count` | `DEF_ITEM_COUNT` | `100` | Items in a list when no `limit` is given | | `--xpub-gap` | `XPUB_GAP` | `32` | Unused addresses kept after the last used one of each chain of an xpub; payments further away are not seen | | `--xpub-gap-init` | `XPUB_GAP_INIT` | `20` | Addresses looked at for an xpub with no transactions; an xpub with none used among them counts as unused |@@ -386,11 +394,16 @@ unhealthy, and stops counting after ten seconds without a report, so that it holds nobody up. - **When Redis loses its data**, the stores notice it by an epoch key that- is gone or changed. Each forgets what it offered, reports unhealthy until- the fleet has a tip again, and asks its peers for their mempools again;- with `--node-rpc` it also reads the node's mempool (`getrawmempool`),- 2,000 transactions every two seconds. Xpub entries are built again as- requests ask for them.+ is gone or changed. Each forgets the fleet tip, reports unhealthy until+ the fleet has a tip again, and fills the pool again. Xpub entries are+ built again as requests ask for them.+- **When the pool starts again**, after a reorganisation deeper than 50+ blocks, the stores fill it again too.+- **Filling the pool again** works best with `--node-rpc`: each store reads+ its node's mempool (`getrawmempool`), 2,000 transactions every two+ seconds. Each also asks its peers for their mempools, once. Bitcoin Cash+ Node answers; Bitcoin Core answers only a store it whitelists, and+ disconnects any other that asks (`docs/fleet-mempool.md`). - **`/events` is the same stream on every store**, written to Redis with each change of the pool and each move of the tip. - **Xpubs** are cached for the whole fleet (`docs/xpub-indexing.md`).@@ -400,7 +413,7 @@ | Option | Environment | Default | Meaning | | --- | --- | --- | --- | | `--max-lagging-blocks` | `MAX_LAGGING_BLOCKS` | `3` | Most blocks the index may lag behind the headers |-| `--max-pending-txs` | `MAX_PENDING_TXS` | `10000` | Most transactions waiting to be imported |+| `--max-pending-txs` | `MAX_PENDING_TXS` | `10000` | Most transactions requested from peers and not received yet | | `--min-peers` | `MIN_PEERS` | `1` | Fewest connected peers | | `--block-timeout` | `BLOCK_TIMEOUT` | `14400` | Most seconds since the last block | | `--tx-timeout` | `TX_TIMEOUT` | `1800` | Most seconds since the last transaction |@@ -432,9 +445,8 @@ Memory used by RocksDB is roughly the block cache plus the memtables, on top of the Haskell heap. `/dbstats` shows where it goes. -Every table file stays open by default, and the index and the filter of-each are written in pieces of 4 KiB that the block cache holds as they are-needed.+Every table file stays open by default, and keeps its index and its+filter in memory while it is open. - **Why all files open:** a lookup in a file that is not open opens it first, which reads its footer and properties again. A store of Bitcoin@@ -447,8 +459,10 @@ outside the block cache, so that data blocks cannot push them out: when they were in the cache and a busy store evicted them, nearly every lookup read megabytes back, 0.8 ms of processor time each and 18 seconds- to import a block of Bitcoin. They come to about 0.34% of the database:- some 6 GB for Bitcoin and 1.6 GB for Bitcoin Cash. Keep all files open+ to import a block of Bitcoin. They come to about 0.9% of the database:+ 16 GB for Bitcoin (a database of 1.7 TB) and 4 GB for Bitcoin Cash+ (461 GB) on api.haskoin.com, on Thu 19 Nov 2026 (`/dbstats`, table+ readers). Keep all files open (above) so that they are read once, when the store starts, which takes longer on a large database. The block cache (`--block-cache-mb`) holds data blocks only.@@ -490,9 +504,10 @@ - **What has them:** the data that is read by key: transactions, spenders, unspent outputs, blocks and balances. The two indices by address, which are scanned, do not.-- **What they cost:** about 1.25 bytes for each key, held in the block- cache. On a chain with billions of keys that is gigabytes: size- `--block-cache-mb` for it, or turn them off with `--no-bloom`.+- **What they cost:** about 1.25 bytes for each key, held in memory with+ the indexes, outside the block cache (above). On a chain with billions+ of keys that is gigabytes: plan memory for it, or turn them off with+ `--no-bloom`. - **An existing database** gets filters for the table files that are written from now on, so their memory comes gradually, as RocksDB compacts. The migration to database version 22 rewrites nearly every@@ -792,7 +807,8 @@ - **Each request reads one snapshot** of the database, at the fleet tip, so everything in an answer is of the same moment. -`docs/one-fleet.md` has the reasons, the measurements and the tests.+`docs/one-fleet.md` has the reasons, and `docs/fleet-mempool.md` the pool+in detail and its tests. The layout of the keys is described in `Haskoin.Store.Database.Types`. ### Upgrading an existing database@@ -848,7 +864,7 @@ What changed for clients in version 23: nothing in the answers. A balance or the spenders of a transaction that recent blocks changed take a few microseconds more to read until RocksDB has compacted them:-`docs/one-fleet.md` has the figures.+`docs/block-import.md` explains why. What changed for clients in version 22: @@ -900,19 +916,21 @@ An entry also holds a gap's worth of addresses after the window, derived ahead of time, so a payment to the next address of a wallet is in its-entry as soon as the pool takes it. `docs/one-fleet.md` has the design and-the tests.+entry as soon as the pool takes it. `docs/xpub-indexing.md` has the design. ### Where the code is | Module | What it does | | --- | --- | | `Haskoin.Store.BlockStore` | Downloads and imports blocks, and hands transactions from peers to the fleet's pool |+| `Haskoin.Store.BlockStore.Sync` | The state of the block download, as a pure state machine |+| `Haskoin.Store.BlockStore.Pending` | Transactions that peers announce, on their way to the pool | | `Haskoin.Store.Logic` | The rules that import and revert blocks | | `Haskoin.Store.Database.Writer` | A batch of writes, with its read cache and its reads ahead | | `Haskoin.Store.Database.Table` | The hash tables of a batch, whose keys are hashed quickly | | `Haskoin.Store.Database.Merge` | The merge operators of balances and spenders, which are C functions (`cbits/`) | | `Haskoin.Store.Database.Query` | The queries of the chain's column families |+| `Haskoin.Store.Database.Reader` | Opening the database, with its migration, column families and metrics | | `Haskoin.Store.Database.Query.Pooled` | The queries of the chain with the fleet's pool on top | | `Haskoin.Store.Database.Views` | Views of the database at recent blocks, one of them the fleet tip | | `Haskoin.Store.Database.Slices` | Stored blocks and transactions read in part, for what an answer needs |@@ -923,13 +941,20 @@ | `Haskoin.Store.XPub.Lock` | The lock in Redis that keeps two stores from reading the same xpub | | `Haskoin.Store.XPub.Service` | The door to the fleet's cache of xpubs: a hit answers, a miss asks the indexer | | `Haskoin.Store.Fleet.Member`, `Haskoin.Store.Fleet.XPub` | A fleet, and its cache |+| `Haskoin.Store.Fleet`, `Haskoin.Store.Fleet.Pool` | The rules of the fleet tip and of the pool, as pure decisions |+| `Haskoin.Store.Fleet.Redis` | Those rules carried out in Redis, one Lua script per step | | `Haskoin.Store.Web`, `Haskoin.Store.Web.Events` | The web server, and `/events` |+| `Haskoin.Store.Web.Router` | Routing of requests to handlers, directly over WAI |+| `Haskoin.Store.Web.Docs` | The API reference that the server serves at its root | | `Haskoin.Store.Web.Pieces` | Answers that are sent while they are read | | `Haskoin.Store.Web.Listen` | A Unix socket to listen on, made by the store or passed by systemd | | `Haskoin.Store.Web.Admission` | The limit on long requests answered at once, and the line in front of it (CoDel) | | `Haskoin.Store.Web.Congestion` | The delay of GHC's scheduler, and the controller that moves the limit with it | | `Haskoin.Store.Runtime` | GHC's garbage collector, as StatsD metrics | | `Haskoin.Store.Exception` | Handlers that let a thread be stopped, and cleanups that cannot be cut short |+| `Haskoin.Store.Manager` | Wires a store together: Redis, RocksDB, the node, the block store and the fleet |+| `Haskoin.Store.Startup` | Errors that stop the store from starting, said so an operator can act |+| `Haskoin.Store.Outage` | Logs once that Redis or another dependency stopped working, and once that it is back | | `Haskoin.Store.Web.Publish` | Checking a transaction and handing it to the peers, or to a full node over RPC | The tests hold the cache to the queries of `Query.Pooled` at the fleet tip@@ -1061,7 +1086,6 @@ | `store.blocks` | gauge | Height of the best block | | `store.headers` | gauge | Height of the best header | | `store.peers` | gauge | Peers online |-| `store.queued_txs` | gauge | Transactions waiting to be imported | | `store.block` | timing | Milliseconds to import a block, with publishing its events | | `store.block_prepare` | timing | Of those, working out what the import needs of the block | | `store.block_prefetch` | timing | Reading ahead what the rules will read |@@ -1172,7 +1196,9 @@ The resident memory of the store is its GHC heap plus RocksDB. - **RocksDB:** the block cache and memtables fill up to `--block-cache-mb`- and `--write-buffer-mb`, then stay there.+ and `--write-buffer-mb`, then stay there. The indexes and filters of the+ open table files come on top, about 0.9% of the database (see+ [RocksDB](#rocksdb)). - **The heap** gives memory back to the system as soon as it no longer needs it. The executable is built with `--disable-delayed-os-memory-return` for that. Without the flag, Linux@@ -1290,12 +1316,13 @@ scripts/bench-web.sh # load-test the API with wrk stack bench haskoin-store:bench:haskoin-store-bench \ --ba 'import +RTS -N4 -A64m' # time the import of full blocks-scripts/stress-cache.sh # stress the Redis cache, with a report+scripts/heap-profile-summary.py FILE.hp # what grew in a heap profile scripts/check-local-deps.sh # what CI checks: no local paths in stack.yaml scripts/install-system-libraries.sh # the C libraries, on Ubuntu or Debian ``` -`stack.yaml` must pin released versions of every dependency. Pointing+`stack.yaml` must pin released versions, or git commits, of every+dependency. Pointing `extra-deps` at a sibling checkout such as `../haskoin-core` is fine for local testing, but CI rejects it. @@ -1311,7 +1338,7 @@ - The first run builds the image `localhost/haskoin-store-test`, which needs network access: Ubuntu 26.04 pinned by digest, Valkey from a fixed snapshot of the Ubuntu archive, and the release tarballs of both nodes, checked against- SHA-256 hashes pinned in `test/Haskoin/Store/Integration/Image.hs`. No+ SHA-256 hashes pinned in `integration/Haskoin/Store/Integration/Image.hs`. No third-party image is used. - Containers are removed when the tests end, data lives in tmpfs mounts and under `/dev/shm`, and every container has a memory limit.@@ -1324,53 +1351,12 @@ than `haskoin-store.cabal`, which [hpack](https://github.com/sol/hpack) generates. -### Stress test of the cache--`scripts/stress-cache.sh [OUT_DIR] [-- OPTIONS]` measures the cache while it-works hard.--What it runs:--1. A regtest chain of Bitcoin Core, Valkey and wrk, in Podman containers- from the test image, and the `haskoin-store` binary with the cache on.-2. A few wallets, whose keys the stress program (`stress/Main.hs`) holds,- make hundreds of thousands of transactions between them. The program- signs them and sends them to the node in batches, with a block after- each batch.-3. Every transaction changes two cached xpubs, so the cache reloads them- over and over. Meanwhile wrk asks the server about the xpubs on five- endpoints: summary, balances, transactions, unspent outputs and- multiaddr.-4. At the end it compares the cached answers with those of the database.--`scripts/stress-report.py` then writes `OUT_DIR/report.md`, with:--- how fast the transactions went in, and how far the server fell behind;-- how many reloads the cache did, how many read the whole xpub, and how- long they took as the xpubs grew;-- the CPU and memory of the server, and the memory of Valkey;-- the commands that blocked Redis for 10 ms or more, from its slow log;-- the latencies of each endpoint over time.--- The defaults are 200,000 transactions between 4 wallets of 200 receiving- and 200 change addresses, 2,000 per block; pass `-- --help` for the- options, and `-- --txs 20000` for a run of a few minutes. A default run- takes about half an hour and up to about 7 GiB of memory.-- The raw measurements stay in `OUT_DIR` as JSON lines (`rounds.jsonl`,- `samples.jsonl`, `wrk.jsonl`), with `summary.json` and the log of the- server.-- No container is left behind, even if the script is interrupted or- killed: the program removes its containers when it ends, and the script- removes any that carry its label and stops the server.-- To look for a leak, run the server with a heap profile by type, which needs- no profiling build, and summarise it: the live heap over time and the types- of closure that grew most.+### Looking for a leak - ```sh- scripts/stress-cache.sh /tmp/run -- --txs 60000 \- --server-rts '-N4 -M2g -T -hT -i10 -po/tmp/heap'- scripts/heap-profile-summary.py /tmp/heap.hp- ```+Run the server with a heap profile by type, which needs no profiling build,+for example `haskoin-store ... +RTS -hT -i10 -po/tmp/heap`, and summarise it+with `scripts/heap-profile-summary.py /tmp/heap.hp`: the live heap over time,+and the types of closure that grew most. ### Measurements for importing blocks @@ -1406,8 +1392,9 @@ - Format the code with `scripts/format` (Ormolu), and keep every line of code shorter than 120 columns. - Put scripts in files of their own rather than embedding them: Lua for- Redis under `src/Haskoin/Store/Fleet/lua/`, shell under `scripts/`, called- from CI rather than written into the workflow.+ Redis under `src/Haskoin/Store/Fleet/lua/` or `src/Haskoin/Store/XPub/lua/`,+ shell under `scripts/`, called from CI rather than written into the+ workflow. - Catch any error with `catchAny`, `handleAny` or `tryAny` of `Haskoin.Store.Exception`, and clean up with its `bracket`, `finally` and `onException`, not with those of `Control.Exception`. A request that
app/Main.hs view
@@ -404,7 +404,9 @@ option auto $ metavar "COUNT" <> long "max-pending-txs"- <> helpEnv "MAX_PENDING_TXS" "Maximum pending txs to fail health check"+ <> helpEnv+ "MAX_PENDING_TXS"+ "Most transactions requested from peers and not received yet, or the health check fails" <> showDefault <> value c.maxPendingTxs maxLaggingBlocks <-@@ -709,7 +711,7 @@ "NODE_RPC" ( unwords [ "Publish transactions over the JSON-RPC of this full node, and through the peers when it fails;",- "fill the shared mempool from it again after Redis loses its data"+ "fill the shared mempool from it again after it is emptied" ] ) <> value c.nodeRpc@@ -779,9 +781,9 @@ <> helpEnv "BLOCK_CACHE_MB" ( unwords- [ "RocksDB block cache in MiB, shared by all column families,",- "holding index and filter blocks too (0 = RocksDB default of",- "32 MiB per column family)"+ [ "RocksDB block cache of data blocks in MiB, shared by all column",+ "families; index and filter blocks stay outside it with their open",+ "tables (0 = RocksDB default of 32 MiB per column family)" ] ) <> showDefault
cbits/haskoin_store_merge.h view
@@ -24,7 +24,7 @@ * else about such input. * * Haskoin.Store.Database.Merge gives them to RocksDB, and- * docs/layered-database.md has the reasons.+ * docs/block-import.md has the reasons. */ #ifndef HASKOIN_STORE_MERGE_H
haskoin-store.cabal view
@@ -5,15 +5,15 @@ -- see: https://github.com/sol/hpack name: haskoin-store-version: 1.16.5+version: 1.16.6 synopsis: Storage and index for Bitcoin and Bitcoin Cash description: An indexer for Bitcoin and Bitcoin Cash. It downloads blocks and mempool- transactions from the peer-to-peer network, indexes them in RocksDB, and- serves a REST API with balances, transactions and unspent outputs of- addresses and extended public keys, a Blockchain.info compatible API, and- events over WebSockets. An optional Redis cache, shared by several stores,- keeps large wallets fast. See the README for installation and- configuration.+ transactions from the peer-to-peer network, indexes the chain in RocksDB and+ the mempool in Redis, and serves a REST API with balances, transactions and+ unspent outputs of addresses and extended public keys, a Blockchain.info+ compatible API, and events over WebSockets. Redis also holds the balances of+ extended public keys, and lets several stores answer as one. See the README+ for installation and configuration. category: Bitcoin, Finance, Network homepage: https://github.com/jprupp/haskoin-store#readme bug-reports: https://github.com/jprupp/haskoin-store/issues@@ -37,7 +37,6 @@ src/Haskoin/Store/XPub/lua/release.lua src/Haskoin/Store/XPub/lua/renew.lua src/Haskoin/Store/XPub/lua/take.lua- stress/wrk.lua swagger.yaml extra-doc-files: README.md@@ -458,74 +457,6 @@ , unordered-containers >=0.2.21 && <0.3 , vault >=0.3.2.0 && <0.4 , vector >=0.13.2.0 && <0.14- , wai >=3.2.5 && <3.3- , wai-extra >=3.1.18 && <3.2- , wai-websockets >=3.0.1.2 && <3.1- , warp >=3.4.17 && <3.5- , warp-tls >=3.4.14 && <3.5- , websockets >=0.13.0.0 && <0.14- , wreq >=0.5.4.5 && <0.6- default-language: GHC2024--benchmark haskoin-store-stress- type: exitcode-stdio-1.0- main-is: Main.hs- other-modules:- Paths_haskoin_store- autogen-modules:- Paths_haskoin_store- hs-source-dirs:- stress- default-extensions:- DuplicateRecordFields- NoFieldSelectors- OverloadedRecordDot- OverloadedStrings- ghc-options: -Wall -Wcompat -Widentities -Wincomplete-record-updates -Wincomplete-uni-patterns -Wmissing-export-lists -Wmissing-home-modules -Wpartial-fields -Wredundant-constraints -threaded -rtsopts -with-rtsopts=-N2- build-depends:- aeson >=2.2.5.1 && <2.3- , async >=2.2.6 && <2.3- , base >=4.21.2.0 && <4.22- , base16 ==2.0.*- , bytes >=0.17.5 && <0.18- , bytestring >=0.12.2.0 && <0.13- , case-insensitive >=1.2.1.0 && <1.3- , cereal >=0.5.8.3 && <0.6- , conduit >=1.3.6.1 && <1.4- , containers ==0.7.*- , data-default >=0.8.0.2 && <0.9- , deepseq >=1.5.1.0 && <1.6- , directory >=1.3.10.1 && <1.4- , file-embed >=0.0.16.0 && <0.1- , hashable >=1.5.1.0 && <1.6- , hashtables >=1.4.2 && <1.5- , haskoin-core >=2.0.1 && <2.1- , haskoin-node >=2.6.1 && <2.7- , haskoin-store- , haskoin-store-data >=3.0.1 && <3.1- , hedis >=0.16.4 && <0.17- , http-types >=0.12.6 && <0.13- , integration- , lens >=5.3.6 && <5.4- , monad-control >=1.0.3.1 && <1.1- , mtl >=2.3.2 && <2.4- , network >=3.2.12.0 && <3.3- , nqe >=0.7.0 && <0.8- , optparse-applicative >=0.19.0.0 && <0.20- , process >=1.6.26.1 && <1.7- , random >=1.3.1 && <1.4- , rocksdb-haskell-jprupp >=2.8.0 && <2.9- , rocksdb-query >=0.7.0 && <0.8- , statsd-rupp >=0.6.0 && <0.7- , stm >=2.5.3.1 && <2.6- , string-conversions >=0.4.0.1 && <0.5- , text >=2.1.4 && <2.2- , time ==1.14.*- , time-manager >=0.4.2 && <0.5- , transformers >=0.6.3.0 && <0.7- , unix >=2.8.8.0 && <2.9- , unordered-containers >=0.2.21 && <0.3- , vault >=0.3.2.0 && <0.4 , wai >=3.2.5 && <3.3 , wai-extra >=3.1.18 && <3.2 , wai-websockets >=3.0.1.2 && <3.1
integration/Haskoin/Store/Integration/Image.hs view
@@ -11,7 +11,7 @@ -- -- The tests use no third-party images. The image starts from the official -- Ubuntu 26.04 image, pinned by the digest of its multi-architecture index,--- installs Valkey (the Redis-compatible server Ubuntu ships) and wrk from the+-- installs Valkey (the Redis-compatible server Ubuntu ships) from the -- Ubuntu archive at a fixed snapshot date (apt checks the signatures of the -- archive; the certificates needed to reach the snapshot service over HTTPS -- come from the regular archive first), and downloads Bitcoin Core and@@ -38,7 +38,6 @@ coreBin, bchnBin, valkeyBin,- wrkBin, ) where @@ -86,23 +85,18 @@ valkeyBin :: String valkeyBin = "/usr/bin/valkey-server" --- | The HTTP load generator wrk in the image, for the stress test of the--- cache.-wrkBin :: String-wrkBin = "/usr/bin/wrk"- -- | Recipe of the image. -- -- Every download is its own layer, ordered from the one that changes least -- often: the packages of the snapshot, then Bitcoin Core, then Bitcoin Cash--- Node, then wrk for the stress test, which came last. Podman keeps the+-- Node. Podman keeps the -- layers it builds and reuses each one whose instruction, and those before -- it, did not change, so moving one node to a new release downloads only -- that release. Nothing in the image depends on the time it is built. -- -- The recipe stays here rather than in a Containerfile of its own: it is -- made from the versions and hashes defined in this module, and the tag of--- the image is a hash of it, so the tests and the stress program, which may+-- the image is a hash of it, so the tests, which may -- run from any directory, need no file to find. It is written one -- instruction and one shell command per line. containerfile :: String@@ -142,10 +136,6 @@ ) bchnAmd64 bchnArm64- <> [ "RUN apt-get update --error-on=any --snapshot " <> snapshot <> " \\",- " && apt-get install -y --no-install-recommends --snapshot " <> snapshot <> " wrk \\",- " && rm -rf /var/lib/apt/lists/*"- ] where -- download a release for the architecture of the image, check its hash, -- and unpack it; @$a@ in the URL stands for the architecture
integration/Haskoin/Store/Integration/Temp.hs view
@@ -6,7 +6,7 @@ -- Stability : experimental -- Portability : POSIX ----- Temporary directories for the tests, the benchmark and the stress test:+-- Temporary directories for the tests and the benchmark: -- made for an action, and removed with all that is in them when it ends. -- -- These are the functions of the same name of "System.IO.Temp", with one
integration/Haskoin/Store/Integration/Valkey.hs view
@@ -6,7 +6,7 @@ -- Stability : experimental -- Portability : POSIX ----- A Redis server for the integration tests and the stress test: Valkey, the+-- A Redis server for the integration tests: Valkey, the -- Redis-compatible server that Ubuntu ships, from the test image. module Haskoin.Store.Integration.Valkey ( redisContainer,
src/Haskoin/Store.hs view
@@ -10,16 +10,18 @@ -- -- Haskoin Store connects to the peer-to-peer network with -- <https://hackage.haskell.org/package/haskoin-node haskoin-node>, downloads--- blocks and mempool transactions, and indexes them in RocksDB. The indices--- answer questions about blocks, transactions, address balances, unspent--- outputs and extended public keys. A REST API built on these indices is in--- "Haskoin.Store.Web". (The Redis cache of extended public keys is being--- rewritten; until it is back, every query reads the database.)+-- blocks and mempool transactions, and indexes the chain in RocksDB. The+-- mempool is one pool in Redis, shared by every store of a fleet, even a+-- fleet of one, and so are the entries of extended public keys+-- (@docs\/one-fleet.md@). The indices answer questions about blocks,+-- transactions, address balances, unspent outputs and extended public keys.+-- A REST API built on them is in "Haskoin.Store.Web". -- -- = Running a store ----- 'withStore' starts everything from a 'StoreConfig': the database, the node--- and the block store that imports data. The 'Store' it returns has a+-- 'withStore' starts everything from a 'StoreConfig': the connection to+-- Redis, the database, the node, the block store that imports data, and the+-- store's membership of its fleet. The 'Store' it returns has a -- 'DatabaseReader'; query it with the functions of -- "Haskoin.Store.Database.Query", which take the reader and run in 'IO'. --@@ -54,6 +56,7 @@ -- * "Haskoin.Store.BlockStore": the process that imports blocks and -- transactions. -- * "Haskoin.Store.Web": the REST API server.+-- -- = Purpose -- -- A block chain is easy to append to and hard to query. Bitcoin nodes store@@ -68,23 +71,23 @@ -- * __No full node required.__ The store talks to the peer-to-peer network itself -- through haskoin-node, so a deployment is one program and a database. ----- * __Every import is atomic.__ A block or a batch of mempool transactions is--- applied as one RocksDB write. A crash leaves the database at the previous--- block, never half-way through one, so there is no recovery step.+-- * __Every import is atomic.__ A block is applied as one RocksDB write. A+-- crash leaves the database at the previous block, never half-way through+-- one, so there is no recovery step. -- -- * __Indices are designed around reads.__ Keys are laid out so that every API -- query is a seek and a short scan, newest first; see -- "Haskoin.Store.Database.Types". -- -- * __Plain functions in 'IO'.__ Queries take the database reader, or a--- 'StoreRead' record of them, which the web server builds with the Redis--- cache in front of the queries about xpubs. Nothing runs in a stack of+-- 'StoreRead' record of them, which the web server builds with the+-- fleet's entries of xpubs in Redis in front of the queries about them. Nothing runs in a stack of -- monad transformers, except the conduits that merge sorted streams. The -- web server runs on WAI with a router of its own. -- -- * __The mempool is a first-class citizen.__ Unconfirmed transactions are indexed--- like confirmed ones, ordered before them, and kept across restarts, because--- wallets care about them most.+-- like confirmed ones, in the pool in Redis, ordered before them, and kept+-- across restarts of a store, because wallets care about them most. module Haskoin.Store ( -- * Store Store (..),
src/Haskoin/Store/BlockStore.hs view
@@ -36,8 +36,8 @@ -- that peers announce and that no store of the fleet has seen ('toFetch'): -- what the fleet saw is a set in Redis, the same for every store and kept -- across restarts, so a transaction is fetched about once for the fleet.--- It asks peers for their whole mempools only to fill--- the pool again after Redis lost it ('blockStoreRefill'): the pool outlives+-- It asks peers for their whole mempools only to fill the pool again+-- after it was emptied ('blockStoreRefill'): the pool outlives -- the store, so a store that starts has nothing to ask for. A transaction -- that arrives goes to 'onTx': the member of the fleet offers it to the -- pool that the fleet shares in Redis, which is the mempool@@ -50,6 +50,7 @@ -- The block store does not listen to the node by itself. -- "Haskoin.Store.Manager" forwards node events to it with the -- @blockStore*@ functions of this module.+-- -- = Why it works this way -- -- * __Blocks come from one peer at a time.__ Blocks must be imported in order,@@ -101,7 +102,6 @@ ( STM, TVar, atomically,- modifyTVar, modifyTVar', newTVarIO, readTVar,@@ -159,11 +159,9 @@ InvVector (..), Message (..), Network (..),- OutPoint (..), Ping (..), Tx (..), TxHash (..),- TxIn (..), Version (..), blockHashToHex, headerHash,@@ -192,7 +190,7 @@ reservePeer, sendMessage, )-import Haskoin.Store.BlockStore.Pending (Pending, PendingTx (..))+import Haskoin.Store.BlockStore.Pending (Pending) import Haskoin.Store.BlockStore.Pending qualified as Pending import Haskoin.Store.BlockStore.Sync ( Accept (..),@@ -277,16 +275,14 @@ config :: !BlockStoreConfig, -- | state of the block download ("Haskoin.Store.BlockStore.Sync") sync :: !(TVar (Sync Peer)),- -- | transactions received and waiting to be imported+ -- | transactions requested from peers -- ("Haskoin.Store.BlockStore.Pending") pending :: !(TVar Pending), -- | StatsD metrics, if enabled metrics :: !(Maybe StoreMetrics),- -- | labels of the peers whose mempool was already requested- syncedMempool :: !(TVar (HashSet Text)),- -- | whether to ask peers for their mempools: from the first refill of- -- the pool on ('blockStoreRefill')- refilling :: !(TVar Bool),+ -- | whether a refill of the pool waits to ask the peers for their+ -- mempools ('blockStoreRefill')+ refill :: !(TVar Bool), -- | what each peer said that it has, by its label: a peer is asked -- only for blocks that it announced knows :: !(TVar (HashMap Text Known)),@@ -314,8 +310,6 @@ blocks :: !StatGauge, -- | height of the best header (@store.headers@) headers :: !StatGauge,- -- | transactions waiting to be imported (@store.queued_txs@)- queuedTxs :: !StatGauge, -- | online peers (@store.peers@) peers :: !StatGauge, -- | milliseconds to import a block (@store.block@)@@ -365,7 +359,6 @@ p <- getPeers cfg.peerMgr blocks <- g s "blocks" (fromIntegral b) headers <- g s "headers" (fromIntegral h.height)- queuedTxs <- g s "queued_txs" 0 peers <- g s "peers" (length p) block <- t s "block" blockPrepare <- t s "block_prepare"@@ -401,13 +394,6 @@ n <- chainGetBest self.config.chain setGauge m.headers (fromIntegral n.height) --- | Update the gauge of transactions waiting to be imported.-setPendingTxs :: BlockStore -> IO ()-setPendingTxs self =- forM_ self.metrics $ \m -> do- p <- readTVarIO self.pending- setGauge m.queuedTxs (Pending.queued p)- -- | Update the gauge of online peers. setPeersConnected :: BlockStore -> IO () setPeersConnected self =@@ -461,8 +447,6 @@ seen :: !([TxHash] -> IO (HashSet TxHash)), -- | blocks to request from the syncing peer in one batch blockWindow :: !Int,- -- | delete transactions older than this number of days from mempool- mempoolTimeout :: !Int, -- | disconnect syncing peer if inactive for this long peerTimeout :: !NominalDiffTime, -- | the longest that the syncing peer may deliver nothing before another@@ -541,10 +525,6 @@ getTxData :: BlockStore -> TxHash -> IO (Maybe TxData) getTxData self = Q.getTxData self.config.db --- | Unspent output of the chain in the database.-getUnspent :: BlockStore -> OutPoint -> IO (Maybe Unspent)-getUnspent self = Q.getUnspent self.config.db- -- | Network of the block store. getNetwork :: BlockStore -> IO Network getNetwork self = return self.config.net@@ -557,8 +537,7 @@ withBlockStore cfg action = do pb <- newTVarIO . Sync.idle =<< getCurrentTime ts <- newTVarIO Pending.empty- sy <- newTVarIO HashSet.empty- refilling <- newTVarIO False+ refill <- newTVarIO False inbox <- newInbox metrics <- newStoreMetrics cfg owner <- (newOwner "block store")@@ -570,8 +549,7 @@ sync = pb, pending = ts, metrics = metrics,- syncedMempool = sy,- refilling = refilling,+ refill = refill, knows = knows, owner = owner }@@ -621,31 +599,42 @@ guardSync :: BlockStore -> IO () -> IO () guardSync self f = isInSync self >>= \s -> when s f --- | Ask a peer for its mempool, once per peer, if in sync and the pool is--- being filled again ('blockStoreRefill').-syncMempool :: BlockStore -> Peer -> IO ()-syncMempool self p =+-- | Ask the peers connected now for their mempools, if a refill waits for+-- it ('blockStoreRefill') and the database is in sync; with no peer, the+-- refill waits for the first one. Called again on each block imported in+-- sync and each peer that connects.+syncMempool :: BlockStore -> IO ()+syncMempool self = guardSync self $ do- s <- readTVarIO self.refilling- let m = self.syncedMempool- y <- not . HashSet.member p.label <$> readTVarIO m- when (s && y) $ do- logDebug self $- "Requesting mempool from peer: " <> p.label- atomically . modifyTVar m $ HashSet.insert p.label- MMempool `sendMessage` p+ peers <- getPeers self.config.peerMgr+ asking <- atomically $ do+ waits <- readTVar self.refill+ when (waits && not (null peers)) $ writeTVar self.refill False+ return (waits && not (null peers))+ when asking $ do+ logInfo self $ "Asking " <> cs (show (length peers)) <> " peers for their mempools"+ forM_ peers $ \op -> MMempool `sendMessage` op.mailbox --- | Ask every peer for its mempool again, and fetch every transaction that--- it announces and the fleet has not seen: for when Redis lost its data,--- the pool with it, and what the fleet had seen--- ('Haskoin.Store.Fleet.Member.noteEpoch').--- The transactions go to 'onTx', which offers them to the pool again.+-- | Ask the peers for their mempools, once, and fetch every transaction+-- that they announce and the fleet has not seen: for when the pool was+-- emptied, because Redis lost its data+-- ('Haskoin.Store.Fleet.Member.noteEpoch') or the pool started again+-- ('Haskoin.Store.Fleet.Member.noteRestarts'). What the fleet had seen+-- went with it, so all of it is fetched. The transactions go to 'onTx',+-- which offers them to the pool again.+--+-- Only the peers connected at that time are asked, and only once. Bitcoin+-- Core closes the connection of a peer that asks for its mempool unless it+-- serves bloom filters (@-peerbloomfilters@, off by default) or it+-- whitelists that peer; Bitcoin Cash Node serves bloom filters by default+-- and answers. Asking every peer that connects, as the store once did,+-- lost its public Bitcoin Core peers one after the other for as long as+-- it ran. A node that the store has over RPC (@--node-rpc@) answers in any+-- case ('Haskoin.Store.Web.refillLoop'). blockStoreRefill :: BlockStore -> IO () blockStoreRefill self = do- atomically $ do- writeTVar self.syncedMempool HashSet.empty- writeTVar self.refilling True- getPeers self.config.peerMgr >>= mapM_ (syncMempool self . (.mailbox))+ atomically $ writeTVar self.refill True+ syncMempool self -- | Import a block from the syncing peer, then ask for more blocks or, if now in -- sync, sync the mempool. Disconnects the peer if it is not the syncing peer, if@@ -697,9 +686,7 @@ transition self $ \st -> ((), Sync.imported now blockhash st) isInSync self >>= \case False -> syncMe self- True -> do- updateOrphans self- syncMempool self peer+ True -> syncMempool self failure e = do logError self $ "Error importing block "@@ -736,7 +723,7 @@ <> p.label killPeer p --- | Queue a transaction received from a peer for import.+-- | A transaction that a peer sent: no longer a request, and on to the fleet member to offer to the pool. processTx :: BlockStore -> Peer -> Tx -> IO () processTx self p tx = do t <- getCurrentTime@@ -748,37 +735,14 @@ changePending self (Pending.answered (txHash tx)) self.config.onTx (fromIntegral (floor (utcTimeToPOSIXSeconds t) :: Integer)) tx --- | Change the mempool transactions on their way into the database.+-- | Change the transactions on their way to the pool of the fleet. changePending :: BlockStore -> (Pending -> Pending) -> IO ()-changePending self f = do- atomically $ modifyTVar' self.pending f- setPendingTxs self+changePending self f = atomically $ modifyTVar' self.pending f --- | Whether a transaction is queued or was requested recently.+-- | Whether a transaction was requested recently. isPending :: BlockStore -> TxHash -> IO Bool isPending self th = Pending.isPending th <$> readTVarIO self.pending --- | Drop queued transactions that are already in the database, and recompute the--- missing parents of the others.-updateOrphans :: BlockStore -> IO ()-updateOrphans self = do- orphans <- Pending.waiting <$> readTVarIO self.pending- updates <- forM orphans $ \p -> do- let h = txHash p.tx- exists h >>= \case- True -> return (h, Nothing)- False -> do- found <- catMaybes <$> mapM (getUnspent self . (.outpoint)) p.tx.inputs- let have = HashSet.fromList (map (.outpoint.hash) found)- return (h, Just (HashSet.difference p.deps have))- changePending self (Pending.refresh (HashMap.fromList updates))- where- exists th =- getTxData self th >>= \case- Nothing -> return False- Just TxData {deleted = True} -> return False- Just TxData {deleted = False} -> return True- -- | Of the transactions that a peer announced, those to fetch: given -- whether one is queued or was requested lately, which of them the fleet -- has seen ('seen' of t'BlockStoreConfig'), and whether one is in the@@ -1105,14 +1069,13 @@ <> cs (show target.height) throwIO $ AncestorNotInChain height h --- | Forget a peer: release it if it was the syncing peer, and forget that its mempool was requested.+-- | Forget a peer: release it if it was the syncing peer. finishPeer :: BlockStore -> Peer -> IO () finishPeer self p = do released <- transition self (Sync.release p) when released $ do logDebug self $ "Releasing peer: " <> p.label void (releasePeer self.owner p)- atomically $ modifyTVar self.syncedMempool $ HashSet.delete p.label -- | Make a peer the syncing peer if there is none and the peer is not busy. trySetPeer :: BlockStore -> Peer -> IO Bool@@ -1143,7 +1106,8 @@ False -> revertToMainChain self >> search self [] -- | Note the height that a newly connected peer gave, then start syncing if--- needed, or request its mempool if in sync.+-- needed, or, if in sync, ask the peers for their mempools if a refill+-- waits for a peer ('syncMempool'). trySyncingPeer :: BlockStore -> Peer -> IO () trySyncingPeer self p = do online <- getOnlinePeer self.config.peerMgr p@@ -1151,7 +1115,7 @@ atomically . modifyTVar' self.knows $ HashMap.insertWith (\_ k -> Known (max height k.height) k.tip) p.label (Known height Nothing) isInSync self >>= \case- True -> syncMempool self p+ True -> syncMempool self False -> trySyncing self -- | Release the syncing peer, if any.@@ -1214,7 +1178,6 @@ changePending self (Pending.prune now) setStoreHeight self setHeadersHeight self- setPendingTxs self setPeersConnected self trySyncing self checkTime self
src/Haskoin/Store/BlockStore/Pending.hs view
@@ -6,167 +6,79 @@ -- Stability : experimental -- Portability : POSIX ----- Mempool transactions on their way into the database, as a pure value with--- transitions.------ A transaction that a peer announces goes through these states:+-- The transactions that the block store asked its peers for and that have+-- not arrived yet, as a pure value with transitions. ----- @--- announced ──► requested ──► received ──► ready ──► taken for import--- │ │ (missing parents arrive)--- ▼ ▼--- forgotten dropped--- (after 20 s) (after an hour)--- @+-- A transaction that a peer announces is requested once, and not again+-- while the request is fresh: another peer that announces it meanwhile is+-- not asked. When it arrives it is no longer a request ('answered'), and+-- goes on to the pool of the fleet. A request that no peer answers is+-- forgotten after 'requestTimeout', so that the next announcement asks+-- again. ----- It is requested once, and not again while the request is fresh. Once--- received, it waits until no parent is missing, and is then taken for--- import together with other ready transactions, parents first.+-- The block store holds no transaction here: what arrives is offered to+-- the pool at once, and what waits for its parents waits in the fleet+-- member ("Haskoin.Store.Fleet.Member"). -- -- = Why a pure value ----- The block store kept the queue and the recent requests in two variables,--- changed by several functions, and forgot each request in a thread of its--- own that slept for twenty seconds: one thread per transaction requested.--- Here every change is a function, the clock is an argument, and requests--- are forgotten by 'prune' on the clock tick of the block store, like the--- transactions that waited too long.+-- The block store kept the recent requests in a variable changed by+-- several functions, and forgot each request in a thread of its own that+-- slept for twenty seconds: one thread per transaction requested. Here+-- every change is a function, the clock is an argument, and requests are+-- forgotten by 'prune' on the clock tick of the block store. module Haskoin.Store.BlockStore.Pending ( -- * State Pending,- PendingTx (..), empty, isPending,- queued, size,- waiting, -- * Transitions requested, answered,- received,- takeReady,- imported,- refresh, prune, -- * Timeouts requestTimeout,- orphanTimeout, ) where import Data.HashMap.Strict (HashMap) import Data.HashMap.Strict qualified as HashMap-import Data.HashSet (HashSet)-import Data.HashSet qualified as HashSet-import Data.Maybe (mapMaybe) import Data.Time.Clock (NominalDiffTime, UTCTime, diffUTCTime)-import Haskoin (Tx, TxHash, txHash)-import Haskoin.Store.Common (sortTxs)---- | Transaction received from a peer and waiting to be imported.-data PendingTx = PendingTx- { -- | when it was received- time :: !UTCTime,- -- | the transaction- tx :: !Tx,- -- | parent transactions that are missing; empty if it can be imported- deps :: !(HashSet TxHash)- }- deriving stock (Show, Eq, Ord)+import Haskoin (TxHash) --- | Transactions requested from peers and received from them.-data Pending = Pending- { -- | received, by hash- txs :: !(HashMap TxHash PendingTx),- -- | requested and not received yet, with when- asked :: !(HashMap TxHash UTCTime)- }+-- | Transactions requested from peers and not received yet, with when.+newtype Pending = Pending (HashMap TxHash UTCTime) deriving stock (Show, Eq) -- | How long a request keeps the transaction from being requested again. requestTimeout :: NominalDiffTime requestTimeout = 20 --- | How long a transaction waits for missing parents.-orphanTimeout :: NominalDiffTime-orphanTimeout = 3600---- | Nothing requested or received.+-- | Nothing requested. empty :: Pending-empty = Pending HashMap.empty HashMap.empty+empty = Pending HashMap.empty --- | Whether a transaction is received or requested, so that it is not+-- | Whether a transaction was requested recently, so that it is not -- requested again. isPending :: TxHash -> Pending -> Bool-isPending h p = HashMap.member h p.txs || HashMap.member h p.asked---- | How many transactions are received and not taken for import yet.-queued :: Pending -> Int-queued p = HashMap.size p.txs+isPending h (Pending asked) = HashMap.member h asked --- | How many transactions are received or requested.+-- | How many transactions are requested and not received yet. size :: Pending -> Int-size p = HashMap.size (HashMap.union (() <$ p.txs) (() <$ p.asked))---- | Received transactions that have missing parents.-waiting :: Pending -> [PendingTx]-waiting p = filter (not . null . (.deps)) (HashMap.elems p.txs)+size (Pending asked) = HashMap.size asked -- | Transactions were requested. requested :: UTCTime -> [TxHash] -> Pending -> Pending-requested now hs p = p {asked = HashMap.union (HashMap.fromList [(h, now) | h <- hs]) p.asked}+requested now hs (Pending asked) = Pending (HashMap.union (HashMap.fromList [(h, now) | h <- hs]) asked) -- | A transaction that was requested arrived, to be offered to the pool -- of the fleet: it is no longer a request. answered :: TxHash -> Pending -> Pending-answered h p = p {asked = HashMap.delete h p.asked}---- | A transaction was received, with the parents that it spends and the--- database does not have. It is no longer a request.-received :: UTCTime -> Tx -> HashSet TxHash -> Pending -> Pending-received now tx deps p =- Pending- { txs = HashMap.insert h (PendingTx now tx deps) p.txs,- asked = HashMap.delete h p.asked- }- where- h = txHash tx---- | Take up to this many transactions with no missing parents, parents--- before children.-takeReady :: Int -> Pending -> ([PendingTx], Pending)-takeReady n p = (taken, p {txs = foldr (HashMap.delete . txHash . (.tx)) p.txs taken})- where- ready = HashMap.filter (null . (.deps)) p.txs- taken =- take n $- mapMaybe ((`HashMap.lookup` ready) . txHash . snd) (sortTxs (map (.tx) (HashMap.elems ready)))---- | A transaction is in the database: transactions waiting for it no longer--- miss it.-imported :: TxHash -> Pending -> Pending-imported h p = p {txs = HashMap.map (\t -> t {deps = HashSet.delete h t.deps}) p.txs}---- | After a block, give each waiting transaction what the database now--- says about it: 'Nothing' if it is in the database already, so it is--- dropped, or the parents that it still misses.-refresh :: HashMap TxHash (Maybe (HashSet TxHash)) -> Pending -> Pending-refresh updates p = p {txs = HashMap.mapMaybeWithKey update p.txs}- where- update h t = case HashMap.lookup h updates of- Nothing -> Just t- Just Nothing -> Nothing- Just (Just deps) -> Just t {deps = deps}+answered h (Pending asked) = Pending (HashMap.delete h asked) --- | Forget requests older than 'requestTimeout', and drop transactions--- received longer than 'orphanTimeout' ago.+-- | Forget requests older than 'requestTimeout'. prune :: UTCTime -> Pending -> Pending-prune now p =- Pending- { txs = HashMap.filter (\t -> age t.time <= orphanTimeout) p.txs,- asked = HashMap.filter (\t -> age t <= requestTimeout) p.asked- }- where- age t = now `diffUTCTime` t+prune now (Pending asked) = Pending (HashMap.filter (\t -> now `diffUTCTime` t <= requestTimeout) asked)
src/Haskoin/Store/Common.hs view
@@ -12,8 +12,9 @@ -- -- 'StoreRead' is the queries that the web server asks about blocks, -- transactions, addresses and extended public keys, as a record of functions--- in 'IO': "Haskoin.Store.Database.Query" gives them for the RocksDB--- database, and the web server puts the Redis cache in front of some.+-- in 'IO': "Haskoin.Store.Database.Query.Pooled" gives them for the RocksDB+-- database with the fleet's pool on top, and the web server puts the cache+-- of xpubs in front of some. -- -- This module also has helpers for pagination ('Limits'), for deriving -- addresses from extended public keys ('scanXPubBals'), for the order of@@ -252,12 +253,12 @@ -- another from one that it mistyped. With this it is told which -- transaction won, and in which block. ----- A store keeps it with the chain: written when the block is imported and--- deleted when it is reverted, for 'replacedKept' blocks. That depends on--- the mempool that the block found as well, which is each store's own, so--- the stores of a fleet answer from what the shared pool keeps instead--- ("Haskoin.Store.Fleet.Pool".'Haskoin.Store.Fleet.Pool.advanceReplacing'),--- which is the same for all of them.+-- The pool that the stores of a fleet share keeps it as the tip moves past+-- the block, for 'replacedKept' blocks, the same for all of them+-- ("Haskoin.Store.Fleet.Pool".'Haskoin.Store.Fleet.Pool.advanceReplacing').+-- Stores before version 1.16.0 kept it with the chain, from their own+-- mempools: nothing writes those records any more, and the import and the+-- revert of blocks delete them as they did. data Replaced = Replaced { -- | the transaction of the block that spends what it, or an ancestor, -- spent
src/Haskoin/Store/Database/Merge.hs view
@@ -64,8 +64,8 @@ -- -- * __Readers pay for the merge.__ A read of a key with merge records -- pending looks into every table file that has one of them, down to the--- value, and merges. @haskoin-store-bench merge@ measures it, and--- @docs\/one-fleet.md@ has the figures.+-- value, and merges: a few microseconds a key, which+-- @haskoin-store-bench merge@ measures (@docs\/block-import.md@). -- -- * __Reverts read.__ Only the import of a block writes merge records. -- Taking a block out of the chain, and everything else that changes a
src/Haskoin/Store/Database/Query/Pooled.hs view
@@ -26,10 +26,11 @@ -- -- = What a pool cannot say ----- A transaction that left a mempool unconfirmed is kept by the store that--- deleted it, and by no other. A fleet cannot answer the same about it from--- every store, so these queries do not know deleted transactions at all: a--- page that starts at one is empty.+-- A transaction that left a mempool unconfirmed is kept nowhere, but for+-- the bodies that a database of version 20 still has of those that its own+-- mempool deleted, which no other store has. A fleet cannot answer the same+-- about them from every store, so these queries do not know deleted+-- transactions at all: a page that starts at one is empty. module Haskoin.Store.Database.Query.Pooled ( PoolReads (..), pooledRead,
src/Haskoin/Store/Database/Reader.hs view
@@ -52,7 +52,6 @@ DataMetrics (..), Derived (..), derivedAddrs,- derivedAddrsCached, createDataMetrics, withLayeredReader, withReader,@@ -82,7 +81,7 @@ import Data.HashMap.Strict (HashMap) import Data.HashMap.Strict qualified as HashMap import Data.IORef (IORef, atomicModifyIORef', newIORef)-import Data.Maybe (isJust, mapMaybe)+import Data.Maybe (fromMaybe, mapMaybe) import Data.Text (Text) import Data.Text qualified as T import Data.Word (Word32)@@ -131,14 +130,7 @@ -- | Addresses of an xpub, from the cache or a new lazy derivation. derivedAddrs :: DatabaseReader -> XPubSpec -> IO Derived-derivedAddrs r xpub = fst <$> derivedAddrsCached r xpub---- | Addresses of an xpub, from the cache or a new lazy derivation, and--- whether the cache had them. A request pays for deriving the addresses of--- an xpub only when it did not ("Haskoin.Store.Web.Tokens"): those that--- the cache has were derived for an earlier request, as far as it needed.-derivedAddrsCached :: DatabaseReader -> XPubSpec -> IO (Derived, Bool)-derivedAddrsCached r xpub = do+derivedAddrs r xpub = do let ref = r.derived fresh = let addrs m =@@ -149,14 +141,14 @@ in Derived (addrs 0) (addrs 1) atomicModifyIORef' ref $ \c -> case HashMap.lookup xpub c.newer of- Just d -> (c, (d, True))+ Just d -> (c, d) Nothing ->- let (d, hit) = maybe (fresh, False) (,True) (HashMap.lookup xpub c.older)+ let d = fromMaybe fresh (HashMap.lookup xpub c.older) c' | HashMap.size c.newer >= derivedCacheSize = DerivedCache {newer = HashMap.singleton xpub d, older = c.newer} | otherwise = c {newer = HashMap.insert xpub d c.newer}- in (c', (d, hit))+ in (c', d) -- | StatsD metrics of database reads. Counters count the items read. data DataMetrics = DataMetrics@@ -292,7 +284,7 @@ maxBackgroundJobs = jobs, blockCacheSize = blockCache, -- index and filter blocks stay with their open tables, outside the- -- block cache, where data blocks cannot push them out: about 0.34%+ -- block cache, where data blocks cannot push them out: about 0.9% -- of the data, held for as long as a table is open cacheIndexAndFilterBlocks = False, partitionIndexAndFilters = False
src/Haskoin/Store/Database/Types.hs view
@@ -400,8 +400,9 @@ -- | Key of what is kept of a mempool transaction that a block replaced -- ('Replaced'), in the @block@ column family, since haskoin-store 1.13.0.--- It is part of the chain: written and deleted only by the import and the--- revert of blocks.+-- It is part of the chain: written by the import of blocks until version+-- 1.16.0, when the shared pool took this over, and deleted by the import+-- and the revert of blocks. data ReplacedKey = ReplacedKey !TxHash deriving (Show, Read, Eq, Ord, Generic, Hashable)
src/Haskoin/Store/Database/Views.hs view
@@ -7,9 +7,9 @@ -- Portability : POSIX -- -- The database as it was at each of its recent blocks: a RocksDB snapshot--- for each, kept until it is no longer needed. Nothing in the store uses--- this yet. It is the first part of the store's side of "A fleet that--- answers as one" in @docs\/one-fleet.md@.+-- for each, kept until it is no longer needed. The member of the fleet+-- keeps them ("Haskoin.Store.Fleet.Member"), and a request is answered from+-- the one at the fleet tip (@docs\/one-fleet.md@). -- -- = Why --
src/Haskoin/Store/Database/Writer.hs view
@@ -13,12 +13,12 @@ -- A 'Writer' records every change in in-memory hash tables, and its reads look -- in those tables first and fall back to the database. When the function given -- to 'runWriter' returns, all the changes are written to RocksDB in a single--- atomic batch. A block or a set of transactions is thus either fully imported--- or not at all.+-- atomic batch. A block is thus either fully imported or not at all. -- -- The writer is a plain handle passed to plain functions in 'IO' ('getTxData', -- 'insertTx', ...), rather than a monad transformer with class instances: the -- import logic says which writer it uses, and needs nothing but 'IO' to run.+-- -- = Why buffer writes -- -- Importing a block touches thousands of keys, and many of them several times:@@ -140,7 +140,6 @@ deleteSpender, -- * Transactions that blocks replaced- insertReplaced, deleteReplaced, getReplacedAt, )@@ -1209,10 +1208,6 @@ -- the first of the union wins: what the batch adds comes before what -- the database has, and no group is both whole and added to return (IntMap.unions (added <> (kept : map snd changed)))---- | Keep what a block replaced.-insertReplaced :: Writer -> TxHash -> Replaced -> IO ()-insertReplaced w h r = T.insert w.memory.replacedTable h (Just r) -- | Forget what was kept of a replaced transaction. deleteReplaced :: Writer -> TxHash -> IO ()
src/Haskoin/Store/Fleet.hs view
@@ -8,9 +8,9 @@ -- -- How stores that share one Redis agree on the block that they all answer -- from, as pure decisions. "Haskoin.Store.Fleet.Redis" carries them out.--- This is the first part of the design in @docs\/one-fleet.md@, "A--- fleet that answers as one", written down so that it can be tested before--- a store depends on it.+-- The design is in @docs\/one-fleet.md@, and every store runs it as a+-- member of a fleet ("Haskoin.Store.Fleet.Member"). The decisions are pure+-- so that they can be tested without Redis. -- -- = The problem --
src/Haskoin/Store/Fleet/Member.hs view
@@ -12,8 +12,8 @@ -- the Redis that the fleet shares, so that every store of the fleet gives -- the same answer to the same request. The rules are in -- "Haskoin.Store.Fleet" and "Haskoin.Store.Fleet.Pool", Redis carries them--- out through "Haskoin.Store.Fleet.Redis", and the reasons are in "A fleet--- that answers as one" in @docs\/one-fleet.md@.+-- out through "Haskoin.Store.Fleet.Redis", and the reasons are in+-- @docs\/one-fleet.md@ and @docs\/fleet-mempool.md@. -- -- = What a member does --@@ -27,10 +27,11 @@ -- which transactions they confirmed and which outputs they spent, and -- across a reorganisation which transactions come back. Across more -- blocks than a store goes through ('farthest'), it moves the tip and--- starts the pool again ('wayBetween'), and every member that sees that--- the pool was emptied offers its whole mempool again ('noteRestarts').+-- starts the pool again ('wayBetween'), which every store then fills+-- again ('noteRestarts'). ----- * __It offers its mempool to the shared pool__ ('offerMempool'), each+-- * __It offers to the shared pool what its peers send it__ ('memberReceived',+-- 'offerMempool'), each -- transaction with whether its inputs are unspent in the store's view at -- the fleet tip, so that the pool only takes what is valid there. What -- the pool decided on is seen by the fleet, in Redis, and no store@@ -48,23 +49,21 @@ -- block in the same commit as the pool when it moves the tip -- ('moveTip'), a reorganisation undone address by address. ----- = Why the member does not withdraw what it deletes+-- = Why the member does not withdraw transactions ----- A store deletes a transaction from its own mempool when it replaces it,--- when a block confirms a conflicting one, and when it is too old. The--- first reaches the pool with the offer of the replacement, which names--- what it replaces. The second reaches it when the tip moves to that block.--- The third is done on the pool itself, by its own times ('expirePool'), so--- that the stores do not have to agree on when each of them first saw a--- transaction. A store that simply does not have a transaction says--- nothing about it: another store has it.+-- A transaction leaves the pool when it is replaced, when a block confirms+-- a conflicting one, and when it is too old. The first comes with the offer+-- of the replacement, which names what it replaces. The second comes when+-- the tip moves to that block. The third is done on the pool itself, by its+-- own times ('expirePool'), so that the stores do not have to agree on when+-- each of them first saw a transaction. A store that simply does not have a+-- transaction says nothing about it: another store has it. module Haskoin.Store.Fleet.Member ( -- * A member Member (..), MemberConfig (..), newMember, noteBest,- memberForget, memberTick, memberLoop, memberAlive,@@ -309,16 +308,12 @@ reported :: !(TVar [BlockHash]), -- | the fleet tip, read after that report tip :: !(TVar (Maybe BlockHash)),- -- | the transactions of the store's mempool that the pool has- offered :: !(TVar (HashSet TxHash)),- -- | those that it refused, or that there was no time to offer, with- -- the times at which they entered the mempool: they are offered again+ -- | the transactions that peers sent and that are still to offer: new,+ -- refused as orphans, or with no time to offer them yet, with the times+ -- at which the store heard of them; they are offered at the next turn.+ -- What the pool has decided on is in the fleet's seen set instead+ -- ('memberSeen'), which Redis keeps for every store. waiting :: !(TVar (HashMap TxHash (UnixTime, Tx))),- -- | how many times the mempool was offered, and at which tip last- rounds :: !(TVar Int),- offeredAt :: !(TVar (Maybe BlockHash)),- -- | whether the best block changed since the mempool was last offered- changed :: !(TVar Bool), -- | whether the store's own health checks pass. While they do not, the -- member does nothing, so that its lease runs out and a store that is -- stuck does not hold the fleet back.@@ -331,23 +326,18 @@ built :: !(TVar (HashMap ByteString Int)), -- | the door to the cache, for the web server door :: !XPubs,- -- | how many times the pool had been emptied at this store's last- -- turn ('noteRestarts')- restarts :: !(TVar (Maybe Integer)), -- | the metrics of the cache, if the configuration has a StatsD client metrics :: !(Maybe CacheMetrics),- -- | transactions that the store deleted from its mempool since it- -- started, up to a hundred thousand. The database keeps nothing of- -- them, and the rule for conflicts in the pool asks about them- -- ('offerMempool').- gone :: !(TVar (HashSet TxHash)), -- | whether the last thing that the member tried with Redis failed, -- so that an outage is logged once, by whichever loop meets it outage :: !Outage, -- | the epoch of Redis at the last turn ('noteEpoch') epoch :: !(TVar (Maybe ByteString)),- -- | how many times this store found that Redis had lost its data: the- -- web server fills the pool again each time ('Haskoin.Store.Web')+ -- | the count of restarts of the pool at the last turn ('noteRestarts')+ restarts :: !(TVar (Maybe Integer)),+ -- | how many times this store found the pool emptied, because Redis+ -- lost its data or the pool started again: the web server fills the+ -- pool again each time ('Haskoin.Store.Web') refills :: !(TVar Int), -- | the last tip that Redis was seen to have, and when, by the clock of -- 'getMonotonicTimeNSec': a request that read nothing of the pool@@ -371,18 +361,13 @@ recent <- newTVarIO [] reported <- newTVarIO [] tip <- newTVarIO Nothing- offered <- newTVarIO HashSet.empty waiting <- newTVarIO HashMap.empty- rounds <- newTVarIO 0- offeredAt <- newTVarIO Nothing- changed <- newTVarIO True healthy <- newTVarIO True- restarts <- newTVarIO Nothing metrics <- mapM newCacheMetrics config.stats- gone <- newTVarIO HashSet.empty outage <- newOutage built <- newTVarIO HashMap.empty epoch <- newTVarIO Nothing+ restarts <- newTVarIO Nothing refills <- newTVarIO 0 seen <- newIORef ("", 0) lock <- redisLock (defaultLockConfig (config.prefix <> "fleet:") config.name) redis xpubId@@ -409,24 +394,9 @@ noteBest :: Member -> BlockHash -> IO () noteBest m block = do takeView m.views block- atomically $ do- modifyTVar' m.recent $ \blocks -> take m.config.depth (block : filter (/= block) blocks)- writeTVar m.changed True+ atomically . modifyTVar' m.recent $ \blocks -> take m.config.depth (block : filter (/= block) blocks) prune m --- | Note that transactions left the store's mempool unconfirmed. The store--- calls this when it deletes them. The member forgets that it offered them:--- one that comes back, as a transaction does that is sent again after it--- was replaced, is then offered again, and takes the place of what replaced--- it if this store's rules say so.-memberForget :: Member -> [TxHash] -> IO ()-memberForget m hs = atomically $ do- modifyTVar' m.offered (\have -> foldr HashSet.delete have hs)- modifyTVar' m.waiting (\held -> foldr HashMap.delete held hs)- modifyTVar' m.gone $ \seen ->- let more = foldr HashSet.insert seen hs- in if HashSet.size more > 100000 then HashSet.fromList hs else more- -- | Let go of the views that the rules do not ask for: all but those of -- the recent blocks, of the last report, and of the tip read after it. prune :: Member -> IO ()@@ -445,20 +415,20 @@ _ -> False -- | One round of what a member does: report, move the tip if the stores--- agree on another block, offer what the pool does not have of the store's--- mempool, let go of old views, and trim what the fleet has seen+-- agree on another block, offer what peers sent that it has not offered+-- yet, let go of old views, and trim what the fleet has seen -- ('fleetTrimSeen'). Call it every second or two, and when -- the best block changes. A store whose own health checks fail must not -- call it: its lease then runs out, and the fleet goes on without it. memberTick :: Member -> IO () memberTick m = do noteEpoch m+ noteRestarts m report m moved <- moveTip m -- the tip is read after the report, as the rule of 'F.keep' asks tip <- fmap decoded <$> fleetTip m.redis m.keys atomically (writeTVar m.tip (fromMaybe Nothing tip))- noteRestarts m prune m cacheGauges m unless moved (expirePool m)@@ -467,26 +437,10 @@ where decoded = either (const Nothing) Just . decode --- | Forget what this store offered to the pool, if the pool was emptied--- since the store's last turn, by this store or by another--- ('fleetRestarts'): the next offer is then of the whole mempool. Without--- this a store went on taking what it had offered to be in the pool, and--- the pool of a fleet that started again had only what came in afterwards.-noteRestarts :: Member -> IO ()-noteRestarts m = do- now <- fleetRestarts m.redis m.keys- atomically $ do- before <- readTVar m.restarts- writeTVar m.restarts (Just now)- when (maybe False (/= now) before) $ do- writeTVar m.offered HashSet.empty- writeTVar m.waiting HashMap.empty- writeTVar m.changed True- -- | Find out whether Redis lost its data since this store's last turn. -- Redis keeps an epoch, a random number that the first store to find it--- missing sets. A store that finds it missing or changed forgets what it--- offered to the pool and its view of the fleet tip, so that it is+-- missing sets. A store that finds it missing or changed forgets its view+-- of the fleet tip, so that it is -- unhealthy until a tip is set again, which its next turn does, since -- every live store reports. It counts the loss in 'refills', and the web -- server then asks the peers, and the node over RPC if it has one, for@@ -504,10 +458,9 @@ writeTVar m.epoch now let lost = isJust before && before /= now when lost $ do- writeTVar m.offered HashSet.empty- writeTVar m.offeredAt Nothing writeTVar m.tip Nothing- writeTVar m.changed True+ -- the count of restarts went with the rest: not a second refill+ writeTVar m.restarts Nothing modifyTVar' m.refills (+ 1) return lost when lost . m.config.note LevelWarn $@@ -517,6 +470,23 @@ redis' :: R.Redis (Either R.Reply a) -> IO a redis' f = R.runRedis m.redis f >>= either (fail . ("Redis error: " <>) . show) return +-- | Find out whether the pool was emptied since this store's last turn+-- ('fleetRestarts'), and if so count a refill, as 'noteEpoch' does for a+-- loss of Redis. The pool forgot with it what the fleet had seen, so what+-- the peers announce from then on is fetched again; the refill brings+-- back what they announced before. The first turn only notes the count.+noteRestarts :: Member -> IO ()+noteRestarts m = do+ now <- fleetRestarts m.redis m.keys+ restarted <- atomically $ do+ before <- readTVar m.restarts+ writeTVar m.restarts (Just now)+ let restarted = maybe False (/= now) before+ when restarted $ modifyTVar' m.refills (+ 1)+ return restarted+ when restarted . m.config.note LevelInfo $+ "The pool started again: it fills again from the node and the peers"+ -- | Take a turn as a member every two seconds, and at once when an action -- that waits for the next best block returns, for as long as the store's -- health checks pass. A turn that fails, as when Redis is away, is tried@@ -877,69 +847,60 @@ mapMaybe (address . (.script)) (IntMap.elems td.prevs) <> mapMaybe (address . (.script)) td.tx.outputs --- | Offer to the pool the transactions of the store's mempool that it does--- not have, oldest first, so that each comes after those that it spends--- from. A store without a view of the fleet tip offers nothing: it cannot--- say what is unspent there.------ A transaction that the pool refuses is offered again at the next round:--- its parents may not be there yet, or the tip may have moved. One that--- conflicts with transactions of the pool takes their place when this--- store deleted every one of them by its own rules, as it does when it--- replaces a transaction, and otherwise stays out: the first of two that--- spend the same output stays.+-- | Offer to the pool the transactions that peers sent this store and that+-- it has not offered yet ('memberReceived'), oldest first, so that each+-- comes after those that it spends from. A store without a view of the+-- fleet tip offers nothing: it cannot say what is unspent there. ----- = Reading no more of the mempool than is new+-- A transaction whose inputs neither the tip nor the pool has, an orphan+-- for now, waits for the next round, as does everything after an offer+-- that the pool turned away because the tip had moved. An orphan is let go+-- 'orphanLife' after the store heard of it: its parent is not coming. One that conflicts+-- with transactions of the pool takes their place: a peer relayed it, so+-- its node took it in place of them. ----- A mempool has tens of thousands of transactions and a round comes every--- two seconds, so a round does not read the whole list. It reads the list--- from the newest, a page at a time, until a page has nothing that it does--- not know, and offers what it found with what is waiting from earlier--- rounds. The whole list is read when the tip has moved or the store's best--- block has changed, since a block or a reorganisation changes the mempool--- anywhere, and every 'wholeEvery' rounds, for whatever the short way--- missed. What the store deletes in between it tells the member--- ('memberForget'). A round offers 'mostOffers'--- transactions at most and leaves the rest waiting, so that a store with a--- large mempool to offer still reports in time to keep its lease.+-- A round offers 'mostOffers' transactions at most and leaves the rest+-- waiting, so that a store with many to offer still reports in time to+-- keep its lease. offerMempool :: Member -> IO () offerMempool m = readTVarIO m.tip >>= \case Nothing -> return () Just tip -> void . withViewAt m.views tip $ \at -> do+ now <- round <$> getPOSIXTime held <- readTVarIO m.waiting let todo = take mostOffers (sortOn (fst . snd) (HashMap.toList held)) pool = redisPool m.redis m.keys (encode tip) (return ())- go at pool tip todo+ go now at pool tip todo where- go _ _ _ [] = return ()- go at pool tip ((h, (time, tx)) : rest) =+ go _ _ _ _ [] = return ()+ go now at pool tip ((h, (time, tx)) : rest) = txDataAt at pool time tx >>= \case -- an input that neither the tip nor the pool has: an orphan for now- Nothing -> go at pool tip rest+ Nothing -> orphan now h time >> go now at pool tip rest Just td -> do unspent <- forM [i.outpoint | i <- tx.inputs, i.outpoint /= nullOutPoint] $ \p -> isJust <$> Q.getUnspent at p let offer = poolOffer m.redis m.keys 32 (Just (encode tip)) (poolTx m.db.net td) unspent- done = atomically $ do- modifyTVar' m.offered (HashSet.insert h)- modifyTVar' m.waiting (HashMap.delete h)+ done = atomically (modifyTVar' m.waiting (HashMap.delete h)) offer [] >>= \case- Taken -> done >> go at pool tip rest- Known -> done >> go at pool tip rest- Orphan _ -> go at pool tip rest+ Taken -> done >> go now at pool tip rest+ Known -> done >> go now at pool tip rest+ Orphan _ -> orphan now h time >> go now at pool tip rest Late -> return () -- a peer relayed it, so its node took it in place of what it -- conflicts with: it replaces them in the pool too- Conflict rivals -> offer rivals >> done >> go at pool tip rest+ Conflict rivals -> offer rivals >> done >> go now at pool tip rest+ orphan now h time =+ when (now > time + orphanLife) $+ atomically (modifyTVar' m.waiting (HashMap.delete h)) -- | A transaction that the store heard from a peer, to offer to the pool at -- the next turn, with the time at which it was heard. There is no mempool -- of the store's own: the pool in Redis is the mempool. memberReceived :: Member -> UnixTime -> Tx -> IO ()-memberReceived m time tx = atomically $ do- have <- HashSet.member (txHash tx) <$> readTVar m.offered- unless have $ modifyTVar' m.waiting (HashMap.insertWith (\_ old -> old) (txHash tx) (time, tx))+memberReceived m time tx =+ atomically $ modifyTVar' m.waiting (HashMap.insertWith (\_ old -> old) (txHash tx) (time, tx)) -- | Of these transactions, those that the block store need not fetch when -- a peer announces them: those that this store holds to offer, and those@@ -1023,6 +984,15 @@ spenders = IntMap.empty } +-- | How long, in seconds, a transaction waits for its parent before it is+-- let go: twenty minutes, as Bitcoin Core keeps its orphans. A parent comes+-- within seconds when it comes at all, in a block or from a peer; one that+-- has not come by then was replaced or never reached the network. Kept for+-- ever, such orphans would pile up, and once they were 'mostOffers', the+-- oldest first, a round would offer nothing else.+orphanLife :: UnixTime+orphanLife = 1200+ -- | The most transactions that one round offers. An offer takes a tenth of -- a millisecond or two, so this is well under the two seconds of a round. mostOffers :: Int@@ -1112,7 +1082,7 @@ -- | 'answeringAt' with the queries ready: those of -- "Haskoin.Store.Database.Query.Pooled" for the view and the pool, and -- the same with the cache of extended public keys in front--- ('cachedRead'), which is what requests are answered with unless they+-- ('fleetSession'), which is what requests are answered with unless they -- ask not to be. answeringCached :: Member -> ByteString -> (StoreRead -> Session -> IO () -> IO a) -> IO (Maybe a) answeringCached m tip action = answeringWith m tip $ \r pool found confirm -> do
src/Haskoin/Store/Fleet/Pool.hs view
@@ -6,17 +6,17 @@ -- Stability : experimental -- Portability : POSIX ----- The mempool that stores sharing one Redis answer from, as pure decisions:--- the second part of "A fleet that answers as one" in--- @docs\/one-fleet.md@, after the fleet tip of "Haskoin.Store.Fleet".--- "Haskoin.Store.Fleet.Redis" carries it out.+-- The mempool that stores sharing one Redis answer from, as pure decisions,+-- beside the fleet tip of "Haskoin.Store.Fleet". It is described in+-- @docs\/fleet-mempool.md@, and the design around it in+-- @docs\/one-fleet.md@. "Haskoin.Store.Fleet.Redis" carries it out. -- -- = The problem ----- Every store has a mempool of its own, filled in the order in which its--- peers sent it transactions, and no two are the same. Stores that answer--- from the same block still answer differently if each lays its own mempool--- on top.+-- A mempool of each store's own would be filled in the order in which its+-- peers sent it transactions, and no two would be the same. Stores that+-- answer from the same block would still answer differently if each laid+-- its own mempool on top. -- -- = One pool, the mempool as of the fleet tip --@@ -25,9 +25,9 @@ -- up to that block, and it changes with the tip in the same step, so that -- no request sees a block without what it did to the pool. ----- A store's own mempool does not go away. It is what the store offers to--- the pool and relays to its peers, and what it answers from when Redis is--- down.+-- No store keeps a mempool of its own. A store offers the pool what its+-- peers send it, and while Redis is down it answers 503 rather than from+-- anything else. -- -- = The rules --
src/Haskoin/Store/Fleet/Redis.hs view
@@ -367,16 +367,20 @@ -- other stores live. The length of the way is the same for all of them, so -- none would ever say what the blocks did, and the tip would stay where it -- is for good: a fleet whose stores came back together after half a day--- away answered nothing from then on. Emptying the pool costs the stores--- an offer of their mempools, which they make when they see that the--- count of 'fleetRestarts' went up.+-- away answered nothing from then on. Emptying the pool costs the+-- transactions that were in it until the stores fill it again: what the+-- fleet has seen is forgotten with it, and the count of 'fleetRestarts'+-- goes up, so that each store reads the mempool of its node and asks its+-- peers for theirs. fleetRestart :: Connection -> FleetKeys -> Int -> Maybe Time -> Maybe ByteString -> ByteString -> IO Moved fleetRestart conn keys tsize clock seen target = advance conn keys tsize clock seen target "2" (PoolChange [] [] [] [] Nothing []) ([], "") --- | How many times the pool was emptied to move the tip. A store that sees--- another number than at its last turn offers all of its mempool again:--- what it offered before is not in the pool any more.+-- | How many times the pool was emptied, by 'fleetRestart' or by a store+-- alone that did not have the old tip. A store that sees another number+-- than at its last turn fills the pool again, as after Redis lost its+-- data: the emptied pool is shared, so every store sees the same count+-- and any of them that can read a mempool helps. fleetRestarts :: Connection -> FleetKeys -> IO Integer fleetRestarts conn keys = maybe 0 (maybe 0 fst . C.readInteger) <$> run conn (R.get (keys.prefix <> "fleet:restarts"))@@ -422,10 +426,11 @@ -- | Something that happened to the fleet, as every store of it tells its -- clients. ----- A store alone tells what its own node saw. The stores of a fleet answer--- from one tip and one pool, and their events have to be those of that tip--- and that pool: otherwise a client would hear of a transaction from one--- server and find that the next does not have it. So the script that+-- Every store is a member of a fleet, even a fleet of one. The stores of a+-- fleet answer from one tip and one pool, and their events have to be those+-- of that tip and that pool, not what each store's own node saw: otherwise+-- a client would hear of a transaction from one server and find that the+-- next does not have it. So the script that -- changes the pool or moves the tip appends what it did to a stream in -- Redis, in the same step, and every store reads the stream. data FleetEvent
src/Haskoin/Store/Fleet/XPub.hs view
@@ -86,7 +86,7 @@ -- -- Every key that a request asks about is cached, and the cache evicts by -- GreedyDual ('Haskoin.Store.XPub.greedyLua'): each--- entry has what it cost to build, at the prices of the token design, a use+-- entry has what it cost to build ('Haskoin.Store.XPub.indexCost'), a use -- gives it the level of the cache plus that cost, and the entry of the -- lowest priority goes first, the level rising to it ('cacheEvict'). A -- wallet of 40,000 addresses outlives an empty one used as recently, and
src/Haskoin/Store/Fleet/lua/pool.lua view
@@ -64,8 +64,9 @@ -- otherwise, since another store may know. With '2' the store has both -- tips and the way between them is longer than a store goes through: that -- is so for every store, so any of them may, or the tip would never move--- again. Each time the pool is emptied a counter goes up, next to the--- tip, which tells the stores to offer their mempools again.+-- again. Each time the pool is emptied, what the fleet has seen is+-- forgotten, since what it decided on is gone, and a counter goes up,+-- next to the tip, which tells the stores to fill the pool again. -- -- What the pool holds for each address, kept in step with its transactions: -- its transactions and the outputs paid to it, both ordered newest first,@@ -99,7 +100,6 @@ local key = { tip = prefix .. 'fleet:tip', moves = prefix .. 'fleet:moves',- restarts = prefix .. 'fleet:restarts', events = prefix .. 'fleet:events', lease = prefix .. 'fleet:lease', recent = prefix .. 'fleet:recent',@@ -124,6 +124,7 @@ lostby = prefix .. 'pool:lostby:', lostat = prefix .. 'pool:lostat', seen = prefix .. 'fleet:seen',+ restarts = prefix .. 'fleet:restarts', of = prefix .. 'fx:of:', entry = prefix .. 'fx:e:', xtip = prefix .. 'fx:tip',@@ -217,8 +218,11 @@ return s and tonumber(s) < 0 end +-- add to a field of the summary of an address; the + 0 turns -0, which taking+-- out an output of value 0 gives, into 0: Redis reads '-0' as no integer, and+-- the error would stop the whole script half way local function add(a, field, by)- redis.call('HINCRBY', key.asum .. a, field, string.format('%.0f', by))+ redis.call('HINCRBY', key.asum .. a, field, string.format('%.0f', by + 0)) end -- put the entries of a transaction in, with sign 1, or take them out, with@@ -561,9 +565,16 @@ if ARGV[8] == '1' and #live ~= 1 then return {'raced'} end- redis.call('INCR', key.restarts) for _, k in ipairs(redis.call('KEYS', prefix .. 'pool:*')) do redis.call('DEL', k)+ end+ -- what the pool decided on is gone: the stores fetch it again when it+ -- is announced, and fill the pool again when the counter goes up. The+ -- first tip of a fleet, or the first after Redis lost its data, empties+ -- nothing, and the loss is a refill of its own (noteEpoch).+ redis.call('DEL', key.seen)+ if tip ~= '' then+ redis.call('INCR', key.restarts) end -- the entries have the pool's layer of the pool that is gone settle(true)
src/Haskoin/Store/Manager.hs view
@@ -6,15 +6,16 @@ -- Stability : experimental -- Portability : POSIX ----- Wires the components of a store together: the RocksDB database, the--- peer-to-peer node, the block store that imports data, and the optional Redis--- cache. Node events are forwarded to the block store and turned into--- 'StoreEvent's, which the cache follows.+-- Wires the components of a store together: the Redis of its fleet, the+-- RocksDB database, the peer-to-peer node, the block store that imports+-- data, and the store's membership of its fleet ("Haskoin.Store.Fleet.Member"),+-- which keeps the shared mempool and the cache of xpubs in Redis. Node+-- events are forwarded to the block store and turned into 'StoreEvent's. -- -- Keeping the wiring in one place means each component only knows about the -- interfaces it uses: the node does not know about the block store, and the--- block store does not know about the cache. 'withStore' starts them in dependency order--- and stops them all together.+-- block store knows the member only through a few callbacks. 'withStore'+-- starts them in dependency order and stops them all together. module Haskoin.Store.Manager ( StoreConfig (..), cachePrefix,@@ -96,7 +97,6 @@ MemberConfig (..), memberCacheLoop, memberEvents,- memberForget, memberLoop, memberReceived, memberSeen,@@ -135,10 +135,10 @@ -- without Redis, when the store is unhealthy and answers 503 fleet :: !(Maybe Member), -- | what the store tells its clients: new best blocks and changes of- -- the mempool. For a store alone they are its own, those that the- -- block store publishes.- -- For a member of a fleet they are those of the fleet, read from the- -- Redis that it shares, so that every server tells the same.+ -- the mempool, those of the fleet, read from the Redis that it+ -- shares, so that every server tells the same. Without a fleet, in+ -- tests of the block store alone, they are those that the block store+ -- publishes. events :: !(Publisher StoreEvent), -- | store events are published here pub :: !(Publisher StoreEvent),@@ -201,12 +201,10 @@ stats :: !(Maybe Stats), -- | Bloom filters on the column families that are read by key bloom :: !Bool,- -- | RocksDB max_open_files ('Nothing' = unlimited, which is what a- -- store with a block cache wants: its index and filter blocks are in- -- the cache, an open file costs a few kilobytes, and a file that is- -- not open has to be opened for every lookup that lands in it. Without- -- a block cache every open file keeps its index and filter in memory,- -- and the limit is what bounds them)+ -- | RocksDB max_open_files ('Nothing' = unlimited: a file that is not+ -- open has to be opened for every lookup that lands in it. Every open+ -- file keeps its index and filter in memory, outside the block cache,+ -- about 0.9% of its data, and the limit is what bounds them) maxOpenFiles :: !(Maybe Int), -- | bytes of RocksDB memtables across all column families ('Nothing' = -- unlimited: each column family keeps up to two 64 MiB memtables)@@ -219,9 +217,9 @@ -- them shorten a backlog of compactions where the disk and the cores -- have room, and take both from the requests where they have not backgroundJobs :: !(Maybe Int),- -- | bytes of RocksDB block cache shared by all column families, which- -- also holds their index and filter blocks ('Nothing' = RocksDB default:- -- 32 MiB per column family, index and filter blocks outside it)+ -- | bytes of RocksDB block cache shared by all column families, for+ -- data blocks; index and filter blocks stay outside it, with their open+ -- tables ('Nothing' = RocksDB default: 32 MiB per column family) blockCache :: !(Maybe Int), -- | log messages here logger :: !Logger@@ -282,9 +280,9 @@ cachePrefix :: Network -> ByteString cachePrefix net = "hs2:" <> C.pack net.name <> ":" --- | The store as a member of a fleet, if the configuration says that it is--- one and has a Redis to share (connected by 'connectRedis', which checked--- that it answers): under a name of its own, with a lease of+-- | The store as a member of its fleet, on the Redis that 'connectRedis'+-- connected and checked ('Nothing' only without one, in tests): under a+-- name of its own, with a lease of -- ten seconds, views of its last six blocks, and the store's limit on the -- age of unconfirmed transactions. storeMember :: StoreConfig -> Maybe Redis.Connection -> DatabaseReader -> IO (Maybe Member)@@ -320,16 +318,15 @@ withAsync (memberLoop member (nextBlock sub)) $ \a -> withAsync (memberCacheLoop member) $ \b -> link a >> link b >> action where- -- until the next block, telling the member what the store deletes+ -- until the next block nextBlock sub = receive sub >>= \case StoreBestBlock _ -> return ()- StoreMempoolDelete h -> memberForget member [h] >> nextBlock sub _ -> nextBlock sub --- | Run an action with the events that the store tells its clients. A--- store alone tells its own. A member of a fleet tells those of the--- fleet, which a thread reads from Redis for as long as the action runs.+-- | Run an action with the events that the store tells its clients: those+-- of the fleet, which a thread reads from Redis for as long as the action+-- runs. Without a fleet, in tests, those that the block store publishes. withFleetEvents :: StoreConfig -> Publisher StoreEvent ->@@ -351,9 +348,9 @@ -- | Open the database described by the configuration, bring it to the -- layout that the store runs on ('upgradeDatabase'), and run an action with--- its reader. A database of version 20 has its mempool moved to an area of--- its own here, once, before anything reads it; older stores refuse the--- database from then on.+-- its reader. A database of version 20 has its mempool taken out here,+-- once, before anything reads it ("Haskoin.Store.Database.Migrate"); older+-- stores refuse the database from then on. connectDB :: StoreConfig -> (DatabaseReader -> IO a) ->@@ -376,7 +373,9 @@ done <- upgradeDatabase (logAt LevelInfo (withSource "Database" cfg.logger)) db when (done.moved > 0) $ logAt LevelInfo (withSource "Database" cfg.logger) $- "Moved " <> cs (show done.moved) <> " mempool transactions to their own area"+ "Took "+ <> cs (show done.moved)+ <> " mempool transactions out of the chain; the pool gets them again from the peers" when (done.rewritten > 0) $ logAt LevelInfo (withSource "Database" cfg.logger) $ "Wrote the records of " <> cs (show done.rewritten) <> " confirmed transactions again, to be written once"@@ -401,7 +400,6 @@ onTx = maybe (\_ _ -> return ()) memberReceived member, seen = maybe (\_ -> return mempty) memberSeen member, blockWindow = cfg.blockWindow,- mempoolTimeout = cfg.mempoolTimeout, peerTimeout = cfg.peerTimeout, blockStall = cfg.blockStall, blockPing = cfg.blockPing,
src/Haskoin/Store/Outage.hs view
@@ -9,8 +9,8 @@ -- Saying once that something the store depends on stopped working, and -- once that it works again. ----- The store goes on without Redis: it answers from its database and tries--- again. Several loops try every second or two, and each used to log every+-- The store keeps running without Redis: it answers 503 until Redis is+-- back, and tries again. Several loops try every second or two, and each used to log every -- failure, so an outage of an hour was thousands of lines that said the -- same, and nothing said when it was over. Here a failure is logged as a -- warning when it is the first after things worked, for debugging while
src/Haskoin/Store/Web.hs view
@@ -21,6 +21,7 @@ -- -- Limits on the size of responses are set with 'WebLimits'. Errors are -- returned as JSON with an HTTP status that depends on the 'Except'.+-- -- = Design of the API -- -- * __One encoding.__ The API speaks JSON, which any language reads, and@@ -452,8 +453,8 @@ -- | blocks that the database may lag behind the headers before the store -- is reported unhealthy maxLaggingBlocks :: !Int,- -- | transactions that may be waiting to be imported before the store is- -- reported unhealthy+ -- | transactions that may be requested from peers and not received yet+ -- before the store is reported unhealthy maxPendingTxs :: !Int, -- | fewest peers before the store is reported unhealthy minPeers :: !Int,@@ -518,7 +519,7 @@ -- | the limit on the long requests answered at once admission :: !Admission, -- | the reads of the caches of xpubs for this request, each xpub of one- -- version, and the check that ends it ('xpubSession'); outside of a+ -- version, and the check that ends it ('Haskoin.Store.Fleet.Member.fleetSession'); outside of a -- request, reads of no request reading :: !Door.Session }@@ -583,7 +584,7 @@ serverErrors :: StatCounter, clientErrors :: StatCounter, -- | requests answered again because an entry of a cache that they read- -- changed meanwhile (@web.reruns@, 'xpubSession')+ -- changed meanwhile (@web.reruns@, 'Haskoin.Store.Fleet.Member.fleetSession') reruns :: StatCounter, -- Admission refused :: StatCounter,@@ -803,8 +804,8 @@ patience = fromIntegral config.limits.requestWait * 1000000 } let db = config.store.db- -- the door to the fleet's cache of xpubs; a store without Redis has- -- none, and answers about no xpub+ -- the door to the fleet's cache of xpubs; only a store without Redis,+ -- in tests, has none, and answers about no xpub xpubDoor = (.door) <$> config.store.fleet stateOf out sess q = WebState@@ -1171,7 +1172,7 @@ -- | A route whose handler, once it has its response and before the response -- is sent, checks that every cached xpub that the request read is still of--- the version that it read ('xpubSession'): if one is not, it throws+-- the version that it read ('Haskoin.Store.Fleet.Member.fleetSession'): if one is not, it throws -- v'EntryMoved', and the request is answered again. The routes of xpubs read -- the caches only before they respond. settled :: WebState -> Route -> Route@@ -2044,7 +2045,7 @@ -- -- The peers are those of the store that take transactions. A transaction -- that was handed over is asked for again after the answer--- ('askingLoop'), unless the store keeps no mempool.+-- ('askingLoop'), until the store has it in the chain or the pool. publishing :: WebState -> Publish.Publishing publishing st = Publish.Publishing@@ -2092,14 +2093,17 @@ Nothing -> maybe (return False) (\m -> isJust <$> memberPoolBody m h) own.fleet ask h = getPeers own.peerMgr >>= mapM_ (\op -> blockStoreTxHash op.mailbox [h] own.block) --- | Fill the pool again each time the member finds that Redis lost its--- data ('Haskoin.Store.Fleet.Member.noteEpoch'): ask every peer for its--- mempool ('blockStoreRefill'), and with the RPC of the store's node--- (@--node-rpc@), read the node's whole mempool, 'refillBatch'--- transactions every two seconds, a turn of the member, each given to the--- member as if a peer had sent it ('memberReceived'). The peers alone may--- not answer: a node answers a request for its mempool only from a peer--- that it allows to use bloom filters.+-- | Fill the pool again each time the member finds it emptied, because+-- Redis lost its data ('Haskoin.Store.Fleet.Member.noteEpoch') or the pool+-- started again ('Haskoin.Store.Fleet.Member.noteRestarts'): ask the peers+-- for their mempools, once ('blockStoreRefill'), and with the RPC of the+-- store's node (@--node-rpc@), read the node's whole mempool,+-- 'refillBatch' transactions every two seconds, a turn of the member, each+-- given to the member as if a peer had sent it ('memberReceived'). The+-- peers alone may not answer: Bitcoin Core answers only a peer that it+-- whitelists or when it serves bloom filters, which it does not by+-- default. The RPC answers in any case, and its transactions are offered+-- whether the fleet has seen them or not. refillLoop :: WebState -> IO () refillLoop st = forM_ own.fleet $ \m -> go m 0 where@@ -2386,10 +2390,11 @@ -- runs, each transaction that enters with what the database has of it -- ('seenOf'), for the clients that choose what they hear about. ----- A store alone writes what it publishes. A member of a fleet writes what--- the fleet tells ('memberEvents'), from the start of what Redis keeps, so+-- A member of a fleet, as every store is, writes what the fleet tells+-- ('memberEvents'), from the start of what Redis keeps, so -- that the logs of all the servers of a fleet hold the same events and a--- client can take up on any of them.+-- client can take up on any of them. Without a fleet, in tests, it writes+-- what the block store publishes. feedEvents :: WebState -> IO () feedEvents st = case own.fleet of Just member -> memberEvents member True (mapM_ happened . fleet)@@ -3922,7 +3927,7 @@ withMetrics st (.health) $ do h <- readTVarIO st.health up <- readTVarIO st.cacheUp- -- a store without a cache has no Redis to be without+ -- Redis is mandatory: the answer always says whether it answers let HealthCheck {blocks, lastBlock, lastTx, pendingTxs, peers, network, version, time} = h return HealthCheck {redis = Just up, ..} @@ -3971,7 +3976,7 @@ where ch = store'.chain --- | Transactions waiting to be imported and the maximum allowed.+-- | Transactions requested from peers and not received yet, and the maximum allowed. pendingTxsHealthCheck :: WebConfig -> IO MaxHealth pendingTxsHealthCheck cfg = do n <- blockStorePendingTxs cfg.store.block@@ -4191,7 +4196,7 @@ cached ask = ask >>= maybe (throwIO EntryMoved) return balances x = r.balances x >>= maybe (asked x (2 :: Int)) return -- once indexed, the cache, read at one version as every read of the- -- request ('xpubSession'); an xpub evicted at once is asked for again+ -- request ('Haskoin.Store.Fleet.Member.fleetSession'); an xpub evicted at once is asked for again asked x tries = case gate of Nothing -> throwIO JobBroke Just xs -> do@@ -4364,7 +4369,7 @@ [ [shown (h.blocks.headers - h.blocks.blocks) <> " blocks behind the headers" | not (isOK h.blocks)], [shown h.lastBlock.age <> " seconds since the last block" | not (isOK h.lastBlock)], [shown h.lastTx.age <> " seconds since the last transaction" | not (isOK h.lastTx)],- [shown h.pendingTxs.count <> " transactions waiting to be imported" | not (isOK h.pendingTxs)],+ [shown h.pendingTxs.count <> " transactions requested from peers and not received" | not (isOK h.pendingTxs)], [shown h.peers.count <> " peers" | not (isOK h.peers)] ] where
src/Haskoin/Store/Web/Publish.hs view
@@ -47,9 +47,9 @@ -- -- = Why it is this way ----- * __No waiting for the echo.__ Until 1.12.1 the request waited, up to--- @--publish-timeout@, for a node to announce the transaction back, and--- said what came of it. The wait was arbitrary, Bitcoin Core does not say+-- * __No waiting for the echo.__ The request could wait for a node to+-- announce the transaction back, and say what came of it. But any limit+-- on that wait would be arbitrary, Bitcoin Core does not say -- when it refuses, and the store cannot see the network: a transaction -- that one node refuses may be taken by others. The client is told what -- the store knows, that it was handed over, and finds out the rest the@@ -66,7 +66,7 @@ -- close its connections to the sender, and Bitcoin Cash Node closes them -- all. So the store sends a peer only what it verified, and checks the -- rules whose breach was measured to cost it its connections: the form of--- the input scripts and the tokens ("docs/publishing.md" has what was+-- the input scripts and the tokens (@docs\/publishing.md@ has what was -- measured). A transaction that only breaks a rule of relay, such as one -- that pays too little, is refused by the nodes without harm. --
src/Haskoin/Store/XPub/Lock.hs view
@@ -50,8 +50,8 @@ -- late, it let several stores read the same xpub for requests that came at -- the same time. ----- A store without Redis has no fleet to share xpubs with: 'noLock' always--- gives the lock.+-- Every store is a member of a fleet and takes its locks in Redis. The+-- tests of the indexer use 'noLock', which always gives the lock. module Haskoin.Store.XPub.Lock ( Lock (..), Taken (..),@@ -102,7 +102,7 @@ Held !Int deriving (Eq, Show) --- | The lock of a store alone: always given, never held by another.+-- | A lock that is always given, never held by another: for the tests. noLock :: Lock k noLock = Lock
src/Haskoin/Store/XPub/Service.hs view
@@ -69,8 +69,7 @@ settle :: IO () } --- | No reads: every read is a miss. Outside of a request, and for a store--- without Redis, which answers about no xpub.+-- | No reads: every read is a miss. For the queries outside of a request. unpinned :: Session unpinned = Session
− stress/Main.hs
@@ -1,911 +0,0 @@-{-# LANGUAGE ApplicativeDo #-}-{-# LANGUAGE RecordWildCards #-}-{-# LANGUAGE TemplateHaskell #-}---- |--- Module : Main--- Copyright : (c) Haskoin Developers--- License : MIT--- Maintainer : jprupp@protonmail.ch--- Stability : experimental--- Portability : POSIX------ Stress test of the Redis cache of xpubs. Run it with--- @scripts/stress-cache.sh@, which builds it, runs it and writes a report.------ It starts a regtest chain of Bitcoin Core, Valkey and wrk in Podman--- containers from the test image ("Haskoin.Store.Integration.Image"), and a--- haskoin-store server with the cache on. A few wallets, whose keys this--- program holds, get many coins; then the program makes transactions between--- them, signed here and sent to the node in batches, and mines a block after--- each batch. Every transaction changes two cached xpubs, so the cache--- reloads them again and again while wrk asks the server about them on five--- endpoints at once. At the end, the cached answers are compared with those--- of the database.------ Everything it measures goes to files in the output directory, as JSON--- lines: @rounds.jsonl@ (one line per block), @samples.jsonl@ (one line per--- second: heights, CPU, memory and the StatsD metrics of the server),--- @wrk.jsonl@ (one line per run of wrk) and @summary.json@.--- @scripts/stress-report.py@ turns them into a report.------ = Why it works this way------ * __Transactions are signed here.__ A node wallet makes a few hundred--- transactions per second at best, and slower as it grows; signing with--- the keys in memory and sending raw transactions in batches keeps the node--- busy only with validating them.--- * __Only confirmed coins are spent.__ A block is mined after each batch, so--- chains of unconfirmed transactions, which the node limits, never form.--- * __The server is a separate process__, the release binary, so that its--- CPU and memory are its own, and it is configured from the command line--- as in production.--- * __Containers are removed__ when the program ends, even when it fails:--- they run with @--rm@ inside brackets, and carry a label with the process--- ID, so that @scripts/stress-cache.sh@ and the next run remove any that a--- killed run left.-module Main (main) where--import Control.Concurrent (threadDelay)-import Control.Concurrent.Async (withAsync)-import Control.Concurrent.MVar (MVar, newMVar, withMVar)-import Control.Concurrent.STM (TVar, atomically, newTVarIO, readTVarIO, writeTVar)-import Control.DeepSeq (force)-import Control.Exception (evaluate)-import Control.Lens ((&), (.~), (?~), (^.))-import Control.Monad (forM, forM_, unless, void, when)-import Data.Aeson (Value (..), eitherDecode, encode, object, (.=))-import Data.Aeson qualified as A-import Data.ByteString qualified as BS-import Data.ByteString.Char8 qualified as C8-import Data.ByteString.Lazy qualified as BL-import Data.Bytes.Put (runPutS)-import Data.Bytes.Serial (serialize)-import Data.FileEmbed (embedFile, makeRelativeToProject)-import Data.Foldable (toList)-import Data.HashMap.Strict (HashMap)-import Data.HashMap.Strict qualified as HashMap-import Data.IORef (IORef, atomicModifyIORef', newIORef)-import Data.List (sort)-import Data.Maybe (fromMaybe, mapMaybe)-import Data.Sequence (Seq)-import Data.Sequence qualified as Seq-import Data.Text (Text)-import Data.Text qualified as T-import Data.Text.Encoding (decodeUtf8)-import Data.Time.Clock (UTCTime, diffUTCTime, getCurrentTime)-import Data.Word (Word64)-import Database.Redis qualified as Redis-import Haskoin hiding (Coin, Key)-import Haskoin.Store.Exception (bracket, tryAny)-import Haskoin.Store.Integration.Image (testImage, valkeyBin, wrkBin)-import Haskoin.Store.Integration.Node-import Haskoin.Store.Integration.Podman-import Haskoin.Store.Integration.Valkey (redisContainer, waitRedis)-import Network.Socket qualified as Socket-import Network.Socket.ByteString qualified as SocketB-import Network.Wreq qualified as Wreq-import Numeric (readHex)-import Options.Applicative-import System.Directory (createDirectoryIfMissing, makeAbsolute)-import System.Exit (ExitCode (..))-import System.IO (Handle, IOMode (..), hFlush, hPutStrLn, stderr, withFile)-import System.Process-import System.Random.Stateful (IOGenM, StdGen, mkStdGen, newIOGenM, uniformRM)-import Text.Read (readMaybe)------------------------------------------------------------------------------------ Options----------------------------------------------------------------------------------data Opts = Opts- { txs :: !Int,- wallets :: !Int,- addresses :: !Int,- blockTxs :: !Int,- batch :: !Int,- connections :: !Int,- wrkSeconds :: !Int,- server :: !FilePath,- serverRts :: !String,- out :: !FilePath,- chainMb :: !Int,- seed :: !Int- }--opts :: Parser Opts-opts = do- txs <- num "txs" 200000 "Transactions to make between the wallets"- wallets <- num "wallets" 4 "Wallets, each an xpub"- addresses <- num "addresses" 200 "Receiving and change addresses of each wallet"- blockTxs <- num "block-txs" 2000 "Transactions per block"- batch <- num "batch" 250 "Transactions per call to the node"- connections <- num "connections" 2 "Connections of wrk to each endpoint"- wrkSeconds <- num "wrk-seconds" 10 "Seconds of each run of wrk"- server <- strOption (long "server" <> metavar "PATH" <> help "haskoin-store binary")- serverRts <-- strOption $- long "server-rts"- <> metavar "FLAGS"- <> value "-N4 -M2g"- <> showDefault- <> help "RTS flags of the server"- out <- strOption (long "out" <> metavar "DIR" <> help "Directory for the results")- chainMb <- num "chain-mb" 1024 "Size of the tmpfs of the chain of the node, in MiB"- seed <- num "seed" 1 "Seed of the random choices"- return Opts {..}- where- num name def h = option auto (long name <> metavar "N" <> value def <> showDefault <> help h)------------------------------------------------------------------------------------ Main----------------------------------------------------------------------------------main :: IO ()-main = do- o <- execParser (info (opts <**> helper) (fullDesc <> progDesc "Stress test of the Redis cache of haskoin-store"))- outDir <- makeAbsolute o.out- createDirectoryIfMissing True outDir- podmanUsable >>= either fail return- say "Building or finding the test image"- image <- testImage- withContainer (valkey image) $ \rc -> do- redisURL <- ("redis://127.0.0.1:" <>) . show <$> hostPort rc 6379- waitRedis redisURL- withContainer (nodeSpec o image) $ \nc -> do- rpcPort <- hostPort nc 18443- p2p <- hostPort nc 18444- let node = Node {kind = BitcoinCore, url = "http://127.0.0.1:" <> show rpcPort <> "/", p2p = p2p}- waitRpc node- withContainer (wrkSpec image outDir) $ \wc ->- withStatsd $ \statsPort stats ->- withRamDir $ \dir ->- withServer o outDir dir redisURL p2p statsPort $ \srv ->- stress o {out = outDir} node redisURL srv stats wc---- | Valkey with room for the bodies of every transaction.-valkey :: String -> ContainerSpec-valkey image =- let spec = redisContainer image- in spec- { options = ["--memory", "1g", "--entrypoint", valkeyBin],- command = map (\a -> if a == "128mb" then "768mb" else a) spec.command- }---- | The regtest node, with a larger chain and mempool than in the tests.-nodeSpec :: Opts -> String -> ContainerSpec-nodeSpec o image =- let spec = nodeContainer BitcoinCore image- in spec- { options = map grow spec.options,- command = map bigger spec.command <> ["-rpcworkqueue=64", "-rpcthreads=8"]- }- where- grow "1g" = "3g"- grow a- | "/chain:" `T.isPrefixOf` T.pack a = "/chain:rw,size=" <> show o.chainMb <> "m,mode=1777"- | otherwise = a- bigger "-dbcache=16" = "-dbcache=128"- bigger "-maxmempool=16" = "-maxmempool=128"- bigger a = a---- | A container that runs wrk against the server, on the network of the--- host, with the output directory, where the scripts of wrk are, mounted.-wrkSpec :: String -> FilePath -> ContainerSpec-wrkSpec image dir =- ContainerSpec- { name = "wrk",- image = image,- options =- [ "--network",- "host",- "--memory",- "512m",- "--security-opt",- "label=disable",- "--volume",- dir <> ":/stress:ro",- "--entrypoint",- "sleep"- ],- command = ["infinity"],- ports = []- }------------------------------------------------------------------------------------ The server----------------------------------------------------------------------------------data Server = Server- { url :: !String,- pid :: !Int- }---- | Run the haskoin-store server while an action runs.-withServer :: Opts -> FilePath -> FilePath -> String -> Int -> Int -> (Server -> IO a) -> IO a-withServer o outDir dir redisURL p2p statsPort act =- withFile (outDir <> "/server.log") WriteMode $ \logH -> do- -- another program could take the port before the server binds it, but- -- the server then fails to start, which is reported- port <- freePort- let gap = o.addresses + 20- args =- [ "--dir",- dir,- "--host",- "127.0.0.1",- "--port",- show port,- "--net",- "btcreg",- "--peer",- "127.0.0.1:" <> show p2p,- "--max-peers",- "1",- "--redis",- "--redis-url",- redisURL,- "--xpub-gap",- show gap,- "--xpub-gap-init",- show gap,- "--statsd",- "--statsd-host",- "127.0.0.1",- "--statsd-port",- show statsPort,- "--statsd-prefix",- "stress",- "--statsd-aggregates",- "--ticker-url",- "http://127.0.0.1:9/",- "--price-history-url",- "http://127.0.0.1:9/",- "+RTS"- ]- <> words o.serverRts- <> ["-RTS"]- cp = (proc o.server args) {std_out = UseHandle logH, std_err = UseHandle logH}- bracket (createProcess cp) stop $ \(_, _, _, ph) -> do- pid <- maybe (fail "The server did not start") (return . fromIntegral) =<< getPid ph- writeFile (outDir <> "/server.pid") (show pid)- let srv = Server {url = "http://127.0.0.1:" <> show port, pid = pid}- waitFor 60 "the server to answer" $ do- getProcessExitCode ph >>= \case- Just c -> fail ("The server exited with " <> show c <> "; see server.log")- Nothing -> either (const False) (const True) <$> tryAny (getJSON srv "/health")- act srv- where- stop (_, _, _, ph) = terminateProcess ph >> void (waitForProcess ph)---- | A free TCP port of the loopback interface.-freePort :: IO Int-freePort =- bracket (Socket.socket Socket.AF_INET Socket.Stream Socket.defaultProtocol) Socket.close $ \s -> do- Socket.bind s (Socket.SockAddrInet 0 (Socket.tupleToHostAddress (127, 0, 0, 1)))- fromIntegral <$> Socket.socketPort s---- | JSON answer of the server.-getJSON :: Server -> String -> IO Value-getJSON srv path = do- let o = Wreq.defaults & Wreq.checkResponse ?~ (\_ _ -> return ())- r <- Wreq.getWith o (srv.url <> path)- either fail return (eitherDecode (r ^. Wreq.responseBody))------------------------------------------------------------------------------------ Wallets------------------------------------------------------------------------------------ | An address and its key.-data Key = Key- { script :: !ScriptOutput,- secret :: !SecKey- }---- | A wallet: its xpub, and its receiving and change addresses.-data Wallet = Wallet- { xpub :: !Text,- receive :: !(Seq Key),- change :: !(Seq Key)- }---- | A coin that a wallet can spend.-data Coin = Coin- { outpoint :: !OutPoint,- value :: !Word64,- wallet :: !Int,- key :: !Key- }---- | The wallets, derived from a fixed seed, and the key of the miner.-makeWallets :: Opts -> ([Wallet], Key)-makeWallets o = ([wallet (fromIntegral i) | i <- [0 .. o.wallets - 1]], keyOf (hardSubKey root 9999) 0 0)- where- root = makeXPrvKey (C8.pack ("haskoin-store-stress-" <> show o.seed))- wallet i =- let w = hardSubKey root i- in Wallet- { xpub = xPubExport btcRegTest (deriveXPubKey w),- receive = Seq.fromList [keyOf w 0 j | j <- [0 .. fromIntegral o.addresses - 1]],- change = Seq.fromList [keyOf w 1 j | j <- [0 .. fromIntegral o.addresses - 1]]- }- keyOf w c j =- let k = prvSubKey (prvSubKey w c) j- a = xPubAddr (deriveXPubKey k)- in Key {script = fromMaybe (error "no script") (addressToOutput a), secret = k.key}---- | Address of a key, as text.-keyAddress :: Key -> Text-keyAddress k = case k.script of- PayPKHash h -> fromMaybe (error "no address") (addrToText btcRegTest (PubKeyAddress h))- _ -> error "not a key hash"---- | Sign a transaction that spends coins.-sign :: Tx -> [Coin] -> Tx-sign tx coins =- either error id $- signTx- btcRegTest- tx- [SigInput c.key.script c.value c.outpoint sigHashAll Nothing | c <- coins]- [c.key.secret | c <- coins]---- | A payment from a coin to another wallet, with change back to the wallet--- of the coin. Gives the transaction and its new coins.-pay :: IOGenM StdGen -> [Wallet] -> Coin -> IO (Tx, [Coin])-pay g ws c = do- w' <- (\i -> (c.wallet + i) `mod` length ws) <$> uniformRM (1, max 1 (length ws - 1)) g- let to = ws !! w'- from = ws !! c.wallet- r <- (`Seq.index` 0) . flip Seq.drop to.receive <$> uniformRM (0, Seq.length to.receive - 1) g- ch <- (`Seq.index` 0) . flip Seq.drop from.change <$> uniformRM (0, Seq.length from.change - 1) g- amount <- uniformRM (10000, max 10000 (c.value `div` 2)) g- -- every output is worth at least 10000, so a coin always pays the fee- let fee = 1000- outs- | c.value >= amount + fee + 10000 = [(r.script, amount), (ch.script, c.value - amount - fee)]- | otherwise = [(r.script, c.value - fee)]- tx = sign (buildTx [c.outpoint] outs) [c]- h = txHash tx- owners = [(w', r), (c.wallet, ch)]- coins = [Coin (OutPoint h i) v w k | (i, (_, v), (w, k)) <- zip3 [0 ..] outs owners]- return (tx, coins)------------------------------------------------------------------------------------ The node------------------------------------------------------------------------------------ | Send transactions to the node in one call. Gives the error of each--- transaction that the node refused.-sendRaw :: Node -> [Tx] -> IO [Maybe Value]-sendRaw node txs = do- let o =- Wreq.defaults- & Wreq.auth ?~ Wreq.basicAuth "test" "test"- & Wreq.checkResponse ?~ (\_ _ -> return ())- & Wreq.header "Content-Type" .~ ["application/json"]- body =- A.toJSON- [ object- [ "jsonrpc" .= ("1.0" :: Text),- "id" .= i,- "method" .= ("sendrawtransaction" :: Text),- "params" .= [encodeHex (runPutS (serialize tx))]- ]- | (i, tx) <- zip [0 :: Int ..] txs- ]- r <- Wreq.postWith o node.url body- case eitherDecode (r ^. Wreq.responseBody) of- Right (Array xs) ->- return- [ case field "error" x of- Just Null -> Nothing- Nothing -> Nothing- e -> e- | x <- toList xs- ]- Right v -> fail ("Unexpected answer to a batch: " <> take 400 (show v))- Left e -> fail ("Bad answer to a batch: " <> e)---- | The coinbase output of a block.-coinbaseOf :: Node -> Text -> IO (OutPoint, Word64)-coinbaseOf node h = do- b <- rpc node "getblock" [String h, Number 2]- case field "tx" b of- Just (Array ts) | Just t <- Seq.lookup 0 (Seq.fromList (toList ts)) -> do- txid <- maybe (fail "no txid") return (textField "txid" t >>= hexToTxHash)- v <- case field "vout" t of- Just (Array os) | o : _ <- toList os, Just v <- field "value" o -> return (sats v)- _ -> fail "no coinbase output"- return (OutPoint txid 0, v)- _ -> fail ("No transactions in block " <> T.unpack h)---- | Height of the best block of the node.-nodeHeight :: Node -> IO Int-nodeHeight node =- rpc node "getblockcount" [] >>= \case- Number n -> return (round n)- v -> fail ("Unexpected block count: " <> show v)------------------------------------------------------------------------------------ The stress test----------------------------------------------------------------------------------stress :: Opts -> Node -> String -> Server -> IORef Stats -> Container -> IO ()-stress o node redisURL srv stats wc = do- g <- newIOGenM (mkStdGen o.seed)- let (ws, miner) = makeWallets o- minerAddr = keyAddress miner- perFanOut = 1000- fanOuts = max 1 ((o.blockTxs * 3 `div` 2 + perFanOut - 1) `div` perFanOut)- writeJSON (o.out <> "/config.json") $- object- [ "txs" .= o.txs,- "wallets" .= o.wallets,- "addresses" .= o.addresses,- "block_txs" .= o.blockTxs,- "batch" .= o.batch,- "connections" .= o.connections,- "wrk_seconds" .= o.wrkSeconds,- "server_rts" .= o.serverRts,- "xpubs" .= map (.xpub) ws- ]- say ("Funding " <> show (length ws) <> " wallets with " <> show (fanOuts * perFanOut) <> " coins")- funded <- generateTo node fanOuts minerAddr- void $ generateTo node 100 minerAddr- coins0 <- fmap concat . forM funded $ \h -> do- (op, v) <- coinbaseOf node h- let n = perFanOut- each = (v - 200000) `div` fromIntegral n- -- every receiving address of every wallet, in turn- targets = take n (cycle [(wi, Seq.index w.receive j) | j <- [0 .. o.addresses - 1], (wi, w) <- zip [0 ..] ws])- tx = sign (buildTx [op] [(k.script, each) | (_, k) <- targets]) [Coin op v (-1) miner]- errs <- sendRaw node [tx]- forM_ errs $ mapM_ (\e -> fail ("The node refused a funding transaction: " <> show e))- return [Coin (OutPoint (txHash tx) i) each wi k | (i, (wi, k)) <- zip [0 ..] targets]- void $ generateTo node 1 minerAddr- say "Waiting for the server to sync and cache the xpubs"- waitSynced node srv- forM_ ws $ \w -> getJSON srv ("/xpub/" <> T.unpack w.xpub <> "/balances")- waitFor 120 "the xpubs to be cached" $ do- conn <- Redis.checkedConnect (either error id (Redis.parseConnectInfo redisURL))- n <- either (const 0) id <$> Redis.runRedis conn (Redis.dbsize)- Redis.disconnect conn- return (n > 0)- writeScripts o- slowlogStart redisURL- stop <- newTVarIO False- start <- getCurrentTime- withFile (o.out <> "/samples.jsonl") WriteMode $ \sh ->- withFile (o.out <> "/wrk.jsonl") WriteMode $ \wh -> do- wlock <- newMVar wh- withAsync (sampler start node srv redisURL stats stop sh) $ \_ ->- withAsync (wrkAll o ws srv wc start stop wlock) $ \_ -> do- (sent, refused) <- withFile (o.out <> "/rounds.jsonl") WriteMode $ \rh ->- rounds o g node ws minerAddr start rh (Seq.fromList coins0)- say "Waiting for the server to catch up"- t0 <- getCurrentTime- waitSynced node srv- catchUp <- (`diffUTCTime` t0) <$> getCurrentTime- atomically (writeTVar stop True)- -- let the last runs of wrk and the sampler finish- threadDelay ((o.wrkSeconds + 3) * 1000000)- say "Comparing cached answers with the database"- checks <- forM ws (consistent srv)- slow <- slowlog redisURL- end <- getCurrentTime- height <- nodeHeight node- writeJSON (o.out <> "/summary.json") $- object- [ "transactions_sent" .= sent,- "transactions_refused" .= refused,- "height" .= height,- "seconds" .= (realToFrac (end `diffUTCTime` start) :: Double),- "catch_up_seconds" .= (realToFrac catchUp :: Double),- "consistency" .= concat checks,- "redis_slow" .= slow- ]- say ("Done: " <> show sent <> " transactions, " <> show refused <> " refused")---- | Make the transactions, a block at a time. Gives how many were sent and--- how many the node refused.-rounds :: Opts -> IOGenM StdGen -> Node -> [Wallet] -> Text -> UTCTime -> Handle -> Seq Coin -> IO (Int, Int)-rounds o g node ws minerAddr start rh = go 0 (0, 0)- where- total = o.txs- go :: Int -> (Int, Int) -> Seq Coin -> IO (Int, Int)- go r (sent, refused) pool- | sent >= total = return (sent, refused)- | otherwise = do- let n = minimum [o.blockTxs, total - sent, Seq.length pool]- (spend, rest) = Seq.splitAt n pool- when (n == 0) $ fail "No coins left to spend"- t0 <- getCurrentTime- made <- mapM (pay g ws) (toList spend)- -- sign now, not when the transactions are sent- mapM_ (evaluate . force . fst) made- t1 <- getCurrentTime- results <- fmap concat . forM (chunks o.batch (zip (toList spend) made)) $ \chunk -> do- errs <- sendRaw node [tx | (_, (tx, _)) <- chunk]- return (zip chunk errs)- t2 <- getCurrentTime- let accepted = [new | ((_, (_, new)), Nothing) <- results]- back = [c | ((c, _), Just _) <- results]- errors = [e | (_, Just e) <- results]- case errors of- e : _ ->- say $- unwords- [ "The node refused",- show (length errors),- "transactions, such as",- take 300 (show e)- ]- [] -> return ()- void $ generateTo node 1 minerAddr- t3 <- getCurrentTime- height <- nodeHeight node- let sent' = sent + length accepted- appendLine rh $- object- [ "round" .= r,- "t" .= secs start t3,- "height" .= height,- "sent" .= sent',- "refused" .= length errors,- "sign_ms" .= ms t0 t1,- "submit_ms" .= ms t1 t2,- "mine_ms" .= ms t2 t3- ]- when (r `mod` 10 == 0) $- say ("Round " <> show r <> ": " <> show sent' <> " transactions, height " <> show height)- go (r + 1) (sent', refused + length errors) (rest <> Seq.fromList (back <> concat accepted))- chunks k xs = case splitAt k xs of- ([], _) -> []- (a, b) -> a : chunks k b---- | Wait until the server has the best block of the node.-waitSynced :: Node -> Server -> IO ()-waitSynced node srv =- waitFor 1800 "the server to sync" $ do- h <- nodeHeight node- serverHeight srv <&&> (== h)- where- a <&&> f = maybe False f <$> a---- | Height of the best block of the server.-serverHeight :: Server -> IO (Maybe Int)-serverHeight srv =- tryAny (getJSON srv "/block/best?notx=true") <&> \case- Right v | Just (Number n) <- field "height" v -> Just (round n)- _ -> Nothing- where- (<&>) = flip fmap---- | Compare the cached answers about an xpub with those of the database,--- for up to a minute, as the cache may still be catching up.-consistent :: Server -> Wallet -> IO [Value]-consistent srv w =- forM paths $ \p -> do- ok <- tryWait (60 :: Int) p- return (object ["xpub" .= w.xpub, "path" .= p, "same" .= ok])- where- paths = ["", "/balances", "/transactions?limit=100", "/unspent?limit=100"]- base = "/xpub/" <> T.unpack w.xpub- sep p = if '?' `elem` p then "&" else "?"- tryWait n p = do- a <- getJSON srv (base <> p)- b <- getJSON srv (base <> p <> sep p <> "nocache=true")- if a == b || n <= 0- then return (a == b)- else threadDelay 1000000 >> tryWait (n - 1) p------------------------------------------------------------------------------------ wrk------------------------------------------------------------------------------------ | Endpoints that wrk asks about, each rotating over every xpub.-endpoints :: [(String, Text -> Text)]-endpoints =- [ ("summary", \x -> "/xpub/" <> x),- ("balances", \x -> "/xpub/" <> x <> "/balances"),- ("transactions", \x -> "/xpub/" <> x <> "/transactions?limit=100"),- ("unspent", \x -> "/xpub/" <> x <> "/unspent?limit=100"),- ("multiaddr", \x -> "/blockchain/multiaddr?active=" <> x <> "&n=50")- ]---- | The script of wrk ("stress/wrk.lua"), copied into the output directory--- where the container of wrk sees it. It asks for the paths it is given in--- turn and prints its results as JSON. It is embedded when the program is--- compiled, so the program finds it wherever it runs.-writeScripts :: Opts -> IO ()-writeScripts o = BS.writeFile (o.out <> "/wrk.lua") $(makeRelativeToProject "stress/wrk.lua" >>= embedFile)---- | Run wrk on every endpoint at once, over and over, until told to stop.-wrkAll :: Opts -> [Wallet] -> Server -> Container -> UTCTime -> TVar Bool -> MVar Handle -> IO ()-wrkAll o ws srv wc start stop lock = go endpoints- where- go [] = return ()- go ((name, path) : rest) = withAsync (loop name (map (path . (.xpub)) ws)) $ \_ -> go rest >> waitStop- waitStop = readTVarIO stop >>= \s -> unless s (threadDelay 500000 >> waitStop)- loop name paths = do- s <- readTVarIO stop- unless s $ do- t <- getCurrentTime- let args =- [ "exec",- wc.cid,- wrkBin,- "-t1",- "-c" <> show o.connections,- "-d" <> show o.wrkSeconds <> "s",- "--timeout",- "10s",- "-s",- "/stress/wrk.lua",- srv.url,- "--"- ]- <> map T.unpack paths- (code, out, err) <- readProcessWithExitCode "podman" args ""- let result = case [l | l <- lines out, take 1 l == "{"] of- l : _ | Just (Object r) <- A.decode (BL.fromStrict (C8.pack l)) -> Object r- _ -> object ["failed" .= True, "exit" .= show code, "stderr" .= take 300 err]- withMVar lock $ \h ->- appendLine h (merge (object ["t" .= secs start t, "endpoint" .= name]) result)- loop name paths- merge (Object a) (Object b) = Object (a <> b)- merge a _ = a------------------------------------------------------------------------------------ Sampling------------------------------------------------------------------------------------ | Write a sample every second until told to stop: the heights of the node--- and the server, the CPU and memory of the server, the memory of Valkey, and--- the StatsD metrics of the server since the last sample.-sampler :: UTCTime -> Node -> Server -> String -> IORef Stats -> TVar Bool -> Handle -> IO ()-sampler start node srv redisURL stats stop h = do- hz <- clockTicks- conn <- Redis.checkedConnect (either error id (Redis.parseConnectInfo redisURL))- let go prevTicks prevTime = do- s <- readTVarIO stop- unless s $ do- threadDelay 1000000- now <- getCurrentTime- ticks <- cpuTicks srv.pid- rss <- rssKb srv.pid- (heap, lazyFree) <- ghcHeapKb srv.pid- nh <- either (const Nothing) Just <$> tryAny (nodeHeight node)- sh <- serverHeight srv- vm <- valkeyMemory conn- m <- drainStats stats- let seconds = realToFrac (now `diffUTCTime` prevTime)- cpu = fromIntegral (max 0 (ticks - prevTicks)) / fromIntegral hz / seconds * 100 :: Double- appendLine h $- object- [ "t" .= secs start now,- "node_height" .= nh,- "server_height" .= sh,- "cpu_percent" .= cpu,- "rss_mb" .= (fromIntegral rss / 1024 :: Double),- "heap_mb" .= (fromIntegral heap / 1024 :: Double),- "heap_lazy_free_mb" .= (fromIntegral lazyFree / 1024 :: Double),- "valkey_mb" .= ((/ (1024 * 1024)) . fromIntegral <$> vm :: Maybe Double),- "counters" .= m.counters,- "gauges" .= m.gauges,- "timings" .= HashMap.map timingStats m.timings- ]- go ticks now- t <- getCurrentTime- ticks0 <- cpuTicks srv.pid- go ticks0 t- where- timingStats vs =- let s = sort vs- n = length s- at p = s !! min (n - 1) (floor (p * fromIntegral n :: Double))- in object ["n" .= n, "p50" .= at 0.5, "p95" .= at 0.95, "max" .= last s, "sum" .= sum s]---- | Ticks of the clock of the kernel per second.-clockTicks :: IO Int-clockTicks = do- r <- tryAny (readProcessWithExitCode "getconf" ["CLK_TCK"] "")- return $ case r of- Right (ExitSuccess, o, _) | Just n <- readMaybe o -> n- _ -> 100---- | CPU time of a process, in ticks of the clock of the kernel.-cpuTicks :: Int -> IO Int-cpuTicks pid = do- r <- tryAny (readFile ("/proc/" <> show pid <> "/stat"))- return $ case r of- -- the fields after the name, which is in parentheses and may have spaces- Right s- | fs <- words (drop 2 (dropWhile (/= ')') s)),- length fs > 13 ->- sum (mapMaybe readMaybe [fs !! 11, fs !! 12])- _ -> 0---- | Resident memory of a process, in KiB.-rssKb :: Int -> IO Int-rssKb pid = do- r <- tryAny (readFile ("/proc/" <> show pid <> "/status"))- return $ case r of- Right s | (v : _) <- [w | l <- lines s, ("VmRSS:" : w : _) <- [words l]], Just n <- readMaybe v -> n- _ -> 0---- | Resident memory of the GHC heap of a process, in KiB: what is resident of--- the address range that the runtime reserves for its heap, 1 TiB from--- @0x4200000000@. The rest of the resident memory is outside the heap:--- RocksDB, buffers of C libraries, stacks and the program itself. A heap that--- grows while its live data does not is memory the runtime took for a peak--- and kept, which a heap profile, made of live data, does not show.------ Also gives how much of it the runtime has given back to the system with--- @MADV_FREE@ (@LazyFree@): free for the kernel to take when it needs memory,--- but counted as resident until it does.-ghcHeapKb :: Int -> IO (Int, Int)-ghcHeapKb pid = do- r <- tryAny (readFile ("/proc/" <> show pid <> "/smaps"))- return $ case r of- Left _ -> (0, 0)- Right s -> (go "Rss:" False 0 (lines s), go "LazyFree:" False 0 (lines s))- where- base = 0x4200000000 :: Integer- top = base + 2 ^ (40 :: Int)- go _ _ n [] = n- go key inHeap n (l : ls)- | (range : _) <- words l,- (a, '-' : _) <- break (== '-') range,- [(start, "")] <- readHex a =- go key (start >= base && start < top) n ls- | inHeap, (k : v : _) <- words l, k == key, Just x <- readMaybe v = go key inHeap (n + x) ls- | otherwise = go key inHeap n ls---- | Memory used by Valkey, in bytes.-valkeyMemory :: Redis.Connection -> IO (Maybe Integer)-valkeyMemory conn =- Redis.runRedis conn Redis.info <&&> \case- Right s ->- case [v | l <- C8.lines s, Just v <- [C8.stripPrefix "used_memory:" l]] of- v : _ -> fst <$> C8.readInteger v- [] -> Nothing- Left _ -> Nothing- where- a <&&> f = f <$> a---- | Make Valkey log every command that takes 10 ms or more, keeping all of--- them: a long script blocks every client of Redis, every server included.-slowlogStart :: String -> IO ()-slowlogStart url = withRedis url $ \conn -> Redis.runRedis conn $ do- send ["CONFIG", "SET", "slowlog-log-slower-than", "10000"]- send ["CONFIG", "SET", "slowlog-max-len", "100000"]- send ["SLOWLOG", "RESET"]- where- send args = void (Redis.sendRequest args :: Redis.Redis (Either Redis.Reply Redis.Reply))---- | The commands that took 10 ms or more: when, how long in milliseconds,--- the command, and how many arguments it had. For a script of the cache,--- also its name and, for a commit, whether it was of a full reload.------ The slow log keeps the first 32 arguments of a command, each cut to 128--- bytes, and ends the list with how many it left out. That is enough: the--- first argument of an EVAL is the script, whose first line names it (as--- @commitScript@), and the number of arguments of a commit grows with the--- size of the change.-slowlog :: String -> IO [Value]-slowlog url = withRedis url $ \conn ->- Redis.runRedis conn (Redis.sendRequest ["SLOWLOG", "GET", "100000"]) <&&> \case- Right (Redis.MultiBulk (Just entries)) -> mapMaybe entry entries- _ -> []- where- a <&&> f = f <$> a- entry (Redis.MultiBulk (Just (_ : Redis.Integer ts : Redis.Integer us : Redis.MultiBulk (Just reply) : _))) =- let args = [decodeUtf8 b | Redis.Bulk (Just b) <- reply]- script = case args of- _ : body : _ -> take 1 [w | w <- T.words body, "Script" `T.isSuffixOf` w]- _ -> []- in Just . object $- [ "time" .= ts,- "ms" .= (fromIntegral us / 1000 :: Double),- "command" .= T.toUpper (T.take 24 (mconcat (take 1 args))),- "args" .= argCount args- ]- <> ["script" .= s | s <- script]- -- ARGV[8] of a commit is the time of a full reload, or empty- <> ["full" .= not (T.null f) | script == ["commitScript"], f <- take 1 (drop 10 args)]- entry _ = Nothing- -- the arguments kept, the last of which may say how many were left out:- -- "... (2048 more arguments)"- argCount args = case reverse args of- l : kept- | Just rest <- T.stripPrefix "... (" l,- [(n, _)] <- reads (T.unpack rest) ->- length kept + n :: Int- _ -> length args--withRedis :: String -> (Redis.Connection -> IO a) -> IO a-withRedis url = bracket (Redis.checkedConnect (either error id (Redis.parseConnectInfo url))) Redis.disconnect------------------------------------------------------------------------------------ StatsD------------------------------------------------------------------------------------ | Metrics received since they were last drained: counters added up,--- gauges at their last value, and every timing.-data Stats = Stats- { counters :: !(HashMap Text Double),- gauges :: !(HashMap Text Double),- timings :: !(HashMap Text [Double])- }---- | Receive the StatsD metrics of the server on a UDP port of the loopback--- interface while an action runs.-withStatsd :: (Int -> IORef Stats -> IO a) -> IO a-withStatsd act =- bracket (Socket.socket Socket.AF_INET Socket.Datagram Socket.defaultProtocol) Socket.close $ \s -> do- Socket.bind s (Socket.SockAddrInet 0 (Socket.tupleToHostAddress (127, 0, 0, 1)))- port <- fromIntegral <$> Socket.socketPort s- ref <- newIORef (Stats HashMap.empty HashMap.empty HashMap.empty)- let receive = do- bs <- SocketB.recv s 65535- forM_ (C8.lines bs) $ \l -> forM_ (parseLine l) $ \m ->- atomicModifyIORef' ref (\st -> (add m st, ()))- receive- withAsync receive $ \_ -> act port ref- where- add (k, v, t) st = case t of- 'c' -> st {counters = HashMap.insertWith (+) k v st.counters}- 'g' -> st {gauges = HashMap.insert k v st.gauges}- _ -> st {timings = HashMap.insertWith (<>) k [v] st.timings}---- | A StatsD line, @key:value|type@ with an optional @|\@rate@: the key--- without the prefix of the server, the value (scaled up by the rate for a--- sampled counter), and the first letter of the type.-parseLine :: C8.ByteString -> Maybe (Text, Double, Char)-parseLine l = do- let (k, rest) = C8.break (== ':') l- (v, t, rate) <- case C8.split '|' (C8.drop 1 rest) of- [v, t] -> Just (v, t, 1)- [v, t, r] | Just ('@', x) <- C8.uncons r -> (v,t,) <$> readMaybe (C8.unpack x)- _ -> Nothing- x <- readMaybe (C8.unpack (C8.dropWhile (== '+') v))- c <- fst <$> C8.uncons t- let key = T.replace "stress.btcreg." "" (decodeUtf8 k)- return (key, if c == 'c' then x / rate else x, c)---- | Take the counters and timings received since the last call; gauges--- keep their values.-drainStats :: IORef Stats -> IO Stats-drainStats ref = atomicModifyIORef' ref $ \st -> (st {counters = HashMap.empty, timings = HashMap.empty}, st)------------------------------------------------------------------------------------ Helpers----------------------------------------------------------------------------------say :: String -> IO ()-say s = hPutStrLn stderr ("stress: " <> s) >> hFlush stderr---- | Try a check every quarter of a second, for up to some seconds.-waitFor :: Int -> String -> IO Bool -> IO ()-waitFor secs' what check = go (secs' * 4)- where- go n =- check >>= \case- True -> return ()- False- | n <= 0 -> fail ("Timed out waiting for " <> what)- | otherwise -> threadDelay 250000 >> go (n - 1)--secs :: UTCTime -> UTCTime -> Double-secs a b = realToFrac (b `diffUTCTime` a)--ms :: UTCTime -> UTCTime -> Double-ms a b = 1000 * secs a b--appendLine :: Handle -> Value -> IO ()-appendLine h v = BL.hPut h (encode v <> "\n") >> hFlush h--writeJSON :: FilePath -> Value -> IO ()-writeJSON path = BL.writeFile path . encode
− stress/wrk.lua
@@ -1,32 +0,0 @@--- The script of wrk for the stress test (stress/Main.hs). It asks for the--- paths given after "--" on the command line in turn, and when wrk is done it--- prints one line of JSON with the results, latencies in microseconds:------ wrk -s wrk.lua http://server -- /xpub/a /xpub/b-local paths = {}-local i = 0--init = function(args)- paths = args-end--request = function()- i = i % #paths + 1- return wrk.format("GET", paths[i])-end--done = function(summary, latency, requests)- local e = summary.errors- local fields = {- '"requests":' .. summary.requests,- '"duration_us":' .. summary.duration,- '"errors":' .. (e.connect + e.read + e.write + e.timeout),- '"non2xx":' .. e.status,- '"p50_us":' .. latency:percentile(50),- '"p90_us":' .. latency:percentile(90),- '"p99_us":' .. latency:percentile(99),- '"max_us":' .. latency.max,- '"mean_us":' .. string.format("%.1f", latency.mean),- }- io.write("{" .. table.concat(fields, ",") .. "}\n")-end
swagger.yaml view
@@ -11,7 +11,7 @@ - url: './' info:- version: &version 1.16.0+ version: &version 1.16.6 title: Haskoin contact: name: JP Rupp@@ -131,6 +131,7 @@ | 404 | `not-found-or-invalid-arg` | The thing does not exist, or a value in the path is not valid. | | 409 | `multiple-tx-index` | A short transaction number matches several transactions. | | 410 | `replaced` | A transaction asked for by its id was replaced by one that a block confirmed; `replaced-by` and `height` say which and where. |+ | 413 | `request-too-large` | The body of `POST /transactions` is longer than a transaction can be. | | 429 | `server-busy` | A wallet asked for is not in the cache yet, and the server cannot have it there soon. Try again after the seconds of its `Retry-After` header. Any endpoint of wallets may answer it; `/events` only for an `xpub` to watch. | | 429 | `server-busy` | The server has more long requests (lists, whole blocks, wallets) than it can answer now. Try again after a second, or ask another server. | | 500 | `server-error` | Something went wrong on the server; it is logged there. |@@ -148,8 +149,8 @@ A 409 comes only from the Blockchain.info endpoints, which can name a transaction by a short number; the error lists the matching ids in `txids`. - The codes `server-timeout` and `request-too-large` are part of the format, for- clients that decode it, but this version of the server does not send them.+ The code `server-timeout` is part of the format, for clients that decode+ it, but this version of the server does not send it. ## Unconfirmed transactions @@ -355,8 +356,8 @@ body: Redis holds the mempool and the cache of extended public keys that every server of the fleet answers from. - The checks are run in the background every 30 seconds (unless the operator- changed it), and this endpoint returns the latest result. It does no work of+ The checks are run in the background every second (unless the operator+ changed it) and at each new block, and this endpoint returns the latest result. It does no work of its own, so it is safe to call as often as a load balancer likes. operationId: healthCheck responses:@@ -392,6 +393,13 @@ ok: true count: 20 min: 1+ redis:+ ok: true+ database:+ ok: true+ writes: flowing+ pending-compaction-bytes: 0+ delayed-write-rate: 0 time: '2021-06-14 10:21:43 UTC' '503': description: unavailable@@ -425,6 +433,13 @@ ok: false count: 0 min: 1+ redis:+ ok: true+ database:+ ok: true+ writes: flowing+ pending-compaction-bytes: 0+ delayed-write-rate: 0 time: '2021-06-14 10:21:43 UTC' '/address/{address}/transactions':@@ -2072,8 +2087,8 @@ $ref: '#/components/schemas/Error' '413': description: |- The body is longer than a transaction of the network can be in- hexadecimal: 8,000,000 characters on networks with SegWit and+ `request-too-large`: the body is longer than a transaction of the+ network can be in hexadecimal: 8,000,000 characters on networks with SegWit and 2,000,000 on Bitcoin Cash, and some white space. content: application/json:@@ -6481,6 +6496,8 @@ - last-tx - pending-txs - peers+ - redis+ - database - net - version - ok@@ -6504,7 +6521,7 @@ - $ref: '#/components/schemas/TimeHealth' pending-txs: description: |- Transactions received but not imported yet.+ Transactions requested from peers and not received yet. allOf: - $ref: '#/components/schemas/MaxHealth' peers:@@ -6515,9 +6532,10 @@ redis: type: object description: |- Whether the Redis cache answers. Only for a server that has one. It does- not count towards `ok`: a server without its Redis still answers every- request, from its own database, and `/health` gives 203.+ Whether the Redis of the fleet answers. It does not count towards `ok`,+ but a server without its Redis answers `/health` with 503 and the header+ `Haskoin-Degraded: redis`, as it has no mempool and no cache of+ extended public keys to answer from. required: - ok properties:@@ -6597,6 +6615,11 @@ type: string description: | Transaction id.+ message:+ type: string+ description: |+ For a transaction that was handed over (202), to whom: a full node+ over RPC, or how many peers took it. BinfoTxIdHash: type: string
test/Haskoin/Store/BlockStore/PendingSpec.hs view
@@ -1,10 +1,7 @@--- | Mempool transactions on their way into the database.+-- | The transactions that the block store requested from peers. module Haskoin.Store.BlockStore.PendingSpec (spec) where -import Data.ByteString (ByteString)-import Data.HashMap.Strict qualified as HashMap-import Data.HashSet qualified as HashSet-import Data.List (elemIndex)+import Data.ByteString.Char8 qualified as C8 import Data.Time.Calendar (fromGregorian) import Data.Time.Clock (UTCTime (..), addUTCTime) import Haskoin@@ -16,79 +13,29 @@ t0 :: UTCTime t0 = UTCTime (fromGregorian 2026 11 9) 0 --- | Transactions where each one spends an output from outside, or an output--- of an earlier one: parents come first.-family :: Gen [Tx]-family = do- n <- choose (1, 12)- go n []- where- go 0 acc = return (reverse acc)- go k acc = do- parents <- sublistOf (map txHash acc)- let outside = OutPoint (TxHash (sha256 ("outside" :: ByteString))) (fromIntegral k)- ins = [TxIn op "" maxBound | op <- outside : [OutPoint h 0 | h <- parents]]- tx = Tx 2 ins [TxOut 1000 "", TxOut 1000 ""] [] (fromIntegral k)- go (k - 1 :: Int) (tx : acc)---- | The parents of a transaction among others.-parentsIn :: [Tx] -> Tx -> [TxHash]-parentsIn txs tx = [op.hash | i <- tx.inputs, let op = i.outpoint, op.hash `elem` map txHash txs]---- | Receive transactions, each missing those of its parents that are not--- received before it.-receiveAll :: [Tx] -> Pending -> Pending-receiveAll txs p0 = foldl (\p tx -> received t0 tx (HashSet.fromList (parentsIn txs tx)) p) p0 txs+-- | Distinct transaction ids.+hashes :: Gen [TxHash]+hashes = do+ n <- choose (1, 12 :: Int)+ return [TxHash (sha256 (C8.pack (show i))) | i <- [1 .. n]] spec :: Spec-spec = describe "Pending transactions" $ do- prop "takes only transactions with no missing parents, parents first" $- forAll family $ \txs -> forAll (sublistOf txs) $ \done -> forAll (choose (0, 20)) $ \n ->- let p = foldr (imported . txHash) (receiveAll txs empty) done- (taken, rest) = takeReady n p- order = map (txHash . (.tx)) taken- in conjoin- [ property (length taken <= n),- property (all (null . (.deps)) taken),- conjoin- [ counterexample "a child before its parent" $- maybe True (> i) (elemIndex h order)- | (i, t) <- zip [0 ..] taken,- h <- [txHash c | c <- txs, txHash t.tx `elem` parentsIn txs c]- ],- queued rest === queued p - length taken- ]-- prop "makes a child ready once all its parents are in the database" $- forAll family $ \txs ->- let p = foldr (imported . txHash) (receiveAll txs empty) txs- in fst (takeReady (length txs) p) `shouldMatchList'` txs-- prop "does not request again what is requested or received" $- forAll family $ \txs ->- let hs = map txHash txs- (asked, got) = splitAt (length hs `div` 2) txs- p = receiveAll got (requested t0 (map txHash asked) empty)+spec = describe "Pending requests" $ do+ prop "does not request again what is requested" $+ forAll hashes $ \hs ->+ let p = requested t0 hs empty in all (`isPending` p) hs .&&. size p === length hs - prop "forgets requests after twenty seconds, and transactions after an hour" $- forAll family $ \txs ->- let hs = map txHash txs- asked = requested t0 hs empty- got = receiveAll txs empty+ prop "forgets a request once its transaction arrives" $+ forAll hashes $ \hs ->+ let (got, still) = splitAt (length hs `div` 2) hs+ p = foldr answered (requested t0 hs empty) got+ in not (any (`isPending` p) got) .&&. all (`isPending` p) still .&&. size p === length still++ prop "forgets requests after twenty seconds" $+ forAll hashes $ \hs ->+ let asked = requested t0 hs empty in conjoin [ property (all (`isPending` prune (addUTCTime 20 t0) asked) hs),- property (not (any (`isPending` prune (addUTCTime 21 t0) asked) hs)),- queued (prune (addUTCTime 3600 t0) got) === length txs,- queued (prune (addUTCTime 3601 t0) got) === 0+ property (not (any (`isPending` prune (addUTCTime 21 t0) asked) hs)) ]-- prop "drops what reached the database and updates what still waits" $- forAll family $ \txs ->- let p = receiveAll txs empty- (gone, still) = splitAt (length txs `div` 2) (map txHash txs)- p' = refresh (HashMap.fromList ([(h, Nothing) | h <- gone] <> [(h, Just HashSet.empty) | h <- still])) p- in queued p' === length still .&&. null (waiting p')- where- shouldMatchList' taken txs =- HashSet.fromList (map (txHash . (.tx)) taken) === HashSet.fromList (map txHash txs)
test/Haskoin/Store/Fleet/CacheSpec.hs view
@@ -31,6 +31,7 @@ BlockHash, BlockHeader (..), BlockNode (..),+ OutPoint (..), Tx (..), TxHash (..), TxIn (..),@@ -106,9 +107,12 @@ it "never answers an entry that a move of the tip is writing" (`withBed` markedTest) it "keeps the cache across a reorganisation, as the chain and the pool give it" (`withBed` reorgTest) it "fills the pool again after Redis lost its data" (`withBed` refillTest)+ it "fills the pool again after it started again, once after Redis lost its data" (`withBed` restartTest) it "keeps what an abandoned block gave back when the chain moves under a move of the tip" $ (`withBed` unsettledTest) it "fetches no transaction that another member received" (`withBed` seenTest)+ it "offers again a transaction that left the pool and is sent again" (`withBed` resendTest)+ it "lets go of an orphan whose parent did not come in twenty minutes" (`withBed` orphanTest) modifyMaxSuccess (const 10) $ it "keeps every cached entry equal to the chain at the tip and the pool" $ \r -> property $ \steps -> ioProperty (withBed r (\bed -> modelTest bed (take 12 steps)) >> return True)@@ -248,6 +252,61 @@ inPool bed tx `shouldReturn` True fetch bed.a [txHash tx, other] `shouldReturn` [other] fetch bed.b [txHash tx, other] `shouldReturn` [other]++-- | A transaction that a replacement pushed out of the pool, and that a+-- peer sends again, takes its place back: a member that kept a set of what+-- it had offered refused it as offered already.+resendTest :: Bed -> IO ()+resendTest bed = do+ coin : _ <- coins bed+ let first = spend [coin] [(script, 100000)]+ second = spend [coin] [(script, 200000)]+ offer bed first+ settle bed+ inPool bed first `shouldReturn` True+ offer bed second+ settle bed+ inPool bed second `shouldReturn` True+ inPool bed first `shouldReturn` False+ offer bed first+ settle bed+ inPool bed first `shouldReturn` True+ inPool bed second `shouldReturn` False++-- | Orphans wait for their parent for twenty minutes from when the member+-- heard of them, and are then let go.+orphanTest :: Bed -> IO ()+orphanTest bed = do+ coin : _ <- coins bed+ now <- round <$> getPOSIXTime+ let parent = spend [coin] [(script, 100000)]+ child v = Tx 1 [TxIn (OutPoint (txHash parent) 0) "" maxBound] [TxOut v script] [] 0+ old = child 1000+ young = child 2000+ memberReceived bed.a (now - 1201) old+ memberReceived bed.a now young+ settle bed+ held <- readTVarIO bed.a.waiting+ HashMap.member (txHash old) held `shouldBe` False+ HashMap.member (txHash young) held `shouldBe` True++-- | Each member counts a refill when the pool starts again, which the count+-- of restarts says; and one only, not two, when Redis loses its data with+-- that count in it.+restartTest :: Bed -> IO ()+restartTest bed = do+ settle bed+ readTVarIO bed.a.refills `shouldReturn` 0+ -- what a restart of the pool does to the count+ void (Redis.runRedis bed.conn (Redis.incr (bed.prefix <> "fleet:restarts")))+ settle bed+ readTVarIO bed.a.refills `shouldReturn` 1+ readTVarIO bed.b.refills `shouldReturn` 1+ lost <- Redis.runRedis bed.conn (Redis.keys (bed.prefix <> "*"))+ forM_ (either (const []) id lost) $ \k -> void (Redis.runRedis bed.conn (Redis.del (k :| [])))+ settle bed+ readTVarIO bed.a.refills `shouldReturn` 2+ readTVarIO bed.b.refills `shouldReturn` 2 refillTest :: Bed -> IO () refillTest bed = do
test/Haskoin/Store/Fleet/RedisSpec.hs view
@@ -108,6 +108,14 @@ fleetTrimSeen conn keys 40 `shouldReturn` 33 fleetTrimSeen conn keys 20 `shouldReturn` 20 drop 10 <$> fleetSeen conn keys (map block [100 .. 129]) `shouldReturn` replicate 20 True+ it "takes out a transaction that pays an output of value 0, as an anchor does" $ \(conn, fresh) -> do+ keys <- fresh+ fleetReport conn keys Nothing 60000 "a" [block 1] [] `shouldReturn` True+ fleetMove conn keys Nothing Nothing `shouldReturn` MovedTo (block 1)+ let anchor = (txOf 7 [out 0 0]) {entries = [PoolTouches "anchor", PoolPays "anchor" (out 7 0) 0 "script"]}+ poolOffer conn keys 4 (Just (block 1)) anchor [True] [] `shouldReturn` Taken+ poolWithdraw conn keys 4 [block 7] `shouldReturn` [block 7]+ poolBody conn keys (block 7) `shouldReturn` Nothing it "tells what happens to the pool and to the tip, in the order in which it happens" $ \(conn, fresh) -> do keys <- fresh fleetReport conn keys Nothing 60000 "a" [block 1] [] `shouldReturn` True@@ -150,19 +158,19 @@ poolSpends conn keys `shouldReturn` [] (Redis.runRedis conn (Redis.keys (keys.prefix <> "pool:*")) >>= either (fail . show) return) `shouldReturn` []- it "empties the pool for any store when the way is too long, and counts it" $ \(conn, fresh) -> do+ it "empties the pool and what the fleet saw for any store when the way is too long" $ \(conn, fresh) -> do keys <- fresh fleetReport conn keys (Just 0) 10 "a" [block 1] [] `shouldReturn` True fleetAdvance conn keys 4 (Just 0) Nothing (block 1) Nothing [] `shouldReturn` MovedTo (block 1) poolOffer conn keys 4 (Just (block 1)) (txOf 7 [out 0 0]) [True] [] `shouldReturn` Taken- before <- fleetRestarts conn keys+ fleetSeen conn keys [block 7] `shouldReturn` [True]+ fleetRestarts conn keys `shouldReturn` 0 -- two stores that both have a view of the tip, so both are live, -- at a block that neither will say the way to fleetReport conn keys (Just 0) 10 "a" [block 2] [block 2, block 1] `shouldReturn` True fleetReport conn keys (Just 0) 10 "b" [block 2] [block 2, block 1] `shouldReturn` True -- a store that does not know the way waits for one that does fleetAdvance conn keys 4 (Just 0) (Just (block 1)) (block 2) Nothing [] `shouldReturn` Raced- fleetRestarts conn keys `shouldReturn` before fleetTip conn keys `shouldReturn` Just (block 1) -- one that knows it and finds it too long does not, or the tip -- would never move again@@ -171,13 +179,13 @@ poolSpends conn keys `shouldReturn` [] (Redis.runRedis conn (Redis.keys (keys.prefix <> "pool:*")) >>= either (fail . show) return) `shouldReturn` []- -- which the stores hear of, to offer their mempools again- fleetRestarts conn keys `shouldReturn` before + 1+ -- the stores fetch its transactions again, and fill it again+ fleetSeen conn keys [block 7] `shouldReturn` [False]+ fleetRestarts conn keys `shouldReturn` 1 -- still only to the block that the live stores agree on, and only -- from the tip that the store saw fleetRestart conn keys 4 (Just 0) (Just (block 2)) (block 3) `shouldReturn` Raced fleetRestart conn keys 4 (Just 0) (Just (block 1)) (block 2) `shouldReturn` Raced- fleetRestarts conn keys `shouldReturn` before + 1 it "reads the pool by the page, as it is at the tip that is asked for" $ \(conn, fresh) -> do keys <- fresh fleetReport conn keys (Just 0) 10 "a" [block 1] [] `shouldReturn` True
test/Haskoin/Store/IntegrationSpec.hs view
@@ -7,24 +7,23 @@ -- Portability : POSIX -- -- Tests of the whole store against regtest chains of Bitcoin Core and Bitcoin--- Cash Node, with a Redis cache, all in Podman containers.+-- Cash Node, with Redis, all in Podman containers. -- -- The unit tests check pieces against models; these check that the pieces -- fit: that the store syncs from a real node, that the xpub endpoints find -- the payments a wallet following the gap would find and no others, and that--- the Redis cache answers exactly like the database. Each chain gets one node,--- one Redis and one store. The store is served twice, by a web server with--- the cache and by one without ('cache' set to 'Nothing'), so both read the--- same database, mempool times included, and every difference between their--- answers is a difference made by the cache.+-- two web servers of one store answer alike. Each chain gets one node, one+-- Redis and one store. The store is served twice: the cache of xpubs cannot+-- be turned off, so both answer xpubs from it, and both read the same+-- database and the same pool. -- -- Every test makes its own xpubs from a fixed seed, sends transactions to -- chosen derivation paths, mines them, and checks the answers against a -- ledger that it keeps itself: balance, transactions, unspent outputs, used -- paths and indices. Then it compares the two web servers on every xpub and -- Blockchain.info endpoint, with pages, starting points and filters. The cache--- writer updates Redis in the background, so the comparison is retried for up--- to 30 seconds before it fails.+-- is filled in the background, so the comparison is retried for up to 30+-- seconds before it fails. -- -- The tests are skipped when Podman cannot run containers, or when -- @HASKOIN_STORE_NO_PODMAN@ is set. Containers are removed when the tests@@ -223,9 +222,10 @@ node :: !Node, net :: !Network, store :: !Store,- -- | base URL of the web server with the cache+ -- | base URL of the first web server cached :: !String,- -- | base URL of the web server without the cache+ -- | base URL of the second web server of the same store, which answers+ -- xpubs from the same cache plain :: !String, -- | address of the wallet of the node, to mine to miner :: !Text,@@ -421,7 +421,7 @@ body :: BL.ByteString -> Value body b = either (const (String (cs b))) id (eitherDecode b) --- | Body of a successful request to the server without the cache.+-- | Body of a successful request to the second web server. plainJSON :: Env -> String -> IO Value plainJSON env path = do (code, v) <- getJSON env.plain path@@ -446,7 +446,7 @@ expectationFailure $ "GET " <> p- <> "\nstatus with and without the cache: "+ <> "\nstatus of the two web servers: " <> show (fst a, fst b) <> "\nfirst difference: " <> difference "" (snd a) (snd b)