packages feed

haskoin-store 1.15.1 → 1.16.0

raw patch · 99 files changed

+18197/−15596 lines, 99 filesdep +asyncdep +directorydep +temporarydep −unliftiodep ~haskoin-coredep ~haskoin-nodedep ~haskoin-store-dataPVP ok

version bump matches the API change (PVP)

Dependencies added: async, directory, temporary, time-manager

Dependencies removed: unliftio

Dependency ranges changed: haskoin-core, haskoin-node, haskoin-store-data, nqe, rocksdb-haskell-jprupp, rocksdb-query, statsd-rupp

API changes (from Hackage documentation)

- Haskoin.Store: CacheConfig :: !String -> !Int -> !Int -> !Int -> !Maybe Stats -> CacheConfig
- Haskoin.Store: [cache] :: Store -> !Maybe Cache
- Haskoin.Store: [ctx] :: StoreRead -> !Ctx
- Haskoin.Store: [maxXPubs] :: CacheConfig -> !Int
- Haskoin.Store: [minAddrs] :: CacheConfig -> !Int
- Haskoin.Store: [minTxs] :: CacheConfig -> !Int
- Haskoin.Store: [noMempool] :: StoreConfig -> !Bool
- Haskoin.Store: [redisMinAddrs] :: StoreConfig -> !Int
- Haskoin.Store: [redisMinTxs] :: StoreConfig -> !Int
- Haskoin.Store: [syncMempool] :: StoreConfig -> !Bool
- Haskoin.Store: [url] :: CacheConfig -> !String
- Haskoin.Store: [wipeMempool] :: StoreConfig -> !Bool
- Haskoin.Store: cacheDelete :: Cache -> XPubSpec -> IO Bool
- Haskoin.Store: cacheEvict :: Cache -> Int -> IO Int
- Haskoin.Store: cacheIsCached :: Cache -> XPubSpec -> IO Bool
- Haskoin.Store: data Cache
- Haskoin.Store: data CacheConfig
- Haskoin.Store: databaseRead :: DatabaseReader -> StoreRead
- Haskoin.Store.BlockStore: [ctx] :: BlockStoreConfig -> !Ctx
- Haskoin.Store.BlockStore: [noMempool] :: BlockStoreConfig -> !Bool
- Haskoin.Store.BlockStore: [syncMempool] :: BlockStoreConfig -> !Bool
- Haskoin.Store.BlockStore: [wipeMempool] :: BlockStoreConfig -> !Bool
- Haskoin.Store.BlockStore.Sync: Download :: !p -> !UTCTime -> ![BlockHash] -> Download p
- Haskoin.Store.BlockStore.Sync: Idle :: Sync p
- Haskoin.Store.BlockStore.Sync: Syncing :: !Download p -> Sync p
- Haskoin.Store.BlockStore.Sync: [blocks] :: Download p -> ![BlockHash]
- Haskoin.Store.BlockStore.Sync: data Download p
- Haskoin.Store.BlockStore.Sync: instance GHC.Classes.Eq p => GHC.Classes.Eq (Haskoin.Store.BlockStore.Sync.Download p)
- Haskoin.Store.BlockStore.Sync: instance GHC.Internal.Show.Show p => GHC.Internal.Show.Show (Haskoin.Store.BlockStore.Sync.Download p)
- Haskoin.Store.Cache: Behind :: Reload
- Haskoin.Store.Cache: CacheConfig :: !String -> !Int -> !Int -> !Int -> !Maybe Stats -> CacheConfig
- Haskoin.Store.Cache: CacheCounters :: !Int -> !Int -> !Int -> CacheCounters
- Haskoin.Store.Cache: Locked :: Reload
- Haskoin.Store.Cache: Reloaded :: Reload
- Haskoin.Store.Cache: [behind] :: CacheCounters -> !Int
- Haskoin.Store.Cache: [fullReloads] :: CacheCounters -> !Int
- Haskoin.Store.Cache: [maxXPubs] :: CacheConfig -> !Int
- Haskoin.Store.Cache: [minAddrs] :: CacheConfig -> !Int
- Haskoin.Store.Cache: [minTxs] :: CacheConfig -> !Int
- Haskoin.Store.Cache: [reloads] :: CacheCounters -> !Int
- Haskoin.Store.Cache: [stats] :: CacheConfig -> !Maybe Stats
- Haskoin.Store.Cache: [url] :: CacheConfig -> !String
- Haskoin.Store.Cache: blockRefScore :: BlockRef -> Double
- Haskoin.Store.Cache: cacheAlive :: Cache -> IO Bool
- Haskoin.Store.Cache: cacheCounters :: Cache -> IO CacheCounters
- Haskoin.Store.Cache: cacheDelete :: Cache -> XPubSpec -> IO Bool
- Haskoin.Store.Cache: cacheEvict :: Cache -> Int -> IO Int
- Haskoin.Store.Cache: cacheIsCached :: Cache -> XPubSpec -> IO Bool
- Haskoin.Store.Cache: cachePrefix :: Network -> ByteString
- Haskoin.Store.Cache: cacheRefresh :: Cache -> XPubSpec -> HashSet TxHash -> IO Reload
- Haskoin.Store.Cache: cacheRequest :: Cache -> XPubSpec -> [XPubBal] -> IO ()
- Haskoin.Store.Cache: cacheTxData :: Cache -> [TxHash] -> IO [Maybe TxData]
- Haskoin.Store.Cache: cacheXPubBals :: Cache -> XPubSpec -> IO (Maybe [XPubBal])
- Haskoin.Store.Cache: cacheXPubTxCount :: Cache -> XPubSpec -> IO (Maybe Word32)
- Haskoin.Store.Cache: cacheXPubTxs :: Cache -> XPubSpec -> Limits -> IO (Maybe [TxRef])
- Haskoin.Store.Cache: cacheXPubUnspents :: Cache -> XPubSpec -> [XPubBal] -> Limits -> IO (Maybe [XPubUnspent])
- Haskoin.Store.Cache: data Cache
- Haskoin.Store.Cache: data CacheConfig
- Haskoin.Store.Cache: data CacheCounters
- Haskoin.Store.Cache: data Reload
- Haskoin.Store.Cache: fullReloadAfter :: XPubSpec -> Integer
- Haskoin.Store.Cache: fullReloadEvery :: Integer
- Haskoin.Store.Cache: instance GHC.Classes.Eq Haskoin.Store.Cache.CacheCounters
- Haskoin.Store.Cache: instance GHC.Internal.Show.Show Haskoin.Store.Cache.CacheCounters
- Haskoin.Store.Cache: scoreBlockRef :: Double -> BlockRef
- Haskoin.Store.Cache: withCache :: CacheConfig -> DatabaseReader -> Chain -> Publisher StoreEvent -> Logger -> (Cache -> IO a) -> IO a
- Haskoin.Store.Cache: xpubId :: XPubSpec -> ByteString
- Haskoin.Store.Cache.Protocol: Behind :: Reload
- Haskoin.Store.Cache.Protocol: Busy :: Claim
- Haskoin.Store.Cache.Protocol: ClaimLater :: OnClaim
- Haskoin.Store.Cache.Protocol: Claimed :: Claim
- Haskoin.Store.Cache.Protocol: Done :: Claim
- Haskoin.Store.Cache.Protocol: Finish :: OnReload
- Haskoin.Store.Cache.Protocol: Hold :: OnReload
- Haskoin.Store.Cache.Protocol: Locked :: Reload
- Haskoin.Store.Cache.Protocol: Release :: OnReload
- Haskoin.Store.Cache.Protocol: ReloadNow :: OnClaim
- Haskoin.Store.Cache.Protocol: Reloaded :: Reload
- Haskoin.Store.Cache.Protocol: Skip :: OnClaim
- Haskoin.Store.Cache.Protocol: afterClaim :: Claim -> OnClaim
- Haskoin.Store.Cache.Protocol: afterReload :: Reload -> OnReload
- Haskoin.Store.Cache.Protocol: data Claim
- Haskoin.Store.Cache.Protocol: data OnClaim
- Haskoin.Store.Cache.Protocol: data OnReload
- Haskoin.Store.Cache.Protocol: data Reload
- Haskoin.Store.Cache.Protocol: instance GHC.Classes.Eq Haskoin.Store.Cache.Protocol.Claim
- Haskoin.Store.Cache.Protocol: instance GHC.Classes.Eq Haskoin.Store.Cache.Protocol.OnClaim
- Haskoin.Store.Cache.Protocol: instance GHC.Classes.Eq Haskoin.Store.Cache.Protocol.OnReload
- Haskoin.Store.Cache.Protocol: instance GHC.Classes.Eq Haskoin.Store.Cache.Protocol.Reload
- Haskoin.Store.Cache.Protocol: instance GHC.Internal.Enum.Bounded Haskoin.Store.Cache.Protocol.Claim
- Haskoin.Store.Cache.Protocol: instance GHC.Internal.Enum.Bounded Haskoin.Store.Cache.Protocol.Reload
- Haskoin.Store.Cache.Protocol: instance GHC.Internal.Enum.Enum Haskoin.Store.Cache.Protocol.Claim
- Haskoin.Store.Cache.Protocol: instance GHC.Internal.Enum.Enum Haskoin.Store.Cache.Protocol.Reload
- Haskoin.Store.Cache.Protocol: instance GHC.Internal.Show.Show Haskoin.Store.Cache.Protocol.Claim
- Haskoin.Store.Cache.Protocol: instance GHC.Internal.Show.Show Haskoin.Store.Cache.Protocol.OnClaim
- Haskoin.Store.Cache.Protocol: instance GHC.Internal.Show.Show Haskoin.Store.Cache.Protocol.OnReload
- Haskoin.Store.Cache.Protocol: instance GHC.Internal.Show.Show Haskoin.Store.Cache.Protocol.Reload
- Haskoin.Store.Cache.Protocol: mayReplace :: Maybe Integer -> BlockHeight -> Bool -> Bool
- Haskoin.Store.Common: [ctx] :: StoreRead -> !Ctx
- Haskoin.Store.Database.Mempool: AddressMempool :: ![TxRef] -> !HashMap OutPoint Word64 -> ![Unspent] -> AddressMempool
- Haskoin.Store.Database.Mempool: MemRecords :: !TxHash -> !UnixTime -> !TxData -> ![(OutPoint, Spender)] -> ![(MemAddrKey, MemAddrVal)] -> MemRecords
- Haskoin.Store.Database.Mempool: [addresses] :: MemRecords -> ![(MemAddrKey, MemAddrVal)]
- Haskoin.Store.Database.Mempool: [body] :: MemRecords -> !TxData
- Haskoin.Store.Database.Mempool: [created] :: AddressMempool -> ![Unspent]
- Haskoin.Store.Database.Mempool: [spends] :: MemRecords -> ![(OutPoint, Spender)]
- Haskoin.Store.Database.Mempool: [spent] :: AddressMempool -> !HashMap OutPoint Word64
- Haskoin.Store.Database.Mempool: [time] :: MemRecords -> !UnixTime
- Haskoin.Store.Database.Mempool: [txid] :: MemRecords -> !TxHash
- Haskoin.Store.Database.Mempool: [txs] :: AddressMempool -> ![TxRef]
- Haskoin.Store.Database.Mempool: addressArea :: [MemRecords] -> [(ByteString, ByteString)]
- Haskoin.Store.Database.Mempool: applyMempool :: Balance -> AddressMempool -> Balance
- Haskoin.Store.Database.Mempool: applySummary :: Balance -> MemSummary -> Balance
- Haskoin.Store.Database.Mempool: data AddressMempool
- Haskoin.Store.Database.Mempool: data MemRecords
- Haskoin.Store.Database.Mempool: deleteRecordOps :: DB -> MemRecords -> [BatchOp]
- Haskoin.Store.Database.Mempool: getAddressMempool :: DatabaseReader -> Address -> IO AddressMempool
- Haskoin.Store.Database.Mempool: getAddressMempools :: DatabaseReader -> [Address] -> IO [AddressMempool]
- Haskoin.Store.Database.Mempool: getMempoolSpenders :: DatabaseReader -> TxHash -> IO (IntMap Spender)
- Haskoin.Store.Database.Mempool: getMempoolTx :: DatabaseReader -> TxHash -> IO (Maybe TxData)
- Haskoin.Store.Database.Mempool: getSummaries :: DatabaseReader -> [Address] -> IO [Maybe MemSummary]
- Haskoin.Store.Database.Mempool: hiddenOutputs :: AddressMempool -> HashMap OutPoint Word64
- Haskoin.Store.Database.Mempool: insertRecordOps :: DB -> MemRecords -> [BatchOp]
- Haskoin.Store.Database.Mempool: instance GHC.Classes.Eq Haskoin.Store.Database.Mempool.AddressMempool
- Haskoin.Store.Database.Mempool: instance GHC.Classes.Eq Haskoin.Store.Database.Mempool.MemRecords
- Haskoin.Store.Database.Mempool: instance GHC.Internal.Show.Show Haskoin.Store.Database.Mempool.AddressMempool
- Haskoin.Store.Database.Mempool: instance GHC.Internal.Show.Show Haskoin.Store.Database.Mempool.MemRecords
- Haskoin.Store.Database.Mempool: mempoolRecords :: Network -> Ctx -> UnixTime -> TxData -> MemRecords
- Haskoin.Store.Database.Mempool: mempoolUnspents :: AddressMempool -> [Unspent]
- Haskoin.Store.Database.Mempool: mergeBalance :: Address -> Maybe Balance -> AddressMempool -> Maybe Balance
- Haskoin.Store.Database.Mempool: rebuildSummaries :: DatabaseReader -> IO Int
- Haskoin.Store.Database.Mempool: recordEntries :: MemRecords -> ([(ByteString, ByteString)], [(ByteString, ByteString)])
- Haskoin.Store.Database.Mempool: summarise :: AddressMempool -> MemSummary
- Haskoin.Store.Database.Mempool: summaryBalance :: Address -> Maybe Balance -> Maybe MemSummary -> Maybe Balance
- Haskoin.Store.Database.Mempool: summaryEntries :: [(MemAddrKey, MemAddrVal)] -> [(MemAddrKey, MemAddrVal)]
- Haskoin.Store.Database.Mempool: wipeMempoolArea :: DatabaseReader -> IO Int
- Haskoin.Store.Database.Mempool: wrongSummaries :: DatabaseReader -> IO [(MemAddrKey, Maybe MemAddrVal, Maybe MemAddrVal)]
- Haskoin.Store.Database.Query: databaseRead :: DatabaseReader -> StoreRead
- Haskoin.Store.Database.Query: getMempool :: DatabaseReader -> IO [(UnixTime, TxHash)]
- Haskoin.Store.Database.Query: getMempoolBefore :: DatabaseReader -> UnixTime -> Int -> IO [(UnixTime, TxHash)]
- Haskoin.Store.Database.Query: getMempoolPage :: DatabaseReader -> Limits -> IO [(UnixTime, TxHash)]
- Haskoin.Store.Database.Query.Layered: InChain :: !BlockRef -> From
- Haskoin.Store.Database.Query.Layered: InMempool :: !UnixTime -> !TxHash -> From
- Haskoin.Store.Database.Query.Layered: Top :: From
- Haskoin.Store.Database.Query.Layered: chainTxsC :: From -> Address -> Iterator -> ConduitT i TxRef IO ()
- Haskoin.Store.Database.Query.Layered: chainUnspentsC :: DatabaseReader -> From -> Address -> Iterator -> ConduitT i Unspent IO ()
- Haskoin.Store.Database.Query.Layered: data From
- Haskoin.Store.Database.Query.Layered: data View
- Haskoin.Store.Database.Query.Layered: getAddressTxs :: View -> Address -> Limits -> IO [TxRef]
- Haskoin.Store.Database.Query.Layered: getAddressUnspents :: View -> Address -> Limits -> IO [Unspent]
- Haskoin.Store.Database.Query.Layered: getAddressesTxCount :: View -> [Address] -> IO Word32
- Haskoin.Store.Database.Query.Layered: getAddressesTxs :: View -> [Address] -> Limits -> IO [TxRef]
- Haskoin.Store.Database.Query.Layered: getAddressesUnspents :: View -> [Address] -> Limits -> IO [Unspent]
- Haskoin.Store.Database.Query.Layered: getBalance :: View -> Address -> IO (Maybe Balance)
- Haskoin.Store.Database.Query.Layered: getBalances :: View -> [Address] -> IO [Balance]
- Haskoin.Store.Database.Query.Layered: getBestBlock :: View -> IO (Maybe BlockHash)
- Haskoin.Store.Database.Query.Layered: getBlock :: View -> BlockHash -> IO (Maybe BlockData)
- Haskoin.Store.Database.Query.Layered: getBlocksAtHeight :: View -> BlockHeight -> IO [BlockHash]
- Haskoin.Store.Database.Query.Layered: getManyTxBytes :: View -> [TxHash] -> IO [Maybe ByteString]
- Haskoin.Store.Database.Query.Layered: getManyTxData :: View -> [TxHash] -> IO [Maybe TxData]
- Haskoin.Store.Database.Query.Layered: getMempool :: View -> IO [(UnixTime, TxHash)]
- Haskoin.Store.Database.Query.Layered: getMempoolPage :: View -> Limits -> IO [(UnixTime, TxHash)]
- Haskoin.Store.Database.Query.Layered: getNumTxData :: View -> Word64 -> IO [TxData]
- Haskoin.Store.Database.Query.Layered: getSpender :: View -> OutPoint -> IO (Maybe Spender)
- Haskoin.Store.Database.Query.Layered: getTxData :: View -> TxHash -> IO (Maybe TxData)
- Haskoin.Store.Database.Query.Layered: getUnspent :: View -> OutPoint -> IO (Maybe Unspent)
- Haskoin.Store.Database.Query.Layered: layeredRead :: View -> StoreRead
- Haskoin.Store.Database.Query.Layered: standaloneRead :: DatabaseReader -> StoreRead
- Haskoin.Store.Database.Query.Layered: viewReader :: View -> DatabaseReader
- Haskoin.Store.Database.Query.Layered: withSpenders :: HashMap TxHash (IntMap Spender) -> TxHash -> TxData -> TxData
- Haskoin.Store.Database.Query.Layered: withView :: DatabaseReader -> (View -> IO a) -> IO a
- Haskoin.Store.Database.Query.Layered: xPubBals :: View -> XPubSpec -> IO [XPubBal]
- Haskoin.Store.Database.Query.Layered: xPubTxCount :: View -> [XPubBal] -> IO Word32
- Haskoin.Store.Database.Query.Layered: xPubTxs :: View -> [XPubBal] -> Limits -> IO [TxRef]
- Haskoin.Store.Database.Query.Layered: xPubUnspents :: View -> [XPubBal] -> Limits -> IO [XPubUnspent]
- Haskoin.Store.Database.Query.Pooled: localPool :: DatabaseReader -> PoolReads
- Haskoin.Store.Database.Reader: [ctx] :: DatabaseReader -> !Ctx
- Haskoin.Store.Database.Reader: [dataMempoolCount] :: DataMetrics -> !StatCounter
- Haskoin.Store.Database.Reader: initRocksDB :: DatabaseReader -> IO ()
- Haskoin.Store.Database.Reader: mempoolAddrCF :: DB -> ColumnFamily
- Haskoin.Store.Database.Reader: mempoolInsertOps :: DB -> TxHash -> UnixTime -> [BatchOp]
- Haskoin.Store.Database.Reader: withDatabaseReader :: Network -> Ctx -> Word32 -> Word32 -> FilePath -> Bool -> Maybe Int -> Maybe Int -> Maybe Int -> Maybe Int -> Maybe DataMetrics -> (DatabaseReader -> IO a) -> IO a
- Haskoin.Store.Database.Types: MemAddrKeyA :: !Address -> MemAddrKey
- Haskoin.Store.Database.Types: MemAddrKeyS :: MemAddrKey
- Haskoin.Store.Database.Types: MemAddrOutKey :: !Address -> !UnixTime -> !OutPoint -> MemAddrKey
- Haskoin.Store.Database.Types: MemAddrOutKeyA :: !Address -> MemAddrKey
- Haskoin.Store.Database.Types: MemAddrOutVal :: !Word64 -> !ByteString -> MemAddrVal
- Haskoin.Store.Database.Types: MemAddrSpentKey :: !Address -> !OutPoint -> MemAddrKey
- Haskoin.Store.Database.Types: MemAddrSpentVal :: !Word64 -> MemAddrVal
- Haskoin.Store.Database.Types: MemAddrSumKey :: !Address -> MemAddrKey
- Haskoin.Store.Database.Types: MemAddrSumVal :: !MemSummary -> MemAddrVal
- Haskoin.Store.Database.Types: MemAddrTxKey :: !Address -> !UnixTime -> !TxHash -> MemAddrKey
- Haskoin.Store.Database.Types: MemAddrTxKeyA :: !Address -> MemAddrKey
- Haskoin.Store.Database.Types: MemAddrTxVal :: MemAddrVal
- Haskoin.Store.Database.Types: MemDeletedKey :: !TxHash -> MemDeletedKey
- Haskoin.Store.Database.Types: MemDeletedKeyP :: !(Word32, Word16) -> MemDeletedKey
- Haskoin.Store.Database.Types: MemDeletedKeyS :: MemDeletedKey
- Haskoin.Store.Database.Types: MemKey :: MemKey
- Haskoin.Store.Database.Types: MemSpenderKey :: !OutPoint -> MemSpenderKey
- Haskoin.Store.Database.Types: MemSpenderKeyB :: MemSpenderKey
- Haskoin.Store.Database.Types: MemSpenderKeyS :: !TxHash -> MemSpenderKey
- Haskoin.Store.Database.Types: MemTimeKey :: !UnixTime -> !TxHash -> MemTimeKey
- Haskoin.Store.Database.Types: MemTimeKeyS :: MemTimeKey
- Haskoin.Store.Database.Types: data MemAddrKey
- Haskoin.Store.Database.Types: data MemAddrVal
- Haskoin.Store.Database.Types: data MemDeletedKey
- Haskoin.Store.Database.Types: data MemKey
- Haskoin.Store.Database.Types: data MemSpenderKey
- Haskoin.Store.Database.Types: data MemTimeKey
- Haskoin.Store.Database.Types: instance Control.DeepSeq.NFData Haskoin.Store.Database.Types.MemAddrVal
- Haskoin.Store.Database.Types: instance Data.Hashable.Class.Hashable Haskoin.Store.Database.Types.MemAddrKey
- Haskoin.Store.Database.Types: instance Data.Hashable.Class.Hashable Haskoin.Store.Database.Types.MemAddrVal
- Haskoin.Store.Database.Types: instance Data.Hashable.Class.Hashable Haskoin.Store.Database.Types.MemDeletedKey
- Haskoin.Store.Database.Types: instance Data.Hashable.Class.Hashable Haskoin.Store.Database.Types.MemSpenderKey
- Haskoin.Store.Database.Types: instance Data.Hashable.Class.Hashable Haskoin.Store.Database.Types.MemTimeKey
- Haskoin.Store.Database.Types: instance Data.Serialize.Serialize Haskoin.Store.Database.Types.MemAddrKey
- Haskoin.Store.Database.Types: instance Data.Serialize.Serialize Haskoin.Store.Database.Types.MemAddrVal
- Haskoin.Store.Database.Types: instance Data.Serialize.Serialize Haskoin.Store.Database.Types.MemDeletedKey
- Haskoin.Store.Database.Types: instance Data.Serialize.Serialize Haskoin.Store.Database.Types.MemKey
- Haskoin.Store.Database.Types: instance Data.Serialize.Serialize Haskoin.Store.Database.Types.MemSpenderKey
- Haskoin.Store.Database.Types: instance Data.Serialize.Serialize Haskoin.Store.Database.Types.MemTimeKey
- Haskoin.Store.Database.Types: instance Database.RocksDB.Query.Key Haskoin.Store.Database.Types.MemAddrKey
- Haskoin.Store.Database.Types: instance Database.RocksDB.Query.Key Haskoin.Store.Database.Types.MemDeletedKey
- Haskoin.Store.Database.Types: instance Database.RocksDB.Query.Key Haskoin.Store.Database.Types.MemKey
- Haskoin.Store.Database.Types: instance Database.RocksDB.Query.Key Haskoin.Store.Database.Types.MemSpenderKey
- Haskoin.Store.Database.Types: instance Database.RocksDB.Query.Key Haskoin.Store.Database.Types.MemTimeKey
- Haskoin.Store.Database.Types: instance Database.RocksDB.Query.KeyValue Haskoin.Store.Database.Types.MemAddrKey Haskoin.Store.Database.Types.MemAddrVal
- Haskoin.Store.Database.Types: instance Database.RocksDB.Query.KeyValue Haskoin.Store.Database.Types.MemDeletedKey Haskoin.Store.Data.TxData
- Haskoin.Store.Database.Types: instance Database.RocksDB.Query.KeyValue Haskoin.Store.Database.Types.MemKey [(Haskoin.Store.Data.UnixTime, Haskoin.Transaction.Common.TxHash)]
- Haskoin.Store.Database.Types: instance Database.RocksDB.Query.KeyValue Haskoin.Store.Database.Types.MemSpenderKey Haskoin.Store.Data.Spender
- Haskoin.Store.Database.Types: instance Database.RocksDB.Query.KeyValue Haskoin.Store.Database.Types.MemTimeKey ()
- Haskoin.Store.Database.Types: instance GHC.Classes.Eq Haskoin.Store.Database.Types.MemAddrKey
- Haskoin.Store.Database.Types: instance GHC.Classes.Eq Haskoin.Store.Database.Types.MemAddrVal
- Haskoin.Store.Database.Types: instance GHC.Classes.Eq Haskoin.Store.Database.Types.MemDeletedKey
- Haskoin.Store.Database.Types: instance GHC.Classes.Eq Haskoin.Store.Database.Types.MemSpenderKey
- Haskoin.Store.Database.Types: instance GHC.Classes.Eq Haskoin.Store.Database.Types.MemTimeKey
- Haskoin.Store.Database.Types: instance GHC.Classes.Ord Haskoin.Store.Database.Types.MemAddrKey
- Haskoin.Store.Database.Types: instance GHC.Classes.Ord Haskoin.Store.Database.Types.MemAddrVal
- Haskoin.Store.Database.Types: instance GHC.Classes.Ord Haskoin.Store.Database.Types.MemDeletedKey
- Haskoin.Store.Database.Types: instance GHC.Classes.Ord Haskoin.Store.Database.Types.MemSpenderKey
- Haskoin.Store.Database.Types: instance GHC.Classes.Ord Haskoin.Store.Database.Types.MemTimeKey
- Haskoin.Store.Database.Types: instance GHC.Internal.Generics.Generic Haskoin.Store.Database.Types.MemAddrKey
- Haskoin.Store.Database.Types: instance GHC.Internal.Generics.Generic Haskoin.Store.Database.Types.MemAddrVal
- Haskoin.Store.Database.Types: instance GHC.Internal.Generics.Generic Haskoin.Store.Database.Types.MemDeletedKey
- Haskoin.Store.Database.Types: instance GHC.Internal.Generics.Generic Haskoin.Store.Database.Types.MemSpenderKey
- Haskoin.Store.Database.Types: instance GHC.Internal.Generics.Generic Haskoin.Store.Database.Types.MemTimeKey
- Haskoin.Store.Database.Types: instance GHC.Internal.Read.Read Haskoin.Store.Database.Types.MemAddrVal
- Haskoin.Store.Database.Types: instance GHC.Internal.Read.Read Haskoin.Store.Database.Types.MemDeletedKey
- Haskoin.Store.Database.Types: instance GHC.Internal.Read.Read Haskoin.Store.Database.Types.MemKey
- Haskoin.Store.Database.Types: instance GHC.Internal.Read.Read Haskoin.Store.Database.Types.MemSpenderKey
- Haskoin.Store.Database.Types: instance GHC.Internal.Read.Read Haskoin.Store.Database.Types.MemTimeKey
- Haskoin.Store.Database.Types: instance GHC.Internal.Show.Show Haskoin.Store.Database.Types.MemAddrKey
- Haskoin.Store.Database.Types: instance GHC.Internal.Show.Show Haskoin.Store.Database.Types.MemAddrVal
- Haskoin.Store.Database.Types: instance GHC.Internal.Show.Show Haskoin.Store.Database.Types.MemDeletedKey
- Haskoin.Store.Database.Types: instance GHC.Internal.Show.Show Haskoin.Store.Database.Types.MemKey
- Haskoin.Store.Database.Types: instance GHC.Internal.Show.Show Haskoin.Store.Database.Types.MemSpenderKey
- Haskoin.Store.Database.Types: instance GHC.Internal.Show.Show Haskoin.Store.Database.Types.MemTimeKey
- Haskoin.Store.Database.Writer: MempoolChanges :: ![TxHash] -> ![TxHash] -> MempoolChanges
- Haskoin.Store.Database.Writer: [added] :: MempoolChanges -> ![TxHash]
- Haskoin.Store.Database.Writer: [removed] :: MempoolChanges -> ![TxHash]
- Haskoin.Store.Database.Writer: addToMempool :: Writer -> TxHash -> UnixTime -> IO ()
- Haskoin.Store.Database.Writer: data MempoolChanges
- Haskoin.Store.Database.Writer: deleteFromMempool :: Writer -> TxHash -> IO ()
- Haskoin.Store.Database.Writer: deleteMempoolAddr :: Writer -> MemAddrKey -> IO ()
- Haskoin.Store.Database.Writer: deleteMempoolSpender :: Writer -> OutPoint -> IO ()
- Haskoin.Store.Database.Writer: deleteMempoolTx :: Writer -> TxHash -> IO ()
- Haskoin.Store.Database.Writer: getMempool :: Writer -> IO [(UnixTime, TxHash)]
- Haskoin.Store.Database.Writer: getMempoolAddr :: Writer -> MemAddrKey -> IO (Maybe MemAddrVal)
- Haskoin.Store.Database.Writer: getMempoolSpender :: Writer -> OutPoint -> IO (Maybe Spender)
- Haskoin.Store.Database.Writer: getMempoolSpenders :: Writer -> TxHash -> Int -> IO (IntMap Spender)
- Haskoin.Store.Database.Writer: getMempoolTx :: Writer -> TxHash -> IO (Maybe TxData)
- Haskoin.Store.Database.Writer: getSpender :: Writer -> OutPoint -> IO (Maybe Spender)
- Haskoin.Store.Database.Writer: inMempool :: Writer -> TxHash -> IO Bool
- Haskoin.Store.Database.Writer: insertMempoolAddr :: Writer -> MemAddrKey -> MemAddrVal -> IO ()
- Haskoin.Store.Database.Writer: insertMempoolSpender :: Writer -> OutPoint -> Spender -> IO ()
- Haskoin.Store.Database.Writer: insertMempoolTx :: Writer -> TxData -> IO ()
- Haskoin.Store.Database.Writer: instance GHC.Classes.Eq Haskoin.Store.Database.Writer.MempoolChanges
- Haskoin.Store.Database.Writer: instance GHC.Internal.Show.Show Haskoin.Store.Database.Writer.MempoolChanges
- Haskoin.Store.Database.Writer: runWriterMempool :: Network -> Ctx -> DatabaseReader -> Logger -> (Writer -> IO a) -> IO (a, MempoolChanges)
- Haskoin.Store.Database.Writer: writerCtx :: Writer -> Ctx
- Haskoin.Store.Fleet.Member: [cacheMinAddrs] :: MemberConfig -> !Int
- Haskoin.Store.Fleet.Member: [cacheMinTxs] :: MemberConfig -> !Int
- Haskoin.Store.Fleet.Member: [lagging] :: Member -> !TVar (Maybe ByteString)
- Haskoin.Store.Fleet.Member: [wanted] :: Member -> !TVar (HashMap ByteString XPubSpec)
- Haskoin.Store.Fleet.Member: syncCache :: Member -> IO ()
- Haskoin.Store.Fleet.Member: wantCached :: Member -> XPubSpec -> [XPubBal] -> IO ()
- Haskoin.Store.Fleet.XPub: CacheEmptied :: CacheMoved
- Haskoin.Store.Fleet.XPub: CacheMoved :: CacheMoved
- Haskoin.Store.Fleet.XPub: CacheRaced :: CacheMoved
- Haskoin.Store.Fleet.XPub: CacheStays :: CacheMoved
- Haskoin.Store.Fleet.XPub: cacheAdvance :: Connection -> FleetKeys -> Maybe ByteString -> ByteString -> Maybe [(Address, Maybe Balance, AddressChange)] -> IO CacheMoved
- Haskoin.Store.Fleet.XPub: cacheTip :: Connection -> FleetKeys -> IO (Maybe ByteString)
- Haskoin.Store.Fleet.XPub: data CacheMoved
- Haskoin.Store.Fleet.XPub: instance GHC.Classes.Eq Haskoin.Store.Fleet.XPub.CacheMoved
- Haskoin.Store.Fleet.XPub: instance GHC.Internal.Show.Show Haskoin.Store.Fleet.XPub.CacheMoved
- Haskoin.Store.Fleet.XPub: nothingPool :: PoolReads
- Haskoin.Store.Logic: DoubleSpend :: ImportException
- Haskoin.Store.Logic: DuplicatePrevOutput :: ImportException
- Haskoin.Store.Logic: InsufficientFunds :: ImportException
- Haskoin.Store.Logic: TxConfirmed :: ImportException
- Haskoin.Store.Logic: UnexpectedCoinbase :: ImportException
- Haskoin.Store.Logic: deleteUnconfirmedTx :: Writer -> Bool -> TxHash -> IO ()
- Haskoin.Store.Logic: newMempoolTx :: Writer -> Tx -> UnixTime -> IO Bool
- Haskoin.Store.Logic: revertBlockAt :: Writer -> UnixTime -> BlockHash -> IO ()
- Haskoin.Store.Logic.Layered: deleteUnconfirmedTx :: Writer -> Bool -> TxHash -> IO ()
- Haskoin.Store.Logic.Layered: importBlock :: Writer -> Block -> BlockNode -> IO (BlockData, [TxData])
- Haskoin.Store.Logic.Layered: initBest :: Writer -> IO ()
- Haskoin.Store.Logic.Layered: newMempoolTx :: Writer -> Tx -> UnixTime -> IO Bool
- Haskoin.Store.Logic.Layered: revertBlock :: Writer -> UnixTime -> BlockHash -> IO ()
- Haskoin.Store.Manager: [cache] :: Store -> !Maybe Cache
- Haskoin.Store.Manager: [ctx] :: Store -> !Ctx
- Haskoin.Store.Manager: [noMempool] :: StoreConfig -> !Bool
- Haskoin.Store.Manager: [redisMinAddrs] :: StoreConfig -> !Int
- Haskoin.Store.Manager: [redisMinTxs] :: StoreConfig -> !Int
- Haskoin.Store.Manager: [syncMempool] :: StoreConfig -> !Bool
- Haskoin.Store.Manager: [wipeMempool] :: StoreConfig -> !Bool
- Haskoin.Store.Web: [noMempool] :: WebConfig -> !Bool
- Haskoin.Store.Web: [noXPub] :: WebConfig -> !Bool
+ Haskoin.Store: XPubConfig :: Settings -> XPubConfig
+ Haskoin.Store: [backgroundJobs] :: StoreConfig -> !Maybe Int
+ Haskoin.Store: [blockPing] :: StoreConfig -> !NominalDiffTime
+ Haskoin.Store: [blockStall] :: StoreConfig -> !NominalDiffTime
+ Haskoin.Store: [indexer] :: XPubConfig -> Settings
+ Haskoin.Store: [xpubs] :: StoreConfig -> !XPubConfig
+ Haskoin.Store: cachePrefix :: Network -> ByteString
+ Haskoin.Store: chainRead :: DatabaseReader -> StoreRead
+ Haskoin.Store: data XPubs
+ Haskoin.Store: defaultXPubConfig :: XPubConfig
+ Haskoin.Store: newtype XPubConfig
+ Haskoin.Store.BlockStore: [blockPing] :: BlockStoreConfig -> !NominalDiffTime
+ Haskoin.Store.BlockStore: [blockStall] :: BlockStoreConfig -> !NominalDiffTime
+ Haskoin.Store.BlockStore: [onTx] :: BlockStoreConfig -> !UnixTime -> Tx -> IO ()
+ Haskoin.Store.BlockStore: [seen] :: BlockStoreConfig -> ![TxHash] -> IO (HashSet TxHash)
+ Haskoin.Store.BlockStore: blockStoreAnnounced :: Peer -> [BlockHash] -> BlockStore -> IO ()
+ Haskoin.Store.BlockStore: blockStoreAnnouncedSTM :: Peer -> [BlockHash] -> BlockStore -> STM ()
+ Haskoin.Store.BlockStore: blockStorePong :: Peer -> Word64 -> BlockStore -> IO ()
+ Haskoin.Store.BlockStore: blockStorePongSTM :: Peer -> Word64 -> BlockStore -> STM ()
+ Haskoin.Store.BlockStore: blockStoreRefill :: BlockStore -> IO ()
+ Haskoin.Store.BlockStore: toFetch :: (TxHash -> IO Bool) -> ([TxHash] -> IO (HashSet TxHash)) -> (TxHash -> IO Bool) -> [TxHash] -> IO [TxHash]
+ Haskoin.Store.BlockStore.Pending: answered :: TxHash -> Pending -> Pending
+ Haskoin.Store.BlockStore.Sync: Asked :: !p -> !UTCTime -> ![BlockHash] -> Asked p
+ Haskoin.Store.BlockStore.Sync: Late :: Accept
+ Haskoin.Store.BlockStore.Sync: Pinged :: !p -> !UTCTime -> !Maybe UTCTime -> Pinged p
+ Haskoin.Store.BlockStore.Sync: Probe :: !Word64 -> ![Pinged p] -> Probe p
+ Haskoin.Store.BlockStore.Sync: Sync :: ![BlockHash] -> !Maybe (Asked p) -> !UTCTime -> !Int -> ![Asked p] -> !Maybe (Probe p) -> Sync p
+ Haskoin.Store.BlockStore.Sync: [answered] :: Pinged p -> !Maybe UTCTime
+ Haskoin.Store.BlockStore.Sync: [at] :: Pinged p -> !UTCTime
+ Haskoin.Store.BlockStore.Sync: [current] :: Sync p -> !Maybe (Asked p)
+ Haskoin.Store.BlockStore.Sync: [nonce] :: Probe p -> !Word64
+ Haskoin.Store.BlockStore.Sync: [owes] :: Asked p -> ![BlockHash]
+ Haskoin.Store.BlockStore.Sync: [passed] :: Sync p -> ![Asked p]
+ Haskoin.Store.BlockStore.Sync: [passes] :: Sync p -> !Int
+ Haskoin.Store.BlockStore.Sync: [pinged] :: Probe p -> ![Pinged p]
+ Haskoin.Store.BlockStore.Sync: [probe] :: Sync p -> !Maybe (Probe p)
+ Haskoin.Store.BlockStore.Sync: [since] :: Asked p -> !UTCTime
+ Haskoin.Store.BlockStore.Sync: [wanted] :: Sync p -> ![BlockHash]
+ Haskoin.Store.BlockStore.Sync: data Asked p
+ Haskoin.Store.BlockStore.Sync: data Pinged p
+ Haskoin.Store.BlockStore.Sync: data Probe p
+ Haskoin.Store.BlockStore.Sync: expired :: NominalDiffTime -> UTCTime -> Sync p -> [p]
+ Haskoin.Store.BlockStore.Sync: idle :: UTCTime -> Sync p
+ Haskoin.Store.BlockStore.Sync: instance GHC.Classes.Eq p => GHC.Classes.Eq (Haskoin.Store.BlockStore.Sync.Asked p)
+ Haskoin.Store.BlockStore.Sync: instance GHC.Classes.Eq p => GHC.Classes.Eq (Haskoin.Store.BlockStore.Sync.Pinged p)
+ Haskoin.Store.BlockStore.Sync: instance GHC.Classes.Eq p => GHC.Classes.Eq (Haskoin.Store.BlockStore.Sync.Probe p)
+ Haskoin.Store.BlockStore.Sync: instance GHC.Internal.Show.Show p => GHC.Internal.Show.Show (Haskoin.Store.BlockStore.Sync.Asked p)
+ Haskoin.Store.BlockStore.Sync: instance GHC.Internal.Show.Show p => GHC.Internal.Show.Show (Haskoin.Store.BlockStore.Sync.Pinged p)
+ Haskoin.Store.BlockStore.Sync: instance GHC.Internal.Show.Show p => GHC.Internal.Show.Show (Haskoin.Store.BlockStore.Sync.Probe p)
+ Haskoin.Store.BlockStore.Sync: overdue :: NominalDiffTime -> UTCTime -> Sync p -> [p]
+ Haskoin.Store.BlockStore.Sync: pass :: Eq p => UTCTime -> p -> Sync p -> (Maybe (p, [BlockHash]), Sync p)
+ Haskoin.Store.BlockStore.Sync: passedPeers :: Sync p -> [p]
+ Haskoin.Store.BlockStore.Sync: patience :: NominalDiffTime -> NominalDiffTime -> Sync p -> NominalDiffTime
+ Haskoin.Store.BlockStore.Sync: ping :: Eq p => UTCTime -> Word64 -> [p] -> Sync p -> (Maybe (Word64, [p]), Sync p)
+ Haskoin.Store.BlockStore.Sync: pong :: Eq p => NominalDiffTime -> UTCTime -> p -> Word64 -> Sync p -> (Bool, Sync p)
+ Haskoin.Store.BlockStore.Sync: probing :: Sync p -> Bool
+ Haskoin.Store.BlockStore.Sync: settle :: Sync p -> Sync p
+ Haskoin.Store.BlockStore.Sync: skip :: Eq p => p -> Sync p -> Sync p
+ Haskoin.Store.BlockStore.Sync: stalled :: NominalDiffTime -> NominalDiffTime -> UTCTime -> Sync p -> Maybe p
+ Haskoin.Store.Common: sortTxsWith :: (a -> TxHash) -> (a -> Tx) -> [a] -> [(Word32, a)]
+ Haskoin.Store.Database.Merge: balanceOperator :: MergeOperator
+ Haskoin.Store.Database.Merge: spenderOperator :: MergeOperator
+ Haskoin.Store.Database.Migrate: recordsVersion :: Word32
+ Haskoin.Store.Database.Query: scan :: (Serialize key, Serialize value) => Iterator -> key -> ConduitT i (key, value) IO ()
+ Haskoin.Store.Database.Query: scanFrom :: (Serialize key, Serialize value) => Iterator -> key -> key -> ConduitT i (key, value) IO ()
+ Haskoin.Store.Database.Query.Pooled: InChain :: !BlockRef -> From
+ Haskoin.Store.Database.Query.Pooled: InMempool :: !UnixTime -> !TxHash -> From
+ Haskoin.Store.Database.Query.Pooled: Top :: From
+ Haskoin.Store.Database.Query.Pooled: applySummary :: Balance -> MemSummary -> Balance
+ Haskoin.Store.Database.Query.Pooled: chainRead :: DatabaseReader -> StoreRead
+ Haskoin.Store.Database.Query.Pooled: data From
+ Haskoin.Store.Database.Query.Pooled: emptyPool :: PoolReads
+ Haskoin.Store.Database.Query.Pooled: summaryBalance :: Address -> Maybe Balance -> Maybe MemSummary -> Maybe Balance
+ Haskoin.Store.Database.Query.Pooled: withSpenders :: HashMap TxHash (IntMap Spender) -> TxHash -> TxData -> TxData
+ Haskoin.Store.Database.Reader: columnFamilyConfig :: Bool -> Bool -> [(String, Config)]
+ Haskoin.Store.Database.Reader: derivedAddrsCached :: DatabaseReader -> XPubSpec -> IO (Derived, Bool)
+ Haskoin.Store.Database.Reader: layeredFamilyConfig :: Bool -> Bool -> [(String, Config)]
+ Haskoin.Store.Database.Reader: mergeGuardFamily :: String
+ Haskoin.Store.Database.Reader: snapshotReader :: DatabaseReader -> DB -> IO DatabaseReader
+ Haskoin.Store.Database.Reader: withReader :: (Bool -> Bool -> [(String, Config)]) -> (DatabaseReader -> IO ()) -> Network -> Word32 -> Word32 -> FilePath -> Bool -> Maybe Int -> Maybe Int -> Maybe Int -> Maybe Int -> Maybe Int -> Maybe DataMetrics -> (DatabaseReader -> IO a) -> IO a
+ Haskoin.Store.Database.Reader: withSnapshotReader :: DatabaseReader -> (DatabaseReader -> IO a) -> IO a
+ Haskoin.Store.Database.Table: class Eq k => TableKey k
+ Haskoin.Store.Database.Table: data Table k v
+ Haskoin.Store.Database.Table: delete :: TableKey k => Table k v -> k -> IO ()
+ Haskoin.Store.Database.Table: insert :: TableKey k => Table k v -> k -> v -> IO ()
+ Haskoin.Store.Database.Table: instance GHC.Classes.Eq k => GHC.Classes.Eq (Haskoin.Store.Database.Table.Key k)
+ Haskoin.Store.Database.Table: instance Haskoin.Store.Database.Table.TableKey (Haskoin.Address.Address, Haskoin.Store.Data.BlockRef, Haskoin.Transaction.Common.OutPoint)
+ Haskoin.Store.Database.Table: instance Haskoin.Store.Database.Table.TableKey (Haskoin.Address.Address, Haskoin.Store.Data.TxRef)
+ Haskoin.Store.Database.Table: instance Haskoin.Store.Database.Table.TableKey (Haskoin.Transaction.Common.TxHash, GHC.Internal.Word.Word32)
+ Haskoin.Store.Database.Table: instance Haskoin.Store.Database.Table.TableKey GHC.Internal.Word.Word32
+ Haskoin.Store.Database.Table: instance Haskoin.Store.Database.Table.TableKey Haskoin.Address.Address
+ Haskoin.Store.Database.Table: instance Haskoin.Store.Database.Table.TableKey Haskoin.Block.Common.BlockHash
+ Haskoin.Store.Database.Table: instance Haskoin.Store.Database.Table.TableKey Haskoin.Transaction.Common.OutPoint
+ Haskoin.Store.Database.Table: instance Haskoin.Store.Database.Table.TableKey Haskoin.Transaction.Common.TxHash
+ Haskoin.Store.Database.Table: instance Haskoin.Store.Database.Table.TableKey k => Data.Hashable.Class.Hashable (Haskoin.Store.Database.Table.Key k)
+ Haskoin.Store.Database.Table: lookup :: TableKey k => Table k v -> k -> IO (Maybe v)
+ Haskoin.Store.Database.Table: new :: IO (Table k v)
+ Haskoin.Store.Database.Table: tableHash :: TableKey k => Int -> k -> Int
+ Haskoin.Store.Database.Table: toList :: TableKey k => Table k v -> IO [(k, v)]
+ Haskoin.Store.Database.Types: addBalVal :: BalVal -> BalVal -> BalVal
+ Haskoin.Store.Database.Writer: BatchReport :: !Maybe BlockTimes -> !Int -> !Int -> BatchReport
+ Haskoin.Store.Database.Writer: BlockTimes :: !Double -> !Double -> !ReadTimes -> !Double -> !Double -> BlockTimes
+ Haskoin.Store.Database.Writer: ReadTimes :: !Double -> !Double -> !Double -> ReadTimes
+ Haskoin.Store.Database.Writer: [apply] :: BlockTimes -> !Double
+ Haskoin.Store.Database.Writer: [balanceReads] :: ReadTimes -> !Double
+ Haskoin.Store.Database.Writer: [block] :: BatchReport -> !Maybe BlockTimes
+ Haskoin.Store.Database.Writer: [bytes] :: BatchReport -> !Int
+ Haskoin.Store.Database.Writer: [operations] :: BatchReport -> !Int
+ Haskoin.Store.Database.Writer: [prefetch] :: BlockTimes -> !Double
+ Haskoin.Store.Database.Writer: [prepare] :: BlockTimes -> !Double
+ Haskoin.Store.Database.Writer: [reads] :: BlockTimes -> !ReadTimes
+ Haskoin.Store.Database.Writer: [spenderReads] :: ReadTimes -> !Double
+ Haskoin.Store.Database.Writer: [unspentReads] :: ReadTimes -> !Double
+ Haskoin.Store.Database.Writer: [write] :: BlockTimes -> !Double
+ Haskoin.Store.Database.Writer: data BatchReport
+ Haskoin.Store.Database.Writer: data BlockTimes
+ Haskoin.Store.Database.Writer: data ReadTimes
+ Haskoin.Store.Database.Writer: insertTxOf :: Writer -> TxHash -> TxData -> IO ()
+ Haskoin.Store.Database.Writer: instance GHC.Classes.Eq Haskoin.Store.Database.Writer.BatchReport
+ Haskoin.Store.Database.Writer: instance GHC.Classes.Eq Haskoin.Store.Database.Writer.BlockTimes
+ Haskoin.Store.Database.Writer: instance GHC.Classes.Eq Haskoin.Store.Database.Writer.ReadTimes
+ Haskoin.Store.Database.Writer: instance GHC.Internal.Show.Show Haskoin.Store.Database.Writer.BatchReport
+ Haskoin.Store.Database.Writer: instance GHC.Internal.Show.Show Haskoin.Store.Database.Writer.BlockTimes
+ Haskoin.Store.Database.Writer: instance GHC.Internal.Show.Show Haskoin.Store.Database.Writer.ReadTimes
+ Haskoin.Store.Database.Writer: mergeBalance :: Writer -> Address -> BalVal -> IO ()
+ Haskoin.Store.Database.Writer: mergeSpender :: Writer -> OutPoint -> Spender -> IO ()
+ Haskoin.Store.Database.Writer: noReads :: ReadTimes
+ Haskoin.Store.Database.Writer: noteBlockTimes :: Writer -> Double -> Double -> ReadTimes -> Double -> IO ()
+ Haskoin.Store.Database.Writer: onAllCores :: Int -> Int -> ([a] -> IO b) -> [a] -> IO [b]
+ Haskoin.Store.Database.Writer: readAheadBalances :: Writer -> [Address] -> IO ()
+ Haskoin.Store.Database.Writer: readAheadSpenders :: Writer -> [OutPoint] -> IO ()
+ Haskoin.Store.Database.Writer: readAheadUnspents :: Writer -> [OutPoint] -> IO ()
+ Haskoin.Store.Database.Writer: readsAhead :: Writer -> Bool
+ Haskoin.Store.Database.Writer: runWriterReport :: Network -> DatabaseReader -> Logger -> (Writer -> IO a) -> IO (a, BatchReport)
+ Haskoin.Store.Database.Writer: stopwatch :: IO a -> IO (a, Double)
+ Haskoin.Store.Database.Writer: withoutReadCache :: Writer -> Writer
+ Haskoin.Store.Exception: bracket :: IO a -> (a -> IO b) -> (a -> IO c) -> IO c
+ Haskoin.Store.Exception: bracket_ :: IO a -> IO b -> IO c -> IO c
+ Haskoin.Store.Exception: catchAny :: IO a -> (SomeException -> IO a) -> IO a
+ Haskoin.Store.Exception: finally :: IO a -> IO b -> IO a
+ Haskoin.Store.Exception: handleAny :: (SomeException -> IO a) -> IO a -> IO a
+ Haskoin.Store.Exception: isSyncException :: SomeException -> Bool
+ Haskoin.Store.Exception: onException :: IO a -> IO b -> IO a
+ Haskoin.Store.Exception: tryAny :: IO a -> IO (Either SomeException a)
+ Haskoin.Store.Fleet.Member: [built] :: Member -> !TVar (HashMap ByteString Int)
+ Haskoin.Store.Fleet.Member: [door] :: Member -> !XPubs
+ Haskoin.Store.Fleet.Member: [epoch] :: Member -> !TVar (Maybe ByteString)
+ Haskoin.Store.Fleet.Member: [indexer] :: MemberConfig -> !Settings
+ Haskoin.Store.Fleet.Member: [lock] :: Member -> !Lock XPubSpec
+ Haskoin.Store.Fleet.Member: [refills] :: Member -> !TVar Int
+ Haskoin.Store.Fleet.Member: [seen] :: Member -> !IORef (ByteString, Word64)
+ Haskoin.Store.Fleet.Member: cacheGauges :: Member -> IO ()
+ Haskoin.Store.Fleet.Member: fleetSession :: Member -> ByteString -> DatabaseReader -> PoolReads -> IO () -> IO Session
+ Haskoin.Store.Fleet.Member: memberNewest :: Member -> IO (Maybe UnixTime)
+ Haskoin.Store.Fleet.Member: memberPoolBody :: Member -> TxHash -> IO (Maybe TxData)
+ Haskoin.Store.Fleet.Member: memberReceived :: Member -> UnixTime -> Tx -> IO ()
+ Haskoin.Store.Fleet.Member: memberSeen :: Member -> [TxHash] -> IO (HashSet TxHash)
+ Haskoin.Store.Fleet.Member: mostSeen :: Int
+ Haskoin.Store.Fleet.Member: noteEpoch :: Member -> IO ()
+ Haskoin.Store.Fleet.Redis: data Script
+ Haskoin.Store.Fleet.Redis: evalScript :: RedisResult a => Script -> [ByteString] -> [ByteString] -> Redis (Either Reply a)
+ Haskoin.Store.Fleet.Redis: fleetCommit :: Connection -> FleetKeys -> Int -> Maybe Time -> Maybe ByteString -> ByteString -> Maybe PoolChange -> [ByteString] -> ByteString -> IO Moved
+ Haskoin.Store.Fleet.Redis: fleetSeen :: Connection -> FleetKeys -> [ByteString] -> IO [Bool]
+ Haskoin.Store.Fleet.Redis: fleetTrimSeen :: Connection -> FleetKeys -> Int -> IO Integer
+ Haskoin.Store.Fleet.Redis: script :: ByteString -> Script
+ Haskoin.Store.Fleet.XPub: AddressMove :: !Address -> !Maybe Balance -> ![TxHash] -> ![(TxHash, BlockRef)] -> ![OutPoint] -> ![Unspent] -> AddressMove
+ Haskoin.Store.Fleet.XPub: BeginRaced :: Begun
+ Haskoin.Store.Fleet.XPub: Begun :: Begun
+ Haskoin.Store.Fleet.XPub: Busy :: Built
+ Haskoin.Store.Fleet.XPub: Moving :: Begun
+ Haskoin.Store.Fleet.XPub: [abandoned] :: AddressMove -> ![TxHash]
+ Haskoin.Store.Fleet.XPub: [added] :: AddressMove -> ![(TxHash, BlockRef)]
+ Haskoin.Store.Fleet.XPub: [address] :: AddressMove -> !Address
+ Haskoin.Store.Fleet.XPub: [balance] :: AddressMove -> !Maybe Balance
+ Haskoin.Store.Fleet.XPub: [gone] :: AddressMove -> ![OutPoint]
+ Haskoin.Store.Fleet.XPub: [short] :: Entry -> !Bool
+ Haskoin.Store.Fleet.XPub: [unspent] :: AddressMove -> ![Unspent]
+ Haskoin.Store.Fleet.XPub: [version] :: Entry -> !ByteString
+ Haskoin.Store.Fleet.XPub: cacheAbort :: Connection -> FleetKeys -> ByteString -> IO ()
+ Haskoin.Store.Fleet.XPub: cacheBegin :: Connection -> FleetKeys -> ByteString -> ByteString -> IO Begun
+ Haskoin.Store.Fleet.XPub: cachePieces :: Connection -> FleetKeys -> ByteString -> ByteString -> [AddressMove] -> IO Bool
+ Haskoin.Store.Fleet.XPub: data AddressMove
+ Haskoin.Store.Fleet.XPub: data Begun
+ Haskoin.Store.Fleet.XPub: instance GHC.Classes.Eq Haskoin.Store.Fleet.XPub.AddressMove
+ Haskoin.Store.Fleet.XPub: instance GHC.Classes.Eq Haskoin.Store.Fleet.XPub.Begun
+ Haskoin.Store.Fleet.XPub: instance GHC.Internal.Show.Show Haskoin.Store.Fleet.XPub.AddressMove
+ Haskoin.Store.Fleet.XPub: instance GHC.Internal.Show.Show Haskoin.Store.Fleet.XPub.Begun
+ Haskoin.Store.Fleet.XPub: lookahead :: DatabaseReader -> PoolReads -> XPubSpec -> [XPubBal] -> IO [XPubBal]
+ Haskoin.Store.Fleet.XPub: tipMoves :: Connection -> FleetKeys -> DatabaseReader -> [TxData] -> [TxData] -> IO [AddressMove]
+ Haskoin.Store.Logic: PreparedBlock :: ![PreparedTx] -> ![(Word32, PreparedTx)] -> !Int -> !Int -> PreparedBlock
+ Haskoin.Store.Logic: PreparedTx :: !Tx -> !TxHash -> ![OutPoint] -> ![Maybe Address] -> !Int -> !Int -> PreparedTx
+ Haskoin.Store.Logic: [base] :: PreparedTx -> !Int
+ Haskoin.Store.Logic: [ordered] :: PreparedBlock -> ![(Word32, PreparedTx)]
+ Haskoin.Store.Logic: [paid] :: PreparedTx -> ![Maybe Address]
+ Haskoin.Store.Logic: [size] :: PreparedTx -> !Int
+ Haskoin.Store.Logic: [spends] :: PreparedTx -> ![OutPoint]
+ Haskoin.Store.Logic: [tx] :: PreparedTx -> !Tx
+ Haskoin.Store.Logic: [txid] :: PreparedTx -> !TxHash
+ Haskoin.Store.Logic: [txs] :: PreparedBlock -> ![PreparedTx]
+ Haskoin.Store.Logic: [weight] :: PreparedBlock -> !Int
+ Haskoin.Store.Logic: data PreparedBlock
+ Haskoin.Store.Logic: data PreparedTx
+ Haskoin.Store.Logic: instance Control.DeepSeq.NFData Haskoin.Store.Logic.PreparedTx
+ Haskoin.Store.Logic: instance GHC.Classes.Eq Haskoin.Store.Logic.PreparedBlock
+ Haskoin.Store.Logic: instance GHC.Classes.Eq Haskoin.Store.Logic.PreparedTx
+ Haskoin.Store.Logic: instance GHC.Internal.Generics.Generic Haskoin.Store.Logic.PreparedTx
+ Haskoin.Store.Logic: instance GHC.Internal.Show.Show Haskoin.Store.Logic.PreparedBlock
+ Haskoin.Store.Logic: instance GHC.Internal.Show.Show Haskoin.Store.Logic.PreparedTx
+ Haskoin.Store.Logic: prepareBlock :: Network -> Block -> IO PreparedBlock
+ Haskoin.Store.Manager: [backgroundJobs] :: StoreConfig -> !Maybe Int
+ Haskoin.Store.Manager: [blockPing] :: StoreConfig -> !NominalDiffTime
+ Haskoin.Store.Manager: [blockStall] :: StoreConfig -> !NominalDiffTime
+ Haskoin.Store.Manager: [xpubs] :: StoreConfig -> !XPubConfig
+ Haskoin.Store.Manager: cachePrefix :: Network -> ByteString
+ Haskoin.Store.Web: JobBroke :: Unindexed
+ Haskoin.Store.Web: NotYet :: !Int -> Unindexed
+ Haskoin.Store.Web: [maxRequests] :: WebLimits -> !Word32
+ Haskoin.Store.Web: [minRequests] :: WebLimits -> !Word32
+ Haskoin.Store.Web: [requestQueue] :: WebLimits -> !Word32
+ Haskoin.Store.Web: [requestWait] :: WebLimits -> !Word32
+ Haskoin.Store.Web: [targetDelay] :: WebLimits -> !Word32
+ Haskoin.Store.Web: askXPubs :: Maybe XPubs -> Aside -> XPubReads -> [XPubSpec] -> IO [(XPubSpec, Maybe [XPubBal])]
+ Haskoin.Store.Web: data Unindexed
+ Haskoin.Store.Web: instance GHC.Classes.Eq Haskoin.Store.Web.Unindexed
+ Haskoin.Store.Web: instance GHC.Internal.Exception.Type.Exception Haskoin.Store.Web.Unindexed
+ Haskoin.Store.Web: instance GHC.Internal.Show.Show Haskoin.Store.Web.Unindexed
+ Haskoin.Store.Web: unindexedResponseOf :: Unindexed -> Response
+ Haskoin.Store.Web: xpubQuery :: Maybe XPubs -> Aside -> Session -> StoreRead -> StoreRead
+ Haskoin.Store.Web.Admission: Aside :: (forall b. () => IO b -> IO b) -> Aside
+ Haskoin.Store.Web.Admission: LineFull :: Refusal
+ Haskoin.Store.Web.Admission: Load :: !Int -> !Int -> !Int -> !Bool -> !Refusals -> Load
+ Haskoin.Store.Web.Admission: Overloaded :: Refusal
+ Haskoin.Store.Web.Admission: Refusals :: !Int -> !Int -> !Int -> Refusals
+ Haskoin.Store.Web.Admission: Report :: !Int -> !Int -> !Int -> !Int -> !Int -> !Bool -> Report
+ Haskoin.Store.Web.Admission: Settings :: !Int -> !Int -> !Int -> !Int -> !Int -> Settings
+ Haskoin.Store.Web.Admission: WaitedTooLong :: Refusal
+ Haskoin.Store.Web.Admission: [busy] :: Load -> !Int
+ Haskoin.Store.Web.Admission: [ceiling] :: Settings -> !Int
+ Haskoin.Store.Web.Admission: [delay] :: Report -> !Int
+ Haskoin.Store.Web.Admission: [floor] :: Settings -> !Int
+ Haskoin.Store.Web.Admission: [full] :: Refusals -> !Int
+ Haskoin.Store.Web.Admission: [lateness] :: Report -> !Int
+ Haskoin.Store.Web.Admission: [limit] :: Load -> !Int
+ Haskoin.Store.Web.Admission: [overloaded] :: Load -> !Bool
+ Haskoin.Store.Web.Admission: [patience] :: Settings -> !Int
+ Haskoin.Store.Web.Admission: [refused] :: Load -> !Refusals
+ Haskoin.Store.Web.Admission: [room] :: Settings -> !Int
+ Haskoin.Store.Web.Admission: [roundTrip] :: Report -> !Int
+ Haskoin.Store.Web.Admission: [shed] :: Refusals -> !Int
+ Haskoin.Store.Web.Admission: [target] :: Settings -> !Int
+ Haskoin.Store.Web.Admission: [used] :: Report -> !Int
+ Haskoin.Store.Web.Admission: [waited] :: Refusals -> !Int
+ Haskoin.Store.Web.Admission: [waiting] :: Load -> !Int
+ Haskoin.Store.Web.Admission: admitted :: Admission -> (Refusal -> IO a) -> IO a -> IO a
+ Haskoin.Store.Web.Admission: admittedAside :: Admission -> (Refusal -> IO a) -> (Aside -> IO a) -> IO a
+ Haskoin.Store.Web.Admission: aside :: Aside -> IO b -> IO b
+ Haskoin.Store.Web.Admission: data Admission
+ Haskoin.Store.Web.Admission: data Load
+ Haskoin.Store.Web.Admission: data Refusal
+ Haskoin.Store.Web.Admission: data Refusals
+ Haskoin.Store.Web.Admission: data Report
+ Haskoin.Store.Web.Admission: data Settings
+ Haskoin.Store.Web.Admission: instance GHC.Classes.Eq Haskoin.Store.Web.Admission.Load
+ Haskoin.Store.Web.Admission: instance GHC.Classes.Eq Haskoin.Store.Web.Admission.Refusal
+ Haskoin.Store.Web.Admission: instance GHC.Classes.Eq Haskoin.Store.Web.Admission.Refusals
+ Haskoin.Store.Web.Admission: instance GHC.Classes.Eq Haskoin.Store.Web.Admission.Report
+ Haskoin.Store.Web.Admission: instance GHC.Classes.Eq Haskoin.Store.Web.Admission.Settings
+ Haskoin.Store.Web.Admission: instance GHC.Internal.Show.Show Haskoin.Store.Web.Admission.Load
+ Haskoin.Store.Web.Admission: instance GHC.Internal.Show.Show Haskoin.Store.Web.Admission.Refusal
+ Haskoin.Store.Web.Admission: instance GHC.Internal.Show.Show Haskoin.Store.Web.Admission.Refusals
+ Haskoin.Store.Web.Admission: instance GHC.Internal.Show.Show Haskoin.Store.Web.Admission.Report
+ Haskoin.Store.Web.Admission: instance GHC.Internal.Show.Show Haskoin.Store.Web.Admission.Settings
+ Haskoin.Store.Web.Admission: interval :: Int
+ Haskoin.Store.Web.Admission: load :: Admission -> IO Load
+ Haskoin.Store.Web.Admission: newAdmission :: Settings -> IO Admission
+ Haskoin.Store.Web.Admission: newAdmissionWith :: IO Int -> Settings -> IO Admission
+ Haskoin.Store.Web.Admission: newtype Aside
+ Haskoin.Store.Web.Admission: regulate :: Admission -> (Report -> IO ()) -> IO ()
+ Haskoin.Store.Web.Admission: setClimbing :: Admission -> Bool -> IO ()
+ Haskoin.Store.Web.Admission: setLimit :: Admission -> Int -> IO ()
+ Haskoin.Store.Web.Admission: sojournTarget :: Int
+ Haskoin.Store.Web.Admission: sweep :: Admission -> IO ()
+ Haskoin.Store.Web.Congestion: Control :: !Int -> !Int -> !Int -> Control
+ Haskoin.Store.Web.Congestion: Delays :: !Int -> !Int -> !Int -> Delays
+ Haskoin.Store.Web.Congestion: Observation :: !Int -> !Int -> !Bool -> !Int -> !Int -> Observation
+ Haskoin.Store.Web.Congestion: Window :: !Int -> !Double -> !Double -> !Double -> !Double -> !Int -> Window
+ Haskoin.Store.Web.Congestion: [ceiling] :: Control -> !Int
+ Haskoin.Store.Web.Congestion: [climbed] :: Window -> !Double
+ Haskoin.Store.Web.Congestion: [collected] :: Delays -> !Int
+ Haskoin.Store.Web.Congestion: [delay] :: Observation -> !Int
+ Haskoin.Store.Web.Congestion: [elapsed] :: Observation -> !Int
+ Haskoin.Store.Web.Congestion: [floor] :: Control -> !Int
+ Haskoin.Store.Web.Congestion: [knee] :: Window -> !Double
+ Haskoin.Store.Web.Congestion: [level] :: Window -> !Double
+ Haskoin.Store.Web.Congestion: [limit] :: Window -> !Int
+ Haskoin.Store.Web.Congestion: [peak] :: Window -> !Double
+ Haskoin.Store.Web.Congestion: [raw] :: Delays -> !Int
+ Haskoin.Store.Web.Congestion: [roundTrip] :: Observation -> !Int
+ Haskoin.Store.Web.Congestion: [saturated] :: Observation -> !Bool
+ Haskoin.Store.Web.Congestion: [scheduling] :: Delays -> !Int
+ Haskoin.Store.Web.Congestion: [sinceCut] :: Window -> !Int
+ Haskoin.Store.Web.Congestion: [target] :: Control -> !Int
+ Haskoin.Store.Web.Congestion: [used] :: Observation -> !Int
+ Haskoin.Store.Web.Congestion: control :: Control -> Observation -> Window -> Window
+ Haskoin.Store.Web.Congestion: cpuTime :: IO Int
+ Haskoin.Store.Web.Congestion: data Control
+ Haskoin.Store.Web.Congestion: data Delays
+ Haskoin.Store.Web.Congestion: data Observation
+ Haskoin.Store.Web.Congestion: data Probe
+ Haskoin.Store.Web.Congestion: data Window
+ Haskoin.Store.Web.Congestion: delays :: [Probe] -> IO Delays
+ Haskoin.Store.Web.Congestion: instance GHC.Classes.Eq Haskoin.Store.Web.Congestion.Control
+ Haskoin.Store.Web.Congestion: instance GHC.Classes.Eq Haskoin.Store.Web.Congestion.Delays
+ Haskoin.Store.Web.Congestion: instance GHC.Classes.Eq Haskoin.Store.Web.Congestion.Observation
+ Haskoin.Store.Web.Congestion: instance GHC.Classes.Eq Haskoin.Store.Web.Congestion.Window
+ Haskoin.Store.Web.Congestion: instance GHC.Internal.Show.Show Haskoin.Store.Web.Congestion.Control
+ Haskoin.Store.Web.Congestion: instance GHC.Internal.Show.Show Haskoin.Store.Web.Congestion.Delays
+ Haskoin.Store.Web.Congestion: instance GHC.Internal.Show.Show Haskoin.Store.Web.Congestion.Observation
+ Haskoin.Store.Web.Congestion: instance GHC.Internal.Show.Show Haskoin.Store.Web.Congestion.Window
+ Haskoin.Store.Web.Congestion: lateness :: [Probe] -> IO Int
+ Haskoin.Store.Web.Congestion: percentile :: Int -> [Int] -> Int
+ Haskoin.Store.Web.Congestion: rising :: Control -> Observation -> Window -> Bool
+ Haskoin.Store.Web.Congestion: snooze :: Int -> IO ()
+ Haskoin.Store.Web.Congestion: weighted :: Int -> [(Int, Int)] -> Int
+ Haskoin.Store.Web.Congestion: window :: Control -> Window
+ Haskoin.Store.Web.Congestion: withProbes :: ([Probe] -> IO a) -> IO a
+ Haskoin.Store.Web.Pieces: pieceOf :: Int -> IO Builder -> Piece
+ Haskoin.Store.Web.Router: Light :: Cost
+ Haskoin.Store.Web.Router: Long :: Cost
+ Haskoin.Store.Web.Router: Short :: Cost
+ Haskoin.Store.Web.Router: [cost] :: Route -> !Cost
+ Haskoin.Store.Web.Router: [name] :: Route -> Text
+ Haskoin.Store.Web.Router: data Cost
+ Haskoin.Store.Web.Router: instance GHC.Classes.Eq Haskoin.Store.Web.Router.Cost
+ Haskoin.Store.Web.Router: instance GHC.Internal.Show.Show Haskoin.Store.Web.Router.Cost
+ Haskoin.Store.XPub: Budget :: IO () -> IO () -> (Int -> IO ()) -> Budget
+ Haskoin.Store.XPub: EntryMoved :: EntryMoved
+ Haskoin.Store.XPub: XPubReads :: (XPubSpec -> IO (Maybe [XPubBal])) -> (XPubSpec -> Limits -> IO (Maybe [TxRef])) -> (XPubSpec -> IO (Maybe Word32)) -> (XPubSpec -> [XPubBal] -> Limits -> IO (Maybe [XPubUnspent])) -> ([TxHash] -> IO [Maybe TxData]) -> ([Maybe TxData] -> IO [Maybe TxData]) -> XPubReads
+ Haskoin.Store.XPub: [balances] :: XPubReads -> XPubSpec -> IO (Maybe [XPubBal])
+ Haskoin.Store.XPub: [bodies] :: XPubReads -> [TxHash] -> IO [Maybe TxData]
+ Haskoin.Store.XPub: [count] :: XPubReads -> XPubSpec -> IO (Maybe Word32)
+ Haskoin.Store.XPub: [finish] :: Budget -> IO ()
+ Haskoin.Store.XPub: [settle] :: Budget -> IO ()
+ Haskoin.Store.XPub: [sized] :: Budget -> Int -> IO ()
+ Haskoin.Store.XPub: [transactions] :: XPubReads -> XPubSpec -> Limits -> IO (Maybe [TxRef])
+ Haskoin.Store.XPub: [unspents] :: XPubReads -> XPubSpec -> [XPubBal] -> Limits -> IO (Maybe [XPubUnspent])
+ Haskoin.Store.XPub: [wallet] :: XPubReads -> [Maybe TxData] -> IO [Maybe TxData]
+ Haskoin.Store.XPub: addressPrice :: Int
+ Haskoin.Store.XPub: bodyPrice :: Int
+ Haskoin.Store.XPub: data Budget
+ Haskoin.Store.XPub: data EntryMoved
+ Haskoin.Store.XPub: data XPubReads
+ Haskoin.Store.XPub: entryPrice :: Int
+ Haskoin.Store.XPub: greedyLua :: ByteString
+ Haskoin.Store.XPub: inSync :: DatabaseReader -> Chain -> IO Bool
+ Haskoin.Store.XPub: indexCost :: Int -> Int -> Int -> Int -> Int
+ Haskoin.Store.XPub: instance GHC.Classes.Eq Haskoin.Store.XPub.EntryMoved
+ Haskoin.Store.XPub: instance GHC.Internal.Exception.Type.Exception Haskoin.Store.XPub.EntryMoved
+ Haskoin.Store.XPub: instance GHC.Internal.Show.Show Haskoin.Store.XPub.EntryMoved
+ Haskoin.Store.XPub: noBudget :: Budget
+ Haskoin.Store.XPub: txAddresses :: Network -> TxData -> [Address]
+ Haskoin.Store.XPub.Indexer: AnotherStore :: !Int -> Holder
+ Haskoin.Store.XPub.Indexer: BeyondHorizon :: Refusal
+ Haskoin.Store.XPub.Indexer: Counts :: !Int -> !Int -> !Int -> !Int -> !Int -> !Refusals -> !Int -> !Int -> Counts
+ Haskoin.Store.XPub.Indexer: Failed :: Answer
+ Haskoin.Store.XPub.Indexer: Indexed :: Answer
+ Haskoin.Store.XPub.Indexer: IndexerStats :: !Int -> !Int -> !Int -> !Int -> !Counts -> !Int -> !Int -> IndexerStats
+ Haskoin.Store.XPub.Indexer: Nobody :: Holder
+ Haskoin.Store.XPub.Indexer: NotInSync :: Refusal
+ Haskoin.Store.XPub.Indexer: OtherStore :: Refusal
+ Haskoin.Store.XPub.Indexer: QueueFull :: Refusal
+ Haskoin.Store.XPub.Indexer: Refusals :: !Int -> !Int -> !Int -> !Int -> !Int -> !Int -> Refusals
+ Haskoin.Store.XPub.Indexer: RefuseKeep :: !Int -> Decision
+ Haskoin.Store.XPub.Indexer: RefuseNothing :: !Refusal -> !Int -> Decision
+ Haskoin.Store.XPub.Indexer: Refused :: !Refusal -> !Int -> Answer
+ Haskoin.Store.XPub.Indexer: Runner :: (Budget -> k -> p -> IO ()) -> (k -> IO Budget) -> IO Bool -> (k -> p -> IO ()) -> (k -> p -> Int -> IO ()) -> (k -> p -> SomeException -> IO ()) -> ([(k, p)] -> IO ()) -> Runner k p
+ Haskoin.Store.XPub.Indexer: Settings :: !Int -> !Int -> !Int -> !Int -> !Int -> Settings
+ Haskoin.Store.XPub.Indexer: Situation :: !Int -> !Int -> !Int -> !Bool -> !Holder -> Situation
+ Haskoin.Store.XPub.Indexer: ThisStore :: Holder
+ Haskoin.Store.XPub.Indexer: TooLong :: Refusal
+ Haskoin.Store.XPub.Indexer: Wait :: Decision
+ Haskoin.Store.XPub.Indexer: WaitedOut :: Refusal
+ Haskoin.Store.XPub.Indexer: [abandon] :: Runner k p -> k -> p -> IO ()
+ Haskoin.Store.XPub.Indexer: [answered] :: Counts -> !Int
+ Haskoin.Store.XPub.Indexer: [beyondHorizon] :: Refusals -> !Int
+ Haskoin.Store.XPub.Indexer: [blocked] :: Runner k p -> k -> p -> Int -> IO ()
+ Haskoin.Store.XPub.Indexer: [budget] :: Runner k p -> k -> IO Budget
+ Haskoin.Store.XPub.Indexer: [counts] :: IndexerStats -> !Counts
+ Haskoin.Store.XPub.Indexer: [declined] :: Counts -> !Int
+ Haskoin.Store.XPub.Indexer: [done] :: Counts -> !Int
+ Haskoin.Store.XPub.Indexer: [dropped] :: Counts -> !Int
+ Haskoin.Store.XPub.Indexer: [errors] :: Counts -> !Int
+ Haskoin.Store.XPub.Indexer: [expected] :: Situation -> !Int
+ Haskoin.Store.XPub.Indexer: [failed] :: Runner k p -> k -> p -> SomeException -> IO ()
+ Haskoin.Store.XPub.Indexer: [forRequests] :: IndexerStats -> !Int
+ Haskoin.Store.XPub.Indexer: [forUpkeep] :: IndexerStats -> !Int
+ Haskoin.Store.XPub.Indexer: [guess] :: Settings -> !Int
+ Haskoin.Store.XPub.Indexer: [hasRoom] :: Situation -> !Bool
+ Haskoin.Store.XPub.Indexer: [holder] :: Situation -> !Holder
+ Haskoin.Store.XPub.Indexer: [horizon] :: Settings -> !Int
+ Haskoin.Store.XPub.Indexer: [index] :: Runner k p -> Budget -> k -> p -> IO ()
+ Haskoin.Store.XPub.Indexer: [lost] :: Counts -> !Int
+ Haskoin.Store.XPub.Indexer: [median] :: IndexerStats -> !Int
+ Haskoin.Store.XPub.Indexer: [notInSync] :: Refusals -> !Int
+ Haskoin.Store.XPub.Indexer: [otherStore] :: Refusals -> !Int
+ Haskoin.Store.XPub.Indexer: [patience] :: Situation -> !Int
+ Haskoin.Store.XPub.Indexer: [prefetched] :: Counts -> !Int
+ Haskoin.Store.XPub.Indexer: [queueFull] :: Refusals -> !Int
+ Haskoin.Store.XPub.Indexer: [reach] :: Situation -> !Int
+ Haskoin.Store.XPub.Indexer: [ready] :: Runner k p -> IO Bool
+ Haskoin.Store.XPub.Indexer: [refused] :: Counts -> !Refusals
+ Haskoin.Store.XPub.Indexer: [renew] :: Runner k p -> [(k, p)] -> IO ()
+ Haskoin.Store.XPub.Indexer: [room] :: Settings -> !Int
+ Haskoin.Store.XPub.Indexer: [running] :: IndexerStats -> !Int
+ Haskoin.Store.XPub.Indexer: [threads] :: IndexerStats -> !Int
+ Haskoin.Store.XPub.Indexer: [tooLong] :: Refusals -> !Int
+ Haskoin.Store.XPub.Indexer: [wait] :: Settings -> !Int
+ Haskoin.Store.XPub.Indexer: [waitedOut] :: Refusals -> !Int
+ Haskoin.Store.XPub.Indexer: [waiting] :: IndexerStats -> !Int
+ Haskoin.Store.XPub.Indexer: [workers] :: Settings -> !Int
+ Haskoin.Store.XPub.Indexer: data Answer
+ Haskoin.Store.XPub.Indexer: data Counts
+ Haskoin.Store.XPub.Indexer: data Decision
+ Haskoin.Store.XPub.Indexer: data Holder
+ Haskoin.Store.XPub.Indexer: data Indexer k p
+ Haskoin.Store.XPub.Indexer: data IndexerStats
+ Haskoin.Store.XPub.Indexer: data Refusal
+ Haskoin.Store.XPub.Indexer: data Refusals
+ Haskoin.Store.XPub.Indexer: data Runner k p
+ Haskoin.Store.XPub.Indexer: data Settings
+ Haskoin.Store.XPub.Indexer: data Situation
+ Haskoin.Store.XPub.Indexer: decide :: Situation -> Decision
+ Haskoin.Store.XPub.Indexer: defaultSettings :: Settings
+ Haskoin.Store.XPub.Indexer: estimate :: Int -> Int -> Int -> Int
+ Haskoin.Store.XPub.Indexer: indexerIdle :: Indexer k p -> IO Bool
+ Haskoin.Store.XPub.Indexer: indexerStats :: Indexer k p -> IO IndexerStats
+ Haskoin.Store.XPub.Indexer: instance GHC.Classes.Eq Haskoin.Store.XPub.Indexer.Answer
+ Haskoin.Store.XPub.Indexer: instance GHC.Classes.Eq Haskoin.Store.XPub.Indexer.Counts
+ Haskoin.Store.XPub.Indexer: instance GHC.Classes.Eq Haskoin.Store.XPub.Indexer.Decision
+ Haskoin.Store.XPub.Indexer: instance GHC.Classes.Eq Haskoin.Store.XPub.Indexer.Holder
+ Haskoin.Store.XPub.Indexer: instance GHC.Classes.Eq Haskoin.Store.XPub.Indexer.IndexerStats
+ Haskoin.Store.XPub.Indexer: instance GHC.Classes.Eq Haskoin.Store.XPub.Indexer.Refusal
+ Haskoin.Store.XPub.Indexer: instance GHC.Classes.Eq Haskoin.Store.XPub.Indexer.Refusals
+ Haskoin.Store.XPub.Indexer: instance GHC.Classes.Eq Haskoin.Store.XPub.Indexer.Settings
+ Haskoin.Store.XPub.Indexer: instance GHC.Classes.Eq Haskoin.Store.XPub.Indexer.Situation
+ Haskoin.Store.XPub.Indexer: instance GHC.Internal.Enum.Bounded Haskoin.Store.XPub.Indexer.Refusal
+ Haskoin.Store.XPub.Indexer: instance GHC.Internal.Enum.Enum Haskoin.Store.XPub.Indexer.Refusal
+ Haskoin.Store.XPub.Indexer: instance GHC.Internal.Show.Show Haskoin.Store.XPub.Indexer.Answer
+ Haskoin.Store.XPub.Indexer: instance GHC.Internal.Show.Show Haskoin.Store.XPub.Indexer.Counts
+ Haskoin.Store.XPub.Indexer: instance GHC.Internal.Show.Show Haskoin.Store.XPub.Indexer.Decision
+ Haskoin.Store.XPub.Indexer: instance GHC.Internal.Show.Show Haskoin.Store.XPub.Indexer.Holder
+ Haskoin.Store.XPub.Indexer: instance GHC.Internal.Show.Show Haskoin.Store.XPub.Indexer.IndexerStats
+ Haskoin.Store.XPub.Indexer: instance GHC.Internal.Show.Show Haskoin.Store.XPub.Indexer.Refusal
+ Haskoin.Store.XPub.Indexer: instance GHC.Internal.Show.Show Haskoin.Store.XPub.Indexer.Refusals
+ Haskoin.Store.XPub.Indexer: instance GHC.Internal.Show.Show Haskoin.Store.XPub.Indexer.Settings
+ Haskoin.Store.XPub.Indexer: instance GHC.Internal.Show.Show Haskoin.Store.XPub.Indexer.Situation
+ Haskoin.Store.XPub.Indexer: median :: Int -> Seq Int -> Int
+ Haskoin.Store.XPub.Indexer: newIndexer :: Settings -> Lock k -> Runner k p -> IO (Indexer k p)
+ Haskoin.Store.XPub.Indexer: newIndexerWith :: IO Int -> Settings -> Lock k -> Runner k p -> IO (Indexer k p)
+ Haskoin.Store.XPub.Indexer: plainRunner :: (Budget -> k -> p -> IO ()) -> Runner k p
+ Haskoin.Store.XPub.Indexer: prefetch :: (Hashable k, Semigroup p) => Indexer k p -> k -> p -> IO Answer
+ Haskoin.Store.XPub.Indexer: renewHeld :: Hashable k => Indexer k p -> IO ()
+ Haskoin.Store.XPub.Indexer: request :: (Hashable k, Semigroup p) => Indexer k p -> k -> p -> IO Answer
+ Haskoin.Store.XPub.Indexer: requestWithin :: (Hashable k, Semigroup p) => Indexer k p -> Int -> k -> p -> IO Answer
+ Haskoin.Store.XPub.Indexer: retryAfter :: Int -> Int
+ Haskoin.Store.XPub.Indexer: runIndexer :: Hashable k => Indexer k p -> IO ()
+ Haskoin.Store.XPub.Indexer: runNext :: Hashable k => Indexer k p -> IO Bool
+ Haskoin.Store.XPub.Indexer: snooze :: Int -> IO ()
+ Haskoin.Store.XPub.Indexer: sweep :: Hashable k => Indexer k p -> IO ()
+ Haskoin.Store.XPub.Indexer: upkeep :: (Hashable k, Semigroup p) => Indexer k p -> k -> p -> IO ()
+ Haskoin.Store.XPub.Indexer: withIndexer :: Hashable k => Settings -> Lock k -> Runner k p -> (Indexer k p -> IO a) -> IO a
+ Haskoin.Store.XPub.Lock: Held :: !Int -> Taken
+ Haskoin.Store.XPub.Lock: Lock :: (k -> Int -> IO Taken) -> ([(k, Int)] -> IO Int) -> (k -> IO ()) -> (k -> IO (Maybe (Int, ByteString))) -> !Int -> Lock k
+ Haskoin.Store.XPub.Lock: LockConfig :: !ByteString -> !ByteString -> !Int -> LockConfig
+ Haskoin.Store.XPub.Lock: Took :: Taken
+ Haskoin.Store.XPub.Lock: [acquire] :: Lock k -> k -> Int -> IO Taken
+ Haskoin.Store.XPub.Lock: [every] :: Lock k -> !Int
+ Haskoin.Store.XPub.Lock: [holder] :: Lock k -> k -> IO (Maybe (Int, ByteString))
+ Haskoin.Store.XPub.Lock: [name] :: LockConfig -> !ByteString
+ Haskoin.Store.XPub.Lock: [prefix] :: LockConfig -> !ByteString
+ Haskoin.Store.XPub.Lock: [release] :: Lock k -> k -> IO ()
+ Haskoin.Store.XPub.Lock: [renew] :: Lock k -> [(k, Int)] -> IO Int
+ Haskoin.Store.XPub.Lock: [ttl] :: LockConfig -> !Int
+ Haskoin.Store.XPub.Lock: data Lock k
+ Haskoin.Store.XPub.Lock: data LockConfig
+ Haskoin.Store.XPub.Lock: data Taken
+ Haskoin.Store.XPub.Lock: defaultLockConfig :: ByteString -> ByteString -> LockConfig
+ Haskoin.Store.XPub.Lock: instance GHC.Classes.Eq Haskoin.Store.XPub.Lock.Taken
+ Haskoin.Store.XPub.Lock: instance GHC.Internal.Show.Show Haskoin.Store.XPub.Lock.Taken
+ Haskoin.Store.XPub.Lock: lockKey :: ByteString -> ByteString -> ByteString
+ Haskoin.Store.XPub.Lock: noLock :: Lock k
+ Haskoin.Store.XPub.Lock: redisLock :: LockConfig -> Connection -> (k -> ByteString) -> IO (Lock k)
+ Haskoin.Store.XPub.Service: Backend :: (XPubSpec -> Int -> IO Answer) -> (XPubSpec -> Int -> IO Bool) -> (XPubSpec -> IO (Maybe Int)) -> (XPubSpec -> IO Bool) -> Backend
+ Haskoin.Store.XPub.Service: BeyondHorizon :: Refusal
+ Haskoin.Store.XPub.Service: Failed :: Answer
+ Haskoin.Store.XPub.Service: Indexed :: Answer
+ Haskoin.Store.XPub.Service: NotInSync :: Refusal
+ Haskoin.Store.XPub.Service: OtherStore :: Refusal
+ Haskoin.Store.XPub.Service: QueueFull :: Refusal
+ Haskoin.Store.XPub.Service: Refused :: !Refusal -> !Int -> Answer
+ Haskoin.Store.XPub.Service: Session :: XPubReads -> IO () -> Session
+ Haskoin.Store.XPub.Service: TooLong :: Refusal
+ Haskoin.Store.XPub.Service: WaitedOut :: Refusal
+ Haskoin.Store.XPub.Service: XPubConfig :: Settings -> XPubConfig
+ Haskoin.Store.XPub.Service: XPubs :: !Backend -> !Int -> XPubs
+ Haskoin.Store.XPub.Service: [ask] :: Backend -> XPubSpec -> Int -> IO Answer
+ Haskoin.Store.XPub.Service: [await] :: Backend -> XPubSpec -> Int -> IO Bool
+ Haskoin.Store.XPub.Service: [backend] :: XPubs -> !Backend
+ Haskoin.Store.XPub.Service: [delete] :: Backend -> XPubSpec -> IO Bool
+ Haskoin.Store.XPub.Service: [eta] :: Backend -> XPubSpec -> IO (Maybe Int)
+ Haskoin.Store.XPub.Service: [indexer] :: XPubConfig -> Settings
+ Haskoin.Store.XPub.Service: [patience] :: XPubs -> !Int
+ Haskoin.Store.XPub.Service: [reads] :: Session -> XPubReads
+ Haskoin.Store.XPub.Service: [settle] :: Session -> IO ()
+ Haskoin.Store.XPub.Service: data Answer
+ Haskoin.Store.XPub.Service: data Backend
+ Haskoin.Store.XPub.Service: data Refusal
+ Haskoin.Store.XPub.Service: data Session
+ Haskoin.Store.XPub.Service: data XPubs
+ Haskoin.Store.XPub.Service: defaultXPubConfig :: XPubConfig
+ Haskoin.Store.XPub.Service: instance GHC.Classes.Eq Haskoin.Store.XPub.Service.XPubConfig
+ Haskoin.Store.XPub.Service: instance GHC.Internal.Show.Show Haskoin.Store.XPub.Service.XPubConfig
+ Haskoin.Store.XPub.Service: newtype XPubConfig
+ Haskoin.Store.XPub.Service: retryAfter :: Int -> Int
+ Haskoin.Store.XPub.Service: unpinned :: Session
+ Haskoin.Store.XPub.Service: xpubAsk :: XPubs -> XPubSpec -> IO Answer
+ Haskoin.Store.XPub.Service: xpubDelete :: XPubs -> XPubSpec -> IO Bool
- Haskoin.Store: Store :: !PeerMgr -> !Chain -> !BlockStore -> !DatabaseReader -> !Maybe Cache -> !Maybe Member -> !Publisher StoreEvent -> !Publisher StoreEvent -> !Network -> !Ctx -> !SockAddr -> WithConnection -> Store
+ Haskoin.Store: Store :: !PeerMgr -> !Chain -> !BlockStore -> !DatabaseReader -> !Maybe Member -> !Publisher StoreEvent -> !Publisher StoreEvent -> !Network -> !SockAddr -> WithConnection -> Store
- Haskoin.Store: StoreConfig :: !Int -> ![String] -> !Bool -> !FilePath -> !Network -> !Maybe String -> !Ctx -> !Word32 -> !Word32 -> !Int -> !Int -> !Int -> !Bool -> !Bool -> !Bool -> !Bool -> !Int -> !Int -> !NominalDiffTime -> !NominalDiffTime -> !SockAddr -> WithConnection -> !Maybe Stats -> !Bool -> !Maybe Int -> !Maybe Int -> !Maybe Int -> !Maybe Int -> !Logger -> StoreConfig
+ Haskoin.Store: StoreConfig :: !Int -> ![String] -> !Bool -> !FilePath -> !Network -> !Maybe String -> !Word32 -> !Word32 -> !Int -> !XPubConfig -> !Int -> !Int -> !NominalDiffTime -> !NominalDiffTime -> !NominalDiffTime -> !NominalDiffTime -> !SockAddr -> WithConnection -> !Maybe Stats -> !Bool -> !Maybe Int -> !Maybe Int -> !Maybe Int -> !Maybe Int -> !Maybe Int -> !Logger -> StoreConfig
- Haskoin.Store: StoreRead :: !Network -> !Ctx -> IO (Maybe BlockHash) -> (BlockHeight -> IO [BlockHash]) -> (BlockHash -> IO (Maybe BlockData)) -> (TxHash -> IO (Maybe TxData)) -> ([TxHash] -> IO [Maybe TxData]) -> ([TxHash] -> IO [Maybe ByteString]) -> (OutPoint -> IO (Maybe Spender)) -> (Address -> IO (Maybe Balance)) -> (OutPoint -> IO (Maybe Unspent)) -> IO [(UnixTime, TxHash)] -> (Limits -> IO [(UnixTime, TxHash)]) -> ([Address] -> Limits -> IO [TxRef]) -> ([Address] -> Limits -> IO [Unspent]) -> !Word32 -> !Word32 -> (Word64 -> IO [TxData]) -> ([Address] -> IO [Balance]) -> (Address -> Limits -> IO [TxRef]) -> (Address -> Limits -> IO [Unspent]) -> (XPubSpec -> IO [XPubBal]) -> (XPubSpec -> [XPubBal] -> Limits -> IO [XPubUnspent]) -> (XPubSpec -> [XPubBal] -> Limits -> IO [TxRef]) -> (XPubSpec -> [XPubBal] -> IO Word32) -> ([Address] -> IO Word32) -> (TxHash -> IO (Maybe Replaced)) -> StoreRead
+ Haskoin.Store: StoreRead :: !Network -> IO (Maybe BlockHash) -> (BlockHeight -> IO [BlockHash]) -> (BlockHash -> IO (Maybe BlockData)) -> (TxHash -> IO (Maybe TxData)) -> ([TxHash] -> IO [Maybe TxData]) -> ([TxHash] -> IO [Maybe ByteString]) -> (OutPoint -> IO (Maybe Spender)) -> (Address -> IO (Maybe Balance)) -> (OutPoint -> IO (Maybe Unspent)) -> IO [(UnixTime, TxHash)] -> (Limits -> IO [(UnixTime, TxHash)]) -> ([Address] -> Limits -> IO [TxRef]) -> ([Address] -> Limits -> IO [Unspent]) -> !Word32 -> !Word32 -> (Word64 -> IO [TxData]) -> ([Address] -> IO [Balance]) -> (Address -> Limits -> IO [TxRef]) -> (Address -> Limits -> IO [Unspent]) -> (XPubSpec -> IO [XPubBal]) -> (XPubSpec -> [XPubBal] -> Limits -> IO [XPubUnspent]) -> (XPubSpec -> [XPubBal] -> Limits -> IO [TxRef]) -> (XPubSpec -> [XPubBal] -> IO Word32) -> ([Address] -> IO Word32) -> (TxHash -> IO (Maybe Replaced)) -> StoreRead
- Haskoin.Store: [fleet] :: StoreConfig -> !Bool
+ Haskoin.Store: [fleet] :: Store -> !Maybe Member
- Haskoin.Store: [stats] :: CacheConfig -> !Maybe Stats
+ Haskoin.Store: [stats] :: StoreConfig -> !Maybe Stats
- Haskoin.Store.BlockStore: BlockStoreConfig :: !Ctx -> !PeerMgr -> !Chain -> !Publisher StoreEvent -> !DatabaseReader -> !Network -> !Bool -> !Bool -> !BlockHash -> IO () -> !Bool -> !Int -> !Int -> !NominalDiffTime -> !Maybe Stats -> !Logger -> BlockStoreConfig
+ Haskoin.Store.BlockStore: BlockStoreConfig :: !PeerMgr -> !Chain -> !Publisher StoreEvent -> !DatabaseReader -> !Network -> !BlockHash -> IO () -> !UnixTime -> Tx -> IO () -> ![TxHash] -> IO (HashSet TxHash) -> !Int -> !Int -> !NominalDiffTime -> !NominalDiffTime -> !NominalDiffTime -> !Maybe Stats -> !Logger -> BlockStoreConfig
- Haskoin.Store.BlockStore.Sync: [peer] :: Download p -> !p
+ Haskoin.Store.BlockStore.Sync: [peer] :: Pinged p -> !p
- Haskoin.Store.BlockStore.Sync: [time] :: Download p -> !UTCTime
+ Haskoin.Store.BlockStore.Sync: [time] :: Sync p -> !UTCTime
- Haskoin.Store.BlockStore.Sync: accept :: Eq p => p -> BlockHash -> Sync p -> Accept
+ Haskoin.Store.BlockStore.Sync: accept :: Eq p => p -> BlockHash -> Sync p -> (Accept, Sync p)
- Haskoin.Store.BlockStore.Sync: start :: UTCTime -> p -> Sync p -> Maybe (Sync p)
+ Haskoin.Store.BlockStore.Sync: start :: Eq p => UTCTime -> p -> Sync p -> Maybe (Sync p)
- Haskoin.Store.Common: StoreRead :: !Network -> !Ctx -> IO (Maybe BlockHash) -> (BlockHeight -> IO [BlockHash]) -> (BlockHash -> IO (Maybe BlockData)) -> (TxHash -> IO (Maybe TxData)) -> ([TxHash] -> IO [Maybe TxData]) -> ([TxHash] -> IO [Maybe ByteString]) -> (OutPoint -> IO (Maybe Spender)) -> (Address -> IO (Maybe Balance)) -> (OutPoint -> IO (Maybe Unspent)) -> IO [(UnixTime, TxHash)] -> (Limits -> IO [(UnixTime, TxHash)]) -> ([Address] -> Limits -> IO [TxRef]) -> ([Address] -> Limits -> IO [Unspent]) -> !Word32 -> !Word32 -> (Word64 -> IO [TxData]) -> ([Address] -> IO [Balance]) -> (Address -> Limits -> IO [TxRef]) -> (Address -> Limits -> IO [Unspent]) -> (XPubSpec -> IO [XPubBal]) -> (XPubSpec -> [XPubBal] -> Limits -> IO [XPubUnspent]) -> (XPubSpec -> [XPubBal] -> Limits -> IO [TxRef]) -> (XPubSpec -> [XPubBal] -> IO Word32) -> ([Address] -> IO Word32) -> (TxHash -> IO (Maybe Replaced)) -> StoreRead
+ Haskoin.Store.Common: StoreRead :: !Network -> IO (Maybe BlockHash) -> (BlockHeight -> IO [BlockHash]) -> (BlockHash -> IO (Maybe BlockData)) -> (TxHash -> IO (Maybe TxData)) -> ([TxHash] -> IO [Maybe TxData]) -> ([TxHash] -> IO [Maybe ByteString]) -> (OutPoint -> IO (Maybe Spender)) -> (Address -> IO (Maybe Balance)) -> (OutPoint -> IO (Maybe Unspent)) -> IO [(UnixTime, TxHash)] -> (Limits -> IO [(UnixTime, TxHash)]) -> ([Address] -> Limits -> IO [TxRef]) -> ([Address] -> Limits -> IO [Unspent]) -> !Word32 -> !Word32 -> (Word64 -> IO [TxData]) -> ([Address] -> IO [Balance]) -> (Address -> Limits -> IO [TxRef]) -> (Address -> Limits -> IO [Unspent]) -> (XPubSpec -> IO [XPubBal]) -> (XPubSpec -> [XPubBal] -> Limits -> IO [XPubUnspent]) -> (XPubSpec -> [XPubBal] -> Limits -> IO [TxRef]) -> (XPubSpec -> [XPubBal] -> IO Word32) -> ([Address] -> IO Word32) -> (TxHash -> IO (Maybe Replaced)) -> StoreRead
- Haskoin.Store.Common: deriveFunction :: Ctx -> DeriveType -> DeriveAddr
+ Haskoin.Store.Common: deriveFunction :: DeriveType -> DeriveAddr
- Haskoin.Store.Database.Query: retrieveManyBytesCF :: Serialize key => DB -> ColumnFamily -> [key] -> IO [Maybe ByteString]
+ Haskoin.Store.Database.Query: retrieveManyBytesCF :: Serialize key => DatabaseReader -> ColumnFamily -> [key] -> IO [Maybe ByteString]
- Haskoin.Store.Database.Query: retrieveManyCF :: (Serialize key, Serialize value) => DB -> ColumnFamily -> [key] -> IO [Maybe value]
+ Haskoin.Store.Database.Query: retrieveManyCF :: (Serialize key, Serialize value) => DatabaseReader -> ColumnFamily -> [key] -> IO [Maybe value]
- Haskoin.Store.Database.Reader: DataMetrics :: !StatCounter -> !StatCounter -> !StatCounter -> !StatCounter -> !StatCounter -> !StatCounter -> !StatCounter -> !StatCounter -> !StatCounter -> !StatCounter -> !StatCounter -> !StatGauge -> DataMetrics
+ Haskoin.Store.Database.Reader: DataMetrics :: !StatCounter -> !StatCounter -> !StatCounter -> !StatCounter -> !StatCounter -> !StatCounter -> !StatCounter -> !StatCounter -> !StatCounter -> !StatCounter -> !StatGauge -> DataMetrics
- Haskoin.Store.Database.Reader: DatabaseReader :: !DB -> !Word32 -> !Word32 -> !Network -> !Maybe DataMetrics -> !Ctx -> !IORef DerivedCache -> DatabaseReader
+ Haskoin.Store.Database.Reader: DatabaseReader :: !DB -> !Word32 -> !Word32 -> !Network -> !Maybe DataMetrics -> !IORef DerivedCache -> DatabaseReader
- Haskoin.Store.Database.Reader: withLayeredReader :: Network -> Ctx -> Word32 -> Word32 -> FilePath -> Bool -> Maybe Int -> Maybe Int -> Maybe Int -> Maybe Int -> Maybe DataMetrics -> (DatabaseReader -> IO a) -> IO a
+ Haskoin.Store.Database.Reader: withLayeredReader :: Network -> Word32 -> Word32 -> FilePath -> Bool -> Maybe Int -> Maybe Int -> Maybe Int -> Maybe Int -> Maybe Int -> Maybe DataMetrics -> (DatabaseReader -> IO a) -> IO a
- Haskoin.Store.Database.Types: toUnspent :: Network -> Ctx -> AddrOutKey -> OutVal -> Maybe Unspent
+ Haskoin.Store.Database.Types: toUnspent :: Network -> AddrOutKey -> OutVal -> Maybe Unspent
- Haskoin.Store.Database.Types: valToUnspent :: Network -> Ctx -> OutPoint -> UnspentVal -> Unspent
+ Haskoin.Store.Database.Types: valToUnspent :: Network -> OutPoint -> UnspentVal -> Unspent
- Haskoin.Store.Database.Writer: runWriter :: Network -> Ctx -> DatabaseReader -> Logger -> (Writer -> IO a) -> IO a
+ Haskoin.Store.Database.Writer: runWriter :: Network -> DatabaseReader -> Logger -> (Writer -> IO a) -> IO a
- Haskoin.Store.Fleet.Member: Between :: !PoolChange -> ![TxData] -> !Bool -> ![BlockHash] -> Between
+ Haskoin.Store.Fleet.Member: Between :: !PoolChange -> ![TxData] -> !Bool -> ![TxData] -> ![BlockHash] -> Between
- Haskoin.Store.Fleet.Member: Member :: !MemberConfig -> !Connection -> !FleetKeys -> !DatabaseReader -> !Views BlockHash -> !TVar [BlockHash] -> !TVar [BlockHash] -> !TVar (Maybe BlockHash) -> !TVar (HashSet TxHash) -> !TVar (HashMap TxHash UnixTime) -> !TVar Int -> !TVar (Maybe BlockHash) -> !TVar Bool -> !TVar Bool -> !TVar (HashMap ByteString XPubSpec) -> !TVar (Maybe ByteString) -> !TVar (Maybe Integer) -> !Maybe CacheMetrics -> !TVar (HashSet TxHash) -> !Outage -> Member
+ Haskoin.Store.Fleet.Member: Member :: !MemberConfig -> !Connection -> !FleetKeys -> !DatabaseReader -> !Views BlockHash -> !TVar [BlockHash] -> !TVar [BlockHash] -> !TVar (Maybe BlockHash) -> !TVar (HashSet TxHash) -> !TVar (HashMap TxHash (UnixTime, Tx)) -> !TVar Int -> !TVar (Maybe BlockHash) -> !TVar Bool -> !TVar Bool -> !Indexer XPubSpec () -> !Lock XPubSpec -> !TVar (HashMap ByteString Int) -> !XPubs -> !TVar (Maybe Integer) -> !Maybe CacheMetrics -> !TVar (HashSet TxHash) -> !Outage -> !TVar (Maybe ByteString) -> !TVar Int -> !IORef (ByteString, Word64) -> Member
- Haskoin.Store.Fleet.Member: MemberConfig :: !ByteString -> !ByteString -> !Word64 -> !Int -> !Word64 -> !Int -> !Int -> !Int -> !Maybe Stats -> !LogLevel -> Text -> IO () -> MemberConfig
+ Haskoin.Store.Fleet.Member: MemberConfig :: !ByteString -> !ByteString -> !Word64 -> !Int -> !Word64 -> !Int -> !Settings -> !Maybe Stats -> !LogLevel -> Text -> IO () -> MemberConfig
- Haskoin.Store.Fleet.Member: [abandoned] :: CacheMetrics -> !StatCounter
+ Haskoin.Store.Fleet.Member: [abandoned] :: Between -> ![TxData]
- Haskoin.Store.Fleet.Member: [waiting] :: Member -> !TVar (HashMap TxHash UnixTime)
+ Haskoin.Store.Fleet.Member: [waiting] :: Member -> !TVar (HashMap TxHash (UnixTime, Tx))
- Haskoin.Store.Fleet.Member: answeringCached :: Member -> ByteString -> (StoreRead -> StoreRead -> IO () -> IO a) -> IO (Maybe a)
+ Haskoin.Store.Fleet.Member: answeringCached :: Member -> ByteString -> (StoreRead -> Session -> IO () -> IO a) -> IO (Maybe a)
- Haskoin.Store.Fleet.Member: blocksBetween :: DatabaseReader -> Maybe BlockHash -> BlockHash -> IO (Maybe Between)
+ Haskoin.Store.Fleet.Member: blocksBetween :: Views BlockHash -> DatabaseReader -> Maybe BlockHash -> BlockHash -> IO (Maybe Between)
- Haskoin.Store.Fleet.Member: changeBetween :: DatabaseReader -> Maybe BlockHash -> BlockHash -> IO (Maybe PoolChange)
+ Haskoin.Store.Fleet.Member: changeBetween :: Views BlockHash -> DatabaseReader -> Maybe BlockHash -> BlockHash -> IO (Maybe PoolChange)
- Haskoin.Store.Fleet.Member: poolTx :: Network -> Ctx -> TxData -> PoolTx
+ Haskoin.Store.Fleet.Member: poolTx :: Network -> TxData -> PoolTx
- Haskoin.Store.Fleet.Member: wayBetween :: DatabaseReader -> Maybe BlockHash -> BlockHash -> IO Way
+ Haskoin.Store.Fleet.Member: wayBetween :: Views BlockHash -> DatabaseReader -> Maybe BlockHash -> BlockHash -> IO Way
- Haskoin.Store.Fleet.XPub: Entry :: ![XPubBal] -> !(Int, Int) -> HashMap Address [Word32] -> Entry
+ Haskoin.Store.Fleet.XPub: Entry :: ![XPubBal] -> !(Int, Int) -> HashMap Address [Word32] -> !ByteString -> !Bool -> Entry
- Haskoin.Store.Fleet.XPub: blockChanges :: Network -> Ctx -> [TxData] -> HashMap Address AddressChange
+ Haskoin.Store.Fleet.XPub: blockChanges :: Network -> [TxData] -> HashMap Address AddressChange
- Haskoin.Store.Fleet.XPub: cacheBuild :: Connection -> FleetKeys -> ByteString -> DatabaseReader -> PoolReads -> Word64 -> XPubSpec -> IO Built
+ Haskoin.Store.Fleet.XPub: cacheBuild :: Connection -> FleetKeys -> ByteString -> DatabaseReader -> PoolReads -> XPubSpec -> IO Built
- Haskoin.Store.Fleet.XPub: cachedCount :: Connection -> FleetKeys -> ByteString -> XPubSpec -> (Int, Int) -> IO (Cached Word32)
+ Haskoin.Store.Fleet.XPub: cachedCount :: Connection -> FleetKeys -> ByteString -> XPubSpec -> (Int, Int) -> IO (Cached (ByteString, Word32))
- Haskoin.Store.Fleet.XPub: cachedEntry :: Connection -> FleetKeys -> ByteString -> Word32 -> Word32 -> Word64 -> XPubSpec -> IO (Cached Entry)
+ Haskoin.Store.Fleet.XPub: cachedEntry :: Connection -> FleetKeys -> ByteString -> Word32 -> Word32 -> Bool -> XPubSpec -> IO (Cached Entry)
- Haskoin.Store.Fleet.XPub: cachedTxs :: Connection -> FleetKeys -> ByteString -> XPubSpec -> (Int, Int) -> From -> Limits -> IO (Cached [TxRef])
+ Haskoin.Store.Fleet.XPub: cachedTxs :: Connection -> FleetKeys -> ByteString -> XPubSpec -> (Int, Int) -> From -> Limits -> IO (Cached (ByteString, [TxRef]))
- Haskoin.Store.Fleet.XPub: cachedUnspents :: Connection -> FleetKeys -> ByteString -> XPubSpec -> Entry -> From -> Limits -> IO (Cached [XPubUnspent])
+ Haskoin.Store.Fleet.XPub: cachedUnspents :: Connection -> FleetKeys -> ByteString -> XPubSpec -> Entry -> From -> Limits -> IO (Cached (ByteString, [XPubUnspent]))
- Haskoin.Store.Manager: Store :: !PeerMgr -> !Chain -> !BlockStore -> !DatabaseReader -> !Maybe Cache -> !Maybe Member -> !Publisher StoreEvent -> !Publisher StoreEvent -> !Network -> !Ctx -> !SockAddr -> WithConnection -> Store
+ Haskoin.Store.Manager: Store :: !PeerMgr -> !Chain -> !BlockStore -> !DatabaseReader -> !Maybe Member -> !Publisher StoreEvent -> !Publisher StoreEvent -> !Network -> !SockAddr -> WithConnection -> Store
- Haskoin.Store.Manager: StoreConfig :: !Int -> ![String] -> !Bool -> !FilePath -> !Network -> !Maybe String -> !Ctx -> !Word32 -> !Word32 -> !Int -> !Int -> !Int -> !Bool -> !Bool -> !Bool -> !Bool -> !Int -> !Int -> !NominalDiffTime -> !NominalDiffTime -> !SockAddr -> WithConnection -> !Maybe Stats -> !Bool -> !Maybe Int -> !Maybe Int -> !Maybe Int -> !Maybe Int -> !Logger -> StoreConfig
+ Haskoin.Store.Manager: StoreConfig :: !Int -> ![String] -> !Bool -> !FilePath -> !Network -> !Maybe String -> !Word32 -> !Word32 -> !Int -> !XPubConfig -> !Int -> !Int -> !NominalDiffTime -> !NominalDiffTime -> !NominalDiffTime -> !NominalDiffTime -> !SockAddr -> WithConnection -> !Maybe Stats -> !Bool -> !Maybe Int -> !Maybe Int -> !Maybe Int -> !Maybe Int -> !Maybe Int -> !Logger -> StoreConfig
- Haskoin.Store.Web: WebConfig :: !String -> !Int -> !Store -> !Int -> !Int -> !Int -> !WebLimits -> !String -> !Bool -> !Maybe Stats -> !Int -> !String -> !String -> !Bool -> !Bool -> !Int -> !Bool -> !Maybe String -> !Maybe (FilePath, FilePath) -> !Logger -> WebConfig
+ Haskoin.Store.Web: WebConfig :: !String -> !Int -> !Store -> !Int -> !Int -> !Int -> !WebLimits -> !String -> !Maybe Stats -> !Int -> !String -> !String -> !Bool -> !Int -> !Bool -> !Maybe String -> !Maybe (FilePath, FilePath) -> !Logger -> WebConfig
- Haskoin.Store.Web: WebLimits :: !Word32 -> !Word32 -> !Word32 -> !Word32 -> !Word32 -> !Word32 -> !Word64 -> !Word64 -> !Word32 -> WebLimits
+ Haskoin.Store.Web: WebLimits :: !Word32 -> !Word32 -> !Word32 -> !Word32 -> !Word32 -> !Word32 -> !Word64 -> !Word64 -> !Word32 -> !Word32 -> !Word32 -> !Word32 -> !Word32 -> !Word32 -> WebLimits
- Haskoin.Store.Web.Events: command :: Ctx -> Word32 -> (XPubSpec -> IO [XPubBal]) -> TVar Watch -> EventCommand -> IO (Either String ())
+ Haskoin.Store.Web.Events: command :: Word32 -> (XPubSpec -> IO [XPubBal]) -> TVar Watch -> EventCommand -> IO (Either String ())
- Haskoin.Store.Web.Events: relevant :: Ctx -> Word32 -> Maybe Seen -> Watch -> (Bool, Watch)
+ Haskoin.Store.Web.Events: relevant :: Word32 -> Maybe Seen -> Watch -> (Bool, Watch)
- Haskoin.Store.Web.Events: seenOf :: Network -> Ctx -> TxData -> Seen
+ Haskoin.Store.Web.Events: seenOf :: Network -> TxData -> Seen
- Haskoin.Store.Web.Pieces: listPieces :: (b -> Encoding) -> [IO [b]] -> IO [Piece]
+ Haskoin.Store.Web.Pieces: listPieces :: (b -> Encoding) -> [(Int, IO [b])] -> IO [Piece]
- Haskoin.Store.Web.Pieces: pieced :: Int -> ([k] -> IO b) -> [k] -> [IO b]
+ Haskoin.Store.Web.Pieces: pieced :: Int -> ([k] -> IO b) -> [k] -> [(Int, IO b)]
- Haskoin.Store.Web.Router: Route :: !Method -> ![Either Text Text] -> ![Text] -> (Req -> IO Response) -> Route
+ Haskoin.Store.Web.Router: Route :: !Method -> ![Either Text Text] -> ![Text] -> (Req -> IO Response) -> !Cost -> Text -> Route
- Haskoin.Store.Web.Router: routerScoped :: [Route] -> (([Route] -> IO (Maybe ResponseReceived)) -> IO (Maybe ResponseReceived)) -> (Req -> Miss -> IO Response) -> Application
+ Haskoin.Store.Web.Router: routerScoped :: [Route] -> (Route -> ([Route] -> IO (Maybe ResponseReceived)) -> IO (Maybe ResponseReceived)) -> (Req -> Miss -> IO Response) -> Application

Files

CHANGELOG.md view
@@ -4,6 +4,320 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html) +## [1.16.0] - 2026-11-18++This release has one design (`docs/one-fleet.md`): Redis is required, every+store is a member of a fleet, the mempool is only in Redis, and every xpub is+answered from the fleet's cache.++### Added+- **Gauges of the latest block's import times** (`store.last_block_ms` and one per stage), which+  keep their value until the next block, so that charts show the right time at any zoom.+- **Admission control that adapts** (`Haskoin.Store.Web.Admission`,+  `Haskoin.Store.Web.Congestion`, `docs/congestion-control.md`). Only so many+  long requests (lists, whole blocks, xpubs) run at once, so that a busy store+  still answers its quick requests at once.+  - **The limit follows the cores.** A probe on each capability waits on an+    eventfd that a system thread rings every 10 ms, and measures how late a+    woken thread runs, less GHC's collector pauses. Every half second the+    controller cuts the limit by a quarter when that delay is over+    `--target-delay` (25 ms) and the cores are used at 90%, at most once a+    round trip (the smoothed time a request holds its slot). The delay+    alone does not cut: requests that wait for a slow disk make it jump+    while the cores have room. It grows along TCP CUBIC's curve, back toward+    the level of the last cut and then past it, while the delay is under the+    target, every slot was used and the cores, smoothed over two seconds,+    are used under 90%. The limit stays between `--min-requests` (two a+    core) and `--max-requests` (sixteen a core; 0 turns it all off).+  - **The line in front is managed by CoDel** (100 ms interval, 5 ms target;+    while overloaded, a request that waited past 10 ms is refused). It holds+    up to `--request-queue` (64) requests for up to `--request-wait` (1 s).+    It refuses nothing while the limit is free to rise (the cores under 90%,+    the limit under its ceiling): such a line waits for the limit to catch+    up, and refusing it cost clients that ask one after the other a second+    of `Retry-After` each. A refused request gets 429 `server-busy`+    with `Retry-After: 1`, which+    clients and nginx take as "come back later"; a load generator retried+    503s at once. A request in line sets no GHC timer: a sweep enforces the+    waits.+  - **A request for an xpub gives its slot back while it waits for the+    indexer** (`Admission.admittedAside`, `aside`), and takes one again+    without the line. The indexer bounds that work; holding the slot left+    plain requests waiting a second behind cold xpubs.+  - **Options:** `--max-requests` (`MAX_REQUESTS`), `--min-requests`+    (`MIN_REQUESTS`), `--target-delay` (`TARGET_DELAY`), `--request-queue`+    (`REQUEST_QUEUE`) and `--request-wait` (`REQUEST_WAIT`).+  - **Metrics:** `web.cap.limit`, `web.cap.delay_us`, `web.cap.lateness_us`,+    `web.cap.overloaded`, `web.cap.used`, `web.cap.round_trip_us`,+    `web.cap.sojourn`, `web.cap.refused_full`,+    `web.cap.refused_waited`, `web.cap.refused_codel`, `web.requests_waiting`,+    and for each long route `web.cap.<route>.running`, `.waited`, `.held` and+    `.refused`.+  - **`haskoin-store-bench admission`** compares no limit, a fixed limit and+    the adaptive one (`bench/Admission.hs`).+- **Xpubs are indexed once, in the background** (`docs/xpub-indexing.md`).+  - **`Haskoin.Store.XPub.Indexer`**: threads that read xpubs into the+    fleet's cache, one a core by default. Jobs that requests asked for go+    before upkeep, and no xpub is queued twice; a request for an xpub that is+    queued or running joins its job. A sweeper thread refuses the requests+    whose time is up, so no GHC timer is set.+  - **`Haskoin.Store.XPub.Lock`**: the lock of an xpub in Redis+    (`<prefix>idx:<id>`), so that one store of the fleet indexes it. Its value+    says which store holds it and when that store expects to be done, by the+    clock of Redis.+  - **`Haskoin.Store.XPub.Service`**: the one door to the cache for the web+    server. `xpubAsk` says what to do on a miss and gives an `Answer`;+    `xpubDelete` takes an xpub out.+  - **`Haskoin.Store.XPub`**: what the door, the indexer and the web server+    share, `XPubReads` and the `Budget` of a job. Every job gets `noBudget`;+    the type stays as the seam through which a job settles and tells its size.+  - **`StoreConfig.xpubs`** (`XPubConfig`, `defaultXPubConfig`) and the+    options `--xpub-workers` (`XPUB_WORKERS`, 0 for one a core), `--xpub-wait`+    (`XPUB_WAIT`, 10 seconds), `--xpub-horizon` (`XPUB_HORIZON`, 30 seconds)+    and `--xpub-queue` (`XPUB_QUEUE`, 1000).+  - **`web.reruns`** counts the requests answered again because an xpub that+    they read changed meanwhile.+- **Timings of the stages of a block import**, for StatsD: `store.block_prepare`,+  `store.block_prefetch`, `store.block_apply` and `store.block_write`, and the+  read ahead by kind of key, `store.block_read_unspents`,+  `store.block_read_spenders` and `store.block_read_balances` (both zero now).+  They say whether a slow block is the disk, RocksDB's cache or the CPU.+- **`haskoin-store-bench import`** times the import of four kinds of full+  block on a database of a million unspent outputs, and prints the stages and+  the size of the batch (`bench/Import.hs`, `docs/block-import.md`).+  **`haskoin-store-bench merge`** times reads of balances and spenders over+  merge records that RocksDB has not compacted yet (`bench/Merge.hs`).+- **`--background-jobs`** (`BACKGROUND_JOBS`): how many threads RocksDB may+  use for flushes and compactions, RocksDB's default unless set. More of them+  shorten a backlog of compactions when the disk and cores have room.+  `StoreConfig` has `backgroundJobs`. Needs `rocksdb-haskell-jprupp` 2.8.0.+- **In the library:**+  - `Haskoin.Store.Database.Writer`: `runWriterReport` gives a `BatchReport`+    with the `BlockTimes` of a block and the size of its batch; the+    `readAhead` functions fill a batch's read cache with multi-gets on all+    cores; `mergeBalance` and `mergeSpender` change a balance or a group of+    spenders without reading it; `withoutReadCache` reads the database every+    time.+  - `Haskoin.Store.Logic.prepareBlock` gives the hashes, addresses and sizes+    of a block.+  - `Haskoin.Store.Database.Table`: the hash tables of a batch, keyed by the+    hash that each key already holds.+  - `Haskoin.Store.Database.Merge`: the merge operators of the balances and+    the spenders.+  - `Haskoin.Store.Database.Migrate`: `recordsVersion`, and a `MigrationError`+    that says in words what is wrong with a version.+  - `Haskoin.Store.Database.Reader` exports `withReader`,+    `columnFamilyConfig`, `layeredFamilyConfig` and `mergeGuardFamily`, for+    tests and benchmarks that open a database with other settings.+  - `Haskoin.Store.Common.sortTxsWith` takes the hashes of the transactions+    from a caller that has them.+  - `Haskoin.Store.Exception`: `catchAny`, `handleAny` and `tryAny` catch any+    error but not what stops a thread from outside; `bracket`, `bracket_`,+    `finally` and `onException` run a cleanup that cannot be interrupted.+  - `Haskoin.Store.Integration.Temp`, in the `integration` library: temporary+    directories for the tests, the benchmark and the stress test.++### Changed+- **The fleet remembers what it saw, in Redis** (`fleet:seen`): the last+  100,000 transactions that the pool decided on, taken, refused as a+  conflict or let go unconfirmed, marked by the pool's script in the same+  step. No store fetches them again when a peer announces them, after a+  restart either; what a store holds unoffered it does not fetch again from+  its own peers. Each store kept its own set in memory before.+- **Indexes and filters stay in memory** with their open table files, outside+  the block cache, so data blocks can no longer evict them (about 0.34% of the+  database).+- **Redis is required, and every store is a member of a fleet**, even a+  fleet of one (`docs/one-fleet.md`). Without Redis, or behind the fleet tip,+  a store is unhealthy: `/health` answers 503, where it was 203 without Redis.+- **The mempool is only in Redis.** A store hands the transactions that its+  peers send to the pool and keeps no mempool of its own; RocksDB holds the+  chain alone. A reverted block's transactions leave RocksDB, and the tip move+  gives them back to the pool. `Haskoin.Store.Logic.Layered` is now+  `Haskoin.Store.Logic`, which imports and reverts blocks, and `Query.Pooled`+  has `chainRead` in place of `databaseRead`.+- **Migration:** a database of version 18, 19 or 20 has its unconfirmed+  transactions taken out of the chain's indices, and the pool gets them again+  from the peers. Nothing is written to the old mempool column families.+- **The tip move and the xpub cache are one Redis commit.** The member that+  moves the fleet tip writes the cache's part in pieces, marking the entries+  it touches (`cacheBegin`, `cachePieces`); one script then moves the pool,+  the fleet tip and the cache tip and stamps the entries (`fleetCommit`). A+  read of a marked entry waits up to two seconds, then has its request+  answered again.+- **A reorganisation never empties the cache.** It is undone address by+  address from the views at recent blocks (`tipMoves`). A store without a view+  of the old tip does not move the tip; only a restart of the pool empties the+  cache.+- **The pool fills again after Redis loses its data.** An epoch key+  (`fleet:epoch`) tells the stores; each asks its peers for their mempools+  again and, with `--node-rpc`, reads its node's, 2,000 transactions every two+  seconds. Xpub entries are built again on demand.+- **Fewer round trips to Redis.** The fleet's Lua scripts are loaded once and+  called by digest (`EVALSHA`), loaded again after `NOSCRIPT`. A request that+  reads nothing of the pool, such as `/block/best`, reuses a tip seen in the+  last 5 ms.+- **Every answer about an xpub comes from the fleet's cache**, never from the+  database. The routes `/xpub/{xpub}` (with `balances`, `transactions`,+  `transactions/full` and `unspent`) and the Blockchain.info routes+  `multiaddr`, `rawaddr`, `unspent`, `balance` and `export-history` read the+  cache, and on a miss wait for the one job that indexes the xpub. One store+  of the fleet builds each entry, holding its dibs in Redis with its+  estimate; the others wait for the commit, which Redis announces. Visible to+  clients:+  - a request that would wait longer than `--xpub-wait` (now ten seconds,+    from five) gets 429 `server-busy` with a `Retry-After` from the one+    estimate in Redis, the same on every store. The estimate of a running job+    comes from its size (`Budget.sized`) and the recent time per+    transaction. The job goes on unless the xpub is beyond `--xpub-horizon`+    or the queue is full. A request for several xpubs is refused with the+    longest wait;+  - a failed job, or an xpub evicted before the request read it twice, gives+    503 with `Retry-After: 1`;+  - every xpub is cached after its first query, small ones included;+  - `swagger.yaml` has the 429 and 503 on each of these routes.+- **A request reads one version of each xpub, never parts of two.** Every+  commit gives an entry a new version, and a read gives the version in the+  same script. A request that finds another version later is answered again+  from the start, before anything was sent (`web.reruns`). Responses built+  from transaction bodies take each transaction's confirmation and each+  output's spender from the entry at that version, read with the bodies in+  one round trip.+- **Lookahead addresses.** An entry holds a gap's worth of unused addresses+  after each chain's window, so a payment to a wallet's next address is in+  its entry as soon as the pool takes it.+- **Redis evicts xpubs by GreedyDual**, keeping longest those that would+  cost most to index again (`Haskoin.Store.XPub.greedyLua`). Entries record+  their cost in `fx:e:<id>:meta`, and `fx:prio` and `fx:level` replace+  `fx:lru`. `cachedEntry` of `Haskoin.Store.Fleet.XPub` takes whether the read+  is a use, and `cacheBuild` takes no time.+- **The filters of `/events`** ask the indexer for an xpub that the cache does+  not have, as the routes do. A refusal answers the stream or the opening of+  a websocket with 429 or 503, and closes an open websocket with 1013.+- **Options that no longer choose anything are gone:** `--redis` (`REDIS`),+  `--fleet` (`FLEET`), `--no-xpub` (`NO_XPUB`), `--no-mempool`+  (`NO_MEMPOOL`), `--sync-mempool` (`SYNC_MEMPOOL`), `--wipe-mempool`+  (`WIPE_MEMPOOL`), `--redis-min-addrs` and `--redis-min-txs`+  (`REDIS_MIN_ADDRS`, `REDIS_MIN_TXS`). `--redis-url` defaults to+  `redis://localhost:6379`. `--redis-max-xpubs` only sets the size of the+  cache, and the store refuses to start below 1. `nocache` is accepted from+  older clients and ignored.+- **Removed from the library:** the database mempool and its copy in memory+  (`Haskoin.Store.Database.Mempool`, `Haskoin.Store.Database.Copy`), the+  layered queries (`Haskoin.Store.Database.Query.Layered`), the Redis cache+  of a store alone (`Haskoin.Store.Cache`), `runWriterMempool`,+  `MempoolChanges`, `importBlockSequential`, `newMempoolTx`,+  `deleteUnconfirmedTx`, and the metrics `store.mempool` and+  `store.block_read_mempool`.+- **A block is imported in stages, several times faster.** A full block took+  a store of mainnet 13 to 18 seconds, with its disk and half its cores idle.+  Now the hashes, addresses and sizes are worked out once on all cores; every+  key the import needs is read ahead with multi-gets into the batch's cache,+  each key once; the rules run in memory; and the batch is serialised on all+  cores and written in one write. The database that comes out is the same,+  byte for byte, as the tests check against the import of 1.15.1.+- **The batch of a block is handed to RocksDB packed**, 256 KiB a call+  (`writePacked` of `rocksdb-haskell-jprupp` 2.8.0): some forty foreign calls+  for a block where there were 65,000. It is still one atomic write.+- **A mined block is not validated.** A block that passed proof of work is+  taken to have valid transactions, so the import reads only what it needs to+  write the block: no spender of the chain, and no output that the block+  itself makes. Only a coinbase is looked up in case it is confirmed already,+  as two of 2010 are. A block that is not valid is written whole or not at+  all; one that spends an output a block has spent is refused as an orphan.+- **Database version 23: a block changes balances and spenders without+  reading them.** It writes how they change as RocksDB merge records, which+  two merge operators that come with the store work out on reads and+  compactions (`Haskoin.Store.Database.Merge`, `cbits/haskoin_store_merge.c`).+  - **Upgrading from version 22 is one write**, made when the store first+    opens the database.+  - **There is no way back.** An older haskoin-store fails to open the+    database, with an error about the column family `merge-operators`, and+    leaves it as it was. Take a copy first if you may want to go back.+  - **Answers are the same.** A balance or spender that recent blocks changed+    costs a read a few microseconds more until RocksDB compacts it.+  - Reverting a block reads and writes whole values, as before, and still+    checks that a balance does not go below zero.+- **Every table file stays open by default** (`--max-open-files 0`, where it+  was 2000): with more files than the limit, a store of Bitcoin Cash opened+  some 70,000 files to send one block. Raise the process's limit on open files+  above the number of table files.+- **The index and the filter of a table file are written in 4 KiB pieces**,+  loaded as needed, where each was one block of up to megabytes. Files already+  on disk keep the old layout until RocksDB rewrites them; until then give+  the block cache about 1% of the database's size. Needs+  `rocksdb-haskell-jprupp` 2.8.0.+- **Lists are read from the database in chunks** (`rocksdb-query` 0.7.0):+  one foreign call for a few hundred entries, in constant stack, where there+  was one call for each entry. Those calls took some 28% of the processor+  time of a busy store.+- **Faster hashing and serialisation.** The hash tables of a batch hash keys+  by the hash they already hold, the block store no longer hashes a block's+  transactions again, and `stack.yaml` takes haskoin-core, haskoin-store-data+  and haskoin-node from commits that serialise on a buffer directly.+- **The heap collects less often.** The executable runs with `-A64m`, 64 MiB+  of new objects for each core between collections, where GHC's default is 4:+  about 60 MiB more memory a core for far fewer pauses. The collector is GHC's+  default again; `GHCRTS=--nonmoving-gc` selects the non-moving one of 1.15.1.+- **No secp256k1 context in the API of the library.** The store is built on+  haskoin-core 2.0.0, haskoin-store-data 3.0.0 and secp256k1-haskell 3.0.0,+  where the context is global and `Ctx` is gone. The server is the same;+  programs that embed the library must change:+  - delete `withContext`, `createContext` and every `ctx` passed to this+    package; the field `ctx` is gone from `StoreConfig`, `Store`,+    `BlockStoreConfig`, `StoreRead` and `DatabaseReader`, and `writerCtx` is+    removed;+  - `withLayeredReader`, `runWriter`, `deriveFunction`, `toUnspent`,+    `valToUnspent`, `poolTx`, `blockChanges`, `seenOf`, `relevant` and+    `command` lost their `Ctx` argument and kept the order of the others;+  - see the "Migrating" sections of the changelogs of haskoin-core and+    haskoin-store-data.+- **No unliftio.** The package uses `base`, `stm` and `async`, and is built on+  nqe 0.7.0 and statsd-rupp 0.6.0, whose functions now run in `IO`. No+  exported type changed. A program that calls nqe or statsd-rupp from another+  monad must lift those calls itself.+- **Peers that hold up blocks are passed over.**+  - A peer that delivers nothing for `--block-stall` seconds (ten by default)+    is passed over and the next fastest asked for the same blocks; the stall+    doubles for each peer passed over, up to the peer timeout.+  - Blocks are asked only from peers that announced them, or that gave a+    height at least that of the block when they connected.+  - The peers that have a block are pinged, and the first to answer is asked;+    one slower than `--block-ping` seconds (five by default) only if none+    answers in time.+  - The peer timeout (`--peer-timeout`) is 30 seconds by default, where it+    was 120. A block of 32 MB must arrive within it, which takes a link of+    9 Mbit/s: set it higher on a slower one.+  - `StoreConfig` and `BlockStoreConfig` have `blockStall` and `blockPing`,+    and the block store takes `blockStoreAnnounced` and `blockStorePong`.+- **`Haskoin.Store.Web.Router`:** a `Route` has a `cost`, light, short or+  long, and the scope of `routerScoped` is told which route matched.++### Fixed+- **`/events` can watch an xpub again.** Its filter read the cache through+  a session with no balances, so every xpub filter failed with 503 or close+  code 1013. It now reads in a session of its own.+- **A store whose last block is old still reports to its fleet.** Reports+  are gated on being in sync and having peers, not on the age of the last+  block, so a quiet chain such as regtest gets a fleet tip.+- **A large wallet is read into the cache ten times faster.** Reloads found+  the bodies to read with `nub`, which is quadratic: on a wallet of 45,000+  transactions that took 26 of the 28 seconds of its first reload, on 1.15+  too. They de-duplicate with a hash set now, in linear time. The addresses+  of a new transaction are no longer de-duplicated at all, as a set joins+  them anyway.+- **Requests no longer queue up behind each other in Warp's timeouts.**+  time-manager 0.3 edits GHC's one timer queue at every pause, resume and+  tickle of a connection's timeout, and under load a simple request waited+  some 40 ms with a core idle. time-manager 0.4.2 is pinned in `stack.yaml`+  and bounded in `package.yaml`.+- **Responses no longer wait for each other for their `Date` header.** Warp+  computed its date once a second, and every response waited when the thread+  computing it was stopped on a busy core. The store now gives every response+  a `Date` of its own, computed twice a second.+- **`store.block` reaches the StatsD server.** It was sent with a sampling+  rate of zero, which sends nothing; now with a rate of one.+ ## [1.15.1] - 2026-11-14  ### Added
README.md view
@@ -53,15 +53,17 @@ - **Index for the questions people ask.** Every index is laid out so that an   API query is one seek and a short scan in RocksDB, newest first, which keeps   response times flat as the chain grows.-- **Never half-write.** Each block, and each batch of mempool transactions, is-  one atomic database write. A crash leaves the store at the previous block,+- **Never half-write.** Each block is one atomic database write, and each+  change of the shared pool one Redis script. A crash leaves the store at the previous block,   with no repair step on restart.-- **Unconfirmed transactions matter.** The mempool is indexed and persisted like-  the chain, ordered before confirmed history, and handles replace-by-fee on-  Bitcoin. A wallet sees a payment the moment the network does.+- **Unconfirmed transactions matter.** The mempool is one pool in Redis that+  every store of a fleet shares, indexed by address, ordered before+  confirmed history, with replace-by-fee on Bitcoin. A wallet sees a+  payment the moment the network does. - **Extended public keys are first class.** Wallets are mostly xpubs, so the   store derives their addresses up to an exact gap, remembers the derivations,-  and can keep large wallets ready in a Redis shared by several stores.+  and keeps wallets ready in a cache in the Redis that the stores of a+  fleet share. - **Bounded resources.** API limits stop one request from monopolising the   server, slow event clients are dropped, and RocksDB memory has a budget. - **Operable.** A health endpoint that tells a load balancer when an instance is@@ -79,9 +81,9 @@ | --------------- | ------------------------------------------------------------------------------------------- | | Networks        | Bitcoin and Bitcoin Cash: mainnet, testnets and regtest                                     | | Addresses       | Balances, transactions and unspent outputs (UTXO) of any address, alone or many at once      |-| Xpubs           | Balances, transactions and UTXO of extended public keys, for legacy, nested SegWit and native SegWit addresses, with an optional Redis cache for large wallets |+| Xpubs           | Balances, transactions and UTXO of extended public keys, for legacy, nested SegWit and native SegWit addresses, from the fleet's cache in Redis | | Blocks and txs  | Lookup by hash, height or time; raw serialized forms; transactions of a block               |-| Mempool         | Persistent, indexed by arrival time, with replace-by-fee on BTC and pruning of old entries   |+| Mempool         | One pool in Redis shared by the fleet, by arrival time, with replace-by-fee on BTC and expiry | | Formats         | JSON                                                                                         | | Live events     | New blocks and transactions as a JSON stream or over a WebSocket                            | | Compatibility   | Endpoints in the style of Blockchain.info under `/blockchain/`                               |@@ -289,19 +291,18 @@  ### 7. Make big wallets fast -Start the store with `--redis` and a Redis (or Valkey) server.+Wallets are answered from the fleet's cache in Redis, which follows every+block and the shared mempool. -- **Large xpubs are cached** after their first query and answered from-  Redis, transactions in full included. Large means at least-  `--redis-min-addrs` used addresses, or at least `--redis-min-txs`-  transactions.-- **Several stores can share one Redis.** They keep each other's cache up-  to date.-- Add `nocache=true` to a query to read the database instead.+- **Every xpub is cached** on its first query. The query waits while one+  store of the fleet reads the wallet, ten seconds at most (`--xpub-wait`).+  If the wallet is not expected that soon, or the wait runs out, the store+  answers 429 with `Retry-After`, the same on every store: ask again then.+- **The cache keeps the costly ones longest**, up to `--redis-max-xpubs`.+  The cache cannot be turned off; that option sets only its size, at least+  1. There is no `nocache`: an answer from the cache is always exact. -Servers behind a load balancer should also be started with `--fleet`-([A fleet of servers](#a-fleet-of-servers)). They then answer every request-alike, and share a cache of xpubs that is built for that.+`docs/xpub-indexing.md` has the details.  ## Configuration @@ -318,7 +319,9 @@ | `--discover` | `DISCOVER` | off | Discover peers through DNS seeds and relayed addresses; on regtest it finds none, so name a `--peer` | | `-p`, `--peer` | `PEER` | none | Connect to `HOST[:PORT]`; repeat for more, space-separated in `PEER` | | `--max-peers` | `MAX_PEERS` | `20` | Most peers to connect to |-| `--peer-timeout` | `PEER_TIMEOUT` | `120` | Seconds before an unresponsive peer is dropped |+| `--peer-timeout` | `PEER_TIMEOUT` | `30` | Seconds before a peer that is silent, or that owes blocks, is dropped |+| `--block-stall` | `BLOCK_STALL` | `10` | Seconds without a block from the syncing peer before another peer is asked for it too |+| `--block-ping` | `BLOCK_PING` | `5` | Seconds that a peer has to answer a ping to be asked for blocks, unless none answers in time | | `--max-peer-life` | `MAX_PEER_LIFE` | `172800` | Seconds after which a peer is dropped, so connections rotate | | `--block-window` | `BLOCK_WINDOW` | `64` | Blocks to request from the syncing peer at a time | @@ -326,9 +329,6 @@  | Option | Environment | Default | Meaning | | --- | --- | --- | --- |-| `--no-mempool` | `NO_MEMPOOL` | off | Do not index unconfirmed transactions |-| `--sync-mempool` | `SYNC_MEMPOOL` | off | Ask each peer for its whole mempool once in sync |-| `--wipe-mempool` | `WIPE_MEMPOOL` | off | Empty the mempool area of the database at start | | `--mempool-timeout` | `MEMPOOL_TIMEOUT` | `14` | Days after which unconfirmed transactions are deleted (0 = never) |  ### Web server and limits@@ -338,13 +338,17 @@ | `--host` / `--port` | `HOST` / `PORT` | `*` / `3000` | Address the API listens on | | `--noweb` | `NO_WEB` | off | Index only, no API | | `--tls-cert` / `--tls-key` | `TLS_CERT` / `TLS_KEY` | none | Serve HTTPS with this certificate chain and private key, both files in PEM |-| `--no-xpub` | `NO_XPUB` | off | Disable the xpub endpoints | | `--no-blockchain-info` | `NO_BLOCKCHAIN_INFO` | off | Disable the `/blockchain/` endpoints and price lookups | | `--max-item-count` | `MAX_ITEM_COUNT` | `200000` | Most items in a list (0 = no limit) | | `--max-full-item-count` | `MAX_FULL_ITEM_COUNT` | `5000` | Most full transactions in a list (0 = no limit) | | `--max-offset` | `MAX_OFFSET` | `50000` | Largest `offset` (0 = no limit) | | `--slow-request` | `SLOW_REQUEST` | `5` | Seconds a request may take before the log warns of it (0 = never) |-| `--node-rpc` | `NODE_RPC` | none | JSON-RPC URL of a full node, such as `http://user:password@127.0.0.1:8332/`, to publish transactions through; without it, or when it does not answer, they go to the peers |+| `--max-requests` | `MAX_REQUESTS` | 16 a core | Most long requests (lists, whole blocks, xpubs) answered at once, the ceiling of a limit that adapts; see [Overload](#overload) (0 = no limit) |+| `--min-requests` | `MIN_REQUESTS` | 2 a core | Fewest long requests answered at once, where the limit starts |+| `--target-delay` | `TARGET_DELAY` | `25` | Milliseconds that a thread ready to run may wait for a core before the limit comes down |+| `--request-queue` | `REQUEST_QUEUE` | `64` | Long requests that may wait for a slot; more are refused with 429 |+| `--request-wait` | `REQUEST_WAIT` | `1` | Seconds that a long request may wait for a slot; then it is refused with 429 |+| `--node-rpc` | `NODE_RPC` | none | JSON-RPC URL of a full node, such as `http://user:password@127.0.0.1:8332/`, to publish transactions through, and to fill the shared mempool from again after Redis loses its data; without it, or when it does not answer, transactions go to the peers | | `--def-item-count` | `DEF_ITEM_COUNT` | `100` | Items in a list when no `limit` is given | | `--xpub-gap` | `XPUB_GAP` | `32` | Unused addresses kept after the last used one of each chain of an xpub; payments further away are not seen | | `--xpub-gap-init` | `XPUB_GAP_INIT` | `20` | Addresses looked at for an xpub with no transactions; an xpub with none used among them counts as unused |@@ -352,71 +356,42 @@ | `--price-history-url` | `PRICE_HISTORY_URL` | Blockchain.info price index | Source of historical prices | | `--ticker-refresh` | `TICKER_REFRESH` | `90000000` | Microseconds between price updates | -### Redis cache+### Cache of xpubs  | Option | Environment | Default | Meaning | | --- | --- | --- | --- |-| `--redis` | `REDIS` | off | Cache large xpubs in Redis |-| `--redis-url` | `REDIS_URL` | local Redis | Redis URL, such as `redis://host:6379` |-| `--redis-min-addrs` | `REDIS_MIN_ADDRS` | `100` | Cache xpubs with at least this many used addresses |-| `--redis-min-txs` | `REDIS_MIN_TXS` | `1000` | Or with at least this many transactions in their addresses |-| `--redis-max-xpubs` | `REDIS_MAX_XPUBS` | `10000` | Evict the least recently used xpubs past this many |--Every key of the cache starts with `hs2:<network>:`, so a Redis can be shared-by stores of several networks, and by stores of older versions, whose cache-uses other keys. `--redis-max-keys` and `--redis-sync-interval` of older-versions are gone.+| `--xpub-workers` | `XPUB_WORKERS` | `0` | Threads that read xpubs into the cache (0 = one per core) |+| `--xpub-wait` | `XPUB_WAIT` | `10` | Most seconds that a request waits for its xpub to be read; one that would wait longer gets 429 |+| `--xpub-horizon` | `XPUB_HORIZON` | `30` | Seconds: an xpub that the store expects to read later than this is not queued at all |+| `--xpub-queue` | `XPUB_QUEUE` | `1000` | Most xpubs that requests wait for in the queue; more are refused with 429 | -### A fleet of servers+### Redis: the fleet  | Option | Environment | Default | Meaning | | --- | --- | --- | --- |-| `--fleet` | `FLEET` | off | Answer as one of the servers that share the Redis of `--redis` |--Servers behind a load balancer import the same blocks and transactions at-slightly different times, so the same request can get different answers-depending on which of them it reaches. Start every one of them with-`--redis`, the same `--redis-url` and `--fleet`, and they all answer from-the same block and the same mempool:+| `--redis-url` | `REDIS_URL` | `redis://localhost:6379` | Redis of the fleet, which every store needs: its tip, mempool and xpub cache |+| `--redis-max-xpubs` | `REDIS_MAX_XPUBS` | `10000` | Size of the xpub cache, at least 1; evicted by cost to index again and by use | -- **The block** is the newest that every live server has. Each server-  imports blocks when it gets them and keeps its database as it was at each-  of its last six, and a request is answered from the block that the fleet-  agreed on. The fleet moves to a new block in one step, when the last-  server has it.-- **The mempool** is one pool in Redis, which every server offers its-  unconfirmed transactions to and all answer from. A transaction has one-  time of arrival there, so lists come in the same order everywhere.-- **A server that falls behind** answers 503 with `behind-fleet` rather-  than something that the others would not say, and stops counting after-  ten seconds without a report, so that it holds nobody up. A server whose-  health check fails stops reporting.-- **After a long outage** the fleet starts its shared mempool again. When-  the servers come back more than 50 blocks past the block that the fleet-  last answered from, the first of them moves the fleet there, empties-  the shared mempool, and all offer their unconfirmed transactions again.-  The log has a warning when it happens.-- **`/events` is the same stream on every server.** What changes the-  shared mempool or moves the fleet to a block writes an event to Redis-  in the same step, and every server sends those events, in that order,-  instead of what its own node saw. A block is told when the whole fleet-  can answer about it. Each server keeps the same last events, so a-  client that reconnects with `after=` may land on any of them.-- **Without Redis** each server answers from its own database, and-  `/health` says 203, as it does for the cache.-- **Large xpubs are cached for the whole fleet**, under the same-  `--redis-min-addrs`, `--redis-min-txs` and `--redis-max-xpubs` as the-  cache of a server alone. Set them the same on every server.-  - A cached answer is the one that the database gives, on every server.-  - The server that moves the fleet to a new block brings the cache along.-    In the moment before it has, and for a wallet that just gained or lost-    an address, requests are answered from the database.-  - A reorganisation empties the cache.-  - A request about a wallet of 464 addresses and 19,499 transactions-    takes 1.3 to 2.2 milliseconds with it, where it took 5 to 38.+Redis is how stores agree, and every store is a member of a fleet, even a+fleet of one. Every key starts with `hs2:<network>:`. -The rules, and the tests that hold Redis to them, are described in "A fleet-that answers as one" in `docs/layered-database.md`.+- **The block** is the newest that every live store has. A request is+  answered from it, from the store's view of its database at that block.+- **The mempool** is one pool in Redis, and only there: a store keeps no+  mempool of its own. Each store offers the transactions that its peers+  send it (`docs/fleet-mempool.md`).+- **A store that falls behind**, or loses Redis, answers 503 and reports+  unhealthy, and stops counting after ten seconds without a report, so+  that it holds nobody up.+- **When Redis loses its data**, the stores notice it by an epoch key that+  is gone or changed. Each forgets what it offered, reports unhealthy until+  the fleet has a tip again, and asks its peers for their mempools again;+  with `--node-rpc` it also reads the node's mempool (`getrawmempool`),+  2,000 transactions every two seconds. Xpub entries are built again as+  requests ask for them.+- **`/events` is the same stream on every store**, written to Redis with+  each change of the pool and each move of the tip.+- **Xpubs** are cached for the whole fleet (`docs/xpub-indexing.md`).  ### Health check @@ -446,20 +421,59 @@ | Option | Environment | Default | Meaning | | --- | --- | --- | --- | | `--no-bloom` | `BLOOM` | on | No Bloom filters on what is read by key; `BLOOM=false` is the same |-| `--max-open-files` | `MAX_OPEN_FILES` | `2000` | Open files; bounds index and filter memory (0 = unlimited) |+| `--max-open-files` | `MAX_OPEN_FILES` | `0` | Table files kept open (0 = all of them) | | `--write-buffer-mb` | `WRITE_BUFFER_MB` | `256` | Memtables across all column families (0 = unlimited) | | `--max-wal-mb` | `MAX_WAL_MB` | `1024` | Write-ahead log size that forces flushes (0 = RocksDB default) |-| `--block-cache-mb` | `BLOCK_CACHE_MB` | `1024` | Shared block cache, holding index and filter blocks too (0 = RocksDB default) |+| `--background-jobs` | `BACKGROUND_JOBS` | `0` | Threads for flushes and compactions together (0 = RocksDB default of two) |+| `--block-cache-mb` | `BLOCK_CACHE_MB` | `1024` | Shared block cache for data blocks; indexes and filters stay in memory outside it (0 = RocksDB default) |  Memory used by RocksDB is roughly the block cache plus the memtables, on top of the Haskell heap. `/dbstats` shows where it goes. +Every table file stays open by default, and the index and the filter of+each are written in pieces of 4 KiB that the block cache holds as they are+needed.++- **Why all files open:** a lookup in a file that is not open opens it+  first, which reads its footer and properties again. A store of Bitcoin+  Cash with 8,357 files and the old limit of 2,000 opened some 70,000 files+  to send one block of 166,739 transactions. An open file costs a file+  descriptor and a few kilobytes, so raise the limit on open files of the+  process (`LimitNOFILE` in systemd) above the number of table files:+  about one for each 64 MiB of database.+- **Indexes and filters stay in memory**, with their open files and+  outside the block cache, so that data blocks cannot push them out: when+  they were in the cache and a busy store evicted them, nearly every+  lookup read megabytes back, 0.8 ms of processor time each and 18 seconds+  to import a block of Bitcoin. They come to about 0.34% of the database:+  some 6 GB for Bitcoin and 1.6 GB for Bitcoin Cash. Keep all files open+  (above) so that they are read once, when the store starts, which takes+  longer on a large database. The block cache (`--block-cache-mb`) holds+  data blocks only.++RocksDB flushes and compacts in the background, with two threads by+default: one for each. `--background-jobs` gives it more, a quarter of them+for flushes and the rest for compactions.++- **When it helps:** a backlog of compactions on a machine whose disk and+  cores have room: the first sync, or the hours after a migration (see+  [Upgrading an existing database](#upgrading-an-existing-database)). One+  thread compacts about 90 MiB a second if the disk keeps up.+- **When it does not:** on a disk that is already the limit. Measured on a+  store of Bitcoin on a SATA SSD, the one compaction thread was on a core a+  third of the time and waiting for the disk the rest, with the machine at+  75% I/O wait. A second thread there would have made every request wait+  longer for the same disk.+- **What it costs:** cores and disk that the requests would otherwise have.+  Leave it at the default on a machine that serves traffic with all its+  cores.+ Bloom filters are on by default. They let RocksDB skip the table files that cannot have a key, which is most of what a lookup of a transaction, an output or a balance would otherwise read.  - **What has them:** the data that is read by key: transactions, spenders,-  unspent outputs, blocks, balances and the mempool. The two indices by+  unspent outputs, blocks and balances. The two indices by   address, which are scanned, do not. - **What they cost:** about 1.25 bytes for each key, held in the block   cache. On a chain with billions of keys that is gigabytes: size@@ -532,7 +546,7 @@   `POST /transactions` may be as long as the longest transaction of the   network in hexadecimal, 8 MB with SegWit and 2 MB on Bitcoin Cash. More   is answered 413 without being read whole.-- **A 503 says when to try again**, in `Retry-After`.+- **A 503 or a 429 says when to try again**, in `Retry-After`. - **HTTPS is optional and off by default.** With `--tls-cert` and   `--tls-key` the server speaks HTTPS on its port, and nothing else. A   public service usually has a proxy in front that holds the certificate;@@ -548,12 +562,12 @@ | Mempool | `/mempool` | | Addresses | `/address/:addr/{transactions,transactions/full,balance,unspent}` and the multi-address forms `/address/{transactions,transactions/full,balances,unspent}?addresses=…` | | Xpubs | `/xpub/:xpub`, `/xpub/:xpub/{transactions,transactions/full,balances,unspent}`, `DELETE /xpub/:xpub` (drop from cache); `derive=legacy\|compat\|segwit` picks the address type (`normal`, the old name of `legacy`, still works) |-| Network | `/peers`, `/health` (503 when unhealthy, 203 without its Redis), `/dbstats` |+| Network | `/peers`, `/health` (503 when unhealthy, Redis included), `/dbstats` | | Events | `/events`, as JSON lines over HTTP or as a WebSocket; `after=` resumes, and a WebSocket client can filter by address, xpub or Bloom filter | | Blockchain.info | `/blockchain/multiaddr`, `/blockchain/balance`, `/blockchain/rawaddr/:addr`, `/blockchain/unspent`, `/blockchain/rawtx/:txid`, `/blockchain/rawblock/:hash`, `/blockchain/latestblock`, `/blockchain/unconfirmed-transactions`, `/blockchain/export-history`, `/blockchain/q/…` | -Turn off the xpub endpoints with `--no-xpub` and the Blockchain.info ones-with `--no-blockchain-info`.+Turn off the Blockchain.info endpoints with `--no-blockchain-info`. The xpub+endpoints are always on: they answer from the fleet's cache in Redis.  ## How it works @@ -564,14 +578,108 @@ ```  - **haskoin-node** keeps a pool of peers and syncs block headers.-- The **block store** downloads blocks from one peer at a time and mempool-  transactions from all of them, and imports them. Each import is one-  atomic RocksDB batch, so a crash never leaves a half-written block.+- The **block store** downloads blocks from one peer at a time, and+  transactions from all of them, which it hands to the fleet's pool. Each+  block is one atomic RocksDB batch, so a crash never leaves a half-written block.+- **A block is imported in stages**, so that a full block takes a second+  or two: what it needs is worked out on all cores, everything it will+  read is read from RocksDB ahead of time, many keys at once, and then the+  rules run in memory. See [Importing a block](#importing-a-block).+- **The database holds the chain alone.** A transaction from a peer goes+  to the pool that the fleet shares in Redis, which is the mempool;+  RocksDB has no mempool of its own.+- **A peer is asked only for blocks that it has announced:** it sent the+  header of the block or of a later one, or gave a height at least that of+  the block when it connected. A peer asked for a block that it does not+  have yet answers nothing.+- **A peer is pinged before it is asked.** The peers that have the block+  are pinged at once, and the first to answer is asked. One that takes+  longer than `--block-ping` seconds is asked only if none answers in+  time. This costs each new block one round trip to the quickest peer.+- **A peer that does not deliver is passed over, not waited for.** If the+  peer that was asked sends nothing for `--block-stall` seconds, another+  is found the same way and asked for the same blocks, and whichever+  delivers first is taken. The stall doubles each time, up to+  `--peer-timeout`, so that a slow link still gets its blocks; a peer that+  still owes blocks after `--peer-timeout` is disconnected. - **Reorganisations** are handled by reverting blocks until the best block   is back on the main chain of headers.-- The **web server** answers from RocksDB, and from Redis for what is-  cached there.+- The **web server** answers from the view of RocksDB at the fleet tip,+  with the fleet's pool in Redis on top, and xpubs from their cache in+  Redis. +### Importing a block++A full block of Bitcoin, 4,000 transactions, took a store of mainnet 13 to+18 seconds to import, with its disk and half its cores idle. The time went+into some 30,000 reads of RocksDB, made one after another from one thread,+many of them of a key that had just been read.++Now a block goes through four stages:++1. **Prepare:** the hash of each transaction, the address of each output+   and the sizes, from the block alone, on all cores.+2. **Read ahead:** every key of the chain that the rules will read, with+   RocksDB multi-gets, eight at a time for each core, into a cache that+   belongs to the batch. Each key is read once.+3. **Apply:** the rules, one transaction after another, as before. They+   find everything in memory.+4. **Write:** one atomic batch, as before, serialised on all cores and+   handed to RocksDB a buffer of 256 KiB at a time.++The balances of the addresses of a block and the spenders of the outputs+that it spends are in none of the stages: since database version 23 they+are not read. The block writes how they change, as RocksDB merge records,+and RocksDB works out the values when they are read. On a store of mainnet+they were more than half of the read ahead.++- **A mined block is not validated.** It passed proof of work and the+  node took it, so the store takes it that every output it spends is+  unspent in the chain or made by the block itself, and that none is+  spent twice. The import reads the outputs that the block spends, which+  it needs to write the block, and nothing to check it: no spender of the+  chain and no output that the block itself makes. Only a coinbase is looked up+  in the chain in case it is confirmed already, as two of 2010 were. A+  block that is not valid is written whole or not at all, and is refused+  when what the import needs to read is not there.+- **What keeps the store's own data right stays:** the order in which+  transactions are applied. Reverts check what they always did.++Measured with `haskoin-store-bench import`, on four cores of an AMD Ryzen+9 3900X, with a database of a million unspent outputs on tmpfs and blocks+of 3,500 transactions, milliseconds a block:++| | 1.15.1 | 1.16.0 |+| --- | ---: | ---: |+| Syncing: the mempool is empty | 1,381 | 382 |+| At the tip: every transaction is in the mempool first | 2,321 | 841 |++Of the 382 ms, 36 prepare, 75 read ahead, 63 apply the rules and 208+write a batch of 65,410 keys and 6.6 MB.++That was with the batch given to RocksDB one key at a time. Packed into+buffers, its write took 86 ms where it took 134, in later runs of the same+benchmark in which the whole block went from 250 to 198 ms, and from 522+to 447 ms at the tip. Without the reads of balances and spenders, in a+third set of runs, the read ahead went from 56 to 21 ms and the block from+189 to 148 ms, and from 433 to 398 ms at the tip. Without the reads that+checked a mined block, in a later set, a block of chains of transactions+went from 161 to 138 ms. These were measured while the database still had+a mempool of its own, which it no longer has. `docs/block-import.md` has+the sets of numbers.++A store of mainnet gains more than the benchmark shows. In the benchmark+a read is 30 microseconds of CPU, as the files are in memory. On the+store of mainnet that this was made for, a read waits for a disk, which+gave five times more reads a second when asked for 32 at a time than one+at a time: its reads are slower than the benchmark's, and gain more from+being made together.++`docs/block-import.md` has the design, what was measured, and what is left.+`docs/block-import.md` says what could bring a block well below a+second, and what each step would do to reads.+The stages are timed for StatsD: see [StatsD](#statsd).+ ### Publishing  The store does not check scripts, so a transaction that a client posts@@ -610,37 +718,56 @@  ### The database -The chain and the mempool are kept apart.+RocksDB holds the chain alone. The mempool is the fleet's pool in Redis.  - **The chain's data is written once and is the same on every server** that   has the same blocks. A confirmed transaction has the time of its block.   What spends its outputs is under keys of its own, so its record never   changes.-- **Unconfirmed transactions live in an area of their own**, and are applied-  on top of the chain when a query is answered. Nothing the mempool does-  changes the chain's data.+- **Unconfirmed transactions live in Redis**, in the pool that the fleet+  shares, and are applied on top of the chain when a query is answered.+  A reverted block's transactions leave RocksDB; the tip move hands them+  back to the pool. - **A transaction that leaves the mempool unconfirmed is forgotten**: it was   replaced, double-spent by a block, or too old. The API answers 404 for   it, or 410, with the transaction that won, for one that a block replaced   in the last 2016 blocks.-- **Each request reads one snapshot** of the database, so everything in an-  answer is of the same moment.+- **Each request reads one snapshot** of the database, at the fleet tip, so+  everything in an answer is of the same moment. -`docs/layered-database.md` has the reasons, the measurements and the tests.+`docs/one-fleet.md` has the reasons, the measurements and the tests. The layout of the keys is described in `Haskoin.Store.Database.Types`.  ### Upgrading an existing database -This is version 22 of the database. The store migrates an older database+This is version 23 of the database. The store migrates an older database the first time it opens it. **There is no way back**: an older store-refuses the database afterwards, so take a copy first.+cannot open the database afterwards, so take a copy first.  | From | What the migration does | How long | | --- | --- | --- |-| 20 | Moves the mempool to its own area | Seconds to minutes |+| 18, 19 | Not migrated: sync again | |+| 20 | Takes the unconfirmed transactions out of the chain's indices; the pool gets them again from the peers | Seconds to minutes | | 21 | Reads every transaction and writes nearly every one again | Many hours for Bitcoin |+| 22 | Writes the new version | At once | -The second step is the long one. Nothing lists the records that need a+**From 22 to 23 nothing in the database changes, and it is still one+way.** From version 23 a block writes the balances of its addresses and+the spenders of the outputs that it spends as RocksDB merge records, which+only a store with the merge operators can read (see+[Importing a block](#importing-a-block)). An older haskoin-store does not+have them:++- **It fails to open the database**, with a RocksDB error that names a+  column family that it does not know, `merge-operators`. That column+  family is empty, and is there for this: without it, RocksDB would open+  the database for the older store and silently drop the merge records+  that were still in its write-ahead log.+- **The database is unchanged by the attempt**, and this version opens it+  as before.+- **To go back to an older version,** restore the copy, or sync again.++The step from 21 is the long one. Nothing lists the records that need a change, and nearly all do.  - **Time:** a database of Bitcoin has more than a billion transactions.@@ -649,9 +776,23 @@ - **Disk:** it needs room for a second copy of the transactions, until   RocksDB has compacted the first away. - **Interrupted,** it starts again and skips what is done.+- **Afterwards** RocksDB writes the spenders once more, in the background,+  while the store runs. The migration writes them in the order of their+  keys, so their files reach the last level without being compacted, and+  RocksDB compacts each such file by itself once the store starts. On a+  store of Bitcoin that was 196 GiB in about two hours on a SATA SSD, with+  the machine waiting for its disk most of that time: the store answered+  and imported blocks slowly until it was over. It needs no more room, and+  nothing has to be done about it. On a faster disk `--background-jobs`+  shortens it. - **Deleted transactions** that an old database still holds are left where   they are and ignored. Finding them would mean reading everything again. +What changed for clients in version 23: nothing in the answers. A balance+or the spenders of a transaction that recent blocks changed take a few+microseconds more to read until RocksDB has compacted them:+`docs/one-fleet.md` has the figures.+ What changed for clients in version 22:  - A confirmed transaction that the server first saw in its mempool used to@@ -679,37 +820,9 @@ - The extra reads happen only for wallets that look empty, and never on   networks without SegWit, which have one kind of address. -### The cache of a server alone--With `--redis`, the cache (`Haskoin.Store.Cache`) keeps large xpubs in-Redis: their balances, transactions, unspent outputs and the bodies of-their transactions. Without Redis, xpub queries read RocksDB directly.--- **It reads the database, it does not guess.** When a block, a mempool-  transaction or a reorganisation touches a cached xpub, the store reads-  what changed from RocksDB, with the same queries that answer uncached-  requests, and writes the differences to Redis in one Lua script.-- **An update costs the size of the change**, not of the xpub. Redis-  answers one client at a time, so this is what keeps it responsive.-- **The script writes nothing if the entry changed since it was read**, so-  an entry is always what the database says.-- **It reads the whole xpub when it cannot be sure** that the entry has-  every earlier change: when it first caches it, when the entry is more-  than a block behind, when an address enters or leaves the gap, when a-  balance does not add up, and every five to ten minutes in any case.-- **Stores that share a Redis** claim each change, so it is applied once. A-  store never replaces an entry that knows more than its own database, and-  a store that is catching up does no cache work.-- **A page of a cached xpub** is one round trip to Redis.--The Lua scripts are in `src/Haskoin/Store/Cache/lua/`, one file each,-compiled into the library. The ways an entry could still come to differ-from the database unnoticed, and what would rule them out, are in-`docs/cache-consistency.md`.- ### The cache of a fleet -A fleet (`--fleet`) has a cache of another kind+The fleet has a cache of xpubs (`Haskoin.Store.Fleet.XPub`), because its servers do not answer from their own databases but from one block and one shared mempool. @@ -722,43 +835,66 @@  The first follows from the chain, so any server writes the same. Nothing has to be claimed or repaired: the server that moves the fleet to a new-block applies what the block did, and a cache that was not brought along-is emptied. `docs/layered-database.md` has the design and the tests.+block applies what the block did, in the same Redis commit that moves the+tip and the pool. A big move is written in pieces under the entries'+version marks, so a request waits or is answered again and never sees half+of it. A reorganisation is undone address by address, from the store's+views of its database at recent blocks: it never empties the cache. +An entry also holds a gap's worth of addresses after the window, derived+ahead of time, so a payment to the next address of a wallet is in its+entry as soon as the pool takes it. `docs/one-fleet.md` has the design and+the tests.+ ### Where the code is  | Module | What it does | | --- | --- |-| `Haskoin.Store.BlockStore` | Downloads and imports blocks and mempool transactions |-| `Haskoin.Store.Logic.Layered` | The rules that import blocks and transactions |-| `Haskoin.Store.Database.Mempool` | The keys of the mempool area, and what it does to balances |-| `Haskoin.Store.Database.Query.Layered` | The queries over both areas, each on a snapshot |+| `Haskoin.Store.BlockStore` | Downloads and imports blocks, and hands transactions from peers to the fleet's pool |+| `Haskoin.Store.Logic` | The rules that import and revert blocks |+| `Haskoin.Store.Database.Writer` | A batch of writes, with its read cache and its reads ahead |+| `Haskoin.Store.Database.Table` | The hash tables of a batch, whose keys are hashed quickly |+| `Haskoin.Store.Database.Merge` | The merge operators of balances and spenders, which are C functions (`cbits/`) |+| `Haskoin.Store.Database.Query` | The queries of the chain's column families |+| `Haskoin.Store.Database.Query.Pooled` | The queries of the chain with the fleet's pool on top |+| `Haskoin.Store.Database.Views` | Views of the database at recent blocks, one of them the fleet tip | | `Haskoin.Store.Database.Slices` | Stored blocks and transactions read in part, for what an answer needs |-| `Haskoin.Store.Database.Migrate` | The migrations from versions 20 and 21 |-| `Haskoin.Store.Cache` | The Redis cache of a server alone |+| `Haskoin.Store.Database.Migrate` | The migrations from versions 20, 21 and 22 |+| `Haskoin.Store.XPub` | The vocabulary of the xpub cache: its reads and the budget of a job |+| `Haskoin.Store.XPub.Indexer` | The threads that read xpubs into the cache, and the queue of their jobs |+| `Haskoin.Store.XPub.Lock` | The lock in Redis that keeps two stores from reading the same xpub |+| `Haskoin.Store.XPub.Service` | The door to the fleet's cache of xpubs: a hit answers, a miss asks the indexer | | `Haskoin.Store.Fleet.Member`, `Haskoin.Store.Fleet.XPub` | A fleet, and its cache | | `Haskoin.Store.Web`, `Haskoin.Store.Web.Events` | The web server, and `/events` | | `Haskoin.Store.Web.Pieces` | Answers that are sent while they are read |+| `Haskoin.Store.Web.Admission` | The limit on long requests answered at once, and the line in front of it (CoDel) |+| `Haskoin.Store.Web.Congestion` | The delay of GHC's scheduler, and the controller that moves the limit with it | | `Haskoin.Store.Runtime` | GHC's garbage collector, as StatsD metrics |+| `Haskoin.Store.Exception` | Handlers that let a thread be stopped, and cleanups that cannot be cut short | | `Haskoin.Store.Web.Publish` | Checking a transaction and handing it to the peers, or to a full node over RPC | -The rules and queries of version 20 (`Haskoin.Store.Logic`,-`Haskoin.Store.Database.Query`) stay in the library as a reference. The-tests hold the new ones to their results, operation by operation and page-by page. Nothing in the store calls them.+The tests hold the cache to the queries of `Query.Pooled` at the fleet tip+with the pool: every cached entry equals what they give, after random+histories of payments, blocks, reorganisations and builds.  ## Monitoring  ### Health +A store is unhealthy when Redis does not answer within two seconds, when+it has no view of the fleet tip, or when its lease in the fleet has+lapsed: it then knows nothing of the mempool, answers no xpub query, and+answers 503. A load balancer should stop sending it requests until+`/health` passes again. A store that syncs from scratch is unhealthy until+it reaches the fleet tip.+ `/health` returns the result of the latest check as JSON. Point your load balancer at it.  | Status | Meaning | | --- | --- | | 200 | Every check passes |-| 203 | Healthy, but its Redis does not answer |-| 503 | A check fails; the body says which |+| 503 | A check fails, Redis included; the body says which |  The check runs every second, and at once when a block arrives, so what `/health` answers is at most a second old. Redis is asked every five@@ -766,23 +902,67 @@  The checks are: how far the index is behind the headers, the age of the last block and of the last transaction, the transactions waiting to be-imported, and the connected peers.--**Without its Redis**, a store that has a cache goes on answering from its-own database:--- `/health` says 203, with the header `Haskoin-Degraded: redis` and-  `"redis": {"ok": false}` in its JSON.-- A request waits at most two seconds for Redis before the store gives up-  on it. The store tries Redis again at every health check.-- Load balancers take 203 for healthy, so the store stays in rotation.-  Alert on anything but 200 to hear of it.+imported, the connected peers, and Redis (`"redis"` in the JSON). -**In a fleet** (`--fleet`), a server that does not have the block that the+**In a fleet**, a server that does not have the block that the fleet answers from can answer any request with 503 and `{"error": "behind-fleet"}`. Have the load balancer try another server, and alert if one keeps saying it: its `/health` will say why. +### Overload++A store asked for more than its cores can do still answers the simple+requests at once. Only so many long requests run at once, and the limit+follows what the cores keep up with.+[docs/congestion-control.md](docs/congestion-control.md) explains the+design.++- **What is limited:** the long routes, those that answer lists, whole+  blocks and xpubs. Each holds a slot until its last byte is sent.+  Requests for one thing by its key, such as a transaction or a balance,+  and those that read nothing from the database, such as `/health`,+  `/peers` and `/events`, are not limited.+- **The limit adapts.** A probe on each core notes how late a woken+  thread runs, less the pauses of GHC's collector. Every half second, if+  the slowest tenth of those delays was over `--target-delay` (25 ms)+  and the cores were used at 90%, the limit comes down by a quarter, at most once in the time a request+  holds its slot. If it was under, every slot was in use and the cores+  were used under 90%, the limit grows as TCP CUBIC's window does: fast+  back toward the level of the last cut, then past it. It stays between+  `--min-requests` (two a core) and `--max-requests` (sixteen a core).+  Requests that wait for the disk give their cores back, so the limit+  grows for them; busy cores stop it growing.+- **A request for an xpub that is not cached** gives its slot back while+  it waits for the indexer, which bounds that work itself.+- **The line** in front of the slots holds up to `--request-queue` (64)+  requests for up to `--request-wait` (1 second). It is managed by CoDel:+  when even the shortest wait in 100 ms was over 5 ms, the line is+  overloaded, and a request that finds no slot free is refused at once.+  While the cores have room and the limit is under its ceiling, the line+  waits for the limit to grow instead.+- **A refused request** gets 429, `{"error": "server-busy"}` and+  `Retry-After: 1`, as nginx's `limit_conn_status 429` does: clients and+  proxies take it to mean "come back later", where some retry a 503 at+  once. Have the load balancer try another server on a 429 (with nginx,+  `proxy_next_upstream error timeout http_429`).+- **Xpubs:** reading an xpub into the cache is the work of the indexer,+  in threads of its own, not of a request. A request that would wait for+  the indexer too long gets 429, with when to come back+  ([Extended public keys](#extended-public-keys)).+- **Behind nginx:**+  - **Keep connections to the store open:** `keepalive` in the+    `upstream`, with `keepalive_timeout` under Warp's idle timeout of 30+    seconds, and `proxy_http_version 1.1` with+    `proxy_set_header Connection ""` in the `location`. A new connection+    for each request makes the slowest requests much slower under load.+  - **Keep clients apart in the proxy.** The store does not tell clients+    apart. `limit_conn` per client address and `limit_req` do.+- **Off:** `--max-requests 0` answers every request as it comes.+- **Watch** `web.cap.limit`, `web.cap.delay_us` and the refusals in+  StatsD ([StatsD](#statsd)). A limit stuck at the ceiling with long waits+  points at the disk; one stuck at the floor with a high delay points at+  work that is not limited.+ ### StatsD  With `--statsd`, the store reports:@@ -790,31 +970,60 @@ | Prefix | What | | --- | --- | | `web.*` | Timing of every kind of request |+| `web.reruns` | Requests answered again because a cached xpub that they read changed meanwhile | | `db.*` | Database reads |-| `cache.*` | Cache hits, misses, reloads and full reloads |-| `store.*` | Block and header heights, mempool size and other gauges |+| `store.*` | Block and header heights and other gauges | | `fleet.cache.*` | What a server of a fleet does with the fleet's cache of xpubs | | `rts.*` | GHC's garbage collector | +The metrics of the block store:++| Metric | Kind | Meaning |+| --- | --- | --- |+| `store.blocks` | gauge | Height of the best block |+| `store.headers` | gauge | Height of the best header |+| `store.peers` | gauge | Peers online |+| `store.queued_txs` | gauge | Transactions waiting to be imported |+| `store.block` | timing | Milliseconds to import a block, with publishing its events |+| `store.block_prepare` | timing | Of those, working out what the import needs of the block |+| `store.block_prefetch` | timing | Reading ahead what the rules will read |+| `store.block_read_unspents` | timing | Of the read ahead: the unspent outputs that the block spends |+| `store.block_read_spenders` | timing | Zero: a block no longer reads spenders, as a mined block is not validated |+| `store.block_read_balances` | timing | Zero since database version 23: a block no longer reads balances |+| `store.block_apply` | timing | The rules, in memory |+| `store.block_write` | timing | Writing the batch to RocksDB |+| `store.last_block_ms`, `store.last_block_prepare_ms`, `store.last_block_prefetch_ms`, `store.last_block_apply_ms`, `store.last_block_write_ms` | gauge | The same times of the latest block, kept until the next one, so that a chart averaging seconds shows them right at any zoom |++- **The four stages add up to nearly all of `store.block`.** A slow+  `store.block_prefetch` is the disk or RocksDB's cache, and the four+  `store.block_read_*` timings say which kind of key it waited for. The+  other three stages are mostly CPU: the write too, which serialises the+  batch and has RocksDB add it to its tables in memory.+- **Every sample of these timings is sent**: there is one a block. Until+  version 1.15.1 `store.block` was made with a sampling rate of zero and+  never reached the StatsD server.+ The metrics of the fleet's cache:  | Metric | Kind | Meaning | | --- | --- | --- | | `fleet.cache.hits` | counter | Requests about an xpub whose balances the cache gave |-| `fleet.cache.misses` | counter | Requests about an xpub that the database answered |+| `fleet.cache.misses` | counter | Requests about an xpub that the cache did not have | | `fleet.cache.builds` | counter | Entries that this server built, or added addresses to | | `fleet.cache.build` | timing | Milliseconds to build an entry | | `fleet.cache.abandoned` | counter | Builds given up because the fleet or the entry moved on |-| `fleet.cache.evictions` | counter | Entries deleted for being the least recently used |+| `fleet.cache.evictions` | counter | Entries evicted, those of the lowest priority first (GreedyDual) | | `fleet.cache.emptied` | counter | Times that this server emptied the cache | | `fleet.cache.xpubs` | gauge | Xpubs in the cache, whichever server cached them | | `fleet.cache.addresses` | gauge | Addresses that those xpubs have between them | -- **Misses include small xpubs**, which are never cached. So the share of-  hits says how much of the load the cache takes, not how well it holds-  what it has.-- **`fleet.cache.emptied`** should move only at a reorganisation or when a-  server stops at the wrong moment. The log says why each time.+- **Misses include xpubs being built**, and those built for other+  addresses than they have now. So the share of hits says how much of the+  load the cache takes, not how well it holds what it has.+- **`fleet.cache.emptied`** moves only when the pool starts again: the way+  to the new tip is longer than the stores go through, or a store that is+  alone does not know it. A reorganisation never empties the cache. The+  log says why each time. - With `--debug` the log also has a line for each entry built and each   block that the cache was brought to. @@ -840,11 +1049,39 @@ - **The executable runs with `+RTS -T`**, which GHC needs to keep these   statistics. Without it, as in a program that uses the library, the log   says so and there are no `rts.*` metrics.-- **The `rts.nonmoving_*` metrics** are the work of the non-moving-  collector, which the executable uses for the old generation (see-  [Memory](#memory)). They stay at zero if the store runs with the copying-  collector instead.+- **The `rts.nonmoving_*` metrics** stay at zero unless the store runs+  with GHC's non-moving collector, as with `GHCRTS=--nonmoving-gc` in its+  environment. The executable uses GHC's default collector (see+  [Memory](#memory)). +The metrics of admission control ([Overload](#overload)):++| Metric | Kind | Meaning |+| --- | --- | --- |+| `web.cap.limit` | gauge | The limit on long requests, every half second |+| `web.cap.delay_us` | gauge | 90th percentile of the scheduler's delay in the last half second, without the collector's pauses, in microseconds |+| `web.cap.lateness_us` | gauge | The same with the collector's pauses |+| `web.cap.used` | gauge | Use of the cores that the controller sees, smoothed over two seconds, in hundredths of the capabilities |+| `web.cap.round_trip_us` | gauge | Time that a long request holds its slot, smoothed, in microseconds: the controller cuts at most once in it |+| `web.cap.overloaded` | gauge | 1 while CoDel finds the line overloaded |+| `web.cap.sojourn` | timing | Milliseconds that an admitted long request waited for its slot |+| `web.cap.refused_full` | counter | Requests refused because the line was full |+| `web.cap.refused_waited` | counter | Requests refused after `--request-wait` |+| `web.cap.refused_codel` | counter | Requests refused by CoDel |+| `web.requests_refused` | counter | Requests refused with `server-busy`, for any reason |+| `web.requests_running` | gauge | Long requests being answered |+| `web.requests_waiting` | gauge | Long requests waiting for a slot |++And for each long route, named by the fixed segments of its path, such as+`address_transactions_full` or `blockchain_multiaddr`:++| Metric | Kind | Meaning |+| --- | --- | --- |+| `web.cap.<route>.running` | gauge | Slots that it holds |+| `web.cap.<route>.waited` | timing | Milliseconds that a request waited for its slot |+| `web.cap.<route>.held` | timing | Milliseconds that a request held its slot, to the last byte |+| `web.cap.<route>.refused` | counter | Requests refused |+ ### Database statistics  `/dbstats` returns the memory that RocksDB uses for each column family, and@@ -861,21 +1098,21 @@   `--disable-delayed-os-memory-return` for that. Without the flag, Linux   keeps counting returned memory as resident until it needs it, and the   heap looks as if it never shrinks after a peak.-- **The old generation is collected while the store runs**, with GHC's-  non-moving collector (`--nonmoving-gc`). The default copying collector-  stops every thread while it copies all live data, which takes longer as-  the heap grows. A store of mainnet holds gigabytes, and under load every-  request waited for it, `/health` included. Small collections of new-  objects still stop every thread, but they are short.-- **The price is memory.** Old objects are not moved together, so the heap-  can hold more than the copying collector would for the same live data.-  To go back to the copying collector, run the store with-  `GHCRTS=--copying-gc` in its environment, or `+RTS --copying-gc -RTS`.+- **The heap collects less often**, with 64 MiB of new objects for each+  core between collections (`-A64m`), where GHC's default is 4. Every+  collection stops every thread. Measured under load on a store of mainnet+  and one of testnet, collections fell from 50 to 110 a second to 2 to 4,+  the time that the stores spent stopped fell by half (11% to 6%, and 24%+  to 14%), and the testnet store answered 19% more requests.+- **The price** is about 60 MiB more memory for each core, and pauses of+  about 30 ms where they were 2 to 4 ms. To change it, set for example+  `GHCRTS=-A16m` in the environment of the store.+- **The collector is GHC's default**, which the executable does not+  choose: in GHC 9.12 it copies. Version 1.15.1 chose the non-moving+  collector; in the same measurements it gained nothing, and with `-A64m`+  the copying one did a little better. `GHCRTS=--nonmoving-gc` selects it. - If you build the store some other way, pass-  `+RTS --nonmoving-gc --disable-delayed-os-memory-return -RTS` yourself.-- **To tell such memory from a leak**, see-  `stress/results/2026-11-09-memory/README.md`: how to read `smaps`, and a-  heap profile of a busy store, with live data flat at about 10 MiB.+  `+RTS -A64m --disable-delayed-os-memory-return -RTS` yourself.  ### Logs @@ -919,14 +1156,14 @@  ```haskell import Data.Default (def)-import Haskoin (btc, textToAddr, withContext)+import Haskoin (btc, textToAddr) import Haskoin.Store.WebClient  main :: IO ()-main = withContext $ \ctx -> do+main = do   let cfg = def {net = btc} -- https://api.haskoin.com/btc/...       Just addr = textToAddr btc "bc1q…"-  r <- apiCall ctx cfg (GetAddrBalance addr)+  r <- apiCall cfg (GetAddrBalance addr)   print r ``` @@ -941,9 +1178,9 @@  - `withStore` starts the node, the block store and the cache from a   `StoreConfig`.-- The functions of `Haskoin.Store.Database.Query.Layered` query its-  database in `IO`, on a view taken with `withView`, such as-  `withView store.db (\v -> getBalance v addr)`.+- The functions of `Haskoin.Store.Database.Query` query the chain in its+  database in `IO`, such as `getBalance store.db addr`, and `chainRead`+  gives them as a `StoreRead`. - Subscribe to `Store.pub` to receive `StoreEvent`s.  Every module has Haddock documentation; build it with `stack haddock`.@@ -970,6 +1207,8 @@ stack test                     # includes a 6000-block heap-growth test scripts/format                 # ormolu, the code style used here scripts/bench-web.sh           # load-test the API with wrk+stack bench haskoin-store:bench:haskoin-store-bench \+  --ba 'import +RTS -N4 -A64m' # time the import of full blocks scripts/stress-cache.sh        # stress the Redis cache, with a report scripts/check-local-deps.sh    # what CI checks: no local paths in stack.yaml scripts/install-system-libraries.sh  # the C libraries, on Ubuntu or Debian@@ -1042,10 +1281,6 @@ - No container is left behind, even if the script is interrupted or   killed: the program removes its containers when it ends, and the script   removes any that carry its label and stops the server.-- Results of past runs are kept in `stress/results/<date>/` for comparison:-  the report, the measurements (without the server log), and a README with-  the commits, the machine and what the run found. A new run worth keeping-  goes there too. - To look for a leak, run the server with a heap profile by type, which needs   no profiling build, and summarise it: the live heap over time and the types   of closure that grew most.@@ -1056,24 +1291,33 @@   scripts/heap-profile-summary.py /tmp/heap.hp   ``` -### Measurements for the layered database+### Measurements for importing blocks -`docs/layered-database/` has the programs behind the numbers in-`docs/layered-database.md`, with their output. They are not part of the-package; each says in its header how to build and run it.+`haskoin-store-bench import [OUTPUTS [TXS [BLOCKS [CACHE_MIB [DIR]]]]]`+(`bench/Import.hs`) times the import of full blocks, in stages with the+reads made ahead, and with every read made when the rules come to it. -- `MigrateBench.hs` times the migration to the layered layout against-  wiping the same mempool.-- `QueryCost.hs` writes the same synthetic history into a database of each-  layout, with each set of rules, and times both: importing blocks and-  mempool transactions, and every kind of query, for addresses and wallets-  that have nothing, a little and much in the mempool. It checks that both-  databases give the same answers.-- `FleetCost.hs` asks one store the same questions alone and as a member-  of a fleet, with its mempool in a pool in Redis, checks that the answers-  are the same, and times both. It asks about wallets a third way too,-  through the fleet's cache of xpubs.+1. It builds a regtest database with a million unspent outputs at half a+   million addresses, closes it so that RocksDB writes it to table files,+   and copies it.+2. It imports the same blocks of 3,500 transactions into each copy, five+   of each of two kinds:+   - with inputs that spend outputs spread over the set;+   - with chains of transactions in the block, so that three inputs in+     four spend an output of the same block, as on Bitcoin mainnet today.+3. It prints milliseconds a block for each stage, and the number of keys+   and bytes of the batch of a block. +Give it the cores of the machine you care about with `+RTS -N4`, and the+allocation area of the store with `-A64m`.++`haskoin-store-bench merge [KEYS [READS [CACHE_MIB [DIR]]]]`+(`bench/Merge.hs`) times what the merge records of blocks cost the readers+of the database: the balance of an address, the spender of an output and+the spenders of a transaction, for keys with no merge record, one, three+and six, each in a table file of its own, and then the same keys once+compacted.+ ### Contributing  Pull requests are welcome.@@ -1081,8 +1325,15 @@ - Format the code with `scripts/format` (Ormolu), and keep every line of   code shorter than 120 columns. - Put scripts in files of their own rather than embedding them: Lua for-  Redis under `src/Haskoin/Store/Cache/lua/`, shell under `scripts/`, called+  Redis under `src/Haskoin/Store/Fleet/lua/`, shell under `scripts/`, called   from CI rather than written into the workflow.+- Catch any error with `catchAny`, `handleAny` or `tryAny` of+  `Haskoin.Store.Exception`, and clean up with its `bracket`, `finally` and+  `onException`, not with those of `Control.Exception`. A request that+  times out and a loop that is cancelled are stopped by an exception: a+  handler for `SomeException` would take it for an error and go on, and a+  cleanup of `Control.Exception` can be cut short, with a snapshot not+  released or a gauge left too high. The module says more. - Add tests for the behaviour you change. - Describe the change under a new version in `CHANGELOG.md`, which follows   [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
app/Main.hs view
@@ -4,6 +4,8 @@  module Main (main) where +import Control.Concurrent (getNumCapabilities)+import Control.Concurrent.Async (withAsync) import Control.Monad (when) import Data.Char (toLower) import Data.Default (Default (..))@@ -15,7 +17,6 @@   ( Network (..),     allNets,     netByName,-    withContext,   ) import Haskoin.Node   ( LogLevel (..),@@ -29,11 +30,14 @@     StoreConfig (..),     WebConfig (..),     WebLimits (..),+    XPubConfig (..),+    defaultXPubConfig,     runWeb,     withStore,   ) import Haskoin.Store.Common (loopForever) import Haskoin.Store.Runtime (runtimeMetrics)+import Haskoin.Store.XPub.Indexer (Settings (..)) import NQE (receive, withSubscription) import Options.Applicative   ( Mod,@@ -58,6 +62,8 @@     switch,     value,   )+import System.Directory (createDirectoryIfMissing, getAppUserDataDirectory)+import System.Environment (lookupEnv) import System.Exit (exitSuccess) import System.FilePath ((</>)) import System.Metrics.StatsD@@ -72,12 +78,6 @@     withStats,   ) import Text.Read (readMaybe)-import UnliftIO.Directory-  ( createDirectoryIfMissing,-    getAppUserDataDirectory,-  )-import UnliftIO.Async (withAsync)-import UnliftIO.Environment (lookupEnv)  -- | Version of the package, set at build time. haskoinStoreVersion :: String@@ -104,17 +104,16 @@     maxLaggingBlocks :: !Int,     minPeers :: !Int,     webLimits :: !WebLimits,-    redis :: !Bool,-    fleet :: !Bool,     redisURL :: !String,-    redisMinAddrs :: !Int,-    redisMinTxs :: !Int,     redisMaxXPubs :: !Int,-    noMempool :: !Bool,-    wipeMempool :: !Bool,-    syncMempool :: !Bool,+    xpubWorkers :: !Int,+    xpubWait :: !Int,+    xpubHorizon :: !Int,+    xpubQueue :: !Int,     blockWindow :: !Int,     peerTimeout :: !Int,+    blockStall :: !Int,+    blockPing :: !Int,     maxPeerLife :: !Int,     mempoolTimeout :: !Int,     maxPeers :: !Int,@@ -128,7 +127,6 @@     tickerURL :: !String,     priceHistoryURL :: !String,     noBlockchainInfo :: !Bool,-    noXPub :: !Bool,     healthCheckInterval :: !Int,     accessLog :: !Bool,     nodeRpc :: !String,@@ -139,6 +137,7 @@     maxOpenFiles :: !Int,     writeBufferMB :: !Int,     maxWalMB :: !Int,+    backgroundJobs :: !Int,     blockCacheMB :: !Int   } @@ -173,30 +172,28 @@     env "MIN_PEERS" 1 readMaybe   webLimits <-     getWebLimits-  redis <--    env "REDIS" False parseBool-  fleet <--    env "FLEET" False parseBool   redisURL <--    env "REDIS_URL" "" pure-  redisMinAddrs <--    env "REDIS_MIN_ADDRS" 100 readMaybe-  redisMinTxs <--    env "REDIS_MIN_TXS" 1000 readMaybe+    env "REDIS_URL" "redis://localhost:6379" pure   redisMaxXPubs <-     env "REDIS_MAX_XPUBS" 10000 readMaybe-  noMempool <--    env "NO_MEMPOOL" False parseBool-  wipeMempool <--    env "WIPE_MEMPOOL" False parseBool-  syncMempool <--    env "SYNC_MEMPOOL" False parseBool+  xpubWorkers <-+    env "XPUB_WORKERS" 0 readMaybe+  xpubWait <-+    env "XPUB_WAIT" 10 readMaybe+  xpubHorizon <-+    env "XPUB_HORIZON" 30 readMaybe+  xpubQueue <-+    env "XPUB_QUEUE" 1000 readMaybe   blockWindow <-     env "BLOCK_WINDOW" 64 readMaybe   mempoolTimeout <-     env "MEMPOOL_TIMEOUT" 14 readMaybe   peerTimeout <--    env "PEER_TIMEOUT" 120 readMaybe+    env "PEER_TIMEOUT" 30 readMaybe+  blockStall <-+    env "BLOCK_STALL" 10 readMaybe+  blockPing <-+    env "BLOCK_PING" 5 readMaybe   maxPeerLife <-     env "MAX_PEER_LIFE" (48 * 3600) readMaybe   maxPeers <-@@ -221,8 +218,6 @@     env "PRICE_HISTORY_URL" priceHistoryString pure   noBlockchainInfo <-     env "NO_BLOCKCHAIN_INFO" False parseBool-  noXPub <--    env "NO_XPUB" False parseBool   healthCheckInterval <-     env "HEALTH_CHECK_INTERVAL" 1 readMaybe   accessLog <-@@ -238,11 +233,13 @@   tlsKey <-     env "TLS_KEY" "" pure   maxOpenFiles <--    env "MAX_OPEN_FILES" 2000 readMaybe+    env "MAX_OPEN_FILES" 0 readMaybe   writeBufferMB <-     env "WRITE_BUFFER_MB" 256 readMaybe   maxWalMB <-     env "MAX_WAL_MB" 1024 readMaybe+  backgroundJobs <-+    env "BACKGROUND_JOBS" 0 readMaybe   blockCacheMB <-     env "BLOCK_CACHE_MB" 1024 readMaybe   return Config {version = False, ..}@@ -274,6 +271,23 @@         env "TX_TIMEOUT" d.txTimeout readMaybe       slowRequest <-         env "SLOW_REQUEST" d.slowRequest readMaybe+      -- the limit on long requests moves between two a core and sixteen a+      -- core, for requests that wait for the disk: 64 on four cores, twice+      -- the 32 reads in flight at which the disk of a production store gave+      -- its most (README, Overload). Two a core, not one: a request spends+      -- part of its time in RocksDB and Redis, off its core, and a slot+      -- handed to a thread that has yet to run is idle until it does.+      cores <- getNumCapabilities+      maxRequests <-+        env "MAX_REQUESTS" (fromIntegral (16 * cores)) readMaybe+      minRequests <-+        env "MIN_REQUESTS" (fromIntegral (2 * cores)) readMaybe+      targetDelay <-+        env "TARGET_DELAY" d.targetDelay readMaybe+      requestQueue <-+        env "REQUEST_QUEUE" d.requestQueue readMaybe+      requestWait <-+        env "REQUEST_WAIT" d.requestWait readMaybe       return WebLimits {..}  -- | Names of the supported networks, separated by pipes.@@ -442,60 +456,94 @@           <> helpEnv "SLOW_REQUEST" "Warn in the log of requests that take this long (0 = never)"           <> showDefault           <> value c.webLimits.slowRequest+    maxRequests <-+      option auto $+        metavar "INT"+          <> long "max-requests"+          <> helpEnv+            "MAX_REQUESTS"+            ( unwords+                [ "Most long requests (lists, whole blocks, xpubs) answered at once;",+                  "the limit adapts below this; the default is sixteen a core (0 = no limit)"+                ]+            )+          <> showDefault+          <> value c.webLimits.maxRequests+    minRequests <-+      option auto $+        metavar "INT"+          <> long "min-requests"+          <> helpEnv+            "MIN_REQUESTS"+            "Fewest long requests answered at once, where the limit starts; the default is two a core"+          <> showDefault+          <> value c.webLimits.minRequests+    targetDelay <-+      option auto $+        metavar "MILLISECONDS"+          <> long "target-delay"+          <> helpEnv+            "TARGET_DELAY"+            "Lower the limit on long requests when threads wait longer than this for a core"+          <> showDefault+          <> value c.webLimits.targetDelay+    requestQueue <-+      option auto $+        metavar "INT"+          <> long "request-queue"+          <> helpEnv "REQUEST_QUEUE" "Long requests that may wait for a slot; more are refused"+          <> showDefault+          <> value c.webLimits.requestQueue+    requestWait <-+      option auto $+        metavar "SECONDS"+          <> long "request-wait"+          <> helpEnv "REQUEST_WAIT" "Refuse a long request that has waited this long for a slot"+          <> showDefault+          <> value c.webLimits.requestWait     return WebLimits {..}-  redis <--    flag c.redis True $-      long "redis"-        <> helpEnv "REDIS" "Redis cache for xpub data"-  fleet <--    flag c.fleet True $-      long "fleet"-        <> helpEnv-          "FLEET"-          ( unwords-              [ "Answer as one of the servers that share the Redis of --redis:",-                "all from the same block and the same mempool"-              ]-          )   redisURL <-     strOption $       metavar "URL"         <> long "redis-url"-        <> helpEnv "REDIS_URL" "URL for Redis cache"+        <> helpEnv "REDIS_URL" "Redis of the fleet: its tip, mempool and xpub cache"+        <> showDefault         <> value c.redisURL-  redisMinAddrs <-+  redisMaxXPubs <-     option auto $       metavar "COUNT"-        <> long "redis-min-addrs"-        <> helpEnv "REDIS_MIN_ADDRS" "Cache xpubs with at least this many used addresses"+        <> long "redis-max-xpubs"+        <> helpEnv "REDIS_MAX_XPUBS" "Size of the xpub cache in Redis, at least 1"         <> showDefault-        <> value c.redisMinAddrs-  redisMinTxs <-+        <> value c.redisMaxXPubs+  xpubWorkers <-     option auto $       metavar "COUNT"-        <> long "redis-min-txs"-        <> helpEnv "REDIS_MIN_TXS" "Cache xpubs with at least this many transactions in their addresses"+        <> long "xpub-workers"+        <> helpEnv "XPUB_WORKERS" "Threads that index xpubs into the cache (0 for one per core)"         <> showDefault-        <> value c.redisMinTxs-  redisMaxXPubs <-+        <> value c.xpubWorkers+  xpubWait <-     option auto $+      metavar "SECONDS"+        <> long "xpub-wait"+        <> helpEnv "XPUB_WAIT" "Most seconds a request waits for an xpub to be indexed"+        <> showDefault+        <> value c.xpubWait+  xpubHorizon <-+    option auto $+      metavar "SECONDS"+        <> long "xpub-horizon"+        <> helpEnv "XPUB_HORIZON" "Take no job for an xpub that would be indexed later than this"+        <> showDefault+        <> value c.xpubHorizon+  xpubQueue <-+    option auto $       metavar "COUNT"-        <> long "redis-max-xpubs"-        <> helpEnv "REDIS_MAX_XPUBS" "Evict the least recently used xpubs from Redis past this many"+        <> long "xpub-queue"+        <> helpEnv "XPUB_QUEUE" "Most xpubs that requests wait for in the queue of the indexer"         <> showDefault-        <> value c.redisMaxXPubs-  noMempool <--    flag c.noMempool True $-      long "no-mempool"-        <> helpEnv "NO_MEMPOOL" "Do not index mempool transactions"-  wipeMempool <--    flag c.wipeMempool True $-      long "wipe-mempool"-        <> helpEnv "WIPE_MEMPOOL" "Wipe indexed mempool at start"-  syncMempool <--    flag c.syncMempool True $-      long "sync-mempool"-        <> helpEnv "SYNC_MEMPOOL" "Attempt to download peer mempools"+        <> value c.xpubQueue   blockWindow <-     option auto $       metavar "COUNT"@@ -514,9 +562,23 @@     option auto $       metavar "SECONDS"         <> long "peer-timeout"-        <> helpEnv "PEER_TIMEOUT" "Unresponsive peer timeout"+        <> helpEnv "PEER_TIMEOUT" "Disconnect a peer that is silent, or owes blocks, for this long"         <> showDefault         <> value c.peerTimeout+  blockStall <-+    option auto $+      metavar "SECONDS"+        <> long "block-stall"+        <> helpEnv "BLOCK_STALL" "Ask another peer for blocks that the syncing peer has not delivered for this long"+        <> showDefault+        <> value c.blockStall+  blockPing <-+    option auto $+      metavar "SECONDS"+        <> long "block-ping"+        <> helpEnv "BLOCK_PING" "Ask a peer for blocks if it answers a ping within this long, or if none does"+        <> showDefault+        <> value c.blockPing   maxPeerLife <-     option auto $       metavar "SECONDS"@@ -589,10 +651,6 @@     flag c.noBlockchainInfo True $       long "no-blockchain-info"         <> helpEnv "NO_BLOCKCHAIN_INFO" "Disable Blockchain.info-style API endpoints"-  noXPub <--    flag c.noXPub True $-      long "no-xpub"-        <> helpEnv "NO_XPUB" "Disable extended public key support"   healthCheckInterval <-     option auto $       metavar "SECONDS"@@ -610,7 +668,11 @@         <> long "node-rpc"         <> helpEnv           "NODE_RPC"-          "Publish transactions over the JSON-RPC of this full node, and through the peers when it fails"+          ( unwords+              [ "Publish transactions over the JSON-RPC of this full node, and through the peers when it fails;",+                "fill the shared mempool from it again after Redis loses its data"+              ]+          )         <> value c.nodeRpc   tlsCert <-     strOption $@@ -635,7 +697,7 @@     option auto $       metavar "COUNT"         <> long "max-open-files"-        <> helpEnv "MAX_OPEN_FILES" "RocksDB max_open_files; bounds index/filter memory (0 = unlimited)"+        <> helpEnv "MAX_OPEN_FILES" "Table files that RocksDB keeps open (0 = all of them)"         <> showDefault         <> value c.maxOpenFiles   writeBufferMB <-@@ -658,6 +720,19 @@           )         <> showDefault         <> value c.maxWalMB+  backgroundJobs <-+    option auto $+      metavar "COUNT"+        <> long "background-jobs"+        <> helpEnv+          "BACKGROUND_JOBS"+          ( unwords+              [ "RocksDB threads for flushes and compactions together",+                "(0 = RocksDB default of two)"+              ]+          )+        <> showDefault+        <> value c.backgroundJobs   blockCacheMB <-     option auto $       metavar "MB"@@ -689,6 +764,9 @@   when c.version $ do     putStrLn haskoinStoreVersion     exitSuccess+  -- the cache of xpubs cannot be turned off: its size is all that is set+  when (c.redisMaxXPubs < 1) $+    error "--redis-max-xpubs must be at least 1"   if null c.peers && not c.discover     then run $ let Config {..} = c in Config {discover = True, ..}     else run c@@ -704,21 +782,20 @@ -- Data goes in a directory named after the network inside the data directory. run :: Config -> IO () run cfg =-  withContext $ \ctx ->-    withLogger level $ \logger -> do-      net <- either error return $ networkReader cfg.net-      with_stats logger net $ \stats -> do-        let dir = cfg.dir </> net.name-        logInfo (withSource "haskoin-store" logger) $-          "Creating working directory (if not found): " <> cs dir-        createDirectoryIfMissing True dir-        run_store logger net ctx stats dir $ \store ->-          if cfg.noWeb-            then withSubscription store.pub $ \sub ->-              loopForever (receive sub)-            else run_web logger stats store+  withLogger level $ \logger -> do+    net <- either error return $ networkReader cfg.net+    with_stats logger net $ \stats -> do+      let dir = cfg.dir </> net.name+      logInfo (withSource "haskoin-store" logger) $+        "Creating working directory (if not found): " <> cs dir+      createDirectoryIfMissing True dir+      run_store logger net stats dir $ \store ->+        if cfg.noWeb+          then withSubscription store.pub $ \sub ->+            loopForever (receive sub)+          else run_web logger stats store   where-    run_store logger net ctx stats dir =+    run_store logger net stats dir =       withStore         StoreConfig           { maxPeers = cfg.maxPeers,@@ -726,20 +803,25 @@             discover = cfg.discover,             db = dir </> "db",             net = net,-            ctx = ctx,-            redis = if cfg.redis then Just cfg.redisURL else Nothing,-            fleet = cfg.fleet,+            redis = Just cfg.redisURL,             gap = cfg.webLimits.xpubGap,             initGap = cfg.webLimits.xpubGapInit,-            redisMinAddrs = cfg.redisMinAddrs,-            redisMinTxs = cfg.redisMinTxs,             redisMaxXPubs = cfg.redisMaxXPubs,-            wipeMempool = cfg.wipeMempool,-            noMempool = cfg.noMempool,-            syncMempool = cfg.syncMempool,+            xpubs =+              defaultXPubConfig+                { indexer =+                    defaultXPubConfig.indexer+                      { workers = cfg.xpubWorkers,+                        wait = cfg.xpubWait * 1000000,+                        horizon = cfg.xpubHorizon * 1000000,+                        room = cfg.xpubQueue+                      }+                },             blockWindow = cfg.blockWindow,             mempoolTimeout = cfg.mempoolTimeout,             peerTimeout = fromIntegral cfg.peerTimeout,+            blockStall = fromIntegral cfg.blockStall,+            blockPing = fromIntegral cfg.blockPing,             maxPeerLife = fromIntegral cfg.maxPeerLife,             connect = withConnection,             stats = stats,@@ -753,6 +835,7 @@               if cfg.maxWalMB > 0                 then Just (cfg.maxWalMB * 1024 * 1024)                 else Nothing,+            backgroundJobs = if cfg.backgroundJobs > 0 then Just cfg.backgroundJobs else Nothing,             blockCache =               if cfg.blockCacheMB > 0                 then Just (cfg.blockCacheMB * 1024 * 1024)@@ -770,12 +853,10 @@             maxLaggingBlocks = cfg.maxLaggingBlocks,             minPeers = cfg.minPeers,             version = haskoinStoreVersion,-            noMempool = cfg.noMempool,             stats = stats,             tickerRefresh = cfg.tickerRefresh,             tickerURL = cfg.tickerURL,             priceHistoryURL = cfg.priceHistoryURL,-            noXPub = cfg.noXPub,             noBlockchainInfo = cfg.noBlockchainInfo,             healthCheckInterval = cfg.healthCheckInterval,             accessLog = cfg.accessLog,
+ bench/Admission.hs view
@@ -0,0 +1,316 @@+-- | Admission control under load: no limit, a fixed one, and the one that+-- adapts, against requests that compute and requests that wait for a disk.+--+-- > haskoin-store-bench admission [MIX LIMIT [SECONDS [CLIENTS]]]+--+-- With no arguments it runs every MIX with every LIMIT, for 60 seconds each.+--+-- Each of CLIENTS (128) threads asks for long requests through+-- "Haskoin.Store.Web.Admission", one after the other, as the connections of+-- a load generator would; a client that is refused waits 100 ms before it+-- asks again, as if it had tried another server. The requests run in the+-- thread of their client, as a request of the web server runs in the thread+-- of its connection. A MIX is what they ask for:+--+-- * @cpu@: 10 ms of computing that allocates, with a map of 10,000 entries+--   held meanwhile, as a response being made holds its data.+--+-- * @disk@: twenty reads from a disk that takes 1 ms a read and does 32 at+--   once, 32,000 reads a second, like the disk of a production store; each+--   is followed by 50 µs of computing. A read waits for its turn at the+--   disk in Haskell, and then in a safe foreign call (@usleep@), which gives+--   the capability back, as a read of RocksDB does.+--+-- * @switch@: @disk@ for the first half of the run, then @cpu@.+--+-- A LIMIT is @off@, @fixed@, four slots a core as the cap of 1.16.0 had,+-- or @fixed=N@, or @adaptive@, between one and sixteen a core, or+-- @adaptive=FLOOR-CEILING[/TARGET]@. The target of the delay is 25 ms+-- unless given, in milliseconds; 64 requests may wait, for a second at+-- most.+--+-- Meanwhile a light request, which computes for 50 µs, is due every 10 ms,+-- on each capability in turn, and one of the threads that wait for them+-- there takes it, as the thread of a connection takes a health check. Its+-- latency is counted from when it was due, so a capability that runs late+-- counts for as many light requests as the others. Every capability has+-- threads of its own for them: GHC moves a thread to another capability+-- only when that one is idle, and sixteen threads that compute, started+-- from one, were found thirteen on one capability and one on each of the+-- others, so where a thread is started says much about how long it waits.+--+-- For the run, for each half of it, and for every five seconds, it prints+-- the long requests answered a second, their latency at the 50th and 99th+-- percentiles, counted from when they asked for a slot, the refusals a+-- second by why (line full, waited too long, CoDel), the latency of the+-- light requests, the limit, and the 90th percentile of the scheduler's+-- delay.+--+-- Run it as the production machine runs the store, on four cores:+--+-- > taskset -c 4-7 haskoin-store-bench admission +RTS -N4 -A64m -T+module Admission (admissionBench) where++import Control.Concurrent+  ( forkIOWithUnmask,+    forkOnWithUnmask,+    getNumCapabilities,+    killThread,+    newEmptyMVar,+    putMVar,+    takeMVar,+    threadDelay,+  )+import Control.Concurrent.Async (withAsync, withAsyncOn)+import Control.Concurrent.QSem (QSem, newQSem, signalQSem, waitQSem)+import Control.Concurrent.STM (atomically, check, newTVarIO, readTVar, writeTVar)+import Control.Exception (bracket, bracket_, evaluate)+import Control.Monad (forM_, forever, replicateM, replicateM_, void, when)+import Data.Bits ((.&.))+import Data.IORef (IORef, atomicModifyIORef', newIORef, readIORef, writeIORef)+import Data.IntMap.Strict qualified as IntMap+import Data.List (intercalate)+import Foreign.C.Types (CInt (..), CUInt (..))+import GHC.Clock (getMonotonicTimeNSec)+import Haskoin.Store.Web.Admission+import Haskoin.Store.Web.Congestion (lateness, percentile, snooze, withProbes)+import System.Mem (performGC)+import Text.Printf (printf)++-- | Sleep so many microseconds in a safe foreign call, which lets go of the+-- capability while it sleeps.+foreign import ccall safe "usleep" c_usleep :: CUInt -> IO CInt++-- | What the long requests do.+data Mix = Cpu | Disk | Switch+  deriving (Eq, Show)++-- | How many long requests are answered at once.+data Limit = Off | Fixed !Int | Adaptive !Int !Int !Int++instance Show Limit where+  show Off = "off"+  show (Fixed n) = "fixed " <> show n+  show (Adaptive lo hi t) = "adaptive " <> show lo <> "-" <> show hi <> ", target " <> show t <> " ms"++-- | Everything a run records, newest first, in nanoseconds.+data Records = Records+  { -- | when a long request ended, and how long it took+    answered :: !(IORef [(Int, Int)]),+    -- | when a long request was refused, and why+    refused :: !(IORef [(Int, Refusal)]),+    -- | when a light request was due, and how long it took from then+    lights :: !(IORef [(Int, Int)]),+    -- | when a step of the controller was, its limit and the delay+    steps :: !(IORef [(Int, Int, Int)])+  }++admissionBench :: [String] -> IO ()+admissionBench args = do+  caps <- getNumCapabilities+  perMs <- calibrate+  printf "%d capabilities, %d steps of computing a millisecond\n" caps perMs+  let limits = [Off, Fixed (4 * caps), Adaptive caps (16 * caps) 25]+  case args of+    [] -> forM_ [Cpu, Disk, Switch] $ \m -> forM_ limits $ \l -> run perMs m l 60 128 >> performGC+    m : l : rest -> do+      let (secs, clients) = case rest of+            [] -> (60, 128)+            [s] -> (read s, 128)+            s : c : _ -> (read s, read c)+      run perMs (mixOf m) (limitOf caps l) secs clients+    _ -> error "usage: haskoin-store-bench admission [MIX LIMIT [SECONDS [CLIENTS]]]"+  where+    mixOf = \case+      "cpu" -> Cpu+      "disk" -> Disk+      "switch" -> Switch+      m -> error ("no such mix: " <> m)+    limitOf caps = \case+      "off" -> Off+      "fixed" -> Fixed (4 * caps)+      "adaptive" -> Adaptive caps (16 * caps) 25+      l+        | ("fixed=", n) <- splitAt 6 l -> Fixed (read n)+        | ("adaptive=", r) <- splitAt 9 l,+          (lo, '-' : r') <- break (== '-') r,+          (hi, t) <- break (== '/') r' ->+            Adaptive (read lo) (read hi) (if null t then 25 else read (drop 1 t))+        | otherwise -> error ("no such limit: " <> l)++-- | One run of so many seconds with so many clients.+run :: Int -> Mix -> Limit -> Int -> Int -> IO ()+run perMs mix limit secs clients = do+  a <- newAdmission settings+  disk <- newQSem 32+  rs <- Records <$> newIORef [] <*> newIORef [] <*> newIORef [] <*> newIORef []+  caps <- getNumCapabilities+  start <- nanos+  let half = start + secs * 500_000_000+      end = start + secs * 1_000_000_000+      kind t = if mix == Switch then (if t < half then Disk else Cpu) else mix+      client = forever $ do+        t0 <- nanos+        r <- admitted a (return . Left) (Right <$> request perMs disk (kind t0) t0)+        t1 <- nanos+        case r of+          Right () -> push rs.answered (t1, t1 - t0)+          -- a client is elsewhere, and its wait costs the server nothing:+          -- not even a timer of GHC's timer manager+          Left why -> push rs.refused (t1, why) >> void (c_usleep 100_000)+      -- on each capability in turn, a light request due every 10 ms, which+      -- one of the threads waiting there takes, as a health check is+      -- answered by the thread of its connection when it hears a request+      light i = do+        -- each thread has a place of its own to be told: a channel that+        -- they all read would wake them one after the other+        dues <- replicateM 128 newEmptyMVar+        let answer due = forever $ do+              t0 <- takeMVar due+              spin (perMs `div` 20)+              t1 <- nanos+              push rs.lights (t0, t1 - t0)+            tick k = do+              let t0 = start + (k * caps + i) * 10_000_000+              t <- nanos+              -- asleep in the system, as GHC's manager of input and output+              -- waits for the network, and with no timer of GHC+              when (t0 > t) (snooze ((t0 - t) `div` 1000))+              putMVar (dues !! (k `mod` length dues)) t0+              when (t0 < end) (tick (k + 1))+            -- unmasked, as 'bracket' masks what it starts, and a thread that+            -- computes with exceptions masked cannot be stopped+            begin d = forkOnWithUnmask i (\unmask -> unmask (answer d))+        bracket (mapM begin dues) (mapM_ killThread) (const (tick 0))+      lights = foldr (\i k -> withAsyncOn i (light i) (const k))+      record r = nanos >>= \t -> push rs.steps (t, r.limit, r.delay)+      -- with no limit there is no controller, but the delay is still worth+      -- seeing+      watch = case limit of+        Off -> withProbes $ \ps -> forever $ do+          threadDelay 500_000+          d <- lateness ps+          t <- nanos+          push rs.steps (t, 0, d)+        _ -> regulate a record+  -- every client is there before any of them asks: under load, a thread+  -- that GHC has yet to run may wait for seconds, and so would the clients+  -- that it was to start+  go <- newTVarIO False+  withAsync watch $ \_ ->+    flip lights [0 .. caps - 1] $+      let ready = atomically (readTVar go >>= check)+          begin = forkIOWithUnmask (\unmask -> unmask (ready >> client))+       in bracket (replicateM clients begin) (mapM_ killThread) $ \_ -> do+            atomically (writeTVar go True)+            threadDelay (secs * 1_000_000)+  printf "\n== %s, %s, %d s, %d clients\n" (show mix) (show limit) secs clients+  putStrLn header+  report rs "run" start end+  when (mix == Switch) $ do+    report rs "disk half" start half+    report rs "cpu half" half end+  putStrLn "every five seconds:"+  forM_ [0, 5 .. secs - 5] $ \s ->+    report rs (printf "%3d s" s) (start + s * 1_000_000_000) (start + (s + 5) * 1_000_000_000)+  where+    settings = case limit of+      Off -> base {ceiling = 0}+      Fixed n -> base {ceiling = n, floor = n}+      Adaptive lo hi t -> base {ceiling = hi, floor = lo, target = t * 1000}+    base = Settings {ceiling = 0, floor = 1, target = 25_000, room = 64, patience = 1_000_000}++-- | What a long request does.+request :: Int -> QSem -> Mix -> Int -> IO ()+request perMs disk mix salt = case mix of+  Disk -> replicateM_ 20 $ do+    bracket_ (waitQSem disk) (signalQSem disk) (void (c_usleep 1000))+    spin (perMs `div` 20)+  _ -> do+    -- the salt keeps GHC from making the map once for every request+    held <- evaluate (IntMap.fromList [(i, i + salt) | i <- [1 .. 10_000]])+    spin (10 * perMs)+    void (evaluate (IntMap.foldl' (+) 0 held))++-- | Compute for so many steps, allocating a number every sixteen, about+-- half a gigabyte a second: a loop that does not allocate never lets GHC+-- take its core, and one that allocates at every step spends its time in+-- the collector.+spin :: Int -> IO ()+spin steps = do+  r <- newIORef (0 :: Int)+  let go :: Int -> Int -> IO ()+      go 0 !acc = writeIORef r acc+      go n !acc+        | n .&. 15 == 0 = writeIORef r acc >> go (n - 1) (acc + n)+        | otherwise = go (n - 1) (acc + n)+  go steps 0++-- | Steps of 'spin' in a millisecond, the best of three, on a quiet process.+calibrate :: IO Int+calibrate = do+  spin 100_000_000+  ts <- mapM (const timed) [1 :: Int, 2, 3]+  return (round (200_000_000 / minimum ts :: Double))+  where+    timed = do+      t0 <- nanos+      spin 200_000_000+      t1 <- nanos+      return (fromIntegral (t1 - t0) / 1e6 :: Double)++-- | The columns of 'report'.+header :: String+header =+  intercalate+    "  "+    [ "window     ",+      "long/s",+      "p50 ms",+      "p99 ms",+      "refused/s full waited codel",+      "light p50 p95 p99 ms",+      "limit",+      "delay ms"+    ]++-- | Print a line of what happened from one time to another.+report :: Records -> String -> Int -> Int -> IO ()+report rs name from to = do+  done <- within fst <$> readIORef rs.answered+  no <- within fst <$> readIORef rs.refused+  ls <- within fst <$> readIORef rs.lights+  ss <- within (\(t, _, _) -> t) <$> readIORef rs.steps+  let per n = fromIntegral n / secs :: Double+      ms = (/ 1e6) . fromIntegral :: Int -> Double+      lat p = ms (percentile p (map snd done))+      light p = ms (percentile p (map snd ls))+      count why = length (filter ((== why) . snd) no)+      mean xs = if null xs then 0 else fromIntegral (sum xs) / fromIntegral (length xs) :: Double+  printf+    "%-11s  %6.0f  %6.1f  %6.1f  %9.1f %4.1f %6.1f %5.1f  %9.1f %3.0f %4.0f     %5.1f  %8.1f\n"+    name+    (per (length done))+    (lat 50)+    (lat 99)+    (per (length no))+    (per (count LineFull))+    (per (count WaitedTooLong))+    (per (count Overloaded))+    (light 50)+    (light 95)+    (light 99)+    (mean [l | (_, l, _) <- ss])+    (mean [d | (_, _, d) <- ss] / 1000)+  where+    secs = fromIntegral (to - from) / 1e9+    within :: (x -> Int) -> [x] -> [x]+    within f = filter (\x -> f x >= from && f x < to)++-- | Add to a record.+push :: IORef [a] -> a -> IO ()+push r x = atomicModifyIORef' r (\xs -> (x : xs, ()))++-- | Nanoseconds of the monotonic clock.+nanos :: IO Int+nanos = fromIntegral <$> getMonotonicTimeNSec
+ bench/Import.hs view
@@ -0,0 +1,409 @@+-- | How long a full block takes to import, in stages with its reads made+-- ahead, and with every read made when the rules come to it.+--+-- > haskoin-store-bench import [OUTPUTS [TXS [BLOCKS [CACHE_MIB [DIR]]]]] +RTS -N4+--+-- It builds a regtest database with OUTPUTS unspent outputs (a million by+-- default) over half as many addresses, with the rules of the store,+-- closes it so that RocksDB writes it all to table files, and makes two+-- copies. It then imports the same blocks of TXS transactions (3,500) into+-- each: into one with 'importBlock' and no read cache ('withoutReadCache'),+-- which reads every key when the rules ask for it; and into the other with 'importBlock' as the store runs it.+--+-- BLOCKS blocks (5) of two kinds are imported, in this order: blocks whose+-- inputs spend outputs spread over the set, and blocks of chains+-- ('Chained'), where three inputs in four spend an output that a+-- transaction of the same block makes, as on Bitcoin mainnet today.+--+-- = What a block is made of+--+-- A transaction of the first kind spends two outputs spread+-- over the set, or one of them and an output of the transaction before it+-- in the block for one in ten, and pays an address that has outputs and a+-- new one. So a block of 3,500 spends about 6,650 unspent outputs, adds a+-- spender to the groups of as many transactions and changes 14,000+-- balances, about what a block of mainnet does. The import in stages reads+-- the outputs, and since+-- database version 23 writes the spenders and the balances as merge+-- records, without reading them ("Haskoin.Store.Database.Merge"). The+-- outputs of the set come from transactions of 50 outputs each, and a+-- block spends outputs of thousands of them.+--+-- A block of the second kind has as many transactions, inputs+-- and outputs, in chains of a hundred ('chainLength'), and spends about+-- 1,850 outputs of the set.+--+-- The database is built by the import in stages, so the balances and the+-- spenders that it starts with are merge records in table files, as on a+-- store.+--+-- The block cache is CACHE_MIB (32), a fraction of the database, so that+-- most reads fetch and decompress a block of a table file, as on a store+-- of mainnet. DIR is where the databases go: a temporary directory unless+-- given.+module Import (importBench) where++import Control.Concurrent (getNumCapabilities)+import Control.DeepSeq (force)+import Control.Exception (evaluate)+import Control.Monad (forM_, void, when)+import Data.ByteString (ByteString)+import Data.ByteString qualified as B+import Data.HashSet qualified as HashSet+import Data.IORef (modifyIORef', newIORef, readIORef)+import Data.List (find)+import Data.Maybe (fromMaybe)+import Data.Serialize (encode)+import Data.Word (Word64)+import Haskoin+import Haskoin.Node (LogLevel (..), Logger (..))+import Haskoin.Store.Data (BlockData (..))+import Haskoin.Store.Database.Query qualified as Q+import Haskoin.Store.Database.Reader+  ( DatabaseReader (..),+    withLayeredReader,+  )+import Haskoin.Store.Database.Writer+  ( BatchReport (..),+    BlockTimes (..),+    ReadTimes (..),+    Writer,+    noReads,+    runWriter,+    runWriterReport,+    stopwatch,+    withoutReadCache,+  )+import Haskoin.Store.Integration.Temp (withSystemTempDirectory, withTempDirectory)+import Haskoin.Store.Logic (importBlock, initBest)+import System.Directory (copyFile, createDirectory, listDirectory)+import System.FilePath ((</>))+import System.IO (hFlush, stdout)+import Text.Printf (printf)++net :: Network+net = btcRegTest++quiet :: Logger+quiet = Logger {level = LevelError, source = "", push = \_ -> return ()}++-- | What to measure.+data Plan = Plan+  { -- | unspent outputs to start with+    outputs :: !Int,+    -- | transactions in a measured block+    txs :: !Int,+    -- | measured blocks of each kind+    blocks :: !Int,+    -- | block cache of RocksDB, in bytes+    cache :: !Int+  }++importBench :: [String] -> IO ()+importBench args = do+  let number i d = maybe d read (lookup i (zip [0 :: Int ..] args))+      plan =+        Plan+          { outputs = number 0 1000000,+            txs = number 1 3500,+            blocks = number 2 5,+            cache = number 3 32 * 1024 * 1024+          }+      inDir = case drop 4 args of+        dir : _ -> withTempDirectory dir "haskoin-store-import-"+        [] -> withSystemTempDirectory "haskoin-store-import-"+  inDir (run plan)++run :: Plan -> FilePath -> IO ()+run plan dir = do+  cores <- getNumCapabilities+  printf "cores used: %d\n" cores+  printf "building a database with %d unspent outputs at %d addresses\n" plan.outputs (addresses plan)+  hFlush stdout+  let before = dir </> "before"+      after = dir </> "after"+  createDirectory before+  (tip, took) <- stopwatch (build plan before)+  printf "built in %.1f s\n" took+  copyDirectory before after+  -- the blocks to measure, the same for both databases, each kind with+  -- outputs of the set of its own+  let n = plan.blocks+      (tip1, empty) = measured plan Spread tip 0 n+      (tip2, emptyChained) = measured plan Chained tip1 n n+  _ <- evaluate (force (map fst (empty <> emptyChained)))+  forM_ [("spread" :: String, empty), ("chained", emptyChained)] $ \(shape, blocks) ->+    printf+      "%s blocks of %d transactions, %d inputs and %d outputs: %d inputs spend an output of the block\n"+      shape+      (plan.txs + 1)+      (inputsOf blocks)+      (outputsOf blocks)+      (insideOf blocks)+  putStrLn ""+  hFlush stdout+  (old, oldBest) <- open plan before $ \db -> do+    a <- mapM (measure db plain) empty+    b <- mapM (measure db plain) emptyChained+    ((a, b),) <$> Q.getBestBlock db+  (new, newBest) <- open plan after $ \db -> do+    a <- mapM (measure db importBlock) empty+    b <- mapM (measure db importBlock) emptyChained+    ((a, b),) <$> Q.getBestBlock db+  when (oldBest /= newBest || oldBest /= Just (headerHash (fst tip2))) $+    fail "the two databases did not end at the last block"+  let (oldEmpty, oldEmptyChained) = old+      (newEmpty, newEmptyChained) = new+  report "outputs spread over the set" oldEmpty newEmpty+  report+    "a block of chains: three inputs in four spend an output of the block"+    oldEmptyChained+    newEmptyChained+  where+    plain w = importBlock (withoutReadCache w)+    inputsOf bs = sum [length tx.inputs | (b, _) <- take 1 bs, tx <- b.txs]+    outputsOf bs = sum [length tx.outputs | (b, _) <- take 1 bs, tx <- b.txs]+    -- inputs of the first block that spend an output of the block itself+    insideOf bs =+      length+        [ ()+        | (b, _) <- take 1 bs,+          let made = HashSet.fromList (map txHash b.txs),+          tx <- b.txs,+          i <- tx.inputs,+          i.outpoint.hash `HashSet.member` made+        ]++-- | Print the mean time of a block, before and after, and the stages.+report :: String -> [(BlockTimes, BatchReport)] -> [(BlockTimes, BatchReport)] -> IO ()+report title before after = do+  let batches = map snd after+      average f = sum (map f batches) `div` max 1 (length batches)+  when (map ((.operations) . snd) before /= map (.operations) batches) $+    fail "the two imports did not write the same number of keys"+  printf "%s (milliseconds a block, mean of %d)\n" title (length new)+  printf "  one batch a block: %d keys set or deleted, %d bytes\n" (average (.operations)) (average (.bytes))+  printf "  %-40s %10s %10s\n" ("" :: String) ("no cache" :: String) ("in stages" :: String)+  line "prepare" (.prepare)+  line "read ahead" (.prefetch)+  line "  unspent outputs" (.reads.unspentReads)+  line "  groups of spenders" (.reads.spenderReads)+  line "  balances" (.reads.balanceReads)+  line "apply the rules (no cache: reads too)" (.apply)+  line "write the batch" (.write)+  line "whole block" total+  printf "  %-40s %20.1fx\n\n" ("faster by" :: String) (mean total old / mean total new)+  hFlush stdout+  where+    old = map fst before+    new = map fst after+    total t = t.prepare + t.prefetch + t.apply + t.write+    mean f ts = 1000 * sum (map f ts) / fromIntegral (max 1 (length ts))+    line :: String -> (BlockTimes -> Double) -> IO ()+    line name f = printf "  %-40s %10.1f %10.1f\n" name (mean f old) (mean f new)++-- | Open a database as the store does, with a block cache of the size of+-- the plan and Bloom filters.+open :: Plan -> FilePath -> (DatabaseReader -> IO a) -> IO a+open plan dir = withLayeredReader net 20 32 dir True Nothing Nothing Nothing Nothing (Just plan.cache) Nothing++-- | Import a block and say how long each stage took.+measure ::+  DatabaseReader ->+  (Writer -> Block -> BlockNode -> IO (BlockData, [a])) ->+  (Block, BlockNode) ->+  IO (BlockTimes, BatchReport)+measure db rules (block, node) = do+  inside <- newIORef 0+  ((_, batch), whole) <- stopwatch . runWriterReport net db quiet $ \w -> do+    (_, took) <- stopwatch (rules w block node)+    modifyIORef' inside (+ took)+  applied <- readIORef inside+  return . (,batch) $ case batch.block of+    Just t -> t+    Nothing -> BlockTimes {prepare = 0, prefetch = 0, reads = noReads, apply = applied, write = whole - applied}++chunks :: Int -> [a] -> [[a]]+chunks _ [] = []+chunks n xs = let (a, b) = splitAt n xs in a : chunks n b++copyDirectory :: FilePath -> FilePath -> IO ()+copyDirectory from to = do+  createDirectory to+  names <- listDirectory from+  forM_ names $ \name -> copyFile (from </> name) (to </> name)++-------------------------------------------------------------------------------+-- The database to start from+-------------------------------------------------------------------------------++-- | Outputs of a funding transaction, after its change.+fanout :: Int+fanout = 50++-- | Funding transactions in a block.+perBlock :: Int+perBlock = 400++-- | Satoshi of each output of the set.+coin :: Word64+coin = 2000++-- | Addresses that the outputs of the set are spread over: each has two.+addresses :: Plan -> Int+addresses plan = max 1 (plan.outputs `div` 2)++-- | A script that pays one of many addresses: to a public key hash for+-- even numbers, to a witness public key hash for odd ones.+payTo :: Int -> ByteString+payTo n+  | even n = "\x76\xa9\x14" <> h <> "\x88\xac"+  | otherwise = "\x00\x14" <> h+  where+    h = B.take 20 (encode (sha256 (encode (fromIntegral n :: Word64))))++coinbase :: BlockHeight -> Tx+coinbase h = Tx 1 [TxIn nullOutPoint (encode h) maxBound] [TxOut 5000000000 (payTo 0)] [] 0++-- | The funding transactions: a chain, each spending the change of the one+-- before and paying 'fanout' outputs of the set. The first spends the+-- coinbase of the first block.+funding :: Plan -> [Tx]+funding plan = go (0 :: Int) (OutPoint (txHash (coinbase 1)) 0) 5000000000+  where+    count = (plan.outputs + fanout - 1) `div` fanout+    go k from value+      | k >= count = []+      | otherwise =+          let left = value - fromIntegral fanout * coin+              paid = [TxOut coin (payTo (1 + (k * fanout + i) `mod` addresses plan)) | i <- [0 .. fanout - 1]]+              tx = Tx 2 [TxIn from "" maxBound] (TxOut left (payTo 0) : paid) [] 0+           in tx : go (k + 1) (OutPoint (txHash tx) 0) left++-- | The outputs of the set, in the order in which the measured blocks+-- spend them: one of each funding transaction in turn, so that the inputs+-- of a block are spread over all of them.+spendable :: Plan -> [OutPoint]+spendable plan =+  [ OutPoint h (fromIntegral i)+  | i <- [1 .. fanout],+    h <- hashes+  ]+  where+    hashes = map txHash (funding plan)++-- | Build the database: the genesis block, a block with the coin, and the+-- blocks of the funding transactions. Gives the header and height of the+-- last block.+build :: Plan -> FilePath -> IO (BlockHeader, BlockHeight)+build plan dir = open plan dir $ \db -> do+  runWriter net db quiet initBest+  let first = mine net.genesisHeader 1 []+      go parent [] = return parent+      go (header, height) (txs : more) = do+        let next = mine header (height + 1) txs+        add db next+        go (tipOf next) more+  add db first+  go (tipOf first) (chunks perBlock (funding plan))+  where+    add db (block, node) = void (runWriter net db quiet (\w -> importBlock w block node))++tipOf :: (Block, BlockNode) -> (BlockHeader, BlockHeight)+tipOf (block, node) = (block.header, node.height)++-- | A block on a parent, with a coinbase in front of the transactions.+mine :: BlockHeader -> BlockHeight -> [Tx] -> (Block, BlockNode)+mine parent height txs = (Block header all', BlockNode header height 0 prev)+  where+    prev = headerHash parent+    all' = coinbase height : txs+    merkle = either error id (buildMerkleRoot (map txHash all'))+    candidate = BlockHeader 0x20000000 prev merkle (parent.timestamp + 600) parent.bits+    header = fromMaybe (error "no nonce") (find (isValidPOW net) (map candidate [0 ..]))++-------------------------------------------------------------------------------+-- The blocks to measure+-------------------------------------------------------------------------------++-- | How the transactions of a measured block are tied to each other.+data Shape+  = -- | nearly every input spends an output of the set: one in twenty+    -- spends an output of the transaction before it in the block+    Spread+  | -- | chains of 'chainLength' transactions, each spending outputs of the+    -- ones before it in its chain: three inputs in four spend an output of+    -- the block+    Chained++-- | Transactions in a chain of a block of chains.+chainLength :: Int+chainLength = 100++-- | Blocks to measure on a tip, given their shape and how many were made+-- before them, which says which outputs of the set are left to spend.+-- Gives the new tip too.+measured ::+  Plan ->+  Shape ->+  (BlockHeader, BlockHeight) ->+  Int ->+  Int ->+  ((BlockHeader, BlockHeight), [(Block, BlockNode)])+measured plan shape tip done count = foldl' step (tip, []) [done .. done + count - 1]+  where+    coins = spendable plan+    step ((header, height), acc) k =+      let block = mine header (height + 1) (payments plan shape k (drop (k * 2 * plan.txs) coins))+       in (tipOf block, acc <> [block])++-- | The transactions of the measured block of a number, given the outputs+-- of the set that are left. Each spends two outputs and makes two, to an+-- address of the set and to a new one. Each input has a witness of the+-- size of a signature and a public key.+--+-- Which outputs a transaction spends depends on the shape of the block:+--+-- * 'Spread': two outputs of the set, but for one transaction in ten,+--   which spends an output of the transaction before it instead of the+--   second.+--+-- * 'Chained': the first transaction of a chain spends two outputs of the+--   set. Each of the others spends the first output of the transaction+--   before it, and with it the second output of the transaction two+--   before, for every other one, or an output of the set. No output is+--   spent twice, and of a chain of a hundred, 148 inputs of 200 spend an+--   output of the block. Bitcoin mainnet had 3,000 to 6,500 such inputs+--   in a block in November 2026, of some 7,000.+payments :: Plan -> Shape -> Int -> [OutPoint] -> [Tx]+payments plan shape k = go (0 :: Int) []+  where+    witness = [B.replicate 72 0x30, B.replicate 33 0x02]+    -- the outputs of the block that a transaction spends, given its number+    -- and the hashes of those before it, the last one first+    inside Spread i (h : _) | i `mod` 10 == 9 = [OutPoint h 0]+    inside Spread _ _ = []+    inside Chained i before = case (i `mod` chainLength, before) of+      (0, _) -> []+      (j, h : h' : _) | j >= 2 && even j -> [OutPoint h 0, OutPoint h' 1]+      (_, h : _) -> [OutPoint h 0]+      (_, []) -> []+    go i before coins+      | i >= plan.txs = []+      | otherwise =+          let made = inside shape i before+              (fresh, rest) = splitAt (2 - length made) coins+              mine' = made <> fresh+              n = k * plan.txs + i+              old = payTo (1 + (n * 7919) `mod` addresses plan)+              new = payTo (addresses plan + 1 + n)+              tx =+                Tx+                  2+                  [TxIn p "" maxBound | p <- mine']+                  [TxOut 1900 old, TxOut 1900 new]+                  (map (const witness) mine')+                  0+           in if length mine' < 2+                then error "not enough outputs: fewer blocks or transactions, or more outputs"+                else tx : go (i + 1) (txHash tx : before) rest
bench/Main.hs view
@@ -6,6 +6,14 @@ -- -- > haskoin-store-bench PORT BLOCKS [REDIS_URL] --+-- Four other things are measured without a server: @haskoin-store-bench+-- micro@ times the pure functions that encode a response,+-- @haskoin-store-bench import@ the import of full blocks ("Import"),+-- @haskoin-store-bench merge@ what the merge records of those blocks cost+-- the readers of the database ("Merge"), and @haskoin-store-bench+-- admission@ how the limit on long requests behaves under load+-- ("Admission").+-- -- Set @BENCH_LOG=1@ to print the log of the store, such as to see why it -- does not sync. --@@ -15,11 +23,31 @@ -- external and change addresses of the benchmark xpub, in turn. Both are -- printed once the chain is synced. With 400 blocks or more, the last one -- also holds 3,000 transactions that spend outputs of the hot address.+--+-- While it serves, the server reads commands from its standard input, each+-- of which mines blocks on its chain and announces them, to measure how+-- the store keeps up with what comes after the xpub is cached:+--+-- * @idle N@: N blocks with nothing but their coinbase, which touch no+--   address of the xpub;+-- * @pay@: a block as those of the chain, which pays nine addresses of the+--   xpub;+-- * @addr M I@: a block that pays the address of index I of chain M of the+--   xpub, once: one past the gap is not seen until a payment within the gap+--   brings it in.+--+-- It prints @mined H@ with the height of the last block once it has+-- announced them. module Main (main) where +import Admission (admissionBench) import Conduit+import Control.Applicative ((<|>))+import Control.Concurrent.Async (withAsync)+import Control.Concurrent.STM (TVar, atomically, modifyTVar', newTVarIO, readTVar, readTVarIO) import Control.DeepSeq (NFData, force)-import Control.Monad (forever, unless)+import Control.Exception (evaluate)+import Control.Monad (forM_, forever, unless) import Data.Aeson (toEncoding) import Data.Aeson.Encoding (encodingToLazyByteString) import Data.ByteString (ByteString)@@ -40,17 +68,23 @@ import Haskoin.Node import Haskoin.Store import Haskoin.Store.Database.Query qualified as Q+import Haskoin.Store.Integration.Temp (withSystemTempDirectory)+import Import (importBench)+import Merge (mergeBench) import NQE import Network.Socket (SockAddr (..)) import System.Environment (getArgs, lookupEnv)+import System.IO (hFlush, isEOF, stdout) import System.Random (randomIO)-import UnliftIO  main :: IO () main = do   args <- getArgs   case args of     ["micro"] -> micro+    "import" : more -> importBench more+    "merge" : more -> mergeBench more+    "admission" : more -> admissionBench more     _ -> server args  server :: [String] -> IO ()@@ -59,24 +93,83 @@         [p, n] -> (read p, read n, Nothing)         [p, n, r] -> (read p, read n, Just r)         _ -> error "usage: haskoin-store-bench PORT BLOCKS [REDIS_URL]"-  createContext >>= \ctx ->-    withSystemTempDirectory "haskoin-store-bench-" $ \dir -> do-      let net = bchRegTest-          blocks = chain ctx net nblocks-      requests <- newTVarIO []-      logged <- maybe False (`notElem` ["", "0"]) <$> lookupEnv "BENCH_LOG"-      let withLog f = if logged then withLogger LevelDebug f else f quiet-      withLog $ \logger -> withStore (storeConfig ctx net dir redis blocks requests logger) $ \store -> do-        withSubscription store.pub $ \sub -> waitHeight store sub nblocks-        T.putStrLn $ "hot address: " <> addrText net (hotAddr ctx)-        T.putStrLn $ "xpub: " <> xPubExport net ctx (benchXPub ctx)-        T.putStrLn $ "xpub address: " <> addrText net (xpubAddr ctx 0 0)-        bh <- Q.getBestBlock store.db-        T.putStrLn $ "best block: " <> maybe "" blockHashToHex bh-        putStrLn $ "serving on port " <> show port-        hFlush stdout-        runWeb (webConfig store port)+  withSystemTempDirectory "haskoin-store-bench-" $ \dir -> do+    let net = bchRegTest+        blocks = chain net nblocks+    mock <- Mock <$> newTVarIO [] <*> newTVarIO []+    logged <- maybe False (`notElem` ["", "0"]) <$> lookupEnv "BENCH_LOG"+    -- MAX_REQUESTS and MIN_REQUESTS: the ceiling and floor of the limit on+    -- the long requests answered at once; no limit by default.+    -- TARGET_DELAY: the scheduler's delay, in milliseconds, over which the+    -- limit comes down; the store's default otherwise.+    capped <- maybe 0 read <$> lookupEnv "MAX_REQUESTS"+    floored <- maybe 1 read <$> lookupEnv "MIN_REQUESTS"+    delay <- maybe (def :: WebLimits).targetDelay read <$> lookupEnv "TARGET_DELAY"+    -- BLOCK_CACHE_MB and WRITE_BUFFER_MB: RocksDB's block cache and write+    -- buffer, for a store whose reads go to the disk; 1024 MB and RocksDB's+    -- default otherwise+    cache <- maybe 1024 read <$> lookupEnv "BLOCK_CACHE_MB"+    buffer <- fmap read <$> lookupEnv "WRITE_BUFFER_MB"+    let withLog f = if logged then withLogger LevelDebug f else f quiet+    withLog $ \logger -> withStore (storeConfig net dir redis blocks mock logger cache buffer) $ \store -> do+      withSubscription store.pub $ \sub -> waitHeight store sub nblocks+      T.putStrLn $ "hot address: " <> addrText net hotAddr+      T.putStrLn $ "xpub: " <> xPubExport net benchXPub+      T.putStrLn $ "xpub address: " <> addrText net (xpubAddr 0 0)+      bh <- Q.getBestBlock store.db+      T.putStrLn $ "best block: " <> maybe "" blockHashToHex bh+      putStrLn $ "serving on port " <> show port+      hFlush stdout+      let limits = def {maxRequests = capped, minRequests = floored, targetDelay = delay}+      -- with BENCH_LOG=1 the web server logs each request too+      withAsync (commands net blocks mock) $ \_ ->+        runWeb ((webConfig store port) {limits = limits, logger = if logged then logger else quiet} :: WebConfig) +-- | What the mock peers share: the blocks mined after the chain, oldest+-- first, and where to announce new ones.+data Mock = Mock+  { extra :: !(TVar [Block]),+    peers :: !(TVar [Mailbox ByteString])+  }++-- | Mine the blocks that the commands on the standard input ask for, on top+-- of the chain and of those mined before, and announce them to every peer.+commands :: Network -> [Block] -> Mock -> IO ()+commands net blocks mock = loop+  where+    loop =+      isEOF >>= \case+        True -> return ()+        False -> do+          ws <- words <$> getLine+          more <- readTVarIO mock.extra+          let tip = last (blocks <> more)+              h0 = length blocks + length more+              new = case ws of+                ["idle", n] -> grow tip (h0 + 1) [const [] | _ <- [1 .. read n :: Int]]+                ["pay"] -> grow tip (h0 + 1) [\h -> spends h (txHash (coinbase (h - 1)))]+                ["addr", m, i] -> grow tip (h0 + 1) [\h -> [toAddr (read m) (read i) (txHash (coinbase (h - 1)))]]+                _ -> []+          unless (null new) $ do+            atomically $ modifyTVar' mock.extra (<> new)+            ps <- readTVarIO mock.peers+            let announce = MHeaders (Headers [(b.header, VarInt (fromIntegral (length b.txs))) | b <- new])+            forM_ ps $ \s -> runPut (putMessage net announce) `send` s+            putStrLn ("mined " <> show (h0 + length new))+            hFlush stdout+          loop+    grow _ _ [] = []+    grow parent h (f : fs) =+      let b = mine net parent.header (coinbase h : f h)+       in b : grow b (h + 1) fs+    toAddr m i cb =+      Tx+        1+        [TxIn (OutPoint cb 0) "" maxBound]+        [TxOut 400000000 (scriptOf (xpubAddr m i)), TxOut 4500000000 hotScript]+        []+        0+ addrText :: Network -> Address -> T.Text addrText net = fromMaybe (error "address") . addrToText net @@ -92,9 +185,11 @@       _ -> Nothing     waitHeight store sub n +-- | The store of the benchmark, with the megabytes of its block cache and+-- of its write buffer, if not RocksDB's default. storeConfig ::-  Ctx -> Network -> FilePath -> Maybe String -> [Block] -> TVar [Int] -> Logger -> StoreConfig-storeConfig ctx net dir redis blocks requests logger =+  Network -> FilePath -> Maybe String -> [Block] -> Mock -> Logger -> Int -> Maybe Int -> StoreConfig+storeConfig net dir redis blocks mock logger cache buffer =   StoreConfig     { maxPeers = 20,       -- the mock peer, by name: since haskoin-node 2.5 the seed of a regtest@@ -103,28 +198,25 @@       discover = False,       db = dir,       net = net,-      ctx = ctx,       redis = redis,       gap = 32,       initGap = 20,-      redisMinAddrs = 20,-      redisMinTxs = 1000,       redisMaxXPubs = 10000,-      noMempool = False,-      wipeMempool = False,-      fleet = False,-      syncMempool = False,+      xpubs = defaultXPubConfig,       blockWindow = 64,       mempoolTimeout = 14,       peerTimeout = 60,+      blockStall = 10,+      blockPing = 5,       maxPeerLife = 48 * 3600,-      connect = mockPeer blocks requests net ad,+      connect = mockPeer blocks mock net ad,       stats = Nothing,       bloom = True,       maxOpenFiles = Nothing,-      writeBufferSize = Nothing,+      writeBufferSize = (* (1024 * 1024)) <$> buffer,       maxWalSize = Nothing,-      blockCache = Just (1024 * 1024 * 1024),+      backgroundJobs = Nothing,+      blockCache = Just (cache * 1024 * 1024),       logger = logger     }   where@@ -144,12 +236,10 @@       minPeers = 0,       limits = def,       version = "bench",-      noMempool = False,       stats = Nothing,       tickerRefresh = 90 * 1000 * 1000,       tickerURL = "",       priceHistoryURL = "",-      noXPub = False,       noBlockchainInfo = False,       healthCheckInterval = 30,       accessLog = False,@@ -162,29 +252,28 @@ -- Synthetic chain -- --------------------- -benchXPub :: Ctx -> XPubKey-benchXPub ctx = deriveXPubKey ctx (makeXPrvKey "haskoin-store benchmark seed")+benchXPub :: XPubKey+benchXPub = deriveXPubKey (makeXPrvKey "haskoin-store benchmark seed") -xpubAddr :: Ctx -> KeyIndex -> KeyIndex -> Address-xpubAddr ctx m i = fst (deriveAddr ctx (pubSubKey ctx (benchXPub ctx) m) i)+xpubAddr :: KeyIndex -> KeyIndex -> Address+xpubAddr m i = fst (deriveAddr (pubSubKey benchXPub m) i) -hotAddr :: Ctx -> Address-hotAddr ctx = pubKeyAddr ctx (derivePublicKey ctx (wrapSecKey True key))+hotAddr :: Address+hotAddr = pubKeyAddr (derivePublicKey (wrapSecKey True key))   where     key = fromMaybe (error "key") (secKey (B.replicate 32 0x42)) -scriptOf :: Ctx -> Address -> ByteString-scriptOf ctx = fromMaybe (error "script") . addressToScriptBS ctx+scriptOf :: Address -> ByteString+scriptOf = fromMaybe (error "script") . addressToScriptBS  -- | Blocks at heights 1 to n on top of the regtest genesis. The last block -- of a chain of 400 blocks or more also spends 3,000 outputs that earlier -- blocks paid to the hot address, one transaction each, so that there is a -- large block to answer about.-chain :: Ctx -> Network -> Int -> [Block]-chain ctx net n = go 1 net.genesisHeader+chain :: Network -> Int -> [Block]+chain net n = go 1 net.genesisHeader   where-    hot = scriptOf ctx (hotAddr ctx)-    coinbase h = Tx 1 [TxIn nullOutPoint (encode (fromIntegral h :: Word32)) maxBound] [TxOut 5000000000 hot] [] 0+    hot = hotScript     -- the transactions of the block at a height, after its coinbase     regular h       | h <= 1 = []@@ -202,19 +291,32 @@       | h <- [2 .. 335],         t <- drop 1 (regular h)       ]-    spends h cb =-      let t1 = Tx 1 [TxIn (OutPoint cb 0) "" maxBound] (replicate 9 (TxOut 500000000 hot)) [] 0-          tk k =-            let j = fromIntegral (h * 9 + k) :: Word32-                a = xpubAddr ctx (j `mod` 2) ((j `div` 2) `mod` 50)-             in Tx-                  1-                  [TxIn (OutPoint (txHash t1) (fromIntegral k)) "" maxBound]-                  [TxOut 400000000 (scriptOf ctx a), TxOut 99000000 hot]-                  []-                  0-       in t1 : map tk [0 .. 8] +-- | The script of the hot address.+hotScript :: ByteString+hotScript = scriptOf hotAddr++-- | The coinbase of the block at a height: it pays the hot address.+coinbase :: Int -> Tx+coinbase h = Tx 1 [TxIn nullOutPoint (encode (fromIntegral h :: Word32)) maxBound] [TxOut 5000000000 hotScript] [] 0++-- | The transactions of a block at a height but its coinbase, spending a+-- coinbase: one that splits it in nine, and nine that each pay an address of+-- the xpub, in turn, and the hot address.+spends :: Int -> TxHash -> [Tx]+spends h cb =+  let t1 = Tx 1 [TxIn (OutPoint cb 0) "" maxBound] (replicate 9 (TxOut 500000000 hotScript)) [] 0+      tk k =+        let j = fromIntegral (h * 9 + k) :: Word32+            a = xpubAddr (j `mod` 2) ((j `div` 2) `mod` 50)+         in Tx+              1+              [TxIn (OutPoint (txHash t1) (fromIntegral k)) "" maxBound]+              [TxOut 400000000 (scriptOf a), TxOut 99000000 hotScript]+              []+              0+   in t1 : map tk [0 .. 8]+ mine :: Network -> BlockHeader -> [Tx] -> Block mine net parent txs = Block header txs   where@@ -233,12 +335,11 @@ -- result between calls. micro :: IO () micro = do-  ctx <- createContext   let net = bchRegTest-      blocks = chain ctx net 66+      blocks = chain net 66       txs = V.fromList [b.txs !! 2 | b <- drop 2 blocks]-      addrs = V.generate 64 (xpubAddr ctx 0 . fromIntegral)-      hot = scriptOf ctx (hotAddr ctx)+      addrs = V.generate 64 (xpubAddr 0 . fromIntegral)+      hot = scriptOf hotAddr       txdata tx =         TxData           { block = BlockRef 3 2,@@ -251,7 +352,7 @@           }       tds = V.map txdata txs       tdbytes = V.map encode tds-      ts = V.map (toTransaction net ctx) tds+      ts = V.map (toTransaction net) tds       pick :: V.Vector a -> Int -> a       pick v i = v V.! (i `mod` 64)   _ <- evaluate (force (tds, tdbytes, ts, addrs))@@ -260,7 +361,7 @@   bench "Address JSON (cashaddr)" (encodingToLazyByteString . marshalEncoding net . pick addrs)   bench "txHash" (txHash . pick txs)   bench "TxData decode" (\i -> either error id (decode (pick tdbytes i)) :: TxData)-  bench "toTransaction" (toTransaction net ctx . pick tds)+  bench "toTransaction" (toTransaction net . pick tds)   bench "Transaction JSON" (encodingToLazyByteString . marshalEncoding net . pick ts)   bench "TxHash JSON" (encodingToLazyByteString . toEncoding . txHash . pick txs)   bench "BlockRef JSON" (\i -> encodingToLazyByteString (toEncoding (BlockRef (fromIntegral i) 1)))@@ -284,14 +385,15 @@ -- Mock peer -- --------------- -mockPeer :: [Block] -> TVar [Int] -> Network -> NetworkAddress -> SockAddr -> WithConnection-mockPeer blocks _requests net ad sa f = do+mockPeer :: [Block] -> Mock -> Network -> NetworkAddress -> SockAddr -> WithConnection+mockPeer blocks mock net ad sa f = do   r <- newInbox   s <- newInbox   let s' = inboxToMailbox s+  atomically $ modifyTVar' mock.peers (s' :)   withAsync (go r s') $ \_ -> do-    let o = awaitForever (`send` r)-        i = forever (receive s >>= yield)+    let o = conduitMailbox r+        i = repeatMC (receive s)     f (Conduits i o) :: IO ()   where     heights = HashMap.fromList (zip (map (headerHash . (.header)) blocks) [1 :: Int ..])@@ -299,16 +401,16 @@     go :: Inbox ByteString -> Mailbox ByteString -> IO ()     go r s = do       nonce <- randomIO-      now <- round <$> liftIO getPOSIXTime+      now <- round <$> getPOSIXTime       let rmt = NetworkAddress 0 (sockToHostAddress sa)           ver = buildVersion net nonce 0 ad rmt now       runPut (putMessage net (MVersion ver)) `send` s       runConduit $-        forever (receive r >>= yield)+        repeatMC (receive r)           .| inc-          .| concatMapC react+          .| concatMapMC (liftIO . react)           .| mapC (runPut . putMessage net)-          .| awaitForever (`send` s)+          .| conduitMailbox s     inc = forever $ do       x <- takeCE 24 .| foldC       case decode x of@@ -318,13 +420,20 @@           case runGet (getMessage net) (x <> y) of             Right m -> yield m             Left e -> error ("mock peer: " <> e)-    react (MPing (Ping n)) = [MPong (Pong n)]-    react (MVersion _) = [MVerAck]-    react (MGetHeaders (GetHeaders _ locator _)) =-      let start = fromMaybe 0 (listToMaybe (mapMaybe (`HashMap.lookup` heights) locator))+    -- the blocks mined after the chain are few: they are looked up in a list+    react :: Message -> IO [Message]+    react (MPing (Ping n)) = return [MPong (Pong n)]+    react (MVersion _) = return [MVerAck]+    react (MGetHeaders (GetHeaders _ locator _)) = do+      more <- atomically (readTVar mock.extra)+      let later = HashMap.fromList (zip (map (headerHash . (.header)) more) [length blocks + 1 ..])+          at l = HashMap.lookup l heights <|> HashMap.lookup l later+          start = fromMaybe 0 (listToMaybe (mapMaybe at locator))           hdr :: Block -> (BlockHeader, VarInt)           hdr b = (b.header, VarInt (fromIntegral (length b.txs)))-       in [MHeaders (Headers (map hdr (take 2000 (drop start blocks))))]-    react (MGetData (GetData ivs)) =-      [MBlock b | InvVector InvBlock h <- ivs, Just b <- [HashMap.lookup (BlockHash h) byHash]]-    react _ = []+      return [MHeaders (Headers (map hdr (take 2000 (drop start (blocks <> more)))))]+    react (MGetData (GetData ivs)) = do+      more <- atomically (readTVar mock.extra)+      let found h = HashMap.lookup h byHash <|> find ((== h) . headerHash . (.header)) more+      return [MBlock b | InvVector InvBlock h <- ivs, Just b <- [found (BlockHash h)]]+    react _ = return []
+ bench/Merge.hs view
@@ -0,0 +1,192 @@+-- | What merge records cost the readers of the database.+--+-- > haskoin-store-bench merge [KEYS [READS [CACHE_MIB [DIR]]]]+--+-- Since database version 23 the import of a block does not read the+-- balances of its addresses or the spenders of the outputs that it spends:+-- it writes merge records, which say how they change, and RocksDB works+-- out the values when they are read ("Haskoin.Store.Database.Merge"). The+-- work has moved to whoever reads a key that has merge records which no+-- compaction has reached yet: RocksDB looks into every table file that has+-- one of them, down to the value, and merges. This says how long that+-- takes.+--+-- It builds a database with KEYS balances and KEYS groups of spenders+-- (100,000) for each of four kinds of key: with no merge record, with one,+-- with three and with six. Every key gets a whole value first, and those+-- are compacted to the last level, where the data of a store is. Then come+-- six rounds: each writes a merge record for the keys that are to have as+-- many, and flushes, so that each round is a table file of its own.+-- RocksDB is kept from compacting them, which a store is not: there, what+-- is measured here lasts until a compaction has merged the records.+--+-- Three reads are timed for each kind of key, READS of them (20,000) at+-- random keys, from one thread, five times over: the balance of an+-- address, the spender of an output, and all the spenders of a+-- transaction, which is a scan with an iterator. Then everything is+-- compacted, which leaves whole values, and the same reads are timed+-- again: the same keys and the same answers, as a store had them before.+--+-- The block cache is CACHE_MIB (256), which holds all of it, so that what+-- is timed is the work of RocksDB and of the merge and not the disk. With+-- a cache too small for the data, such as 1, every block that is looked+-- into is fetched and decompressed again, as on a store whose cache is a+-- fraction of its database. DIR is where the database goes: a temporary+-- directory unless given.+module Merge (mergeBench) where++import Control.Monad (forM, forM_, unless, when)+import Data.ByteString (ByteString)+import Data.ByteString qualified as B+import Data.ByteString.Char8 qualified as C+import Data.IntMap.Strict qualified as IntMap+import Data.List (sort)+import Data.Maybe (fromMaybe)+import Data.Serialize (encode)+import Data.Word (Word32, Word64)+import Database.RocksDB+  ( BatchOp (..),+    Config (..),+    Property (..),+    compactRange,+    flush,+    getPropertyCF,+    write,+  )+import GHC.Clock (getMonotonicTime)+import Haskoin+import Haskoin.Store.Data (Balance (..), Spender (..))+import Haskoin.Store.Database.Query qualified as Q+import Haskoin.Store.Database.Reader+  ( DatabaseReader (..),+    balanceCF,+    layeredFamilyConfig,+    spenderCF,+    withReader,+  )+import Haskoin.Store.Database.Types (BalKey (..), BalVal (..), SpenderKey (..), SpenderVal (..))+import Haskoin.Store.Integration.Temp (withSystemTempDirectory, withTempDirectory)+import System.IO (hFlush, stdout)+import Text.Printf (printf)++net :: Network+net = btcRegTest++-- | The numbers of merge records that the four kinds of key have.+pending :: [Int]+pending = [0, 1, 3, 6]++mergeBench :: [String] -> IO ()+mergeBench args = do+  let number i d = maybe d read (lookup i (zip [0 :: Int ..] args))+      keys = number 0 100000+      reads' = number 1 20000+      cache = number 2 256 * 1024 * 1024+      inDir = case drop 3 args of+        dir : _ -> withTempDirectory dir "haskoin-store-merge-"+        [] -> withSystemTempDirectory "haskoin-store-merge-"+  inDir $ \dir -> open cache dir $ \db -> do+    let balances = balanceCF db.db+        spenders = spenderCF db.db+        both action = action balances >> action spenders+        files cf = maybe "?" C.unpack <$> getPropertyCF db.db cf (NumFilesAtLevel 0)+    printf "%d balances and %d groups of spenders for each number of merge records\n" keys keys+    hFlush stdout+    -- a whole value for every key, at the last level+    forM_ pending $ \kind ->+      forM_ (chunks 5000 [0 .. keys - 1]) $ \some ->+        write db.db $+          concat+            [ [ PutCF balances (encode (BalKey (address kind i))) (encode whole),+                PutCF spenders (encode (SpenderKey (made kind i) 0)) (encode (SpenderVal (entries [0, 1])))+              ]+            | i <- some+            ]+    both (flush db.db . Just)+    both (\cf -> compactRange db.db (Just cf) Nothing Nothing)+    -- a merge record a round for the keys that are to have as many, each+    -- round in a table file of its own+    forM_ [1 .. maximum pending] $ \turn -> do+      forM_ (filter (>= turn) pending) $ \kind ->+        forM_ (chunks 5000 [0 .. keys - 1]) $ \some ->+          write db.db $+            concat+              [ [ MergeCF balances (encode (BalKey (address kind i))) (encode difference),+                  MergeCF spenders (encode (SpenderKey (made kind i) 0)) (encode (SpenderVal (entries [1 + turn])))+                ]+              | i <- some+              ]+      both (flush db.db . Just)+    level0 <- (,) <$> files balances <*> files spenders+    printf "table files above the last level: %s of balances, %s of spenders\n\n" (fst level0) (snd level0)+    hFlush stdout+    let measure = forM pending $ \kind -> do+          let picked = take reads' [n `mod` keys | n <- iterate (\n -> (n * 7919 + 104729) `mod` 1000003) (kind + 1)]+              time action = do+                -- once unmeasured, so that the cache has what there is+                mapM_ action picked+                passes <- forM [1 .. 5 :: Int] $ \_ -> do+                  t0 <- getMonotonicTime+                  mapM_ action picked+                  t1 <- getMonotonicTime+                  return (1e6 * (t1 - t0) / fromIntegral (length picked))+                return (sort passes !! 2)+          balance <- time $ \i -> do+            b <- Q.getBalance db (address kind i)+            unless (fmap (.txs) b == Just (fromIntegral (1 + kind))) (fail "a balance is not what was merged")+          spender <- time $ \i -> do+            found <- Q.getChainSpender db (OutPoint (made kind i) (if kind == 0 then 1 else 2))+            when (found == Nothing) (fail "a spender is not there")+          scan <- time $ \i -> do+            m <- Q.getChainSpenders db (made kind i)+            unless (IntMap.size m == 2 + kind) (fail "the spenders are not those that were merged")+          return (kind, balance :: Double, spender :: Double, scan :: Double)+    merged <- measure+    both (\cf -> compactRange db.db (Just cf) Nothing Nothing)+    compacted <- measure+    printf "microseconds a read, median of 5 passes of %d reads at random keys, one thread\n" reads'+    printf "block cache of %d MiB\n\n" (cache `div` (1024 * 1024))+    let table :: String -> ((Int, Double, Double, Double) -> Double) -> IO ()+        table title of' = do+          printf "%s\n" title+          printf "  %-22s %16s %18s\n" (s "merge records") (s "in table files") (s "after compaction")+          forM_ (zip merged compacted) $ \(m@(kind, _, _, _), c) ->+            printf "  %-22d %16.1f %18.1f\n" kind (of' m) (of' c)+          printf "\n"+    table "the balance of an address (a point read)" (\(_, x, _, _) -> x)+    table "the spender of an output (a point read of its group)" (\(_, _, x, _) -> x)+    table "the spenders of a transaction (a scan of its groups)" (\(_, _, _, x) -> x)+    hFlush stdout+  where+    s :: String -> String+    s = id+    whole = BalVal {confirmed = 5000, unconfirmed = 0, utxo = 2, txs = 1, received = 5000}+    difference = BalVal {confirmed = 1000, unconfirmed = 0, utxo = 1, txs = 1, received = 1000}+    entries outputs = IntMap.fromList [(o, Spender (made 9 o) (fromIntegral o)) | o <- outputs]++-- | Open a database as the store does, with a block cache and Bloom+-- filters, but for the compactions of RocksDB, which are off: a merge+-- record stays in the table file that it was flushed to.+open :: Int -> FilePath -> (DatabaseReader -> IO a) -> IO a+open cache dir =+  withReader families (\_ -> return ()) net 20 32 dir True Nothing Nothing Nothing Nothing (Just cache) Nothing+  where+    families bloom cached =+      [(name, config {disableAutoCompactions = True}) | (name, config) <- layeredFamilyConfig bloom cached]++-- | The address of a kind of key and a number.+address :: Int -> Int -> Address+address kind i =+  fromMaybe (error "no address") . eitherToMaybe . scriptToAddressBS net $+    "\x76\xa9\x14" <> B.take 20 (hashOf kind i) <> "\x88\xac"++-- | The transaction of a kind of key and a number.+made :: Int -> Int -> TxHash+made kind i = TxHash (sha256 (hashOf kind i))++hashOf :: Int -> Int -> ByteString+hashOf kind i = encode (sha256 (encode (fromIntegral kind :: Word32, fromIntegral i :: Word64)))++chunks :: Int -> [a] -> [[a]]+chunks _ [] = []+chunks n xs = let (a, b) = splitAt n xs in a : chunks n b
+ cbits/haskoin_store_merge.c view
@@ -0,0 +1,294 @@+/*+ * The merge operators of haskoin-store. See haskoin_store_merge.h for what+ * each function is for and the formats that it reads and writes.+ */++#include "haskoin_store_merge.h"++#include <stdint.h>+#include <stdlib.h>+#include <string.h>++/* The numbers of a balance. */+#define BALANCE_FIELDS 5++/* The most entries that the byte in front of a group can count. */+#define MOST_SPENDERS 255++/* Say that a merge failed, the way RocksDB's C API wants it said. */+static char *fail(unsigned char *success, size_t *new_value_length)+{+    *success = 0;+    *new_value_length = 0;+    return NULL;+}++/* A number of 64 bits, most significant byte first. */+static uint64_t read_be64(const unsigned char *bytes)+{+    uint64_t number = 0;+    int i;++    for (i = 0; i < 8; i++) {+        number = (number << 8) | bytes[i];+    }+    return number;+}++static void write_be64(unsigned char *bytes, uint64_t number)+{+    int i;++    for (i = 7; i >= 0; i--) {+        bytes[i] = (unsigned char)(number & 0xff);+        number >>= 8;+    }+}++/*+ * Add the five numbers of a balance, or of a merge record, to five sums.+ * Returns zero if it is not of forty bytes.+ */+static int add_balance(uint64_t *sums, const char *bytes, size_t length)+{+    int i;++    if (length != HASKOIN_STORE_BALANCE_SIZE) {+        return 0;+    }+    for (i = 0; i < BALANCE_FIELDS; i++) {+        /* unsigned, so that it wraps around at 2^64 */+        sums[i] += read_be64((const unsigned char *)bytes + 8 * i);+    }+    return 1;+}++/*+ * The sum of a first balance, if there is one, and of some more.+ */+static char *sum_balances(+    const char *first,+    size_t first_length,+    const char *const *list,+    const size_t *lengths,+    int count,+    unsigned char *success,+    size_t *new_value_length)+{+    uint64_t sums[BALANCE_FIELDS] = {0, 0, 0, 0, 0};+    unsigned char *sum;+    int i;++    if (first != NULL && !add_balance(sums, first, first_length)) {+        return fail(success, new_value_length);+    }+    for (i = 0; i < count; i++) {+        if (!add_balance(sums, list[i], lengths[i])) {+            return fail(success, new_value_length);+        }+    }+    sum = malloc(HASKOIN_STORE_BALANCE_SIZE);+    if (sum == NULL) {+        return fail(success, new_value_length);+    }+    for (i = 0; i < BALANCE_FIELDS; i++) {+        write_be64(sum + 8 * i, sums[i]);+    }+    *success = 1;+    *new_value_length = HASKOIN_STORE_BALANCE_SIZE;+    return (char *)sum;+}++char *haskoin_store_balance_full_merge(+    void *state,+    const char *key,+    size_t key_length,+    const char *existing_value,+    size_t existing_value_length,+    const char *const *operands_list,+    const size_t *operands_list_length,+    int num_operands,+    unsigned char *success,+    size_t *new_value_length)+{+    (void)state;+    (void)key;+    (void)key_length;+    return sum_balances(+        existing_value,+        existing_value_length,+        operands_list,+        operands_list_length,+        num_operands,+        success,+        new_value_length);+}++char *haskoin_store_balance_partial_merge(+    void *state,+    const char *key,+    size_t key_length,+    const char *const *operands_list,+    const size_t *operands_list_length,+    int num_operands,+    unsigned char *success,+    size_t *new_value_length)+{+    (void)state;+    (void)key;+    (void)key_length;+    return sum_balances(NULL, 0, operands_list, operands_list_length, num_operands, success, new_value_length);+}++/*+ * The number of entries that a group says it has, or -1 if its length is+ * not that of so many entries after the byte that counts them.+ */+static int spender_count(const char *group, size_t length)+{+    size_t count;++    if (length < 1) {+        return -1;+    }+    count = (unsigned char)group[0];+    if (length != 1 + count * HASKOIN_STORE_SPENDER_SIZE) {+        return -1;+    }+    return (int)count;+}++/*+ * Put the entries of a group into a list of "*used" entries that is in+ * ascending order of output, each where its output belongs, and in place of+ * the entry that the list has for that output if it has one. The list has+ * room for every entry that it will be given.+ *+ * The number of the output is the first four bytes of an entry, most+ * significant first, so the order of the outputs is the order of those+ * bytes.+ */+static void add_spenders(unsigned char *entries, size_t *used, const char *group, int count)+{+    const size_t size = HASKOIN_STORE_SPENDER_SIZE;+    int i;++    for (i = 0; i < count; i++) {+        const char *entry = group + 1 + (size_t)i * size;+        size_t at = 0;+        int order = 1;++        /* groups are of sixteen outputs: a walk is as good as a search */+        while (at < *used) {+            order = memcmp(entry, entries + at * size, 4);+            if (order <= 0) {+                break;+            }+            at += 1;+        }+        if (at == *used || order < 0) {+            /* a new output: make room for it */+            memmove(entries + (at + 1) * size, entries + at * size, (*used - at) * size);+            *used += 1;+        }+        memcpy(entries + at * size, entry, size);+    }+}++/*+ * The union of a first group, if there is one, and of some more, from the+ * oldest to the newest.+ */+static char *join_spenders(+    const char *first,+    size_t first_length,+    const char *const *list,+    const size_t *lengths,+    int count,+    unsigned char *success,+    size_t *new_value_length)+{+    const size_t size = HASKOIN_STORE_SPENDER_SIZE;+    size_t most = 0;+    size_t used = 0;+    unsigned char *joined;+    int entries;+    int i;++    /* every group is checked before anything is read from one */+    if (first != NULL) {+        entries = spender_count(first, first_length);+        if (entries < 0) {+            return fail(success, new_value_length);+        }+        most += (size_t)entries;+    }+    for (i = 0; i < count; i++) {+        entries = spender_count(list[i], lengths[i]);+        if (entries < 0) {+            return fail(success, new_value_length);+        }+        most += (size_t)entries;+    }++    /* the byte that counts, and room for as many entries as there can be */+    joined = malloc(1 + most * size);+    if (joined == NULL) {+        return fail(success, new_value_length);+    }+    if (first != NULL) {+        add_spenders(joined + 1, &used, first, spender_count(first, first_length));+    }+    for (i = 0; i < count; i++) {+        add_spenders(joined + 1, &used, list[i], spender_count(list[i], lengths[i]));+    }+    if (used > MOST_SPENDERS) {+        free(joined);+        return fail(success, new_value_length);+    }+    joined[0] = (unsigned char)used;+    *success = 1;+    *new_value_length = 1 + used * size;+    return (char *)joined;+}++char *haskoin_store_spender_full_merge(+    void *state,+    const char *key,+    size_t key_length,+    const char *existing_value,+    size_t existing_value_length,+    const char *const *operands_list,+    const size_t *operands_list_length,+    int num_operands,+    unsigned char *success,+    size_t *new_value_length)+{+    (void)state;+    (void)key;+    (void)key_length;+    return join_spenders(+        existing_value,+        existing_value_length,+        operands_list,+        operands_list_length,+        num_operands,+        success,+        new_value_length);+}++char *haskoin_store_spender_partial_merge(+    void *state,+    const char *key,+    size_t key_length,+    const char *const *operands_list,+    const size_t *operands_list_length,+    int num_operands,+    unsigned char *success,+    size_t *new_value_length)+{+    (void)state;+    (void)key;+    (void)key_length;+    return join_spenders(NULL, 0, operands_list, operands_list_length, num_operands, success, new_value_length);+}
+ cbits/haskoin_store_merge.h view
@@ -0,0 +1,124 @@+/*+ * The merge operators of haskoin-store: the C functions that RocksDB calls+ * to work out an address balance and a group of spenders from the merge+ * records that the import of a block writes.+ *+ * Why: importing a block changes the balance of every address that it pays+ * or spends from, and adds a spender to the group of every output that it+ * spends. Each of those was read from the database, changed and written+ * back, and on a store of Bitcoin those reads were most of the time of a+ * block. A merge record says how a value changes and is written without+ * reading it. RocksDB works out the value later, when it is read or when a+ * compaction brings the records and the value together, with the functions+ * below. They run on the threads of RocksDB, so they are C.+ *+ * Both formats are made so that a merge record looks like a value. Then+ * applying records to a value and combining records with each other are+ * the same operation, and a record that is merged with no value is the+ * value.+ *+ * The functions have the types that rocksdb_mergeoperator_create of+ * rocksdb/c.h asks for. They return memory from malloc(), which RocksDB+ * frees. When their input is not of the format they fail: they store 0 in+ * "*success" and in "*new_value_length" and return NULL. They do nothing+ * else about such input.+ *+ * Haskoin.Store.Database.Merge gives them to RocksDB, and+ * docs/layered-database.md has the reasons.+ */++#ifndef HASKOIN_STORE_MERGE_H+#define HASKOIN_STORE_MERGE_H++#include <stddef.h>++/* Bytes of an address balance: five numbers of eight bytes. */+#define HASKOIN_STORE_BALANCE_SIZE 40++/*+ * Bytes of an entry of a group of spenders: the number of the output in+ * its transaction (4), the hash of the transaction that spends it (32) and+ * the number of the input that does (4).+ */+#define HASKOIN_STORE_SPENDER_SIZE 40++/*+ * Address balances: the values of the "balance" column family.+ *+ * A value is five unsigned numbers of 64 bits, most significant byte+ * first: the confirmed balance, the unconfirmed balance, the number of+ * unspent outputs, the number of transactions, and the amount received. It+ * is what the store has always written (BalVal).+ *+ * A merge record has the same forty bytes: five differences, each to be+ * added to its number modulo 2^64, so that a number goes down by adding+ * its two's complement. A key without a value counts from five zeros.+ *+ * The full merge adds the records to the value, and the partial merge adds+ * records to each other. Addition modulo 2^64 does not care about order or+ * grouping, so both are the same sum.+ */+char *haskoin_store_balance_full_merge(+    void *state,+    const char *key,+    size_t key_length,+    const char *existing_value,+    size_t existing_value_length,+    const char *const *operands_list,+    const size_t *operands_list_length,+    int num_operands,+    unsigned char *success,+    size_t *new_value_length);++char *haskoin_store_balance_partial_merge(+    void *state,+    const char *key,+    size_t key_length,+    const char *const *operands_list,+    const size_t *operands_list_length,+    int num_operands,+    unsigned char *success,+    size_t *new_value_length);++/*+ * Groups of spenders: the values of the "spender" column family.+ *+ * A value is one byte with the number of its entries, then the entries in+ * ascending order of their first field, each of+ * HASKOIN_STORE_SPENDER_SIZE bytes: the number of an output in its+ * transaction, the hash of the transaction that spends it, and the number+ * of the input that does, the two numbers of 32 bits with the most+ * significant byte first. It is what the store has written since database+ * version 22 (SpenderVal).+ *+ * A merge record is a value of the same format, with the entries to add.+ * Merging is the union: every entry of the value and of the records, and+ * for an output that several have, the entry of the newest. The result is+ * in ascending order of output again.+ *+ * The full merge and the partial merge are the same union, the first with+ * the value as the oldest of what it is given.+ */+char *haskoin_store_spender_full_merge(+    void *state,+    const char *key,+    size_t key_length,+    const char *existing_value,+    size_t existing_value_length,+    const char *const *operands_list,+    const size_t *operands_list_length,+    int num_operands,+    unsigned char *success,+    size_t *new_value_length);++char *haskoin_store_spender_partial_merge(+    void *state,+    const char *key,+    size_t key_length,+    const char *const *operands_list,+    const size_t *operands_list_length,+    int num_operands,+    unsigned char *success,+    size_t *new_value_length);++#endif
+ cbits/haskoin_store_probe.c view
@@ -0,0 +1,162 @@+/*+ * The bell of the probes of Haskoin.Store.Web.Congestion: a thread of the+ * system, outside of GHC, that writes to one eventfd for each probe every+ * period, and notes when it wrote and how long GHC's collector had run by+ * then. The probe, a Haskell thread, waits for its eventfd in+ * threadWaitRead, as a connection waits for its socket: GHC's manager of+ * input and output sees the eventfd in epoll and wakes the probe, which+ * then waits in its capability's queue of threads that are ready, as a+ * request does. From the write to the moment the probe runs is how long a+ * request that the network woke would have waited.+ */+#include <errno.h>+#include <pthread.h>+#include <stdint.h>+#include <stdlib.h>+#include <sys/eventfd.h>+#include <time.h>+#include <unistd.h>++#include "Rts.h"++struct haskoin_probe {+    int fd;+    pthread_mutex_t lock;+    /* whether a write is waiting for the probe to take it */+    int pending;+    /* when the oldest write that the probe did not take was made, in+     * nanoseconds of the monotonic clock */+    int64_t written;+    /* nanoseconds that GHC's collector had run by then */+    int64_t collected;+};++struct haskoin_probes {+    int n;+    int period_us;+    /* whether GHC keeps its statistics (+RTS -T) */+    int stats;+    int stop;+    pthread_t thread;+    struct haskoin_probe probe[];+};++static int64_t monotonic(void)+{+    struct timespec t;+    clock_gettime(CLOCK_MONOTONIC, &t);+    return (int64_t)t.tv_sec * 1000000000 + t.tv_nsec;+}++static void add_nanos(struct timespec *t, int64_t ns)+{+    int64_t s = t->tv_nsec + ns;+    t->tv_sec += s / 1000000000;+    t->tv_nsec = s % 1000000000;+}++static void *ring(void *arg)+{+    struct haskoin_probes *ps = arg;+    struct timespec next;+    clock_gettime(CLOCK_MONOTONIC, &next);+    while (!__atomic_load_n(&ps->stop, __ATOMIC_ACQUIRE)) {+        add_nanos(&next, (int64_t)ps->period_us * 1000);+        /* a bell that fell behind starts again from now, rather than+         * ringing many times at once to catch up */+        int64_t now = monotonic();+        if ((int64_t)next.tv_sec * 1000000000 + next.tv_nsec < now) {+            next.tv_sec = now / 1000000000;+            next.tv_nsec = now % 1000000000;+        }+        while (clock_nanosleep(CLOCK_MONOTONIC, TIMER_ABSTIME, &next, NULL) == EINTR) {+        }+        int64_t collected = 0;+        if (ps->stats) {+            RTSStats s;+            getRTSStats(&s);+            collected = (int64_t)s.gc_elapsed_ns;+        }+        for (int i = 0; i < ps->n; i++) {+            struct haskoin_probe *p = &ps->probe[i];+            pthread_mutex_lock(&p->lock);+            if (!p->pending) {+                p->pending = 1;+                p->written = monotonic();+                p->collected = collected;+            }+            pthread_mutex_unlock(&p->lock);+            uint64_t one = 1;+            ssize_t w = write(p->fd, &one, sizeof one);+            (void)w;+        }+    }+    return NULL;+}++static void release(struct haskoin_probes *ps, int made)+{+    for (int i = 0; i < made; i++) {+        pthread_mutex_destroy(&ps->probe[i].lock);+        close(ps->probe[i].fd);+    }+    free(ps);+}++/* Make n probes that ring every period, with their eventfds in fds, and+ * start the bell; NULL if the system refused. */+struct haskoin_probes *haskoin_probes_start(int n, int period_us, int stats, int *fds)+{+    struct haskoin_probes *ps = calloc(1, sizeof *ps + (size_t)n * sizeof(struct haskoin_probe));+    if (ps == NULL)+        return NULL;+    ps->n = n;+    ps->period_us = period_us;+    ps->stats = stats;+    for (int i = 0; i < n; i++) {+        int fd = eventfd(0, EFD_NONBLOCK | EFD_CLOEXEC);+        if (fd < 0) {+            release(ps, i);+            return NULL;+        }+        ps->probe[i].fd = fd;+        pthread_mutex_init(&ps->probe[i].lock, NULL);+        fds[i] = fd;+    }+    if (pthread_create(&ps->thread, NULL, ring, ps) != 0) {+        release(ps, n);+        return NULL;+    }+    return ps;+}++/* Stop the bell and let go of everything but the eventfds, which the caller+ * closes through GHC's manager of input and output. */+void haskoin_probes_stop(struct haskoin_probes *ps)+{+    __atomic_store_n(&ps->stop, 1, __ATOMIC_RELEASE);+    pthread_join(ps->thread, NULL);+    for (int i = 0; i < ps->n; i++)+        pthread_mutex_destroy(&ps->probe[i].lock);+    free(ps);+}++/* Take the oldest write that probe i did not take, emptying its eventfd:+ * 1 with when it was made and the collector's time then, or 0 if there is+ * none, which happens when two writes were taken at once. */+int haskoin_probe_take(struct haskoin_probes *ps, int i, int64_t *written, int64_t *collected)+{+    struct haskoin_probe *p = &ps->probe[i];+    uint64_t count;+    ssize_t r = read(p->fd, &count, sizeof count);+    (void)r;+    pthread_mutex_lock(&p->lock);+    int had = p->pending;+    if (had) {+        *written = p->written;+        *collected = p->collected;+        p->pending = 0;+    }+    pthread_mutex_unlock(&p->lock);+    return had;+}
haskoin-store.cabal view
@@ -5,7 +5,7 @@ -- see: https://github.com/sol/hpack  name:           haskoin-store-version:        1.15.1+version:        1.16.0 synopsis:       Storage and index for Bitcoin and Bitcoin Cash description:    An indexer for Bitcoin and Bitcoin Cash. It downloads blocks and mempool                 transactions from the peer-to-peer network, indexes them in RocksDB, and@@ -26,18 +26,17 @@ tested-with:     GHC == 9.12.4 extra-source-files:-    src/Haskoin/Store/Cache/lua/bodies.lua-    src/Haskoin/Store/Cache/lua/commit.lua-    src/Haskoin/Store/Cache/lua/delete.lua-    src/Haskoin/Store/Cache/lua/evict.lua-    src/Haskoin/Store/Cache/lua/page.lua-    src/Haskoin/Store/Cache/lua/spends.lua-    src/Haskoin/Store/Cache/lua/unlock.lua+    cbits/haskoin_store_merge.h     src/Haskoin/Store/Fleet/lua/move.lua     src/Haskoin/Store/Fleet/lua/pool.lua     src/Haskoin/Store/Fleet/lua/read.lua     src/Haskoin/Store/Fleet/lua/report.lua     src/Haskoin/Store/Fleet/lua/xpub.lua+    src/Haskoin/Store/XPub/lua/greedy.lua+    src/Haskoin/Store/XPub/lua/peek.lua+    src/Haskoin/Store/XPub/lua/release.lua+    src/Haskoin/Store/XPub/lua/renew.lua+    src/Haskoin/Store/XPub/lua/take.lua     stress/wrk.lua     swagger.yaml extra-doc-files:@@ -54,35 +53,39 @@       Haskoin.Store.BlockStore       Haskoin.Store.BlockStore.Pending       Haskoin.Store.BlockStore.Sync-      Haskoin.Store.Cache-      Haskoin.Store.Cache.Protocol       Haskoin.Store.Common-      Haskoin.Store.Database.Mempool+      Haskoin.Store.Database.Merge       Haskoin.Store.Database.Migrate       Haskoin.Store.Database.Query-      Haskoin.Store.Database.Query.Layered       Haskoin.Store.Database.Query.Pooled       Haskoin.Store.Database.Reader       Haskoin.Store.Database.Slices+      Haskoin.Store.Database.Table       Haskoin.Store.Database.Types       Haskoin.Store.Database.Views       Haskoin.Store.Database.Writer+      Haskoin.Store.Exception       Haskoin.Store.Fleet       Haskoin.Store.Fleet.Member       Haskoin.Store.Fleet.Pool       Haskoin.Store.Fleet.Redis       Haskoin.Store.Fleet.XPub       Haskoin.Store.Logic-      Haskoin.Store.Logic.Layered       Haskoin.Store.Manager       Haskoin.Store.Outage       Haskoin.Store.Runtime       Haskoin.Store.Web+      Haskoin.Store.Web.Admission+      Haskoin.Store.Web.Congestion       Haskoin.Store.Web.Docs       Haskoin.Store.Web.Events       Haskoin.Store.Web.Pieces       Haskoin.Store.Web.Publish       Haskoin.Store.Web.Router+      Haskoin.Store.XPub+      Haskoin.Store.XPub.Indexer+      Haskoin.Store.XPub.Lock+      Haskoin.Store.XPub.Service   other-modules:       Paths_haskoin_store   autogen-modules:@@ -95,8 +98,15 @@       OverloadedRecordDot       OverloadedStrings   ghc-options: -Wall -Wcompat -Widentities -Wincomplete-record-updates -Wincomplete-uni-patterns -Wmissing-export-lists -Wmissing-home-modules -Wpartial-fields -Wredundant-constraints+  cc-options: -Wall+  include-dirs:+      cbits+  c-sources:+      cbits/haskoin_store_merge.c+      cbits/haskoin_store_probe.c   build-depends:       aeson >=2.2.5.1 && <2.3+    , async >=2.2.6 && <2.3     , base >=4.21.2.0 && <4.22     , base16 ==2.0.*     , bytes >=0.17.5 && <0.18@@ -110,26 +120,26 @@     , file-embed >=0.0.16.0 && <0.1     , hashable >=1.5.1.0 && <1.6     , hashtables >=1.4.2 && <1.5-    , haskoin-core >=1.6.4 && <1.7-    , haskoin-node >=2.5.1 && <2.6-    , haskoin-store-data >=2.0.2 && <2.1+    , haskoin-core >=2.0.0 && <2.1+    , haskoin-node >=2.6.0 && <2.7+    , haskoin-store-data >=3.0.0 && <3.1     , hedis >=0.16.4 && <0.17     , http-types >=0.12.6 && <0.13     , lens >=5.3.6 && <5.4     , monad-control >=1.0.3.1 && <1.1     , mtl >=2.3.2 && <2.4     , network >=3.2.12.0 && <3.3-    , nqe >=0.6.9 && <0.7+    , nqe >=0.7.0 && <0.8     , random >=1.3.1 && <1.4-    , rocksdb-haskell-jprupp >=2.7.1 && <2.8-    , rocksdb-query >=0.6.1 && <0.7-    , statsd-rupp >=0.5.0.2 && <0.6+    , rocksdb-haskell-jprupp >=2.8.0 && <2.9+    , rocksdb-query >=0.7.0 && <0.8+    , statsd-rupp >=0.6.0 && <0.7     , stm >=2.5.3.1 && <2.6     , string-conversions >=0.4.0.1 && <0.5     , text >=2.1.4 && <2.2     , time ==1.14.*+    , time-manager >=0.4.2 && <0.5     , transformers >=0.6.3.0 && <0.7-    , unliftio >=0.2.25.1 && <0.3     , unordered-containers >=0.2.21 && <0.3     , vault >=0.3.2.0 && <0.4     , wai >=3.2.5 && <3.3@@ -146,6 +156,7 @@       Haskoin.Store.Integration.Image       Haskoin.Store.Integration.Node       Haskoin.Store.Integration.Podman+      Haskoin.Store.Integration.Temp       Haskoin.Store.Integration.Valkey   other-modules:       Paths_haskoin_store@@ -161,6 +172,7 @@   ghc-options: -Wall -Wcompat -Widentities -Wincomplete-record-updates -Wincomplete-uni-patterns -Wmissing-export-lists -Wmissing-home-modules -Wpartial-fields -Wredundant-constraints   build-depends:       aeson >=2.2.5.1 && <2.3+    , async >=2.2.6 && <2.3     , base >=4.21.2.0 && <4.22     , base16 ==2.0.*     , bytes >=0.17.5 && <0.18@@ -171,30 +183,33 @@     , containers ==0.7.*     , data-default >=0.8.0.2 && <0.9     , deepseq >=1.5.1.0 && <1.6+    , directory >=1.3.10.1 && <1.4     , file-embed >=0.0.16.0 && <0.1     , hashable >=1.5.1.0 && <1.6     , hashtables >=1.4.2 && <1.5-    , haskoin-core >=1.6.4 && <1.7-    , haskoin-node >=2.5.1 && <2.6-    , haskoin-store-data >=2.0.2 && <2.1+    , haskoin-core >=2.0.0 && <2.1+    , haskoin-node >=2.6.0 && <2.7+    , haskoin-store+    , haskoin-store-data >=3.0.0 && <3.1     , hedis >=0.16.4 && <0.17     , http-types >=0.12.6 && <0.13     , lens >=5.3.6 && <5.4     , monad-control >=1.0.3.1 && <1.1     , mtl >=2.3.2 && <2.4     , network >=3.2.12.0 && <3.3-    , nqe >=0.6.9 && <0.7+    , nqe >=0.7.0 && <0.8     , process >=1.6.26.1 && <1.7     , random >=1.3.1 && <1.4-    , rocksdb-haskell-jprupp >=2.7.1 && <2.8-    , rocksdb-query >=0.6.1 && <0.7-    , statsd-rupp >=0.5.0.2 && <0.6+    , rocksdb-haskell-jprupp >=2.8.0 && <2.9+    , rocksdb-query >=0.7.0 && <0.8+    , statsd-rupp >=0.6.0 && <0.7     , stm >=2.5.3.1 && <2.6     , string-conversions >=0.4.0.1 && <0.5+    , temporary ==1.3.*     , text >=2.1.4 && <2.2     , time ==1.14.*+    , time-manager >=0.4.2 && <0.5     , transformers >=0.6.3.0 && <0.7-    , unliftio >=0.2.25.1 && <0.3     , unordered-containers >=0.2.21 && <0.3     , vault >=0.3.2.0 && <0.4     , wai >=3.2.5 && <3.3@@ -219,9 +234,10 @@       NoFieldSelectors       OverloadedRecordDot       OverloadedStrings-  ghc-options: -Wall -Wcompat -Widentities -Wincomplete-record-updates -Wincomplete-uni-patterns -Wmissing-export-lists -Wmissing-home-modules -Wpartial-fields -Wredundant-constraints -threaded -rtsopts "-with-rtsopts=-N -T --nonmoving-gc --disable-delayed-os-memory-return"+  ghc-options: -Wall -Wcompat -Widentities -Wincomplete-record-updates -Wincomplete-uni-patterns -Wmissing-export-lists -Wmissing-home-modules -Wpartial-fields -Wredundant-constraints -threaded -rtsopts "-with-rtsopts=-N -T -A64m --disable-delayed-os-memory-return"   build-depends:       aeson >=2.2.5.1 && <2.3+    , async >=2.2.6 && <2.3     , base >=4.21.2.0 && <4.22     , base16 ==2.0.*     , bytes >=0.17.5 && <0.18@@ -232,32 +248,33 @@     , containers ==0.7.*     , data-default >=0.8.0.2 && <0.9     , deepseq >=1.5.1.0 && <1.6+    , directory >=1.3.10.1 && <1.4     , file-embed >=0.0.16.0 && <0.1     , filepath >=1.5.5.0 && <1.6     , hashable >=1.5.1.0 && <1.6     , hashtables >=1.4.2 && <1.5-    , haskoin-core >=1.6.4 && <1.7-    , haskoin-node >=2.5.1 && <2.6+    , haskoin-core >=2.0.0 && <2.1+    , haskoin-node >=2.6.0 && <2.7     , haskoin-store-    , haskoin-store-data >=2.0.2 && <2.1+    , haskoin-store-data >=3.0.0 && <3.1     , hedis >=0.16.4 && <0.17     , http-types >=0.12.6 && <0.13     , lens >=5.3.6 && <5.4     , monad-control >=1.0.3.1 && <1.1     , mtl >=2.3.2 && <2.4     , network >=3.2.12.0 && <3.3-    , nqe >=0.6.9 && <0.7+    , nqe >=0.7.0 && <0.8     , optparse-applicative >=0.19.0.0 && <0.20     , random >=1.3.1 && <1.4-    , rocksdb-haskell-jprupp >=2.7.1 && <2.8-    , rocksdb-query >=0.6.1 && <0.7-    , statsd-rupp >=0.5.0.2 && <0.6+    , rocksdb-haskell-jprupp >=2.8.0 && <2.9+    , rocksdb-query >=0.7.0 && <0.8+    , statsd-rupp >=0.6.0 && <0.7     , stm >=2.5.3.1 && <2.6     , string-conversions >=0.4.0.1 && <0.5     , text >=2.1.4 && <2.2     , time ==1.14.*+    , time-manager >=0.4.2 && <0.5     , transformers >=0.6.3.0 && <0.7-    , unliftio >=0.2.25.1 && <0.3     , unordered-containers >=0.2.21 && <0.3     , vault >=0.3.2.0 && <0.4     , wai >=3.2.5 && <3.3@@ -275,29 +292,32 @@   other-modules:       Haskoin.Store.BlockStore.PendingSpec       Haskoin.Store.BlockStore.SyncSpec-      Haskoin.Store.Cache.ProtocolSpec-      Haskoin.Store.CacheSpec       Haskoin.Store.CommonSpec+      Haskoin.Store.Database.MergeSpec       Haskoin.Store.Database.MigrateSpec-      Haskoin.Store.Database.Query.LayeredSpec       Haskoin.Store.Database.SlicesSpec       Haskoin.Store.Database.ViewsSpec-      Haskoin.Store.Fleet.MemberSpec+      Haskoin.Store.Database.WriterSpec+      Haskoin.Store.ExceptionSpec+      Haskoin.Store.Fleet.CacheSpec       Haskoin.Store.Fleet.PoolSpec       Haskoin.Store.Fleet.RedisSpec       Haskoin.Store.Fleet.XPubSpec       Haskoin.Store.FleetSpec       Haskoin.Store.Histories+      Haskoin.Store.Integration.TempSpec       Haskoin.Store.IntegrationSpec-      Haskoin.Store.Logic.LayeredSpec       Haskoin.Store.LogicSpec       Haskoin.Store.OutageSpec       Haskoin.Store.TestBlocks+      Haskoin.Store.Web.AdmissionSpec+      Haskoin.Store.Web.CongestionSpec       Haskoin.Store.Web.EventsSpec       Haskoin.Store.Web.PiecesSpec       Haskoin.Store.Web.PublishSpec       Haskoin.Store.Web.RouterSpec       Haskoin.Store.WebSpec+      Haskoin.Store.XPub.IndexerSpec       Haskoin.StoreSpec       Paths_haskoin_store   autogen-modules:@@ -313,6 +333,7 @@   build-depends:       QuickCheck >=2.17.1.0 && <2.18     , aeson >=2.2.5.1 && <2.3+    , async >=2.2.6 && <2.3     , base >=4.21.2.0 && <4.22     , base16 ==2.0.*     , base64 >=1.0.1 && <1.1@@ -324,14 +345,16 @@     , containers ==0.7.*     , data-default >=0.8.0.2 && <0.9     , deepseq >=1.5.1.0 && <1.6+    , directory >=1.3.10.1 && <1.4     , fast-logger >=3.2.9 && <3.3     , file-embed >=0.0.16.0 && <0.1+    , filepath >=1.5.5.0 && <1.6     , hashable >=1.5.1.0 && <1.6     , hashtables >=1.4.2 && <1.5-    , haskoin-core >=1.6.4 && <1.7-    , haskoin-node >=2.5.1 && <2.6+    , haskoin-core >=2.0.0 && <2.1+    , haskoin-node >=2.6.0 && <2.7     , haskoin-store-    , haskoin-store-data >=2.0.2 && <2.1+    , haskoin-store-data >=3.0.0 && <3.1     , hedis >=0.16.4 && <0.17     , hspec >=2.11.18 && <2.12     , http-types >=0.12.6 && <0.13@@ -340,18 +363,18 @@     , monad-control >=1.0.3.1 && <1.1     , mtl >=2.3.2 && <2.4     , network >=3.2.12.0 && <3.3-    , nqe >=0.6.9 && <0.7+    , nqe >=0.7.0 && <0.8     , process >=1.6.26.1 && <1.7     , random >=1.3.1 && <1.4-    , rocksdb-haskell-jprupp >=2.7.1 && <2.8-    , rocksdb-query >=0.6.1 && <0.7-    , statsd-rupp >=0.5.0.2 && <0.6+    , rocksdb-haskell-jprupp >=2.8.0 && <2.9+    , rocksdb-query >=0.7.0 && <0.8+    , statsd-rupp >=0.6.0 && <0.7     , stm >=2.5.3.1 && <2.6     , string-conversions >=0.4.0.1 && <0.5     , text >=2.1.4 && <2.2     , time ==1.14.*+    , time-manager >=0.4.2 && <0.5     , transformers >=0.6.3.0 && <0.7-    , unliftio >=0.2.25.1 && <0.3     , unordered-containers >=0.2.21 && <0.3     , vault >=0.3.2.0 && <0.4     , wai >=3.2.5 && <3.3@@ -368,6 +391,9 @@   type: exitcode-stdio-1.0   main-is: Main.hs   other-modules:+      Admission+      Import+      Merge       Paths_haskoin_store   autogen-modules:       Paths_haskoin_store@@ -381,6 +407,7 @@   ghc-options: -Wall -Wcompat -Widentities -Wincomplete-record-updates -Wincomplete-uni-patterns -Wmissing-export-lists -Wmissing-home-modules -Wpartial-fields -Wredundant-constraints -threaded -rtsopts -with-rtsopts=-N   build-depends:       aeson >=2.2.5.1 && <2.3+    , async >=2.2.6 && <2.3     , base >=4.21.2.0 && <4.22     , base16 ==2.0.*     , bytes >=0.17.5 && <0.18@@ -391,30 +418,33 @@     , containers ==0.7.*     , data-default >=0.8.0.2 && <0.9     , deepseq >=1.5.1.0 && <1.6+    , directory >=1.3.10.1 && <1.4     , file-embed >=0.0.16.0 && <0.1+    , filepath >=1.5.5.0 && <1.6     , hashable >=1.5.1.0 && <1.6     , hashtables >=1.4.2 && <1.5-    , haskoin-core >=1.6.4 && <1.7-    , haskoin-node >=2.5.1 && <2.6+    , haskoin-core >=2.0.0 && <2.1+    , haskoin-node >=2.6.0 && <2.7     , haskoin-store-    , haskoin-store-data >=2.0.2 && <2.1+    , haskoin-store-data >=3.0.0 && <3.1     , hedis >=0.16.4 && <0.17     , http-types >=0.12.6 && <0.13+    , integration     , lens >=5.3.6 && <5.4     , monad-control >=1.0.3.1 && <1.1     , mtl >=2.3.2 && <2.4     , network >=3.2.12.0 && <3.3-    , nqe >=0.6.9 && <0.7+    , nqe >=0.7.0 && <0.8     , random >=1.3.1 && <1.4-    , rocksdb-haskell-jprupp >=2.7.1 && <2.8-    , rocksdb-query >=0.6.1 && <0.7-    , statsd-rupp >=0.5.0.2 && <0.6+    , rocksdb-haskell-jprupp >=2.8.0 && <2.9+    , rocksdb-query >=0.7.0 && <0.8+    , statsd-rupp >=0.6.0 && <0.7     , stm >=2.5.3.1 && <2.6     , string-conversions >=0.4.0.1 && <0.5     , text >=2.1.4 && <2.2     , time ==1.14.*+    , time-manager >=0.4.2 && <0.5     , transformers >=0.6.3.0 && <0.7-    , unliftio >=0.2.25.1 && <0.3     , unordered-containers >=0.2.21 && <0.3     , vault >=0.3.2.0 && <0.4     , vector >=0.13.2.0 && <0.14@@ -444,6 +474,7 @@   ghc-options: -Wall -Wcompat -Widentities -Wincomplete-record-updates -Wincomplete-uni-patterns -Wmissing-export-lists -Wmissing-home-modules -Wpartial-fields -Wredundant-constraints -threaded -rtsopts -with-rtsopts=-N2   build-depends:       aeson >=2.2.5.1 && <2.3+    , async >=2.2.6 && <2.3     , base >=4.21.2.0 && <4.22     , base16 ==2.0.*     , bytes >=0.17.5 && <0.18@@ -454,13 +485,14 @@     , containers ==0.7.*     , data-default >=0.8.0.2 && <0.9     , deepseq >=1.5.1.0 && <1.6+    , directory >=1.3.10.1 && <1.4     , file-embed >=0.0.16.0 && <0.1     , hashable >=1.5.1.0 && <1.6     , hashtables >=1.4.2 && <1.5-    , haskoin-core >=1.6.4 && <1.7-    , haskoin-node >=2.5.1 && <2.6+    , haskoin-core >=2.0.0 && <2.1+    , haskoin-node >=2.6.0 && <2.7     , haskoin-store-    , haskoin-store-data >=2.0.2 && <2.1+    , haskoin-store-data >=3.0.0 && <3.1     , hedis >=0.16.4 && <0.17     , http-types >=0.12.6 && <0.13     , integration@@ -468,19 +500,19 @@     , monad-control >=1.0.3.1 && <1.1     , mtl >=2.3.2 && <2.4     , network >=3.2.12.0 && <3.3-    , nqe >=0.6.9 && <0.7+    , nqe >=0.7.0 && <0.8     , optparse-applicative >=0.19.0.0 && <0.20     , process >=1.6.26.1 && <1.7     , random >=1.3.1 && <1.4-    , rocksdb-haskell-jprupp >=2.7.1 && <2.8-    , rocksdb-query >=0.6.1 && <0.7-    , statsd-rupp >=0.5.0.2 && <0.6+    , rocksdb-haskell-jprupp >=2.8.0 && <2.9+    , rocksdb-query >=0.7.0 && <0.8+    , statsd-rupp >=0.6.0 && <0.7     , stm >=2.5.3.1 && <2.6     , string-conversions >=0.4.0.1 && <0.5     , text >=2.1.4 && <2.2     , time ==1.14.*+    , time-manager >=0.4.2 && <0.5     , transformers >=0.6.3.0 && <0.7-    , unliftio >=0.2.25.1 && <0.3     , unordered-containers >=0.2.21 && <0.3     , vault >=0.3.2.0 && <0.4     , wai >=3.2.5 && <3.3
integration/Haskoin/Store/Integration/Image.hs view
@@ -47,9 +47,9 @@ import Data.Serialize (encode) import Data.Text qualified as T import Haskoin (encodeHex, sha256)+import Haskoin.Store.Integration.Temp (withSystemTempDirectory) import System.Exit (ExitCode (..)) import System.Process (readProcessWithExitCode)-import UnliftIO.Temporary (withSystemTempDirectory)  -- | Ubuntu 26.04, by the digest of its multi-architecture index. base :: String
integration/Haskoin/Store/Integration/Node.hs view
@@ -43,6 +43,8 @@   ) where +import Control.Concurrent (threadDelay)+import Control.Exception (Exception, throwIO) import Control.Lens ((&), (?~), (^.)) import Control.Monad (void) import Data.Aeson (Value (..), eitherDecode, object, (.=))@@ -53,11 +55,10 @@ import Data.Text qualified as T import Data.Word (Word64) import Haskoin (Network, bchRegTest, btcRegTest)+import Haskoin.Store.Exception (tryAny) import Haskoin.Store.Integration.Image (bchnBin, coreBin) import Haskoin.Store.Integration.Podman (ContainerSpec (..)) import Network.Wreq qualified as Wreq-import UnliftIO (Exception, throwIO, tryAny)-import UnliftIO.Concurrent (threadDelay)  -- | Which node software. data NodeKind = BitcoinCore | BitcoinCashNode
integration/Haskoin/Store/Integration/Podman.hs view
@@ -46,13 +46,14 @@ import Data.Char (isDigit, isSpace) import Data.List (dropWhileEnd) import Data.Text qualified as T+import Haskoin.Store.Exception (bracket, onException, tryAny)+import Haskoin.Store.Integration.Temp (withTempDirectory)+import System.Directory (doesDirectoryExist, doesPathExist, getTemporaryDirectory) import System.Environment (lookupEnv) import System.Exit (ExitCode (..)) import System.IO (hPutStrLn, stderr) import System.Process (getCurrentPid, readProcessWithExitCode)-import UnliftIO (bracket, onException, timeout, tryAny)-import UnliftIO.Directory (doesDirectoryExist, doesPathExist, getTemporaryDirectory)-import UnliftIO.Temporary (withTempDirectory)+import System.Timeout (timeout)  -- | Whether Podman can run containers here, or why not. Set -- @HASKOIN_STORE_NO_PODMAN@ to a value other than @0@ to skip the tests that
+ integration/Haskoin/Store/Integration/Temp.hs view
@@ -0,0 +1,45 @@+-- |+-- Module      : Haskoin.Store.Integration.Temp+-- Copyright   : (c) Haskoin Developers+-- License     : MIT+-- Maintainer  : jprupp@protonmail.ch+-- Stability   : experimental+-- Portability : POSIX+--+-- Temporary directories for the tests, the benchmark and the stress test:+-- made for an action, and removed with all that is in them when it ends.+--+-- These are the functions of the same name of "System.IO.Temp", with one+-- difference: the directory is removed by the 'bracket' of+-- "Haskoin.Store.Exception", whose cleanup cannot be cut short. A test that+-- is stopped from outside while it removes its database, by a timeout or by+-- a second Ctrl-C, still removes all of it, where the others could leave+-- half of it behind. A database of a test is tens of megabytes, often in+-- memory ('Haskoin.Store.Integration.Podman.withRamDir'), so what is left+-- behind matters.+module Haskoin.Store.Integration.Temp+  ( withSystemTempDirectory,+    withTempDirectory,+  )+where++import Control.Exception (IOException, handle)+import Haskoin.Store.Exception (bracket)+import System.Directory (removeDirectoryRecursive)+import System.IO.Temp (createTempDirectory, getCanonicalTemporaryDirectory)++-- | Run an action with a new directory under the temporary directory of+-- the system, and remove it afterwards, however the action ends. The name+-- of the directory starts with the text given.+withSystemTempDirectory :: String -> (FilePath -> IO a) -> IO a+withSystemTempDirectory template action = do+  tmp <- getCanonicalTemporaryDirectory+  withTempDirectory tmp template action++-- | 'withSystemTempDirectory' for a directory under the one given. A+-- failure to remove the directory is ignored: the result or the error of+-- the action is what matters to the caller.+withTempDirectory :: FilePath -> String -> (FilePath -> IO a) -> IO a+withTempDirectory parent template = bracket (createTempDirectory parent template) remove+  where+    remove = handle (\(_ :: IOException) -> return ()) . removeDirectoryRecursive
integration/Haskoin/Store/Integration/Valkey.hs view
@@ -14,11 +14,11 @@   ) where +import Control.Concurrent (threadDelay) import Database.Redis qualified as Redis+import Haskoin.Store.Exception (bracket, tryAny) import Haskoin.Store.Integration.Image (valkeyBin) import Haskoin.Store.Integration.Podman (ContainerSpec (..))-import UnliftIO (bracket, tryAny)-import UnliftIO.Concurrent (threadDelay)  -- | Valkey, the Redis-compatible server of Ubuntu, from the test image, listening on every interface of the container -- (only its published port is reachable, on 127.0.0.1), with nothing saved
src/Haskoin/Store.hs view
@@ -37,22 +37,20 @@ --   addresses, and 'StoreRead', the queries of a store as a record of --   functions in 'IO'. -- * "Haskoin.Store.Database.Reader": opening the RocksDB database.--- * "Haskoin.Store.Database.Query": queries against RocksDB, as plain---   functions.--- * "Haskoin.Store.Database.Mempool", "Haskoin.Store.Database.Migrate",---   "Haskoin.Store.Logic.Layered" and---   "Haskoin.Store.Database.Query.Layered": a layout of the database in---   which the mempool is kept apart from the chain, the migration to it, the---   rules that import blocks and transactions in it, and its queries. The---   store does not use them yet; see @docs\/layered-database.md@.+-- * "Haskoin.Store.Database.Query": queries of the chain in RocksDB, as+--   plain functions, and "Haskoin.Store.Database.Query.Pooled", which lays+--   the fleet's pool over them: what the store answers with.+-- * "Haskoin.Store.Logic": the rules that import and revert blocks, and+--   "Haskoin.Store.Database.Migrate", which brings an older database to the+--   layout of the chain alone. The database has no mempool; see+--   @docs\/one-fleet.md@. -- * "Haskoin.Store.Fleet": how stores that share a Redis agree on the block --   that they all answer from, as pure decisions, and --   "Haskoin.Store.Fleet.Redis", which carries them out.---   "Haskoin.Store.Fleet.Pool" has the rules of the mempool that they share.---   Nothing uses them yet.+--   "Haskoin.Store.Fleet.Pool" has the rules of the mempool that they share,+--   and "Haskoin.Store.Fleet.Member" is a store as a member of the fleet. -- * "Haskoin.Store.Database.Views": the database as it was at each recent --   block, for a store that answers from the block its fleet agrees on.---   Nothing uses it yet. -- * "Haskoin.Store.BlockStore": the process that imports blocks and --   transactions. -- * "Haskoin.Store.Web": the REST API server.@@ -98,17 +96,16 @@     module Haskoin.Store.Database.Reader,     module Haskoin.Store.Database.Types,     module Haskoin.Store.Data,+    cachePrefix, -    -- * Cache-    Cache,-    CacheConfig (..),-    cacheIsCached,-    cacheDelete,-    cacheEvict,+    -- * Caches of extended public keys+    XPubConfig (..),+    defaultXPubConfig,+    XPubs,      -- * Store Reader     StoreRead (..),-    databaseRead,+    chainRead,     Limits (..),     Start (..), @@ -122,12 +119,11 @@ where  import Haskoin.Store.BlockStore-import Haskoin.Store.Cache import Haskoin.Store.Common import Haskoin.Store.Data--- the instances of the store classes for the database reader-import Haskoin.Store.Database.Query (databaseRead)+import Haskoin.Store.Database.Query.Pooled (chainRead) import Haskoin.Store.Database.Reader import Haskoin.Store.Database.Types import Haskoin.Store.Manager import Haskoin.Store.Web+import Haskoin.Store.XPub.Service (XPubConfig (..), XPubs, defaultXPubConfig)
src/Haskoin/Store/BlockStore.hs view
@@ -17,23 +17,31 @@ -- -- When the best header of the node is ahead of the best block in the database, -- the block store picks one online peer as its syncing peer and asks it for--- blocks in batches of 'blockWindow'. Each block is imported as soon as it--- arrives. If the best block of the database is no longer in the main chain of--- headers, blocks are reverted until it is. A syncing peer that sends a block--- that was not requested, cannot find a block, or stays silent for longer than--- 'peerTimeout' is disconnected. The state of the download, and every rule+-- blocks in batches of 'blockWindow'. Only a peer that has announced the next+-- block is picked ('Known'): the peers that have are pinged, and the first to+-- answer is the one, or a late one if none answers within 'blockPing'. Each+-- block is imported as soon as it arrives. If the best block of the database is no longer in the main chain of+-- headers, blocks are reverted until it is. A syncing peer that delivers+-- nothing for 'blockStall' is passed over: another is found the same way and+-- asked for the same blocks, and whichever delivers first is taken. A peer that sends a+-- block that it was not asked for, cannot find a block, or still owes blocks+-- after 'peerTimeout' is disconnected. The state of the download, and every rule -- about it, is the pure state machine of "Haskoin.Store.BlockStore.Sync"; -- this module carries out what its transitions decide. ----- = Mempool+-- = Transactions ----- Once the database is in sync, the block store asks each peer for its mempool--- (if 'syncMempool' is set) and requests every transaction that peers--- announce. Received transactions are queued and imported in batches on every--- clock tick, in dependency order. Transactions that spend unknown outputs are--- kept for up to an hour, waiting for their parents. Mempool transactions older--- than 'mempoolTimeout' days are deleted. The transactions on their way in,--- requested or received, are the pure value of+-- Once the database is in sync, the block store requests every transaction+-- that peers announce and that no store of the fleet has seen ('toFetch'):+-- what the fleet saw is a set in Redis, the same for every store and kept+-- across restarts, so a transaction is fetched about once for the fleet.+-- It asks peers for their whole mempools only to fill+-- the pool again after Redis lost it ('blockStoreRefill'): the pool outlives+-- the store, so a store that starts has nothing to ask for. A transaction+-- that arrives goes to 'onTx': the member of the fleet offers it to the+-- pool that the fleet shares in Redis, which is the mempool+-- ("Haskoin.Store.Fleet.Member"). The database holds the chain alone. The+-- transactions on their way in, requested, are the pure value of -- "Haskoin.Store.BlockStore.Pending". -- -- = Feeding the block store@@ -55,14 +63,6 @@ -- * __Mempool sync waits for the tip.__ Unconfirmed transactions only make sense --   against the current chain, so they are requested once the database is in --   sync.------ * __Orphans get a grace period.__ A transaction can arrive before its parent.---   It is held for up to an hour, waiting for the parent, instead of being---   dropped and requested again.------ * __Imports are batched.__ Mempool transactions are imported up to 2000 at a---   time on each clock tick, which amortises the cost of a database write over---   many transactions during bursts. module Haskoin.Store.BlockStore   ( -- * Block Store     BlockStore,@@ -78,6 +78,10 @@     blockStoreBlockSTM,     blockStoreNotFound,     blockStoreNotFoundSTM,+    blockStoreAnnounced,+    blockStoreAnnouncedSTM,+    blockStorePong,+    blockStorePongSTM,     blockStoreTx,     blockStoreTxSTM,     blockStoreTxHash,@@ -85,19 +89,37 @@     blockStorePendingTxs,     blockStorePendingTxsSTM,     needsMoreBlocks,+    blockStoreRefill,+    toFetch,   ) where +import Control.Concurrent (threadDelay)+import Control.Concurrent.Async (link, withAsync)+import Control.Concurrent.STM+  ( STM,+    TVar,+    atomically,+    modifyTVar,+    modifyTVar',+    newTVarIO,+    readTVar,+    readTVarIO,+    stateTVar,+    writeTVar,+  )+import Control.Exception (Exception, throwIO, try) import Control.Monad-  ( forM,+  ( filterM,+    forM,     forM_,     unless,     void,     when,     (>=>),   )-import Data.Bool (bool) import Data.ByteString qualified as B+import Data.HashMap.Strict (HashMap) import Data.HashMap.Strict qualified as HashMap import Data.HashSet (HashSet) import Data.HashSet qualified as HashSet@@ -124,22 +146,24 @@   ( defaultTimeLocale,     formatTime,   )+import Data.Word (Word64) import Haskoin   ( Block (..),     BlockHash (..),     BlockHeader (..),     BlockHeight,     BlockNode (..),-    Ctx,     GetData (..),     InvType (..),     InvVector (..),     Message (..),     Network (..),     OutPoint (..),+    Ping (..),     Tx (..),     TxHash (..),     TxIn (..),+    Version (..),     blockHashToHex,     headerHash,     txHash,@@ -158,6 +182,7 @@     chainGetBest,     chainGetBlock,     chainGetParents,+    getOnlinePeer,     getPeers,     killPeer,     logAt,@@ -170,25 +195,25 @@ import Haskoin.Store.BlockStore.Pending qualified as Pending import Haskoin.Store.BlockStore.Sync   ( Accept (..),-    Sync (..),+    Sync,     needsMoreBlocks,   ) import Haskoin.Store.BlockStore.Sync qualified as Sync import Haskoin.Store.Common (StoreEvent (..), loopForever) import Haskoin.Store.Data-import Haskoin.Store.Database.Mempool (wipeMempoolArea)-import Haskoin.Store.Database.Query (getMempoolBefore) import Haskoin.Store.Database.Query qualified as Q-import Haskoin.Store.Database.Query.Layered qualified as L import Haskoin.Store.Database.Reader-import Haskoin.Store.Database.Writer (MempoolChanges (..), Writer, runWriterMempool)+import Haskoin.Store.Database.Writer+  ( BatchReport (..),+    BlockTimes (..),+    Writer,+    runWriterReport,+  ) import Haskoin.Store.Database.Writer qualified as W-import Haskoin.Store.Logic (ImportException (..))-import Haskoin.Store.Logic.Layered-  ( deleteUnconfirmedTx,+import Haskoin.Store.Logic+  ( ImportException (..),     importBlock,     initBest,-    newMempoolTx,     revertBlock,   ) import NQE@@ -205,25 +230,7 @@     sendSTM,   ) import System.Metrics.StatsD-import System.Random (randomRIO)-import UnliftIO-  ( Exception,-    STM,-    TVar,-    atomically,-    catch,-    link,-    modifyTVar,-    modifyTVar',-    newTVarIO,-    readTVar,-    readTVarIO,-    stateTVar,-    throwIO,-    try,-    withAsync,-  )-import UnliftIO.Concurrent (threadDelay)+import System.Random (randomIO, randomRIO)  -- | Messages handled by the block store process. data BlockStoreMessage@@ -237,6 +244,11 @@     BlockReceived !Peer !Block   | -- | a peer could not find blocks     BlockNotFound !Peer ![BlockHash]+  | -- | a peer announced blocks: the last header of those it sent, or the+    -- blocks of an inventory+    BlockAnnounced !Peer ![BlockHash]+  | -- | a peer answered a ping with a nonce+    BlockPong !Peer !Word64   | -- | a peer sent a transaction     TxRefReceived !Peer !Tx   | -- | a peer announced transactions@@ -254,8 +266,6 @@     CorruptDatabase   | -- | the node has no header at this height below this block     AncestorNotInChain !BlockHeight !BlockHash-  | -- | importing mempool transactions failed-    MempoolImportFailed   deriving (Show, Eq, Ord, Exception)  -- | Block store process state.@@ -273,20 +283,29 @@     metrics :: !(Maybe StoreMetrics),     -- | labels of the peers whose mempool was already requested     syncedMempool :: !(TVar (HashSet Text)),+    -- | whether to ask peers for their mempools: from the first refill of+    -- the pool on ('blockStoreRefill')+    refilling :: !(TVar Bool),+    -- | what each peer said that it has, by its label: a peer is asked+    -- only for blocks that it announced+    knows :: !(TVar (HashMap Text Known)),     -- | who reserves the syncing peer, so that no one else releases it-    owner :: !Owner,-    -- | transactions that left the mempool unconfirmed since the store-    -- started, as many as 'mostForgotten'. The database keeps nothing of-    -- them, and peers go on announcing them: without this the store would-    -- fetch each one again, only to refuse it again.-    forgotten :: !(TVar (HashSet TxHash))+    owner :: !Owner   } --- | How many deleted transactions the store remembers not to fetch again.--- Past that many it forgets them all and starts over, which costs a few--- fetches that come to nothing.-mostForgotten :: Int-mostForgotten = 100000+-- | The blocks that a peer said it has. A peer is asked for a block only+-- if it has announced it, or a block that comes after it: one that is+-- asked for a block that it does not have yet answers nothing, or that it+-- cannot find it, and the block arrives a stall later from another.+data Known = Known+  { -- | the height that it gave when it connected: it is taken to have+    -- the blocks of the main chain up to there+    height :: !BlockHeight,+    -- | the block with the most work of those it announced since, with+    -- the headers that it sent or in an inventory: it has that block and+    -- those before it+    tip :: !(Maybe BlockNode)+  }  -- | StatsD metrics of the block store. data StoreMetrics = StoreMetrics@@ -298,17 +317,45 @@     queuedTxs :: !StatGauge,     -- | online peers (@store.peers@)     peers :: !StatGauge,-    -- | transactions in the mempool (@store.mempool@)-    mempool :: !StatGauge,     -- | milliseconds to import a block (@store.block@)-    block :: !StatTiming+    block :: !StatTiming,+    -- | milliseconds of each stage of the import of a block+    -- ('Haskoin.Store.Logic.importBlock'): working out what it+    -- needs of the block (@store.block_prepare@)+    blockPrepare :: !StatTiming,+    -- | reading ahead what the rules will read (@store.block_prefetch@)+    blockPrefetch :: !StatTiming,+    -- | of the read ahead, each kind of key+    -- ('Haskoin.Store.Database.Writer.ReadTimes'): the unspent outputs+    -- that the block spends (@store.block_read_unspents@)+    blockReadUnspents :: !StatTiming,+    -- | the groups of spenders of those outputs+    -- (@store.block_read_spenders@): zero, as a block reads none+    blockReadSpenders :: !StatTiming,+    -- | the balances of the addresses of the block+    -- (@store.block_read_balances@): zero, as a block reads none+    blockReadBalances :: !StatTiming,+    -- | the rules, in memory (@store.block_apply@)+    blockApply :: !StatTiming,+    -- | writing the batch to RocksDB (@store.block_write@)+    blockWrite :: !StatTiming,+    -- | the same times of the latest block, as gauges that keep their value+    -- until the next block (@store.last_block_ms@, @store.last_block_prepare_ms@,+    -- @store.last_block_prefetch_ms@, @store.last_block_apply_ms@ and+    -- @store.last_block_write_ms@): a timing has a value only in the second of+    -- its block, so a chart that averages seconds shows it smaller the more it+    -- is zoomed out; a gauge shows the time of the latest block at any zoom+    lastBlock :: !StatGauge,+    lastPrepare :: !StatGauge,+    lastPrefetch :: !StatGauge,+    lastApply :: !StatGauge,+    lastWrite :: !StatGauge   }  -- | Register the block store metrics, if StatsD is configured, with their current values. newStoreMetrics :: BlockStoreConfig -> IO (Maybe StoreMetrics) newStoreMetrics cfg =   forM cfg.stats $ \s -> do-    m <- Q.getMempool cfg.db     b <-       Q.getBestBlock cfg.db >>= \case         Nothing -> return 0@@ -319,12 +366,26 @@     headers <- g s "headers" (fromIntegral h.height)     queuedTxs <- g s "queued_txs" 0     peers <- g s "peers" (length p)-    mempool <- g s "mempool" (length m)-    block <- t s "block" 0+    block <- t s "block"+    blockPrepare <- t s "block_prepare"+    blockPrefetch <- t s "block_prefetch"+    blockReadUnspents <- t s "block_read_unspents"+    blockReadSpenders <- t s "block_read_spenders"+    blockReadBalances <- t s "block_read_balances"+    blockApply <- t s "block_apply"+    blockWrite <- t s "block_write"+    lastBlock <- g s "last_block_ms" 0+    lastPrepare <- g s "last_block_prepare_ms" 0+    lastPrefetch <- g s "last_block_prefetch_ms" 0+    lastApply <- g s "last_block_apply_ms" 0+    lastWrite <- g s "last_block_write_ms" 0     return StoreMetrics {..}   where     g s x = newStatGauge s ("store." <> x)-    t s x = newStatTiming s ("store." <> x)+    -- every sample is sent: there is one a block. The timing of a block+    -- was made with a rate of 0 until version 1.15.1, which sends none,+    -- so it never reached the StatsD server+    t s x = newStatTiming s ("store." <> x) 1  -- | Update the gauge of the best block height. setStoreHeight :: BlockStore -> IO ()@@ -363,15 +424,14 @@         Right x -> do           end <- round . (* 1000) <$> getPOSIXTime           addTiming m.block (end - start)+          setGauge m.lastBlock (end - start)           return (Right x)         Left y -> return (Left y)     Nothing -> go  -- | Configuration for a block store. data BlockStoreConfig = BlockStoreConfig-  { -- | secp256k1 context-    ctx :: !Ctx,-    -- | peer manager from running node+  { -- | peer manager from running node     peerMgr :: !PeerMgr,     -- | chain from a running node     chain :: !Chain,@@ -381,23 +441,35 @@     db :: !DatabaseReader,     -- | network constants     net :: !Network,-    -- | do not index new mempool transactions-    noMempool :: !Bool,-    -- | wipe mempool at start-    wipeMempool :: !Bool,     -- | what to do with the best block right after it changes, and when the     -- store starts, before anything else is written to the database: a     -- member of a fleet takes a view of the database for the block here     -- ('Haskoin.Store.Fleet.Member.noteBest')     onBest :: !(BlockHash -> IO ()),-    -- | sync mempool from peers-    syncMempool :: !Bool,+    -- | what to do with a transaction that a peer sent: the member of the+    -- fleet offers it to the pool, the mempool, which is in Redis alone+    -- ('Haskoin.Store.Fleet.Member.memberReceived')+    onTx :: !(UnixTime -> Tx -> IO ()),+    -- | of these transactions, those that the store does not fetch when a+    -- peer announces them: those that it holds to offer to the pool, and+    -- those that the pool has decided on, taken, refused as a conflict or+    -- let go unconfirmed. The database keeps+    -- nothing of the mempool, so without this every store would fetch each+    -- transaction from every peer that announces it+    -- ('Haskoin.Store.Fleet.Member.memberSeen')+    seen :: !([TxHash] -> IO (HashSet TxHash)),     -- | blocks to request from the syncing peer in one batch     blockWindow :: !Int,     -- | delete transactions older than this number of days from mempool     mempoolTimeout :: !Int,     -- | disconnect syncing peer if inactive for this long     peerTimeout :: !NominalDiffTime,+    -- | ask another peer for the blocks that the syncing peer has not+    -- delivered for this long ("Haskoin.Store.BlockStore.Sync")+    blockStall :: !NominalDiffTime,+    -- | a peer is asked for blocks if it answers a ping within this long,+    -- or if none does ("Haskoin.Store.BlockStore.Sync")+    blockPing :: !NominalDiffTime,     -- | StatsD client, if metrics are enabled     stats :: !(Maybe Stats),     -- | log messages here@@ -412,54 +484,47 @@ logError self = logAt LevelError self.config.logger  -- | Run an import against the database, returning the import error if it fails.-runImport :: BlockStore -> Network -> Ctx -> (Writer -> IO a) -> IO (Either ImportException a)-runImport self net ctx f = fmap fst <$> runImportMempool self net ctx f---- | Like 'runImport', but also return how the import changed the mempool.--- The mempool gauge is kept up to date from those changes instead of--- counting the whole mempool on every clock tick.-runImportMempool :: BlockStore -> Network -> Ctx -> (Writer -> IO a) -> IO (Either ImportException (a, MempoolChanges))-runImportMempool self net ctx f = do-  r <- try $ runWriterMempool net ctx self.config.db self.config.logger f-  forM_ r $ \(_, c) ->+runImport :: BlockStore -> Network -> (Writer -> IO a) -> IO (Either ImportException a)+runImport self net f = do+  r <- try $ runWriterReport net self.config.db self.config.logger f+  forM_ r $ \(_, report) ->     forM_       self.metrics-      ( \m -> do-          incrementGauge m.mempool (length c.added)-          decrementGauge m.mempool (length c.removed)+      ( \m ->+          -- only a batch that imported a block has these+          forM_ report.block $ \t -> do+            addTiming m.blockPrepare (milliseconds t.prepare)+            addTiming m.blockPrefetch (milliseconds t.prefetch)+            addTiming m.blockReadUnspents (milliseconds t.reads.unspentReads)+            addTiming m.blockReadSpenders (milliseconds t.reads.spenderReads)+            addTiming m.blockReadBalances (milliseconds t.reads.balanceReads)+            addTiming m.blockApply (milliseconds t.apply)+            addTiming m.blockWrite (milliseconds t.write)+            setGauge m.lastPrepare (milliseconds t.prepare)+            setGauge m.lastPrefetch (milliseconds t.prefetch)+            setGauge m.lastApply (milliseconds t.apply)+            setGauge m.lastWrite (milliseconds t.write)       )-  return r+  return (fst <$> r)+  where+    milliseconds :: Double -> Int+    milliseconds = round . (* 1000) --- | Run an import and publish the transactions that it added to the--- mempool or removed from it, then the new best block if there is one.--- Transactions of the block are not published.-runImportNotify :: BlockStore -> Maybe Block -> (Writer -> IO a) -> IO (Either ImportException a)+-- | Run an import, then publish the new best block if there is one.+runImportNotify ::+  BlockStore -> Maybe BlockHash -> (Writer -> IO a) -> IO (Either ImportException a) runImportNotify self block f = do   net <- getNetwork self-  ctx <- getCtx self-  runImportMempool self net ctx f >>= \case+  runImport self net f >>= \case     Left e -> return (Left e)-    Right (x, c) -> do+    Right x -> do       let l = self.config.pub-      let deleted = filter notInBlock c.removed-      atomically . modifyTVar' self.forgotten $ \seen ->-        let more = foldr HashSet.insert seen deleted-         in if HashSet.size more > mostForgotten then HashSet.fromList deleted else more-      -- one that came back is not forgotten any more-      atomically (modifyTVar' self.forgotten (\seen -> foldr HashSet.delete seen c.added))-      forM_ deleted $ \h ->-        publish (StoreMempoolDelete h) l-      forM_ (filter notInBlock c.added) $ \h ->-        publish (StoreMempoolNew h) l-      forM_ block $ \b -> do+      forM_ block $ \h -> do         -- before the event, so that whoever hears of the block finds what         -- was done for it-        self.config.onBest (headerHash b.header)-        publish (StoreBestBlock (headerHash b.header)) l+        self.config.onBest h+        publish (StoreBestBlock h) l       return (Right x)-  where-    inBlock = maybe HashSet.empty (HashSet.fromList . map txHash . (.txs)) block-    notInBlock = not . (`HashSet.member` inBlock)  -- | Run a database query from the block store. -- | Best block in the database.@@ -470,36 +535,32 @@ getBlock :: BlockStore -> BlockHash -> IO (Maybe BlockData) getBlock self = Q.getBlock self.config.db --- | Data of a transaction in the database: confirmed, in the mempool, or--- deleted from it and marked so.+-- | Data of a confirmed transaction in the database. getTxData :: BlockStore -> TxHash -> IO (Maybe TxData)-getTxData self h = L.withView self.config.db (`L.getTxData` h)+getTxData self = Q.getTxData self.config.db --- | Unspent output in the database, of the chain or of the mempool.+-- | Unspent output of the chain in the database. getUnspent :: BlockStore -> OutPoint -> IO (Maybe Unspent)-getUnspent self p = L.withView self.config.db (`L.getUnspent` p)+getUnspent self = Q.getUnspent self.config.db  -- | Network of the block store. getNetwork :: BlockStore -> IO Network getNetwork self = return self.config.net --- | Secp256k1 context of the block store.-getCtx :: BlockStore -> IO Ctx-getCtx self = return self.config.ctx- -- | Run block store process. withBlockStore ::   BlockStoreConfig ->   (BlockStore -> IO a) ->   IO a withBlockStore cfg action = do-  pb <- newTVarIO Idle+  pb <- newTVarIO . Sync.idle =<< getCurrentTime   ts <- newTVarIO Pending.empty   sy <- newTVarIO HashSet.empty+  refilling <- newTVarIO False   inbox <- newInbox   metrics <- newStoreMetrics cfg   owner <- (newOwner "block store")-  forgotten <- newTVarIO HashSet.empty+  knows <- newTVarIO HashMap.empty   let r =         BlockStore           { mailbox = inboxToMailbox inbox,@@ -508,8 +569,9 @@             pending = ts,             metrics = metrics,             syncedMempool = sy,-            owner = owner,-            forgotten = forgotten+            refilling = refilling,+            knows = knows,+            owner = owner           }   withAsync ((go r inbox)) $ \a -> do     link a@@ -517,20 +579,11 @@   where     go self inbox = do       ini-      wipe       run       where-        -- the mempool is an area of its own, so wiping it is deleting the-        -- keys of that area: nothing of the chain has to be put right-        wipe = when cfg.wipeMempool $ do-          keys <- wipeMempoolArea cfg.db-          forM_ self.metrics $ \m -> setGauge m.mempool 0-          logAt LevelInfo cfg.logger $-            "Wiped the mempool: " <> cs (show keys) <> " keys deleted"         ini = do           net <- getNetwork self-          ctx <- getCtx self-          runImport self net ctx initBest >>= \case+          runImport self net initBest >>= \case             Left e -> do               logError self $                 "Could not initialize: " <> cs (show e)@@ -566,15 +619,12 @@ guardSync :: BlockStore -> IO () -> IO () guardSync self f = isInSync self >>= \s -> when s f --- | Run an action only if the mempool is indexed.-guardMempool :: BlockStore -> IO () -> IO ()-guardMempool self f = unless self.config.noMempool f---- | Ask a peer for its mempool, once per peer, if in sync and mempool sync is enabled.+-- | Ask a peer for its mempool, once per peer, if in sync and the pool is+-- being filled again ('blockStoreRefill'). syncMempool :: BlockStore -> Peer -> IO () syncMempool self p =-  guardMempool self . guardSync self $ do-    let s = self.config.syncMempool+  guardSync self $ do+    s <- readTVarIO self.refilling     let m = self.syncedMempool     y <- not . HashSet.member p.label <$> readTVarIO m     when (s && y) $ do@@ -583,14 +633,28 @@       atomically . modifyTVar m $ HashSet.insert p.label       MMempool `sendMessage` p +-- | Ask every peer for its mempool again, and fetch every transaction that+-- it announces and the fleet has not seen: for when Redis lost its data,+-- the pool with it, and what the fleet had seen+-- ('Haskoin.Store.Fleet.Member.noteEpoch').+-- The transactions go to 'onTx', which offers them to the pool again.+blockStoreRefill :: BlockStore -> IO ()+blockStoreRefill self = do+  atomically $ do+    writeTVar self.syncedMempool HashSet.empty+    writeTVar self.refilling True+  getPeers self.config.peerMgr >>= mapM_ (syncMempool self . (.mailbox))+ -- | Import a block from the syncing peer, then ask for more blocks or, if now in -- sync, sync the mempool. Disconnects the peer if it is not the syncing peer, if -- the block is unknown to the node, or if the block cannot be imported. processBlock :: BlockStore -> Peer -> Block -> IO () processBlock self peer block =-  Sync.accept peer blockhash <$> readTVarIO self.sync >>= \case+  transition self (Sync.accept peer blockhash) >>= \case     NotSyncingPeer -> refuse "Non-syncing peer"     NotRequested -> refuse "Syncing peer"+    -- a peer that was passed over for being slow, and another was faster+    Late -> logDebug self $ "Block " <> hexhash <> " from peer " <> peer.label <> " came after another's"     Requested ->       getBlockNode self blockhash >>= \case         Nothing -> do@@ -606,7 +670,9 @@               <> blockText node Nothing               <> " from peer "               <> peer.label-          ret <- withBlockTiming self $ runImportNotify self (Just block) (\wr -> importBlock wr block node)+          ret <-+            withBlockTiming self $+              runImportNotify self (Just blockhash) (\wr -> importBlock wr block node)           case ret of             Left e -> failure e             Right _ -> success node@@ -670,21 +736,15 @@  -- | Queue a transaction received from a peer for import. processTx :: BlockStore -> Peer -> Tx -> IO ()-processTx self p tx = guardMempool self $ do+processTx self p tx = do   t <- getCurrentTime   logDebug self $     "Received tx "       <> txHashToHex (txHash tx)       <> " by peer: "       <> p.label-  changePending self (Pending.received t tx HashSet.empty)---- | Forget old requests, and drop transactions that waited for their parents--- for too long.-pruneOrphans :: BlockStore -> IO ()-pruneOrphans self = guardMempool self $ do-  now <- getCurrentTime-  changePending self (Pending.prune now)+  changePending self (Pending.answered (txHash tx))+  self.config.onTx (fromIntegral (floor (utcTimeToPOSIXSeconds t) :: Integer)) tx  -- | Change the mempool transactions on their way into the database. changePending :: BlockStore -> (Pending -> Pending) -> IO ()@@ -696,14 +756,6 @@ isPending :: BlockStore -> TxHash -> IO Bool isPending self th = Pending.isPending th <$> readTVarIO self.pending --- | Take up to this many queued transactions that have no missing parents, in--- dependency order, out of the queue.-pendingTxs :: BlockStore -> Int -> IO [PendingTx]-pendingTxs self i = do-  selected <- atomically $ stateTVar self.pending (Pending.takeReady i)-  setPendingTxs self-  return selected- -- | Drop queued transactions that are already in the database, and recompute the -- missing parents of the others. updateOrphans :: BlockStore -> IO ()@@ -725,146 +777,33 @@         Just TxData {deleted = True} -> return False         Just TxData {deleted = False} -> return True --- | Whether the store has a transaction, or deleted it not long ago: in--- both cases there is nothing to fetch.-known :: BlockStore -> TxHash -> IO (Maybe ())-known self h =-  HashSet.member h <$> readTVarIO self.forgotten >>= \case-    True -> return (Just ())-    False -> (() <$) <$> getTxData self h---- | Queue a transaction that spends unknown outputs, recording which parents are missing.-newOrphanTx ::-  BlockStore ->-  UTCTime ->-  Tx ->-  Writer ->-  IO ()-newOrphanTx self time tx wr = do-  logAt LevelDebug self.config.logger $-    "Import tx "-      <> txHashToHex (txHash tx)-      <> ": Orphan"-  unspents <- catMaybes <$> mapM (W.getUnspent wr) prevs-  let have = HashSet.fromList (map (.outpoint) unspents)-      missing = HashSet.map (.hash) (HashSet.difference (HashSet.fromList prevs) have)-  atomically . modifyTVar' self.pending $ Pending.received time tx missing-  where-    prevs = map (.outpoint) tx.inputs---- | Import a transaction into the mempool. Returns 'True' if imported. A--- transaction that spends unknown outputs is queued as an orphan; other errors--- are ignored.-importMempoolTx :: BlockStore -> UTCTime -> Tx -> Writer -> IO Bool-importMempoolTx block_read time tx wr =-  catch new_mempool_tx handle_error-  where-    tx_hash = txHash tx-    handle_error :: ImportException -> IO Bool-    handle_error Orphan = do-      newOrphanTx block_read time tx wr-      return False-    handle_error _ = return False-    seconds = floor (utcTimeToPOSIXSeconds time)-    new_mempool_tx = do-      t <- newMempoolTx wr tx seconds-      logAt LevelDebug block_read.config.logger $-        bool "Already have" "Imported" t-          <> " tx "-          <> txHashToHex (txHash tx)-      when t . atomically . modifyTVar' block_read.pending $ Pending.imported tx_hash-      return t---- | Import up to 2000 queued transactions.-processMempool :: BlockStore -> IO ()-processMempool self = guardMempool self $ do-  txs <- pendingTxs self 2000-  let block_read = self-  unless (null txs) $ import_txs block_read txs-  where-    run_import block_read wr p =-      importMempoolTx block_read p.time p.tx wr-    import_txs block_read txs =-      runImportNotify self Nothing (\wr -> mapM (run_import block_read wr) txs) >>= \case-        Left e -> report_error e-        Right _ -> return ()-    report_error e = do-      logError self $-        "Error processing mempool: " <> cs (show e)-      throwIO e---- | Delete up to 1000 mempool transactions older than the configured number of days.-pruneMempool :: BlockStore -> IO ()-pruneMempool self =-  guardMempool self $ do-    let days = self.config.mempoolTimeout-    when (days > 0) $ do-      time <- (floor . utcTimeToPOSIXSeconds) <$> getCurrentTime-      let thresh = time - (fromIntegral days * 24 * 60 * 60)-      -- runs on every clock tick: read only the expired entries, oldest-      -- first, from the index by time-      let db = self.config.db-      txs <- map snd <$> (getMempoolBefore db thresh 1000)-      unless (null txs) $ do-        logInfo self $-          "Deleting " <> cs (show (length txs)) <> " old transactions from mempool"-        runImportNotify self Nothing (\wr -> mapM_ (deleteUnconfirmedTx wr False) txs) >>= \case-          Left e ->-            logError self $-              "Error pruning mempool: " <> cs (show e)-          Right () -> return ()+-- | Of the transactions that a peer announced, those to fetch: given+-- whether one is queued or was requested lately, which of them the fleet+-- has seen ('seen' of t'BlockStoreConfig'), and whether one is in the+-- database. The fleet is asked once for all of them, since an inventory can+-- name thousands, and only for those that are not queued.+toFetch :: (TxHash -> IO Bool) -> ([TxHash] -> IO (HashSet TxHash)) -> (TxHash -> IO Bool) -> [TxHash] -> IO [TxHash]+toFetch pending seen stored hs = do+  fresh <- filterM (fmap not . pending) hs+  have <- if null fresh then return HashSet.empty else seen fresh+  filterM (fmap not . stored) (filter (not . (`HashSet.member` have)) fresh)  -- | Request from a peer the announced transactions that are neither queued,--- recently requested nor in the database. Only while in sync.+-- recently requested, seen by the fleet nor in the database ('toFetch').+-- Only while in sync. processTxs :: BlockStore -> Peer -> [TxHash] -> IO ()-processTxs self p hs = guardMempool self . guardSync self $ do+processTxs self p hs = guardSync self $ do+  xs <- toFetch (isPending self) self.config.seen stored hs   logDebug self $-    "Received inventory with "+    "Requesting "+      <> cs (show (length xs))+      <> " of "       <> cs (show (length hs))-      <> " transactions from peer: "+      <> " transactions announced by peer: "       <> p.label-  xs <- catMaybes <$> zip_counter process_tx   unless (null xs) $ go xs   where-    len = length hs-    zip_counter = forM (zip [(1 :: Int) ..] hs) . uncurry-    process_tx i h =-      isPending self h >>= \case-        True -> do-          logDebug self $-            "Tx "-              <> cs (show i)-              <> "/"-              <> cs (show len)-              <> " "-              <> txHashToHex h-              <> ": "-              <> "Already Pending"-          return Nothing-        False ->-          known self h >>= \case-            Just _ -> do-              logDebug self $-                "Tx "-                  <> cs (show i)-                  <> "/"-                  <> cs (show len)-                  <> " "-                  <> txHashToHex h-                  <> ": "-                  <> "Already Imported"-              return Nothing-            Nothing -> do-              logDebug self $-                "Tx "-                  <> cs (show i)-                  <> "/"-                  <> cs (show len)-                  <> " "-                  <> txHashToHex h-                  <> ": "-                  <> "Requesting"-              return (Just h)+    stored h = isJust <$> getTxData self h     go xs = do       now <- getCurrentTime       changePending self (Pending.requested now xs)@@ -874,29 +813,168 @@           msg = MGetData (GetData vec)       msg `sendMessage` p --- | Disconnect the syncing peer if it has been silent for too long.+-- | Deal with peers that do not deliver: disconnect the syncing peer if it+-- has been silent for the timeout, else look for another peer to ask for its+-- blocks if it has been silent for the stall; and disconnect the peers passed+-- over before that still owe their blocks after the timeout. If peers were+-- pinged and none answered in time, ask one of them all the same. checkTime :: BlockStore -> IO () checkTime self = do   now <- getCurrentTime-  Sync.timedOut self.config.peerTimeout now <$> readTVarIO self.sync >>= \case-    Nothing -> return ()+  st <- readTVarIO self.sync+  let timeout = self.config.peerTimeout+  case Sync.timedOut timeout now st of     Just p -> do       logWarn self $ "Syncing peer timeout: " <> p.label       killPeer p+    Nothing -> forM_ (Sync.stalled self.config.blockStall timeout now st) $ \slow ->+      search self (slow : Sync.passedPeers st)+  forM_ (Sync.expired timeout now st) $ \p -> do+    logWarn self $ "Peer did not deliver its blocks: " <> p.label+    killPeer p+  late <- Sync.overdue self.config.blockPing now <$> readTVarIO self.sync+  unless (null late) $ do+    logWarn self $+      "No peer answered a ping within "+        <> cs (show self.config.blockPing)+        <> ": asking one all the same"+    void (firstM (choose self now) late)+    transition self $ \s -> ((), Sync.settle s)+  where+    firstM _ [] = return Nothing+    firstM f (x : xs) = f x >>= \ok -> if ok then return (Just x) else firstM f xs +-- | Look for a peer to ask for blocks: ping those that have the block that+-- is needed, leaving out the ones given, fastest first. The first to+-- answer is asked ('processPong'). Peers that were pinged already in this+-- search are not pinged again, so this may be called at every tick, and+-- reaches the peers that announce the block while the search goes on.+search :: BlockStore -> [Peer] -> IO ()+search self asked =+  needed self >>= \case+    Nothing -> return ()+    Just node -> do+      -- the peers that still owe blocks from before come last+      slow <- Sync.passedPeers <$> readTVarIO self.sync+      peers <- filter (`notElem` asked) . map (.mailbox) <$> getPeers self.config.peerMgr+      able <- filterM (peerHas self node) (filter (`notElem` slow) peers <> filter (`elem` slow) peers)+      now <- getCurrentTime+      fresh <- randomIO+      transition self (Sync.ping now fresh able) >>= \case+        Nothing -> return ()+        Just (nonce, ps) -> forM_ ps $ \p -> do+          logDebug self $ "Pinging peer before asking it for blocks: " <> p.label+          MPing (Ping nonce) `sendMessage` p++-- | The block that a peer must have to be asked: the last one that the+-- syncing peer was asked for and has not delivered, since its blocks are+-- what the next peer is asked for; or, with nothing asked, the block after+-- the best one. A peer that has a block has those before it. Nothing if+-- there is no block to ask for.+needed :: BlockStore -> IO (Maybe BlockNode)+needed self =+  Sync.lastRequested <$> readTVarIO self.sync >>= \case+    Just h -> getBlockNode self h+    Nothing -> do+      bb <- getBest self+      bh <- chainGetBest self.config.chain+      if bb.height < bh.height+        then Just <$> findAncestor self (bb.height + 1) bh+        else return Nothing++-- | Whether a peer said that it has a block: it gave a height at least+-- that of the block when it connected, or it announced the block or one+-- that comes after it.+peerHas :: BlockStore -> BlockNode -> Peer -> IO Bool+peerHas self node p =+  HashMap.lookup p.label <$> readTVarIO self.knows >>= \case+    Nothing -> return False+    Just k+      | node.height <= k.height -> return True+      | Just t <- k.tip,+        node.height <= t.height ->+          (== Just node) <$> chainGetAncestor self.config.chain node.height t+      | otherwise -> return False++-- | Note the blocks that a peer announced, and look for a peer to ask if+-- blocks are needed: this one can now be asked for them. Blocks whose+-- headers the node does not have are ignored: a peer that announces one in+-- an inventory is asked for its header by the node, and announces it again+-- with that.+processAnnounced :: BlockStore -> Peer -> [BlockHash] -> IO ()+processAnnounced self p hs = do+  nodes <- catMaybes <$> mapM (getBlockNode self) hs+  forM_ nodes $ \n ->+    atomically . modifyTVar' self.knows $+      HashMap.insertWith (\_ k -> raise n k) p.label (Known 0 (Just n))+  unless (null nodes) (trySyncing self)+  where+    raise n k+      | maybe True ((< n.work) . (.work)) k.tip = Known k.height (Just n)+      | otherwise = k++-- | A peer answered a ping: if it was pinged to be asked for blocks and+-- answered in time, ask it.+processPong :: BlockStore -> Peer -> Word64 -> IO ()+processPong self p nonce = do+  now <- getCurrentTime+  chosen <- transition self (Sync.pong self.config.blockPing now p nonce)+  when chosen (void (choose self now p))++-- | Ask a peer that the search found for blocks: make it the syncing peer+-- if there is none, or pass the syncing peer over for it if that one is+-- still not delivering. Says whether the search is over; it is not if the+-- peer is busy with something else and cannot be locked.+choose :: BlockStore -> UTCTime -> Peer -> IO Bool+choose self now p = do+  st <- readTVarIO self.sync+  case Sync.syncingPeer st of+    Nothing ->+      trySetPeer self p >>= \case+        False -> return False+        True -> syncMe self >> return True+    Just _+      | isJust (Sync.stalled self.config.blockStall self.config.peerTimeout now st) -> passOver self now p+      -- the syncing peer delivered in the meantime+      | otherwise -> True <$ transition self (\s -> ((), Sync.settle s))++-- | Ask a peer for the blocks that the syncing peer is not delivering, and+-- make it the syncing peer. The one passed over stays connected, and its+-- blocks are still taken if they come first. Says whether it was done.+passOver :: BlockStore -> UTCTime -> Peer -> IO Bool+passOver self now p =+  reservePeer self.owner p >>= \case+    False -> return False+    True ->+      transition self (Sync.pass now p) >>= \case+        Nothing -> False <$ releasePeer self.owner p+        Just (old, blocks) -> do+          logWarn self $+            "Peer "+              <> old.label+              <> " is not delivering: asking "+              <> p.label+              <> " for "+              <> cs (show (length blocks))+              <> " blocks"+          void (releasePeer self.owner old)+          unless (null blocks) $ MGetData (GetData (map inv blocks)) `sendMessage` p+          return True+  where+    inv = InvVector kind . (.get)+    kind = if self.config.net.segWit then InvWitnessBlock else InvBlock+ -- | Revert blocks until the best block in the database is in the main chain of headers. revertToMainChain :: BlockStore -> IO () revertToMainChain self = do   h <- headerHash . (.header) <$> getBest self   let ch = self.config.chain   net <- getNetwork self-  ctx <- getCtx self   (chainBlockMain ch h) >>= \x -> unless x $ do     logWarn self $       "Reverting best block: "         <> blockHashToHex h-    now <- floor . utcTimeToPOSIXSeconds <$> getCurrentTime-    runImport self net ctx (\wr -> revertBlock wr now h) >>= \case+    runImport self net (\wr -> revertBlock wr h) >>= \case       Left e -> do         logError self $           "Could not revert block "@@ -969,15 +1047,38 @@       bb <- getSyncBest self       bh <- chainGetBest self.config.chain       when (bb /= bh) $ do-        bns <- sel bb bh-        logDebug self $-          "Requesting "-            <> fromString (show (length bns))-            <> " blocks from peer: "-            <> p.label-        transition self $ \st -> ((), Sync.request (map (headerHash . (.header)) bns) st)-        MGetData (GetData (map inv bns)) `sendMessage` p+        bns <- sel bb bh >>= reach p+        idle <- null . Sync.outstanding <$> readTVarIO self.sync+        if null bns+          then when idle $ do+            -- it has delivered what it had, and has not announced what+            -- comes next: let it go, for a peer that has to be found+            logDebug self $ "Syncing peer has not announced the next block: " <> p.label+            released <- transition self (Sync.release p)+            when released (void (releasePeer self.owner p))+          else request p bns   where+    request p bns = do+      logDebug self $+        "Requesting "+          <> fromString (show (length bns))+          <> " blocks from peer: "+          <> p.label+      transition self $ \st -> ((), Sync.request (map (headerHash . (.header)) bns) st)+      MGetData (GetData (map inv bns)) `sendMessage` p+    -- the blocks that the peer has announced: all of them if it has the+    -- last, since they are a chain, or else those up to the first that+    -- it has not+    reach _ [] = return []+    reach p bns =+      peerHas self (last bns) p >>= \case+        True -> return bns+        False -> takeWhileM (\n -> peerHas self n p) bns+    takeWhileM _ [] = return []+    takeWhileM f (x : xs) =+      f x >>= \case+        True -> (x :) <$> takeWhileM f xs+        False -> return []     inv = InvVector kind . (.get) . headerHash . (.header)     kind = if self.config.net.segWit then InvWitnessBlock else InvBlock     -- The blocks right after the last requested one, up to one window or@@ -1029,7 +1130,7 @@             else void (releasePeer self.owner p)           return started --- | If not in sync and there is no syncing peer, pick the first free peer and start syncing.+-- | If not in sync and there is no syncing peer, look for one ('search'). trySyncing :: BlockStore -> IO () trySyncing self =   isInSync self >>= \case@@ -1037,27 +1138,24 @@     False ->       isJust . Sync.syncingPeer <$> readTVarIO self.sync >>= \case         True -> return ()-        False -> getPeers self.config.peerMgr >>= recurse . map (.mailbox)-  where-    recurse [] = return ()-    recurse (p : ps) =-      trySetPeer self p >>= \case-        False -> recurse ps-        True -> syncMe self+        False -> revertToMainChain self >> search self [] --- | Start syncing against a newly connected peer if needed, or request its mempool if in sync.+-- | Note the height that a newly connected peer gave, then start syncing if+-- needed, or request its mempool if in sync. trySyncingPeer :: BlockStore -> Peer -> IO ()-trySyncingPeer self p =+trySyncingPeer self p = do+  online <- getOnlinePeer self.config.peerMgr p+  let height = maybe 0 (fromIntegral . max 0 . (.startHeight)) (online >>= (.version))+  atomically . modifyTVar' self.knows $+    HashMap.insertWith (\_ k -> Known (max height k.height) k.tip) p.label (Known height Nothing)   isInSync self >>= \case     True -> syncMempool self p-    False ->-      trySetPeer self p >>= \case-        False -> return ()-        True -> syncMe self+    False -> trySyncing self  -- | Release the syncing peer, if any. clearSyncingState :: BlockStore -> IO ()-clearSyncingState self =+clearSyncingState self = do+  transition self $ \s -> ((), Sync.settle s)   Sync.syncingPeer <$> readTVarIO self.sync >>= mapM_ (finishPeer self)  -- | Handle a message sent to the block store.@@ -1076,6 +1174,7 @@ processBlockStoreMessage self (BlockPeerDisconnect p) = do   logDebug self $     "Peer disconnected: " <> p.label+  atomically $ modifyTVar' self.knows $ HashMap.delete p.label   finishPeer self p processBlockStoreMessage self (BlockReceived p b) = do   logDebug self $@@ -1089,6 +1188,8 @@       <> ": "       <> T.unwords (map blockHashToHex bs)   processNoBlocks self p bs+processBlockStoreMessage self (BlockAnnounced p bs) = processAnnounced self p bs+processBlockStoreMessage self (BlockPong p n) = processPong self p n processBlockStoreMessage self (TxRefReceived p tx) = do   logDebug self $     "Transaction received from peer "@@ -1105,14 +1206,15 @@   processTxs self p ts processBlockStoreMessage self (BlockPing r) = do   logDebug self "Internal clock event"+  -- a request that no peer answered, as a node does not for a transaction+  -- that it has not announced to the store, may be made again+  now <- getCurrentTime+  changePending self (Pending.prune now)   setStoreHeight self   setHeadersHeight self   setPendingTxs self   setPeersConnected self   trySyncing self-  processMempool self-  pruneOrphans self-  pruneMempool self   checkTime self   atomically (r ()) @@ -1152,6 +1254,22 @@   Peer -> [BlockHash] -> BlockStore -> IO () blockStoreNotFound peer blocks store =   BlockNotFound peer blocks `send` store.mailbox++-- | Tell the block store that a peer announced blocks.+blockStoreAnnounced :: Peer -> [BlockHash] -> BlockStore -> IO ()+blockStoreAnnounced peer blocks store = BlockAnnounced peer blocks `send` store.mailbox++-- | Tell the block store that a peer answered a ping.+blockStorePong :: Peer -> Word64 -> BlockStore -> IO ()+blockStorePong peer nonce store = BlockPong peer nonce `send` store.mailbox++-- | 'blockStoreAnnounced' in 'STM'.+blockStoreAnnouncedSTM :: Peer -> [BlockHash] -> BlockStore -> STM ()+blockStoreAnnouncedSTM peer blocks store = BlockAnnounced peer blocks `sendSTM` store.mailbox++-- | 'blockStorePong' in 'STM'.+blockStorePongSTM :: Peer -> Word64 -> BlockStore -> STM ()+blockStorePongSTM peer nonce store = BlockPong peer nonce `sendSTM` store.mailbox  -- | Give the block store a transaction received from a peer. blockStoreTx ::
src/Haskoin/Store/BlockStore/Pending.hs view
@@ -43,6 +43,7 @@      -- * Transitions     requested,+    answered,     received,     takeReady,     imported,@@ -116,6 +117,11 @@ -- | Transactions were requested. requested :: UTCTime -> [TxHash] -> Pending -> Pending requested now hs p = p {asked = HashMap.union (HashMap.fromList [(h, now) | h <- hs]) p.asked}++-- | A transaction that was requested arrived, to be offered to the pool+-- of the fleet: it is no longer a request.+answered :: TxHash -> Pending -> Pending+answered h p = p {asked = HashMap.delete h p.asked}  -- | A transaction was received, with the parents that it spends and the -- database does not have. It is no longer a request.
src/Haskoin/Store/BlockStore/Sync.hs view
@@ -8,12 +8,12 @@ -- -- The state of the block download, as a pure state machine. ----- The block store downloads blocks from one peer at a time. It is either--- 'Idle', with no syncing peer, or 'Syncing' from one peer, with the blocks--- requested from it and not yet imported, in chain order. Every change of--- that state is one of the transitions of this module, and the block store--- only carries out what they decide: lock or release a peer, send a request,--- disconnect a peer.+-- The block store asks one peer at a time for blocks, the syncing peer. The+-- state holds the blocks that are wanted, in chain order, the peer that is+-- being asked, and the peers that were asked before it and have yet to+-- deliver. Every change of that state is one of the transitions of this+-- module, and the block store only carries out what they decide: lock or+-- release a peer, send a request, disconnect a peer. -- -- = Why a state machine --@@ -24,71 +24,155 @@ -- visible reason. Pure transitions can be tested on their own, with any -- sequence of events, and the rules are in one place: ----- * There is at most one syncing peer, and only 'start' sets one, from 'Idle'.--- * Only blocks requested from the syncing peer are accepted ('accept'), so a---   block from another peer, or one nobody asked for, is a reason to---   disconnect the peer that sent it.--- * A block is requested at most once while it is outstanding ('request').--- * The peer times out when it has delivered nothing for a while---   ('timedOut'); delivering a block or starting resets the clock.+-- * There is at most one syncing peer, and only 'start' and 'pass' set one.+-- * A block is accepted only from a peer that was asked for it ('accept'),+--   so a block that nobody asked that peer for is a reason to disconnect it.+-- * A block is wanted at most once ('request').+-- * A peer that has delivered nothing for a while is passed over+--   ('stalled', 'pass'), and one that has delivered nothing for long is+--   disconnected ('timedOut', 'expired'). --+-- = A peer that does not deliver+--+-- A peer may announce a block and never send it, or send it slowly. There+-- used to be one timeout for that, the one that says a peer is dead, of two+-- minutes: the store waited it out, disconnected the peer, and asked the+-- next. A new block could take two minutes and six seconds to arrive on a+-- store with twenty good peers.+--+-- Now a peer that delivers nothing for a short time, the stall, is passed+-- over: the next peer is asked for the same blocks, and becomes the syncing+-- peer. The one before is not disconnected. It is still owed its blocks,+-- and if it sends one that is still wanted, the block is imported: whoever+-- delivers first wins. One that is not wanted any more is dropped, and is+-- no reason to disconnect anyone. Only a peer that has still not delivered+-- when the long timeout runs out is disconnected, as before.+--+-- The stall doubles each time a peer is passed over, up to the long+-- timeout, and goes back to its start when a block is imported. A store on+-- a link that cannot bring a block within the first stall therefore still+-- gets it, from the first peer that it asked, at the price of having asked+-- others too. Making the single timeout short was the alternative: it+-- disconnects every peer in turn on such a link, each before its block is+-- whole, and the store never syncs.+--+-- = Who is asked+--+-- A peer is asked for a block only if it answers a ping first ('ping',+-- 'pong'). The peers that could be asked are all pinged at once, and the+-- first to answer becomes the syncing peer: the one that is quickest now,+-- which the median of its old pings, by which the node sorts its peers,+-- does not tell. A peer that takes longer than the wait to answer is not+-- asked while another may still answer in time. Only when every peer that+-- was pinged is late is one of them asked all the same ('overdue'): a+-- store on a slow link has only late peers, and must still sync.+--+-- Which peers could be asked is not decided here: the block store pings+-- only those that have announced the block that it needs.+-- -- The peer is a type parameter so that the tests can use plain numbers. module Haskoin.Store.BlockStore.Sync   ( -- * State     Sync (..),-    Download (..),+    Asked (..),+    idle,     syncingPeer,     outstanding,     lastRequested,+    passedPeers,+    Probe (..),+    Pinged (..),+    probing,      -- * Transitions     start,+    pass,     release,     Accept (..),     accept,     imported,     request,     forgetRequests,+    ping,+    pong,+    skip,+    settle,      -- * Decisions     wantsMore,     needsMoreBlocks,+    patience,+    stalled,     timedOut,+    expired,+    overdue,   ) where -import Data.List (delete)+import Data.List (delete, find, nub, sortOn)+import Data.Maybe (isJust, isNothing) import Data.Time.Clock (NominalDiffTime, UTCTime, diffUTCTime)+import Data.Word (Word64) import Haskoin (BlockHash)  -- | State of the block download from peers of type @p@.-data Sync p-  = -- | no syncing peer-    Idle-  | -- | downloading from a peer-    Syncing !(Download p)+data Sync p = Sync+  { -- | blocks requested and not yet imported, in chain order+    wanted :: ![BlockHash],+    -- | the syncing peer, locked for the block store, if there is one+    current :: !(Maybe (Asked p)),+    -- | when a block was last imported, or the syncing peer last changed+    time :: !UTCTime,+    -- | syncing peers passed over since a block was last imported+    passes :: !Int,+    -- | peers that were passed over and still owe blocks+    passed :: ![Asked p],+    -- | the peers that were pinged to find the next syncing peer, if one+    -- is being looked for+    probe :: !(Maybe (Probe p))+  }   deriving stock (Eq, Show) --- | A download from the syncing peer.-data Download p = Download-  { -- | the peer, locked for the block store-    peer :: !p,-    -- | when it last delivered a block, or when syncing started-    time :: !UTCTime,-    -- | blocks requested and not yet imported, in chain order-    blocks :: ![BlockHash]+-- | Peers that were pinged, one of which is to be asked for blocks. All+-- are pinged with the same nonce, so that their answers are told from+-- those to the pings of the node.+data Probe p = Probe+  { nonce :: !Word64,+    -- | never empty, and in the order that they were pinged+    pinged :: ![Pinged p]   }   deriving stock (Eq, Show) +-- | A peer that was pinged, when, and when it answered if it was too late+-- to be chosen for it.+data Pinged p = Pinged+  { peer :: !p,+    at :: !UTCTime,+    answered :: !(Maybe UTCTime)+  }+  deriving stock (Eq, Show)++-- | A peer and the blocks that it was asked for and has not sent.+data Asked p = Asked+  { peer :: !p,+    -- | when it became the syncing peer or, once passed over, when it+    -- last delivered: the start of its silence+    since :: !UTCTime,+    owes :: ![BlockHash]+  }+  deriving stock (Eq, Show)++-- | No syncing peer, and nothing wanted.+idle :: UTCTime -> Sync p+idle now = Sync {wanted = [], current = Nothing, time = now, passes = 0, passed = [], probe = Nothing}+ -- | The syncing peer, if any. syncingPeer :: Sync p -> Maybe p-syncingPeer Idle = Nothing-syncingPeer (Syncing d) = Just d.peer+syncingPeer s = (.peer) <$> s.current --- | Blocks requested from the syncing peer and not yet imported.+-- | Blocks requested and not yet imported. outstanding :: Sync p -> [BlockHash]-outstanding Idle = []-outstanding (Syncing d) = d.blocks+outstanding = (.wanted)  -- | The last block requested, if any is outstanding: the next request -- starts after it.@@ -97,68 +181,163 @@   [] -> Nothing   bs -> Just (last bs) --- | Start syncing from a peer, if there is no syncing peer yet. The caller--- locks the peer first, and frees it again if this gives 'Nothing'.-start :: UTCTime -> p -> Sync p -> Maybe (Sync p)-start now p Idle = Just (Syncing (Download p now []))-start _ _ (Syncing _) = Nothing+-- | The peers that were passed over and still owe blocks.+passedPeers :: Sync p -> [p]+passedPeers s = map (.peer) s.passed --- | Stop syncing from a peer, when it disconnects or the store is in sync.--- Gives the new state, and whether the peer was the syncing peer, in which--- case the caller frees it. Another peer leaves the state as it is.+-- | Whether peers were pinged and none was chosen yet.+probing :: Sync p -> Bool+probing s = isJust s.probe++-- | Start syncing from a peer, if there is no syncing peer. The caller must+-- have locked the peer, and releases it if this gives 'Nothing'. The search+-- for a peer is over.+start :: (Eq p) => UTCTime -> p -> Sync p -> Maybe (Sync p)+start now p s = case s.current of+  Nothing -> Just (enlist now p [] s) {time = now, wanted = [], passes = 0}+  Just _ -> Nothing++-- | Make a peer the syncing peer, owed some blocks. If it was passed over+-- before, it still owes what it owed then: a peer is in the state once.+enlist :: (Eq p) => UTCTime -> p -> [BlockHash] -> Sync p -> Sync p+enlist now p hs s =+  s+    { current = Just (Asked p now (before <> filter (`notElem` before) hs)),+      passed = filter ((/= p) . (.peer)) s.passed,+      probe = Nothing+    }+  where+    before = concat [a.owes | a <- s.passed, a.peer == p]++-- | Pass over the syncing peer for another, which the caller has locked,+-- and gives what the new peer is to be asked for: the blocks that are+-- wanted. The peer passed over is given too, for the caller to release; it+-- stays owed its blocks, and has been silent since the clock last started.+-- Nothing changes without a syncing peer, or if the peer is the syncing peer.+pass :: (Eq p) => UTCTime -> p -> Sync p -> (Maybe (p, [BlockHash]), Sync p)+pass now p s = case s.current of+  Nothing -> (Nothing, s)+  Just old+    | old.peer == p -> (Nothing, s)+    | otherwise ->+        ( Just (old.peer, s.wanted),+          (enlist now p s.wanted s {passed = [old {since = s.time} | not (null old.owes)] <> s.passed})+            { time = now,+              passes = s.passes + 1+            }+        )++-- | Forget a peer, as when it disconnects. Says whether it was the syncing+-- peer, for the caller to unlock it; the blocks wanted are then forgotten+-- with it, for the next syncing peer to be asked from the best block on. release :: (Eq p) => p -> Sync p -> (Bool, Sync p)-release p (Syncing d)-  | d.peer == p = (True, Idle)-release _ s = (False, s)+release p s = case s.current of+  Just a | a.peer == p -> (True, others {current = Nothing, wanted = [], passes = 0})+  _ -> (False, others)+  where+    others = skip p s {passed = filter ((/= p) . (.peer)) s.passed}  -- | What to do with a block that a peer sent. data Accept-  = -- | it was requested from this peer: import it+  = -- | this peer was asked for it and it is wanted: import it     Requested-  | -- | the peer is not the syncing peer+  | -- | this peer was asked for it, and another delivered it first: drop it+    Late+  | -- | the peer was not asked for any block     NotSyncingPeer-  | -- | the syncing peer sent a block that is not outstanding+  | -- | the peer was asked for blocks, and not for this one     NotRequested   deriving stock (Eq, Show) --- | Whether to import a block that a peer sent. Anything but 'Requested'--- means the peer misbehaves, or that the requests were forgotten after a--- reorganisation and the block is from the old chain.-accept :: (Eq p) => p -> BlockHash -> Sync p -> Accept-accept p h = \case-  Syncing d-    | d.peer /= p -> NotSyncingPeer-    | h `elem` d.blocks -> Requested-    | otherwise -> NotRequested-  Idle -> NotSyncingPeer+-- | Decide what to do with a block that a peer sent, and note that the peer+-- no longer owes it.+accept :: (Eq p) => p -> BlockHash -> Sync p -> (Accept, Sync p)+accept p h s+  | Just a <- s.current,+    a.peer == p = case sent a of+      Nothing -> (NotRequested, s)+      Just a' -> (verdict, s {current = Just a'})+  | Just a <- lookupPassed = case sent a of+      Nothing -> (NotRequested, s)+      Just a' -> (verdict, s {passed = [a' | not (null a'.owes)] <> filter ((/= p) . (.peer)) s.passed})+  | otherwise = (NotSyncingPeer, s)+  where+    lookupPassed = case filter ((== p) . (.peer)) s.passed of+      a : _ -> Just a+      [] -> Nothing+    sent a+      | h `elem` a.owes = Just a {owes = delete h a.owes}+      | otherwise = Nothing+    verdict = if h `elem` s.wanted then Requested else Late --- | A requested block was imported: it is no longer outstanding, and the--- peer is not idle.+-- | A block was imported: it is no longer wanted, and the clock and the+-- stall start again. imported :: UTCTime -> BlockHash -> Sync p -> Sync p-imported _ _ Idle = Idle-imported now h (Syncing d) = Syncing d {time = now, blocks = delete h d.blocks}+imported now h s = s {wanted = delete h s.wanted, time = now, passes = 0} --- | Blocks were requested from the syncing peer. Blocks already outstanding--- are not added twice. Nothing changes when idle, as there is nobody to ask.+-- | Blocks were requested from the syncing peer. Those already wanted are+-- not added again. Nothing changes without a syncing peer. request :: [BlockHash] -> Sync p -> Sync p-request _ Idle = Idle-request hs (Syncing d) = Syncing d {blocks = foldl' add d.blocks hs}+request hs s = case s.current of+  Nothing -> s+  Just a -> s {wanted = s.wanted <> new, current = Just a {owes = a.owes <> new}}   where-    add bs h = if h `elem` bs then bs else bs <> [h]+    new = foldl' (\bs h -> if h `elem` bs || h `elem` s.wanted then bs else bs <> [h]) [] hs --- | Forget the outstanding requests, after blocks were reverted: they--- continued a chain that is no longer the best. Blocks of them that still--- arrive are 'NotRequested'.+-- | Forget the blocks that are wanted, as when the chain is reorganised:+-- they will be asked for again from the new best block. The peers that+-- were asked still owe them, so that sending one is no fault of theirs. forgetRequests :: Sync p -> Sync p-forgetRequests Idle = Idle-forgetRequests (Syncing d) = Syncing d {blocks = []}+forgetRequests s = s {wanted = []} --- | The syncing peer, if it is time to ask it for more blocks, given the--- size of the window.+-- | Peers are to be pinged, to ask the first that answers for blocks. Gives+-- the nonce to ping them with and those to ping: the ones that were not+-- pinged already in this search, which goes on with the nonce that it+-- started with. The nonce given is used if this starts the search.+ping :: (Eq p) => UTCTime -> Word64 -> [p] -> Sync p -> (Maybe (Word64, [p]), Sync p)+ping now fresh ps s = case new of+  [] -> (Nothing, s)+  _ -> (Just (n, new), s {probe = Just (Probe n (old <> [Pinged p now Nothing | p <- new]))})+  where+    (n, old) = maybe (fresh, []) (\b -> (b.nonce, b.pinged)) s.probe+    new = nub (filter (`notElem` map (.peer) old) ps)++-- | A peer answered a ping. Says whether it is to be asked for blocks: it+-- was pinged in this search and answered within the wait. It is then out+-- of the search, which the caller ends by making it the syncing peer+-- ('start', 'pass'), or which goes on without it if the peer cannot be+-- locked. A late answer is kept, for 'overdue' to prefer the peer to+-- those that never answered.+pong :: (Eq p) => NominalDiffTime -> UTCTime -> p -> Word64 -> Sync p -> (Bool, Sync p)+pong wait now p n s = case s.probe of+  Just b+    | b.nonce == n,+      Just g <- find ((== p) . (.peer)) b.pinged,+      isNothing g.answered ->+        if now `diffUTCTime` g.at <= wait+          then (True, skip p s)+          else (False, s {probe = Just (Probe n (map late b.pinged))})+  _ -> (False, s)+  where+    late g = if g.peer == p then Pinged p g.at (Just now) else g++-- | Leave a peer out of the search for a syncing peer, which ends if it+-- was the last one pinged.+skip :: (Eq p) => p -> Sync p -> Sync p+skip p s = case s.probe of+  Nothing -> s+  Just b -> case filter ((/= p) . (.peer)) b.pinged of+    [] -> s {probe = Nothing}+    rest -> s {probe = Just (Probe b.nonce rest)}++-- | End the search for a syncing peer.+settle :: Sync p -> Sync p+settle s = s {probe = Nothing}++-- | The syncing peer, if it should be asked for the next batch of blocks. wantsMore :: Int -> Sync p -> Maybe p-wantsMore _ Idle = Nothing-wantsMore window (Syncing d)-  | needsMoreBlocks window (length d.blocks) = Just d.peer+wantsMore window s+  | needsMoreBlocks window (length s.wanted) = syncingPeer s   | otherwise = Nothing  -- | Whether to ask for the next batch, given the window size and how many@@ -169,10 +348,44 @@ needsMoreBlocks :: Int -> Int -> Bool needsMoreBlocks window n = n < max 1 (window `div` 4) --- | The syncing peer, if it has delivered nothing for longer than the--- timeout.+-- | How long the syncing peer has to deliver a block before another is+-- asked: the stall, doubled for each peer passed over since a block was+-- last imported, and never longer than the timeout.+patience :: NominalDiffTime -> NominalDiffTime -> Sync p -> NominalDiffTime+patience stall timeout s = min timeout (stall * 2 ^ min 30 s.passes)++-- | The peers to ask all the same, best first, if none of those pinged+-- answered within the wait and none can any more: there is no other+-- option. Those that answered late come first, in the order that they+-- answered, then those that have not answered, in the order that they were+-- pinged. None while a peer that was pinged may still answer in time.+overdue :: NominalDiffTime -> UTCTime -> Sync p -> [p]+overdue wait now s = case s.probe of+  Just b+    | all late b.pinged ->+        map (.peer) (sortOn (.answered) (filter (isJust . (.answered)) b.pinged))+          <> map (.peer) (filter (isNothing . (.answered)) b.pinged)+  _ -> []+  where+    late g = isJust g.answered || now `diffUTCTime` g.at > wait++-- | The syncing peer, if blocks are wanted and none was imported for+-- longer than its 'patience': time to ask another peer, with 'pass'.+stalled :: NominalDiffTime -> NominalDiffTime -> UTCTime -> Sync p -> Maybe p+stalled stall timeout now s+  | null s.wanted = Nothing+  | now `diffUTCTime` s.time > patience stall timeout s = syncingPeer s+  | otherwise = Nothing++-- | The syncing peer, if no block was imported for longer than the+-- timeout: nobody could be asked in its place, and it is to be+-- disconnected. timedOut :: NominalDiffTime -> UTCTime -> Sync p -> Maybe p-timedOut _ _ Idle = Nothing-timedOut timeout now (Syncing d)-  | now `diffUTCTime` d.time > timeout = Just d.peer+timedOut timeout now s+  | now `diffUTCTime` s.time > timeout = syncingPeer s   | otherwise = Nothing++-- | The peers passed over that still owe blocks longer than the timeout+-- after they were asked: they are to be disconnected.+expired :: NominalDiffTime -> UTCTime -> Sync p -> [p]+expired timeout now s = [a.peer | a <- s.passed, now `diffUTCTime` a.since > timeout]
− src/Haskoin/Store/Cache.hs
@@ -1,1535 +0,0 @@-{-# LANGUAGE MultiWayIf #-}-{-# LANGUAGE RecordWildCards #-}-{-# LANGUAGE TemplateHaskell #-}-{-# OPTIONS_GHC -Wno-ambiguous-fields #-}---- |--- Module      : Haskoin.Store.Cache--- Copyright   : (c) Haskoin Developers--- License     : MIT--- Maintainer  : jprupp@protonmail.ch--- Stability   : experimental--- Portability : POSIX------ Redis cache of extended public keys (xpubs), shared by several stores.------ An xpub with many used addresses is slow to answer from the database: every--- address has to be derived and read. Wallets of exchanges and payment--- processors are like that, and are asked about all the time. The cache keeps--- everything that the web server needs to answer about such an xpub in Redis:--- the balances of its addresses up to the gap, its transactions, its unspent--- outputs, and the bodies of its transactions with who spends their outputs.------ = One source of truth------ The data of a cached xpub always comes from the database, read with the--- same functions that answer uncached queries--- ("Haskoin.Store.Database.Query"), and is never computed from what the--- cache guesses a change did, as the cache that this one replaced did, and--- drifted. Whenever a block or a mempool transaction touches one of its--- addresses, or a reorganisation changes one of its transactions, the store--- that saw it reads from the database what that change affects, the--- transactions and the outputs they create and spend, and writes the--- differences in one atomic step ('cacheRefresh'). It reads the whole xpub--- instead when it caches it, and whenever it cannot be sure that the entry--- has every earlier change: checks against the balances of every address of--- the xpub, and a full reload every ten minutes, keep a missed change from--- staying. Reloads of one xpub are coalesced, so a burst of payments costs at--- most one reload in progress and one waiting.------ Everything a response needs is in Redis, including the bodies of the--- transactions: a store that has not yet seen a transaction of a cached xpub--- still serves it in full. Bodies are kept once per transaction and shared by--- the xpubs that refer to them; the last xpub to let go of one deletes it.--- Bodies are stored without the spenders of their outputs, which change--- without touching the xpub (an output paying someone else is spent by a--- transaction that has nothing to do with the xpub). The spenders are in a--- separate hash per transaction, kept up to date by every store as it imports--- transactions, and filled in when a body is read.------ = Several stores, one Redis------ Stores index the same chain at slightly different times, and see mempool--- transactions in different orders. Three rules keep them from getting in--- each other's way:------ * __Each change is applied once.__ A store claims each change it sees, per---   cached xpub that it touches (@ev:\<event\>:\<id\>@). The first store to---   claim it reloads the xpub and marks the claim done; the others skip it. A---   store that is catching up finds its events done and does no work, and a---   store whose database does not have the best header ignores events---   altogether.------ * __No store writes what knows less than Redis.__ A store does not replace---   an entry stamped with a higher block than its own best block, nor one---   with a transaction that its database has never seen; it gives the change---   up for a store that knows more. So stores never undo each other's work.------ * __No entry refers to what Redis does not have.__ An entry is written in---   one Lua script together with the bodies it needs, and the script writes---   nothing if a body would be missing. A store that is behind can serve a---   cached xpub in full, transactions it has not imported included. Bodies---   that Redis already has are rewritten just before, in small scripts, so---   that a large block does not hold Redis for long; a body may so be a---   moment newer than an entry that has it, never older.------ The first two rules are the pure decisions of--- "Haskoin.Store.Cache.Protocol", where a model test checks them with several--- stores catching up in random interleavings.------ A lock per xpub keeps two stores from reloading it at the same time, and--- every change of several keys of an entry is a Lua script, so readers never--- see half of one. Stores that answer from Redis serve the same data; stores whose--- databases disagree can still give different answers for xpubs that are not--- cached.------ = Keys------ Every key starts with @hs2:\<network\>:@, so this cache can share a Redis with--- the cache of older versions of the store, whose keys have no such prefix,--- and with stores of other networks.------ * @x:\<id\>:meta@: hash with the serialized xpub (@spec@), the height of---   the best block it was read at (@height@), the format version (@ver@,---   now 3), the generation of the entry (@gen@), which every reload---   increments, and the time of its last full reload in milliseconds---   (@full@).---   @\<id\>@ is the hexadecimal of the first 16 bytes of the SHA-256 of the---   serialized xpub.--- * @x:\<id\>:bal@: balances of the addresses up to the gap, serialized.--- * @x:\<id\>:txs@: sorted set of transaction hashes, scored by 'blockRefScore'.--- * @x:\<id\>:utxo@: sorted set of serialized unspent outputs, scored the same.--- * @x:\<id\>:uop@: hash of serialized outpoint to its member of @utxo@, so---   that an output can be removed by its outpoint alone, though its member---   changes with its block. Entries of version 2 have none; their next full---   reload builds it.--- * @x:\<id\>:addrs@: set of the addresses of the balances.--- * @a:\<address\>@: set of the ids of the cached xpubs with this address.--- * @tx:\<txid\>@: transaction body, serialized 'TxData' without spenders.--- * @sp:\<txid\>@: hash of output index to spender.--- * @txref:\<txid\>@: set of the ids of the cached xpubs with this transaction.--- * @lru@: sorted set of ids by time of last use, for eviction.--- * @lock:\<id\>@: lock held while an xpub is reloaded.--- * @ev:\<event\>:\<id\>@: claim of a change of a cached xpub: @busy@ while a---   store applies it, for two minutes at most, then @done@ for an hour.-module Haskoin.Store.Cache-  ( -- * Running the cache-    Cache,-    CacheConfig (..),-    withCache,-    cacheAlive,--    -- * Reading-    cacheXPubBals,-    cacheXPubTxs,-    cacheXPubTxCount,-    cacheXPubUnspents,-    cacheTxData,-    cacheIsCached,-    CacheCounters (..),-    cacheCounters,--    -- * Changing-    cacheRequest,-    Reload (..),-    cacheRefresh,-    cacheDelete,-    cacheEvict,--    -- * Internals--    -- | Exported for tests.-    cachePrefix,-    xpubId,-    blockRefScore,-    scoreBlockRef,-    fullReloadEvery,-    fullReloadAfter,-  )-where--import Control.Monad (forM, forM_, join, unless, void, when)-import Data.Bits (shift, (.&.), (.|.))-import Data.ByteString (ByteString)-import Data.ByteString qualified as BS-import Data.ByteString.Char8 qualified as C8-import Data.Either (rights)-import Data.FileEmbed (embedFile, makeRelativeToProject)-import Data.Functor ((<&>))-import Data.HashMap.Strict (HashMap)-import Data.HashMap.Strict qualified as HashMap-import Data.HashSet (HashSet)-import Data.HashSet qualified as HashSet-import Data.IntMap.Strict qualified as IntMap-import Data.List (nub)-import Data.List.NonEmpty qualified as NonEmpty-import Data.Maybe (catMaybes, fromMaybe, isJust, isNothing, maybeToList)-import Data.Serialize (decode, encode)-import Data.String.Conversions (cs)-import Data.Text (Text)-import Data.Time.Clock.POSIX (getPOSIXTime)-import Data.Word (Word32, Word64)-import Database.Redis (Connection, Redis, Reply)-import Database.Redis qualified as R-import Haskoin-  ( Address,-    BlockHash,-    BlockHeader (..),-    BlockNode (..),-    Network (..),-    OutPoint (..),-    Tx (..),-    TxHash,-    TxIn (..),-    TxOut (..),-    encodeHex,-    headerHash,-    scriptToAddressBS,-    sha256,-    txHash,-  )-import Haskoin.Node (Chain, LogLevel (..), Logger, chainGetBest, logAt)-import Haskoin.Store.Cache.Protocol-  ( Claim (..),-    OnClaim (..),-    OnReload (..),-    Reload (..),-    afterClaim,-    afterReload,-    mayReplace,-  )-import Haskoin.Store.Common (Limits (..), Start (..), StoreEvent (..), loopForever)-import Haskoin.Store.Data-import Haskoin.Store.Database.Query qualified as Q-import Haskoin.Store.Database.Query.Layered qualified as L-import Haskoin.Store.Database.Reader (DatabaseReader (..))-import Haskoin.Store.Outage (Outage, attempt, attempting, newOutage)-import NQE (Inbox, Publisher, receive, withSubscription)-import System.Metrics.StatsD-  ( StatCounter,-    StatTiming,-    Stats,-    addTiming,-    incrementCounter,-    newStatCounter,-    newStatTiming,-  )-import System.Random (randomIO)-import Text.Read (readMaybe)-import UnliftIO-  ( IORef,-    TVar,-    atomicModifyIORef',-    atomically,-    checkSTM,-    handleAny,-    link,-    modifyTVar',-    newIORef,-    newTVarIO,-    pooledForConcurrentlyN_,-    readIORef,-    readTVar,-    timeout,-    withAsync,-    writeIORef,-    writeTVar,-  )-import UnliftIO.Concurrent (forkIO, threadDelay)---- | Settings of the cache.-data CacheConfig = CacheConfig-  { -- | Redis URL, such as @redis:\/\/localhost:6379@; empty for the default-    url :: !String,-    -- | cache xpubs with at least this many used addresses-    minAddrs :: !Int,-    -- | or with at least this many transactions in their addresses-    minTxs :: !Int,-    -- | evict the least recently used xpubs past this many-    maxXPubs :: !Int,-    -- | StatsD client, to report the metrics of the cache-    stats :: !(Maybe Stats)-  }---- | StatsD metrics of the cache.-data CacheMetrics = CacheMetrics-  { -- | xpubs answered from the cache (@cache.hits@)-    hits :: !StatCounter,-    -- | xpubs asked about that were not cached (@cache.misses@)-    misses :: !StatCounter,-    -- | xpubs read again from the database and written (@cache.reloads@)-    reloads :: !StatCounter,-    -- | reloads refused because the cache had newer data (@cache.stale@)-    stale :: !StatCounter,-    -- | xpubs evicted (@cache.evictions@)-    evictions :: !StatCounter,-    -- | reloads that read the whole xpub rather than what changed-    -- (@cache.full_reloads@)-    fullReloads :: !StatCounter,-    -- | milliseconds to reload an xpub (@cache.reload@)-    reloadTime :: !StatTiming-  }---- | Register the metrics of the cache.-newCacheMetrics :: Stats -> IO CacheMetrics-newCacheMetrics s = do-  hits <- c "hits"-  misses <- c "misses"-  reloads <- c "reloads"-  stale <- c "stale"-  evictions <- c "evictions"-  fullReloads <- c "full_reloads"-  reloadTime <- newStatTiming s "cache.reload" 10-  return CacheMetrics {..}-  where-    c x = newStatCounter s ("cache." <> x) 10---- | Read the database through a view of it as it is now: the mempool is--- kept apart from the chain and laid over it when read, and the two have to--- be read as they were at one moment--- ("Haskoin.Store.Database.Query.Layered"). Transactions that left the--- mempool unconfirmed are among what these reads give, marked as deleted:--- the cache has to hear of them to take them out.-viewed :: Cache -> (L.View -> IO a) -> IO a-viewed c = L.withView c.db---- | A running cache.-data Cache = Cache-  { redis :: !Connection,-    prefix :: !ByteString,-    db :: !DatabaseReader,-    chain :: !Chain,-    minAddrs :: !Int,-    minTxs :: !Int,-    maxXPubs :: !Int,-    logger :: !Logger,-    metrics :: !(Maybe CacheMetrics),-    -- | what this store did to the cache since it started-    counters :: !(IORef CacheCounters),-    -- | work for the worker, newest first-    jobs :: !(TVar [Job]),-    -- | when this store last marked each xpub as used, by id, in-    -- milliseconds; see 'touchEvery'-    touched :: !(IORef (HashMap ByteString Integer)),-    -- | transactions that this store deleted since it started, as far as-    -- the cache heard of them, up to a hundred thousand. The database-    -- keeps nothing of a deleted transaction, so without this one that-    -- Redis has and the database does not could as well be one that this-    -- store has not seen yet, which an entry must not lose.-    lost :: !(IORef (HashSet TxHash)),-    -- | whether the last thing that the worker tried failed, so that an-    -- outage of Redis is logged once-    outage :: !Outage-  }---- | What a store did to the cache since it started.-data CacheCounters = CacheCounters-  { -- | xpubs read again from the database and written-    reloads :: !Int,-    -- | reloads given up because Redis knew more-    behind :: !Int,-    -- | reloads that read the whole xpub rather than what changed-    fullReloads :: !Int-  }-  deriving (Eq, Show)---- | What this store did to the cache since it started.-cacheCounters :: Cache -> IO CacheCounters-cacheCounters c = readIORef c.counters---- | Work for the worker of the cache.-data Job-  = -- | an event of the store, which may change cached xpubs-    JobEvent !StoreEvent-  | -- | cache an xpub that was asked about-    JobCache !XPubSpec-  | -- | try again to claim changes of an xpub that another store was busy-    -- with: the transactions that changed it, and the claims-    JobClaim !XPubSpec !(HashSet TxHash) ![ByteString]-  | -- | reload an xpub for changes whose claims this store holds-    JobHeld !XPubSpec !(HashSet TxHash) ![ByteString]---- | Connect to Redis and run the process that keeps the cache up to date with--- the events of the store, while an action runs.-withCache ::-  CacheConfig ->-  DatabaseReader ->-  Chain ->-  Publisher StoreEvent ->-  Logger ->-  (Cache -> IO a) ->-  IO a-withCache cfg db chain pub logger action = do-  info <--    if null cfg.url-      then return R.defaultConnectInfo-      else either fail return (R.parseConnectInfo cfg.url)-  conn <- R.checkedConnect info-  jobs <- newTVarIO []-  counters <- newIORef (CacheCounters 0 0 0)-  touched <- newIORef HashMap.empty-  lost <- newIORef HashSet.empty-  outage <- newOutage-  metrics <- mapM newCacheMetrics cfg.stats-  let cache =-        Cache-          { redis = conn,-            prefix = cachePrefix db.net,-            db = db,-            chain = chain,-            minAddrs = cfg.minAddrs,-            minTxs = cfg.minTxs,-            maxXPubs = cfg.maxXPubs,-            logger = logger,-            metrics = metrics,-            counters = counters,-            jobs = jobs,-            touched = touched,-            lost = lost,-            outage = outage-          }-  withSubscription pub $ \events ->-    withAsync (listener cache events) $ \a1 ->-      withAsync (worker cache) $ \a2 -> do-        link a1-        link a2-        action cache---- | Prefix of every key of the cache for a network.-cachePrefix :: Network -> ByteString-cachePrefix net = "hs2:" <> C8.pack net.name <> ":"---- | Identifier of an xpub in the keys.-xpubId :: XPubSpec -> ByteString-xpubId = cs . encodeHex . BS.take 16 . encode . sha256 . encode--xkey :: Cache -> XPubSpec -> ByteString -> ByteString-xkey c x suffix = c.prefix <> "x:" <> xpubId x <> ":" <> suffix--txKey :: Cache -> TxHash -> ByteString-txKey c h = c.prefix <> "tx:" <> encode h--spKey :: Cache -> TxHash -> ByteString-spKey c h = c.prefix <> "sp:" <> encode h--addrKey :: Cache -> Address -> ByteString-addrKey c a = c.prefix <> "a:" <> encode a--lruKey :: Cache -> ByteString-lruKey c = c.prefix <> "lru"---- | Count something in the metrics, if they are enabled.-count :: Cache -> (CacheMetrics -> StatCounter) -> Int -> IO ()-count c f n = mapM_ (\m -> incrementCounter (f m) n) c.metrics--log' :: Cache -> LogLevel -> Text -> IO ()-log' c l = logAt l c.logger---- | Whether Redis answers a ping within two seconds. The health check of--- the web server asks, so that a store whose Redis is gone says that it is--- answering from its own database alone.-cacheAlive :: Cache -> IO Bool-cacheAlive c =-  handleAny (\_ -> return False) $-    timeout 2000000 (R.runRedis c.redis R.ping) <&> \case-      Just (Right _) -> True-      _ -> False---- | Run Redis commands, throwing if Redis answers with an error.-run :: Cache -> Redis (Either Reply a) -> IO a-run c f =-  R.runRedis c.redis f >>= \case-    Right x -> return x-    Left e -> fail ("Redis error: " <> show e)------------------------------------------------------------------------------------ Scores------------------------------------------------------------------------------------ | Score of a position in a Redis sorted set, such that sorting by score--- ascending sorts from newest to oldest. Mempool transactions get negative--- scores from their time, so they come first; confirmed ones get positive--- scores that decrease with height and position. Height and position are--- packed into the 53 bits that a double holds exactly.------ Between equal scores Redis orders members by their bytes, which for--- transaction hashes and unspent outputs is the order that the database uses--- ('Haskoin.Store.Common.txRefKey', 'Haskoin.Store.Common.unspentKey'), so a--- cached page is the page the database gives.-blockRefScore :: BlockRef -> Double-blockRefScore BlockRef {height = h, position = p} =-  fromIntegral (0x001fffffffffffff - (h' .|. p'))-  where-    h' = (fromIntegral h .&. 0x07ffffff) `shift` 26 :: Word64-    p' = (fromIntegral p .&. 0x03ffffff) :: Word64-blockRefScore MemRef {timestamp = t} = negate t'-  where-    t' = fromIntegral (t .&. 0x001fffffffffffff)---- | Inverse of 'blockRefScore'.-scoreBlockRef :: Double -> BlockRef-scoreBlockRef s-  | s < 0 = MemRef {timestamp = n}-  | otherwise = BlockRef {height = h, position = p}-  where-    n = truncate (abs s) :: Word64-    m = 0x001fffffffffffff - n-    h = fromIntegral (m `shift` (-26))-    p = fromIntegral (m .&. 0x03ffffff)------------------------------------------------------------------------------------ Reading------------------------------------------------------------------------------------ | Balances of a cached xpub, or 'Nothing' if it is not cached. Marks the--- xpub as used. An entry that cannot be decoded is deleted and counts as not--- cached.-cacheXPubBals :: Cache -> XPubSpec -> IO (Maybe [XPubBal])-cacheXPubBals c x = do-  touch <- touchArgs c x-  r <- run c $ do-    b <- R.get (xkey c x "bal")-    -- marking an xpub that is not cached is harmless: eviction removes it-    forM_ touch $ \now -> R.zadd (lruKey c) [(fromIntegral now, xpubId x)]-    return b-  case r of-    Nothing -> count c (.misses) 1 >> return Nothing-    Just bs -> case decode bs of-      Right bals -> count c (.hits) 1 >> return (Just bals)-      Left _ -> do-        count c (.misses) 1-        log' c LevelWarn "Cached xpub balances cannot be decoded; deleting them"-        void (cacheDelete c x)-        return Nothing---- | Whether an xpub is cached.-cacheIsCached :: Cache -> XPubSpec -> IO Bool-cacheIsCached c x = run c (R.exists (xkey c x "meta"))---- | Number of transactions of a cached xpub.-cacheXPubTxCount :: Cache -> XPubSpec -> IO (Maybe Word32)-cacheXPubTxCount c x = do-  touch <- touchArgs c x-  -- sent together, in one round trip-  (cached, n) <- run c $ do-    e <- R.exists (xkey c x "meta")-    z <- R.zcard (xkey c x "txs")-    forM_ touch $ \now -> R.zadd (lruKey c) [(fromIntegral now, xpubId x)]-    return ((,) <$> e <*> z)-  return (if cached then Just (fromIntegral n) else Nothing)---- | Page of the transactions of a cached xpub, newest first, as the database--- would give it.-cacheXPubTxs :: Cache -> XPubSpec -> Limits -> IO (Maybe [TxRef])-cacheXPubTxs c x limits = do-  xs <- page c x "txs" limits (\th -> (0, encode th))-  return $ xs <&> \ys -> [TxRef {txid = t, block = scoreBlockRef sc} | (bs, sc) <- ys, Right t <- [decode bs]]---- | Page of the unspent outputs of a cached xpub, newest first, given its--- balances, which give the paths of the addresses.-cacheXPubUnspents :: Cache -> XPubSpec -> [XPubBal] -> Limits -> IO (Maybe [XPubUnspent])-cacheXPubUnspents c x bals limits = do-  -- members are serialized unspent outputs: their block position, which is-  -- nine bytes long, then their output point-  xs <- page c x "utxo" limits (\th -> (9, encode (OutPoint th 0)))-  let paths = HashMap.fromList [(b.balance.address, b.path) | b <- bals]-  return $-    xs <&> \ys ->-      [ XPubUnspent {unspent = u, path = p}-      | (bs, _) <- ys,-        Right u <- [decode bs],-        Just a <- [u.address],-        Just p <- [HashMap.lookup a paths]-      ]---- | How often a store marks an xpub as used, in milliseconds. Marking it on--- every read would make every read a write; eviction only needs to know--- which xpubs were used recently.-touchEvery :: Integer-touchEvery = 10000---- | The time to mark an xpub as used with, if this store has not marked it--- in the last 'touchEvery' milliseconds.-touchArgs :: Cache -> XPubSpec -> IO (Maybe Integer)-touchArgs c x = do-  now <- nowMs-  atomicModifyIORef' c.touched $ \m ->-    case HashMap.lookup (xpubId x) m of-      Just t | now - t < touchEvery -> (m, Nothing)-      _ ->-        -- forget everything now and then, so that the map stays small-        let m' = if HashMap.size m > 100000 then HashMap.empty else m-         in (HashMap.insert (xpubId x) now m', Just now)---- | A page of a sorted set of a cached xpub, with the scores of its members,--- or 'Nothing' if the xpub is not cached. One round trip to Redis: the--- script finds where the page starts, as the database would, and marks the--- xpub as used if it is time to.------ A page that starts at a transaction starts at its position, which is the--- score of the transaction in the sorted set of transactions of the xpub.--- Members of the same score whose bytes, after a number of bytes, sort--- before the bytes that the function gives for the transaction are skipped,--- as the database skips them. A start transaction that is not in the xpub is--- looked up in Redis or in the database, in a second round trip; a page--- with a start that is found nowhere is empty.-page ::-  Cache ->-  XPubSpec ->-  ByteString ->-  Limits ->-  (TxHash -> (Int, ByteString)) ->-  IO (Maybe [(ByteString, Double)])-page c x set limits tieBytes = do-  touch <- touchArgs c x-  let common =-        [ xkey c x "meta",-          xkey c x set,-          xkey c x "txs",-          lruKey c,-          maybe "" num touch,-          xpubId x,-          num limits.offset,-          num limits.limit-        ]-      call extra = run c (R.eval pageScript [] (common <> extra))-  r <- case limits.start of-    Nothing -> call ["top"]-    Just (AtBlock h) -> call ["score", num' (blockRefScore (BlockRef h maxBound))]-    Just (AtTx th) -> do-      let (skip, bytes) = tieBytes th-      call ["tx", encode th, num skip, bytes] >>= \case-        ["nostart"] ->-          startOf th >>= \case-            Nothing -> return ["ok"]-            Just ref -> call ["at", num' (blockRefScore ref), num skip, bytes]-        y -> return y-  case r of-    "ok" : rest -> return (Just (pairs rest))-    _ -> return Nothing-  where-    pairs (m : sc : rest) | Just d <- readMaybe (C8.unpack sc) = (m, d) : pairs rest-    pairs _ = []-    startOf th =-      cacheTxData c [th] >>= \case-        [Just td] -> return (Just td.block)-        _ -> fmap (.block) <$> viewed c (\v -> L.getTxData v th)---- | Bodies of transactions that Redis has, in the order of the hashes, with--- the spenders of their outputs filled in. 'Nothing' for those it does not--- have.-cacheTxData :: Cache -> [TxHash] -> IO [Maybe TxData]-cacheTxData _ [] = return []-cacheTxData c hs = do-  bodies <- run c (R.mget (NonEmpty.fromList (map (txKey c) hs)))-  spenders <- run c $ do-    rs <- forM (zip hs bodies) $ \(h, b) ->-      if isJust b then fmap Just <$> R.hgetall (spKey c h) else return (pure Nothing)-    return (sequence rs)-  return (zipWith fill bodies spenders)-  where-    fill (Just b) sp-      | Right td <- decode b =-          let ss =-                IntMap.fromList-                  [ (fromIntegral i, s)-                  | (k, v) <- fromMaybe [] sp,-                    Right (i :: Word32) <- [decode k],-                    Right s <- [decode v]-                  ]-           in Just (td :: TxData) {spenders = ss}-    fill _ _ = Nothing------------------------------------------------------------------------------------ Changing------------------------------------------------------------------------------------ | Cache an xpub that was just read from the database, if it has enough--- used addresses, or enough transactions: a wallet that reuses a few--- addresses for many payments is as slow to read from the database as one--- with many addresses. Transactions are counted per address, so one that--- touches two addresses counts twice, which is close enough to decide. The--- xpub is loaded in the background; the caller keeps the answer it has.-cacheRequest :: Cache -> XPubSpec -> [XPubBal] -> IO ()-cacheRequest c x bals =-  when (length used >= c.minAddrs || sum (map (.balance.txs) used) >= fromIntegral c.minTxs) $-    push c (JobCache x)-  where-    used = filter (not . nullBalance . (.balance)) bals---- | Give the worker something to do.-push :: Cache -> Job -> IO ()-push c j = atomically (modifyTVar' c.jobs (j :))---- | Give the worker something to do in two seconds.-later :: Cache -> Job -> IO ()-later c j = void . forkIO $ threadDelay 2000000 >> push c j---- | Remove an xpub from the cache. Returns whether it was cached.-cacheDelete :: Cache -> XPubSpec -> IO Bool-cacheDelete c x = do-  r <- run c (R.eval deleteScript [] [c.prefix, xpubId x])-  return (r == (1 :: Integer))---- | Evict the least recently used xpubs until at most the given number are--- cached. Returns how many were evicted. The cache does this by itself after--- caching an xpub, down to its configured maximum.-cacheEvict :: Cache -> Int -> IO Int-cacheEvict c n = do-  evicted <- fromInteger <$> run c (R.eval evictScript [] [c.prefix, num n])-  count c (.evictions) evicted-  return evicted---- | Bring the entry of an xpub in Redis up to date with the database, unless--- another store holds its lock or Redis knows more than this store (see--- 'Reload'). The transactions given are the ones that changed it: their--- bodies are written again even if Redis has them.------ A store never writes data that knows less than what is in Redis, so stores--- that see the chain and the mempool at slightly different times do not undo--- each other's work. Every body that the entry refers to is in Redis before--- the entry is: the changes and the missing bodies are written in one Lua--- script, which writes nothing if a body is missing, so a store that is--- behind never finds a cached transaction that it cannot serve. Only the--- differences are written, and the script writes nothing if the entry--- changed since it was read (its generation), so Redis is held for the size--- of the change, not of the xpub. Bodies that Redis already has, such as--- those of the transactions that a block confirms, are rewritten first, in--- small scripts ('rewriteBodies'), so that the confirmation of a large block--- does not hold Redis in one long script.------ = Reading only what changed------ Working out the differences can take one of two paths.------ * __Incremental__ ('incrementalDiff'): only the transactions that changed---   are read, with the outputs that they create and spend at addresses of---   the xpub. The stress test (@stress\/results@) showed that reading the---   whole xpub from both the database and Redis was almost all the time of a---   reload, growing with the size of the xpub to seconds at 100,000---   transactions; the changes of one block or transaction are a few reads,---   whatever the size of the xpub.------ * __Full__: the whole xpub is read from the database and from Redis, and---   compared. This is what a first caching does, and the fallback whenever---   the incremental path cannot be sure of its answer.------ The incremental path relies on every change of the xpub having been--- applied to the entry, which is what the claims of events are for, but a--- store can miss events: while it catches up, after a restart, or after an--- error. So it is only taken when the entry shows no sign of having missed--- one, and it checks its own result:------ * the entry is in the current format, stamped with the best block of this---   store or the one before it (a larger gap means blocks it may have---   missed), and was fully reloaded less than 'fullReloadAfter' ago;------ * the balances of every address of the xpub are read from the database,---   which is cheap, a read per address; an address that no changed---   transaction touches must have the balance that Redis has. Under load it---   often does not, because the database is already ahead of the events---   being applied, so the newer transactions of those addresses are added to---   the changes first ('catchUp'), and only a balance that they do not---   explain means a change was missed;------ * no address with transactions enters or leaves the gap, as its history---   would have to be read whole.------ Otherwise the full path runs. A missed change that leaves every balance--- as it was, such as the confirmation of a transaction that only spends from--- the xpub, is the one thing the checks cannot see, and the regular full--- reload bounds how long it can last.-cacheRefresh :: Cache -> XPubSpec -> HashSet TxHash -> IO Reload-cacheRefresh c x changed =-  fromMaybe Locked <$> withLock c x (timed (go (3 :: Int) []))-  where-    go tries extra = do-      m <- readMeta c x-      height <- bestHeight c-      now <- nowMs-      if not (mayReplace m.stamp height False)-        then behind-        else do-          bals <- viewed c (\v -> L.xPubBals v x)-          fast <--            if incremental m height now-              then incrementalDiff c x changed m bals-              else return Fallback-          diff <- case fast of-            Apply d -> return (Just d)-            KnowsMore -> return Nothing-            Fallback -> fullDiff c x changed m height bals now-          case diff of-            Nothing -> behind-            Just d -> commit tries extra m height now bals d-    commit tries extra m height now bals d = do-      bodies <- catMaybes <$> viewed c (\v -> L.getManyTxData v (nub (extra <> d.wanted))) >>= rewriteBodies c-      let args =-            [c.prefix, xpubId x, num height, num now, encode x, encode bals, m.gen]-              <> [if d.full then num now else ""]-              <> counted d.addrAdd-              <> counted d.addrDel-              <> counted2 [(t, num' sc) | (t, sc) <- d.txAdd]-              <> counted d.txDel-              <> counted3 [(op, u, num' sc) | (op, u, sc) <- d.utxoAdd]-              <> counted2 d.utxoDel-              <> bodyArgs bodies-      run c (R.eval commitScript [] args) >>= \case-        ["ok"] -> do-          void (cacheEvict c c.maxXPubs)-          count c (.reloads) 1-          when d.full $ count c (.fullReloads) 1-          atomicModifyIORef' c.counters $ \k ->-            ( k {reloads = k.reloads + 1, fullReloads = k.fullReloads + fromEnum d.full} :: CacheCounters,-              ()-            )-          return Reloaded-        ("missing" : missing)-          | tries > 0 -> go (tries - 1) (rights (map decode missing))-        ["changed"]-          | tries > 0 -> go (tries - 1) extra-        _ -> behind-    -- whether the entry may be brought up to date incrementally-    incremental m height now =-      m.ver == "3"-        && not (HashSet.null changed)-        && maybe False (\s -> toInteger height - s <= 1) m.stamp-        && maybe False (\f -> now - f < fullReloadAfter x) m.full-    behind = do-      count c (.stale) 1-      atomicModifyIORef' c.counters (\k -> (k {behind = k.behind + 1} :: CacheCounters, ()))-      return Behind-    timed f = do-      start <- nowMs-      r <- f-      end <- nowMs-      forM_ c.metrics $ \m -> addTiming m.reloadTime (fromInteger (end - start))-      return r---- | Longest time, in milliseconds, between full reloads of an entry: ten--- minutes; each xpub has its own interval up to this ('fullReloadAfter'). It--- bounds how long a change that the checks of the incremental path cannot--- see stays out of the cache. A full reload of the largest xpubs of the--- stress test takes a few seconds and holds the xpub in memory while it--- runs, so it is kept rare.-fullReloadEvery :: Integer-fullReloadEvery = 10 * 60 * 1000---- | Time, in milliseconds, after its last full reload when an entry is read--- whole again: between half of 'fullReloadEvery' and all of it, the same for--- an xpub on every store, and different from one xpub to the next.------ With one interval for every entry, entries cached together are reloaded--- whole together, every time. In the full stress test the four xpubs did so--- every ten minutes, and the heap of the server went from under 100 MiB to--- 785 MiB for a few seconds, holding all four at once. Each xpub gets its own--- interval instead, so such entries drift apart from their first scheduled--- full reload. It comes from the hash of the xpub, not from a random number,--- so that it needs no state and every store sharing Redis agrees on it.-fullReloadAfter :: XPubSpec -> Integer-fullReloadAfter x = half + spread `mod` half-  where-    half = fullReloadEvery `div` 2-    spread = BS.foldl' (\a w -> a * 256 + toInteger w) 0 (BS.take 4 (encode (sha256 (encode x))))---- | What to write to bring an entry up to date. Members of the sets are--- serialized: addresses, transaction hashes, and unspent outputs, which go--- with their outpoints.-data Diff = Diff-  { addrAdd :: ![ByteString],-    addrDel :: ![ByteString],-    -- | transactions to add or move, with their scores-    txAdd :: ![(ByteString, Double)],-    txDel :: ![ByteString],-    -- | unspent outputs to add, with their outpoints and scores-    utxoAdd :: ![(ByteString, ByteString, Double)],-    -- | outpoints of unspent outputs to remove, with the member if known; the-    -- script finds it by its outpoint otherwise-    utxoDel :: ![(ByteString, ByteString)],-    -- | transactions whose bodies to write-    wanted :: ![TxHash],-    -- | whether the whole xpub was read, which the entry records-    full :: !Bool-  }---- | The outcome of trying the incremental path.-data Incremental-  = -- | these are the differences-    Apply !Diff-  | -- | Redis has a changed transaction that this store has never seen-    KnowsMore-  | -- | the incremental path cannot be sure; read the whole xpub-    Fallback---- | The differences of an entry from the transactions that changed and the--- balances of the xpub, without reading its history; see 'cacheRefresh'.------ When the balances show that the database knows of changes beyond the ones--- given, the newest transactions of the addresses whose balances moved are--- added to them ('catchUp'), and the differences are worked out again; only--- if that does not account for every balance does it fall back.-incrementalDiff :: Cache -> XPubSpec -> HashSet TxHash -> Meta -> [XPubBal] -> IO Incremental-incrementalDiff c x changed m bals =-  case (decode m.bal :: Either String [XPubBal], m.stamp) of-    (Right old, Just stamp) ->-      incrementalFrom c x old bals changed >>= \case-        Missed missed ->-          catchUp c x stamp missed >>= \case-            Just extra-              | not (extra `HashSet.isSubsetOf` changed) ->-                  incrementalFrom c x old bals (changed <> extra) <&> \case-                    Missed _ -> Fallback-                    Settled r -> r-            _ -> return Fallback-        Settled r -> return r-    _ -> return Fallback---- | The outcome of working out the differences from a set of changes.-data Attempt-  = -- | the balances of these addresses moved, but no change touches them-    Missed ![Address]-  | Settled !Incremental---- | The differences of an entry from its old balances, its new ones, and the--- transactions that changed it.-incrementalFrom :: Cache -> XPubSpec -> [XPubBal] -> [XPubBal] -> HashSet TxHash -> IO Attempt-incrementalFrom c x old bals changed = do-  let hashes = HashSet.toList changed-  stored <- viewed c (\v -> L.getManyTxData v hashes)-  -- those that the database does not have because this store deleted-  -- them count as known, and as deleted-  gone <- readIORef c.lost-  dropped <- lostTxs c [h | (h, Nothing) <- zip hashes stored, HashSet.member h gone]-  let found = HashMap.fromList [(txHash td.tx, td) | td <- dropped]-      tds = [maybe (HashMap.lookup h found) Just td | (h, td) <- zip hashes stored]-  let newMap = HashMap.fromList [(b.balance.address, b.balance) | b <- bals]-      oldMap = HashMap.fromList [(b.balance.address, b.balance) | b <- old]-      entering = [b | (a, b) <- HashMap.toList newMap, not (HashMap.member a oldMap)]-      leaving = [b | (a, b) <- HashMap.toList oldMap, not (HashMap.member a newMap)]-      known = catMaybes tds-      unknown = [h | (h, Nothing) <- zip hashes tds]-      ours a = HashMap.member a newMap-      touched = HashSet.fromList [a | td <- known, a <- txAddresses c td, ours a]-      missed =-        [ a-        | (a, b) <- HashMap.toList newMap,-          Just o <- [HashMap.lookup a oldMap],-          o /= b,-          not (HashSet.member a touched)-        ]-  if-    | any (\b -> b.txs > 0) (entering <> leaving) -> return (Settled Fallback)-    | not (null missed) -> return (Missed missed)-    | otherwise -> do-        inRedis <- run c (sequence <$> mapM (R.zscore (xkey c x "txs") . encode) unknown)-        if any isJust inRedis-          then return (Settled KnowsMore)-          else do-            let active td = not td.deleted && any ours (txAddresses c td)-                ops = nub (concatMap (outpointsAt c ours) known)-            us <- viewed c (\v -> mapM (L.getUnspent v) ops)-            return . Settled . Apply $-              Diff-                { addrAdd = [encode b.address | b <- entering],-                  addrDel = [encode b.address | b <- leaving],-                  txAdd = [(encode (txHash td.tx), blockRefScore td.block) | td <- known, active td],-                  txDel = [encode (txHash td.tx) | td <- known, not (active td)],-                  utxoAdd = [(encode op, encode u, blockRefScore u.block) | (op, Just u) <- zip ops us],-                  utxoDel = [(encode op, "") | (op, Nothing) <- zip ops us],-                  wanted = [txHash td.tx | td <- known, active td],-                  full = False-                }---- | Changes of an entry that the database has and that it was not told of:--- what a store finds when its database runs ahead of the events it is--- applying, as it does under load, when the balances it reads after a batch--- already include the transactions of the next. 'Nothing' if there are more--- than 'catchUpMost', when reading the whole xpub is as cheap.------ The entry is stamped with a block that this store had whole, so what can--- have changed since is:------ * transactions in the mempool or in a later block, which are first in the---   index of an address, newest first, and are read from it for each address---   whose balance moved, down to the stamp;------ * transactions in the mempool when the entry was written that have been---   confirmed or dropped since, found from the mempool transactions of the---   entry, which are few.------ A reorganisation that replaces the block of the stamp is not covered, and--- the balances catch it: if the transactions found do not account for every--- balance, the full path runs.-catchUp :: Cache -> XPubSpec -> Integer -> [Address] -> IO (Maybe (HashSet TxHash))-catchUp c x stamp addrs = do-  newer <- sequence <$> mapM addressNewer addrs-  pool <- run c (R.zrangebyscoreLimit (xkey c x "txs") (-1 / 0) (-0.5) 0 (toInteger catchUpMost + 1))-  let poolHashes = rights (map decode pool)-  tds <- viewed c (\v -> L.getManyTxData v poolHashes)-  let settled = [h | (h, td) <- zip poolHashes tds, maybe True left td]-      found = HashSet.fromList (settled <> concat (fromMaybe [] newer))-  return $-    if isNothing newer || length pool > catchUpMost || HashSet.size found > catchUpMost-      then Nothing-      else Just found-  where-    -- a mempool transaction of the entry that is no longer in the mempool-    left td = td.deleted || not (isMem td.block)-    isMem MemRef {} = True-    isMem BlockRef {} = False-    after t = case t.block of-      MemRef {} -> True-      BlockRef {height = h} -> toInteger h > stamp-    chunk = 100 :: Word32-    addressNewer a = loop 0 []-      where-        loop off acc = do-          ts <- viewed c (\v -> L.getAddressTxs v a Limits {limit = chunk, offset = off, start = Nothing})-          let (new, rest) = span after ts-              acc' = acc <> map (.txid) new-          if-            | length acc' > catchUpMost -> return Nothing-            | not (null rest) || length ts < fromIntegral chunk -> return (Just acc')-            | otherwise -> loop (off + chunk) acc'---- | Most transactions that 'catchUp' adds to the changes of a reload. Each--- costs a few reads; a full reload of the xpubs of the stress test costs--- tens of thousands.-catchUpMost :: Int-catchUpMost = 1000---- | The differences of an entry from the whole xpub, read from the database--- and from Redis. 'Nothing' if Redis has a transaction of the xpub that this--- store has never seen.-fullDiff :: Cache -> XPubSpec -> HashSet TxHash -> Meta -> Word32 -> [XPubBal] -> Integer -> IO (Maybe Diff)-fullDiff c x changed m height bals _ = do-  st <- readStored c x-  txs <- viewed c (\v -> L.xPubTxs v bals def')-  let newTxs = [(encode t.txid, blockRefScore t.block) | t <- txs]-      newTxSet = HashSet.fromList (map fst newTxs)-      txDel = [t | t <- HashMap.keys st.txs, not (HashSet.member t newTxSet)]-  -- a transaction that Redis has and this store has never seen. The store-  -- keeps nothing of a transaction that it deleted, so one that it does-  -- not have is taken for deleted, and not for unseen, if the cache heard-  -- of its deletion, or if the entry has it as unconfirmed for more than-  -- ten minutes: a store that is in step with the others has seen a-  -- transaction of the mempool by then, and one that is not does not get-  -- here.-  let asked = [(h, sc) | t <- txDel, Right h <- [decode t], Just sc <- [HashMap.lookup t st.txs]]-  known <- viewed c (\v -> L.getManyTxData v (map fst asked))-  gone <- readIORef c.lost-  now <- (`div` 1000) <$> nowMs-  let stale sc = sc < 0 && fromIntegral now - negate sc > 600-      unseen = [h | ((h, sc), Nothing) <- zip asked known, not (HashSet.member h gone), not (stale sc)]-  if not (mayReplace m.stamp height (not (null unseen)))-    then return Nothing-    else do-      utxos <- viewed c (\v -> L.xPubUnspents v bals def')-      let txAdd = [(t, sc) | (t, sc) <- newTxs, HashMap.lookup t st.txs /= Just sc]-          newUs = [(encode u.unspent.outpoint, encode u.unspent, blockRefScore u.unspent.block) | u <- utxos]-          newUSet = HashSet.fromList [u | (_, u, _) <- newUs]-          -- an entry of an older format has no index of outpoints yet, so-          -- every output is written, to build it-          utxoAdd-            | m.ver == "3" = [n | n@(_, u, _) <- newUs, not (HashSet.member u st.utxos)]-            | otherwise = newUs-          utxoDel =-            [ (either (const "") (encode . (.outpoint)) (decode u :: Either String Unspent), u)-            | u <- HashSet.toList st.utxos,-              not (HashSet.member u newUSet)-            ]-          newAs = HashSet.fromList [encode b.balance.address | b <- bals]-      return . Just $-        Diff-          { addrAdd = [a | a <- HashSet.toList newAs, not (HashSet.member a st.addrs)],-            addrDel = [a | a <- HashSet.toList st.addrs, not (HashSet.member a newAs)],-            txAdd,-            txDel,-            utxoAdd,-            utxoDel,-            -- bodies of the transactions that are new or moved, and of those-            -- that changed; the others are in Redis, kept there by this entry-            wanted =-              nub $-                [h | (t, _) <- txAdd, Right h <- [decode t]]-                  <> [t.txid | t <- txs, t.txid `HashSet.member` changed],-            full = True-          }-  where-    def' = Limits {limit = 0, offset = 0, start = Nothing}---- | Addresses that a transaction spends from or pays to.-txAddresses :: Cache -> TxData -> [Address]-txAddresses c td =-  nub . rights $-    map (scriptToAddressBS c.db.net c.db.ctx . (.script)) (IntMap.elems td.prevs)-      <> map (scriptToAddressBS c.db.net c.db.ctx . (.script)) td.tx.outputs---- | Outpoints whose unspent output a transaction may change at addresses that--- pass a test: those it creates, and those it spends.-outpointsAt :: Cache -> (Address -> Bool) -> TxData -> [OutPoint]-outpointsAt c ours td =-  [OutPoint (txHash td.tx) i | (i, o) <- zip [0 ..] td.tx.outputs, at o.script]-    <> [ inp.outpoint-       | (i, inp) <- zip [0 :: Int ..] td.tx.inputs,-         Just prev <- [IntMap.lookup i td.prevs],-         at prev.script-       ]-  where-    at s = either (const False) ours (scriptToAddressBS c.db.net c.db.ctx s)---- | What a reload needs to know of an entry first: the height it is stamped--- with, its generation, which every commit increments, the version of its--- format, the time of its last full reload, and its balances, serialized.-data Meta = Meta-  { stamp :: !(Maybe Integer),-    gen :: !ByteString,-    ver :: !ByteString,-    full :: !(Maybe Integer),-    bal :: !ByteString-  }---- | Read what a reload needs to know of an entry, in one round trip.-readMeta :: Cache -> XPubSpec -> IO Meta-readMeta c x = do-  (fs, b) <- run c $ do-    fs <- R.hmget (xkey c x "meta") ("height" NonEmpty.:| ["gen", "ver", "full"])-    b <- R.get (xkey c x "bal")-    return ((,) <$> fs <*> b)-  let field k = join (lookup k (zip [0 :: Int ..] fs))-      number = (readMaybe . C8.unpack =<<)-  return-    Meta-      { stamp = number (field 0),-        gen = fromMaybe "" (field 1),-        ver = fromMaybe "" (field 2),-        full = number (field 3),-        bal = fromMaybe "" b-      }---- | The sets of an entry, as a full reload compares them with the database.-data Stored = Stored-  { txs :: !(HashMap ByteString Double),-    utxos :: !(HashSet ByteString),-    addrs :: !(HashSet ByteString)-  }---- | Read the sets of an entry. The sorted sets are read in chunks, so that--- no command holds Redis for long however large the xpub. The chunks are not--- read atomically, but nothing changes the entry while this store holds its--- lock except deleting or evicting it, which changes its generation, and the--- commit checks the generation.-readStored :: Cache -> XPubSpec -> IO Stored-readStored c x = do-  as <- run c (R.smembers (xkey c x "addrs"))-  ts <- chunks (\i j -> R.zrangeWithscores (xkey c x "txs") i j)-  us <- chunks (\i j -> R.zrange (xkey c x "utxo") i j)-  return-    Stored-      { txs = HashMap.fromList ts,-        utxos = HashSet.fromList us,-        addrs = HashSet.fromList as-      }-  where-    size = 5000 :: Integer-    chunks :: (Integer -> Integer -> Redis (Either Reply [a])) -> IO [a]-    chunks f = loop 0-      where-        loop i = do-          xs <- run c (f i (i + size - 1))-          if toInteger (length xs) < size then return xs else (xs <>) <$> loop (i + size)---- | Rewrite the bodies that Redis already has, in scripts of 'bodyChunk'--- bodies each, and give back the others, for the commit to write.------ A block confirms many transactions of a busy xpub at once, and each needs--- its body written again, with its block and spenders. In one script, with--- the scores and outputs of the entry, that held Redis for 10 to 30 ms at--- the confirmation of each block in the stress test, while every other--- client waited. Bodies are shared by every entry that has the transaction,--- and are written whole from the database, so writing them before the entry--- is safe: a body is never older than the entry that refers to it, which is--- what matters (see 'cacheRefresh'), and other entries already see bodies--- written by commits of other xpubs before their own. Only bodies that are--- there are rewritten, so that a commit that then fails leaves nothing--- behind; and if one goes between these scripts and the commit, the commit--- reports it missing and is retried with it.-rewriteBodies :: Cache -> [TxData] -> IO [TxData]-rewriteBodies c tds =-  fmap concat . forM (chunks tds) $ \ch -> do-    absent <- HashSet.fromList <$> run c (R.eval bodiesScript [] (c.prefix : bodyArgs ch))-    return [td | td <- ch, encode (txHash td.tx) `HashSet.member` absent]-  where-    chunks [] = []-    chunks xs = let (a, b) = splitAt bodyChunk xs in a : chunks b---- | Bodies rewritten by one script of 'rewriteBodies': each is a few--- commands, so a chunk holds Redis for a millisecond or two.-bodyChunk :: Int-bodyChunk = 200---- | Arguments for bodies and their spenders: the number of bodies, the hash--- and body of each, the number of spenders, and the transaction, output index--- and spender of each.-bodyArgs :: [TxData] -> [ByteString]-bodyArgs tds =-  counted2 [(encode (txHash td.tx), encode (td :: TxData) {spenders = IntMap.empty}) | td <- tds]-    <> counted3-      [ (encode (txHash td.tx), encode (fromIntegral i :: Word32), encode s)-      | td <- tds,-        (i, s) <- IntMap.toList td.spenders-      ]--counted :: [ByteString] -> [ByteString]-counted xs = num (length xs) : xs--counted2 :: [(ByteString, ByteString)] -> [ByteString]-counted2 xs = num (length xs) : concat [[a, b] | (a, b) <- xs]--counted3 :: [(ByteString, ByteString, ByteString)] -> [ByteString]-counted3 xs = num (length xs) : concat [[a, b, d] | (a, b, d) <- xs]--num :: (Integral a) => a -> ByteString-num = C8.pack . show . toInteger--num' :: Double -> ByteString-num' = C8.pack . show--nowMs :: IO Integer-nowMs = round . (* 1000) <$> getPOSIXTime---- | Height of the best block of the database, 0 if none.-bestHeight :: Cache -> IO Word32-bestHeight c =-  Q.getBestBlock c.db >>= \case-    Nothing -> return 0-    Just h -> maybe 0 (.height) <$> Q.getBlock c.db h---- | Whether the database has the best header of the node.-inSync :: Cache -> IO Bool-inSync c =-  Q.getBestBlock c.db >>= \case-    Nothing -> return False-    Just bb -> do-      cb <- chainGetBest c.chain-      return (cb.height > 0 && headerHash cb.header == bb)---- | Run an action holding the lock of an xpub. 'Nothing' if another store--- holds it. The lock expires after two minutes in case its holder dies.-withLock :: Cache -> XPubSpec -> IO a -> IO (Maybe a)-withLock c x f = do-  token <- C8.pack . show <$> (randomIO :: IO Word64)-  let key = c.prefix <> "lock:" <> xpubId x-      opts =-        R.SetOpts-          { R.setSeconds = Nothing,-            R.setMilliseconds = Just 120000,-            R.setUnixSeconds = Nothing,-            R.setUnixMilliseconds = Nothing,-            R.setCondition = Just R.Nx,-            R.setKeepTTL = False-          }-  R.runRedis c.redis (R.setOpts key token opts) >>= \case-    Right R.Ok -> do-      r <- f-      void (run c (R.eval unlockScript [key] [token] :: Redis (Either Reply Integer)))-      return (Just r)-    _ -> return Nothing------------------------------------------------------------------------------------ Keeping up with the store------------------------------------------------------------------------------------ | Do the work of the cache, forever.------ A store that is not in sync ignores events and requests: the stores that--- are ahead have already done or will do the work, and this one would only--- find that Redis knows more. Once in sync, each event is claimed per cached--- xpub that it changes ('claim'): the first store to claim a change reloads--- the xpub and marks the claim done, and the others skip it, so a change is--- applied once however many stores share the Redis, and stores that catch up--- find their events done. A change that a store could not apply because--- Redis knew more is released for another store to take; one that another--- store is busy with is looked at again later.-worker :: Cache -> IO ()-worker c = do-  lastBest <- newIORef Nothing-  loopForever $ do-    js <- atomically $ do-      xs <- readTVar c.jobs-      checkSTM (not (null xs))-      writeTVar c.jobs []-      return (reverse xs)-    synced <- inSync c-    if synced-      then void (attempt c.outage (log' c) "update the cache" (work c lastBest js))-      else releaseClaims c (concat [ks | JobHeld _ _ ks <- js])---- | Do a batch of jobs.-work :: Cache -> IORef (Maybe BlockHash) -> [Job] -> IO ()-work c lastBest js = do-  fromEvents <- concat <$> mapM (eventChanges c lastBest) [e | JobEvent e <- js]-  let toClaim = fromEvents <> [(k, x, ts) | JobClaim x ts ks <- js, k <- ks]-  claimed <- fmap catMaybes . forM toClaim $ \(k, x, ts) ->-    afterClaim <$> claim c k >>= \case-      ReloadNow -> return (Just (x, (ts, [k])))-      Skip -> return Nothing-      ClaimLater -> later c (JobClaim x ts [k]) >> return Nothing-  let held =-        HashMap.fromListWith-          (\(a, b) (a', b') -> (a <> a', b <> b'))-          (claimed <> [(x, (ts, ks)) | JobHeld x ts ks <- js])-  pooledForConcurrentlyN_ parallelReloads (HashMap.toList held) $ \(x, (ts, ks)) ->-    afterReload <$> reload x ts >>= \case-      Finish -> finishClaims c ks-      Release -> releaseClaims c ks-      Hold -> extendClaims c ks >> later c (JobHeld x ts ks)-  pooledForConcurrentlyN_ parallelReloads (nub [x | JobCache x <- js, not (HashMap.member x held)]) $ \x ->-    cacheIsCached c x >>= \case-      True -> return ()-      False ->-        reload x HashSet.empty >>= \case-          Locked -> later c (JobCache x)-          _ -> return ()-  where-    reload x ts =-      fromMaybe Locked <$> attempting c.outage (log' c) "cache an xpub" (cacheRefresh c x ts)---- | How many xpubs a store reloads at once. Each holds the lock of its xpub,--- so two stores never reload the same one; reloading several at once keeps--- one large xpub from holding up the others.-parallelReloads :: Int-parallelReloads = 4---- | Changes that an event makes to cached xpubs: for each xpub, the key of--- the claim for the change, and the transactions that make it. Also keeps the--- spenders of cached transactions up to date, which is harmless to repeat.-eventChanges :: Cache -> IORef (Maybe BlockHash) -> StoreEvent -> IO [(ByteString, XPubSpec, HashSet TxHash)]-eventChanges c lastBest e = do-  (name, tds) <- case e of-    StoreBestBlock h -> do-      prev <- readIORef lastBest-      writeIORef lastBest (Just h)-      ("b:" <> encode h,) <$> blockTxs c prev h-    StoreMempoolNew t -> ("t+:" <> encode t,) . maybeToList <$> viewed c (\v -> L.getTxData v t)-    -- the store keeps nothing of a transaction that it deleted. Redis has-    -- the body of every transaction of a cached xpub, so if it does not-    -- have this one, no entry does either-    StoreMempoolDelete t -> do-      noteLost c [t]-      ("t-:" <> encode t,) <$> lostTxs c [t]-    _ -> return ("", [])-  if null tds-    then return []-    else do-      updateSpenders c tds-      affected <- affectedXPubs c tds-      return [(c.prefix <> "ev:" <> name <> ":" <> xpubId x, x, ts) | (x, ts) <- affected]---- | Claim a change. A claim is held for two minutes, so that a store that--- dies while holding it does not keep others from the change for long.-claim :: Cache -> ByteString -> IO Claim-claim c k = do-  let opts =-        R.SetOpts-          { R.setSeconds = Nothing,-            R.setMilliseconds = Just 120000,-            R.setUnixSeconds = Nothing,-            R.setUnixMilliseconds = Nothing,-            R.setCondition = Just R.Nx,-            R.setKeepTTL = False-          }-  R.runRedis c.redis (R.setOpts k "busy" opts) >>= \case-    Right R.Ok -> return Claimed-    _ ->-      run c (R.get k) <&> \case-        Just "done" -> Done-        _ -> Busy---- | Mark changes as applied, for an hour: long enough for every store in--- sync to see the same events.-finishClaims :: Cache -> [ByteString] -> IO ()-finishClaims _ [] = return ()-finishClaims c ks =-  void . run c $ sequence <$> mapM (\k -> R.setex k 3600 "done") ks---- | Give up changes, for another store to apply.-releaseClaims :: Cache -> [ByteString] -> IO ()-releaseClaims _ [] = return ()-releaseClaims c ks = void (run c (R.del (NonEmpty.fromList ks)))---- | Hold changes for another two minutes.-extendClaims :: Cache -> [ByteString] -> IO ()-extendClaims _ [] = return ()-extendClaims c ks = void . run c $ sequence <$> mapM (`R.pexpire` 120000) ks---- | Pass the events of the store that may change cached xpubs to the worker.-listener :: Cache -> Inbox StoreEvent -> IO ()-listener c events =-  loopForever $-    receive events >>= \case-      e@StoreBestBlock {} -> push c (JobEvent e)-      e@StoreMempoolNew {} -> push c (JobEvent e)-      e@StoreMempoolDelete {} -> push c (JobEvent e)-      _ -> return ()---- | Transactions of a new best block, and of the blocks that a--- reorganisation took out of the chain since the previous best block.-blockTxs :: Cache -> Maybe BlockHash -> BlockHash -> IO [TxData]-blockTxs c prev h =-  Q.getBlock c.db h >>= \case-    Nothing -> return []-    Just bd -> do-      reverted <- case prev of-        Just p | p /= bd.header.prev && p /= h -> walk p-        _ -> return []-      let hashes = bd.txs <> reverted-      found <- viewed c (\v -> L.getManyTxData v hashes)-      -- a transaction of a reverted block that did not go back to the-      -- mempool, as one that spends its coinbase does not, is gone from-      -- the database. Redis has its body if a cached xpub has it.-      let missing = [h' | (h', Nothing) <- zip hashes found]-      noteLost c missing-      (catMaybes found <>) <$> lostTxs c missing-  where-    -- blocks that are no longer in the main chain, from the old best block-    walk p =-      Q.getBlock c.db p >>= \case-        Just b | not b.main -> (b.txs <>) <$> walk b.header.prev-        _ -> return []---- | Note that the store deleted transactions.-noteLost :: Cache -> [TxHash] -> IO ()-noteLost _ [] = return ()-noteLost c hs = atomicModifyIORef' c.lost $ \seen ->-  let more = foldr HashSet.insert seen hs-   in (if HashSet.size more > 100000 then HashSet.fromList hs else more, ())---- | The bodies that Redis has of transactions that the store deleted,--- marked as deleted. Redis has the body of every transaction of a cached--- xpub, so one that it does not have is in no entry.-lostTxs :: Cache -> [TxHash] -> IO [TxData]-lostTxs c hs = map (\td -> (td :: TxData) {deleted = True}) . catMaybes <$> cacheTxData c hs---- | Cached xpubs with an address that these transactions spend from or pay--- to, with the transactions that touch each.-affectedXPubs :: Cache -> [TxData] -> IO [(XPubSpec, HashSet TxHash)]-affectedXPubs c tds = do-  let net = c.db.net-      ctx = c.db.ctx-      addrsOf :: TxData -> [Address]-      addrsOf td =-        nub . rights $-          map (scriptToAddressBS net ctx . (.script)) (IntMap.elems td.prevs)-            <> map (scriptToAddressBS net ctx . (.script)) td.tx.outputs-      byAddr =-        HashMap.fromListWith (<>) [(a, HashSet.singleton (txHash td.tx)) | td <- tds, a <- addrsOf td]-      addrs = HashMap.keys byAddr-  ids <- run c (sequence <$> mapM (R.smembers . addrKey c) addrs)-  let byId =-        HashMap.fromListWith-          (<>)-          [(i, ts) | (a, is) <- zip addrs ids, i <- is, Just ts <- [HashMap.lookup a byAddr]]-      spec i = R.hget (c.prefix <> "x:" <> i <> ":meta") "spec"-  specs <- run c (sequence <$> mapM spec (HashMap.keys byId))-  return-    [ (x, ts)-    | (Just bs, ts) <- zip specs (HashMap.elems byId),-      Right x <- [decode bs]-    ]---- | Record who spends the outputs of cached transactions: these--- transactions spend them, or no longer do if they were deleted.-updateSpenders :: Cache -> [TxData] -> IO ()-updateSpenders c tds =-  unless (null quads) $-    void (run c (R.eval spendsScript [] args :: Redis (Either Reply Integer)))-  where-    args = c.prefix : num (length quads) : concat quads-    quads =-      [ [ encode i.outpoint.hash,-          encode i.outpoint.index,-          if td.deleted then "" else encode (Spender (txHash td.tx) n),-          encode (txHash td.tx)-        ]-      | td <- tds,-        (n, i) <- zip [0 ..] td.tx.inputs-      ]------------------------------------------------------------------------------------ Lua scripts------------------------------------------------------------------------------------ The scripts that Redis runs are in files of their own under--- @src\/Haskoin\/Store\/Cache\/lua@, where they can be read, linted and run--- with @redis-cli --eval@ on their own, and are embedded here when the module--- is compiled. Embedding them, rather than reading them when the server--- starts, keeps the library free of data files that a program using it would--- have to install and find. Their comments are sent with every call, so keep--- them short; the full description of each is the Haddock of its binding.---- | Delete an xpub. Arguments: prefix, id. Returns 1 if it was cached.-deleteScript :: ByteString-deleteScript = $(makeRelativeToProject "src/Haskoin/Store/Cache/lua/delete.lua" >>= embedFile)---- | Evict the least recently used xpubs past a number. Arguments: prefix,--- maximum. Returns how many were evicted.-evictScript :: ByteString-evictScript = $(makeRelativeToProject "src/Haskoin/Store/Cache/lua/evict.lua" >>= embedFile)---- | A page of a sorted set of a cached xpub; see 'page'. Arguments: the--- keys of the meta hash, of the sorted set and of the sorted set of--- transactions, the key of the set of recent use, the time to mark the xpub--- used with (empty for not now), the id of the xpub, the offset and the limit--- (0 for no limit), and how the page starts: @top@; @score@ and a score;--- @at@, a score, a number of bytes and the bytes of the start; or @tx@, a--- transaction hash, a number of bytes and the bytes of the start. Returns--- @{\"none\"}@ if the xpub is not cached, @{\"nostart\"}@ if the--- transaction of @tx@ is not one of the xpub, or @{\"ok\"}@ followed by--- each member and its score.------ Bytes are compared one by one: Lua compares strings by the collation of--- the locale of the server, which need not be the order of their bytes that--- Redis and the database sort members by.-pageScript :: ByteString-pageScript = $(makeRelativeToProject "src/Haskoin/Store/Cache/lua/page.lua" >>= embedFile)---- | Apply the differences of an entry of an xpub, unless the stored one is--- stamped with a higher block, or is not the generation they were computed--- from. Arguments: prefix, id, height, time of use, serialized xpub,--- balances, generation (empty for none); then, each preceded by its number,--- the addresses to add, the addresses to remove, the transactions to add or--- move with their scores, the transactions to remove, the unspent outputs to--- add with their scores, the unspent outputs to remove, the bodies to write--- with their hashes, and the spenders with their transaction and output--- index. Returns @{\"stale\"}@ if the stored entry has a higher block,--- @{\"changed\"}@ if its generation is not the one given,--- @{\"missing\", ...}@ with the hashes of added transactions that have no--- body in Redis nor in the arguments, or @{\"ok\"}@. Nothing is written--- unless it returns @{\"ok\"}@.------ A transaction removed from the entry loses this xpub from its set of--- xpubs, and its body goes when no cached xpub has it any more.-commitScript :: ByteString-commitScript = $(makeRelativeToProject "src/Haskoin/Store/Cache/lua/commit.lua" >>= embedFile)---- | Rewrite bodies that Redis has, with their spenders; see 'rewriteBodies'.--- Arguments: prefix, then as at the end of 'commitScript': the number of--- bodies, the hash and body of each, the number of spenders, and the--- transaction, output index and spender of each. Returns the hashes of the--- bodies that Redis does not have, which it does not write.-bodiesScript :: ByteString-bodiesScript = $(makeRelativeToProject "src/Haskoin/Store/Cache/lua/bodies.lua" >>= embedFile)---- | Record or forget spenders of outputs of cached transactions. Arguments:--- prefix, number of entries, then for each: hash of the transaction that--- created the output, output index, spender (empty to forget) and hash of--- the spending transaction. Only transactions whose body is cached are--- touched, and a spender is only forgotten if it is still the one given.-spendsScript :: ByteString-spendsScript = $(makeRelativeToProject "src/Haskoin/Store/Cache/lua/spends.lua" >>= embedFile)---- | Release a lock if it still holds the given token. Keys: lock. Arguments:--- token.-unlockScript :: ByteString-unlockScript = $(makeRelativeToProject "src/Haskoin/Store/Cache/lua/unlock.lua" >>= embedFile)
− src/Haskoin/Store/Cache/Protocol.hs
@@ -1,122 +0,0 @@--- |--- Module      : Haskoin.Store.Cache.Protocol--- Copyright   : (c) Haskoin Developers--- License     : MIT--- Maintainer  : jprupp@protonmail.ch--- Stability   : experimental--- Portability : POSIX------ The rules that stores sharing one Redis follow to apply each change of a--- cached xpub once, as pure decisions. "Haskoin.Store.Cache" carries them--- out against Redis.------ Each change (a block, or a transaction entering or leaving the mempool,--- for one xpub) has a claim in Redis that goes through these states:------ @---            claim            reload succeeds---   absent ─────────► busy ─────────────────► done (for an hour)---     ▲                 │---     └─────────────────┘ Redis knew more: release--- @------ A store that finds the claim busy tries again later; one that finds it--- done skips the change. While it holds the claim, it reloads the xpub from--- its database, unless the entry in Redis knows more than the database--- ('mayReplace'), in which case it releases the claim and writes nothing.------ = Why these rules------ * __A change is applied once.__ The first store to see a change claims it;---   stores that see it later, because they are behind, find it done. A set of---   stores catching up does no work for changes already applied.--- * __Stores never undo each other.__ An entry is stamped with the best block---   of the store that wrote it, and no store replaces an entry stamped with a---   higher block, or one with a transaction it has never seen. Without this---   rule, a store that is behind would replace a newer entry with an older---   one, and the next store would put the newer one back.--- * __An entry is complete.__ The entry is written with every transaction---   body that it needs, in one script, so a store that is behind can serve---   it without its database.-module Haskoin.Store.Cache.Protocol-  ( -- * Claims-    Claim (..),-    OnClaim (..),-    afterClaim,--    -- * Reloads-    Reload (..),-    OnReload (..),-    afterReload,-    mayReplace,-  )-where--import Haskoin (BlockHeight)---- | The state of the claim of a change, as a store that tries to claim it--- finds it.-data Claim-  = -- | it was absent: this store holds it now-    Claimed-  | -- | another store applied the change-    Done-  | -- | another store holds the claim-    Busy-  deriving stock (Eq, Show, Enum, Bounded)---- | What to do after trying to claim a change.-data OnClaim-  = -- | reload the xpub, holding the claim-    ReloadNow-  | -- | nothing: the change was applied-    Skip-  | -- | try to claim it again later, in case its holder gives it up-    ClaimLater-  deriving stock (Eq, Show)---- | What to do after trying to claim a change.-afterClaim :: Claim -> OnClaim-afterClaim = \case-  Claimed -> ReloadNow-  Done -> Skip-  Busy -> ClaimLater---- | What became of a reload.-data Reload-  = -- | the entry was replaced-    Reloaded-  | -- | Redis knows more about the xpub than this store: its entry is stamped-    -- with a higher block, or has a transaction this store has never seen. It-    -- is left for a store that knows more.-    Behind-  | -- | another store holds the lock of the xpub-    Locked-  deriving stock (Eq, Show, Enum, Bounded)---- | What to do with the claims of the changes that a reload was for.-data OnReload-  = -- | mark them done-    Finish-  | -- | give them up, for a store that knows more-    Release-  | -- | keep holding them, and reload again later-    Hold-  deriving stock (Eq, Show)---- | What to do with the claims of the changes that a reload was for.-afterReload :: Reload -> OnReload-afterReload = \case-  Reloaded -> Finish-  Behind -> Release-  Locked -> Hold---- | Whether a store may replace the entry of an xpub: given the block height--- that the entry is stamped with, if there is an entry, the height of the--- best block of the store, and whether the entry has a transaction that the--- store has never seen. An entry of the same height may be replaced, as it--- can differ only in the mempool. The commit script checks the height again,--- atomically, as another store may write between the check and the commit.-mayReplace :: Maybe Integer -> BlockHeight -> Bool -> Bool-mayReplace stamp height unknownTxs =-  maybe True (<= toInteger height) stamp && not unknownTxs
− src/Haskoin/Store/Cache/lua/bodies.lua
@@ -1,25 +0,0 @@--- Rewrite bodies that Redis has, ahead of a commit: bodiesScript in Haskoin.Store.Cache.-local p = ARGV[1]-local nb = tonumber(ARGV[2])-local bi = 3-local si = bi + 2 * nb + 1-local ns = tonumber(ARGV[si - 1])-local written = {}-local absent = {}-for j = 0, nb - 1 do-  local t = ARGV[bi + 2 * j]-  if redis.call('EXISTS', p .. 'tx:' .. t) == 1 then-    redis.call('SET', p .. 'tx:' .. t, ARGV[bi + 2 * j + 1])-    redis.call('DEL', p .. 'sp:' .. t)-    written[t] = true-  else-    table.insert(absent, t)-  end-end-for j = 0, ns - 1 do-  local k = si + 3 * j-  if written[ARGV[k]] then-    redis.call('HSET', p .. 'sp:' .. ARGV[k], ARGV[k + 1], ARGV[k + 2])-  end-end-return absent
− src/Haskoin/Store/Cache/lua/commit.lua
@@ -1,92 +0,0 @@--- Apply the differences of an entry of an xpub: commitScript in Haskoin.Store.Cache.-local p = ARGV[1]-local id = ARGV[2]-local h = tonumber(ARGV[3])-local xk = p .. 'x:' .. id .. ':'-local meta = xk .. 'meta'-local oh = redis.call('HGET', meta, 'height')-if oh and h < tonumber(oh) then return {'stale'} end-if (redis.call('HGET', meta, 'gen') or '') ~= ARGV[7] then return {'changed'} end-local full = ARGV[8]-local i = 9-local function list(width)-  local n = tonumber(ARGV[i])-  local s = i + 1-  i = s + width * n-  return s, n-end-local aa, naa = list(1)-local ad, nad = list(1)-local ta, nta = list(2)-local td, ntd = list(1)-local ua, nua = list(3)-local ud, nud = list(2)-local bi, nb = list(2)-local si, ns = list(3)-local given = {}-for j = 0, nb - 1 do given[ARGV[bi + 2 * j]] = true end-local missing = {'missing'}-for j = 0, nta - 1 do-  local t = ARGV[ta + 2 * j]-  if not given[t] and redis.call('EXISTS', p .. 'tx:' .. t) == 0 then-    table.insert(missing, t)-  end-end-if #missing > 1 then return missing end-for j = 0, nad - 1 do-  local a = ARGV[ad + j]-  redis.call('SREM', p .. 'a:' .. a, id)-  redis.call('SREM', xk .. 'addrs', a)-end-for j = 0, naa - 1 do-  local a = ARGV[aa + j]-  redis.call('SADD', p .. 'a:' .. a, id)-  redis.call('SADD', xk .. 'addrs', a)-end-for j = 0, ntd - 1 do-  local t = ARGV[td + j]-  local rk = p .. 'txref:' .. t-  redis.call('ZREM', xk .. 'txs', t)-  redis.call('SREM', rk, id)-  if redis.call('SCARD', rk) == 0 then-    redis.call('DEL', rk, p .. 'tx:' .. t, p .. 'sp:' .. t)-  end-end-for j = 0, nta - 1 do-  local t = ARGV[ta + 2 * j]-  redis.call('ZADD', xk .. 'txs', ARGV[ta + 2 * j + 1], t)-  redis.call('SADD', p .. 'txref:' .. t, id)-end--- unspent outputs by outpoint: uop maps each to its member, which changes--- with the block of the output, so that one can go by its outpoint alone-local uop = xk .. 'uop'-for j = 0, nud - 1 do-  local op, m = ARGV[ud + 2 * j], ARGV[ud + 2 * j + 1]-  if m == '' then m = redis.call('HGET', uop, op) end-  if m then redis.call('ZREM', xk .. 'utxo', m) end-  redis.call('HDEL', uop, op)-end-for j = 0, nua - 1 do-  local op, m = ARGV[ua + 3 * j], ARGV[ua + 3 * j + 1]-  local old = redis.call('HGET', uop, op)-  if old and old ~= m then redis.call('ZREM', xk .. 'utxo', old) end-  redis.call('ZADD', xk .. 'utxo', ARGV[ua + 3 * j + 2], m)-  redis.call('HSET', uop, op, m)-end-for j = 0, nb - 1 do-  local t = ARGV[bi + 2 * j]-  redis.call('SET', p .. 'tx:' .. t, ARGV[bi + 2 * j + 1])-  redis.call('DEL', p .. 'sp:' .. t)-end-for j = 0, ns - 1 do-  local k = si + 3 * j-  if given[ARGV[k]] then-    redis.call('HSET', p .. 'sp:' .. ARGV[k], ARGV[k + 1], ARGV[k + 2])-  end-end-redis.call('SET', xk .. 'bal', ARGV[6])-redis.call('HSET', meta, 'spec', ARGV[5], 'height', ARGV[3], 'ver', '3')-if full ~= '' then redis.call('HSET', meta, 'full', full) end-redis.call('HINCRBY', meta, 'gen', 1)-redis.call('ZADD', p .. 'lru', ARGV[4], id)-return {'ok'}
− src/Haskoin/Store/Cache/lua/delete.lua
@@ -1,18 +0,0 @@--- Delete an xpub: deleteScript in Haskoin.Store.Cache. evict.lua removes--- xpubs the same way.-local p = ARGV[1]-local id = ARGV[2]-local xk = p .. 'x:' .. id .. ':'-for _, a in ipairs(redis.call('SMEMBERS', xk .. 'addrs')) do-  redis.call('SREM', p .. 'a:' .. a, id)-end-for _, t in ipairs(redis.call('ZRANGE', xk .. 'txs', 0, -1)) do-  local rk = p .. 'txref:' .. t-  redis.call('SREM', rk, id)-  if redis.call('SCARD', rk) == 0 then-    redis.call('DEL', rk, p .. 'tx:' .. t, p .. 'sp:' .. t)-  end-end-local existed = redis.call('DEL', xk .. 'meta', xk .. 'bal', xk .. 'txs', xk .. 'utxo', xk .. 'addrs', xk .. 'uop')-redis.call('ZREM', p .. 'lru', id)-if existed > 0 then return 1 else return 0 end
− src/Haskoin/Store/Cache/lua/evict.lua
@@ -1,23 +0,0 @@--- Evict the least recently used xpubs past a number: evictScript in--- Haskoin.Store.Cache. Each xpub goes as delete.lua removes one.-local p = ARGV[1]-local max = tonumber(ARGV[2])-local n = 0-while redis.call('ZCARD', p .. 'lru') > max do-  local id = redis.call('ZRANGE', p .. 'lru', 0, 0)[1]-  local xk = p .. 'x:' .. id .. ':'-  for _, a in ipairs(redis.call('SMEMBERS', xk .. 'addrs')) do-    redis.call('SREM', p .. 'a:' .. a, id)-  end-  for _, t in ipairs(redis.call('ZRANGE', xk .. 'txs', 0, -1)) do-    local rk = p .. 'txref:' .. t-    redis.call('SREM', rk, id)-    if redis.call('SCARD', rk) == 0 then-      redis.call('DEL', rk, p .. 'tx:' .. t, p .. 'sp:' .. t)-    end-  end-  redis.call('DEL', xk .. 'meta', xk .. 'bal', xk .. 'txs', xk .. 'utxo', xk .. 'addrs', xk .. 'uop')-  redis.call('ZREM', p .. 'lru', id)-  n = n + 1-end-return n
− src/Haskoin/Store/Cache/lua/page.lua
@@ -1,39 +0,0 @@--- A page of a sorted set of a cached xpub: pageScript in Haskoin.Store.Cache.-local set, txs = ARGV[2], ARGV[3]-if redis.call('EXISTS', ARGV[1]) == 0 then return {'none'} end-if ARGV[5] ~= '' then redis.call('ZADD', ARGV[4], ARGV[5], ARGV[6]) end-local off = tonumber(ARGV[7])-local lim = tonumber(ARGV[8])-local mode = ARGV[9]-local function lt(a, b)-  for k = 1, math.min(#a, #b) do-    local x, y = string.byte(a, k), string.byte(b, k)-    if x ~= y then return x < y end-  end-  return #a < #b-end-local function out(r)-  local o = {'ok'}-  for _, v in ipairs(r) do table.insert(o, v) end-  return o-end-if mode == 'top' then-  local stop = -1-  if lim > 0 then stop = off + lim - 1 end-  return out(redis.call('ZRANGE', set, off, stop, 'WITHSCORES'))-end-local score = ARGV[10]-local skip = 0-if mode == 'tx' then-  score = redis.call('ZSCORE', txs, ARGV[10])-  if not score then return {'nostart'} end-end-if mode ~= 'score' then-  local cut, bytes = tonumber(ARGV[11]), ARGV[12]-  for _, m in ipairs(redis.call('ZRANGEBYSCORE', set, score, score)) do-    if lt(string.sub(m, cut + 1), bytes) then skip = skip + 1 end-  end-end-local count = -1-if lim > 0 then count = lim end-return out(redis.call('ZRANGEBYSCORE', set, score, '+inf', 'WITHSCORES', 'LIMIT', off + skip, count))
− src/Haskoin/Store/Cache/lua/spends.lua
@@ -1,20 +0,0 @@--- Record or forget spenders of outputs of cached transactions: spendsScript in Haskoin.Store.Cache.-local p = ARGV[1]-local n = tonumber(ARGV[2])-local i = 3-for j = 1, n do-  local t = ARGV[i]-  if redis.call('EXISTS', p .. 'tx:' .. t) == 1 then-    local sk = p .. 'sp:' .. t-    if ARGV[i + 2] ~= '' then-      redis.call('HSET', sk, ARGV[i + 1], ARGV[i + 2])-    else-      local cur = redis.call('HGET', sk, ARGV[i + 1])-      if cur and string.sub(cur, 1, 32) == ARGV[i + 3] then-        redis.call('HDEL', sk, ARGV[i + 1])-      end-    end-  end-  i = i + 4-end-return 1
− src/Haskoin/Store/Cache/lua/unlock.lua
@@ -1,5 +0,0 @@--- Release a lock if it still holds the given token: unlockScript in Haskoin.Store.Cache.-if redis.call('GET', KEYS[1]) == ARGV[1] then-  return redis.call('DEL', KEYS[1])-end-return 0
src/Haskoin/Store/Common.hs view
@@ -55,6 +55,7 @@     applyLimit,     applyLimitC,     sortTxs,+    sortTxsWith,     nub',     microseconds,     streamThings,@@ -87,13 +88,13 @@ import Data.Maybe (catMaybes, fromMaybe, mapMaybe) import Data.Ord (Down (..)) import Data.Serialize (Serialize (..), encode)-import GHC.Generics (Generic) import Data.Time.Clock.System   ( getSystemTime,     systemNanoseconds,     systemSeconds,   ) import Data.Word (Word32, Word64)+import GHC.Generics (Generic) import Haskoin import Haskoin.Node (Chain, Peer, chainRunHeaders) import Haskoin.Store.Data@@ -158,19 +159,17 @@   def = defaultLimits  -- | The queries that the web server answers with, as plain functions in--- 'IO'. "Haskoin.Store.Database.Query" gives them for the database--- ('Haskoin.Store.Database.Query.databaseRead'); the web server puts the--- Redis cache in front of some of them.+-- 'IO'. "Haskoin.Store.Database.Query.Pooled" gives them for the chain+-- with the fleet's pool on top ('Haskoin.Store.Database.Query.Pooled.pooledRead');+-- the web server puts the cache of xpubs in front of some of them. -- -- A record of functions rather than a class: a class needs a monad to pick -- its instance, and the store has no monads of its own to pick it with, only -- 'IO'. The record makes the source of every answer an ordinary value that--- can be passed around and replaced, as @nocache=true@ does.+-- can be passed around and replaced, as the door to the cache of xpubs does. data StoreRead = StoreRead   { -- | network the store indexes     net :: !Network,-    -- | context for secp256k1 operations, needed to parse and derive keys-    ctx :: !Ctx,     -- | hash of the best block imported, or 'Nothing' if none yet     getBestBlock :: IO (Maybe BlockHash),     -- | hashes of all blocks imported at a height, side chains included@@ -321,10 +320,10 @@ -- | Address derivation function for a type of address: legacy pay-to-public-key -- hash, pay-to-witness-public-key-hash nested in pay-to-script-hash, or native -- pay-to-witness-public-key-hash.-deriveFunction :: Ctx -> DeriveType -> DeriveAddr-deriveFunction ctx DeriveNormal i = fst . deriveAddr ctx i-deriveFunction ctx DeriveP2SH i = fst . deriveCompatWitnessAddr ctx i-deriveFunction ctx DeriveP2WPKH i = fst . deriveWitnessAddr ctx i+deriveFunction :: DeriveType -> DeriveAddr+deriveFunction DeriveNormal i = fst . deriveAddr i+deriveFunction DeriveP2SH i = fst . deriveCompatWitnessAddr i+deriveFunction DeriveP2WPKH i = fst . deriveWitnessAddr i  -- | Balances of the addresses of an extended public key, derived up to the -- gap: every address of each chain up to the last used one, followed by@@ -419,16 +418,16 @@  -- | Transaction with its metadata, or 'Nothing' if it is unknown. getTransaction :: StoreRead -> TxHash -> IO (Maybe Transaction)-getTransaction r h = fmap (toTransactionWithId r.net r.ctx h) <$> r.getTxData h+getTransaction r h = fmap (toTransactionWithId r.net h) <$> r.getTxData h  -- | Many transactions, in the order of their hashes, read together. Each is -- read by its id, so its id is not computed again ('toTransactionWithId'). getTransactions :: StoreRead -> [TxHash] -> IO [Maybe Transaction]-getTransactions r hs = zipWith (fmap . toTransactionWithId r.net r.ctx) hs <$> r.getManyTxData hs+getTransactions r hs = zipWith (fmap . toTransactionWithId r.net) hs <$> r.getManyTxData hs  -- | Transactions that match a short numeric identifier; see 'getNumTxData'. getNumTransaction :: StoreRead -> Word64 -> IO [Transaction]-getNumTransaction r i = map (toTransaction r.net r.ctx) <$> r.getNumTxData i+getNumTransaction r i = map (toTransaction r.net) <$> r.getNumTxData i  -- | First block in the main chain whose timestamp is at or after the given time, -- or 'Nothing' if there is none.@@ -530,20 +529,30 @@ -- | Sort transactions so that every transaction comes after those it spends. -- Each transaction is returned with its position in the original list. sortTxs :: [Tx] -> [(Word32, Tx)]-sortTxs txs = ts+sortTxs = sortTxsWith txHash id++-- | 'sortTxs' for transactions that come with something else, given how to+-- get the hash and the transaction of each. The order is the one that+-- 'sortTxs' gives the same transactions.+--+-- It is for a caller that has the hashes already. A hash is two rounds of+-- SHA-256 over the whole transaction, and working out those of a block+-- again here cost more than the sort itself.+sortTxsWith :: forall a. (a -> TxHash) -> (a -> Tx) -> [a] -> [(Word32, a)]+sortTxsWith hashOf txOf xs = ts   where     hm :: M.HashMap TxHash Int-    hm = M.fromList $ zip (map txHash txs) [0 ..]-    ns :: [(Tx, Int, [Int])]+    hm = M.fromList $ zip (map hashOf xs) [0 ..]+    ns :: [(a, Int, [Int])]     ns =-      let is tx = nub' $ map (.outpoint.hash) tx.inputs-          hs tx = mapMaybe (`M.lookup` hm) (is tx)-          tp i tx = (tx, i, hs tx)-       in zipWith tp [0 ..] txs+      let is x = nub' $ map (.outpoint.hash) (txOf x).inputs+          hs x = mapMaybe (`M.lookup` hm) (is x)+          tp i x = (x, i, hs x)+       in zipWith tp [0 ..] xs     (g, n, _) = G.graphFromEdges ns     vs = G.reverseTopSort g     ts =-      let f (tx, i, _) = (fromIntegral i, tx)+      let f (x, i, _) = (fromIntegral i, x)        in map (f . n) vs  -- | Remove duplicates from a list. The order of the result is unspecified.
− src/Haskoin/Store/Database/Mempool.hs
@@ -1,505 +0,0 @@--- |--- Module      : Haskoin.Store.Database.Mempool--- Copyright   : (c) Haskoin Developers--- License     : MIT--- Maintainer  : jprupp@protonmail.ch--- Stability   : experimental--- Portability : POSIX------ The mempool area of the layered layout of the database, and how it is--- applied on top of the chain. The design, and the reasons for it, are in--- @docs\/layered-database.md@. This is the layout that the store runs on,--- since version 21. Its rules are checked by the tests of--- "Haskoin.Store.Database.Migrate" against the answers of a database of--- version 20, in which the mempool was mixed into the chain. The queries--- that apply them are in "Haskoin.Store.Database.Query.Layered".------ = Two layers------ In the layered layout the column families of the chain hold only what--- follows from the blocks of the main chain, and change only when a block is--- imported or reverted. Everything about unconfirmed transactions is in two--- column families of its own, 'mempoolCF' and 'mempoolAddrCF'. A query reads--- the chain, reads what the mempool has for the same transaction, output or--- address, and puts the two together. Emptying the mempool area leaves the--- chain as it is, with an empty mempool on top.------ = Every key belongs to one transaction------ 'mempoolRecords' gives the keys that a mempool transaction contributes.--- They depend on that transaction alone: on its inputs, with the outputs--- they spend, which it carries with it, and on its outputs. The mempool area--- is the union of the keys of its transactions. So a transaction enters the--- mempool by writing its keys and leaves it by deleting them, in any order,--- and when a block confirms some transactions and not their descendants,--- nothing has to be worked out again for those that stay.------ The alternative is to keep, for the mempool, the same values that the--- chain has: a balance, a list of unspent outputs. Such values depend on the--- chain under them. A balance with the mempool applied is wrong as soon as a--- block pays the address, and every block would have to find and correct--- the values it made stale. The keys here say what the mempool does, not--- what the result is, so they stay right whatever happens to the chain.------ What a query needs is worked out when it is answered, from the few entries--- that the mempool has for an address, an t'AddressMempool':------ * an output that the mempool creates and does not spend is an unspent---   output of the mempool ('mempoolUnspents');------ * an output that the mempool spends and did not create belongs to the---   chain, which still has it as unspent, and is left out of what the chain---   says ('hiddenOutputs');------ * an output that the mempool both creates and spends shows up in neither,---   and only counts towards what the address received.------ = The summary of an address------ The entries of an address are few for nearly every address, and many--- thousands for some. So each address that the mempool touches also has a--- t'MemSummary', one key with what its entries add up to ('summarise'). A--- balance is the balance of the chain and the summary ('applySummary'), and--- the queries read the entries themselves only for lists, as far as a page--- goes.------ The summary is not the key of one transaction, but it is a function of--- the entries of its address and of nothing else, so it shares what makes--- them safe: nothing that happens to the chain makes it wrong.--- 'wrongSummaries' checks that, and 'rebuildSummaries' writes them all--- again from the entries.-module Haskoin.Store.Database.Mempool-  ( -- * What a transaction contributes-    MemRecords (..),-    mempoolRecords,-    recordEntries,-    insertRecordOps,-    deleteRecordOps,--    -- * Reading the mempool area-    AddressMempool (..),-    getAddressMempool,-    getAddressMempools,-    getMempoolTx,-    getMempoolSpenders,-    wipeMempoolArea,--    -- * Applying the mempool on top of the chain-    mempoolUnspents,-    hiddenOutputs,-    applyMempool,-    mergeBalance,--    -- * The summary of an address-    summarise,-    summaryEntries,-    applySummary,-    summaryBalance,-    getSummaries,-    rebuildSummaries,-    wrongSummaries,-    addressArea,-  )-where--import Conduit (runConduit, sinkList, (.|))-import Data.ByteString (ByteString)-import Data.HashMap.Strict (HashMap)-import Data.HashMap.Strict qualified as HashMap-import Data.HashSet qualified as HashSet-import Data.IntMap.Strict (IntMap)-import Data.IntMap.Strict qualified as IntMap-import Data.Maybe (fromMaybe, isNothing, mapMaybe)-import Data.Serialize (Serialize, decode, encode)-import Data.Word (Word32, Word64)-import Database.RocksDB-  ( BatchOp (..),-    ColumnFamily,-    DB,-    Iterator,-    getCF,-    iterFirst,-    iterKey,-    iterNext,-    multiGetCF,-    withIterCF,-  )-import Database.RocksDB.Query-  ( matching,-    matchingAsListCF,-    retrieveCF,-    writeBatch,-  )-import Haskoin-  ( Address,-    Ctx,-    Network,-    OutPoint (..),-    Tx (..),-    TxHash,-    TxIn (..),-    TxOut (..),-    eitherToMaybe,-    nullOutPoint,-    scriptToAddressBS,-    txHash,-  )-import Haskoin.Store.Common (nub')-import Haskoin.Store.Data-  ( Balance (..),-    BlockRef (..),-    Prev (..),-    Spender (..),-    TxData (..),-    TxRef (..),-    UnixTime,-    Unspent (..),-    zeroBalance,-  )-import Haskoin.Store.Database.Reader-  ( DatabaseReader (..),-    mempoolAddrCF,-    mempoolCF,-  )-import Haskoin.Store.Database.Types-  ( MemAddrKey (..),-    MemAddrVal (..),-    MemSpenderKey (..),-    MemSummary (..),-    MemTimeKey (..),-    MemTxKey (..),-    TxKey (..),-  )------------------------------------------------------------------------------------ What a transaction contributes------------------------------------------------------------------------------------ | What one mempool transaction contributes to the mempool area.-data MemRecords = MemRecords-  { -- | hash of the transaction-    txid :: !TxHash,-    -- | time it entered the mempool: its place in the list of the mempool-    -- and in the index by time-    time :: !UnixTime,-    -- | its body, without spenders and at a mempool position-    body :: !TxData,-    -- | the outputs it spends, each with the input that spends it-    spends :: ![(OutPoint, Spender)],-    -- | for each address it touches: the transaction itself, the outputs of-    -- the address that it spends and the outputs that it pays to the-    -- address-    addresses :: ![(MemAddrKey, MemAddrVal)]-  }-  deriving (Eq, Show)---- | What a mempool transaction that entered the mempool at a time--- contributes to the mempool area.------ It is a function of the transaction and that time and of nothing else:--- not of the chain, and not of the other transactions in the mempool. The--- body is stored without spenders, which have keys of their own, and as not--- deleted. The flag for replace-by-fee is kept as given; it is the one--- thing here that can depend on other transactions, because a transaction--- inherits it from unconfirmed ancestors, and it is corrected when one of--- them is confirmed.-mempoolRecords :: Network -> Ctx -> UnixTime -> TxData -> MemRecords-mempoolRecords net ctx time td =-  MemRecords-    { txid = h,-      time,-      body = (td :: TxData) {block = MemRef time, deleted = False, spenders = IntMap.empty},-      spends = [(op, Spender h k) | (k, op) <- spent],-      addresses =-        [(MemAddrTxKey a time h, MemAddrTxVal) | a <- touched]-          <> [ (MemAddrSpentKey a op, MemAddrSpentVal p.value)-             | (k, op) <- spent,-               Just p <- [IntMap.lookup (fromIntegral k) td.prevs],-               Just a <- [address p.script]-             ]-          <> [ (MemAddrOutKey a time (OutPoint h i), MemAddrOutVal o.value o.script)-             | (i, o) <- zip [0 ..] td.tx.outputs,-               Just a <- [address o.script]-             ]-    }-  where-    h = txHash td.tx-    spent = [(k, i.outpoint) | (k, i) <- zip [0 :: Word32 ..] td.tx.inputs, i.outpoint /= nullOutPoint]-    address = eitherToMaybe . scriptToAddressBS net ctx-    touched =-      nub' $-        mapMaybe (address . (.script)) (IntMap.elems td.prevs)-          <> mapMaybe (address . (.script)) td.tx.outputs---- | The keys and values of a transaction, serialized, for each of the two--- column families of the mempool area: the mempool column family first,--- then the one of address entries.-recordEntries :: MemRecords -> ([(ByteString, ByteString)], [(ByteString, ByteString)])-recordEntries r =-  ( [ kv (MemTxKey r.txid) r.time,-      kv (MemTimeKey r.time r.txid) (),-      kv (TxKey r.txid) r.body-    ]-      <> [kv (MemSpenderKey op) s | (op, s) <- r.spends],-    [kv k v | (k, v) <- r.addresses]-  )-  where-    kv :: (Serialize k, Serialize v) => k -> v -> (ByteString, ByteString)-    kv k v = (encode k, encode v)---- | Operations that write the keys of a transaction to the mempool area.-insertRecordOps :: DB -> MemRecords -> [BatchOp]-insertRecordOps db r =-  [PutCF (mempoolCF db) k v | (k, v) <- general]-    <> [PutCF (mempoolAddrCF db) k v | (k, v) <- byAddress]-  where-    (general, byAddress) = recordEntries r---- | Operations that delete the keys of a transaction from the mempool area.-deleteRecordOps :: DB -> MemRecords -> [BatchOp]-deleteRecordOps db r =-  [DelCF (mempoolCF db) k | (k, _) <- general]-    <> [DelCF (mempoolAddrCF db) k | (k, _) <- byAddress]-  where-    (general, byAddress) = recordEntries r------------------------------------------------------------------------------------ Reading the mempool area------------------------------------------------------------------------------------ | What the mempool does at an address. It is small, usually empty, and--- read with one scan of the entries of the address.-data AddressMempool = AddressMempool-  { -- | mempool transactions that spend from or pay to the address, newest-    -- first, in the order of 'Haskoin.Store.Common.txRefKey'-    txs :: ![TxRef],-    -- | outputs of the address that mempool transactions spend, with their-    -- amounts, whether a block or the mempool created them-    spent :: !(HashMap OutPoint Word64),-    -- | outputs that mempool transactions pay to the address, whether the-    -- mempool spends them or not, newest first, in the order of-    -- 'Haskoin.Store.Common.unspentKey'-    created :: ![Unspent]-  }-  deriving (Eq, Show)---- | What the mempool does at an address.-getAddressMempool :: DatabaseReader -> Address -> IO AddressMempool-getAddressMempool r a = withIterCF r.db (mempoolAddrCF r.db) (`addressMempool` a)---- | What the mempool does at each of some addresses, in their order.------ All of them are read through one iterator, which is moved to each address--- in turn. A query about an extended public key asks about hundreds of--- addresses, nearly all with nothing in the mempool; opening an iterator for--- each would cost more than the reads, each of which stops at the filter.-getAddressMempools :: DatabaseReader -> [Address] -> IO [AddressMempool]-getAddressMempools _ [] = return []-getAddressMempools r as = withIterCF r.db (mempoolAddrCF r.db) $ \it -> mapM (addressMempool it) as---- | What the mempool does at an address, read with an iterator on the--- column family of the entries by address, which is left wherever the scan--- ended.-addressMempool :: Iterator -> Address -> IO AddressMempool-addressMempool it a = do-  rows <- runConduit $ matching it (MemAddrKeyA a) .| sinkList-  return-    AddressMempool-      { txs = [TxRef {block = MemRef t, txid = h} | (MemAddrTxKey _ t h, _) <- rows],-        spent = HashMap.fromList [(p, v) | (MemAddrSpentKey _ p, MemAddrSpentVal v) <- rows],-        created =-          [ Unspent {block = MemRef t, outpoint = p, value = v, script = s, address = Just a}-          | (MemAddrOutKey _ t p, MemAddrOutVal v s) <- rows-          ]-      }---- | Body of a transaction that is in the mempool, without its spenders.-getMempoolTx :: DatabaseReader -> TxHash -> IO (Maybe TxData)-getMempoolTx r h = retrieveCF r.db (mempoolCF r.db) (TxKey h)---- | Mempool inputs that spend the outputs of a transaction, by output index.--- The transaction may be confirmed or in the mempool.-getMempoolSpenders :: DatabaseReader -> TxHash -> IO (IntMap Spender)-getMempoolSpenders r h = do-  rows <- matchingAsListCF r.db (mempoolCF r.db) (MemSpenderKeyS h)-  return $ IntMap.fromList [(fromIntegral p.index, s) | (MemSpenderKey p, s) <- rows]---- | Delete every key of the mempool area, and say how many there were. The--- chain is not touched: afterwards the database is that of a store whose--- mempool is empty, as if every unconfirmed transaction had been deleted one--- by one, except that it does not remember having seen them.------ This is all that wiping the mempool takes in the layered layout, and what--- can be done if the mempool area is ever found to be wrong.-wipeMempoolArea :: DatabaseReader -> IO Int-wipeMempoolArea r = sum <$> mapM wipe [mempoolCF r.db, mempoolAddrCF r.db]-  where-    wipe cf = go 0-      where-        go n =-          firstKeys cf >>= \case-            [] -> return n-            ks -> writeBatch r.db (map (DelCF cf) ks) >> go (n + length ks)-    firstKeys :: ColumnFamily -> IO [ByteString]-    firstKeys cf = withIterCF r.db cf $ \it -> iterFirst it >> collect it (10000 :: Int)-    collect _ 0 = return []-    collect it n =-      iterKey it >>= \case-        Nothing -> return []-        Just k -> iterNext it >> (k :) <$> collect it (n - 1)------------------------------------------------------------------------------------ Applying the mempool on top of the chain------------------------------------------------------------------------------------ | Outputs of an address that the mempool created and has not spent,--- newest first.-mempoolUnspents :: AddressMempool -> [Unspent]-mempoolUnspents m = [u | u <- m.created, not (HashMap.member u.outpoint m.spent)]---- | Outputs of an address that the chain has as unspent and that the mempool--- spends, with their amounts: those that the mempool spends and did not--- create itself.-hiddenOutputs :: AddressMempool -> HashMap OutPoint Word64-hiddenOutputs m = HashMap.filterWithKey (\p _ -> not (HashSet.member p made)) m.spent-  where-    made = HashSet.fromList (map (.outpoint) m.created)---- | The balance that the chain has for an address, with what the mempool--- does at the address applied: the balance as the API gives it.------ As in version 20, an output of a block that a mempool transaction spends no--- longer counts as confirmed balance, and the unconfirmed balance is what--- the mempool pays to the address and has not spent.------ The amounts taken from the chain can never be more than it has, as long--- as every output that the mempool spends exists. If they ever are, the--- result stops at zero rather than wrapping around.-applyMempool :: Balance -> AddressMempool -> Balance-applyMempool c m = applySummary c (summarise m)---- | Balance of an address as the API gives it, from its balance in the--- chain, if it has one, and what the mempool does at it ('applyMempool').--- 'Nothing' if the address was never used in either.------ Version 20 answers with a balance of zero, rather than with 'Nothing',--- for an address whose only transactions were deleted from the mempool: it--- leaves the balance that it wrote for them behind. The API shows an unknown--- address as a balance of zero, so the two are the same to it.-mergeBalance :: Address -> Maybe Balance -> AddressMempool -> Maybe Balance-mergeBalance a chain m-  | isNothing chain && null m.txs && HashMap.null m.spent && null m.created = Nothing-  | otherwise = Just (applyMempool (fromMaybe (zeroBalance a) chain) m)------------------------------------------------------------------------------------ The summary of an address------------------------------------------------------------------------------------ | What the entries of an address add up to.-summarise :: AddressMempool -> MemSummary-summarise m =-  MemSummary-    { txs = fromIntegral (length m.txs),-      paid = sum (map (.value) m.created),-      paidCount = fromIntegral (length m.created),-      spent = sum (HashMap.elems m.spent),-      spentCount = fromIntegral (HashMap.size m.spent),-      both = sum (map (.value) inside),-      bothCount = fromIntegral (length inside)-    }-  where-    inside = [u | u <- m.created, HashMap.member u.outpoint m.spent]---- | The summaries that go with some entries by address, which may be of--- many addresses: what the column family must have next to them, for the--- migration, for repairs and for the tests.-summaryEntries :: [(MemAddrKey, MemAddrVal)] -> [(MemAddrKey, MemAddrVal)]-summaryEntries rows =-  [ (MemAddrSumKey a, MemAddrSumVal (summarise m))-  | (a, m) <- HashMap.toList (HashMap.fromListWith (flip merge) (mapMaybe single rows))-  ]-  where-    entries ts sp cr = AddressMempool {txs = ts, spent = sp, created = cr}-    merge x y = entries (x.txs <> y.txs) (x.spent <> y.spent) (x.created <> y.created)-    single = \case-      (MemAddrTxKey a t h, _) -> Just (a, entries [TxRef {block = MemRef t, txid = h}] HashMap.empty [])-      (MemAddrSpentKey a p, MemAddrSpentVal v) -> Just (a, entries [] (HashMap.singleton p v) [])-      (MemAddrOutKey a t p, MemAddrOutVal v s) ->-        let u = Unspent {block = MemRef t, outpoint = p, value = v, script = s, address = Just a}-         in Just (a, entries [] HashMap.empty [u])-      _ -> Nothing---- | The balance that the chain has for an address with the summary of what--- the mempool does at it applied: what 'applyMempool' gives, from the--- summary instead of the entries.-applySummary :: Balance -> MemSummary -> Balance-applySummary c s =-  Balance-    { address = c.address,-      confirmed = c.confirmed `minus` (s.spent `minus` s.both),-      unconfirmed = c.unconfirmed + (s.paid `minus` s.both),-      utxo = (c.utxo `minus` (s.spentCount `minus` s.bothCount)) + (s.paidCount `minus` s.bothCount),-      txs = c.txs + s.txs,-      received = c.received + s.paid-    }-  where-    minus x y = if y > x then 0 else x - y---- | Balance of an address as the API gives it, from its balance in the--- chain and its summary in the mempool, either of which it may not have:--- what 'mergeBalance' gives, without reading the entries.-summaryBalance :: Address -> Maybe Balance -> Maybe MemSummary -> Maybe Balance-summaryBalance _ chain Nothing = chain-summaryBalance a chain (Just s) = Just (applySummary (fromMaybe (zeroBalance a) chain) s)---- | The summaries of addresses, in their order, read by key with one--- multi-get: 'Nothing' for an address that the mempool does not touch,--- which is nearly all of them, and which the filter of the column family--- answers.-getSummaries :: DatabaseReader -> [Address] -> IO [Maybe MemSummary]-getSummaries r as =-  map (>>= decoded) <$> case map (encode . MemAddrSumKey) as of-    [] -> return []-    -- one key is cheaper to read by itself than in a multi-get-    [key] -> pure <$> getCF r.db (mempoolAddrCF r.db) key-    keys -> multiGetCF r.db (mempoolAddrCF r.db) keys-  where-    decoded bytes = case decode bytes of-      Right (MemAddrSumVal s) -> Just s-      _ -> Nothing---- | The addresses whose summary is not what their entries add up to, each--- with the summary that it should have and the one that it has. There are--- none unless something is broken: this is what the tests hold the rules--- to after every operation.-wrongSummaries :: DatabaseReader -> IO [(MemAddrKey, Maybe MemAddrVal, Maybe MemAddrVal)]-wrongSummaries r = do-  rows <- matchingAsListCF r.db (mempoolAddrCF r.db) MemAddrKeyS-  let found = HashMap.fromList [(k, v) | (k@MemAddrSumKey {}, v) <- rows]-      expected = HashMap.fromList (summaryEntries rows)-  return-    [ (k, HashMap.lookup k expected, HashMap.lookup k found)-    | k <- HashMap.keys (HashMap.union expected found),-      HashMap.lookup k expected /= HashMap.lookup k found-    ]---- | Everything that the column family of entries by address holds for a--- mempool of some transactions, serialized, in no order: the entries of--- each, and the summaries of the addresses that they touch.-addressArea :: [MemRecords] -> [(ByteString, ByteString)]-addressArea records = [(encode k, encode v) | (k, v) <- entries <> summaryEntries entries]-  where-    entries = concatMap (.addresses) records---- | Work every summary out again from the entries, and say how many--- addresses have one. The migration does this once everything is moved, and--- it is what puts the summaries right if they are ever in doubt.-rebuildSummaries :: DatabaseReader -> IO Int-rebuildSummaries r = do-  rows <- matchingAsListCF r.db cf MemAddrKeyS-  let old = [DelCF cf (encode k) | (k@MemAddrSumKey {}, _ :: MemAddrVal) <- rows]-      new = summaryEntries rows-  writeBatch r.db (old <> [PutCF cf (encode k) (encode v) | (k, v) <- new])-  return (length new)-  where-    cf = mempoolAddrCF r.db
+ src/Haskoin/Store/Database/Merge.hs view
@@ -0,0 +1,114 @@+-- |+-- Module      : Haskoin.Store.Database.Merge+-- Copyright   : (c) Haskoin Developers+-- License     : MIT+-- Maintainer  : jprupp@protonmail.ch+-- Stability   : experimental+-- Portability : POSIX+--+-- The merge operators of the database: how RocksDB works out an address+-- balance and a group of spenders from the merge records that the import+-- of a block writes.+--+-- = The problem+--+-- A block changes the balance of every address that it pays or spends+-- from, and adds a spender to the group of every output that it spends.+-- Each was read, changed and written back. On a store of Bitcoin mainnet+-- those two kinds of read were most of the reads of a block: 300 to 850 ms+-- for the groups of spenders and 70 to 300 for the balances, of a read+-- ahead of 640 to 1,440 ms. And they were read only to be changed.+--+-- = Merge records+--+-- RocksDB has a third kind of write besides a put and a delete: a merge+-- record, which says how a value changes and is written without reading+-- anything. The /merge operator/ of the column family works out the value+-- later, from the value that the key had and the records written since:+-- when the key is read, and when a compaction brings them together, after+-- which the key holds a plain value again.+--+-- So the import of a block writes, for each address, the differences to+-- its balance, and for each group of outputs, the spenders to add+-- ('Haskoin.Store.Database.Writer.mergeBalance',+-- 'Haskoin.Store.Database.Writer.mergeSpender'), and reads neither.+--+-- = The two operators+--+-- They are C functions, in @cbits\/haskoin_store_merge.c@, because RocksDB+-- calls them from its own threads. A merge record has the format of a+-- value in both, so that applying records to a value and combining records+-- with each other are one operation:+--+-- * __Balances__ ('balanceOperator'): a record is five differences in the+--   forty bytes of a 'Haskoin.Store.Database.Types.BalVal', and merging+--   adds each of the five numbers modulo 2^64. A key without a value+--   counts from zero.+--+-- * __Groups of spenders__ ('spenderOperator'): a record is a+--   t'Haskoin.Store.Database.Types.SpenderVal' with the entries to add,+--   and merging is the union, the newer entry winning for an output that+--   two have.+--+-- = What it asks+--+-- * __Every program that opens the database sets the operators__: they are+--   in the configuration of the two column families+--   ('Haskoin.Store.Database.Reader.columnFamilyConfig'), so nothing in+--   this package can open it without. RocksDB itself does not refuse to:+--   it then drops the merge records that are still in its write-ahead log,+--   and cannot read the others. That is why a database that this version+--   has opened has a column family that older versions do not know of, and+--   why they can no longer open it+--   ('Haskoin.Store.Database.Reader.layeredFamilyConfig').+--+-- * __Readers pay for the merge.__ A read of a key with merge records+--   pending looks into every table file that has one of them, down to the+--   value, and merges. @haskoin-store-bench merge@ measures it, and+--   @docs\/one-fleet.md@ has the figures.+--+-- * __Reverts read.__ Only the import of a block writes merge records.+--   Taking a block out of the chain, and everything else that changes a+--   balance or a group of spenders, reads the value and writes it whole,+--   or deletes it: a put or a delete hides the merge records under it.+module Haskoin.Store.Database.Merge+  ( balanceOperator,+    spenderOperator,+  )+where++import Database.RocksDB (FullMerge, MergeOperator (..), PartialMerge)+import Foreign.Ptr (FunPtr)++foreign import ccall "&haskoin_store_balance_full_merge" balanceFullMerge :: FunPtr FullMerge++foreign import ccall "&haskoin_store_balance_partial_merge" balancePartialMerge :: FunPtr PartialMerge++foreign import ccall "&haskoin_store_spender_full_merge" spenderFullMerge :: FunPtr FullMerge++foreign import ccall "&haskoin_store_spender_partial_merge" spenderPartialMerge :: FunPtr PartialMerge++-- | The merge operator of the column family of address balances. A merge+-- record is a 'Haskoin.Store.Database.Types.BalVal' of five differences,+-- each added to its number modulo 2^64, so that a number goes down by+-- adding its two's complement. A balance that is not there counts from+-- zero.+balanceOperator :: MergeOperator+balanceOperator =+  MergeFunctions+    { mergeName = "haskoin-store-balance-1",+      fullMerge = balanceFullMerge,+      partialMerge = balancePartialMerge+    }++-- | The merge operator of the column family of the spenders of outputs. A+-- merge record is a t'Haskoin.Store.Database.Types.SpenderVal' with the+-- spenders to add to a group, and merging is the union of the group and+-- the records, the newest entry winning for an output that several have.+spenderOperator :: MergeOperator+spenderOperator =+  MergeFunctions+    { mergeName = "haskoin-store-spender-1",+      fullMerge = spenderFullMerge,+      partialMerge = spenderPartialMerge+    }
src/Haskoin/Store/Database/Migrate.hs view
@@ -9,10 +9,11 @@ -- Stability   : experimental -- Portability : POSIX ----- Migration of a database of version 20 to the layered layout, version 21,--- of @docs\/layered-database.md@: the column families of the chain are left--- with nothing from the mempool, and the mempool gets an area of its own--- ("Haskoin.Store.Database.Mempool").+-- Migration of a database of version 20 to the layout that holds the chain+-- alone, version 21 and after: the column families of the chain are left+-- with nothing from the mempool. The mempool is the pool that the fleet+-- shares in Redis (@docs\/one-fleet.md@), so the transactions are dropped:+-- the pool hears of them again from peers. -- -- The store runs it when it opens a database ('upgradeDatabase'). It is -- what makes the change of layout possible without syncing the chain again.@@ -24,9 +25,10 @@ -- outputs it spends as spent, with itself as their spender in the bodies of -- the transactions that created them, and all of it in the balances. The -- migration takes each mempool transaction out of those indices, exactly as--- deleting it does in version 20, and writes it to the mempool area instead.--- When it is done, the chain's column families hold what they would hold if--- the mempool had been wiped, and the mempool area holds the mempool.+-- deleting it does in version 20. When it is done, the chain's column+-- families hold what they would hold if the mempool had been wiped. The+-- list of the mempool, in its own column family, is read and left there:+-- nothing reads it afterwards. -- -- It reads and writes only what the mempool touches, so it takes a time in -- proportion to the mempool, not to the chain: about what wiping the@@ -34,21 +36,19 @@ -- -- = Why its own rules ----- Taking a transaction out of the indices of version 20 is what--- "Haskoin.Store.Logic" does today when it deletes one. The migration does--- not call that code: the rules of the store will change with the layout,--- and a migration has to go on reading version 20 as it was. It carries the--- few rules it needs, and the tests check them against "Haskoin.Store.Logic"--- while both exist.+-- Taking a transaction out of the indices of version 20 is what the rules+-- of version 20 did when they deleted one. Those rules are gone, and a+-- migration has to go on reading version 20 as it was, so it carries the+-- few rules it needs. -- -- = Safe to interrupt -- -- A mempool can hold hundreds of thousands of transactions, too many for one -- write, so the migration works in batches. Each batch is one atomic write--- that moves whole transactions, descendants before the transactions they--- spend. After any batch the database is a valid mix: the transactions--- already moved are in the mempool area, and the others are still in the--- chain's indices, as a database of version 20 with a smaller mempool. The+-- that takes out whole transactions, descendants before the transactions+-- they spend. After any batch the database is a valid mix: the transactions+-- already taken out are gone, and the others are still in the chain's+-- indices, as a database of version 20 with a smaller mempool. The -- migration can stop there and carry on later. -- -- The first write sets the new version and a marker, t'MigrationKey', so@@ -75,7 +75,7 @@ -- spenders of its outputs are under keys of their own -- as t'Haskoin.Store.Database.Types.SpenderKey' says, and its record is written -- when a block confirms it and deleted if that block is reverted, and--- nothing else ('Haskoin.Store.Logic.Layered.importBlock').+-- nothing else ('Haskoin.Store.Logic.importBlock'). -- 'migrateRecords' brings the records that are there already to that: it -- gives each the time of its block, and moves its spenders to their keys. -- It reads every transaction, and writes nearly every one, since most@@ -84,11 +84,28 @@ -- whole transactions, the keys with the record that they come out of, and -- running it again skips what is done, so it can be interrupted and -- started over.+--+-- = Version 23: merge records for balances and spenders+--+-- From version 23 the import of a block does not read the balances of its+-- addresses or the groups of spenders of the outputs that it spends: it+-- writes how they change, as merge records, which RocksDB combines with+-- the values when they are read ("Haskoin.Store.Database.Merge"). What a+-- database of version 22 holds is what version 23 reads, so there is+-- nothing to migrate: 'upgradeDatabase' writes the version.+--+-- It cannot go back. A store that does not know the merge operators could+-- not read the merge records, and RocksDB would drop those of its+-- write-ahead log when it opened the database for such a store. So a+-- database that this version has opened has a column family that older+-- versions do not know of, and RocksDB refuses to open it for them+-- ('Haskoin.Store.Database.Reader.layeredFamilyConfig'). module Haskoin.Store.Database.Migrate   ( upgradeDatabase,     currentVersion,     migrateRecords,     layeredVersion,+    recordsVersion,     Migration (..),     MigrationError (..),     MigrationOptions (..),@@ -98,14 +115,17 @@   ) where -import Control.Monad (forM_, unless, void, when)+import Control.Exception (Exception (..), throwIO)+import Control.Monad (forM_, unless, when)+import Data.ByteString qualified as B import Data.Default (def) import Data.HashMap.Strict (HashMap) import Data.HashMap.Strict qualified as HashMap import Data.HashSet (HashSet) import Data.HashSet qualified as HashSet+import Data.IORef (IORef, modifyIORef', newIORef, readIORef, writeIORef) import Data.IntMap.Strict qualified as IntMap-import Data.Maybe (isJust, mapMaybe)+import Data.Maybe (isJust, isNothing, mapMaybe) import Data.Serialize (decode, encode) import Data.String.Conversions (cs) import Data.Text (Text)@@ -120,6 +140,7 @@     iterNext,     withIterConfig,   )+import Database.RocksDB qualified as RocksDB import Database.RocksDB.Query   ( deleteOp,     deleteOpCF,@@ -155,14 +176,12 @@     UnixTime,     confirmed,   )-import Haskoin.Store.Database.Mempool (insertRecordOps, mempoolRecords, rebuildSummaries) import Haskoin.Store.Database.Query qualified as Q import Haskoin.Store.Database.Reader   ( DatabaseReader (..),     addrOutCF,     addrTxCF,     balanceCF,-    initRocksDB,     mempoolCF,     spenderCF,     txCF,@@ -184,25 +203,24 @@     VersionKey (..),     spenderGroup,   )-import UnliftIO-  ( Exception,-    IORef,-    modifyIORef',-    newIORef,-    readIORef,-    throwIO,-    writeIORef,-  )  -- | Version of the layered layout as it was first written, with the time -- at which a store first saw a transaction kept in the chain. layeredVersion :: Word32 layeredVersion = 21 --- | Version that the store runs on: the layered layout, with the time of--- its block for every confirmed transaction.+-- | Version of the layered layout with the time of its block for every+-- confirmed transaction, and the spenders of its outputs under keys of+-- their own: what 'migrateRecords' brings a database of 'layeredVersion'+-- to.+recordsVersion :: Word32+recordsVersion = 22++-- | Version that the store runs on: that of 'recordsVersion', in which the+-- import of a block writes merge records for balances and spenders+-- ("Haskoin.Store.Database.Merge"). currentVersion :: Word32-currentVersion = 22+currentVersion = 23  -- | Version that the migration starts from. mixedVersion :: Word32@@ -210,12 +228,12 @@  -- | What a migration did. data Migration = Migration-  { -- | mempool transactions moved to the mempool area by this run+  { -- | mempool transactions taken out of the chain's indices by this run     moved :: !Int,-    -- | writes to the database that moved them+    -- | writes to the database that took them out     batches :: !Int,-    -- | mempool transactions checked at the end, moved by this run or by-    -- an earlier one that was interrupted+    -- | mempool transactions checked at the end, taken out by this run or+    -- by an earlier one that was interrupted     checked :: !Int,     -- | records of confirmed transactions that this run wrote again, with     -- the time of their block and without the spenders of their outputs@@ -231,14 +249,38 @@   | -- | the database is not as its version says it should be; nothing was     -- written for the transaction that showed it     Inconsistent !Text-  deriving (Eq, Show, Exception)+  deriving (Eq, Show) +-- | In words, for whoever reads the last line of a store that did not+-- start: a version above 'currentVersion' is that of a newer store.+instance Exception MigrationError where+  displayException = \case+    UnknownVersion (Just v)+      | v > currentVersion ->+          unwords+            [ "The database is of version",+              show v <> ",",+              "written by a newer haskoin-store: this one runs on version",+              show currentVersion,+              "and cannot read it. Nothing was changed."+            ]+      | otherwise ->+          unwords+            [ "The database is of version",+              show v <> ",",+              "which this haskoin-store cannot bring to version",+              show currentVersion <> ".",+              "Nothing was changed."+            ]+    UnknownVersion Nothing -> "The database has no version where one was expected. Nothing was changed."+    Inconsistent what -> "The database is not as its version says: " <> cs what+ -- | How to run a migration. data MigrationOptions = MigrationOptions-  { -- | mempool transactions to move in each write+  { -- | mempool transactions to take out in each write     chunk :: !Int,-    -- | called after each write with the number of transactions moved so-    -- far; the tests interrupt the migration from here+    -- | called after each write with the number of transactions taken out+    -- so far; the tests interrupt the migration from here     afterBatch :: Int -> IO (),     -- | where to say what is going on     report :: Text -> IO ()@@ -257,15 +299,34 @@ -- | Bring a database opened with -- 'Haskoin.Store.Database.Reader.withLayeredReader' to the layout that the -- store runs on, whatever it was, telling a function what is being done:--- a new database gets the current version, one of version 18 or 19 is--- first brought to version 20 as it always was, one of version 20 is--- migrated to the layered layout ('migrateLayered'), and one of version 21+-- a new database gets the current version, one of version 18, 19 or 20+-- is migrated to the layout of the chain alone ('migrateLayered'), with+-- its unconfirmed transactions taken out of the chain, one of version 21 -- gets the time of its block for every confirmed transaction--- ('migrateRecords'), each from the start or after an earlier run that was--- interrupted. Throws a 'MigrationError' for any other version.+-- ('migrateRecords'), and one of version 22 gets the current version, each+-- from the start or after an earlier run that was interrupted. Throws a+-- 'MigrationError' for any other version, that of a newer store among+-- them, without writing anything. -- -- This is the point of no return for a database of an older version: a -- store older than the layout refuses it afterwards.+--+-- = From version 22 to 23+--+-- Nothing that a database of version 22 holds changes: its balances and+-- its groups of spenders are the values that version 23 reads, and what+-- is new is the merge records that blocks will write from now on. So the+-- step is one write, of the version, made last: every state that an+-- earlier migration can be interrupted in is one of the versions before,+-- as it was, and is carried on from as it was.+--+-- What keeps an older store away from the merge records is not the+-- version, which it would only read after RocksDB had opened the database+-- for it and dropped the merge records of its write-ahead log. It is the+-- column family that 'Haskoin.Store.Database.Reader.withLayeredReader'+-- adds, which makes RocksDB refuse to open the database for a store that+-- does not know it ('Haskoin.Store.Database.Reader.layeredFamilyConfig').+-- That happens when the database is opened, before this function runs. upgradeDatabase :: (Text -> IO ()) -> DatabaseReader -> IO Migration upgradeDatabase report r = do   version <- retrieve r.db VersionKey@@ -275,31 +336,50 @@       writeBatch r.db [insertOp VersionKey currentVersion]       return nothing     Just v-      | v == 18 || v == 19 -> do-          report $ "Upgrading the database from version " <> cs (show v) <> " to " <> cs (show mixedVersion)-          initRocksDB r-          layered >>= times       | v == currentVersion && marker == Nothing -> return nothing+      -- no migration leaves its marker with this version+      | v >= currentVersion -> throwIO (UnknownVersion version)+      | v == recordsVersion && marker == Nothing -> merges nothing       -- the times were being set when the store last stopped-      | v == currentVersion && marker == Just layeredVersion -> times nothing-      | v == currentVersion -> throwIO (UnknownVersion version)-    Just (_ :: Word32) -> layered >>= times+      | v == recordsVersion && marker == Just layeredVersion -> times nothing >>= merges+      | v == recordsVersion -> throwIO (UnknownVersion version)+      -- versions 18 and 19 have the chain of version 20, and a mempool that+      -- the migration to the layered layout reads where it is+      | v == 18 || v == 19 -> do+          report $ "Upgrading the database from version " <> cs (show v)+          writeBatch r.db [insertOp VersionKey mixedVersion]+          layered >>= times >>= merges+    Just (_ :: Word32) -> layered >>= times >>= merges   where     nothing = Migration {moved = 0, batches = 0, checked = 0, rewritten = 0}     layered = migrateLayeredWith defaultMigrationOptions {report = report} r     times done = do       rewritten <- migrateRecords report r       return (done :: Migration) {rewritten}+    -- from 'recordsVersion', with no migration under way+    merges done = do+      report $+        "Upgrading the database from version "+          <> cs (show recordsVersion)+          <> " to "+          <> cs (show currentVersion)+          <> ": blocks will write merge records, which no older haskoin-store can read"+      writeBatch r.db [insertOp VersionKey currentVersion]+      return done  -- | Bring the records of the confirmed transactions of a database of -- version 21 to what version 22 writes, and make the database one of that -- version; or carry that on in a database where it was interrupted. Each -- record gets the time of its block, and the spenders of its outputs go -- from the record to keys of their own. Says how many records it wrote.--- Throws a 'MigrationError' for a database of another version, and for--- one with a confirmed transaction at a height where the main chain has--- no block.+-- Does nothing to a database that is past it, of version 22 or of the+-- current one. Throws a 'MigrationError' for a database of another+-- version, and for one with a confirmed transaction at a height where the+-- main chain has no block. --+-- The spenders are written as whole values, not as merge records: a+-- record is read here, and a transaction is taken once.+-- -- Every record is read, and nearly every one is written: the whole of the -- largest column family, which on a database of Bitcoin is many hours, -- and room on the disk for a second copy of it until RocksDB has@@ -318,11 +398,12 @@   if     | version == Just layeredVersion && marker == Nothing -> do         report "Making the record of every confirmed transaction one that is written once: this reads them all"-        writeBatch db [insertOp VersionKey currentVersion, insertOp MigrationKey layeredVersion]+        writeBatch db [insertOp VersionKey recordsVersion, insertOp MigrationKey layeredVersion]         run-    | version == Just currentVersion && marker == Just layeredVersion -> do+    | version == Just recordsVersion && marker == Just layeredVersion -> do         report "Carrying on with the records of confirmed transactions, from the first"         run+    | version == Just recordsVersion && marker == Nothing -> return 0     | version == Just currentVersion && marker == Nothing -> return 0     | otherwise -> throwIO (UnknownVersion version)   where@@ -412,7 +493,7 @@     -- a database that is past this migration, whatever came after it     | version == Just layeredVersion && marker == Nothing ->         return Migration {moved = 0, batches = 0, checked = 0, rewritten = 0}-    | version == Just currentVersion ->+    | version == Just recordsVersion || version == Just currentVersion ->         return Migration {moved = 0, batches = 0, checked = 0, rewritten = 0}     | version == Just layeredVersion && marker == Just mixedVersion -> do         opts.report "Carrying on with the migration to the layered layout"@@ -420,29 +501,35 @@     | version == Just mixedVersion -> do         listed <- mempoolList         mapM_ (preflight r) listed-        opts.report $ "Moving " <> cs (show (length listed)) <> " mempool transactions to their own area"+        opts.report $+          "Taking "+            <> cs (show (length listed))+            <> " mempool transactions out of the chain: the mempool is the pool that the fleet shares"         -- from here on a store of version 20 refuses the database         writeBatch db [insertOp VersionKey layeredVersion, insertOp MigrationKey mixedVersion]         run     | otherwise -> throwIO (UnknownVersion version)   where     db = r.db+    -- the keys of versions 19 and 20, one for each transaction, and the+    -- list of version 18 under one key of its own (@MemKey@, 0x07), both+    -- left where they are: read again after an interruption, they list the+    -- same transactions, and those already taken out are skipped     mempoolList = do       rows <- matchingAsListCF db (mempoolCF db) MemTxKeyS-      return [h | (MemTxKey h, _ :: UnixTime) <- rows]+      legacy <- RocksDB.get db (B.singleton 0x07)+      let listed = maybe [] (either (const []) id . decode) legacy :: [(UnixTime, TxHash)]+      return (nub' ([h | (MemTxKey h, _ :: UnixTime) <- rows] <> map snd listed))     run = do       env <- newEnv opts r       listed <- mempoolList       mapM_ (moveTx env) listed       flush env       mapM_ (verify r) listed-      -- the summaries are worked out from the entries, all at once: doing it-      -- again after an interruption gives the same-      void (rebuildSummaries r)       writeBatch db [deleteOp MigrationKey]       moved <- readIORef env.total       batches <- readIORef env.written-      opts.report $ "Moved " <> cs (show moved) <> " mempool transactions"+      opts.report $ "Took " <> cs (show moved) <> " mempool transactions out of the chain"       return Migration {moved, batches, checked = length listed, rewritten = 0}  -------------------------------------------------------------------------------@@ -463,15 +550,15 @@     -- | everything else that the batch writes, newest first: each of these     -- keys is written without being read, and in this order     ops :: !(IORef [BatchOp]),-    -- | transactions that the batch moves+    -- | transactions that the batch takes out     moving :: !(IORef (HashSet TxHash)),-    -- | transactions moved by this run+    -- | transactions taken out by this run     total :: !(IORef Int),     -- | batches written by this run     written :: !(IORef Int)   } --- | A migration that has moved nothing yet, with an empty batch.+-- | A migration that has taken nothing out yet, with an empty batch. newEnv :: MigrationOptions -> DatabaseReader -> IO Env newEnv options reader = do   bodies <- newIORef HashMap.empty@@ -535,20 +622,19 @@       | otherwise = insertOpCF (balanceCF db) (BalKey a) b  ---------------------------------------------------------------------------------- Moving a transaction+-- Taking a transaction out ------------------------------------------------------------------------------- --- | Whether a transaction is already in the mempool area.+-- | Whether a transaction is out of the chain's indices already: in this+-- batch, or in an earlier write, which left it no body. isMoved :: Env -> TxHash -> IO Bool isMoved env h = do   now <- HashSet.member h <$> readIORef env.moving-  if now then return True else isJust <$> getCF db (mempoolCF db) (encode (TxKey h))-  where-    db = env.reader.db+  if now then return True else isNothing <$> chainBody env h --- | Move a mempool transaction, and first the mempool transactions that--- spend its outputs: taking those out is what makes its outputs unspent--- again, as they must be for its own turn.+-- | Take a mempool transaction out of the chain's indices, and first the+-- mempool transactions that spend its outputs: taking those out is what+-- makes its outputs unspent again, as they must be for its own turn. moveTx :: Env -> TxHash -> IO () moveTx env h = do   done <- isMoved env h@@ -558,17 +644,15 @@     -- read again: moving the spenders removed them from the body     td' <- mempoolBody     unless (IntMap.null td'.spenders) . stop $ "still has spenders after they were moved"-    time <- case td'.block of-      MemRef t -> return t+    case td'.block of+      MemRef _ -> return ()       BlockRef {} -> stop "is confirmed"     takeOut env td'-    mapM_ (op env) (insertRecordOps db (mempoolRecords env.reader.net env.reader.ctx time td'))     modifyIORef' env.moving (HashSet.insert h)     modifyIORef' env.total (+ 1)     count <- HashSet.size <$> readIORef env.moving     when (count >= env.options.chunk) (flush env)   where-    db = env.reader.db     stop :: Text -> IO a     stop why = throwIO . Inconsistent $ "Mempool transaction " <> txHashToHex h <> " " <> why     mempoolBody =@@ -580,7 +664,7 @@  -- | Take a mempool transaction that nothing spends out of the indices of -- the chain, as deleting it does in version 20, except that its body is--- removed and not kept as deleted: it is not deleted, it moves.+-- removed and not kept as deleted. takeOut :: Env -> TxData -> IO () takeOut env td = do   -- the entries of its addresses, and their counts of transactions@@ -625,7 +709,7 @@   where     db = env.reader.db     h = txHash td.tx-    address = eitherToMaybe . scriptToAddressBS env.reader.net env.reader.ctx+    address = eitherToMaybe . scriptToAddressBS env.reader.net     touched =       nub' $         mapMaybe (address . (.script)) (IntMap.elems td.prevs)@@ -652,26 +736,13 @@       throwIO . Inconsistent $         "The mempool lists " <> txHashToHex h <> ", which is not an unconfirmed transaction" --- | After everything is moved: a mempool transaction is in the mempool--- area and has left nothing in the chain's column families, and every--- output it spends is there to be spent, in the mempool or unspent in the--- chain.+-- | After everything is taken out: a mempool transaction has left no body+-- in the chain's column families. Its outputs went with its body, in the+-- same batch. verify :: DatabaseReader -> TxHash -> IO () verify r h = do-  td <--    retrieveCF db (mempoolCF db) (TxKey h) >>= \case-      Just td -> return (td :: TxData)-      Nothing -> stop "is not in the mempool area"   inChain <- isJust <$> getCF db (txCF db) (encode (TxKey h))   when inChain $ stop "still has a body in the chain"-  forM_ (zip [0 ..] td.tx.outputs) $ \(i, _) -> do-    left <- isJust <$> getCF db (unspentCF db) (encode (UnspentKey (OutPoint h i)))-    when left $ stop "still has an unspent output in the chain"-  forM_ [i.outpoint | i <- td.tx.inputs, i.outpoint /= nullOutPoint] $ \p -> do-    inMempool <- isJust <$> getCF db (mempoolCF db) (encode (TxKey p.hash))-    unspent <- isJust <$> getCF db (unspentCF db) (encode (UnspentKey p))-    unless (inMempool || unspent) . void . stop $-      "spends an output of " <> txHashToHex p.hash <> " that is neither in the mempool nor unspent"   where     db = r.db     stop :: Text -> IO a
src/Haskoin/Store/Database/Query.hs view
@@ -9,11 +9,9 @@ -- Queries against the RocksDB database, as plain functions in 'IO' that take -- the 'DatabaseReader' as their first argument. ----- These read one layer. On a database of version 20 that is everything,--- the mempool included, and the tests use them so. On the layout that the--- store runs on, they give what the chain has and nothing of the mempool,--- but for its list: "Haskoin.Store.Database.Query.Layered" calls them for--- the chain's part of its answers, and is what the store answers with.+-- They read the chain, which is all that the database has: the mempool is+-- the pool that the fleet shares in Redis. "Haskoin.Store.Database.Query.Pooled"+-- puts the two together, and is what the store answers with. -- -- = Why plain functions --@@ -28,10 +26,7 @@ -- Streams are still conduits, run in 'IO': they are a good fit for merging the -- sorted indices of many addresses and reading only as far as a page needs. module Haskoin.Store.Database.Query-  ( -- * All of them-    databaseRead,--    -- * Blocks+  ( -- * Blocks     getBestBlock,     getBlocksAtHeight,     getBlock,@@ -59,11 +54,6 @@     getAddressUnspents,     getAddressesUnspents, -    -- * Mempool-    getMempool,-    getMempoolPage,-    getMempoolBefore,-     -- * Extended public keys     xPubBals,     xPubUnspents,@@ -73,13 +63,15 @@      -- * Building blocks -    -- | What the queries above are made of, for the queries of the layered-    -- layout in "Haskoin.Store.Database.Query.Layered", which read the same-    -- column families for the part of their answers that is in the chain.+    -- | What the queries above are made of, for the queries of+    -- "Haskoin.Store.Database.Query.Pooled", which read the same column+    -- families for the part of their answers that is in the chain.     count,     withManyIters,     retrieveManyCF,     retrieveManyBytesCF,+    scan,+    scanFrom,     joinConduits,     usedAddresses,   )@@ -88,23 +80,23 @@ import Conduit   ( ConduitT,     concatC,-    concatMapC,     dropWhileC,     lift,+    liftIO,     mapC,     runConduit,     sinkList,-    takeC,-    takeWhileC,     yield,     (.|),   )-import Control.Monad (forM, (>=>))+import Control.Exception (mask_)+import Control.Monad (forM, forM_, when, (>=>)) import Data.Bits ((.&.)) import Data.ByteString (ByteString) import Data.ByteString qualified as BS import Data.Default (def) import Data.Functor ((<&>))+import Data.IORef (modifyIORef', newIORef, readIORef) import Data.IntMap.Strict qualified as IntMap import Data.Map.Strict qualified as Map import Data.Maybe (catMaybes, listToMaybe)@@ -112,27 +104,20 @@ import Data.Serialize (Serialize, decode, encode) import Data.Word (Word32, Word64) import Database.RocksDB-  ( ColumnFamily,-    DB,+  ( ChunkConfig (..),+    ColumnFamily,     IterConfig (..),     Iterator,     createIteratorConfig,     getCF,-    iterKey,-    iterLast,-    iterNext,-    iterPrev,+    iterChunk,     iterSeek,-    iterValid,     multiGetCF,     prefixUpperBound,-    withIterCF,     withIterConfig,   ) import Database.RocksDB.Query-  ( matching,-    matchingAsListCF,-    matchingSkip,+  ( matchingAsListCF,     retrieve,     retrieveCF,   )@@ -148,12 +133,10 @@   ( Limits (..),     Replaced,     Start (..),-    StoreRead (..),     applyLimitsC,     deOffset,     joinDescStreams,     mergeDescGroups,-    notDeleted,     scanXPubBals,     txRefKey,     unspentKey,@@ -162,53 +145,9 @@ import Haskoin.Store.Database.Reader import Haskoin.Store.Database.Slices (blockData, txBytes) import Haskoin.Store.Database.Types+import Haskoin.Store.Exception (bracket, bracket_) import System.Metrics.StatsD (decrementGauge, incrementCounter, incrementGauge)-import UnliftIO-  ( bracket,-    bracket_,-    mask_,-    modifyIORef',-    newIORef,-    readIORef,-  ) --- | The queries of a database as a 'StoreRead', for code that takes its--- queries as a value, such as the web server, which puts the Redis cache in--- front of some of them. Transactions that were deleted are left out--- ('notDeleted'); the functions of this module still return them, for the--- code that imports transactions.-databaseRead :: DatabaseReader -> StoreRead-databaseRead r =-  StoreRead-    { net = r.net,-      ctx = r.ctx,-      getBestBlock = getBestBlock r,-      getBlocksAtHeight = getBlocksAtHeight r,-      getBlock = getBlock r,-      getTxData = fmap notDeleted . getTxData r,-      getManyTxData = fmap (map notDeleted) . getManyTxData r,-      getManyTxBytes = getManyTxBytes r,-      getSpender = getSpender r,-      getBalance = getBalance r,-      getUnspent = getUnspent r,-      getMempool = getMempool r,-      getMempoolPage = getMempoolPage r,-      getAddressesTxs = getAddressesTxs r,-      getAddressesUnspents = getAddressesUnspents r,-      initialGap = r.initGap,-      maxGap = r.maxGap,-      getNumTxData = fmap (filter (not . (.deleted))) . getNumTxData r,-      getBalances = getBalances r,-      getAddressTxs = getAddressTxs r,-      getAddressUnspents = getAddressUnspents r,-      xPubBals = xPubBals r,-      xPubUnspents = \_ -> xPubUnspents r,-      xPubTxs = \_ -> xPubTxs r,-      xPubTxCount = \_ -> xPubTxCount r,-      getAddressesTxCount = getAddressesTxCount r,-      getReplaced = getReplaced r-    }- -- | Count something in the metrics, if they are enabled. count :: DatabaseReader -> (DataMetrics -> IO ()) -> IO () count r f = mapM_ f r.metrics@@ -279,7 +218,7 @@ -- multi-get, with the inputs of blocks that spend their outputs. getManyTxData :: DatabaseReader -> [TxHash] -> IO [Maybe TxData] getManyTxData r ths = do-  found <- retrieveManyCF r.db (txCF r.db) (map TxKey ths)+  found <- retrieveManyCF r (txCF r.db) (map TxKey ths)   count r $ \s -> incrementCounter s.dataTxCount (length (catMaybes found))   spent <- withChainSpenders r [(h, td) | (h, Just td) <- zip ths found]   return (fill found spent)@@ -293,7 +232,7 @@ -- without the spenders of their outputs. getManyTxBytes :: DatabaseReader -> [TxHash] -> IO [Maybe ByteString] getManyTxBytes r ths = do-  found <- map (>>= liveTxBytes) <$> retrieveManyBytesCF r.db (txCF r.db) (map TxKey ths)+  found <- map (>>= liveTxBytes) <$> retrieveManyBytesCF r (txCF r.db) (map TxKey ths)   count r $ \s -> incrementCounter s.dataTxCount (length (catMaybes found))   return found @@ -344,7 +283,7 @@       asked = concatMap keys (filter narrow tds)   found <- case asked of     [] -> return []-    _ -> retrieveManyCF r.db (spenderCF r.db) asked+    _ -> retrieveManyCF r (spenderCF r.db) asked   let byKey = Map.fromList [(k, m) | (k, Just (SpenderVal m)) <- zip asked found]       keyed x = IntMap.unions [m | k <- keys x, Just m <- [Map.lookup k byKey]]   forM tds $ \x@(h, td) -> do@@ -399,7 +338,7 @@ getBalances _ [] = return [] getBalances r as = do   count r $ \s -> incrementCounter s.dataBalanceCount (length as)-  vals <- retrieveManyCF r.db (balanceCF r.db) (map BalKey as)+  vals <- retrieveManyCF r (balanceCF r.db) (map BalKey as)   return (zipWith f as vals)   where     f a Nothing = zeroBalance a@@ -412,7 +351,7 @@     Nothing -> return Nothing     Just val -> do       count r $ \s -> incrementCounter s.dataUnspentCount 1-      return (Just (valToUnspent r.net r.ctx p val))+      return (Just (valToUnspent r.net p val))  -- | Transactions of an address, newest first. getAddressTxs :: DatabaseReader -> Address -> Limits -> IO [TxRef]@@ -467,20 +406,20 @@     f _ _ = undefined     pipe = case s of       Nothing ->-        matching it (AddrTxKeyA a)+        scan it (AddrTxKeyA a)       Just (AtBlock bh) ->-        matchingSkip it (AddrTxKeyA a) (AddrTxKeyB a (BlockRef bh maxBound))+        scanFrom it (AddrTxKeyA a) (AddrTxKeyB a (BlockRef bh maxBound))       Just (AtTx txh) ->         lift (getTxRecord r txh) >>= \case           Just TxData {block = b@BlockRef {}} ->-            matchingSkip it (AddrTxKeyA a) (AddrTxKeyB a b)+            scanFrom it (AddrTxKeyA a) (AddrTxKeyB a b)           Just TxData {block = b@MemRef {}} ->             -- Mempool transactions that arrived in the same second are             -- keyed by hash, so seek to the time and drop those before it.             let start = txRefKey (TxRef b txh)                 before (AddrTxKey _ t, ()) = txRefKey t > start                 before _ = undefined-             in matchingSkip it (AddrTxKeyA a) (AddrTxKeyB a b)+             in scanFrom it (AddrTxKeyA a) (AddrTxKeyB a b)                   .| (dropWhileC before >> mapC id)           Nothing -> return () @@ -493,84 +432,28 @@   Iterator ->   ConduitT i Unspent IO () unspentConduit r a s it =-  x .| mapC (uncurry (toUnspent r.net r.ctx)) .| concatC+  x .| mapC (uncurry (toUnspent r.net)) .| concatC   where     x = case s of       Nothing ->-        matching it (AddrOutKeyA a)+        scan it (AddrOutKeyA a)       Just (AtBlock h) ->-        matchingSkip it (AddrOutKeyA a) (AddrOutKeyB a (BlockRef h maxBound))+        scanFrom it (AddrOutKeyA a) (AddrOutKeyB a (BlockRef h maxBound))       Just (AtTx txh) ->         lift (getTxRecord r txh) >>= \case           Just TxData {block = b@BlockRef {}} ->-            matchingSkip it (AddrOutKeyA a) (AddrOutKeyB a b)+            scanFrom it (AddrOutKeyA a) (AddrOutKeyB a b)           Just TxData {block = b@MemRef {}} ->             -- The outputs of the start transaction come first among those             -- of the same second whose hashes are not smaller.             let start = (b, Down (encode (OutPoint txh 0)))                 before (AddrOutKey _ ref p, _) = (ref, Down (encode p)) > start                 before _ = undefined-             in matchingSkip it (AddrOutKeyA a) (AddrOutKeyB a b)+             in scanFrom it (AddrOutKeyA a) (AddrOutKeyB a b)                   .| (dropWhileC before >> mapC id)           Nothing -> return ()  ---------------------------------------------------------------------------------- Mempool------------------------------------------------------------------------------------ | The whole mempool, newest first.-getMempool :: DatabaseReader -> IO [(UnixTime, TxHash)]-getMempool r = getMempoolPage r def---- | Page of the mempool, newest first, from the offset and limit of the--- 'Limits'; its start is ignored. Reads only the entries of the page from--- the index by time.-getMempoolPage :: DatabaseReader -> Limits -> IO [(UnixTime, TxHash)]-getMempoolPage r limits = do-  count r $ \s -> incrementCounter s.dataMempoolCount 1-  withIterCF r.db (mempoolCF r.db) $ \it ->-    runConduit $-      mempoolNewestC it-        .| applyLimitsC limits-        .| sinkList---- | Mempool transactions that entered it before the given time, oldest--- first, at most as many as given. Reads only those entries of the index.-getMempoolBefore :: DatabaseReader -> UnixTime -> Int -> IO [(UnixTime, TxHash)]-getMempoolBefore r time n =-  withIterCF r.db (mempoolCF r.db) $ \it ->-    runConduit $-      matching it MemTimeKeyS-        .| concatMapC entry-        .| takeWhileC ((< time) . fst)-        .| takeC n-        .| sinkList-  where-    entry (MemTimeKey t h, ()) = Just (t, h)-    entry (MemTimeKeyS, ()) = Nothing---- | Mempool from the index by time, newest first. The index sorts last in--- the mempool column family, so seek past its prefix and walk backwards--- until a key is no longer in it.-mempoolNewestC :: Iterator -> ConduitT i (UnixTime, TxHash) IO ()-mempoolNewestC it = do-  lift $ do-    -- 0x0c is the first prefix after the 0x0b of 'MemTimeKey'-    iterSeek it (BS.singleton 0x0c)-    iterValid it >>= \case-      True -> iterPrev it-      False -> iterLast it-  go-  where-    go =-      lift (iterKey it) >>= \case-        Just k | Right (MemTimeKey t h) <- decode k -> do-          yield (t, h)-          lift (iterPrev it)-          go-        _ -> return ()--------------------------------------------------------------------------------- -- Extended public keys ------------------------------------------------------------------------------- @@ -644,23 +527,45 @@ usedAddresses :: [XPubBal] -> [Address] usedAddresses = map (.address) . filter (not . nullBalance) . map (.balance) +-- | The keys of one prefix that an iterator walks, with those already read+-- from it: 'countDistinctSuffixes' takes them one at a time, and the+-- iterator gives them a few hundred at a time.+data Suffixes = Suffixes+  { prefix :: !BS.ByteString,+    iterator :: !Iterator,+    -- | keys read and not yet taken, without their prefix+    ahead :: ![BS.ByteString],+    -- | whether the iterator has more of them+    more :: !Bool+  }+ -- | Number of distinct keys, after their prefix, in a set of iterators that -- each walk the keys with one prefix, in order.+--+-- The keys are read in chunks, one foreign call for a few hundred of them.+-- Read one at a time, each key cost the thread its core for the call and a+-- wait to get one back, which on a busy server was most of the time of+-- counting the transactions of a large wallet. countDistinctSuffixes :: [(BS.ByteString, Iterator)] -> IO Int countDistinctSuffixes streams = do   heads <- mapM start streams   go 0 (Map.fromListWith (<>) [(k, [x]) | (k, x) <- catMaybes heads])   where-    start x@(pfx, it) = iterSeek it pfx >> current x-    current x@(pfx, it) =-      iterKey it <&> \case-        Just k | pfx `BS.isPrefixOf` k -> Just (BS.drop (BS.length pfx) k, x)-        _ -> Nothing-    advance x@(_, it) = iterNext it >> current x+    start (pfx, it) = iterSeek it pfx >> next (Suffixes pfx it [] True)+    -- the next key of a stream, and the stream after it+    next s = case s.ahead of+      k : ks -> return (Just (k, s {ahead = ks} :: Suffixes))+      []+        | s.more -> do+            (rows, more) <- iterChunk s.iterator def {chunkPrefix = s.prefix}+            case map (BS.drop (BS.length s.prefix) . fst) rows of+              [] -> return Nothing+              k : ks -> return (Just (k, s {ahead = ks, more = more} :: Suffixes))+        | otherwise -> return Nothing     go !n m = case Map.minViewWithKey m of       Nothing -> return n       Just ((_, xs), m') -> do-        nexts <- catMaybes <$> mapM advance xs+        nexts <- catMaybes <$> mapM next xs         go (n + 1) (foldr (\(k, x) -> Map.insertWith (<>) k [x]) m' nexts)  -------------------------------------------------------------------------------@@ -682,9 +587,8 @@ -- A scan that stops by looking at the key that it has reached makes -- RocksDB find that key first, and if the keys after the prefix were -- deleted, it steps over the marker of every one of them to get there.--- With an upper bound it stops at the bound. The summary of the mempool--- area and the layout of the chain keep such runs of deleted keys short as--- things are, so this is for the day that they do not: after a+-- With an upper bound it stops at the bound. The layout of the chain keeps+-- such runs of deleted keys short as things are, so this is for the day that they do not: after a -- reorganisation that reverts many blocks, or a change of layout. scanOf :: (Serialize k) => k -> IterConfig scanOf prefix = def {upperBound = prefixUpperBound (encode prefix)}@@ -721,15 +625,43 @@ -- set up and a third of one for each key, as measured, so it is the slower -- way to read one key, and most requests about one transaction or one -- address ask for one.-retrieveManyCF :: (Serialize key, Serialize value) => DB -> ColumnFamily -> [key] -> IO [Maybe value]-retrieveManyCF db cf keys = map (fmap (either error id . decode)) <$> retrieveManyBytesCF db cf keys+retrieveManyCF :: (Serialize key, Serialize value) => DatabaseReader -> ColumnFamily -> [key] -> IO [Maybe value]+retrieveManyCF r cf keys = map (fmap (either error id . decode)) <$> retrieveManyBytesCF r cf keys  -- | The bytes of the values of many keys, as 'retrieveManyCF' reads them, -- for values that are read in part ("Haskoin.Store.Database.Slices").-retrieveManyBytesCF :: (Serialize key) => DB -> ColumnFamily -> [key] -> IO [Maybe ByteString]+retrieveManyBytesCF :: (Serialize key) => DatabaseReader -> ColumnFamily -> [key] -> IO [Maybe ByteString] retrieveManyBytesCF _ _ [] = return []-retrieveManyBytesCF db cf [key] = pure <$> getCF db cf (encode key)-retrieveManyBytesCF db cf keys = multiGetCF db cf (map encode keys)+retrieveManyBytesCF r cf [key] = pure <$> getCF r.db cf (encode key)+retrieveManyBytesCF r cf keys = multiGetCF r.db cf (map encode keys)++-- | The entries of the keys that start with a prefix, read in chunks as+-- the consumer asks for them: what 'Database.RocksDB.Query.matching' does,+-- with each chunk read in a safe call into RocksDB.+-- The chunks start at 16 entries, so that a consumer that takes a few+-- makes the database read few, and grow four times at each call up to+-- 256, and to a mebibyte at most.+scan :: (Serialize key, Serialize value) => Iterator -> key -> ConduitT i (key, value) IO ()+scan it base = scanFrom it base base++-- | Like 'scan', but from the second key, or after it if it has no entry:+-- for the pages after the first, which seek straight to where they start.+scanFrom ::+  (Serialize key, Serialize value) =>+  Iterator ->+  key ->+  key ->+  ConduitT i (key, value) IO ()+scanFrom it base start = do+  liftIO (iterSeek it (encode start))+  go 16+  where+    prefix = encode base+    go n = do+      let chunk = def {chunkPrefix = prefix, chunkEntries = n, chunkBytes = 1_048_576}+      (entries, more) <- liftIO (iterChunk it chunk)+      forM_ entries $ \(k, v) -> yield (either error id (decode k), either error id (decode v))+      when more (go (min 256 (n * 4)))  -- | Merge streams sorted newest first, drop duplicates, and apply the offset -- and limit.
− src/Haskoin/Store/Database/Query/Layered.hs
@@ -1,827 +0,0 @@--- |--- Module      : Haskoin.Store.Database.Query.Layered--- Copyright   : (c) Haskoin Developers--- License     : MIT--- Maintainer  : jprupp@protonmail.ch--- Stability   : experimental--- Portability : POSIX------ The queries of "Haskoin.Store.Database.Query" for the layered layout of--- the database, in which the chain and the mempool are kept apart; see--- @docs\/layered-database.md@. These are the queries that the store--- answers with. Each answers as the query of the same name answers on a--- database of version 20, pages included. The tests check that, on--- databases migrated from that version and on databases built by the rules--- of "Haskoin.Store.Logic.Layered".------ = Two sources------ A query reads the column families of the chain, with the functions of--- "Haskoin.Store.Database.Query", reads what the mempool area has for the--- same transaction, output or address, with those of--- "Haskoin.Store.Database.Mempool", and puts the two together.------ = Why a view------ The two reads must see the database at the same moment. A block could--- confirm a transaction between them. Read from the mempool before the block--- and from the chain after it, the transaction would be listed twice, and--- its payment counted twice in a balance; read the other way around, it--- would be missing. The queries of version 20 need no such care, because each of their--- answers is one key or one scan.------ So every query takes a t'View', a snapshot of the database, and 'withView'--- is the only way to get one: a query cannot be given the database itself by--- mistake. The web server is meant to take one view for each request. That--- also makes the several queries of a request agree with each other, which--- they did not have to in version 20.------ = Pages------ Lists are ordered newest first, and unconfirmed transactions are newer--- than any block. The list of an address is therefore what the mempool has--- for it followed by what the chain has, and that is the sequence that the--- single index of version 20 holds. A page is cut from it by the same--- rules: where it starts, then the offset, then the limit. Both parts are--- streamed, each from its own index, only as far as the page needs.------ = Reading no more than the answer------ Nearly every address has nothing in the mempool, and a few have thousands--- of entries there. Each address that the mempool touches has a summary of--- its entries, a t'MemSummary', which every query about an address reads--- first, by key. A balance is the chain's balance with the summary applied.--- A list reads the entries only of the addresses that have a summary, and--- the summary says how many outputs the mempool spends, so that no more are--- looked for. The first version of these queries scanned the entries of--- every address for every question, which measured ten thousand times--- slower for a busy address, and a hundred times slower for any address--- after a block, when the scan steps over all that the block deleted.-module Haskoin.Store.Database.Query.Layered-  ( -- * A view of the database-    View,-    withView,-    layeredRead,-    standaloneRead,-    viewReader,--    -- * Parts shared with the queries of a fleet-    From (..),-    chainTxsC,-    chainUnspentsC,-    withSpenders,--    -- * Blocks-    getBestBlock,-    getBlocksAtHeight,-    getBlock,--    -- * Transactions-    getTxData,-    getManyTxData,-    getManyTxBytes,-    getNumTxData,-    getSpender,--    -- * Addresses-    getBalance,-    getBalances,-    getUnspent,-    getAddressTxs,-    getAddressesTxs,-    getAddressUnspents,-    getAddressesUnspents,--    -- * Mempool-    getMempool,-    getMempoolPage,--    -- * Extended public keys-    xPubBals,-    xPubUnspents,-    xPubTxs,-    xPubTxCount,-    getAddressesTxCount,-  )-where--import Conduit-  ( ConduitT,-    await,-    awaitForever,-    concatMapC,-    lift,-    mapC,-    runConduit,-    sinkList,-    yield,-    yieldMany,-    (.|),-  )-import Control.Monad (forM)-import Data.Bits ((.&.))-import Control.Applicative ((<|>))-import Data.ByteString qualified as BS-import Data.Functor ((<&>))-import Data.HashMap.Strict (HashMap)-import Data.HashMap.Strict qualified as HashMap-import Data.HashSet qualified as HashSet-import Data.IntMap.Strict (IntMap)-import Data.IntMap.Strict qualified as IntMap-import Data.List (partition)-import Data.Map.Strict (Map)-import Data.Map.Strict qualified as Map-import Data.Maybe (catMaybes, isJust)-import Data.Serialize (Serialize, encode)-import Data.Word (Word32, Word64)-import Database.RocksDB (ColumnFamily, Iterator, multiGetCF, withSnapshot)-import Database.RocksDB.Query-  ( matching,-    matchingAsListCF,-    matchingSkip,-    retrieveCF,-  )-import Haskoin-  ( Address,-    BlockHash,-    BlockHeight,-    OutPoint (..),-    Tx (..),-    TxHash,-    TxOut (..),-    eitherToMaybe,-    scriptToAddressBS,-  )-import Haskoin.Store.Common-  ( Limits (..),-    Start (..),-    StoreRead (..),-    applyLimitsC,-    deOffset,-    mergeDescGroups,-    notDeleted,-    scanXPubBals,-    txRefKey,-    unspentKey,-  )-import Haskoin.Store.Data-  ( Balance (..),-    BlockData,-    BlockRef (..),-    Spender,-    TxData (..),-    TxRef (..),-    UnixTime,-    Unspent (..),-    XPubBal (..),-    XPubSpec,-    XPubUnspent (..),-  )-import Haskoin.Store.Database.Mempool qualified as M-import Haskoin.Store.Database.Query qualified as Q-import Haskoin.Store.Database.Reader-  ( DataMetrics (..),-    DatabaseReader (..),-    Derived (..),-    addrOutCF,-    addrTxCF,-    derivedAddrs,-    mempoolAddrCF,-    mempoolCF,-    txCF,-  )-import Haskoin.Store.Database.Types-  ( AddrOutKey (..),-    AddrTxKey (..),-    MemAddrKey (..),-    MemAddrVal (..),-    MemSpenderKey (..),-    MemSummary (..),-    OutVal,-    TxKey (..),-    decodeTxKey,-    toUnspent,-  )-import System.Metrics.StatsD (incrementCounter)------------------------------------------------------------------------------------ A view of the database------------------------------------------------------------------------------------ | A database of the layered layout as it was at one moment: both of its--- areas, read through one RocksDB snapshot. Whatever is written after the--- view was taken, the queries on it go on answering from that moment.-newtype View = View DatabaseReader---- | The reader of a view: the database as it was at the moment of the--- view. The queries of "Haskoin.Store.Database.Query" on it give what the--- chain had then.-viewReader :: View -> DatabaseReader-viewReader (View r) = r---- | Run an action with a view of a database as it is now.------ Taking a view costs about a microsecond, so there is no reason to share--- one between requests, and a reason not to: RocksDB keeps the data that a--- view can see for as long as the view is open.------ The view, and anything made from it, such as the queries of 'layeredRead',--- must not be used after the action returns: the snapshot is released then,--- and RocksDB would be handed a pointer to freed memory. An action that--- gives back something lazy or something that reads later, such as the body--- of a streamed response, has to run inside.-withView :: DatabaseReader -> (View -> IO a) -> IO a-withView r action = withSnapshot r.db $ \snapshot -> action (View r {db = snapshot})---- | The queries of a view as a t'StoreRead', for the web server, as--- 'Q.databaseRead' gives those of a database of version 20.-layeredRead :: View -> StoreRead-layeredRead v@(View r) =-  StoreRead-    { net = r.net,-      ctx = r.ctx,-      getBestBlock = getBestBlock v,-      getBlocksAtHeight = getBlocksAtHeight v,-      getBlock = getBlock v,-      getTxData = fmap notDeleted . getTxData v,-      getManyTxData = fmap (map notDeleted) . getManyTxData v,-      getManyTxBytes = getManyTxBytes v,-      getSpender = getSpender v,-      getBalance = getBalance v,-      getUnspent = getUnspent v,-      getMempool = getMempool v,-      getMempoolPage = getMempoolPage v,-      getAddressesTxs = getAddressesTxs v,-      getAddressesUnspents = getAddressesUnspents v,-      initialGap = r.initGap,-      maxGap = r.maxGap,-      getNumTxData = fmap (filter (not . (.deleted))) . getNumTxData v,-      getBalances = getBalances v,-      getAddressTxs = getAddressTxs v,-      getAddressUnspents = getAddressUnspents v,-      xPubBals = xPubBals v,-      xPubUnspents = \_ -> xPubUnspents v,-      xPubTxs = \_ -> xPubTxs v,-      xPubTxCount = \_ -> xPubTxCount v,-      getAddressesTxCount = getAddressesTxCount v,-      getReplaced = Q.getReplaced r-    }---- | The queries of a database as a t'StoreRead', each on a view of its--- own, taken when it is asked. For code that asks one thing at a time and--- has no request to hold a view for. Queries asked one after the other may--- see different moments, so what needs several of them to agree takes a--- view with 'withView'.-standaloneRead :: DatabaseReader -> StoreRead-standaloneRead r =-  StoreRead-    { net = r.net,-      ctx = r.ctx,-      getBestBlock = one (.getBestBlock),-      getBlocksAtHeight = \h -> one (\q -> q.getBlocksAtHeight h),-      getBlock = \h -> one (\q -> q.getBlock h),-      getTxData = \h -> one (\q -> q.getTxData h),-      getManyTxData = \hs -> one (\q -> q.getManyTxData hs),-      getManyTxBytes = \hs -> one (\q -> q.getManyTxBytes hs),-      getSpender = \p -> one (\q -> q.getSpender p),-      getBalance = \a -> one (\q -> q.getBalance a),-      getUnspent = \p -> one (\q -> q.getUnspent p),-      getMempool = one (.getMempool),-      getMempoolPage = \l -> one (\q -> q.getMempoolPage l),-      getAddressesTxs = \as l -> one (\q -> q.getAddressesTxs as l),-      getAddressesUnspents = \as l -> one (\q -> q.getAddressesUnspents as l),-      initialGap = r.initGap,-      maxGap = r.maxGap,-      getNumTxData = \n -> one (\q -> q.getNumTxData n),-      getBalances = \as -> one (\q -> q.getBalances as),-      getAddressTxs = \a l -> one (\q -> q.getAddressTxs a l),-      getAddressUnspents = \a l -> one (\q -> q.getAddressUnspents a l),-      xPubBals = \x -> one (\q -> q.xPubBals x),-      xPubUnspents = \x bs l -> one (\q -> q.xPubUnspents x bs l),-      xPubTxs = \x bs l -> one (\q -> q.xPubTxs x bs l),-      xPubTxCount = \x bs -> one (\q -> q.xPubTxCount x bs),-      getAddressesTxCount = \as -> one (\q -> q.getAddressesTxCount as),-      getReplaced = \h -> one (\q -> q.getReplaced h)-    }-  where-    one :: (StoreRead -> IO a) -> IO a-    one f = withView r (f . layeredRead)------------------------------------------------------------------------------------ Blocks------------------------------------------------------------------------------------ | Hash of the best block, if the database has any.-getBestBlock :: View -> IO (Maybe BlockHash)-getBestBlock (View r) = Q.getBestBlock r---- | Hashes of the blocks at a height, of the main chain and of side chains.-getBlocksAtHeight :: View -> BlockHeight -> IO [BlockHash]-getBlocksAtHeight (View r) = Q.getBlocksAtHeight r---- | Data of a block.-getBlock :: View -> BlockHash -> IO (Maybe BlockData)-getBlock (View r) = Q.getBlock r------------------------------------------------------------------------------------ Transactions------------------------------------------------------------------------------------ | Data of a transaction, with the spenders of its outputs, those in blocks--- and those in the mempool. A transaction in the mempool comes from the--- mempool area, and a confirmed one from the chain. One that left the--- mempool unconfirmed is not known: nothing is kept of it, and the bodies--- that version 20 left in the chain for such transactions are passed over.-getTxData :: View -> TxHash -> IO (Maybe TxData)-getTxData v h =-  getManyTxData v [h] <&> \case-    [found] -> found-    _ -> Nothing---- | Data of many transactions, in the order of the hashes. Each column--- family that has to be read is read with one multi-get.-getManyTxData :: View -> [TxHash] -> IO [Maybe TxData]-getManyTxData (View r) hs = do-  found <- bodies r hs-  spent <- mempoolSpenders r (Map.toList found)-  let tds = [withSpenders spent h <$> Map.lookup h found | h <- hs]-  Q.count r $ \s -> incrementCounter s.dataTxCount (length (catMaybes tds))-  return tds---- | Transactions as stored, in the format of the network, from the mempool--- area or the chain, as 'getManyTxData' finds them, without the deleted--- ones and without reading the spenders of their outputs.-getManyTxBytes :: View -> [TxHash] -> IO [Maybe BS.ByteString]-getManyTxBytes (View r) hs = do-  unconfirmed <- look (mempoolCF r.db) hs-  chain <- look (txCF r.db) (filter (`Map.notMember` unconfirmed) hs)-  let found = [Map.lookup h unconfirmed <|> Map.lookup h chain >>= Q.liveTxBytes | h <- hs]-  Q.count r $ \s -> incrementCounter s.dataTxCount (length (catMaybes found))-  return found-  where-    look _ [] = return Map.empty-    look cf ks = Map.fromList . catMaybes . zipWith (fmap . (,)) ks <$> Q.retrieveManyBytesCF r.db cf (map TxKey ks)---- | Transactions whose hash, read as a number, matches a short identifier of--- the Blockchain.info API, in the order of their hashes. There may be more--- than one.-getNumTxData :: View -> Word64 -> IO [TxData]-getNumTxData (View r) i = do-  unconfirmed <- matchingAsListCF r.db (mempoolCF r.db) (TxKeyS prefix)-  stored <- matchingAsListCF r.db (txCF r.db) (TxKeyS prefix)-  let rows = [(h, td) | (TxKey h, td) <- stored]-  chain <- zip (map fst rows) <$> Q.withChainSpenders r rows-  let found =-        preferred-          (Map.fromList [(h, td) | (TxKey h, td) <- unconfirmed])-          (Map.fromList chain)-      matches = filter (seventh . fst) (Map.toAscList found)-  spent <- mempoolSpenders r matches-  Q.count r $ \s -> incrementCounter s.dataTxCount (length matches)-  return [withSpenders spent h td | (h, td) <- matches]-  where-    (prefix, bits) = decodeTxKey i-    -- the keys match on the first six bytes of the hash; the identifier-    -- also has the five high bits of the seventh-    seventh h = BS.index (encode h) 6 .&. 0xf8 == bits---- | Input that spends an output, in the mempool or in a block.-getSpender :: View -> OutPoint -> IO (Maybe Spender)-getSpender (View r) p =-  retrieveCF r.db (mempoolCF r.db) (MemSpenderKey p) >>= \case-    Just s -> return (Just s)-    Nothing -> Q.getChainSpender r p---- | The bodies of transactions by hash, each from where it is kept, with--- the spenders that blocks give to its outputs and without those of the--- mempool.------ The mempool area is asked first, although most transactions asked for are--- confirmed: with its filter, finding that a transaction is not there costs--- a fraction of reading a body, and the chain is then asked only for the--- others.-bodies :: DatabaseReader -> [TxHash] -> IO (Map TxHash TxData)-bodies r hs = do-  unconfirmed <- look (mempoolCF r.db) TxKey hs-  chain <- look (txCF r.db) TxKey (filter (`Map.notMember` unconfirmed) hs) >>= spent-  return (preferred unconfirmed chain)-  where-    -- the spenders that blocks give to the outputs of confirmed-    -- transactions are under keys of their own-    spent found = do-      let rows = Map.toAscList found-      Map.fromDistinctAscList . zip (map fst rows) <$> Q.withChainSpenders r rows-    look :: (Serialize key) => ColumnFamily -> (TxHash -> key) -> [TxHash] -> IO (Map TxHash TxData)-    look _ _ [] = return Map.empty-    look cf key ks =-      Map.fromList . catMaybes . zipWith (fmap . (,)) ks-        <$> Q.retrieveManyCF r.db cf (map key ks)---- | The body to answer with for each transaction, given the bodies that the--- mempool area has for transactions in the mempool and those that the--- chain has. A transaction is in one place only. A body that version 20--- left in the chain for a transaction that it deleted is no body at all:--- finding them all to delete them would mean reading every transaction of--- the database, so they stay where they are and are passed over.-preferred :: Map TxHash TxData -> Map TxHash TxData -> Map TxHash TxData-preferred unconfirmed chain = unconfirmed <> Map.filter (not . (.deleted)) chain---- | The spenders that the mempool has for the outputs of transactions,--- given their bodies as 'bodies' gives them.------ Only the outputs that no block spends are looked up, since the mempool--- cannot spend the others: all of them with one multi-get, which the filter--- of the column family answers for the many outputs that the mempool does--- not spend. A transaction with more than 'scanFrom' such outputs has its--- spenders read with a scan of their common prefix instead.-mempoolSpenders :: DatabaseReader -> [(TxHash, TxData)] -> IO (HashMap TxHash (IntMap Spender))-mempoolSpenders r txs = do-  found <- case points of-    [] -> return []-    _ -> Q.retrieveManyCF r.db (mempoolCF r.db) (map MemSpenderKey points)-  scanned <- mapM (\(h, _) -> (h,) <$> M.getMempoolSpenders r h) wide-  return . HashMap.fromListWith IntMap.union $-    [(p.hash, IntMap.singleton (fromIntegral p.index) s) | (p, Just s) <- zip points found]-      <> scanned-  where-    open :: [(TxHash, [Word32])]-    open =-      [ (h, [i | (i, _) <- zip [0 ..] td.tx.outputs, IntMap.notMember (fromIntegral i) td.spenders])-      | (h, td) <- txs,-        not td.deleted-      ]-    (narrow, wide) = partition (null . drop scanFrom . snd) open-    points = [OutPoint h i | (h, is) <- narrow, i <- is]---- | Number of outputs of a transaction above which the mempool spenders of--- the transaction are read with a scan. A scan costs about as much as eight--- lookups in a multi-get whatever it finds, by the measurements in--- @docs\/layered-database.md@.-scanFrom :: Int-scanFrom = 8---- | Add the spenders that the mempool has for the outputs of a transaction--- to those in its body.-withSpenders :: HashMap TxHash (IntMap Spender) -> TxHash -> TxData -> TxData-withSpenders spent h td = case HashMap.lookup h spent of-  Nothing -> td-  Just unconfirmed -> (td :: TxData) {spenders = IntMap.union td.spenders unconfirmed}------------------------------------------------------------------------------------ Addresses------------------------------------------------------------------------------------ | Balance of an address, with the mempool applied. 'Nothing' if neither--- the chain nor the mempool has anything for the address; see--- 'M.mergeBalance' for where version 20 says zero instead.------ It is two reads by key, the balance of the chain and the summary of the--- mempool, however much the mempool does at the address.-getBalance :: View -> Address -> IO (Maybe Balance)-getBalance (View r) a = M.summaryBalance a <$> Q.getBalance r a <*> (only <$> M.getSummaries r [a])-  where-    only = \case-      [s] -> s-      _ -> Nothing---- | Balances of addresses, in the same order, with the mempool applied.--- Addresses never used get a zero balance. Two multi-gets, one in each area.-getBalances :: View -> [Address] -> IO [Balance]-getBalances (View r) as = zipWith apply <$> Q.getBalances r as <*> M.getSummaries r as-  where-    apply c = maybe c (M.applySummary c)---- | Unspent output, if it exists and neither a block nor the mempool spends--- it: one that the chain has, or one of a mempool transaction.-getUnspent :: View -> OutPoint -> IO (Maybe Unspent)-getUnspent (View r) p =-  retrieveCF r.db (mempoolCF r.db) (MemSpenderKey p) >>= \case-    Just (_ :: Spender) -> return Nothing-    Nothing ->-      Q.getUnspent r p >>= \case-        Just u -> return (Just u)-        Nothing -> (>>= output) <$> M.getMempoolTx r p.hash-  where-    output td = case drop (fromIntegral p.index) td.tx.outputs of-      o : _ ->-        Just-          Unspent-            { block = td.block,-              outpoint = p,-              value = o.value,-              script = o.script,-              address = eitherToMaybe (scriptToAddressBS r.net r.ctx o.script)-            }-      [] -> Nothing---- | Transactions of an address, newest first.-getAddressTxs :: View -> Address -> Limits -> IO [TxRef]-getAddressTxs v a = getAddressesTxs v [a]---- | Transactions of any of the addresses, newest first, each once.-getAddressesTxs :: View -> [Address] -> Limits -> IO [TxRef]-getAddressesTxs (View r) as limits = paged r limits $ \from -> do-  txs <- withMempoolIters r from as (const True) $ \mempools ->-    Q.withManyIters r (addrTxCF r.db) (map AddrTxKeyA as) $ \its ->-      Q.joinConduits txRefKey (zipWith3 (addressTxsC from) as (map (fmap snd) mempools) its) limits-  Q.count r $ \s -> incrementCounter s.dataAddrTxCount (length txs)-  return txs---- | Unspent outputs of an address, newest first.-getAddressUnspents :: View -> Address -> Limits -> IO [Unspent]-getAddressUnspents v a = getAddressesUnspents v [a]---- | Unspent outputs of any of the addresses, newest first.-getAddressesUnspents :: View -> [Address] -> Limits -> IO [Unspent]-getAddressesUnspents (View r) as limits = paged r limits $ \from -> do-  us <- withUnspentStreams r from as $ \streams ->-    Q.joinConduits unspentKey streams limits-  Q.count r $ \s -> incrementCounter s.dataUnspentCount (length us)-  return us------------------------------------------------------------------------------------ Pages------------------------------------------------------------------------------------ | Where a page starts in a list that has the mempool's part first and the--- chain's part after it.-data From-  = -- | at the newest item-    Top-  | -- | in the mempool's part, at the transaction that entered the mempool-    -- at this time and has this hash, or where it would be-    InMempool !UnixTime !TxHash-  | -- | in the chain's part, at this position: nothing of the mempool is in-    -- the page-    InChain !BlockRef---- | Where the page of a query starts, or 'Nothing' if it starts at a--- transaction that is not known, which makes the page empty.------ A transaction that left the mempool unconfirmed is not known, so a page--- that starts at one is empty.-pageStart :: DatabaseReader -> Maybe Start -> IO (Maybe From)-pageStart _ Nothing = return (Just Top)-pageStart _ (Just (AtBlock height)) = return (Just (InChain (BlockRef height maxBound)))-pageStart r (Just (AtTx h)) =-  bodies r [h] <&> \found -> case (.block) <$> Map.lookup h found of-    Nothing -> Nothing-    Just (MemRef time) -> Just (InMempool time h)-    Just position@BlockRef {} -> Just (InChain position)---- | Answer a query from where its page starts.-paged :: DatabaseReader -> Limits -> (From -> IO [a]) -> IO [a]-paged r limits answer = pageStart r limits.start >>= maybe (return []) answer---- | Run an action with, for each address, its summary and an iterator on--- its entries in the mempool area, or 'Nothing' for an address whose--- entries the list does not need: one that the mempool does not touch, which--- is nearly all of them, one whose summary says that there is nothing of--- the kind wanted, and any address when the page starts in the chain, as--- every page after the first of a long history does.------ The summaries are read by key in one multi-get, which the filter of the--- column family answers for the addresses that have none. No iterator is--- moved to an address with nothing there: such a seek costs ten times a--- read by key, and far more when it has to step over what a block just--- deleted.-withMempoolIters ::-  DatabaseReader ->-  From ->-  [Address] ->-  (MemSummary -> Bool) ->-  ([Maybe (MemSummary, Iterator)] -> IO a) ->-  IO a-withMempoolIters r from as wanted action = do-  summaries <- case from of-    InChain _ -> return (map (const Nothing) as)-    _ -> map (>>= \s -> if wanted s then Just s else Nothing) <$> M.getSummaries r as-  Q.withManyIters r (mempoolAddrCF r.db) [MemAddrKeyA a | (a, Just _) <- zip as summaries] (action . pair summaries)-  where-    pair (Just s : rest) (it : its) = Just (s, it) : pair rest its-    pair (_ : rest) its = Nothing : pair rest its-    pair [] _ = []---- | Transactions of an address, newest first, from where a page starts:--- those of the mempool that belong in it, streamed from the entries of the--- address if it has any, then those of the chain, streamed from its index.--- Neither is read further than the page goes.-addressTxsC :: From -> Address -> Maybe Iterator -> Iterator -> ConduitT i TxRef IO ()-addressTxsC from a mempool it = mapM_ (mempoolTxsC from a) mempool >> chainTxsC from a it---- | Transactions of an address in the chain, newest first, from where a--- page starts, streamed from the index of the address.-chainTxsC :: From -> Address -> Iterator -> ConduitT i TxRef IO ()-chainTxsC from a it = chain .| concatMapC ref-  where-    chain = case from of-      InChain position -> matchingSkip it (AddrTxKeyA a) (AddrTxKeyB a position)-      _ -> matching it (AddrTxKeyA a)-    ref :: (AddrTxKey, ()) -> Maybe TxRef-    ref (AddrTxKey _ t, ()) = Just t-    ref _ = Nothing---- | Mempool transactions of an address, newest first, from where a page--- starts. Those that entered the mempool in the same second are in the--- order of their hashes, which is the order of their keys.-mempoolTxsC :: From -> Address -> Iterator -> ConduitT i TxRef IO ()-mempoolTxsC from a it = rows .| concatMapC ref-  where-    rows = case from of-      Top -> matching it (MemAddrTxKeyA a)-      InMempool time h -> matchingSkip it (MemAddrTxKeyA a) (MemAddrTxKey a time h)-      InChain _ -> return ()-    ref :: (MemAddrKey, MemAddrVal) -> Maybe TxRef-    ref (MemAddrTxKey _ t h, _) = Just TxRef {block = MemRef t, txid = h}-    ref _ = Nothing---- | Run an action with the stream of unspent outputs of each address, from--- where a page starts.------ The summaries are read whatever the start of the page: the chain still--- has, as unspent, the outputs that the mempool spends, on every page.-withUnspentStreams ::-  DatabaseReader ->-  From ->-  [Address] ->-  ([ConduitT i Unspent IO ()] -> IO a) ->-  IO a-withUnspentStreams r from as action = do-  summaries <- M.getSummaries r as-  let fresh = case from of-        InChain _ -> map (const Nothing) as-        _ -> map (>>= \s -> if s.paidCount > s.bothCount then Just s else Nothing) summaries-  Q.withManyIters r (mempoolAddrCF r.db) [MemAddrKeyA a | (a, Just _) <- zip as fresh] $ \mits ->-    Q.withManyIters r (addrOutCF r.db) (map AddrOutKeyA as) $ \its ->-      action (zipWith3 stream (zip as summaries) (pair fresh mits) its)-  where-    stream (a, s) = addressUnspentsC r from a s-    pair (Just _ : rest) (it : its) = Just it : pair rest its-    pair (_ : rest) its = Nothing : pair rest its-    pair [] _ = []---- | Unspent outputs of an address, newest first, from where a page starts,--- given the summary of what the mempool does at the address, if anything,--- and an iterator on its entries if the page has some: the outputs that the--- mempool created and has not spent, then those of the chain that the--- mempool does not spend, streamed from the index of the address.------ Whether the mempool spends an output is a read by key, and the summary--- says how many of them there are to find: none for nearly every address,--- so that nothing is read, and when the last one is found the reads stop.-addressUnspentsC ::-  DatabaseReader ->-  From ->-  Address ->-  Maybe MemSummary ->-  Maybe Iterator ->-  Iterator ->-  ConduitT i Unspent IO ()-addressUnspentsC r from a summary mempool it = do-  mapM_ (\mit -> fresh mit .| concatMapC created .| hide (maybe 0 (.bothCount) summary) 1) mempool-  chainUnspentsC r from a it .| hide (maybe 0 (\s -> s.spentCount - s.bothCount) summary) 1-  where-    -- the outputs of the transaction that the page starts at come first-    -- among those of the same second whose hashes are not smaller-    fresh mit = case from of-      Top -> matching mit (MemAddrOutKeyA a)-      InMempool time h -> matchingSkip mit (MemAddrOutKeyA a) (MemAddrOutKey a time (OutPoint h 0))-      InChain _ -> return ()-    created :: (MemAddrKey, MemAddrVal) -> Maybe Unspent-    created (MemAddrOutKey _ t p, MemAddrOutVal v s) =-      Just Unspent {block = MemRef t, outpoint = p, value = v, script = s, address = Just a}-    created _ = Nothing-    -- let outputs through, but for those that the mempool spends, of which-    -- there are no more than a number. They are asked about several at a-    -- time, in one multi-get, which costs a fifth of asking about each: one-    -- at first, then twice as many each time. A page of one address is then-    -- a few reads, and the first output of each of the hundreds of-    -- addresses of a wallet, which is all that a merge of them may take,-    -- does not cost a batch of rows read ahead for each.-    hide :: Word64 -> Int -> ConduitT Unspent Unspent IO ()-    hide 0 _ = awaitForever yield-    hide n size =-      gather size >>= \case-        [] -> return ()-        batch -> do-          found <- lift (spentAmong batch)-          yieldMany [u | (u, False) <- zip batch found]-          hide (n - min n (fromIntegral (length (filter id found)))) (min hideBatch (size * 2))-    -- whether the mempool spends each of some outputs of the address: a-    -- read by key for each of a few, and one multi-get for more, which-    -- only then costs less-    spentAmong :: [Unspent] -> IO [Bool]-    spentAmong batch-      | length batch < 8 = forM batch $ \u -> do-          found <- retrieveCF r.db (mempoolAddrCF r.db) (MemAddrSpentKey a u.outpoint)-          return (isJust (found :: Maybe MemAddrVal))-      | otherwise =-          map isJust <$> multiGetCF r.db (mempoolAddrCF r.db) [encode (MemAddrSpentKey a u.outpoint) | u <- batch]-    gather :: Int -> ConduitT Unspent o IO [Unspent]-    gather 0 = return []-    gather k =-      await >>= \case-        Nothing -> return []-        Just u -> (u :) <$> gather (k - 1)---- | The most outputs that are asked about at once when looking for those--- that the mempool spends. More would save little, and each one is a row--- read ahead of what a page may need.-hideBatch :: Int-hideBatch = 32---- | Unspent outputs of an address in the chain, newest first, from where a--- page starts, streamed from the index of the address. Those that the--- mempool spends are among them.-chainUnspentsC :: DatabaseReader -> From -> Address -> Iterator -> ConduitT i Unspent IO ()-chainUnspentsC r from a it = chain .| concatMapC output-  where-    chain = case from of-      InChain position -> matchingSkip it (AddrOutKeyA a) (AddrOutKeyB a position)-      _ -> matching it (AddrOutKeyA a)-    output :: (AddrOutKey, OutVal) -> Maybe Unspent-    output = uncurry (toUnspent r.net r.ctx)------------------------------------------------------------------------------------ Mempool------------------------------------------------------------------------------------ | The whole mempool, newest first.-getMempool :: View -> IO [(UnixTime, TxHash)]-getMempool (View r) = Q.getMempool r---- | Page of the mempool, newest first, from the offset and limit of the--- t'Limits'; its start is ignored. The list of the mempool and its index by--- time are the same in both layouts.-getMempoolPage :: View -> Limits -> IO [(UnixTime, TxHash)]-getMempoolPage (View r) = Q.getMempoolPage r------------------------------------------------------------------------------------ Extended public keys------------------------------------------------------------------------------------ | Balances of the addresses of an extended public key, derived up to the--- exact gap (see 'scanXPubBals'), with the mempool applied. An address that--- only the mempool has used counts as used, as it did in version 20.-xPubBals :: View -> XPubSpec -> IO [XPubBal]-xPubBals v@(View r) xpub = do-  Derived ext chg <- derivedAddrs r xpub-  bals <- scanXPubBals (getBalances v) r.initGap r.maxGap ext chg-  Q.count r $ \s -> incrementCounter s.dataXPubBals (length bals)-  return bals---- | Unspent outputs of an extended public key, newest first, given its--- balances from @xPubBals@. The addresses are read in step, each only as far--- as the page needs, as 'Q.xPubUnspents' reads them.-xPubUnspents :: View -> [XPubBal] -> Limits -> IO [XPubUnspent]-xPubUnspents (View r) xbals limits = paged r limits $ \from -> do-  let quota = case (deOffset limits).limit of-        0 -> Nothing-        l -> Just l-      tag b = mapC (\u -> XPubUnspent {path = b.path, unspent = u})-  us <- withUnspentStreams r from (map (.balance.address) funded) $ \streams ->-    runConduit $-      mergeDescGroups (.unspent.block) (unspentKey . (.unspent)) quota (zipWith (\b s -> s .| tag b) funded streams)-        .| applyLimitsC limits-        .| sinkList-  Q.count r $ \s -> incrementCounter s.dataXPubUnspents (length us)-  return us-  where-    funded = filter ((> 0) . (.balance.utxo)) xbals---- | Transactions of an extended public key, newest first, given its balances--- from @xPubBals@.-xPubTxs :: View -> [XPubBal] -> Limits -> IO [TxRef]-xPubTxs v@(View r) xbals limits = do-  txs <- getAddressesTxs v (Q.usedAddresses xbals) limits-  Q.count r $ \s -> incrementCounter s.dataXPubTxs (length txs)-  return txs---- | Number of transactions of an extended public key, given its balances--- from @xPubBals@.-xPubTxCount :: View -> [XPubBal] -> IO Word32-xPubTxCount v@(View r) xbals = do-  Q.count r $ \s -> incrementCounter s.dataXPubTxCount 1-  getAddressesTxCount v (Q.usedAddresses xbals)---- | Number of distinct transactions of a set of addresses: each transaction--- is counted once even when it touches several of them.------ Those of the chain are counted from the keys of its index, as--- 'Q.getAddressesTxCount' counts them, and those of the mempool from the--- summaries of the addresses: nothing more is read when none of them, or--- only one, has transactions in the mempool, and otherwise the transactions--- of those that have are read, to count each once. No transaction is in--- both areas.-getAddressesTxCount :: View -> [Address] -> IO Word32-getAddressesTxCount _ [] = return 0-getAddressesTxCount (View r) as = do-  inBlocks <- Q.getAddressesTxCount r as-  inMempool <- withMempoolIters r Top as ((> 0) . (.txs)) $ \mempools ->-    case [(a, m) | (a, Just m) <- zip as mempools] of-      [] -> return 0-      [(_, (s, _))] -> return (fromIntegral s.txs)-      several -> do-        txs <- forM several $ \(a, (_, it)) -> runConduit (mempoolTxsC Top a it .| mapC (.txid) .| sinkList)-        return (HashSet.size (HashSet.fromList (concat txs)))-  return (inBlocks + fromIntegral inMempool)
src/Haskoin/Store/Database/Query/Pooled.hs view
@@ -6,32 +6,23 @@ -- Stability   : experimental -- Portability : POSIX ----- The queries of the store with a mempool that is not the store's own laid--- over the chain: the shared pool of a fleet of stores, which--- "Haskoin.Store.Fleet.Redis" keeps in Redis. See "A fleet that answers as--- one" in @docs\/layered-database.md@.+-- The queries of the store: the chain, as the database has it, with the+-- mempool laid over it. The mempool is the pool that the stores of a fleet+-- share, which "Haskoin.Store.Fleet.Redis" keeps in Redis; the database+-- has no mempool of its own. ----- = Why another set of queries+-- = Why the pool comes as functions ----- "Haskoin.Store.Database.Query.Layered" answers from the two areas of one--- database, read through one snapshot. Stores that share a Redis are to--- give the same answer to the same request, and their own mempools differ.--- So each answers from the chain as it was at the block that the fleet--- agreed on, which every store has the same, and from the one pool that--- they share. The chain's part of an answer is read as the layered queries--- read it, with the same code. The mempool's part comes through a+-- Stores that share a Redis are to give the same answer to the same+-- request. So each answers from the chain as it was at the block that the+-- fleet agreed on, which every store has the same, read through a view of+-- its database at that block ("Haskoin.Store.Database.Views"), and from the+-- one pool that they share. The mempool's part comes through a -- t'PoolReads', a handful of functions that say what a pool has: its -- summaries, the entries of an address by the page, bodies, spenders and--- its list.------ The alternative was to give the layered queries the same handful of--- functions for the store's own mempool area. They read that area through--- iterators that are made once for a request and moved from address to--- address, which is where their speed comes from, and which a function--- that returns a page would hide. The two sets share what is the same, and--- the tests hold this one to the answers of the other: 'localPool' is the--- store's own mempool area as a t'PoolReads', and with it every query here--- must answer as the layered one does.+-- its list. A pool far away answers each in one round trip, and+-- 'emptyPool' answers for no pool at all, which is how the chain alone is+-- read ('chainRead'). -- -- = What a pool cannot say --@@ -42,8 +33,13 @@ module Haskoin.Store.Database.Query.Pooled   ( PoolReads (..),     pooledRead,-    localPool,+    emptyPool,+    chainRead,     pageStart,+    From (..),+    withSpenders,+    summaryBalance,+    applySummary,   ) where @@ -51,6 +47,7 @@   ( ConduitT,     await,     awaitForever,+    concatMapC,     lift,     mapC,     runConduit,@@ -59,23 +56,24 @@     yieldMany,     (.|),   )+import Control.Applicative ((<|>)) import Control.Monad (forM) import Data.Bits ((.&.))-import Control.Applicative ((<|>))+import Data.ByteString qualified as BS import Data.Bytes.Put (runPutS) import Data.Bytes.Serial (serialize)-import Data.ByteString qualified as BS import Data.Functor ((<&>))+import Data.HashMap.Strict (HashMap) import Data.HashMap.Strict qualified as HashMap import Data.HashSet qualified as HashSet import Data.IntMap.Strict qualified as IntMap import Data.Map.Strict (Map) import Data.Map.Strict qualified as Map-import Data.Maybe (catMaybes, isJust)+import Data.Maybe (catMaybes, fromMaybe, isJust) import Data.Serialize (encode) import Data.Word (Word16, Word32, Word64) import Database.RocksDB (Iterator)-import Database.RocksDB.Query (matchingAsListCF, retrieveCF)+import Database.RocksDB.Query (matchingAsListCF) import Haskoin   ( Address,     OutPoint (..),@@ -108,29 +106,26 @@     Unspent (..),     XPubBal (..),     XPubUnspent (..),+    zeroBalance,   )-import Haskoin.Store.Database.Mempool qualified as M import Haskoin.Store.Database.Query qualified as Q-import Haskoin.Store.Database.Query.Layered (From (..), chainTxsC, chainUnspentsC, withSpenders) import Haskoin.Store.Database.Reader   ( DatabaseReader (..),     Derived (..),     addrOutCF,     addrTxCF,     derivedAddrs,-    mempoolAddrCF,-    mempoolCF,     txCF,+    withSnapshotReader,   ) import Haskoin.Store.Database.Types   ( AddrOutKey (..),     AddrTxKey (..),-    MemAddrKey (..),-    MemAddrVal (..),-    MemSpenderKey (..),     MemSummary (..),+    OutVal,     TxKey (..),     decodeTxKey,+    toUnspent,   )  -- | What a mempool has, as much as the queries need of it. Each function@@ -176,7 +171,6 @@ pooledRead r pool =   StoreRead     { net = r.net,-      ctx = r.ctx,       getBestBlock = Q.getBestBlock r,       getBlocksAtHeight = Q.getBlocksAtHeight r,       getBlock = Q.getBlock r,@@ -213,10 +207,72 @@       [found] -> found       _ -> Nothing +-- | A pool that has nothing. The queries of this module with it answer+-- from the chain alone.+emptyPool :: PoolReads+emptyPool =+  PoolReads+    { summaries = return . map (const Nothing),+      addressTxs = \_ _ _ -> return [],+      addressOuts = \_ _ _ -> return [],+      spentOutputs = \_ ps -> return (map (const False) ps),+      bodies = return . map (const (Nothing, IntMap.empty)),+      spenders = return . map (const Nothing),+      page = \_ _ -> return [],+      startingWith = \_ -> return [],+      replaced = \_ -> return Nothing+    }++-- | The queries of the chain alone, each on a snapshot of the database of+-- its own, taken when it is asked: for code that asks one thing at a time,+-- outside of a request, such as the block store and the web server before+-- it has a view of the fleet tip. Queries asked one after the other may see+-- different blocks.+chainRead :: DatabaseReader -> StoreRead+chainRead r =+  StoreRead+    { net = r.net,+      getBestBlock = one (.getBestBlock),+      getBlocksAtHeight = \h -> one (\q -> q.getBlocksAtHeight h),+      getBlock = \h -> one (\q -> q.getBlock h),+      getTxData = \h -> one (\q -> q.getTxData h),+      getManyTxData = \hs -> one (\q -> q.getManyTxData hs),+      getManyTxBytes = \hs -> one (\q -> q.getManyTxBytes hs),+      getSpender = \p -> one (\q -> q.getSpender p),+      getBalance = \a -> one (\q -> q.getBalance a),+      getUnspent = \p -> one (\q -> q.getUnspent p),+      getMempool = return [],+      getMempoolPage = \_ -> return [],+      getAddressesTxs = \as l -> one (\q -> q.getAddressesTxs as l),+      getAddressesUnspents = \as l -> one (\q -> q.getAddressesUnspents as l),+      initialGap = r.initGap,+      maxGap = r.maxGap,+      getNumTxData = \n -> one (\q -> q.getNumTxData n),+      getBalances = \as -> one (\q -> q.getBalances as),+      getAddressTxs = \a l -> one (\q -> q.getAddressTxs a l),+      getAddressUnspents = \a l -> one (\q -> q.getAddressUnspents a l),+      xPubBals = \x -> one (\q -> q.xPubBals x),+      xPubUnspents = \x bs l -> one (\q -> q.xPubUnspents x bs l),+      xPubTxs = \x bs l -> one (\q -> q.xPubTxs x bs l),+      xPubTxCount = \x bs -> one (\q -> q.xPubTxCount x bs),+      getAddressesTxCount = \as -> one (\q -> q.getAddressesTxCount as),+      getReplaced = \_ -> return Nothing+    }+  where+    one :: (StoreRead -> IO a) -> IO a+    one f = withSnapshotReader r (\s -> f (pooledRead s emptyPool))+ ------------------------------------------------------------------------------- -- Transactions ------------------------------------------------------------------------------- +-- | Add the spenders that the pool has for the outputs of a transaction+-- to those in its body.+withSpenders :: HashMap TxHash (IntMap.IntMap Spender) -> TxHash -> TxData -> TxData+withSpenders spent h td = case HashMap.lookup h spent of+  Nothing -> td+  Just unconfirmed -> (td :: TxData) {spenders = IntMap.union td.spenders unconfirmed}+ -- | Data of many transactions, in the order of the hashes: those of the -- pool from it, the others from the chain, each with the spenders that -- blocks and the pool give to its outputs. Deleted ones are not known.@@ -237,7 +293,7 @@       rest = filter (`Map.notMember` unconfirmed) hs   chain <-     Map.fromList . catMaybes . zipWith (\h v -> (,) h <$> (v >>= Q.liveTxBytes)) rest-      <$> Q.retrieveManyBytesCF r.db (txCF r.db) (map TxKey rest)+      <$> Q.retrieveManyBytesCF r (txCF r.db) (map TxKey rest)   return [Map.lookup h unconfirmed <|> Map.lookup h chain | h <- hs]  -- | The data of transactions by hash: those that the pool has, and those@@ -254,7 +310,7 @@   chain <- case rest of     [] -> return Map.empty     _ -> do-      found <- catMaybes . zipWith (fmap . (,)) rest <$> Q.retrieveManyCF r.db (txCF r.db) (map TxKey rest)+      found <- catMaybes . zipWith (fmap . (,)) rest <$> Q.retrieveManyCF r (txCF r.db) (map TxKey rest)       -- with the spenders that blocks give to their outputs       Map.fromList . zip (map fst found) <$> Q.withChainSpenders r found   return (Map.mapWithKey (withSpenders spent) (unconfirmed <> Map.mapMaybe notDeleted' chain))@@ -287,7 +343,7 @@  -- | Balance of an address, with the pool applied. getBalance :: DatabaseReader -> PoolReads -> Address -> IO (Maybe Balance)-getBalance r pool a = M.summaryBalance a <$> Q.getBalance r a <*> (only <$> pool.summaries [a])+getBalance r pool a = summaryBalance a <$> Q.getBalance r a <*> (only <$> pool.summaries [a])   where     only = \case       [s] -> s@@ -298,8 +354,29 @@ getBalances _ _ [] = return [] getBalances r pool as = zipWith apply <$> Q.getBalances r as <*> pool.summaries as   where-    apply c = maybe c (M.applySummary c)+    apply c = maybe c (applySummary c) +-- | The balance that the chain has for an address with the summary of what+-- the pool does at it applied.+applySummary :: Balance -> MemSummary -> Balance+applySummary c s =+  Balance+    { address = c.address,+      confirmed = c.confirmed `minus` (s.spent `minus` s.both),+      unconfirmed = c.unconfirmed + (s.paid `minus` s.both),+      utxo = (c.utxo `minus` (s.spentCount `minus` s.bothCount)) + (s.paidCount `minus` s.bothCount),+      txs = c.txs + s.txs,+      received = c.received + s.paid+    }+  where+    minus x y = if y > x then 0 else x - y++-- | Balance of an address as the API gives it, from its balance in the+-- chain and its summary in the pool, either of which it may not have.+summaryBalance :: Address -> Maybe Balance -> Maybe MemSummary -> Maybe Balance+summaryBalance _ chain Nothing = chain+summaryBalance a chain (Just s) = Just (applySummary (fromMaybe (zeroBalance a) chain) s)+ -- | Unspent output, if it exists and neither a block nor the pool spends -- it. getUnspent :: DatabaseReader -> PoolReads -> OutPoint -> IO (Maybe Unspent)@@ -322,10 +399,46 @@               outpoint = p,               value = o.value,               script = o.script,-              address = eitherToMaybe (scriptToAddressBS r.net r.ctx o.script)+              address = eitherToMaybe (scriptToAddressBS r.net o.script)             }       [] -> Nothing +-- | Where a page starts in a list that has the pool's part first and the+-- chain's part after it.+data From+  = -- | at the newest item+    Top+  | -- | in the pool's part, at the transaction that entered the pool at+    -- this time and has this hash, or where it would be+    InMempool !UnixTime !TxHash+  | -- | in the chain's part, at this position: nothing of the pool is in+    -- the page+    InChain !BlockRef++-- | Transactions of an address in the chain, newest first, from where a+-- page starts, streamed from the index of the address.+chainTxsC :: From -> Address -> Iterator -> ConduitT i TxRef IO ()+chainTxsC from a it = chain .| concatMapC ref+  where+    chain = case from of+      InChain position -> Q.scanFrom it (AddrTxKeyA a) (AddrTxKeyB a position)+      _ -> Q.scan it (AddrTxKeyA a)+    ref :: (AddrTxKey, ()) -> Maybe TxRef+    ref (AddrTxKey _ t, ()) = Just t+    ref _ = Nothing++-- | Unspent outputs of an address in the chain, newest first, from where a+-- page starts, streamed from the index of the address. Those that the+-- pool spends are among them.+chainUnspentsC :: DatabaseReader -> From -> Address -> Iterator -> ConduitT i Unspent IO ()+chainUnspentsC r from a it = chain .| concatMapC output+  where+    chain = case from of+      InChain position -> Q.scanFrom it (AddrOutKeyA a) (AddrOutKeyB a position)+      _ -> Q.scan it (AddrOutKeyA a)+    output :: (AddrOutKey, OutVal) -> Maybe Unspent+    output = uncurry (toUnspent r.net)+ -- | Where the page of a query starts, or 'Nothing' if it starts at a -- transaction that neither the pool nor the chain has. pageStart :: DatabaseReader -> PoolReads -> Maybe Start -> IO (Maybe From)@@ -497,50 +610,3 @@       txs <- forM several $ \(a, _) -> runConduit (poolTxsC pool Top 256 a True .| mapC (.txid) .| sinkList)       return (HashSet.size (HashSet.fromList (concat txs)))   return (inBlocks + fromIntegral inPool)------------------------------------------------------------------------------------ The store's own mempool area as a pool------------------------------------------------------------------------------------ | The mempool area of a database, as a reader has it, as a pool. The--- queries of this module with it must answer as those of--- "Haskoin.Store.Database.Query.Layered" do, but for deleted transactions:--- that is what the tests check, and what this is for.-localPool :: DatabaseReader -> PoolReads-localPool r =-  PoolReads-    { summaries = M.getSummaries r,-      addressTxs = \a start n -> do-        rows <- entries a-        let listed = [(t, h) | (MemAddrTxKey _ t h, _) <- rows]-            from (t, h) = dropWhile (\(t', h') -> t' > t || (t' == t && h' < h)) listed-        return (take n (maybe listed from start)),-      addressOuts = \a start n -> do-        rows <- entries a-        let spent = HashSet.fromList [p | (MemAddrSpentKey _ p, _) <- rows]-            made t p v s = Unspent {block = MemRef t, outpoint = p, value = v, script = s, address = Just a}-            listed = [(made t p v s, HashSet.member p spent) | (MemAddrOutKey _ t p, MemAddrOutVal v s) <- rows]-            place (u, _) = case u.block of-              MemRef t -> (t, encode u.outpoint)-              BlockRef {} -> (0, encode u.outpoint)-            from (t, p) = dropWhile (\x -> let (t', p') = place x in t' > t || (t' == t && p' < encode p)) listed-        return (take n (maybe listed from start)),-      spentOutputs = \a ps ->-        forM ps $ \p ->-          isJust <$> (retrieveCF r.db (mempoolAddrCF r.db) (MemAddrSpentKey a p) :: IO (Maybe MemAddrVal)),-      bodies = \hs -> case hs of-        [] -> return []-        _ -> zip <$> Q.retrieveManyCF r.db (mempoolCF r.db) (map TxKey hs) <*> mapM (M.getMempoolSpenders r) hs,-      spenders = \ps -> case ps of-        [] -> return []-        _ -> Q.retrieveManyCF r.db (mempoolCF r.db) (map MemSpenderKey ps),-      page = \offset limit -> Q.getMempoolPage r Limits {limit = limit, offset = offset, start = Nothing},-      startingWith = \prefix -> do-        rows <- matchingAsListCF r.db (mempoolCF r.db) (TxKeyS prefix)-        return [h | (TxKey h, _ :: TxData) <- rows],-      -- what the blocks replaced, as the store's own database keeps it-      replaced = Q.getReplaced r-    }-  where-    entries :: Address -> IO [(MemAddrKey, MemAddrVal)]-    entries a = matchingAsListCF r.db (mempoolAddrCF r.db) (MemAddrKeyA a)
src/Haskoin/Store/Database/Reader.hs view
@@ -8,11 +8,16 @@ -- Stability   : experimental -- Portability : POSIX ----- The RocksDB database of the store: opening it ('withDatabaseReader'), which+-- The RocksDB database of the store: opening it ('withLayeredReader'), which -- also creates or migrates it, its column families, its metrics, and the -- memo of the addresses of extended public keys. The queries are in -- "Haskoin.Store.Database.Query". --+-- Two of the column families, the balances and the spenders, are opened+-- with a merge operator ("Haskoin.Store.Database.Merge"): what a reader+-- gets from them is always a value, which RocksDB may have worked out from+-- merge records on the way.+-- -- Data is split across column families by kind; see 'addrTxCF' and the other -- column family accessors. The keys stored in each are described in -- "Haskoin.Store.Database.Types".@@ -25,6 +30,16 @@ -- its own key prefix length for filters and seeks, and keep scans over one kind -- of data from wading through another. --+-- = The column families of the old mempool+--+-- Two column families, @mempool@ and @mempool-addr@, held the mempool of+-- the store until version 1.16.0. The mempool is now the pool that the+-- fleet shares in Redis, and nothing reads or writes them, but for the+-- migration of a database of version 20 ("Haskoin.Store.Database.Migrate"),+-- which reads the list of its mempool there. They are still opened:+-- RocksDB opens a database only with all of its column families, and+-- keeping them lets an older store open the database again.+-- -- = Why addresses of xpubs are cached -- -- Deriving an address from an extended public key costs an elliptic curve@@ -37,10 +52,13 @@     DataMetrics (..),     Derived (..),     derivedAddrs,+    derivedAddrsCached,     createDataMetrics,-    withDatabaseReader,     withLayeredReader,-    initRocksDB,+    withReader,+    columnFamilyConfig,+    layeredFamilyConfig,+    mergeGuardFamily,     memoryReport,     addrTxCF,     addrOutCF,@@ -51,8 +69,10 @@     heightCF,     balanceCF,     mempoolCF,-    mempoolAddrCF,-    mempoolInsertOps,++    -- * Snapshots+    snapshotReader,+    withSnapshotReader,   ) where @@ -61,19 +81,18 @@ import Data.Default (def) import Data.HashMap.Strict (HashMap) import Data.HashMap.Strict qualified as HashMap-import Data.Maybe (fromMaybe, isJust, mapMaybe)+import Data.IORef (IORef, atomicModifyIORef', newIORef)+import Data.Maybe (isJust, mapMaybe) import Data.Text (Text) import Data.Text qualified as T import Data.Word (Word32) import Database.RocksDB hiding (withDB)-import Database.RocksDB.Query import Haskoin import Haskoin.Store.Common import Haskoin.Store.Data-import Haskoin.Store.Database.Types+import Haskoin.Store.Database.Merge (balanceOperator, spenderOperator) import System.Metrics.StatsD import Text.Read (readMaybe)-import UnliftIO  -- | Open database and its settings. data DatabaseReader = DatabaseReader@@ -87,8 +106,6 @@     net :: !Network,     -- | StatsD metrics, if enabled     metrics :: !(Maybe DataMetrics),-    -- | secp256k1 context-    ctx :: !Ctx,     -- | addresses derived from recently queried xpubs     derived :: !(IORef DerivedCache)   }@@ -114,26 +131,32 @@  -- | Addresses of an xpub, from the cache or a new lazy derivation. derivedAddrs :: DatabaseReader -> XPubSpec -> IO Derived-derivedAddrs r xpub = do+derivedAddrs r xpub = fst <$> derivedAddrsCached r xpub++-- | Addresses of an xpub, from the cache or a new lazy derivation, and+-- whether the cache had them. A request pays for deriving the addresses of+-- an xpub only when it did not ("Haskoin.Store.Web.Tokens"): those that+-- the cache has were derived for an earlier request, as far as it needed.+derivedAddrsCached :: DatabaseReader -> XPubSpec -> IO (Derived, Bool)+derivedAddrsCached r xpub = do   let ref = r.derived-      ctx = r.ctx       fresh =         let addrs m =               deriveAddresses-                (deriveFunction ctx xpub.deriv)-                (pubSubKey ctx xpub.key m)+                (deriveFunction xpub.deriv)+                (pubSubKey xpub.key m)                 0          in Derived (addrs 0) (addrs 1)   atomicModifyIORef' ref $ \c ->     case HashMap.lookup xpub c.newer of-      Just d -> (c, d)+      Just d -> (c, (d, True))       Nothing ->-        let d = fromMaybe fresh (HashMap.lookup xpub c.older)+        let (d, hit) = maybe (fresh, False) (,True) (HashMap.lookup xpub c.older)             c'               | HashMap.size c.newer >= derivedCacheSize =                   DerivedCache {newer = HashMap.singleton xpub d, older = c.newer}               | otherwise = c {newer = HashMap.insert xpub d c.newer}-         in (c', d)+         in (c', (d, hit))  -- | StatsD metrics of database reads. Counters count the items read. data DataMetrics = DataMetrics@@ -143,8 +166,6 @@     dataBlockCount :: !StatCounter,     -- | transactions read (@db.txs@)     dataTxCount :: !StatCounter,-    -- | mempool entries read (@db.mempool@)-    dataMempoolCount :: !StatCounter,     -- | balances read (@db.balances@)     dataBalanceCount :: !StatCounter,     -- | unspent outputs read (@db.unspents@)@@ -169,7 +190,6 @@   dataBestCount <- newStatCounter s "db.best_block" n   dataBlockCount <- newStatCounter s "db.blocks" n   dataTxCount <- newStatCounter s "db.txs" n-  dataMempoolCount <- newStatCounter s "db.mempool" n   dataBalanceCount <- newStatCounter s "db.balances" n   dataUnspentCount <- newStatCounter s "db.unspents" n   dataAddrTxCount <- newStatCounter s "db.address_txs" n@@ -182,65 +202,29 @@   where     n = 10 --- | Version of the layout that 'withDatabaseReader' opens, in which the--- mempool is written into the indices of the chain. Databases of versions--- 18 and 19 are migrated to it when opened that way; other versions are--- refused.------ The store runs on a later one,--- 'Haskoin.Store.Database.Migrate.currentVersion', and opens its database--- with 'withLayeredReader' and 'Haskoin.Store.Database.Migrate.upgradeDatabase'.--- This version is kept for the tests, which hold the new layout to the--- answers of this one, and as the way through for databases of 18 and 19.------ Bumping the version is how a change to the database layout is made safe: an--- old database is either migrated in place when it is opened or refused, never--- read with the wrong layout.-dataVersion :: Word32-dataVersion = 20---- | Open the database and run a query against it. Creates the database and its--- column families if missing, and migrates it from older versions.+-- | Open the database and run an action with it. Creates the database and+-- its column families if missing ('layeredFamilyConfig'). ----- Arguments, in order: network, secp256k1 context, initial xpub gap, maximum--- xpub gap, directory, whether to use Bloom filters, maximum open files, total--- memtable size in bytes, write-ahead log size in bytes after which column--- families are flushed, block cache size in bytes, and metrics. 'Nothing' leaves--- a setting to RocksDB.-withDatabaseReader ::-  Network ->-  Ctx ->-  Word32 ->-  Word32 ->-  FilePath ->-  Bool ->-  Maybe Int ->-  Maybe Int ->-  Maybe Int ->-  Maybe Int ->-  Maybe DataMetrics ->-  (DatabaseReader -> IO a) ->-  IO a-withDatabaseReader = withReader columnFamilyConfig initRocksDB---- | Open a database with the column families of the layered layout of--- @docs\/layered-database.md@: those of 'withDatabaseReader' and one more,--- for what the mempool does at each address. The arguments are those of--- 'withDatabaseReader'.+-- Arguments, in order: network, initial xpub gap, maximum xpub gap, directory,+-- whether to use Bloom filters, maximum open files, total memtable size in+-- bytes, write-ahead log size in bytes after which column families are+-- flushed, threads for flushes and compactions together, block cache size in+-- bytes, and metrics. 'Nothing' leaves a setting to RocksDB. -- -- It neither checks nor sets the version of the database: -- 'Haskoin.Store.Database.Migrate.upgradeDatabase' does, and has to be -- called before anything reads the database. The store opens its database -- with the two. ----- Opening a database this way adds the column family to it. RocksDB only--- opens a database when given all its column families, so a store that does--- not know this one, such as any through 'withDatabaseReader', can no--- longer open the database afterwards. That is the point of no return of--- the migration.+-- Opening a database this way adds the column families of the layout to+-- it. RocksDB only opens a database when given all its column families, so+-- a store that does not know them can no longer open the database+-- afterwards: not one older than database+-- version 23, which does not know the empty one and could not read the+-- merge records of the balances and the spenders. That is the point of no+-- return of an upgrade. withLayeredReader ::   Network ->-  Ctx ->   Word32 ->   Word32 ->   FilePath ->@@ -249,6 +233,7 @@   Maybe Int ->   Maybe Int ->   Maybe Int ->+  Maybe Int ->   Maybe DataMetrics ->   (DatabaseReader -> IO a) ->   IO a@@ -257,12 +242,19 @@ -- | Open a database with these column families, given whether to use Bloom -- filters and whether there is a shared block cache, and run a check on it -- before anything else. The other arguments are those of--- 'withDatabaseReader'.+-- 'withLayeredReader'.+--+-- The store opens its database with 'withLayeredReader'. This is for the+-- tests and the benchmarks, which open one with the column families of+-- 'layeredFamilyConfig' changed: without the one that older versions do+-- not have, as such a version left a database, or with RocksDB kept from+-- compacting. The families must be those of 'columnFamilyConfig' or of+-- 'layeredFamilyConfig' in their order, since the column families are+-- found by their place ('balanceCF' and the others). withReader ::   (Bool -> Bool -> [(String, Config)]) ->   (DatabaseReader -> IO ()) ->   Network ->-  Ctx ->   Word32 ->   Word32 ->   FilePath ->@@ -271,11 +263,12 @@   Maybe Int ->   Maybe Int ->   Maybe Int ->+  Maybe Int ->   Maybe DataMetrics ->   (DatabaseReader -> IO a) ->   IO a-withReader families check net ctx igap gap dir bloom maxOpenFiles writeBuffers maxWal blockCache stats m =-  withDBCF dir cfg (families bloom (isJust blockCache)) $ \db -> do+withReader families check net igap gap dir bloom maxOpenFiles writeBuffers maxWal jobs blockCache stats m =+  withDBCF dir cfg (families bloom False) $ \db -> do     derived <- newIORef (DerivedCache HashMap.empty HashMap.empty)     let bdb =           DatabaseReader@@ -284,7 +277,6 @@               net = net,               initGap = igap,               metrics = stats,-              ctx = ctx,               derived = derived             }     check bdb@@ -297,8 +289,13 @@           maxFiles = maxOpenFiles,           dbWriteBufferSize = writeBuffers,           maxTotalWalSize = maxWal,+          maxBackgroundJobs = jobs,           blockCacheSize = blockCache,-          cacheIndexAndFilterBlocks = isJust blockCache+          -- index and filter blocks stay with their open tables, outside the+          -- block cache, where data blocks cannot push them out: about 0.34%+          -- of the data, held for as long as a table is open+          cacheIndexAndFilterBlocks = False,+          partitionIndexAndFilters = False         }  -- | Memory that RocksDB uses outside the GHC heap, per column family:@@ -358,55 +355,80 @@ -- indices, and 13 without, in the layout of version 20. No filter was ever -- built before rocksdb-haskell-jprupp 2.6.2, so it never happened to a -- store, and filters are on by default now for everything else.+--+-- = The two that merge+--+-- The column families of the balances and of the spenders have a merge+-- operator each ("Haskoin.Store.Database.Merge"), since database version+-- 23: the import of a block writes merge records to them, and RocksDB+-- works out the values. They are set here, in the one place that every+-- way of opening the database goes through, because a database with merge+-- records must never be opened without them. columnFamilyConfig :: Bool -> Bool -> [(String, Config)] columnFamilyConfig bloom cached =   [ ("addr-tx", scanned (Just 22)),     ("addr-out", scanned (Just 22)),     ("tx", c (Just 33)),-    ("spender", c (Just 33)),+    ("spender", (c (Just 33)) {mergeOperator = Just spenderOperator}),     ("unspent", c (Just 37)),     ("block", c (Just 33)),     ("height", c Nothing),-    ("balance", c (Just 22)),+    ("balance", (c (Just 22)) {mergeOperator = Just balanceOperator}),     ("mempool", c Nothing)   ]   where-    c p = def {prefixLength = p, bloomFilter = bloom, cacheIndexAndFilterBlocks = cached}+    c p =+      def+        { prefixLength = p,+          bloomFilter = bloom,+          cacheIndexAndFilterBlocks = cached,+          partitionIndexAndFilters = cached+        }     scanned p = (c p) {bloomFilter = False} --- | Column families of the layered layout: those of 'columnFamilyConfig',--- and one for the entries of the mempool by address ('MemAddrKey').+-- | Column families of the layout that the store runs on: those of+-- 'columnFamilyConfig', one that held the entries of the old mempool by+-- address, and one that is empty and keeps older versions out. The two of+-- the old mempool have nothing written to them any more (see the head of+-- this module). ----- The two column families of the mempool always have Bloom filters, whatever--- the setting for the others. Almost every lookup in them is for something--- that is not there: a confirmed transaction, an output that no mempool--- transaction spends, an address with nothing in the mempool. A filter--- answers those without reading a block, and for column families this small--- it takes a few megabytes; on the column families of the chain the same--- filters would take gigabytes, which is why they are optional there.+-- = The empty one ----- The entries by address are in a column family of their own because their--- filter is on the first 22 bytes of the key, the kind of key and the--- address: a scan of an address with nothing in the mempool then stops at--- the filter. The mempool column family is read by whole keys and scanned--- across prefixes, for the index by time, so it cannot have a filter by--- prefix.+-- Since database version 23 the balances and the spenders have merge+-- records, which only RocksDB's merge operators can read. A store of an+-- older version does not set them, and RocksDB opens the database for it+-- all the same: it then drops, without a word, the merge records that are+-- still in the write-ahead log, which is the balances and the spenders of+-- the last blocks, and the store finds out about the version afterwards. ----- That scan through a filter is what 'columnFamilyConfig' keeps out of the--- indices of the chain, and it is as slow here after a block, which deletes--- the keys of the transactions it confirms: 1.6 milliseconds for an address--- with nothing, measured. It is one of the things that have to change before--- this layout is used; @docs\/layered-database.md@ says how.+-- RocksDB does refuse to open a database when it is not given all of its+-- column families, and it does so before it reads the write-ahead log. So+-- the layout has a column family that those versions do not know of,+-- 'mergeGuardFamily', with nothing in it. They fail to open the database,+-- with an error that names it, and leave the database as it was. layeredFamilyConfig :: Bool -> Bool -> [(String, Config)] layeredFamilyConfig bloom cached =   map mempool (columnFamilyConfig bloom cached)-    <> [("mempool-addr", c (Just 22))]+    <> [("mempool-addr", c (Just 22)), (mergeGuardFamily, def)]   where-    c p = def {prefixLength = p, bloomFilter = True, cacheIndexAndFilterBlocks = cached}+    c p =+      def+        { prefixLength = p,+          bloomFilter = True,+          cacheIndexAndFilterBlocks = cached,+          partitionIndexAndFilters = cached+        }     mempool (name, cfg)       | name == "mempool" = (name, c Nothing)       | otherwise = (name, cfg) +-- | The name of the column family that has nothing in it, and is there so+-- that a store older than database version 23 cannot open the database:+-- see 'layeredFamilyConfig'. The name is what the error of such a store+-- says is missing.+mergeGuardFamily :: String+mergeGuardFamily = "merge-operators"+ -- | Column family for transactions by address ('AddrTxKey'). addrTxCF :: DB -> ColumnFamily addrTxCF db = columnFamilies db !! 0@@ -440,56 +462,18 @@ balanceCF :: DB -> ColumnFamily balanceCF db = columnFamilies db !! 7 --- | Column family for mempool transactions ('MemTxKey' and 'MemTimeKey').--- In the layered layout it also has their bodies, under t'TxKey', and--- their spenders, under t'MemSpenderKey'.+-- | Column family of the old mempool: the list of a database of version+-- 20 ('MemTxKey'), which only its migration reads. mempoolCF :: DB -> ColumnFamily mempoolCF db = columnFamilies db !! 8 --- | Column family for what the mempool does at each address ('MemAddrKey').--- Only a database opened with 'withLayeredReader' has it.-mempoolAddrCF :: DB -> ColumnFamily-mempoolAddrCF db = columnFamilies db !! 9---- | Set the version of a new database, migrate an old one, or call 'error' if the version is not supported.-initRocksDB :: DatabaseReader -> IO ()-initRocksDB DatabaseReader {db = db} =-  retrieve db VersionKey >>= \case-    Just v-      | v == dataVersion -> return ()-      | v == 19 -> migrateMempoolIndex db-      | v == 18 -> migrateMempool db-      | otherwise -> error "Incorrect RocksDB database version"-    Nothing -> setInitRocksDB db---- | Version 18 kept the whole mempool as one list under 'MemKey'. Move each--- transaction to its own key in the mempool column family, and index it by--- time, in the same batch that sets the new version.-migrateMempool :: DB -> IO ()-migrateMempool db = do-  mempool <- fromMaybe [] <$> (retrieve db MemKey :: IO (Maybe [(UnixTime, TxHash)]))-  writeBatch db $-    concat [mempoolInsertOps db h t | (t, h) <- mempool]-      <> [deleteOp MemKey, insertOp VersionKey dataVersion]---- | Version 19 kept one key per mempool transaction, but no index by time,--- so every page of the mempool had to read and sort all of it. Build the--- index in the same batch that sets the new version.-migrateMempoolIndex :: DB -> IO ()-migrateMempoolIndex db = do-  mempool <- matchingAsListCF db (mempoolCF db) MemTxKeyS-  writeBatch db $-    [insertOpCF (mempoolCF db) (MemTimeKey t h) () | (MemTxKey h, t) <- mempool]-      <> [insertOp VersionKey dataVersion]---- | Operations that add a transaction to the mempool: its key and its entry--- in the index by time.-mempoolInsertOps :: DB -> TxHash -> UnixTime -> [BatchOp]-mempoolInsertOps db h t =-  [ insertOpCF (mempoolCF db) (MemTxKey h) t,-    insertOpCF (mempoolCF db) (MemTimeKey t h) ()-  ]+-- | A reader of a snapshot of the database of another reader: the same+-- in everything, but that it reads the snapshot.+snapshotReader :: DatabaseReader -> DB -> IO DatabaseReader+snapshotReader r snapshot = return r {db = snapshot} --- | Set the version of a new database.-setInitRocksDB :: DB -> IO ()-setInitRocksDB db = insert db VersionKey dataVersion+-- | Run an action with a reader of a snapshot of the database as it is now,+-- for reads that must see one moment. The reader must not be used after+-- the action returns: the snapshot is released then.+withSnapshotReader :: DatabaseReader -> (DatabaseReader -> IO a) -> IO a+withSnapshotReader r action = withSnapshot r.db $ \snapshot -> snapshotReader r snapshot >>= action
+ src/Haskoin/Store/Database/Table.hs view
@@ -0,0 +1,127 @@+-- |+-- Module      : Haskoin.Store.Database.Table+-- Copyright   : (c) Haskoin Developers+-- License     : MIT+-- Maintainer  : jprupp@protonmail.ch+-- Stability   : experimental+-- Portability : POSIX+--+-- The hash tables of a batch of writes ("Haskoin.Store.Database.Writer"):+-- mutable tables whose keys are hashed quickly.+--+-- = Why not the 'Hashable' instances of the keys+--+-- A batch keeps its changes in hash tables keyed by outputs, addresses and+-- pairs of them, and importing a block makes some forty insertions and+-- lookups for each transaction. The 'Hashable' instances of those types are+-- derived generically, which walks the structure of the type for each+-- hash: 0.4 microseconds for an output or an address, 0.8 for an address+-- with a transaction, where hashing the 32 bytes of a transaction hash+-- takes 0.03. An insertion under an address and a transaction took 3.4+-- microseconds, and the tables were most of the time of a block once its+-- reads were out of the way.+--+-- Every one of those keys holds a hash already: of a transaction, a public+-- key or a script. Those bytes are as good as random, so 'tableHash' mixes+-- them and nothing else: the same output of two transactions, or two+-- outputs of the same address, differ in them. The insertion above takes+-- 0.6 microseconds with it.+--+-- The alternatives were faster instances in haskoin-core, which is where+-- they belong and would serve everyone, but is another package with its+-- own releases; and tables keyed by the serialised keys, which hash fast+-- too, but serialising a key costs more than this hash, and has to be done+-- for lookups as well.+module Haskoin.Store.Database.Table+  ( Table,+    TableKey (..),+    new,+    lookup,+    insert,+    delete,+    toList,+  )+where++import Data.Coerce (coerce)+import Data.HashTable.IO qualified as H+import Data.Hashable (Hashable (..))+import Data.Word (Word32)+import Haskoin (Address (..), BlockHash, OutPoint (..), TxHash)+import Haskoin.Store.Data (BlockRef, TxRef (..))+import Prelude hiding (lookup)++-- | A mutable hash table, for one thread at a time.+newtype Table k v = Table (H.BasicHashTable (Key k) v)++-- | A key as the table holds it, hashed with 'tableHash'.+newtype Key k = Key k+  deriving (Eq)++instance (TableKey k) => Hashable (Key k) where+  hashWithSalt salt (Key k) = tableHash salt k++-- | What can be the key of a t'Table'.+class (Eq k) => TableKey k where+  -- | Mix a key into a salt. Two keys that are equal must give the same+  -- result; two that are not should differ in what is mixed, though they+  -- need not: the table compares the keys themselves in the end.+  tableHash :: Int -> k -> Int++instance TableKey TxHash where+  tableHash = hashWithSalt++instance TableKey BlockHash where+  tableHash = hashWithSalt++instance TableKey Word32 where+  tableHash = hashWithSalt++-- | The hash of the transaction and the index of the output.+instance TableKey OutPoint where+  tableHash salt p = salt `hashWithSalt` p.hash `hashWithSalt` p.index++-- | The hash that the address holds, for the usual kinds of address, each+-- with a salt of its own, so that a script and a key with the same hash+-- stay apart. The others are hashed as their type says.+instance TableKey Address where+  tableHash salt = \case+    PubKeyAddress h -> hashWithSalt salt h+    ScriptAddress h -> hashWithSalt (salt + 1) h+    WitnessPubKeyAddress h -> hashWithSalt (salt + 2) h+    WitnessScriptAddress h -> hashWithSalt (salt + 3) h+    a -> hashWithSalt (salt + 4) a++-- | A group of outputs of a transaction: its hash and the number of the+-- group.+instance TableKey (TxHash, Word32) where+  tableHash salt (h, g) = salt `hashWithSalt` h `hashWithSalt` g++-- | The address and the hash of the transaction. The position of the+-- transaction is left out: a transaction has one at a time.+instance TableKey (Address, TxRef) where+  tableHash salt (a, t) = tableHash salt a `hashWithSalt` t.txid++-- | The address and the output, without the position of its transaction.+instance TableKey (Address, BlockRef, OutPoint) where+  tableHash salt (a, _, p) = tableHash (tableHash salt a) p++-- | An empty table.+new :: IO (Table k v)+new = Table <$> H.new++-- | The value of a key, if the table has it.+lookup :: (TableKey k) => Table k v -> k -> IO (Maybe v)+lookup (Table t) k = H.lookup t (Key k)++-- | Set the value of a key.+insert :: (TableKey k) => Table k v -> k -> v -> IO ()+insert (Table t) k = H.insert t (Key k)++-- | Take a key out of the table, if it is there.+delete :: (TableKey k) => Table k v -> k -> IO ()+delete (Table t) k = H.delete t (Key k)++-- | Every key with its value, in no particular order.+toList :: forall k v. (TableKey k) => Table k v -> IO [(k, v)]+toList (Table t) = coerce <$> (H.toList t :: IO [(Key k, v)])
src/Haskoin/Store/Database/Types.hs view
@@ -33,7 +33,7 @@ -- +-----------+--------------+----------------------------------------+ -- | 0x06      | 'AddrOutKey' | 'OutVal'                               | -- +-----------+--------------+----------------------------------------+--- | 0x07      | 'MemKey'     | whole mempool (legacy)                 |+-- | 0x07      | @MemKey@     | whole mempool (legacy)                 | -- +-----------+--------------+----------------------------------------+ -- | 0x08      | 'MemTxKey'   | time a transaction entered the mempool | -- +-----------+--------------+----------------------------------------+@@ -41,29 +41,27 @@ -- +-----------+--------------+----------------------------------------+ -- | 0x0a      | 'VersionKey' | database version                       | -- +-----------+--------------+----------------------------------------+--- | 0x0b      | 'MemTimeKey' | nothing                                |+-- | 0x0b      | @MemTimeKey@ | nothing                                | -- +-----------+--------------+----------------------------------------+ ----- = Keys of the layered layout+-- = Keys of the old mempool, and later ones ----- They belong to the layout that the store runs on, that of--- @docs\/layered-database.md@, version 22, in which the column families of--- the chain hold nothing from the mempool and everything about the mempool--- is in two column families of its own. t'TxKey' is used there too, for the--- bodies of mempool transactions. "Haskoin.Store.Database.Mempool" reads--- and writes them, and "Haskoin.Store.Database.Migrate" moves a database of--- version 20 to that layout.+-- They are the keys of the two column families that held the mempool of+-- the store until version 1.16.0, when the mempool moved to the pool of the+-- fleet, in Redis. Nothing reads or writes them any more; they are kept to+-- say what such a database holds. t'MemSummary' lives on as what the pool+-- says of an address ("Haskoin.Store.Database.Query.Pooled"). -- -- +-----------+------------------+-------------------------------------------+ -- | Prefix    | Key              | Value                                     | -- +===========+==================+===========================================+--- | 0x0c      | t'MemSpenderKey' | 'Spender': mempool input that spends an   |+-- | 0x0c      | @MemSpenderKey@  | 'Spender': mempool input that spends an   | -- |           |                  | output                                    | -- +-----------+------------------+-------------------------------------------+--- | 0x0d      | 'MemAddrKey'     | 'MemAddrVal': what the mempool does at an |+-- | 0x0d      | @MemAddrKey@     | @MemAddrVal@: what the mempool did at an  | -- |           |                  | address                                   | -- +-----------+------------------+-------------------------------------------+--- | 0x0e      | t'MemDeletedKey' | 'TxData' of a transaction that left the   |+-- | 0x0e      | @MemDeletedKey@  | 'TxData' of a transaction that left the   | -- |           |                  | mempool unconfirmed: not written any more | -- +-----------+------------------+-------------------------------------------+ -- | 0x0f      | t'MigrationKey'  | version a migration in progress started   |@@ -72,6 +70,10 @@ -- | 0x10      | t'SpenderKey'    | t'SpenderVal': inputs of blocks that      | -- |           |                  | spend outputs of a transaction            | -- +-----------+------------------+-------------------------------------------++--+-- Since version 23 the values of 'BalKey' and of t'SpenderKey' can also+-- be written as merge records, which RocksDB combines into the value:+-- "Haskoin.Store.Database.Merge". module Haskoin.Store.Database.Types   ( AddrTxKey (..),     AddrOutKey (..),@@ -79,14 +81,8 @@     BlockKey (..),     BalKey (..),     HeightKey (..),-    MemKey (..),     MemTxKey (..),-    MemTimeKey (..),-    MemSpenderKey (..),-    MemAddrKey (..),-    MemAddrVal (..),     MemSummary (..),-    MemDeletedKey (..),     MigrationKey (..),     ReplacedKey (..),     ReplacedAtKey (..),@@ -101,6 +97,7 @@     BalVal (..),     valToBalance,     balanceToVal,+    addBalVal,     UnspentVal (..),     toUnspent,     unspentToVal,@@ -127,7 +124,6 @@     getBytes,     getWord16be,     getWord32be,-    getWord64be,     getWord8,     putWord32be,     putWord64be,@@ -148,7 +144,6 @@     eitherToMaybe,     scriptToAddressBS,   )-import Haskoin.Crypto import Haskoin.Store.Common (Replaced) import Haskoin.Store.Data   ( Balance (..),@@ -320,37 +315,20 @@ instance KeyValue UnspentKey UnspentVal  -- | Build an 'Unspent' from an address output entry. Returns 'Nothing' for a short key.-toUnspent :: Network -> Ctx -> AddrOutKey -> OutVal -> Maybe Unspent-toUnspent net ctx (AddrOutKey _address block outpoint) (OutVal value script) =+toUnspent :: Network -> AddrOutKey -> OutVal -> Maybe Unspent+toUnspent net (AddrOutKey _address block outpoint) (OutVal value script) =   Just     Unspent       { block = block,         value = value,         script = script,         outpoint = outpoint,-        address = eitherToMaybe (scriptToAddressBS net ctx script)+        address = eitherToMaybe (scriptToAddressBS net script)       }-toUnspent _ _ _ _ = Nothing---- | Mempool transaction database key. Legacy: databases of version 18 kept the--- whole mempool under this key. It is only read to migrate them.-data MemKey-  = MemKey-  deriving (Show, Read)--instance Serialize MemKey where-  -- 0x07-  put MemKey = putWord8 0x07-  get = do-    guard . (== 0x07) =<< getWord8-    return MemKey--instance Key MemKey--instance KeyValue MemKey [(UnixTime, TxHash)]+toUnspent _ _ _ = Nothing  -- | Key of one mempool transaction in the mempool column family, with the--- time it entered the mempool as its value. Replaces 'MemKey', which kept+-- time it entered the mempool as its value. Replaces @MemKey@, which kept -- the whole mempool in a single value. data MemTxKey   = -- | key of a mempool transaction@@ -374,184 +352,16 @@  instance KeyValue MemTxKey UnixTime --- | Index of the mempool by the time each transaction entered it, in the--- mempool column family next to 'MemTxKey'. Keys sort by time, then by--- hash, as the pairs in @getMempool@ do, so a page of the mempool is read--- with a short scan instead of loading and sorting every entry.-data MemTimeKey-  = -- | entry of a mempool transaction, by time and hash-    MemTimeKey !UnixTime !TxHash-  | -- | short key that matches every entry-    MemTimeKeyS-  deriving (Show, Read, Eq, Ord, Generic, Hashable)--instance Serialize MemTimeKey where-  -- 0x0b · UnixTime (big endian) · TxHash-  put (MemTimeKey t h) = do-    put MemTimeKeyS-    putWord64be t-    put h-  -- 0x0b-  put MemTimeKeyS = putWord8 0x0b-  get = do-    guard . (== 0x0b) =<< getWord8-    MemTimeKey <$> getWord64be <*> get--instance Key MemTimeKey--instance KeyValue MemTimeKey ()---- | Key of the input of a mempool transaction that spends an output, in the--- mempool column family of the layered layout. The output may be one of a--- confirmed transaction or of another mempool transaction: the key is the--- same, so nothing has to change here when the transaction that created the--- output is confirmed.------ Spenders in the chain's column families are part of the body of the--- transaction whose output they spend. A mempool spender cannot be: writing--- it there would let the mempool change the data of the chain.-data MemSpenderKey-  = -- | key of the mempool spender of an output-    MemSpenderKey !OutPoint-  | -- | short key that matches the mempool spenders of the outputs of a-    -- transaction-    MemSpenderKeyS !TxHash-  | -- | short key that matches every mempool spender-    MemSpenderKeyB-  deriving (Show, Read, Eq, Ord, Generic, Hashable)--instance Serialize MemSpenderKey where-  -- 0x0c · TxHash · Index-  put (MemSpenderKey OutPoint {hash = h, index = i}) = do-    put (MemSpenderKeyS h)-    put i-  -- 0x0c · TxHash-  put (MemSpenderKeyS h) = do-    put MemSpenderKeyB-    put h-  -- 0x0c-  put MemSpenderKeyB = putWord8 0x0c-  get = do-    guard . (== 0x0c) =<< getWord8-    h <- get-    i <- get-    return $ MemSpenderKey OutPoint {hash = h, index = i}--instance Key MemSpenderKey--instance KeyValue MemSpenderKey Spender---- | Key of something that the mempool does at an address, in the column--- family of mempool address records of the layered layout. There are three--- kinds, and all those of an address are read with one scan of its prefix.------ Every entry depends on one mempool transaction alone: on its inputs, with--- the outputs they spend, and on its outputs. None says whether an output--- that is spent was created by a confirmed transaction or by another--- mempool transaction, nor whether an output that is created is spent. So--- a transaction is added by writing its entries and removed by deleting--- them, whatever else is in the mempool, and nothing here changes when a--- block confirms some transactions and not others. What a query needs is--- worked out from the entries of the address when it is answered; see--- "Haskoin.Store.Database.Mempool".------ Times are stored complemented, as in a mempool 'BlockRef', so that entries--- come out newest first, which is the order of the API.-data MemAddrKey-  = -- | a mempool transaction that spends from or pays to the address, by-    -- the time it entered the mempool-    MemAddrTxKey !Address !UnixTime !TxHash-  | -- | an output of the address that a mempool transaction spends-    MemAddrSpentKey !Address !OutPoint-  | -- | an output that a mempool transaction pays to the address, spent or-    -- not, by the time the transaction entered the mempool-    MemAddrOutKey !Address !UnixTime !OutPoint-  | -- | what the entries of an address add up to; see t'MemSummary'-    MemAddrSumKey !Address-  | -- | short key that matches the transactions of an address-    MemAddrTxKeyA !Address-  | -- | short key that matches the outputs paid to an address-    MemAddrOutKeyA !Address-  | -- | short key that matches every entry of an address-    MemAddrKeyA !Address-  | -- | short key that matches the entries of every address-    MemAddrKeyS-  deriving (Show, Eq, Ord, Generic, Hashable)--instance Serialize MemAddrKey where-  -- 0x0d · Address · 0x00 · time · TxHash-  put (MemAddrTxKey a t h) = do-    put (MemAddrKeyA a)-    putWord8 0x00-    putWord64be (maxBound - t)-    put h-  -- 0x0d · Address · 0x01 · OutPoint-  put (MemAddrSpentKey a p) = do-    put (MemAddrKeyA a)-    putWord8 0x01-    put p-  -- 0x0d · Address · 0x02 · time · OutPoint-  put (MemAddrOutKey a t p) = do-    put (MemAddrKeyA a)-    putWord8 0x02-    putWord64be (maxBound - t)-    put p-  -- 0x0d · Address · 0x03-  put (MemAddrSumKey a) = do-    put (MemAddrKeyA a)-    putWord8 0x03-  -- 0x0d · Address · 0x00-  put (MemAddrTxKeyA a) = do-    put (MemAddrKeyA a)-    putWord8 0x00-  -- 0x0d · Address · 0x02-  put (MemAddrOutKeyA a) = do-    put (MemAddrKeyA a)-    putWord8 0x02-  -- 0x0d · Address-  put (MemAddrKeyA a) = do-    put MemAddrKeyS-    put a-  -- 0x0d-  put MemAddrKeyS = putWord8 0x0d-  get = do-    guard . (== 0x0d) =<< getWord8-    a <- get-    getWord8 >>= \case-      0x00 -> MemAddrTxKey a . (maxBound -) <$> getWord64be <*> get-      0x01 -> MemAddrSpentKey a <$> get-      0x02 -> MemAddrOutKey a . (maxBound -) <$> getWord64be <*> get-      0x03 -> return (MemAddrSumKey a)-      _ -> fail "Cannot decode MemAddrKey"--instance Key MemAddrKey---- | Value of a 'MemAddrKey': nothing for a transaction, the amount in--- satoshi of an output that is spent, and the amount and script of an output--- that is created. The first byte says which, so that a value can be decoded--- without its key.-data MemAddrVal-  = -- | value of a 'MemAddrTxKey'-    MemAddrTxVal-  | -- | value of a 'MemAddrSpentKey': amount of the output-    MemAddrSpentVal !Word64-  | -- | value of a 'MemAddrOutKey': amount and script of the output-    MemAddrOutVal !Word64 !ByteString-  | -- | value of a 'MemAddrSumKey'-    MemAddrSumVal !MemSummary-  deriving (Show, Read, Eq, Ord, Generic, Hashable, NFData)---- | What the entries of an address in the mempool area add up to, kept so--- that its balance is one read by key however many entries there are, and--- so that an address with nothing in the mempool is found by a key that is--- not there instead of a scan.+-- | What the mempool does at an address, added up: what the pool of the+-- fleet gives for an address ("Haskoin.Store.Fleet.Pool"), so that its+-- balance is one read however many transactions touch it. ----- It is a function of the entries of the address, and of nothing else: not--- of the chain. Whether an output that the mempool spends is one of the--- chain or one that the mempool made is the difference between 'spent' and--- 'both', and 'both' only changes when one of the two transactions enters--- or leaves the mempool. So the summary stays right when a block confirms--- some transactions and not others, like the entries themselves.+-- It is a function of the transactions of the pool that touch the address,+-- and of nothing else: not of the chain. Whether an output that the pool+-- spends is one of the chain or one that the pool made is the difference+-- between 'spent' and 'both', and 'both' only changes when one of the two+-- transactions enters or leaves the pool. So the summary stays right when+-- a block confirms some transactions and not others. data MemSummary = MemSummary   { -- | mempool transactions that touch the address     txs :: !Word64,@@ -570,64 +380,6 @@   }   deriving (Show, Read, Eq, Ord, Generic, Hashable, Serialize, NFData) -instance Serialize MemAddrVal where-  put MemAddrTxVal = putWord8 0x00-  put (MemAddrSpentVal v) = putWord8 0x01 >> putWord64be v-  put (MemAddrOutVal v s) = putWord8 0x02 >> putWord64be v >> put s-  put (MemAddrSumVal s) = putWord8 0x03 >> put s-  get =-    getWord8 >>= \case-      0x00 -> return MemAddrTxVal-      0x01 -> MemAddrSpentVal <$> getWord64be-      0x02 -> MemAddrOutVal <$> getWord64be <*> get-      0x03 -> MemAddrSumVal <$> get-      _ -> fail "Cannot decode MemAddrVal"--instance KeyValue MemAddrKey MemAddrVal---- | Key of the body of a transaction that left the mempool without being--- confirmed, in the mempool column family of the layered layout: it was--- replaced, it conflicted with a block, or it was too old. The store keeps--- such bodies, marked as deleted, so that the API can go on saying what--- became of a transaction that it once showed.------ Version 20 keeps them in the column family of transactions, under--- t'TxKey'. There, deleting a mempool transaction writes to the data of the--- chain, which the layered layout does not allow.------ Nothing writes or reads this key any more: since version 22 the store--- keeps nothing of a transaction that leaves the mempool unconfirmed. A--- database that has some has them until its mempool is wiped.-data MemDeletedKey-  = -- | key of a deleted transaction-    MemDeletedKey !TxHash-  | -- | short key that matches the deleted transactions whose hash starts-    -- with these six bytes, as 'TxKeyS' does for the others; see-    -- 'decodeTxKey'-    MemDeletedKeyP !(Word32, Word16)-  | -- | short key that matches every deleted transaction-    MemDeletedKeyS-  deriving (Show, Read, Eq, Ord, Generic, Hashable)--instance Serialize MemDeletedKey where-  -- 0x0e · TxHash-  put (MemDeletedKey h) = do-    put MemDeletedKeyS-    put h-  -- 0x0e · first six bytes of a TxHash-  put (MemDeletedKeyP h) = do-    put MemDeletedKeyS-    put h-  -- 0x0e-  put MemDeletedKeyS = putWord8 0x0e-  get = do-    guard . (== 0x0e) =<< getWord8-    MemDeletedKey <$> get--instance Key MemDeletedKey--instance KeyValue MemDeletedKey TxData- -- | Key that is present while a migration of the database is under way, -- with the version that it started from. A migration that works in several -- batches sets it with the first and removes it with the last, so that a@@ -753,6 +505,19 @@ -- | The inputs of blocks that spend outputs of a group, by the number of -- the output in its transaction. A group none of whose outputs is spent -- has no key.+--+-- The value is one byte with the number of entries, then the entries in+-- ascending order of output, each of forty bytes: the number of the output+-- (four bytes, most significant first), the hash of the transaction that+-- spends it (32) and the number of the input that does (four, most+-- significant first).+--+-- Since database version 23 the column family has merge records too, which+-- the import of a block writes instead of reading a group to add to it+-- ("Haskoin.Store.Database.Merge"). A merge record is of this type and+-- these bytes, with the spenders to add. RocksDB joins them with the group+-- when it is read, the newest entry winning for an output that several+-- have, and a reader never sees one. newtype SpenderVal = SpenderVal (IntMap Spender)   deriving (Show, Eq) @@ -855,6 +620,17 @@  -- | Balance of an address as stored in the database. The address itself is in -- the key.+--+-- The value is the five numbers, each of eight bytes with the most+-- significant first: forty bytes.+--+-- Since database version 23 the column family has merge records too, which+-- the import of a block writes instead of reading a balance to change it+-- ("Haskoin.Store.Database.Merge"). A merge record is of this type and+-- these bytes, with five differences in place of the five numbers: each is+-- added to its number modulo 2^64, so that a number goes down by the two's+-- complement of what is taken off. RocksDB adds them up when the balance+-- is read ('addBalVal' is the same sum), and a reader never sees one. data BalVal = BalVal   { -- | confirmed balance in satoshi     confirmed :: !Word64,@@ -879,6 +655,20 @@ balanceToVal Balance {..} =   BalVal {..} +-- | The sum of two stored balances, or of a balance and its differences,+-- or of two sets of differences: each number modulo 2^64. It is what the+-- merge operator of the column family does+-- ('Haskoin.Store.Database.Merge.balanceOperator').+addBalVal :: BalVal -> BalVal -> BalVal+addBalVal a b =+  BalVal+    { confirmed = a.confirmed + b.confirmed,+      unconfirmed = a.unconfirmed + b.unconfirmed,+      utxo = a.utxo + b.utxo,+      txs = a.txs + b.txs,+      received = a.received + b.received+    }+ -- | Default balance for an address. instance Default BalVal where   def =@@ -906,8 +696,8 @@ unspentToVal Unspent {..} = (outpoint, UnspentVal {..})  -- | Build an 'Unspent' from its outpoint and stored value, decoding the address from the script.-valToUnspent :: Network -> Ctx -> OutPoint -> UnspentVal -> Unspent-valToUnspent net ctx outpoint UnspentVal {..} =+valToUnspent :: Network -> OutPoint -> UnspentVal -> Unspent+valToUnspent net outpoint UnspentVal {..} =   Unspent {..}   where-    address = eitherToMaybe (scriptToAddressBS net ctx script)+    address = eitherToMaybe (scriptToAddressBS net script)
src/Haskoin/Store/Database/Views.hs view
@@ -9,7 +9,7 @@ -- The database as it was at each of its recent blocks: a RocksDB snapshot -- for each, kept until it is no longer needed. Nothing in the store uses -- this yet. It is the first part of the store's side of "A fleet that--- answers as one" in @docs\/layered-database.md@.+-- answers as one" in @docs\/one-fleet.md@. -- -- = Why --@@ -42,22 +42,14 @@   ) where +import Control.Concurrent.STM (TVar, atomically, newTVarIO, readTVar, readTVarIO, stateTVar, writeTVar)+import Control.Exception (mask_) import Control.Monad (forM_, when) import Data.Map.Strict (Map) import Data.Map.Strict qualified as Map import Database.RocksDB (DB, Snapshot, createSnapshot, releaseSnapshot)-import Haskoin.Store.Database.Reader (DatabaseReader (..))-import UnliftIO-  ( TVar,-    atomically,-    bracket,-    mask_,-    newTVarIO,-    readTVar,-    readTVarIO,-    stateTVar,-    writeTVar,-  )+import Haskoin.Store.Database.Reader (DatabaseReader (..), snapshotReader)+import Haskoin.Store.Exception (bracket)  -- | The views of a database, by block. data Views b = Views@@ -119,7 +111,7 @@           writeTVar views.held (Map.insert block h {readers = h.readers + 1} m)           return (Just h)         _ -> return Nothing-    run h = action views.reader {db = fst h.snapshot}+    run h = snapshotReader views.reader (fst h.snapshot) >>= action     leave Nothing = return ()     leave (Just _) = do       free <- atomically . stateTVar views.held $ \m -> case Map.lookup block m of
src/Haskoin/Store/Database/Writer.hs view
@@ -27,717 +27,1246 @@ -- key, and a single atomic batch makes the whole import all-or-nothing. A crash -- or an invalid block leaves the database exactly as it was. ----- = Two layouts------ The writes of the first groups below are to the column families that the--- store uses today, where the chain and the mempool share their indices.--- The last group is for the layered layout of @docs\/layered-database.md@,--- in which the mempool has an area of its own: bodies, spenders, address--- entries and deleted transactions of the mempool. A batch can write to--- both areas, and is still one atomic write. Only--- "Haskoin.Store.Logic.Layered" uses that group, on a database opened with--- 'Haskoin.Store.Database.Reader.withLayeredReader'; a batch that does not--- use it writes nothing there.-module Haskoin.Store.Database.Writer-  ( -- * Batches-    Writer,-    MempoolChanges (..),-    runWriter,-    runWriterMempool,-    writerNet,-    writerCtx,-    writerLogger,--    -- * Reads, with the changes of the batch applied-    getBestBlock,-    getBlocksAtHeight,-    getBlock,-    getTxData,-    getSpender,-    getUnspent,-    getBalance,-    getMempool,--    -- * Writes-    setBest,-    insertBlock,-    setBlocksAtHeight,-    insertTx,-    insertAddrTx,-    deleteAddrTx,-    insertAddrUnspent,-    deleteAddrUnspent,-    addToMempool,-    deleteFromMempool,-    inMempool,-    setBalance,-    insertUnspent,-    deleteUnspent,-    deleteTxData,-    deleteBalance,--    -- * The mempool area of the layered layout-    getMempoolTx,-    getMempoolSpender,-    getMempoolSpenders,-    getChainSpender,-    getChainSpenders,-    insertSpender,-    deleteSpender,-    insertMempoolTx,-    deleteMempoolTx,-    insertMempoolSpender,-    deleteMempoolSpender,-    insertMempoolAddr,-    deleteMempoolAddr,-    getMempoolAddr,--    -- * Transactions that blocks replaced-    insertReplaced,-    deleteReplaced,-    getReplacedAt,-  )-where--import Control.Monad (forM)-import Data.Functor ((<&>))-import Data.HashMap.Strict qualified as HashMap-import Data.HashTable.IO qualified as H-import Data.IntMap.Strict (IntMap)-import Data.IntMap.Strict qualified as IntMap-import Data.List (sortOn)-import Data.Maybe (catMaybes, isJust)-import Data.Ord (Down (..))-import Data.Word (Word32)-import Database.RocksDB (BatchOp, DB)-import Database.RocksDB.Query-  ( deleteOp,-    deleteOpCF,-    insertOp,-    insertOpCF,-    retrieveCF,-    writeBatch,-  )-import Haskoin-  ( Address,-    BlockHash,-    BlockHeight,-    Ctx,-    Network,-    OutPoint (..),-    TxHash,-    headerHash,-    txHash,-  )-import Haskoin.Node (Logger)-import Haskoin.Store.Common (Replaced (..))-import Haskoin.Store.Data-import Haskoin.Store.Database.Mempool qualified as M-import Haskoin.Store.Database.Query qualified as Q-import Haskoin.Store.Database.Reader-import Haskoin.Store.Database.Types-import UnliftIO-  ( IORef,-    liftIO,-    newIORef,-    readIORef,-    writeIORef,-  )---- | A batch of writes: the database it reads from and writes to, the--- changes made so far, and where to log. Make one with 'runWriter'.-data Writer = Writer-  { -- | database to read from and write to-    reader :: !DatabaseReader,-    -- | changes made so far-    memory :: !Memory,-    -- | log messages here-    logger :: !Logger-  }---- | Network of the database of a batch.-writerNet :: Writer -> Network-writerNet w = w.reader.net---- | Secp256k1 context of the database of a batch.-writerCtx :: Writer -> Ctx-writerCtx w = w.reader.ctx---- | Logger passed to 'runWriter'.-writerLogger :: Writer -> Logger-writerLogger w = w.logger---- | Change to the best block: 'Nothing' if unchanged, @Just Nothing@ if deleted.-type BestRef = IORef (Maybe (Maybe BlockHash))---- | Changed blocks: 'Nothing' for a deletion.-type BlockTable = H.BasicHashTable BlockHash (Maybe BlockData)---- | Changed lists of blocks at a height.-type HeightTable = H.BasicHashTable BlockHeight [BlockHash]---- | Changed transactions: 'Nothing' for a deletion.-type TxTable = H.BasicHashTable TxHash (Maybe TxData)---- | Changed unspent outputs: 'Nothing' for a deletion.-type UnspentTable = H.BasicHashTable OutPoint (Maybe Unspent)---- | Changed balances: 'Nothing' for a deletion.-type BalanceTable = H.BasicHashTable Address (Maybe Balance)---- | Changed address transactions: 'Nothing' for a deletion.-type AddrTxTable = H.BasicHashTable (Address, TxRef) (Maybe ())---- | Changed address unspent outputs: 'Nothing' for a deletion.-type AddrOutTable = H.BasicHashTable (Address, BlockRef, OutPoint) (Maybe OutVal)---- | Mempool changes in this batch: 'Nothing' for a deletion.-type MempoolTable = H.BasicHashTable TxHash (Maybe UnixTime)---- | Changed groups of spenders of outputs in the chain.-type SpenderTable = H.BasicHashTable (TxHash, Word32) (IntMap Spender)---- | Changed mempool spenders of outputs: 'Nothing' for a deletion.-type MemSpenderTable = H.BasicHashTable OutPoint (Maybe Spender)---- | Changed entries of the mempool by address: 'Nothing' for a deletion.-type MemAddrTable = H.BasicHashTable MemAddrKey (Maybe MemAddrVal)---- | Changed records of transactions that blocks replaced: 'Nothing' for a--- deletion.-type ReplacedTable = H.BasicHashTable TxHash (Maybe Replaced)---- | Changes made in a batch, not yet written to the database.-data Memory = Memory-  { -- | network constants-    net :: !Network,-    -- | secp256k1 context-    ctx :: !Ctx,-    -- | best block-    best :: !BestRef,-    -- | blocks-    blockTable :: !BlockTable,-    -- | blocks at each height-    heightTable :: !HeightTable,-    -- | transactions-    txTable :: !TxTable,-    -- | unspent outputs-    unspentTable :: !UnspentTable,-    -- | address balances-    balanceTable :: !BalanceTable,-    -- | address transactions-    addressTable :: !AddrTxTable,-    -- | address unspent outputs-    outputTable :: !AddrOutTable,-    -- | mempool-    mempoolTable :: !MempoolTable,-    -- | bodies of mempool transactions, in the mempool area-    memTxTable :: !TxTable,-    -- | mempool spenders of outputs, in the mempool area-    memSpenderTable :: !MemSpenderTable,-    -- | entries of the mempool by address, in the mempool area-    memAddrTable :: !MemAddrTable,-    -- | inputs of blocks that spend outputs, by group of outputs-    -- as t'SpenderKey' says, each group as the batch leaves it: empty for one-    -- that has no spent output left-    spenderTable :: !SpenderTable,-    -- | what is kept of transactions that blocks replaced-    replacedTable :: !ReplacedTable-  }---- | Set the best block.-setBest :: Writer -> BlockHash -> IO ()-setBest w h = writeIORef w.memory.best (Just (Just h))---- | Add or replace the data of a block.-insertBlock :: Writer -> BlockData -> IO ()-insertBlock w b = H.insert w.memory.blockTable (headerHash b.header) (Just b)---- | Set the hashes of the blocks at a height.-setBlocksAtHeight :: Writer -> [BlockHash] -> BlockHeight -> IO ()-setBlocksAtHeight w h g = H.insert w.memory.heightTable g h---- | Add or replace the data of a transaction.-insertTx :: Writer -> TxData -> IO ()-insertTx w t = H.insert w.memory.txTable (txHash t.tx) (Just t)---- | Add a transaction to the index of an address.-insertAddrTx :: Writer -> Address -> TxRef -> IO ()-insertAddrTx w a t = H.insert w.memory.addressTable (a, t) (Just ())---- | Remove a transaction from the index of an address.-deleteAddrTx :: Writer -> Address -> TxRef -> IO ()-deleteAddrTx w a t = H.insert w.memory.addressTable (a, t) Nothing---- | Add an unspent output to the index of an address.-insertAddrUnspent :: Writer -> Address -> Unspent -> IO ()-insertAddrUnspent w a u =-  H.insert w.memory.outputTable (a, u.block, u.outpoint) (Just (OutVal u.value u.script))---- | Remove an unspent output from the index of an address.-deleteAddrUnspent :: Writer -> Address -> Unspent -> IO ()-deleteAddrUnspent w a u =-  H.insert w.memory.outputTable (a, u.block, u.outpoint) Nothing---- | Add a transaction to the mempool with the time it arrived.-addToMempool :: Writer -> TxHash -> UnixTime -> IO ()-addToMempool w x t = H.insert w.memory.mempoolTable x (Just t)---- | Remove a transaction from the mempool.-deleteFromMempool :: Writer -> TxHash -> IO ()-deleteFromMempool w x = H.insert w.memory.mempoolTable x Nothing---- | Set the balance of an address.-setBalance :: Writer -> Balance -> IO ()-setBalance w b = H.insert w.memory.balanceTable b.address (Just b)---- | Add an unspent output.-insertUnspent :: Writer -> Unspent -> IO ()-insertUnspent w u = H.insert w.memory.unspentTable (fst (unspentToVal u)) (Just u)---- | Remove an unspent output.-deleteUnspent :: Writer -> OutPoint -> IO ()-deleteUnspent w p = H.insert w.memory.unspentTable p Nothing---- | Remove the data of a transaction.-deleteTxData :: Writer -> TxHash -> IO ()-deleteTxData w h = H.insert w.memory.txTable h Nothing---- | Remove the balance of an address, as that of one never used.-deleteBalance :: Writer -> Address -> IO ()-deleteBalance w a = H.insert w.memory.balanceTable a Nothing---- | Add or replace the body of a mempool transaction in the mempool area.-insertMempoolTx :: Writer -> TxData -> IO ()-insertMempoolTx w t = H.insert w.memory.memTxTable (txHash t.tx) (Just t)---- | Remove the body of a mempool transaction from the mempool area.-deleteMempoolTx :: Writer -> TxHash -> IO ()-deleteMempoolTx w h = H.insert w.memory.memTxTable h Nothing---- | Record the mempool input that spends an output.-insertMempoolSpender :: Writer -> OutPoint -> Spender -> IO ()-insertMempoolSpender w p s = H.insert w.memory.memSpenderTable p (Just s)---- | Forget the mempool input that spends an output.-deleteMempoolSpender :: Writer -> OutPoint -> IO ()-deleteMempoolSpender w p = H.insert w.memory.memSpenderTable p Nothing---- | Add an entry of the mempool by address.-insertMempoolAddr :: Writer -> MemAddrKey -> MemAddrVal -> IO ()-insertMempoolAddr w k v = H.insert w.memory.memAddrTable k (Just v)---- | Remove an entry of the mempool by address.-deleteMempoolAddr :: Writer -> MemAddrKey -> IO ()-deleteMempoolAddr w k = H.insert w.memory.memAddrTable k Nothing---- | An entry of the mempool by address, with the changes of the batch--- applied.-getMempoolAddr :: Writer -> MemAddrKey -> IO (Maybe MemAddrVal)-getMempoolAddr w k = getLayered w (\s -> H.lookup s.memAddrTable k) (\r -> retrieveCF r.db (mempoolAddrCF r.db) k)---- | Read a value from the changes of the batch, or from the database if the--- batch did not change it.-getLayered :: Writer -> (Memory -> IO (Maybe a)) -> (DatabaseReader -> IO a) -> IO a-getLayered w f g =-  f w.memory >>= \case-    Just x -> return x-    Nothing -> g w.reader---- | Run a batch of writes and commit all its changes to the database in one--- atomic write. Reads within the batch see its own changes. If the function--- throws, nothing is written.-runWriter ::-  Network ->-  Ctx ->-  DatabaseReader ->-  Logger ->-  (Writer -> IO a) ->-  IO a-runWriter net ctx bdb logger f =-  fst <$> runWriterMempool net ctx bdb logger f---- | Transactions that a batch added to the mempool or removed from it,--- compared to what the database held before the batch was written.-data MempoolChanges = MempoolChanges-  { -- | transactions that entered the mempool-    added :: ![TxHash],-    -- | transactions that left the mempool-    removed :: ![TxHash]-  }-  deriving (Eq, Show)---- | Like 'runWriter', but also say how the batch changed the mempool, so--- that callers do not have to read the whole mempool before and after.-runWriterMempool ::-  Network ->-  Ctx ->-  DatabaseReader ->-  Logger ->-  (Writer -> IO a) ->-  IO (a, MempoolChanges)-runWriterMempool net ctx bdb@DatabaseReader {db} logger f = do-  hm <- newMemory net ctx-  x <- f Writer {reader = bdb, memory = hm, logger}-  ops <- hashMapOps db hm-  (mops, changes) <- mempoolOps db hm.mempoolTable-  writeBatch db (ops <> mops)-  return (x, changes)---- | Database operations for all changes of a batch except the mempool.-hashMapOps :: DB -> Memory -> IO [BatchOp]-hashMapOps db Memory {..} =-  mconcat-    <$> sequence-      [ bestBlockOp best,-        blockHashOps db blockTable,-        blockHeightOps db heightTable,-        txOps db txTable,-        balOps db balanceTable,-        addrTxOps db addressTable,-        addrOutOps db outputTable,-        unspentOps db unspentTable,-        memTxOps db memTxTable,-        memSpenderOps db memSpenderTable,-        memAddrOps db memAddrTable,-        spenderOps db spenderTable,-        replacedOps db replacedTable-      ]---- | Operation for the change of the best block, if any.-bestBlockOp :: BestRef -> IO [BatchOp]-bestBlockOp r =-  readIORef r >>= \case-    Nothing -> return []-    Just Nothing -> return [deleteOp BestKey]-    Just (Just b) -> return [insertOp BestKey b]---- | Operations for changed blocks.-blockHashOps :: DB -> BlockTable -> IO [BatchOp]-blockHashOps db t = map (uncurry f) <$> (H.toList t)-  where-    f k (Just d) = insertOpCF (blockCF db) (BlockKey k) d-    f k Nothing = deleteOpCF (blockCF db) (BlockKey k)---- | Operations for changed records of replaced transactions: the record,--- and its entry at the height of its block. A record that is deleted, or--- kept again at another height, takes its old entry with it.-replacedOps :: DB -> ReplacedTable -> IO [BatchOp]-replacedOps db t = concat <$> (H.toList t >>= mapM f)-  where-    f (h, new) = do-      old <- retrieveCF db (blockCF db) (ReplacedKey h) :: IO (Maybe Replaced)-      let forget =-            [ deleteOpCF (heightCF db) (ReplacedAtKey o.height h)-            | Just o <- [old],-              fmap (.height) new /= Just o.height-            ]-      return . (forget <>) $ case new of-        Just r -> [insertOpCF (blockCF db) (ReplacedKey h) r, insertOpCF (heightCF db) (ReplacedAtKey r.height h) ()]-        Nothing -> [deleteOpCF (blockCF db) (ReplacedKey h)]---- | Operations for changed block heights.-blockHeightOps :: DB -> HeightTable -> IO [BatchOp]-blockHeightOps db t = map (uncurry f) <$> (H.toList t)-  where-    f = insertOpCF (heightCF db) . HeightKey---- | Operations for changed transactions.-txOps :: DB -> TxTable -> IO [BatchOp]-txOps db t = map (uncurry f) <$> (H.toList t)-  where-    f k (Just t') = insertOpCF (txCF db) (TxKey k) t'-    f k Nothing = deleteOpCF (txCF db) (TxKey k)---- | Operations for changed balances.-balOps :: DB -> BalanceTable -> IO [BatchOp]-balOps db t = map (uncurry f) <$> (H.toList t)-  where-    f a (Just b) = insertOpCF (balanceCF db) (BalKey a) (balanceToVal b)-    f a Nothing = deleteOpCF (balanceCF db) (BalKey a)---- | Operations for changed address transactions.-addrTxOps :: DB -> AddrTxTable -> IO [BatchOp]-addrTxOps db t = map (uncurry f) <$> (H.toList t)-  where-    f (a, t') (Just ()) = insertOpCF (addrTxCF db) (AddrTxKey a t') ()-    f (a, t') Nothing = deleteOpCF (addrTxCF db) (AddrTxKey a t')---- | Operations for changed address unspent outputs.-addrOutOps :: DB -> AddrOutTable -> IO [BatchOp]-addrOutOps db t = map (uncurry f) <$> (H.toList t)-  where-    f (a, b, p) (Just l) =-      insertOpCF-        (addrOutCF db)-        (AddrOutKey a b p)-        l-    f (a, b, p) Nothing =-      deleteOpCF-        (addrOutCF db)-        (AddrOutKey a b p)---- | Operations for the mempool changes of a batch. Each changed entry is--- compared to the database: its time is needed to update the index by--- time, and the comparison says which transactions really entered or left--- the mempool.-mempoolOps :: DB -> MempoolTable -> IO ([BatchOp], MempoolChanges)-mempoolOps db t = do-  changes <- H.toList t-  rows <- forM changes $ \(h, new) -> do-    old <- retrieveCF db cf (MemTxKey h)-    return (h, old, new)-  return-    ( concatMap ops rows,-      MempoolChanges-        { added = [h | (h, Nothing, Just _) <- rows],-          removed = [h | (h, Just _, Nothing) <- rows]-        }-    )-  where-    cf = mempoolCF db-    ops (_, Nothing, Nothing) = []-    ops (h, Nothing, Just n) = mempoolInsertOps db h n-    ops (h, Just o, Nothing) =-      [deleteOpCF cf (MemTxKey h), deleteOpCF cf (MemTimeKey o h)]-    ops (h, Just o, Just n)-      | o == n = []-      | otherwise = deleteOpCF cf (MemTimeKey o h) : mempoolInsertOps db h n---- | Operations for changed bodies of mempool transactions.-memTxOps :: DB -> TxTable -> IO [BatchOp]-memTxOps db t = map (uncurry f) <$> H.toList t-  where-    f h (Just td) = insertOpCF (mempoolCF db) (TxKey h) td-    f h Nothing = deleteOpCF (mempoolCF db) (TxKey h)---- | Operations for the changed groups of spenders of the chain.-spenderOps :: DB -> SpenderTable -> IO [BatchOp]-spenderOps db t = map (uncurry f) <$> H.toList t-  where-    f (h, g) m-      | IntMap.null m = deleteOpCF (spenderCF db) (SpenderKey h g)-      | otherwise = insertOpCF (spenderCF db) (SpenderKey h g) (SpenderVal m)---- | Operations for changed mempool spenders.-memSpenderOps :: DB -> MemSpenderTable -> IO [BatchOp]-memSpenderOps db t = map (uncurry f) <$> H.toList t-  where-    f p (Just s) = insertOpCF (mempoolCF db) (MemSpenderKey p) s-    f p Nothing = deleteOpCF (mempoolCF db) (MemSpenderKey p)---- | Operations for changed entries of the mempool by address. Their column--- family is only looked up if there are any, as a database of the layout in--- use today does not have it.-memAddrOps :: DB -> MemAddrTable -> IO [BatchOp]-memAddrOps db t = map (uncurry f) <$> H.toList t-  where-    f k (Just v) = insertOpCF (mempoolAddrCF db) k v-    f k Nothing = deleteOpCF (mempoolAddrCF db) k---- | Operations for changed unspent outputs.-unspentOps :: DB -> UnspentTable -> IO [BatchOp]-unspentOps db t = map (uncurry f) <$> (H.toList t)-  where-    f p (Just u) =-      insertOpCF (unspentCF db) (UnspentKey p) (snd (unspentToVal u))-    f p Nothing =-      deleteOpCF (unspentCF db) (UnspentKey p)---- | Best block, with the changes of the batch applied.-getBestBlock :: Writer -> IO (Maybe BlockHash)-getBestBlock w = getLayered w getBestBlockH Q.getBestBlock---- | Blocks at a height, with the changes of the batch applied.-getBlocksAtHeight :: Writer -> BlockHeight -> IO [BlockHash]-getBlocksAtHeight w bh = getLayered w (getBlocksAtHeightH bh) (`Q.getBlocksAtHeight` bh)---- | Data of a block, with the changes of the batch applied.-getBlock :: Writer -> BlockHash -> IO (Maybe BlockData)-getBlock w bh = getLayered w (getBlockH bh) (`Q.getBlock` bh)---- | Data of a transaction, deleted ones included, with the changes of the--- batch applied.-getTxData :: Writer -> TxHash -> IO (Maybe TxData)-getTxData w th = getLayered w (getTxDataH th) (`Q.getTxRecord` th)---- | Spender of an output, with the changes of the batch applied.-getSpender :: Writer -> OutPoint -> IO (Maybe Spender)-getSpender w op = getLayered w (getSpenderH op) fromRecord-  where-    -- the rules of version 20 keep spenders in the record of the-    -- transaction that made the output, and this is theirs-    fromRecord r = (>>= IntMap.lookup (fromIntegral op.index) . (.spenders)) <$> Q.getTxRecord r op.hash---- | Balance of an address, with the changes of the batch applied.-getBalance :: Writer -> Address -> IO (Maybe Balance)-getBalance w a = getLayered w (getBalanceH a) (`Q.getBalance` a)---- | Unspent output, with the changes of the batch applied.-getUnspent :: Writer -> OutPoint -> IO (Maybe Unspent)-getUnspent w op = getLayered w (getUnspentH op) (`Q.getUnspent` op)---- | Whether a transaction is in the mempool, with the changes of the batch--- applied. It reads the small key that lists the transaction, where--- 'getMempoolTx' reads and decodes its whole body: a rule that only needs--- to know asks this.-inMempool :: Writer -> TxHash -> IO Bool-inMempool w h =-  H.lookup w.memory.mempoolTable h >>= \case-    Just change -> return (isJust change)-    Nothing -> isJust <$> (retrieveCF w.reader.db (mempoolCF w.reader.db) (MemTxKey h) :: IO (Maybe UnixTime))---- | Body of a transaction in the mempool area, without its spenders, with--- the changes of the batch applied.-getMempoolTx :: Writer -> TxHash -> IO (Maybe TxData)-getMempoolTx w h = getLayered w (\s -> H.lookup s.memTxTable h) (`M.getMempoolTx` h)---- | Mempool input that spends an output, with the changes of the batch--- applied.-getMempoolSpender :: Writer -> OutPoint -> IO (Maybe Spender)-getMempoolSpender w p = getLayered w (\s -> H.lookup s.memSpenderTable p) stored-  where-    stored r = retrieveCF r.db (mempoolCF r.db) (MemSpenderKey p)---- | Mempool inputs that spend the outputs of a transaction, by output--- index, with the changes of the batch applied. The second argument is the--- number of outputs of the transaction: what the database has for it is--- read with one scan, and the changes of the batch are looked up for each--- output.-getMempoolSpenders :: Writer -> TxHash -> Int -> IO (IntMap Spender)-getMempoolSpenders w h outputs = do-  stored <- M.getMempoolSpenders w.reader h-  changed <- catMaybes <$> mapM change [0 .. outputs - 1]-  return $ IntMap.mapMaybe id (IntMap.union (IntMap.fromList changed) (IntMap.map Just stored))-  where-    change i = fmap (i,) <$> H.lookup w.memory.memSpenderTable (OutPoint h (fromIntegral i))---- | The spenders of a group of outputs of a transaction, with the changes--- of the batch applied.-spenderGroupOf :: Writer -> TxHash -> Word32 -> IO (IntMap Spender)-spenderGroupOf w h g =-  H.lookup w.memory.spenderTable (h, g) >>= \case-    Just m -> return m-    Nothing ->-      retrieveCF w.reader.db (spenderCF w.reader.db) (SpenderKey h g) <&> \case-        Just (SpenderVal m) -> m-        Nothing -> IntMap.empty---- | Note that an input of a block spends an output, in the group of--- outputs that it belongs to. The record of the transaction that made the--- output is not read and not written.-insertSpender :: Writer -> OutPoint -> Spender -> IO ()-insertSpender w p s = do-  let g = p.index `div` spenderGroup-  m <- spenderGroupOf w p.hash g-  H.insert w.memory.spenderTable (p.hash, g) (IntMap.insert (fromIntegral p.index) s m)---- | Note that no block spends an output any more.-deleteSpender :: Writer -> OutPoint -> IO ()-deleteSpender w p = do-  let g = p.index `div` spenderGroup-  m <- spenderGroupOf w p.hash g-  H.insert w.memory.spenderTable (p.hash, g) (IntMap.delete (fromIntegral p.index) m)---- | The input of a block that spends an output, with the changes of the--- batch applied.-getChainSpender :: Writer -> OutPoint -> IO (Maybe Spender)-getChainSpender w p = IntMap.lookup (fromIntegral p.index) <$> spenderGroupOf w p.hash (p.index `div` spenderGroup)---- | The inputs of blocks that spend the outputs of a transaction, by--- output, with the changes of the batch applied, given the number of its--- outputs: what the database has, read with one scan, with the groups--- that the batch changed in place of the stored ones.-getChainSpenders :: Writer -> TxHash -> Int -> IO (IntMap Spender)-getChainSpenders w h outputs = do-  stored <- Q.getChainSpenders w.reader h-  let groups = [0 .. (fromIntegral (max 1 outputs) - 1) `div` spenderGroup]-  changed <- catMaybes <$> mapM (\g -> fmap (g,) <$> H.lookup w.memory.spenderTable (h, g)) groups-  let replaced = [g | (g, _) <- changed]-      kept = IntMap.filterWithKey (\i _ -> (fromIntegral i `div` spenderGroup) `notElem` replaced) stored-  return (IntMap.unions (kept : map snd changed))---- | Keep what a block replaced.-insertReplaced :: Writer -> TxHash -> Replaced -> IO ()-insertReplaced w h r = H.insert w.memory.replacedTable h (Just r)---- | Forget what was kept of a replaced transaction.-deleteReplaced :: Writer -> TxHash -> IO ()-deleteReplaced w h = H.insert w.memory.replacedTable h Nothing---- | The transactions that the block at a height replaced, with the changes--- of the batch applied.-getReplacedAt :: Writer -> BlockHeight -> IO [TxHash]-getReplacedAt w height = do-  stored <- Q.getReplacedAt w.reader height-  changes <- H.toList w.memory.replacedTable-  let changed = HashMap.fromList changes-      kept = filter (not . (`HashMap.member` changed)) stored-      added = [h | (h, Just r) <- changes, r.height == height]-  return (kept <> added)---- | The mempool in the database with the changes of this batch applied,--- newest first.-getMempool :: Writer -> IO [(UnixTime, TxHash)]-getMempool w = do-  stored <- Q.getMempool w.reader-  changes <- H.toList w.memory.mempoolTable-  let changed = HashMap.fromList changes-      kept = filter (not . (`HashMap.member` changed) . snd) stored-      added = [(t, h) | (h, Just t) <- changes]-  return $ sortOn Down (kept <> added)---- | Empty set of changes.-newMemory ::-  Network ->-  Ctx ->-  IO Memory-newMemory net ctx = do-  best <- newIORef Nothing-  blockTable <- liftIO H.new-  heightTable <- liftIO H.new-  txTable <- liftIO H.new-  unspentTable <- liftIO H.new-  balanceTable <- liftIO H.new-  addressTable <- liftIO H.new-  outputTable <- liftIO H.new-  mempoolTable <- liftIO H.new-  memTxTable <- liftIO H.new-  memSpenderTable <- liftIO H.new-  memAddrTable <- liftIO H.new-  spenderTable <- liftIO H.new-  replacedTable <- liftIO H.new-  return Memory {..}---- | Read the best block from the changes of the batch: 'Nothing' if unchanged, @Just Nothing@ if deleted.-getBestBlockH :: Memory -> IO (Maybe (Maybe BlockHash))-getBestBlockH = readIORef . (.best)---- | Read the blocks at a height from the changes of the batch: 'Nothing' if unchanged, @Just Nothing@ if deleted.-getBlocksAtHeightH :: BlockHeight -> Memory -> IO (Maybe [BlockHash])-getBlocksAtHeightH h s = H.lookup s.heightTable h---- | Read a block from the changes of the batch: 'Nothing' if unchanged, @Just Nothing@ if deleted.-getBlockH :: BlockHash -> Memory -> IO (Maybe (Maybe BlockData))-getBlockH h s = H.lookup s.blockTable h---- | Read a transaction from the changes of the batch: 'Nothing' if unchanged, @Just Nothing@ if deleted.-getTxDataH :: TxHash -> Memory -> IO (Maybe (Maybe TxData))-getTxDataH t s = H.lookup s.txTable t---- | Read the spender of an output from the transaction changed in the batch:--- 'Nothing' if the transaction did not change, @Just Nothing@ if the output is--- not spent.-getSpenderH :: OutPoint -> Memory -> IO (Maybe (Maybe Spender))-getSpenderH op s = do-  fmap (f =<<) <$> getTxDataH op.hash s-  where-    f = IntMap.lookup (fromIntegral op.index) . (.spenders)---- | Read the balance of an address from the changes of the batch: 'Nothing' if unchanged, @Just Nothing@ if deleted.-getBalanceH :: Address -> Memory -> IO (Maybe (Maybe Balance))-getBalanceH a s = H.lookup s.balanceTable a---- | Read an unspent output from the changes of the batch: 'Nothing' if unchanged, @Just Nothing@ if deleted.-getUnspentH :: OutPoint -> Memory -> IO (Maybe (Maybe Unspent))-getUnspentH p Memory {..} = H.lookup unspentTable p+-- = Reading each value once+--+-- A batch also remembers what it read from the database (its /read cache/):+-- a read looks in the changes of the batch, then in what the batch read+-- before, and only then in the database. The database does not change while+-- a batch is open, as the block store is its only writer and writes one+-- batch at a time, so a value read once is good for the rest of the batch.+-- The changes are always looked at first: the cache can never hide one.+--+-- The cache is there so that the reads of a batch can be done ahead of the+-- rules that need them. Importing a block reads an unspent output for every+-- input, thousands of keys, and the rules ask for them one at a time from+-- one thread. The functions under "Reading ahead" read any number of keys+-- with RocksDB's multi-get, split over all cores, and put the answers in+-- the cache, those that say "not there" too. The rules then run as they are+-- and find everything in memory. A key that was not read ahead is read from+-- the database when it is asked for, and cached, so that reading ahead is+-- never needed to be right: it only moves the time of the reads.+--+-- 'withoutReadCache' gives a writer that reads the database every time, as+-- every batch did until version 1.15.1: the benchmark compares the two.+--+-- = Merges: changes to values that the batch has not read+--+-- The import of a block changes the balance of every address that it+-- touches and adds a spender to the group of every output that it spends.+-- It does not need to know either value, and reading them was most of the+-- reads of a block. So a batch can hold, for a balance or a group of+-- spenders, a change to a value that it has not read: the differences to+-- the five numbers of a balance ('mergeBalance'), the spenders to add to a+-- group ('mergeSpender'). They are written as merge records, one for each+-- key, which RocksDB combines with the value when it is read+-- ("Haskoin.Store.Database.Merge").+--+-- For each of those keys a batch holds /either/ a whole value, which it+-- writes as a put or a delete as before, /or/ pending merge data, never+-- both:+--+-- * a merge on a key that has a whole value in the batch is applied to+--   that value, in memory;+--+-- * a whole value written for a key that has pending merge data replaces+--   the data. Whoever worked that value out read the key through the+--   batch, and so saw the data applied;+--+-- * a read through the batch gives the value of the database with the+--   pending merge data applied: 'getBalance', 'getChainSpender',+--   'getChainSpenders'. It reads the database for it, like any read of a+--   key that the batch has not read: what the merge saves is the read of+--   a key that nobody asks for.+--+-- Only the import of a block merges+-- ('Haskoin.Store.Logic.importBlock'). Reverting a block reads each value+-- and writes it whole.+module Haskoin.Store.Database.Writer+  ( -- * Batches+    Writer,+    runWriter,+    runWriterReport,+    BatchReport (..),+    writerNet,+    writerLogger,+    withoutReadCache,++    -- * Reading ahead+    readsAhead,+    readAheadUnspents,+    readAheadBalances,+    readAheadSpenders,++    -- * Timing the import of a block+    BlockTimes (..),+    ReadTimes (..),+    noReads,+    noteBlockTimes,+    stopwatch,++    -- * Work on all cores+    onAllCores,++    -- * Reads, with the changes of the batch applied+    getBestBlock,+    getBlocksAtHeight,+    getBlock,+    getTxData,+    getUnspent,+    getBalance,++    -- * Writes+    setBest,+    insertBlock,+    setBlocksAtHeight,+    insertTx,+    insertTxOf,+    insertAddrTx,+    deleteAddrTx,+    insertAddrUnspent,+    deleteAddrUnspent,+    setBalance,+    mergeBalance,+    insertUnspent,+    deleteUnspent,+    deleteTxData,+    deleteBalance,++    -- * Spenders of the chain+    getChainSpender,+    getChainSpenders,+    insertSpender,+    mergeSpender,+    deleteSpender,++    -- * Transactions that blocks replaced+    insertReplaced,+    deleteReplaced,+    getReplacedAt,+  )+where++import Control.Concurrent (getNumCapabilities)+import Control.Concurrent.Async (wait, withAsyncOn)+import Control.DeepSeq (NFData, force)+import Control.Exception (evaluate)+import Control.Monad (filterM, forM, when)+import Data.ByteString qualified as B+import Data.ByteString.Builder.Extra (Next (..), runBuilder)+import Data.ByteString.Internal (ByteString (BS))+import Data.Default (def)+import Data.Functor ((<&>))+import Data.HashMap.Strict qualified as HashMap+import Data.IORef (IORef, modifyIORef', newIORef, readIORef, writeIORef)+import Data.IntMap.Strict (IntMap)+import Data.IntMap.Strict qualified as IntMap+import Data.Maybe (catMaybes, fromMaybe, isJust)+import Data.Serialize (Put, Serialize, decode, encode, put, runPut)+import Data.Serialize.Put (execPut, putByteString)+import Data.Word (Word32, Word8)+import Database.RocksDB+  ( BatchOp (..),+    ColumnFamily,+    DB,+    PackedOp (..),+    multiGetCF,+    packOps,+    packedHeaderSize,+    pokePackedHeader,+    writePacked,+  )+import Database.RocksDB.Query+  ( Key,+    KeyValue,+    deleteOp,+    deleteOpCF,+    insertOp,+    insertOpCF,+    retrieveCF,+    writeBatch,+  )+import Foreign.ForeignPtr (ForeignPtr, plusForeignPtr, withForeignPtr)+import Foreign.Ptr (plusPtr)+import GHC.Clock (getMonotonicTime)+import GHC.ForeignPtr (mallocPlainForeignPtrBytes)+import Haskoin+  ( Address,+    BlockHash,+    BlockHeight,+    Network,+    OutPoint (..),+    TxHash,+    headerHash,+    txHash,+  )+import Haskoin.Node (Logger)+import Haskoin.Store.Common (Replaced (..))+import Haskoin.Store.Data+import Haskoin.Store.Database.Query qualified as Q+import Haskoin.Store.Database.Reader+import Haskoin.Store.Database.Table (Table, TableKey)+import Haskoin.Store.Database.Table qualified as T+import Haskoin.Store.Database.Types+import System.Metrics.StatsD (incrementCounter)++-- | A batch of writes: the database it reads from and writes to, the+-- changes made so far, and where to log. Make one with 'runWriter'.+data Writer = Writer+  { -- | database to read from and write to+    reader :: !DatabaseReader,+    -- | changes made so far, and values read+    memory :: !Memory,+    -- | log messages here+    logger :: !Logger,+    -- | whether a read looks in the values read before going to the+    -- database, and adds to them ('withoutReadCache')+    cached :: !Bool+  }++-- | The same batch, for reads that go to the database every time the batch+-- has not changed what they ask for, as every read did before the read+-- cache, and writes that land where those of the writer given do. Reading+-- ahead does nothing through it.+--+-- It is there for the tests, which run the rules through it to check that+-- the cache changes nothing, and for the benchmark, which measures what the+-- cache and reading ahead gain.+withoutReadCache :: Writer -> Writer+withoutReadCache w = w {cached = False}++-- | Whether reading ahead does anything for this writer: not through+-- 'withoutReadCache'.+readsAhead :: Writer -> Bool+readsAhead w = w.cached++-- | Network of the database of a batch.+writerNet :: Writer -> Network+writerNet w = w.reader.net++-- | Logger passed to 'runWriter'.+writerLogger :: Writer -> Logger+writerLogger w = w.logger++-- | Change to the best block: 'Nothing' if unchanged, @Just Nothing@ if deleted.+type BestRef = IORef (Maybe (Maybe BlockHash))++-- | Changed blocks: 'Nothing' for a deletion.+type BlockTable = Table BlockHash (Maybe BlockData)++-- | Changed lists of blocks at a height.+type HeightTable = Table BlockHeight [BlockHash]++-- | Changed transactions: 'Nothing' for a deletion.+type TxTable = Table TxHash (Maybe TxData)++-- | Changed unspent outputs: 'Nothing' for a deletion.+type UnspentTable = Table OutPoint (Maybe Unspent)++-- | Changed balances: 'Nothing' for a deletion.+type BalanceTable = Table Address (Maybe Balance)++-- | Differences to balances that the batch has not read, each the sum of+-- what the batch does to its address: what a merge record will hold.+type BalanceMergeTable = Table Address BalVal++-- | Changed address transactions: 'Nothing' for a deletion.+type AddrTxTable = Table (Address, TxRef) (Maybe ())++-- | Changed address unspent outputs: 'Nothing' for a deletion.+type AddrOutTable = Table (Address, BlockRef, OutPoint) (Maybe OutVal)++-- | Changed groups of spenders of outputs in the chain.+type SpenderTable = Table (TxHash, Word32) (IntMap Spender)++-- | Changed records of transactions that blocks replaced: 'Nothing' for a+-- deletion.+type ReplacedTable = Table TxHash (Maybe Replaced)++-- | Values that a batch read from the database, by key, with 'Nothing' for+-- a key that the database does not have. They are what the database held+-- when the batch started, and holds until the batch is written.+data ReadCache = ReadCache+  { -- | records of transactions of the chain+    txs :: !TxTable,+    -- | unspent outputs+    unspents :: !UnspentTable,+    -- | address balances+    balances :: !BalanceTable,+    -- | groups of spenders of outputs of the chain, empty for one that the+    -- database does not have+    spenders :: !SpenderTable+  }++-- | Changes made in a batch, not yet written to the database, and what the+-- batch read.+data Memory = Memory+  { -- | network constants+    net :: !Network,+    -- | best block+    best :: !BestRef,+    -- | blocks+    blockTable :: !BlockTable,+    -- | blocks at each height+    heightTable :: !HeightTable,+    -- | transactions+    txTable :: !TxTable,+    -- | unspent outputs+    unspentTable :: !UnspentTable,+    -- | address balances that the batch has as whole values+    balanceTable :: !BalanceTable,+    -- | differences to the balances of addresses that are not in+    -- 'balanceTable': an address is in one of the two at most+    balanceMerges :: !BalanceMergeTable,+    -- | address transactions+    addressTable :: !AddrTxTable,+    -- | address unspent outputs+    outputTable :: !AddrOutTable,+    -- | inputs of blocks that spend outputs, by group of outputs+    -- as t'SpenderKey' says, each group as the batch leaves it: empty for one+    -- that has no spent output left+    spenderTable :: !SpenderTable,+    -- | spenders to add to the groups that are not in 'spenderTable', which+    -- the batch has not read: a group is in one of the two at most+    spenderMerges :: !SpenderTable,+    -- | what is kept of transactions that blocks replaced+    replacedTable :: !ReplacedTable,+    -- | values read from the database+    cache :: !ReadCache,+    -- | how long the stages of the imports of blocks took, if any block was+    -- imported+    times :: !(IORef (Maybe BlockTimes))+  }++-- | Set the best block.+setBest :: Writer -> BlockHash -> IO ()+setBest w h = writeIORef w.memory.best (Just (Just h))++-- | Add or replace the data of a block.+insertBlock :: Writer -> BlockData -> IO ()+insertBlock w b = T.insert w.memory.blockTable (headerHash b.header) (Just b)++-- | Set the hashes of the blocks at a height.+setBlocksAtHeight :: Writer -> [BlockHash] -> BlockHeight -> IO ()+setBlocksAtHeight w h g = T.insert w.memory.heightTable g h++-- | Add or replace the data of a transaction.+insertTx :: Writer -> TxData -> IO ()+insertTx w t = insertTxOf w (txHash t.tx) t++-- | 'insertTx' given the hash of the transaction, for a caller that has+-- it.+insertTxOf :: Writer -> TxHash -> TxData -> IO ()+insertTxOf w h t = T.insert w.memory.txTable h (Just t)++-- | Add a transaction to the index of an address.+insertAddrTx :: Writer -> Address -> TxRef -> IO ()+insertAddrTx w a t = T.insert w.memory.addressTable (a, t) (Just ())++-- | Remove a transaction from the index of an address.+deleteAddrTx :: Writer -> Address -> TxRef -> IO ()+deleteAddrTx w a t = T.insert w.memory.addressTable (a, t) Nothing++-- | Add an unspent output to the index of an address.+insertAddrUnspent :: Writer -> Address -> Unspent -> IO ()+insertAddrUnspent w a u =+  T.insert w.memory.outputTable (a, u.block, u.outpoint) (Just (OutVal u.value u.script))++-- | Remove an unspent output from the index of an address.+deleteAddrUnspent :: Writer -> Address -> Unspent -> IO ()+deleteAddrUnspent w a u =+  T.insert w.memory.outputTable (a, u.block, u.outpoint) Nothing++-- | Set the balance of an address: the whole value, in place of whatever+-- the batch had for it. Differences that the batch held for the address+-- ('mergeBalance') are dropped: a caller that worked the balance out from+-- 'getBalance' has them in it.+setBalance :: Writer -> Balance -> IO ()+setBalance w b = do+  T.insert w.memory.balanceTable b.address (Just b)+  T.delete w.memory.balanceMerges b.address++-- | Add differences to the balance of an address without reading it: to+-- each of its five numbers, modulo 2^64, so that a number goes down by the+-- two's complement of what is taken off. A balance that is not there+-- counts from zero.+--+-- If the batch has the balance as a whole value, the differences are+-- applied to it. Otherwise they are added to those that the batch holds+-- for the address, and written as one merge record+-- ("Haskoin.Store.Database.Merge").+--+-- Nothing checks that a number does not go below zero, since that takes+-- the value, and the balance is never deleted, even if every number comes+-- to zero. It is for the import of a block, which only adds to the count+-- of transactions of an address. What takes a block out reads the balance+-- and writes it whole ('getBalance', 'setBalance', 'deleteBalance').+mergeBalance :: Writer -> Address -> BalVal -> IO ()+mergeBalance w a d =+  T.lookup w.memory.balanceTable a >>= \case+    Just whole ->+      let sum' = addBalVal (maybe def balanceToVal whole) d+       in T.insert w.memory.balanceTable a (Just (valToBalance a sum'))+    Nothing -> do+      pending <- T.lookup w.memory.balanceMerges a+      T.insert w.memory.balanceMerges a (maybe d (`addBalVal` d) pending)++-- | Add an unspent output.+insertUnspent :: Writer -> Unspent -> IO ()+insertUnspent w u = T.insert w.memory.unspentTable (fst (unspentToVal u)) (Just u)++-- | Remove an unspent output.+deleteUnspent :: Writer -> OutPoint -> IO ()+deleteUnspent w p = T.insert w.memory.unspentTable p Nothing++-- | Remove the data of a transaction.+deleteTxData :: Writer -> TxHash -> IO ()+deleteTxData w h = T.insert w.memory.txTable h Nothing++-- | Remove the balance of an address, as that of one never used, with any+-- differences that the batch held for it ('mergeBalance').+deleteBalance :: Writer -> Address -> IO ()+deleteBalance w a = do+  T.insert w.memory.balanceTable a Nothing+  T.delete w.memory.balanceMerges a++-- | Read a value from the changes of the batch, or from the database if the+-- batch did not change it. For what a batch reads once or twice: the best+-- block, the blocks at a height.+getLayered :: Writer -> (Memory -> IO (Maybe a)) -> (DatabaseReader -> IO a) -> IO a+getLayered w f g =+  f w.memory >>= \case+    Just x -> return x+    Nothing -> g w.reader++-- | Read the value of a key: from the changes of the batch, or else from+-- what the batch read before, or else from the database, which is then+-- remembered. The changes come first, so that nothing read can hide one.+getCached ::+  (TableKey k) =>+  Writer ->+  (Memory -> Table k v) ->+  (ReadCache -> Table k v) ->+  k ->+  (DatabaseReader -> IO v) ->+  IO v+getCached w changes cache k stored =+  T.lookup (changes w.memory) k >>= \case+    Just x -> return x+    Nothing -> remembered w cache k (stored w.reader)++-- | What the database has for a key: as read before in this batch, or+-- read now and remembered.+remembered :: (TableKey k) => Writer -> (ReadCache -> Table k v) -> k -> IO v -> IO v+remembered w cache k stored+  | not w.cached = stored+  | otherwise =+      T.lookup table k >>= \case+        Just x -> return x+        Nothing -> do+          x <- stored+          T.insert table k x+          return x+  where+    table = cache w.memory.cache++-------------------------------------------------------------------------------+-- Reading ahead+-------------------------------------------------------------------------------++-- | Multi-gets run at once for each core: 32 on four cores.+--+-- A multi-get is a call into RocksDB that does not hold a core of the+-- Haskell runtime, and it reads its keys one after another. On a database+-- that is not in memory most of its time is spent waiting for the disk,+-- so how many reads the disk is given at once is how many multi-gets run+-- at once. The disk of the store that this was made for gave 5,800 random+-- reads a second one at a time and 31,000 a second 32 at a time.+--+-- When the files are in memory the reads are work for the cores, and more+-- multi-gets than cores only take turns: the benchmark took as long with+-- two for each core as with eight.+readsPerCore :: Int+readsPerCore = 8++-- | Fewest keys worth a multi-get of their own.+smallestRead :: Int+smallestRead = 32++-- | Read the values of many keys of a column family with RocksDB's+-- multi-get, the keys split over 'readsPerCore' calls a core that run at+-- the same time, and work out something from each value in the thread that+-- read it. One multi-get uses one core and reads one key at a time,+-- however many keys it is given: the time of a block went into such reads,+-- with three cores and the disk idle.+--+-- The results are in no particular order.+readMany ::+  (Serialize k, NFData v) =>+  DB ->+  ColumnFamily ->+  (a -> k) ->+  (a -> Maybe ByteString -> v) ->+  [a] ->+  IO [(a, v)]+readMany db cf key value items = concat <$> onAllCores readsPerCore smallestRead chunk items+  where+    chunk xs = do+      found <- multiGetCF db cf (map (encode . key) xs)+      forM (zip xs found) $ \(x, bytes) -> (x,) <$> evaluate (force (value x bytes))++-- | Split a list in pieces, as many for each core as the first number says+-- and none shorter than the second unless it is the only one, and work on+-- the pieces at the same time. The results are in the order of the pieces.+--+-- Each piece is given to a thread that is bound to a core, the pieces in+-- turn. Left to the scheduler, new threads start on the core of the thread+-- that made them and move to idle cores only as it gets to them, which for+-- work of a few milliseconds was most of the time: sixteen pieces of+-- serialising took 35 ms that way and 26 ms this way, against 80 ms in one+-- thread.+onAllCores :: Int -> Int -> ([a] -> IO b) -> [a] -> IO [b]+onAllCores perCore least work items = do+  cores <- getNumCapabilities+  let pieces = max 1 (cores * perCore)+      size = max least ((length items + pieces - 1) `div` pieces)+  case chunksOf size items of+    [] -> return []+    [one] -> pure <$> work one+    many -> go (zip [0 ..] many)+  where+    -- every thread is stopped if one of them fails+    go [] = return []+    go ((core, piece) : more) =+      withAsyncOn core (work piece) $ \running -> do+        others <- go more+        (: others) <$> wait running++-- | A list in pieces of a length, the last one shorter.+chunksOf :: Int -> [a] -> [[a]]+chunksOf _ [] = []+chunksOf n xs = let (a, b) = splitAt n xs in a : chunksOf n b++-- | A change of a batch that is not serialised yet.+data Change+  = -- | set a key of a column family to a value+    Set !ColumnFamily Put Put+  | -- | delete a key of a column family+    Drop !ColumnFamily Put+  | -- | add a merge record to a key of a column family, for its merge+    -- operator to combine with the value of the key+    Join !ColumnFamily Put Put+  | -- | an operation that is serialised already+    Ready !BatchOp++-- | The change that sets a key of a column family, with the types that+-- 'Database.RocksDB.Query.insertOpCF' asks for: the value must be of the+-- type that the key is declared to have ('KeyValue').+setOpCF :: (KeyValue k v, Serialize k, Serialize v) => ColumnFamily -> k -> v -> Change+setOpCF cf k v = Set cf (put k) (put v)+  where+    -- never looked at: it is what asks for the pair of types to be declared+    _declared = insertOpCF cf k v++-- | The change that deletes a key of a column family.+dropOpCF :: (Key k, Serialize k) => ColumnFamily -> k -> Change+dropOpCF cf k = Drop cf (put k)+  where+    _declared = deleteOpCF cf k++-- | The change that adds a merge record to a key of a column family. The+-- record is of the type of the values of the key: both merge operators of+-- the store take records in the format of a value+-- ("Haskoin.Store.Database.Merge").+joinOpCF :: (KeyValue k v, Serialize k, Serialize v) => ColumnFamily -> k -> v -> Change+joinOpCF cf k v = Join cf (put k) (put v)+  where+    _declared = insertOpCF cf k v++-- | The changes of a batch, serialised for RocksDB in one of two forms.+data Encoded+  = -- | a few operations, each with byte strings of its own+    Plain [BatchOp]+  | -- | chunks of packed records for 'writePacked', in the order of the+    -- changes, with the number of records and the bytes of their keys and+    -- values+    Packed [ByteString] !Int !Int++-- | Serialise the changes of a batch for RocksDB, on all cores, in the+-- order of the changes.+--+-- = Why not 'encode' for each key and value+--+-- A block changes a hundred thousand keys, nearly all of them shorter than+-- a hundred bytes. 'encode' takes a buffer of four kilobytes for each key+-- and each value, writes into it and copies the result out: a gigabyte of+-- buffers for a block. And every byte string that it returns is memory+-- that the collector may not move, which GHC hands out under a lock that+-- all cores share, so that doing it on four cores was barely faster than+-- on one: 82 ms against 127 for the batch of a block.+--+-- Here each core serialises its share of the changes into a few large+-- buffers, one after another ('arenaSize'). Nothing is taken for a single+-- key, and the cores do not wait for each other.+--+-- = Why packed records, and not an operation for each key+--+-- The buffers could hold just the keys and the values, each a slice of one+-- in a 'BatchOp', for RocksDB to be given the operations one foreign call+-- at a time. That was how it was done first, and it was measured. A thread+-- gives up its capability for such a call and has to get one back, and+-- the calls for the 65,000 keys of a block were 43 ms of the 102 that it+-- took to hand RocksDB the batch and have it written, with nothing else+-- running. Next to busy threads each call waits for a capability+-- (@docs\/block-import.md@).+--+-- So what is written into a buffer is the batch itself, in the form that+-- 'writePacked' of the RocksDB binding takes: for each change a header+-- with its column family, its lengths and what it does, then the key, then+-- the value. A buffer is handed over in one foreign call, a few thousand+-- changes at a time, and a change has no byte string, no 'BatchOp' and no+-- list cell of its own. RocksDB copies the bytes when it is given a+-- buffer, and the buffers go when the batch is written.+--+-- The header of a record comes before its key and holds the lengths of+-- the key and of the value, which are only known once both are+-- serialised. So room is left for it, the key and the value are serialised+-- after that room, and the header is filled in last ('packChange').+--+-- = Small batches+--+-- A batch of a few changes, such as that of the genesis block, is+-- serialised with 'encode' and given to RocksDB one operation at a time, as+-- before. A buffer of this size for each would be the waste that this is+-- meant to remove, and what a few dozen foreign calls cost is not worth a+-- second way to pack. It also keeps the old way in use, which the tests+-- compare the new one against: the same changes must leave the same+-- database whether they are written in one large batch or a few at a time.+encodeChanges :: [Change] -> IO Encoded+encodeChanges changes+  | null (drop 255 changes) = return (Plain (map plain changes))+  | otherwise = do+      pieces <- onAllCores 2 512 inArenas changes+      return $+        Packed+          (concat [chunks | (chunks, _, _) <- pieces])+          (sum [records | (_, records, _) <- pieces])+          (sum [size | (_, _, size) <- pieces])+  where+    inArenas piece = do+      arena <- newIORef =<< newArena+      mapM_ (packChange arena) piece+      Arena buffer start used chunks records size <- readIORef arena+      return (reverse (sealed buffer start used chunks), records, size)++-- | The operation of a change, with byte strings of its own.+plain :: Change -> BatchOp+plain = \case+  Set cf k v -> PutCF cf (runPut k) (runPut v)+  Drop cf k -> DelCF cf (runPut k)+  Join cf k v -> MergeCF cf (runPut k) (runPut v)+  Ready op -> op++-- | Bytes of the key of an operation, and of its value or merge record if+-- it has one.+opBytes :: BatchOp -> Int+opBytes = \case+  Put k v -> B.length k + B.length v+  PutCF _ k v -> B.length k + B.length v+  Del k -> B.length k+  DelCF _ k -> B.length k+  Merge k v -> B.length k + B.length v+  MergeCF _ k v -> B.length k + B.length v++-- | Where one core packs its share of the changes of a batch.+--+-- The fields are: the buffer that records are written into, one after+-- another; where in it the records start that are in no chunk yet; where+-- the next record goes; the chunks so far, the last one first; the+-- number of records in the chunks and in the buffer, with the bytes of+-- their keys and values.+--+-- A chunk is a slice of a buffer, so the buffer lives as long as its+-- chunks, and a buffer can give more than one: see 'packChange'.+data Arena = Arena !(ForeignPtr Word8) !Int !Int [ByteString] !Int !Int++-- | Bytes of the buffer of an t'Arena': room for a few thousand keys and+-- values, and for nearly any transaction. It is also the most that RocksDB+-- is handed in one foreign call, which then takes a fraction of a+-- millisecond.+arenaSize :: Int+arenaSize = 256 * 1024++newArena :: IO Arena+newArena = do+  buffer <- mallocPlainForeignPtrBytes arenaSize+  return (Arena buffer 0 0 [] 0 0)++-- | The chunks of an arena, the last one first, with the records of its+-- buffer that are in no chunk yet as one more.+sealed :: ForeignPtr Word8 -> Int -> Int -> [ByteString] -> [ByteString]+sealed buffer start used chunks+  | used > start = BS (buffer `plusForeignPtr` start) (used - start) : chunks+  | otherwise = chunks++-- | Whether a record could be written at the end of a buffer.+data Fit+  = -- | it is there, with so many bytes of key and so many of value+    Fits !Int !Int+  | -- | the buffer has no room for it+    NoRoom+  | -- | it brings a large byte string of its own, which is not for copying+    -- into a shared buffer+    Alone++-- | Pack a change as a record at the end of what an arena holds: room for+-- the header, the key, the value, and then the header, now that the+-- lengths are known.+--+-- A record that does not fit in what is left of the buffer starts a new+-- one. One that does not fit in an empty buffer either, or that brings a+-- large byte string of its own (a script of eight kilobytes or more), is+-- serialised by itself as 'encode' would, and is a chunk of its own.+-- The order of the changes is the order of the records, so the records+-- before it are made a chunk first, and those after it go on in the same+-- buffer.+packChange :: IORef Arena -> Change -> IO ()+packChange ref change = case change of+  Set cf k v -> record (Just cf) PackedPut k (Just v)+  Drop cf k -> record (Just cf) PackedDel k Nothing+  Join cf k v -> record (Just cf) PackedMerge k (Just v)+  Ready (PutCF cf k v) -> record (Just cf) PackedPut (putByteString k) (Just (putByteString v))+  Ready (DelCF cf k) -> record (Just cf) PackedDel (putByteString k) Nothing+  Ready (MergeCF cf k v) -> record (Just cf) PackedMerge (putByteString k) (Just (putByteString v))+  Ready (Put k v) -> record Nothing PackedPut (putByteString k) (Just (putByteString v))+  Ready (Del k) -> record Nothing PackedDel (putByteString k) Nothing+  Ready (Merge k v) -> record Nothing PackedMerge (putByteString k) (Just (putByteString v))+  where+    record cf op k v = do+      Arena buffer start used chunks records size <- readIORef ref+      let body = used + packedHeaderSize+          key = execPut k+          value = execPut <$> v+      fit <-+        if body > arenaSize+          then return NoRoom+          else withForeignPtr buffer $ \base -> do+            let header = base `plusPtr` used+                done klen vlen = do+                  pokePackedHeader header cf op klen vlen+                  return (Fits klen vlen)+            (klen, next) <- runBuilder key (base `plusPtr` body) (arenaSize - body)+            case (next, value) of+              (Done, Nothing) -> done klen 0+              (Done, Just builder) -> do+                let room = arenaSize - body - klen+                (vlen, next') <- runBuilder builder (base `plusPtr` (body + klen)) room+                case next' of+                  Done -> done klen vlen+                  More _ _ -> return NoRoom+                  Chunk _ _ -> return Alone+              (More _ _, _) -> return NoRoom+              (Chunk _ _, _) -> return Alone+      case fit of+        Fits klen vlen -> do+          let bytes = klen + vlen+          writeIORef ref (Arena buffer start (body + bytes) chunks (records + 1) (size + bytes))+        NoRoom | used > 0 -> do+          fresh <- mallocPlainForeignPtrBytes arenaSize+          writeIORef ref (Arena fresh 0 0 (sealed buffer start used chunks) records size)+          record cf op k v+        _ -> do+          let whole = plain change+              before = sealed buffer start used chunks+          writeIORef ref $+            Arena buffer used used (packOps [whole] : before) (records + 1) (size + opBytes whole)++-- | Decode a value read from the database.+decoded :: (Serialize v) => ByteString -> v+decoded = either error id . decode++-- | Read ahead: put what the database has for some keys in the read cache+-- of the batch, those that it does not have too, given how to read them+-- all. Keys that the batch has read already are not read again, and+-- nothing is read through 'withoutReadCache'.+readAhead ::+  (TableKey k) =>+  Writer ->+  (ReadCache -> Table k v) ->+  [k] ->+  ([k] -> IO [(k, v)]) ->+  IO ()+readAhead w cache keys reading = when w.cached $ do+  let table = cache w.memory.cache+  asked <- T.new+  -- each key once, and none that was read before+  let fresh k =+        T.lookup table k >>= \case+          Just _ -> return False+          Nothing ->+            T.lookup asked k >>= \case+              Just () -> return False+              Nothing -> T.insert asked k () >> return True+  new <- filterM fresh keys+  reading new >>= mapM_ (uncurry (T.insert table))++-- | Read unspent outputs ahead of the reads of the batch, with multi-gets.+readAheadUnspents :: Writer -> [OutPoint] -> IO ()+readAheadUnspents w ps = readAhead w (.unspents) ps $ \new -> do+  found <- readMany db (unspentCF db) UnspentKey (\p -> fmap (valToUnspent w.reader.net p . decoded)) new+  Q.count w.reader $ \s -> incrementCounter s.dataUnspentCount (length (filter (isJust . snd) found))+  return found+  where+    db = w.reader.db++-- | Read balances ahead of the reads of the batch. The import of a block+-- does not need it any more, as it changes balances without reading them+-- ('mergeBalance'): it is for rules that read them.+readAheadBalances :: Writer -> [Address] -> IO ()+readAheadBalances w as = readAhead w (.balances) as $ \new -> do+  Q.count w.reader $ \s -> incrementCounter s.dataBalanceCount (length new)+  readMany db (balanceCF db) BalKey (\a -> fmap (valToBalance a . decoded)) new+  where+    db = w.reader.db++-- | Read ahead the spenders in the chain of the groups of outputs that+-- these outputs are in: what 'getChainSpender' and 'insertSpender' read+-- for each of them. The import of a block does not need it: it adds+-- spenders without reading them ('mergeSpender'), and does not ask who+-- spends an output in the chain, as a mined block is not validated+-- ('Haskoin.Store.Logic.importBlock'). It is for rules that read+-- them.+readAheadSpenders :: Writer -> [OutPoint] -> IO ()+readAheadSpenders w ps =+  readAhead w (.spenders) [(p.hash, p.index `div` spenderGroup) | p <- ps] $+    readMany db (spenderCF db) (uncurry SpenderKey) (\_ -> maybe IntMap.empty (group . decoded))+  where+    db = w.reader.db+    group (SpenderVal m) = m++-------------------------------------------------------------------------------+-- Timing the import of a block+-------------------------------------------------------------------------------++-- | How long each stage of the import of a block took, in seconds. A batch+-- that imported several blocks has the sum for each stage.+data BlockTimes = BlockTimes+  { -- | working out, from the block alone, what the import needs of each+    -- transaction: its hash, the addresses of its outputs, its size+    prepare :: !Double,+    -- | reading ahead what the import reads from the database+    prefetch :: !Double,+    -- | how much of that each kind of key took+    reads :: !ReadTimes,+    -- | the rules: every change of the block, made in memory+    apply :: !Double,+    -- | encoding the changes and writing them to RocksDB in one batch+    write :: !Double+  }+  deriving (Eq, Show)++-- | How long the read ahead of a block took for each kind of key, in+-- seconds. Only the unspent outputs are read now, with multi-gets on all+-- cores. The balances and the groups of spenders of the chain are not read+-- at all any more: their two times are kept, as zero, for what reads them+-- from a report.+data ReadTimes = ReadTimes+  { -- | the outputs that the block spends, as unspent outputs of the chain+    unspentReads :: !Double,+    -- | groups of spenders in the chain: nothing, as the import of a block+    -- adds spenders without reading them since database version 23, and+    -- no longer reads any to check the block+    spenderReads :: !Double,+    -- | the balances of the addresses that the block pays or spends from:+    -- nothing since database version 23, as the import of a block changes+    -- them without reading them+    balanceReads :: !Double+  }+  deriving (Eq, Show)++-- | No time spent reading: of an import that did not read ahead.+noReads :: ReadTimes+noReads = ReadTimes {unspentReads = 0, spenderReads = 0, balanceReads = 0}++-- | Note how long the first three stages of the import of a block took,+-- and each kind of key of the read ahead. 'runWriterReport' adds how long+-- the batch took to write.+noteBlockTimes :: Writer -> Double -> Double -> ReadTimes -> Double -> IO ()+noteBlockTimes w prepare prefetch kinds apply =+  modifyIORef' w.memory.times $ \case+    Nothing -> Just BlockTimes {prepare, prefetch, reads = kinds, apply, write = 0}+    Just t ->+      Just+        (t :: BlockTimes)+          { prepare = t.prepare + prepare,+            prefetch = t.prefetch + prefetch,+            reads =+              ReadTimes+                { unspentReads = t.reads.unspentReads + kinds.unspentReads,+                  spenderReads = t.reads.spenderReads + kinds.spenderReads,+                  balanceReads = t.reads.balanceReads + kinds.balanceReads+                },+            apply = t.apply + apply+          }++-- | Run an action and say how many seconds it took.+stopwatch :: IO a -> IO (a, Double)+stopwatch action = do+  start <- getMonotonicTime+  x <- action+  end <- getMonotonicTime+  return (x, end - start)++-- | Run a batch of writes and commit all its changes to the database in one+-- atomic write. Reads within the batch see its own changes. If the function+-- throws, nothing is written.+runWriter ::+  Network ->+  DatabaseReader ->+  Logger ->+  (Writer -> IO a) ->+  IO a+runWriter net bdb logger f =+  fst <$> runWriterReport net bdb logger f++-- | What a batch did, besides what its function returned.+data BatchReport = BatchReport+  { -- | how long the stages of the import of a block took, if it imported+    -- one+    block :: !(Maybe BlockTimes),+    -- | keys that it set, deleted or wrote a merge record for+    operations :: !Int,+    -- | bytes of those keys, and of the values and merge records that it+    -- wrote: about what RocksDB appends to its write-ahead log for the+    -- batch+    bytes :: !Int+  }+  deriving (Eq, Show)++-- | Like 'runWriter', but say everything about the batch, in a+-- t'BatchReport': for the metrics of the block store, and for the+-- benchmark.+--+-- The changes are still written in one batch, with one write to the+-- write-ahead log, however many there are: that is what makes a block all+-- or nothing, and a disk that is slow to sync is asked to once.+runWriterReport ::+  Network ->+  DatabaseReader ->+  Logger ->+  (Writer -> IO a) ->+  IO (a, BatchReport)+runWriterReport net bdb@DatabaseReader {db} logger f = do+  hm <- newMemory net+  let w = Writer {reader = bdb, memory = hm, logger, cached = True}+  x <- f w+  ((operations, bytes), took) <- stopwatch $ do+    ops <- hashMapOps db hm+    encodeChanges ops >>= \case+      Plain batch -> do+        writeBatch db batch+        return (length batch, sum (map opBytes batch))+      Packed chunks records size -> do+        writePacked db chunks+        return (records, size)+  times <- readIORef hm.times+  let block = (\t -> (t :: BlockTimes) {write = took}) <$> times+  return (x, BatchReport {block, operations, bytes})++-- | Database operations for all changes of a batch.+hashMapOps :: DB -> Memory -> IO [Change]+hashMapOps db Memory {..} =+  mconcat+    <$> sequence+      [ bestBlockOp best,+        blockHashOps db blockTable,+        blockHeightOps db heightTable,+        txOps db txTable,+        balOps db balanceTable,+        balMergeOps db balanceMerges,+        addrTxOps db addressTable,+        addrOutOps db outputTable,+        unspentOps db unspentTable,+        spenderOps db spenderTable,+        spenderMergeOps db spenderMerges,+        replacedOps db replacedTable+      ]++-- | Operation for the change of the best block, if any.+bestBlockOp :: BestRef -> IO [Change]+bestBlockOp r =+  readIORef r >>= \case+    Nothing -> return []+    Just Nothing -> return [Ready (deleteOp BestKey)]+    Just (Just b) -> return [Ready (insertOp BestKey b)]++-- | Operations for changed blocks.+blockHashOps :: DB -> BlockTable -> IO [Change]+blockHashOps db t = map (uncurry f) <$> (T.toList t)+  where+    f k (Just d) = setOpCF (blockCF db) (BlockKey k) d+    f k Nothing = dropOpCF (blockCF db) (BlockKey k)++-- | Operations for changed records of replaced transactions: the record,+-- and its entry at the height of its block. A record that is deleted, or+-- kept again at another height, takes its old entry with it.+replacedOps :: DB -> ReplacedTable -> IO [Change]+replacedOps db t = concat <$> (T.toList t >>= mapM f)+  where+    f (h, new) = do+      old <- retrieveCF db (blockCF db) (ReplacedKey h) :: IO (Maybe Replaced)+      let forget =+            [ dropOpCF (heightCF db) (ReplacedAtKey o.height h)+            | Just o <- [old],+              fmap (.height) new /= Just o.height+            ]+      return . (forget <>) $ case new of+        Just r -> [setOpCF (blockCF db) (ReplacedKey h) r, setOpCF (heightCF db) (ReplacedAtKey r.height h) ()]+        Nothing -> [dropOpCF (blockCF db) (ReplacedKey h)]++-- | Operations for changed block heights.+blockHeightOps :: DB -> HeightTable -> IO [Change]+blockHeightOps db t = map (uncurry f) <$> (T.toList t)+  where+    f = setOpCF (heightCF db) . HeightKey++-- | Operations for changed transactions.+txOps :: DB -> TxTable -> IO [Change]+txOps db t = map (uncurry f) <$> (T.toList t)+  where+    f k (Just t') = setOpCF (txCF db) (TxKey k) t'+    f k Nothing = dropOpCF (txCF db) (TxKey k)++-- | Operations for changed balances.+balOps :: DB -> BalanceTable -> IO [Change]+balOps db t = map (uncurry f) <$> (T.toList t)+  where+    f a (Just b) = setOpCF (balanceCF db) (BalKey a) (balanceToVal b)+    f a Nothing = dropOpCF (balanceCF db) (BalKey a)++-- | A merge record for each address whose balance the batch changes+-- without having read it: the sum of the differences.+balMergeOps :: DB -> BalanceMergeTable -> IO [Change]+balMergeOps db t = map (\(a, d) -> joinOpCF (balanceCF db) (BalKey a) d) <$> T.toList t++-- | Operations for changed address transactions.+addrTxOps :: DB -> AddrTxTable -> IO [Change]+addrTxOps db t = map (uncurry f) <$> (T.toList t)+  where+    f (a, t') (Just ()) = setOpCF (addrTxCF db) (AddrTxKey a t') ()+    f (a, t') Nothing = dropOpCF (addrTxCF db) (AddrTxKey a t')++-- | Operations for changed address unspent outputs.+addrOutOps :: DB -> AddrOutTable -> IO [Change]+addrOutOps db t = map (uncurry f) <$> (T.toList t)+  where+    f (a, b, p) (Just l) =+      setOpCF+        (addrOutCF db)+        (AddrOutKey a b p)+        l+    f (a, b, p) Nothing =+      dropOpCF+        (addrOutCF db)+        (AddrOutKey a b p)++-- | Operations for the changed groups of spenders of the chain.+spenderOps :: DB -> SpenderTable -> IO [Change]+spenderOps db t = map (uncurry f) <$> T.toList t+  where+    f (h, g) m+      | IntMap.null m = dropOpCF (spenderCF db) (SpenderKey h g)+      | otherwise = setOpCF (spenderCF db) (SpenderKey h g) (SpenderVal m)++-- | A merge record for each group of spenders that the batch adds to+-- without having read it: the spenders to add.+spenderMergeOps :: DB -> SpenderTable -> IO [Change]+spenderMergeOps db t =+  map (\((h, g), m) -> joinOpCF (spenderCF db) (SpenderKey h g) (SpenderVal m)) <$> T.toList t++-- | Operations for changed unspent outputs.+unspentOps :: DB -> UnspentTable -> IO [Change]+unspentOps db t = map (uncurry f) <$> (T.toList t)+  where+    f p (Just u) =+      setOpCF (unspentCF db) (UnspentKey p) (snd (unspentToVal u))+    f p Nothing =+      dropOpCF (unspentCF db) (UnspentKey p)++-- | Best block, with the changes of the batch applied.+getBestBlock :: Writer -> IO (Maybe BlockHash)+getBestBlock w = getLayered w getBestBlockH Q.getBestBlock++-- | Blocks at a height, with the changes of the batch applied.+getBlocksAtHeight :: Writer -> BlockHeight -> IO [BlockHash]+getBlocksAtHeight w bh = getLayered w (getBlocksAtHeightH bh) (`Q.getBlocksAtHeight` bh)++-- | Data of a block, with the changes of the batch applied.+getBlock :: Writer -> BlockHash -> IO (Maybe BlockData)+getBlock w bh = getLayered w (getBlockH bh) (`Q.getBlock` bh)++-- | Data of a transaction, deleted ones included, with the changes of the+-- batch applied.+getTxData :: Writer -> TxHash -> IO (Maybe TxData)+getTxData w th = getCached w (.txTable) (.txs) th (`Q.getTxRecord` th)++-- | Balance of an address, with the changes of the batch applied: the+-- whole value if the batch has one, and otherwise what the database has+-- with the differences that the batch holds for the address added to it+-- ('mergeBalance').+getBalance :: Writer -> Address -> IO (Maybe Balance)+getBalance w a =+  T.lookup w.memory.balanceTable a >>= \case+    Just whole -> return whole+    Nothing -> do+      stored <- remembered w (.balances) a (Q.getBalance w.reader a)+      T.lookup w.memory.balanceMerges a <&> \case+        Nothing -> stored+        Just d -> Just (valToBalance a (addBalVal (maybe def balanceToVal stored) d))++-- | Unspent output, with the changes of the batch applied.+getUnspent :: Writer -> OutPoint -> IO (Maybe Unspent)+getUnspent w op = getCached w (.unspentTable) (.unspents) op (`Q.getUnspent` op)++-- | The spenders of a group of outputs of a transaction, with the changes+-- of the batch applied: the whole group if the batch has one, and+-- otherwise what the database has with the spenders that the batch is to+-- add to it ('mergeSpender'), which win for an output that both have.+spenderGroupOf :: Writer -> TxHash -> Word32 -> IO (IntMap Spender)+spenderGroupOf w h g =+  T.lookup w.memory.spenderTable (h, g) >>= \case+    Just whole -> return whole+    Nothing -> do+      stored <-+        remembered w (.spenders) (h, g) $+          retrieveCF db (spenderCF db) (SpenderKey h g) <&> \case+            Just (SpenderVal m) -> m+            Nothing -> IntMap.empty+      maybe stored (`IntMap.union` stored) <$> T.lookup w.memory.spenderMerges (h, g)+  where+    db = w.reader.db++-- | Note that an input of a block spends an output, in the group of+-- outputs that it belongs to, by reading the group and writing it whole.+-- The record of the transaction that made the output is not read and not+-- written. Spenders that the batch was to add to the group ('mergeSpender')+-- are in the group that is read, and so in the one that is written.+insertSpender :: Writer -> OutPoint -> Spender -> IO ()+insertSpender w p s = do+  let g = p.index `div` spenderGroup+  m <- spenderGroupOf w p.hash g+  T.insert w.memory.spenderTable (p.hash, g) (IntMap.insert (fromIntegral p.index) s m)+  T.delete w.memory.spenderMerges (p.hash, g)++-- | Note that an input of a block spends an output, without reading the+-- group of outputs that it belongs to.+--+-- If the batch has the group as a whole value, the spender is put in it.+-- Otherwise it is added to those that the batch holds for the group, which+-- are written as one merge record: RocksDB joins it with the group when+-- the group is read ("Haskoin.Store.Database.Merge").+--+-- It is for the import of a block. 'insertSpender' does the same by+-- reading the group, as taking a spender out has to ('deleteSpender').+mergeSpender :: Writer -> OutPoint -> Spender -> IO ()+mergeSpender w p s =+  T.lookup w.memory.spenderTable key >>= \case+    Just whole -> T.insert w.memory.spenderTable key (IntMap.insert i s whole)+    Nothing -> do+      pending <- T.lookup w.memory.spenderMerges key+      T.insert w.memory.spenderMerges key (IntMap.insert i s (fromMaybe IntMap.empty pending))+  where+    key = (p.hash, p.index `div` spenderGroup)+    i = fromIntegral p.index++-- | Note that no block spends an output any more, by reading the group of+-- outputs that it belongs to and writing it whole, with the spenders that+-- the batch was to add to it ('mergeSpender').+deleteSpender :: Writer -> OutPoint -> IO ()+deleteSpender w p = do+  let g = p.index `div` spenderGroup+  m <- spenderGroupOf w p.hash g+  T.insert w.memory.spenderTable (p.hash, g) (IntMap.delete (fromIntegral p.index) m)+  T.delete w.memory.spenderMerges (p.hash, g)++-- | The input of a block that spends an output, with the changes of the+-- batch applied.+getChainSpender :: Writer -> OutPoint -> IO (Maybe Spender)+getChainSpender w p = IntMap.lookup (fromIntegral p.index) <$> spenderGroupOf w p.hash (p.index `div` spenderGroup)++-- | The inputs of blocks that spend the outputs of a transaction, by+-- output, with the changes of the batch applied, given the number of its+-- outputs: what the database has, read with one scan, with the groups+-- that the batch has whole in place of the stored ones, and the spenders+-- that the batch is to add to the other groups ('mergeSpender').+getChainSpenders :: Writer -> TxHash -> Int -> IO (IntMap Spender)+getChainSpenders w h outputs = do+  stored <- Q.getChainSpenders w.reader h+  let groups = [0 .. (fromIntegral (max 1 outputs) - 1) `div` spenderGroup]+  changed <- catMaybes <$> mapM (\g -> fmap (g,) <$> T.lookup w.memory.spenderTable (h, g)) groups+  added <- catMaybes <$> mapM (\g -> T.lookup w.memory.spenderMerges (h, g)) groups+  let replaced = [g | (g, _) <- changed]+      kept = IntMap.filterWithKey (\i _ -> (fromIntegral i `div` spenderGroup) `notElem` replaced) stored+  -- the first of the union wins: what the batch adds comes before what+  -- the database has, and no group is both whole and added to+  return (IntMap.unions (added <> (kept : map snd changed)))++-- | Keep what a block replaced.+insertReplaced :: Writer -> TxHash -> Replaced -> IO ()+insertReplaced w h r = T.insert w.memory.replacedTable h (Just r)++-- | Forget what was kept of a replaced transaction.+deleteReplaced :: Writer -> TxHash -> IO ()+deleteReplaced w h = T.insert w.memory.replacedTable h Nothing++-- | The transactions that the block at a height replaced, with the changes+-- of the batch applied.+getReplacedAt :: Writer -> BlockHeight -> IO [TxHash]+getReplacedAt w height = do+  stored <- Q.getReplacedAt w.reader height+  changes <- T.toList w.memory.replacedTable+  let changed = HashMap.fromList changes+      kept = filter (not . (`HashMap.member` changed)) stored+      added = [h | (h, Just r) <- changes, r.height == height]+  return (kept <> added)++-- | Empty set of changes.+newMemory ::+  Network ->+  IO Memory+newMemory net = do+  best <- newIORef Nothing+  blockTable <- T.new+  heightTable <- T.new+  txTable <- T.new+  unspentTable <- T.new+  balanceTable <- T.new+  balanceMerges <- T.new+  addressTable <- T.new+  outputTable <- T.new+  spenderTable <- T.new+  spenderMerges <- T.new+  replacedTable <- T.new+  cache <- newReadCache+  times <- newIORef Nothing+  return Memory {..}++-- | Nothing read yet.+newReadCache :: IO ReadCache+newReadCache = do+  txs <- T.new+  unspents <- T.new+  balances <- T.new+  spenders <- T.new+  return ReadCache {..}++-- | Read the best block from the changes of the batch: 'Nothing' if unchanged, @Just Nothing@ if deleted.+getBestBlockH :: Memory -> IO (Maybe (Maybe BlockHash))+getBestBlockH = readIORef . (.best)++-- | Read the blocks at a height from the changes of the batch: 'Nothing' if unchanged, @Just Nothing@ if deleted.+getBlocksAtHeightH :: BlockHeight -> Memory -> IO (Maybe [BlockHash])+getBlocksAtHeightH h s = T.lookup s.heightTable h++-- | Read a block from the changes of the batch: 'Nothing' if unchanged, @Just Nothing@ if deleted.+getBlockH :: BlockHash -> Memory -> IO (Maybe (Maybe BlockData))+getBlockH h s = T.lookup s.blockTable h
+ src/Haskoin/Store/Exception.hs view
@@ -0,0 +1,170 @@+-- |+-- Module      : Haskoin.Store.Exception+-- Copyright   : (c) Haskoin Developers+-- License     : MIT+-- Maintainer  : jprupp@protonmail.ch+-- Stability   : experimental+-- Portability : POSIX+--+-- The few functions about exceptions that the store needs and that+-- "Control.Exception" does not have: handlers for any error that let a+-- thread be stopped, and cleanups that cannot be cut short. It is an+-- internal module, for the store, its tests and its tools.+--+-- = Why+--+-- A thread of the store is stopped from outside all the time: a request+-- that takes too long gets a 'System.Timeout.timeout', a loop that runs+-- under 'Control.Concurrent.Async.withAsync' is cancelled when its owner is+-- done, the web server kills the thread of a client that went away. Each of+-- those is an exception thrown to the thread, an /asynchronous/ one, and+-- two things in "Control.Exception" get in its way:+--+-- * A handler for 'SomeException' catches it too. A loop that logs every+--   failure and goes on (@forever (action \`catch\` log)@) then logs its own+--   cancellation and goes on, and never stops; a request that falls back to+--   an empty answer on any error answers its own timeout that way.+--   'catchAny', 'handleAny' and 'tryAny' catch every exception but those,+--   which they let through.+--+-- * The cleanup of 'Control.Exception.bracket' and+--   'Control.Exception.finally' runs with asynchronous exceptions masked,+--   but a second one still gets in where the cleanup waits for something,+--   and the rest of the cleanup is then skipped: a snapshot of the database+--   is not released, an iterator stays open, a gauge is left one too+--   high for good. 'bracket', 'bracket_', 'finally' and 'onException' here+--   run the cleanup under 'uninterruptibleMask_', so that it runs to its+--   end. And if the cleanup fails as well, the exception of the body is the+--   one thrown, since it is the one that says what went wrong.+--+-- = When to use which+--+-- Use these everywhere in the store in place of the functions of the same+-- name of "Control.Exception", and import the rest from there: 'throwIO',+-- 'Control.Exception.mask_', 'Control.Exception.evaluate' and so on.+-- Catching one type of synchronous error, such as an @ImportException@,+-- needs nothing from here: 'Control.Exception.catch',+-- 'Control.Exception.handle' and 'Control.Exception.try' do it right.+--+-- A cleanup under these functions must be short and sure to end, since+-- nothing can stop it: release, decrement, unsubscribe, or cancel a thread+-- that can be cancelled. A cleanup that may wait for long belongs in+-- "Control.Exception"'s functions, or better, should not wait.+--+-- = Where they come from+--+-- The names and the behaviour are those of the @unliftio@ library, which+-- the store used until 1.15.1 and whose functions run in any monad that+-- can be run in 'IO'. The store runs in 'IO' alone, so it keeps these few+-- functions for 'IO' and not the library. Writing 'uninterruptibleMask_'+-- around every cleanup, and a test of the type of exception in every+-- handler, was the alternative: something to forget at every use, where+-- this is one place to get it right.+--+-- What tells an asynchronous exception from another is its type, not how+-- it was thrown: it is one that is made a 'SomeException' through+-- t'SomeAsyncException', as 'Control.Exception.AsyncException',+-- 'Control.Concurrent.Async.AsyncCancelled' and the exception of+-- 'System.Timeout.timeout' are. So do not throw an ordinary exception to+-- another thread with 'Control.Exception.throwTo': handlers there take it+-- for an error of the code that they guard.+module Haskoin.Store.Exception+  ( -- * Handlers for any error+    catchAny,+    handleAny,+    tryAny,+    isSyncException,++    -- * Cleanups that run to their end+    bracket,+    bracket_,+    finally,+    onException,+  )+where++import Control.Exception+  ( SomeAsyncException (..),+    SomeException,+    fromException,+    mask,+    throwIO,+    uninterruptibleMask,+    uninterruptibleMask_,+  )+import Control.Exception qualified as E++-- | Whether an exception is an error of the code that threw it, and not one+-- thrown to a thread to stop it: whether it is not a t'SomeAsyncException'.+isSyncException :: SomeException -> Bool+isSyncException e = case fromException e of+  Just (SomeAsyncException _) -> False+  Nothing -> True++-- | Run an action, and a handler if it throws any exception but an+-- asynchronous one, which goes on to the caller as if there were no+-- handler. The handler runs with asynchronous exceptions masked, as that+-- of 'Control.Exception.catch' does.+catchAny :: IO a -> (SomeException -> IO a) -> IO a+catchAny action handler =+  action `E.catch` \e ->+    if isSyncException e+      then handler e+      else throwIO e++-- | 'catchAny' with the handler first.+handleAny :: (SomeException -> IO a) -> IO a -> IO a+handleAny = flip catchAny++-- | Run an action and give its result, or the exception that it threw+-- unless that is an asynchronous one, which is thrown on.+tryAny :: IO a -> IO (Either SomeException a)+tryAny action = (Right <$> action) `catchAny` (return . Left)++-- | Get a resource, use it, and release it however the use ends. Getting+-- it runs with asynchronous exceptions masked, and releasing it with them+-- masked uninterruptibly, so that a resource that was got is released and+-- the release is not cut short. If both the use and the release throw, the+-- exception of the use is the one thrown.+bracket :: IO a -> (a -> IO b) -> (a -> IO c) -> IO c+bracket acquire release use = mask $ \restore -> do+  x <- acquire+  E.try (restore (use x)) >>= \case+    Left (e :: SomeException) -> do+      -- the release may fail too, but the failure of the use is the one+      -- that says what went wrong+      _ :: Either SomeException b <- E.try (uninterruptibleMask_ (release x))+      throwIO e+    Right y -> do+      _ <- uninterruptibleMask_ (release x)+      return y++-- | 'bracket' for a resource that is not a value: something to do before+-- an action and something to undo after it, however it ends.+bracket_ :: IO a -> IO b -> IO c -> IO c+bracket_ before after action = bracket before (const after) (const action)++-- | Run an action, then a cleanup, however the action ends. The cleanup+-- runs with asynchronous exceptions masked uninterruptibly. If both throw,+-- the exception of the action is the one thrown.+finally :: IO a -> IO b -> IO a+finally action cleanup = uninterruptibleMask $ \restore ->+  E.try (restore action) >>= \case+    Left (e :: SomeException) -> do+      _ :: Either SomeException b <- E.try cleanup+      throwIO e+    Right x -> do+      _ <- cleanup+      return x++-- | Run an action, and a cleanup if it throws any exception, asynchronous+-- ones included, before the exception is thrown on. The cleanup runs with+-- asynchronous exceptions masked uninterruptibly, and an exception of its+-- own is dropped.+onException :: IO a -> IO b -> IO a+onException action cleanup = uninterruptibleMask $ \restore ->+  E.try (restore action) >>= \case+    Left (e :: SomeException) -> do+      _ :: Either SomeException b <- E.try cleanup+      throwIO e+    Right x -> return x
src/Haskoin/Store/Fleet.hs view
@@ -8,7 +8,7 @@ -- -- How stores that share one Redis agree on the block that they all answer -- from, as pure decisions. "Haskoin.Store.Fleet.Redis" carries them out.--- This is the first part of the design in @docs\/layered-database.md@, "A+-- This is the first part of the design in @docs\/one-fleet.md@, "A -- fleet that answers as one", written down so that it can be tested before -- a store depends on it. --
src/Haskoin/Store/Fleet/Member.hs view
@@ -13,7 +13,7 @@ -- the same answer to the same request. The rules are in -- "Haskoin.Store.Fleet" and "Haskoin.Store.Fleet.Pool", Redis carries them -- out through "Haskoin.Store.Fleet.Redis", and the reasons are in "A fleet--- that answers as one" in @docs\/layered-database.md@.+-- that answers as one" in @docs\/one-fleet.md@. -- -- = What a member does --@@ -32,7 +32,9 @@ -- -- * __It offers its mempool to the shared pool__ ('offerMempool'), each --   transaction with whether its inputs are unspent in the store's view at---   the fleet tip, so that the pool only takes what is valid there.+--   the fleet tip, so that the pool only takes what is valid there. What+--   the pool decided on is seen by the fleet, in Redis, and no store+--   fetches it again when a peer announces it ('memberSeen'). -- -- * __It answers from the view at the fleet tip with the pool on top__ --   ('answering', 'redisPool'), or says that it is behind when it has no@@ -41,9 +43,10 @@ -- * __It keeps the fleet's cache of extended public keys__ --   ("Haskoin.Store.Fleet.XPub"): it answers the queries about a key from --   it when the cache has the key ('answeringCached'), builds the entries---   that requests missed ('memberCacheLoop'), brings the cache to the new---   block when it moves the tip, and empties a cache that was left behind---   ('syncCache').+--   that requests missed ('memberCacheLoop'), with the lookahead of each+--   ('Haskoin.Store.Fleet.XPub.lookahead'), and brings the cache to the new+--   block in the same commit as the pool when it moves the tip+--   ('moveTip'), a reorganisation undone address by address. -- -- = Why the member does not withdraw what it deletes --@@ -66,6 +69,7 @@     memberLoop,     memberAlive,     memberEvents,+    noteEpoch,      -- * Answering     Standing (..),@@ -79,8 +83,8 @@     -- * The cache of extended public keys     CacheMetrics (..),     memberCacheLoop,-    wantCached,-    syncCache,+    fleetSession,+    cacheGauges,      -- * The parts of a tick     report,@@ -92,12 +96,32 @@     farthest,     changeBetween,     offerMempool,+    memberReceived,+    memberSeen,+    mostSeen,+    memberPoolBody,+    memberNewest,     expirePool,     poolTx,   ) where -import Control.Exception (Exception, throwIO)+import Control.Concurrent (threadDelay)+import Control.Concurrent.Async (link, withAsync)+import Control.Concurrent.STM+  ( TVar,+    atomically,+    check,+    modifyTVar',+    newTVarIO,+    orElse,+    readTVar,+    readTVarIO,+    registerDelay,+    retry,+    writeTVar,+  )+import Control.Exception (Exception, handle, throwIO) import Control.Monad (filterM, forM, forM_, unless, void, when) import Data.ByteString (ByteString) import Data.ByteString qualified as B@@ -108,8 +132,9 @@ import Data.HashMap.Strict qualified as HashMap import Data.HashSet (HashSet) import Data.HashSet qualified as HashSet+import Data.IORef (IORef, modifyIORef', newIORef, readIORef, writeIORef) import Data.IntMap.Strict qualified as IntMap-import Data.List (nub, sortOn)+import Data.List (nub, partition, sortOn) import Data.Map.Strict qualified as Map import Data.Maybe (catMaybes, fromMaybe, isJust, isNothing, mapMaybe) import Data.Serialize (Serialize, decode, encode, get, getWord32be, putWord32be, runGet, runPut)@@ -119,50 +144,59 @@ import Data.Time.Clock.POSIX (getPOSIXTime) import Data.Word (Word64) import Database.Redis qualified as R+import GHC.Clock (getMonotonicTimeNSec) import Haskoin   ( BlockHash,     BlockHeader (..),-    Ctx,     Network (..),     OutPoint (..),     Tx (..),     TxHash,     TxIn (..),+    TxOut (..),     blockHashToHex,+    eitherToMaybe,     headerHash,     nullOutPoint,+    scriptToAddressBS,     txHash,   ) import Haskoin.Node (LogLevel (..))-import Haskoin.Store.Common (Limits (..), Replaced (..), StoreRead (..))+import Haskoin.Store.Common (Limits (..), Replaced (..), StoreRead (..), loopForever) import Haskoin.Store.Data-  ( Balance (..),-    BlockData (..),+  ( BlockData (..),     BlockRef (..),+    Prev (..),     Spender (..),     TxData (..),     UnixTime,     Unspent (..),-    XPubBal (..),     XPubSpec,-    nullBalance,   )-import Haskoin.Store.Database.Mempool qualified as M import Haskoin.Store.Database.Query qualified as Q-import Haskoin.Store.Database.Query.Layered qualified as L-import Haskoin.Store.Database.Query.Pooled (PoolReads (..), pageStart, pooledRead)-import Haskoin.Store.Database.Reader (DatabaseReader (..), txCF)-import Haskoin.Store.Database.Types-  ( MemAddrKey (..),-    MemAddrVal (..),-    MemSummary (..),-    TxKey (..),-  )+import Haskoin.Store.Database.Query.Pooled (From, PoolReads (..), pageStart, pooledRead)+import Haskoin.Store.Database.Reader (DatabaseReader (..), txCF, withSnapshotReader)+import Haskoin.Store.Database.Types (MemSummary (..), TxKey (..)) import Haskoin.Store.Database.Views (Views, heldViews, keepViews, newViews, takeView, withViewAt)+import Haskoin.Store.Exception (handleAny, tryAny) import Haskoin.Store.Fleet qualified as F import Haskoin.Store.Fleet.Redis import Haskoin.Store.Fleet.XPub import Haskoin.Store.Outage (Outage, attempt, newOutage)+import Haskoin.Store.XPub (EntryMoved (..), XPubReads (..))+import Haskoin.Store.XPub.Indexer+  ( Indexer,+    Runner (..),+    Settings (..),+    newIndexer,+    plainRunner,+    prefetch,+    requestWithin,+    runIndexer,+  )+import Haskoin.Store.XPub.Lock (Lock (..), defaultLockConfig, redisLock)+import Haskoin.Store.XPub.Service (Backend (..), Session (..), XPubs (..))+import System.IO (fixIO) import System.Metrics.StatsD   ( StatCounter,     StatGauge,@@ -175,24 +209,8 @@     newStatTiming,     setGauge,   )-import UnliftIO-  ( TVar,-    atomically,-    handle,-    handleAny,-    modifyIORef',-    modifyTVar',-    newIORef,-    newTVarIO,-    readIORef,-    readTVar,-    readTVarIO,-    retrySTM,-    stateTVar,-    timeout,-    writeIORef,-    writeTVar,-  )+import System.Random (randomIO)+import System.Timeout (timeout)  -- | How a store takes part in a fleet. data MemberConfig = MemberConfig@@ -211,13 +229,12 @@     -- for never     expiry :: !Word64,     -- | how many extended public keys the fleet's cache of them may hold-    -- ("Haskoin.Store.Fleet.XPub"), the least recently used going first, or-    -- 0 not to use the cache+    -- ("Haskoin.Store.Fleet.XPub"), evicted by GreedyDual: at least 1, as+    -- the cache cannot be turned off     cacheMax :: !Int,-    -- | cache a key that has at least this many used addresses-    cacheMinAddrs :: !Int,-    -- | or at least this many transactions in its addresses-    cacheMinTxs :: !Int,+    -- | the settings of the indexer of xpubs: its threads, and how long a+    -- request waits for an xpub ("Haskoin.Store.XPub.Indexer")+    indexer :: !Settings,     -- | StatsD client, to report what the cache of extended public keys     -- does, as t'CacheMetrics' lists     stats :: !(Maybe Stats),@@ -228,8 +245,7 @@   }  -- | StatsD metrics of the fleet's cache of extended public keys, as this--- store sees it. They have the names of those of "Haskoin.Store.Cache"--- under @fleet@, so that a dashboard reads both alike.+-- store sees it, under @fleet.cache@. data CacheMetrics = CacheMetrics   { -- | requests about a key whose balances the cache gave     -- (@fleet.cache.hits@)@@ -244,8 +260,8 @@     -- | builds that this store gave up, because the tip moved or another     -- store wrote the entry (@fleet.cache.abandoned@)     abandoned :: !StatCounter,-    -- | entries that this store deleted for being the least recently used-    -- (@fleet.cache.evictions@)+    -- | entries that this store evicted, those of the lowest priority by+    -- GreedyDual (@fleet.cache.evictions@)     evictions :: !StatCounter,     -- | times that this store emptied the cache (@fleet.cache.emptied@)     emptied :: !StatCounter,@@ -297,7 +313,7 @@     offered :: !(TVar (HashSet TxHash)),     -- | those that it refused, or that there was no time to offer, with     -- the times at which they entered the mempool: they are offered again-    waiting :: !(TVar (HashMap TxHash UnixTime)),+    waiting :: !(TVar (HashMap TxHash (UnixTime, Tx))),     -- | how many times the mempool was offered, and at which tip last     rounds :: !(TVar Int),     offeredAt :: !(TVar (Maybe BlockHash)),@@ -307,13 +323,14 @@     -- member does nothing, so that its lease runs out and a store that is     -- stuck does not hold the fleet back.     healthy :: !(TVar Bool),-    -- | the extended public keys that requests asked about and the cache-    -- did not have, by their names in Redis: 'memberCacheLoop' builds-    -- their entries-    wanted :: !(TVar (HashMap ByteString XPubSpec)),-    -- | the fleet tip at which the cache was found at another block, at-    -- the last turn-    lagging :: !(TVar (Maybe ByteString)),+    -- | the indexer that builds the entries of the xpubs that requests+    -- miss, one store of the fleet for each, holding its dibs in Redis+    indexer :: !(Indexer XPubSpec ()),+    lock :: !(Lock XPubSpec),+    -- | how many builds Redis announced, by xpub, for those that wait+    built :: !(TVar (HashMap ByteString Int)),+    -- | the door to the cache, for the web server+    door :: !XPubs,     -- | how many times the pool had been emptied at this store's last     -- turn ('noteRestarts')     restarts :: !(TVar (Maybe Integer)),@@ -326,9 +343,27 @@     gone :: !(TVar (HashSet TxHash)),     -- | whether the last thing that the member tried with Redis failed,     -- so that an outage is logged once, by whichever loop meets it-    outage :: !Outage+    outage :: !Outage,+    -- | the epoch of Redis at the last turn ('noteEpoch')+    epoch :: !(TVar (Maybe ByteString)),+    -- | how many times this store found that Redis had lost its data: the+    -- web server fills the pool again each time ('Haskoin.Store.Web')+    refills :: !(TVar Int),+    -- | the last tip that Redis was seen to have, and when, by the clock of+    -- 'getMonotonicTimeNSec': a request that read nothing of the pool+    -- takes it for the tip for 'seenFor', instead of asking Redis+    -- ('answeringWith')+    seen :: !(IORef (ByteString, Word64))   } +-- | For how long, in nanoseconds, a tip that Redis was seen to have is+-- taken to be the tip still, by a request that reads nothing of the pool.+-- Asking Redis is a round trip for every such request, and a request a+-- few milliseconds earlier would have had that tip anyway. It never mixes+-- tips: a read of the pool checks the tip in Redis in the same step.+seenFor :: Word64+seenFor = 5000000+ -- | A member with no views yet, on a connection to the Redis of the fleet. newMember :: MemberConfig -> R.Connection -> DatabaseReader -> IO Member newMember config redis db = do@@ -342,13 +377,30 @@   offeredAt <- newTVarIO Nothing   changed <- newTVarIO True   healthy <- newTVarIO True-  wanted <- newTVarIO HashMap.empty-  lagging <- newTVarIO Nothing   restarts <- newTVarIO Nothing   metrics <- mapM newCacheMetrics config.stats   gone <- newTVarIO HashSet.empty   outage <- newOutage-  return Member {keys = fleetKeys config.prefix 32, ..}+  built <- newTVarIO HashMap.empty+  epoch <- newTVarIO Nothing+  refills <- newTVarIO 0+  seen <- newIORef ("", 0)+  lock <- redisLock (defaultLockConfig (config.prefix <> "fleet:") config.name) redis xpubId+  fixIO $ \m -> do+    let runner = (plainRunner (\_ x () -> buildFor m x)) {ready = isJust <$> readTVarIO m.tip}+    indexer <- newIndexer config.indexer lock runner+    let door =+          XPubs+            { backend =+                Backend+                  { ask = \x w -> requestWithin indexer w x (),+                    await = awaitBuilt m,+                    eta = \x -> fmap fst <$> lock.holder x,+                    delete = \x -> True <$ cacheDrop redis (fleetKeys config.prefix 32) x+                  },+              patience = config.indexer.wait+            }+    return Member {keys = fleetKeys config.prefix 32, ..}  -- | Take a view of the database for its best block. The block store calls -- this right after it imports or reverts a block, and when it starts,@@ -394,11 +446,13 @@  -- | One round of what a member does: report, move the tip if the stores -- agree on another block, offer what the pool does not have of the store's--- mempool, and let go of old views. Call it every second or two, and when+-- mempool, let go of old views, and trim what the fleet has seen+-- ('fleetTrimSeen'). Call it every second or two, and when -- the best block changes. A store whose own health checks fail must not -- call it: its lease then runs out, and the fleet goes on without it. memberTick :: Member -> IO () memberTick m = do+  noteEpoch m   report m   moved <- moveTip m   -- the tip is read after the report, as the rule of 'F.keep' asks@@ -406,9 +460,10 @@   atomically (writeTVar m.tip (fromMaybe Nothing tip))   noteRestarts m   prune m-  syncCache m+  cacheGauges m   unless moved (expirePool m)   offerMempool m+  void (fleetTrimSeen m.redis m.keys mostSeen)   where     decoded = either (const Nothing) Just . decode @@ -428,6 +483,40 @@       writeTVar m.waiting HashMap.empty       writeTVar m.changed True +-- | Find out whether Redis lost its data since this store's last turn.+-- Redis keeps an epoch, a random number that the first store to find it+-- missing sets. A store that finds it missing or changed forgets what it+-- offered to the pool and its view of the fleet tip, so that it is+-- unhealthy until a tip is set again, which its next turn does, since+-- every live store reports. It counts the loss in 'refills', and the web+-- server then asks the peers, and the node over RPC if it has one, for+-- their mempools, which fill the pool again through 'memberReceived'.+-- Entries of the cache of extended public keys are built again on demand.+--+-- Without an epoch, a loss would show only in the count of restarts of+-- the pool going back, which says nothing when it was 0 before.+noteEpoch :: Member -> IO ()+noteEpoch m = do+  mine <- C.pack . show <$> (randomIO :: IO Word64)+  now <- redis' (R.setnx key mine >> R.get key)+  lost <- atomically $ do+    before <- readTVar m.epoch+    writeTVar m.epoch now+    let lost = isJust before && before /= now+    when lost $ do+      writeTVar m.offered HashSet.empty+      writeTVar m.offeredAt Nothing+      writeTVar m.tip Nothing+      writeTVar m.changed True+      modifyTVar' m.refills (+ 1)+    return lost+  when lost . m.config.note LevelWarn $+    "Redis lost its data: the pool fills again from the peers, and xpub entries are built again on demand"+  where+    key = m.keys.prefix <> "fleet:epoch"+    redis' :: R.Redis (Either R.Reply a) -> IO a+    redis' f = R.runRedis m.redis f >>= either (fail . ("Redis error: " <>) . show) return+ -- | Take a turn as a member every two seconds, and at once when an action -- that waits for the next best block returns, for as long as the store's -- health checks pass. A turn that fails, as when Redis is away, is tried@@ -458,7 +547,7 @@   where     go at = do       heard <- attempt m.outage m.config.note "read the events of the fleet" (listen at)-      unless heard (void (timeout 1000000 (atomically retrySTM) :: IO (Maybe ())))+      unless heard (void (timeout 1000000 (atomically retry) :: IO (Maybe ())))       go at     -- what there is after the place, waiting a second for it     listen at = do@@ -483,10 +572,18 @@     main h = maybe False (.main) <$> Q.getBlock m.db h  -- | Move the fleet tip to the block that the live stores agree on, if it is--- another one, with what the blocks in between did to the pool. Says--- whether it moved. When this store cannot say what they did, the pool is--- emptied: Redis allows that to a store that is alone, and to any store--- when the way is too long for all of them (v'TooFar').+-- another one, with what the blocks in between did to the pool and to the+-- cache of extended public keys, in one commit. Says whether it moved.+--+-- The move begins under a token ('cacheBegin'), which has builds wait;+-- what the blocks did at the addresses that entries have, a reorganisation+-- undone address by address, is written in pieces ('tipMoves',+-- 'cachePieces'); then 'fleetCommit' moves the pool, the tip and the cache+-- together. A move that is refused is aborted, and the entries that it+-- wrote are dropped ('cacheAbort'). When this store cannot say what the+-- blocks did, the pool is emptied, and the cache with it: Redis allows that+-- to a store that is alone, and to any store when the way is too long for+-- all of them (v'TooFar'). moveTip :: Member -> IO Bool moveTip m = do   seen <- fleetTip m.redis m.keys@@ -502,23 +599,51 @@       fleet = F.Fleet {tip = seen, reports}   case F.agreed now fleet of     Just target | Just target /= seen -> do+      token <- C.pack . show <$> (randomIO :: IO Word64)+      cacheBegin m.redis m.keys token (fromMaybe "" seen) >>= \case+        Moving -> do+          m.config.note LevelDebug "Fleet tip not moved: another store is moving it"+          return False+        BeginRaced -> return False+        Begun -> moveWith seen token target+    agreed -> do+      when (isNothing agreed) . m.config.note LevelDebug $+        "The live stores have no block in common: " <> shown (Map.size reports) <> " reports"+      return False+  where+    -- the move under a token of the cache: what the blocks did at the+    -- addresses of the entries is written in pieces, and the commit moves+    -- the pool, the tip and the cache together+    moveWith seen token target = do       way <- case (traverse decode seen, decode target) of-        (Right from, Right to) -> wayBetween m.db from to+        (Right from, Right to) -> wayBetween m.views m.db from to         _ -> return NoWay       let between = case way of             Way b -> Just b             _ -> Nothing           path = maybe [] (map encode . (.path)) between-      moved <- case way of-        TooFar -> do-          m.config.note LevelWarn $-            "The fleet tip is more than "-              <> shown farthest-              <> " blocks from "-              <> blockName target-              <> ": starting the shared mempool again there"-          fleetRestart m.redis m.keys 32 Nothing seen target-        _ -> fleetAdvance m.redis m.keys 32 Nothing seen target ((.pool) <$> between) path+      -- what the blocks did at the addresses of the entries, read at the+      -- new tip: without a view of it, this store does not move the tip+      written <- case (way, decode target) of+        (Way b, Right to) ->+          withViewAt m.views to (\r -> tipMoves m.redis m.keys r b.abandoned b.added) >>= \case+            Just moves -> cachePieces m.redis m.keys token (fromMaybe "" seen) moves+            Nothing -> return False+        (Way _, _) -> return False+        _ -> return True+      moved <-+        if not written+          then return Raced+          else case way of+            TooFar -> do+              m.config.note LevelWarn $+                "The fleet tip is more than "+                  <> shown farthest+                  <> " blocks from "+                  <> blockName target+                  <> ": starting the shared mempool again there"+              fleetRestart m.redis m.keys 32 Nothing seen target+            _ -> fleetCommit m.redis m.keys 32 Nothing seen target ((.pool) <$> between) path token       m.config.note LevelDebug . mconcat $         [ "Fleet tip from ",           maybe "none" blockName seen,@@ -534,13 +659,13 @@             TooFar -> " (the way is too long)"         ]       case moved of-        MovedTo _ -> advanceCache m seen target between >> return True-        _ -> return False-    agreed -> do-      when (isNothing agreed) . m.config.note LevelDebug $-        "The live stores have no block in common: " <> shown (Map.size reports) <> " reports"-      return False-  where+        MovedTo _ -> do+          let restarted = case way of+                Way _ -> False+                _ -> True+          when restarted $ wasEmptied m target "the shared mempool started again"+          return True+        _ -> False <$ cacheAbort m.redis m.keys token     redis' :: R.Redis (Either R.Reply a) -> IO a     redis' f = R.runRedis m.redis f >>= either (fail . ("Redis error: " <>) . show) return     chunks bytes@@ -565,8 +690,8 @@ -- 'Nothing' if the database does not have one of the tips, if the second -- is not on its main chain, or if the way is longer than 'farthest' -- blocks on either side.-changeBetween :: DatabaseReader -> Maybe BlockHash -> BlockHash -> IO (Maybe PoolChange)-changeBetween db from to = fmap (.pool) <$> blocksBetween db from to+changeBetween :: Views BlockHash -> DatabaseReader -> Maybe BlockHash -> BlockHash -> IO (Maybe PoolChange)+changeBetween views db from to = fmap (.pool) <$> blocksBetween views db from to  -- | What the blocks between two tips did. data Between = Between@@ -577,17 +702,20 @@     added :: ![TxData],     -- | whether blocks of the way to the first tip were abandoned     reorganised :: !Bool,+    -- | the transactions of those abandoned blocks, in their order, as the+    -- view of the database at the first tip has them+    abandoned :: ![TxData],     -- | those blocks of the main chain, oldest first, the second tip last:     -- what the fleet tells its clients that the tip moved through     path :: ![BlockHash]   } --- | 'changeBetween', with the transactions that the new blocks confirm,--- which is what the cache of extended public keys is brought to the new--- tip with ('advanceCache').-blocksBetween :: DatabaseReader -> Maybe BlockHash -> BlockHash -> IO (Maybe Between)-blocksBetween db from to =-  wayBetween db from to <&> \case+-- | 'changeBetween', with the transactions that the new blocks confirm and+-- those of the abandoned blocks, which is what the cache of extended public+-- keys is brought to the new tip with ('moveTip').+blocksBetween :: Views BlockHash -> DatabaseReader -> Maybe BlockHash -> BlockHash -> IO (Maybe Between)+blocksBetween views db from to =+  wayBetween views db from to <&> \case     Way between -> Just between     _ -> Nothing @@ -606,10 +734,17 @@ -- store does not know. The fleet treats them differently: a tip that no -- store will ever say the way from has to be left behind -- ('fleetRestart'), and one that another store may know has to wait.-wayBetween :: DatabaseReader -> Maybe BlockHash -> BlockHash -> IO Way-wayBetween db from to = L.withView db $ \view -> do-  let r = L.viewReader view-      -- the blocks from one that is not on the main chain back to it+--+-- The transactions of the blocks that a reorganisation abandoned are no+-- longer in the database: reverting a block takes them out+-- ("Haskoin.Store.Logic"). They are read from the store's view of the+-- database at the first tip, which it keeps ('noteBest'). Without that+-- view the store cannot say what they held, and the way is v'NoWay': it+-- does not move the tip, which keeps the cache of extended public keys+-- whole, since a reorganisation never empties it. Another store may know.+wayBetween :: Views BlockHash -> DatabaseReader -> Maybe BlockHash -> BlockHash -> IO Way+wayBetween views db from to = withSnapshotReader db $ \r -> do+  let -- the blocks from one that is not on the main chain back to it       back :: Int -> BlockHash -> IO (Either Way ([BlockData], BlockData))       back n h         | n > farthest = return (Left TooFar)@@ -639,68 +774,92 @@       | bd.main ->           down 0 fork bd >>= \case             Left none -> return none-            Right added -> do-              old <- concat <$> mapM (bodies view) (reverse abandoned)-              new <- concat <$> mapM (confirmedBy r) (reverse added)-              -- those of the abandoned blocks that this store has as-              -- replaced: every store had them confirmed, and says the same-              let named h rep = encode h <> encode rep.by <> runPut (putWord32be rep.height)-              lostOnes <- fmap catMaybes . forM [h | (h, Nothing) <- old] $ \h ->-                fmap (named h) <$> Q.getReplaced r h-              let pool =-                    PoolChange-                      { returned = [poolTx r.net r.ctx td | (_, Just td) <- old, not (coinbase td)],-                        gone = [encode h | (h, found) <- old, maybe True coinbase found],-                        confirmed = map (encode . txHash . (.tx)) new,-                        -- with what spends each output and where, for what-                        -- the blocks replaced-                        spent =-                          [ encode i.outpoint <> encode (txHash td.tx) <> runPut (putWord32be h)-                          | td <- new,-                            BlockRef {height = h} <- [td.block],-                            i <- td.tx.inputs,-                            i.outpoint /= nullOutPoint-                          ],-                        height = Just bd.height,-                        lost = lostOnes-                      }-              let path = map (headerHash . (.header)) (reverse added)-              return (Way Between {pool, added = new, reorganised = not (null abandoned), path})+            Right added -> maybe (return NoWay) (wayWith r bd abandoned added) =<< abandonedBodies (reverse abandoned)     _ -> return NoWay   where-    -- the transactions of an abandoned block, each with what the store has-    -- of it now, if it still has it and did not delete it-    bodies view bd = do-      found <- L.getManyTxData view bd.txs-      return [(h, found' >>= live) | (h, found') <- zip bd.txs found]+    wayWith r bd abandoned added old = do+      new <- concat <$> mapM (confirmedBy r) (reverse added)+      -- what the abandoned blocks give back enters the pool now, not at+      -- the time of its block, which the pool would expire at once+      now <- round <$> getPOSIXTime+      -- those of the abandoned blocks that this store has as+      -- replaced: every store had them confirmed, and says the same+      let named h rep = encode h <> encode rep.by <> runPut (putWord32be rep.height)+      lostOnes <- fmap catMaybes . forM [h | (h, Nothing) <- old] $ \h ->+        fmap (named h) <$> Q.getReplaced r h+      let pool =+            PoolChange+              { returned = [poolTx r.net (td :: TxData) {block = MemRef now} | (_, Just td) <- old, not (coinbase td)],+                gone = [encode h | (h, found) <- old, maybe True coinbase found],+                confirmed = map (encode . txHash . (.tx)) new,+                -- with what spends each output and where, for what+                -- the blocks replaced+                spent =+                  [ encode i.outpoint <> encode (txHash td.tx) <> runPut (putWord32be h)+                  | td <- new,+                    BlockRef {height = h} <- [td.block],+                    i <- td.tx.inputs,+                    i.outpoint /= nullOutPoint+                  ],+                height = Just bd.height,+                lost = lostOnes+              }+      let path = map (headerHash . (.header)) (reverse added)+      let held = [td | (_, Just td) <- old]+      return (Way Between {pool, added = new, reorganised = not (null abandoned), abandoned = held, path})+    -- the transactions of the abandoned blocks, each with what the view at+    -- the first tip has of it; 'Nothing' without that view+    abandonedBodies [] = return (Just [])+    abandonedBodies bds = do+      let hs = concatMap (.txs) bds+      case from of+        Nothing -> return Nothing+        Just tip -> withViewAt views tip (\old -> zip hs . map (>>= live) <$> bodiesIn old hs)+    bodiesIn old hs = Q.retrieveManyCF old (txCF old.db) (map TxKey hs)     live td = if td.deleted then Nothing else Just td     confirmedBy :: DatabaseReader -> BlockData -> IO [TxData]-    confirmedBy r bd = catMaybes <$> Q.retrieveManyCF r.db (txCF r.db) (map TxKey bd.txs)+    confirmedBy r bd = catMaybes <$> Q.retrieveManyCF r (txCF r.db) (map TxKey bd.txs)     coinbase td = any ((== nullOutPoint) . (.outpoint)) td.tx.inputs --- | A transaction as the pool keeps it: its body without spenders, and--- what it does at each address, as "Haskoin.Store.Database.Mempool" works--- them out for the store's own mempool area.-poolTx :: Network -> Ctx -> TxData -> PoolTx-poolTx net ctx td =+-- | A transaction as the pool keeps it: its body without spenders, at the+-- time that it entered the pool, and what it does at each address: the+-- transaction touches it, spends an output of it, or pays to it. It is a+-- function of the transaction and that time alone.+poolTx :: Network -> TxData -> PoolTx+poolTx net td =   PoolTx-    { txid = encode records.txid,+    { txid = encode h,       time,-      inputs = [encode op | (op, _) <- records.spends],-      body = encode records.body,+      inputs = [encode op | op <- spent],+      body = encode (td :: TxData) {block = MemRef time, deleted = False, spenders = IntMap.empty},       signals = net.replaceByFee && any ((< 0xffffffff - 1) . (.sequence)) td.tx.inputs,-      entries = mapMaybe entry records.addresses+      entries =+        [PoolTouches (encode a) | a <- touched]+          <> [ PoolSpends (encode a) (encode i.outpoint) p.value+             | (k, i) <- zip [0 :: Int ..] td.tx.inputs,+               i.outpoint /= nullOutPoint,+               Just p <- [IntMap.lookup k td.prevs],+               Just a <- [address p.script]+             ]+          <> [ PoolPays (encode a) (encode (OutPoint h i)) o.value o.script+             | (i, o) <- zip [0 ..] td.tx.outputs,+               Just a <- [address o.script]+             ]     }   where+    h = txHash td.tx+    -- never 0, which in an entry is the chain's layer+    -- ("Haskoin.Store.Fleet.XPub".'score'); what a reorganisation gives+    -- back comes with the time it is given back ('wayBetween')     time = case td.block of-      MemRef t -> t-      BlockRef {} -> 0-    records = M.mempoolRecords net ctx time td-    entry = \case-      (MemAddrTxKey a _ _, _) -> Just (PoolTouches (encode a))-      (MemAddrSpentKey a p, MemAddrSpentVal v) -> Just (PoolSpends (encode a) (encode p) v)-      (MemAddrOutKey a _ p, MemAddrOutVal v s) -> Just (PoolPays (encode a) (encode p) v s)-      _ -> Nothing+      MemRef t -> max 1 t+      BlockRef {} -> max 1 td.timestamp+    spent = [i.outpoint | i <- td.tx.inputs, i.outpoint /= nullOutPoint]+    address = eitherToMaybe . scriptToAddressBS net+    touched =+      nub $+        mapMaybe (address . (.script)) (IntMap.elems td.prevs)+          <> mapMaybe (address . (.script)) td.tx.outputs  -- | Offer to the pool the transactions of the store's mempool that it does -- not have, oldest first, so that each comes after those that it spends@@ -732,82 +891,122 @@   readTVarIO m.tip >>= \case     Nothing -> return ()     Just tip -> void . withViewAt m.views tip $ \at -> do-      before <- readTVarIO m.offeredAt-      (n, dirty) <- atomically $ do-        writeTVar m.offeredAt (Just tip)-        (,) <$> stateTVar m.rounds (\k -> (k, k + 1)) <*> stateTVar m.changed (,False)-      todo <- if dirty || before /= Just tip || n `mod` wholeEvery == 0 then everything else newest-      let (now, later) = splitAt mostOffers (sortOn fst todo)-      atomically (modifyTVar' m.waiting (HashMap.union (HashMap.fromList [(h, time) | (time, h) <- later])))-      go at tip now-  where-    -- all of the mempool that the pool does not have, forgetting what has-    -- left the mempool-    everything = do-      mempool <- Q.getMempool m.db-      have <- readTVarIO m.offered-      let kept = HashSet.intersection have (HashSet.fromList (map snd mempool))-      atomically (writeTVar m.offered kept >> writeTVar m.waiting HashMap.empty)-      return [(time, h) | (time, h) <- mempool, not (HashSet.member h kept)]-    -- the newest of the mempool that this member has not seen, and what is-    -- waiting-    newest = do-      have <- readTVarIO m.offered       held <- readTVarIO m.waiting-      let known h = HashSet.member h have || HashMap.member h held-          scan offset = do-            page <- Q.getMempoolPage m.db Limits {limit = pageSize, offset, start = Nothing}-            let fresh = filter (not . known . snd) page-            if null fresh || length page < fromIntegral pageSize-              then return fresh-              else (fresh <>) <$> scan (offset + pageSize)-      found <- scan 0-      return (found <> [(time, h) | (h, time) <- HashMap.toList held])-    pageSize = 256-    go _ _ [] = return ()-    go at tip ((time, h) : rest) =-      M.getMempoolTx m.db h >>= \case-        Nothing -> forget h >> go at tip rest+      let todo = take mostOffers (sortOn (fst . snd) (HashMap.toList held))+          pool = redisPool m.redis m.keys (encode tip) (return ())+      go at pool tip todo+  where+    go _ _ _ [] = return ()+    go at pool tip ((h, (time, tx)) : rest) =+      txDataAt at pool time tx >>= \case+        -- an input that neither the tip nor the pool has: an orphan for now+        Nothing -> go at pool tip rest         Just td -> do-          let tx = poolTx m.db.net m.db.ctx td-          unspent <- forM [i.outpoint | i <- td.tx.inputs, i.outpoint /= nullOutPoint] $ \p ->+          unspent <- forM [i.outpoint | i <- tx.inputs, i.outpoint /= nullOutPoint] $ \p ->             isJust <$> Q.getUnspent at p-          let offer = poolOffer m.redis m.keys 32 (Just (encode tip)) tx unspent+          let offer = poolOffer m.redis m.keys 32 (Just (encode tip)) (poolTx m.db.net td) unspent               done = atomically $ do                 modifyTVar' m.offered (HashSet.insert h)                 modifyTVar' m.waiting (HashMap.delete h)-              wait = atomically (modifyTVar' m.waiting (HashMap.insert h time))           offer [] >>= \case-            Taken -> done >> go at tip rest-            Known -> done >> go at tip rest-            Orphan _ -> wait >> go at tip rest-            -- the tip moved: the next round reads the whole mempool+            Taken -> done >> go at pool tip rest+            Known -> done >> go at pool tip rest+            Orphan _ -> go at pool tip rest             Late -> return ()-            Conflict rivals -> do-              mine <- mapM deletedHere rivals-              if and mine-                then-                  offer rivals >>= \case-                    Taken -> done-                    _ -> wait-                else wait-              go at tip rest-    forget h = atomically (modifyTVar' m.waiting (HashMap.delete h))-    deletedHere rival = case decode rival of-      Left _ -> return False-      -- it deleted it, and has not taken it back since. A store that was-      -- started again does not remember, and leaves the conflict as it-      -- is: the first of the two stays in the pool.-      Right h -> do-        deleted <- HashSet.member h <$> readTVarIO m.gone-        back <- isJust <$> M.getMempoolTx m.db h-        return (deleted && not back)+            -- a peer relayed it, so its node took it in place of what it+            -- conflicts with: it replaces them in the pool too+            Conflict rivals -> offer rivals >> done >> go at pool tip rest --- | How many rounds of offers go by between two that read the whole--- mempool: five minutes of them.-wholeEvery :: Int-wholeEvery = 150+-- | A transaction that the store heard from a peer, to offer to the pool at+-- the next turn, with the time at which it was heard. There is no mempool+-- of the store's own: the pool in Redis is the mempool.+memberReceived :: Member -> UnixTime -> Tx -> IO ()+memberReceived m time tx = atomically $ do+  have <- HashSet.member (txHash tx) <$> readTVar m.offered+  unless have $ modifyTVar' m.waiting (HashMap.insertWith (\_ old -> old) (txHash tx) (time, tx)) +-- | Of these transactions, those that the block store need not fetch when+-- a peer announces them: those that this store holds to offer, and those+-- that the fleet has seen ('fleetTrimSeen'), asked of Redis in one round+-- trip. The fleet sees a transaction only once the pool has decided on it,+-- so what this store received and has not offered yet is known here alone.+-- Without an answer from Redis within a second, only what the store holds;+-- the store then fetches the rest, and finds out at the offer what the pool+-- has.+memberSeen :: Member -> [TxHash] -> IO (HashSet TxHash)+memberSeen m hs = do+  held <- readTVarIO m.waiting+  let (mine, others) = partition (`HashMap.member` held) hs+  fleet <-+    if null others+      then return HashSet.empty+      else+        handleAny (\_ -> return HashSet.empty) $+          timeout 1000000 (fleetSeen m.redis m.keys (map encode others)) <&> \case+            Just flags -> HashSet.fromList [h | (h, True) <- zip others flags]+            Nothing -> HashSet.empty+  return (HashSet.union (HashSet.fromList mine) fleet)++-- | How many transactions the fleet remembers having seen, the newest+-- ('fleetTrimSeen'), trimmed at each turn of a member. A peer announces a transaction within seconds of having+-- it, so a hundred thousand covers far more than the time in which the+-- peers of the fleet announce one, at any rate that Bitcoin carries.+mostSeen :: Int+mostSeen = 100000++-- | The body of a transaction of the pool, for the filters of @/events@.+memberPoolBody :: Member -> TxHash -> IO (Maybe TxData)+memberPoolBody m h =+  readTVarIO m.tip >>= \case+    Nothing -> return Nothing+    Just tip ->+      (redisPool m.redis m.keys (encode tip) (return ())).bodies [h] <&> \case+        [(td, _)] -> td+        _ -> Nothing++-- | The time at which the newest transaction of the pool entered it, if the+-- pool has any and the member can read it at the tip that it knows: what+-- tells the health check that transactions still come in.+memberNewest :: Member -> IO (Maybe UnixTime)+memberNewest m =+  readTVarIO m.tip >>= \case+    Nothing -> return Nothing+    Just tip ->+      handleAny (\_ -> return Nothing) $+        (redisPool m.redis m.keys (encode tip) (return ())).page 0 1 <&> \case+          (t, _) : _ -> Just t+          [] -> Nothing++-- | A transaction with the outputs that it spends, from the view at the tip+-- or from the pool; 'Nothing' if one of them is in neither.+txDataAt :: DatabaseReader -> PoolReads -> UnixTime -> Tx -> IO (Maybe TxData)+txDataAt at pool time tx = do+  prevs <- forM (zip [0 :: Int ..] tx.inputs) $ \(n, i) ->+    if i.outpoint == nullOutPoint+      then return (Just Nothing)+      else+        Q.getUnspent at i.outpoint >>= \case+          Just u -> return (Just (Just (n, Prev u.script u.value)))+          Nothing ->+            pool.bodies [i.outpoint.hash] >>= \case+              [(Just parent, _)]+                | fromIntegral i.outpoint.index < length parent.tx.outputs ->+                    let o = parent.tx.outputs !! fromIntegral i.outpoint.index+                     in return (Just (Just (n, Prev o.script o.value)))+              _ -> return Nothing+  return $ do+    ps <- sequence prevs+    return+      TxData+        { block = MemRef time,+          tx,+          prevs = IntMap.fromList (catMaybes ps),+          deleted = False,+          rbf = False,+          timestamp = time,+          spenders = IntMap.empty+        }+ -- | The most transactions that one round offers. An offer takes a tenth of -- a millisecond or two, so this is well under the two seconds of a round. mostOffers :: Int@@ -865,7 +1064,8 @@ -- asking. Every read of the pool checks it and throws v'TipMoved' if the tip -- is another one. The action is also given a check to run when it has what -- it needs: it asks Redis for the tip if no read of the pool has done so,--- and throws the same. An answer that passed either is one that every+-- and no request saw that tip in Redis in the last 'seenFor', and throws+-- the same. An answer that passed either is one that every -- store of the fleet gave at that moment. answeringAt :: Member -> ByteString -> (DatabaseReader -> PoolReads -> IO () -> IO a) -> IO (Maybe a) answeringAt m tip action = answeringWith m tip (\r pool _ confirm -> action r pool confirm)@@ -878,13 +1078,19 @@   Left _ -> return Nothing   Right block -> withViewAt m.views block $ \r -> do     checked <- newIORef False-    let confirm =+    let saw = getMonotonicTimeNSec >>= \t -> writeIORef m.seen (tip, t)+        confirm =           readIORef checked >>= \case             True -> return ()             False -> do-              now <- memberTip m-              if now == Just tip then writeIORef checked True else throwIO TipMoved-        found = writeIORef checked True+              (last', at) <- readIORef m.seen+              t <- getMonotonicTimeNSec+              if last' == tip && t - at < seenFor+                then writeIORef checked True+                else do+                  now <- memberTip m+                  if now == Just tip then saw >> writeIORef checked True else throwIO TipMoved+        found = saw >> writeIORef checked True     action r (redisPool m.redis m.keys tip found) found confirm  -- | 'answeringAt' with the queries ready: those of@@ -892,176 +1098,143 @@ -- the same with the cache of extended public keys in front -- ('cachedRead'), which is what requests are answered with unless they -- ask not to be.-answeringCached :: Member -> ByteString -> (StoreRead -> StoreRead -> IO () -> IO a) -> IO (Maybe a)+answeringCached :: Member -> ByteString -> (StoreRead -> Session -> IO () -> IO a) -> IO (Maybe a) answeringCached m tip action = answeringWith m tip $ \r pool found confirm -> do-  cached <- cachedRead m tip r pool found-  action (pooledRead r pool) cached confirm+  sess <- fleetSession m tip r pool found+  action (pooledRead r pool) sess confirm  ------------------------------------------------------------------------------- -- The cache of extended public keys ------------------------------------------------------------------------------- --- | The queries of a request with the fleet's cache of extended public--- keys in front of those about them: given the tip that the request--- answers from, the view at that block, the pool, and what to run after a--- read that found the tip where it should be.------ The balances of a key come from the cache if it has the key with the--- addresses that it has now. The other three queries about a key are given--- its balances by the caller, and are answered from the cache when this--- same request got them there: the cache is then asked for an entry with--- as many addresses, so that an entry that changed in between is not--- used. Whatever the cache does not answer the database does, and a key--- that it did not have is put down to be built ('wantCached'). A read that--- finds the tip moved throws v'TipMoved', like those of the pool.-cachedRead :: Member -> ByteString -> DatabaseReader -> PoolReads -> IO () -> IO StoreRead-cachedRead m tip r pool found-  | m.config.cacheMax <= 0 = return base-  | otherwise = do-      seen <- newIORef HashMap.empty-      let hit :: Cached a -> IO (Maybe a)-          hit = \case-            Hit x -> Just x <$ found-            Miss -> return Nothing-            Moved -> throwIO TipMoved-          entryOf x = HashMap.lookup (xpubId x) <$> readIORef seen-          -- from the cache if this request has the entry of the key, and-          -- from where the page starts if it has a start-          paged :: XPubSpec -> Limits -> IO [a] -> (Entry -> L.From -> IO (Cached [a])) -> IO [a]-          paged x limits direct cached =-            entryOf x >>= \case-              Nothing -> direct-              Just e ->-                pageStart r pool limits.start >>= \case-                  Nothing -> return []-                  Just from -> cached e from >>= hit >>= maybe direct return-      return-        base-          { xPubBals = \x -> do-              now <- milliseconds-              cachedEntry m.redis m.keys tip r.initGap r.maxGap now x >>= hit >>= \case-                Just e -> do-                  tally m (.hits) 1-                  e.balances <$ modifyIORef' seen (HashMap.insert (xpubId x) e)-                Nothing -> do-                  tally m (.misses) 1-                  bals <- base.xPubBals x-                  wantCached m x bals-                  return bals,-            xPubTxs = \x bals limits ->-              paged x limits (base.xPubTxs x bals limits) $ \e from ->-                cachedTxs m.redis m.keys tip x e.sizes from limits,-            xPubUnspents = \x bals limits ->-              paged x limits (base.xPubUnspents x bals limits) $ \e from ->-                cachedUnspents m.redis m.keys tip x e from limits,-            xPubTxCount = \x bals ->-              entryOf x >>= \case-                Nothing -> base.xPubTxCount x bals-                Just e ->-                  cachedCount m.redis m.keys tip x e.sizes >>= hit >>= maybe (base.xPubTxCount x bals) return-          }-  where-    base = pooledRead r pool+-- | The reads of the fleet's cache of xpubs for one request, at the tip+-- that it answers from, with its view and the pool: the balances of a key+-- if the cache has it with the addresses that it has now, and its other+-- lists from the entry with as many addresses. A read that finds the tip+-- moved throws v'TipMoved'; a miss is 'Nothing', and the door asks.+fleetSession :: Member -> ByteString -> DatabaseReader -> PoolReads -> IO () -> IO Session+fleetSession m tip r pool found = do+  seen <- newIORef HashMap.empty+  let hit :: Cached a -> IO (Maybe a)+      hit = \case+        Hit x -> Just x <$ found+        Miss -> return Nothing+        Moved -> throwIO TipMoved+      entryOf x = HashMap.lookup (xpubId x) <$> readIORef seen+      -- a list of the version of the entry that the request read, or the+      -- request is answered again+      same :: Entry -> Maybe (ByteString, b) -> IO (Maybe b)+      same e = \case+        Just (v, a) | v == e.version -> return (Just a)+        Just _ -> throwIO EntryMoved+        Nothing -> return Nothing+      paged :: XPubSpec -> Limits -> (Entry -> From -> IO (Cached (ByteString, [a]))) -> IO (Maybe [a])+      paged x limits cached =+        entryOf x >>= \case+          Nothing -> throwIO EntryMoved+          Just e ->+            pageStart r pool limits.start >>= \case+              Nothing -> return (Just [])+              Just from -> cached e from >>= hit >>= same e+  return+    Session+      { reads =+          XPubReads+            { balances = \x ->+                cachedEntry m.redis m.keys tip r.initGap r.maxGap True x >>= hit >>= \case+                  Just e -> do+                    -- read again, as after a miss: of the version read first+                    entryOf x >>= mapM_ (\old -> when (old.version /= e.version) (throwIO EntryMoved))+                    tally m (.hits) 1+                    -- payments used up most of the lookahead: the indexer+                    -- extends it, and the entry answers meanwhile+                    when e.short (void (prefetch m.indexer x ()))+                    Just e.balances <$ modifyIORef' seen (HashMap.insert (xpubId x) e)+                  Nothing -> Nothing <$ tally m (.misses) 1,+              transactions = \x limits ->+                paged x limits $ \e from -> cachedTxs m.redis m.keys tip x e.sizes from limits,+              unspents = \x _ limits -> paged x limits $ \e from -> cachedUnspents m.redis m.keys tip x e from limits,+              count = \x ->+                entryOf x >>= \case+                  Nothing -> throwIO EntryMoved+                  Just e -> cachedCount m.redis m.keys tip x e.sizes >>= hit >>= same e,+              bodies = return . map (const Nothing),+              wallet = return+            },+        settle = return ()+      } --- | The time in milliseconds, by which the cache orders the keys by use.+-- | The time in milliseconds, by which a build is timed. milliseconds :: IO Word64 milliseconds = round . (* 1000) <$> getPOSIXTime --- | Put an extended public key down to be cached, given its balances, if--- it is worth it: a key with few addresses and few transactions is read--- from the database as fast as from Redis. The thresholds are those of the--- configuration, and count as "Haskoin.Store.Cache" counts them.-wantCached :: Member -> XPubSpec -> [XPubBal] -> IO ()-wantCached m x bals =-  when (m.config.cacheMax > 0 && (length used >= m.config.cacheMinAddrs || txs >= m.config.cacheMinTxs)) $-    atomically . modifyTVar' m.wanted $ \w ->-      if HashMap.size w < 1000 then HashMap.insert (xpubId x) x w else w+-- | Run the indexer of the fleet's cache of xpubs, and hear the builds that+-- Redis announces, for as long as the store runs. The indexer is apart from+-- the turns of the member because an entry of a wallet with a long history+-- takes seconds to read, and a member that does not report for that long+-- loses its lease.+memberCacheLoop :: Member -> IO a+memberCacheLoop m = withAsync hear $ \a -> link a >> runIndexer m.indexer >> loopForever (return ())   where-    used = filter (not . nullBalance . (.balance)) bals-    txs = fromIntegral (sum (map (.balance.txs) used)) :: Int+    hear = loopForever $ do+      ctl <- R.newPubSubController [(m.config.prefix <> "fleet:built", heard)] []+      _ <- tryAny (R.pubSubForever m.redis ctl (return ()))+      atomically (modifyTVar' m.built (HashMap.map (+ 1)))+    heard i = atomically (modifyTVar' m.built (HashMap.insertWith (+) i 1)) --- | Build the entries of the keys that requests asked about and the cache--- did not have, one at a time, for as long as the store runs. A build that--- fails is told to a function. The least recently used entries are deleted--- when there are more than the configuration allows.+-- | Wait at most so many microseconds for Redis to announce a build of the+-- entry of a key; whether it did.+awaitBuilt :: Member -> XPubSpec -> Int -> IO Bool+awaitBuilt m x us = do+  n0 <- HashMap.lookupDefault 0 (xpubId x) <$> readTVarIO m.built+  timer <- registerDelay (max 1 us)+  atomically $+    (HashMap.lookupDefault 0 (xpubId x) <$> readTVar m.built >>= check . (> n0) >> return True)+      `orElse` (readTVar timer >>= check >> return False)++-- | Build the entry of a key at the fleet tip, the job of the indexer, and+-- announce it; then evict past the size of the cache. ----- This is apart from the turns of the member because an entry of a wallet--- with a long history takes seconds to read from the database, and a--- member that does not report for that long loses its lease.-memberCacheLoop :: Member -> IO a-memberCacheLoop m = go+-- A build waits while a move of the tip is under way, a few milliseconds+-- at a time, and is then made at the new tip, which it reads from Redis:+-- the member's own may be a turn old.+buildFor :: Member -> XPubSpec -> IO ()+buildFor m spec = do+  now <- milliseconds+  building (200 :: Int) >>= \case+    Nothing -> return ()+    Just built -> do+      let name = decodeUtf8 (xpubId spec)+      took <- subtract now <$> milliseconds+      _ <- R.runRedis m.redis (R.publish (m.config.prefix <> "fleet:built") (xpubId spec))+      case built of+        Just (Built n) | n > 0 -> do+          tally m (.builds) 1+          forM_ m.metrics (\x -> addTiming x.buildTime (fromIntegral took))+          m.config.note LevelDebug $+            "Cached xpub " <> name <> ": " <> shown n <> " addresses added in " <> shown took <> " ms"+          gone <- cacheEvict m.redis m.keys m.config.cacheMax+          when (gone > 0) $ tally m (.evictions) gone+        Just Abandoned -> tally m (.abandoned) 1+        _ -> return ()   where-    go = do-      spec <- atomically $ do-        w <- readTVar m.wanted-        case HashMap.toList w of-          [] -> retrySTM-          (k, x) : _ -> x <$ writeTVar m.wanted (HashMap.delete k w)-      void (attempt m.outage m.config.note "cache an xpub for the fleet" (build spec))-      go-    build spec =-      readTVarIO m.tip >>= \case-        Nothing -> return ()-        Just block -> do-          now <- milliseconds-          let tip = encode block-              name = decodeUtf8 (xpubId spec)+    building tries =+      memberTip m >>= \case+        Nothing -> return Nothing+        Just tip -> do           built <-             handle (\TipMoved -> return (Just Abandoned)) $-              answeringAt m tip (\r pool _ -> cacheBuild m.redis m.keys tip r pool now spec)-          took <- subtract now <$> milliseconds+              answeringAt m tip (\r pool _ -> cacheBuild m.redis m.keys tip r pool spec)           case built of-            Just (Built n) | n > 0 -> do-              tally m (.builds) 1-              forM_ m.metrics (\x -> addTiming x.buildTime (fromIntegral took))-              m.config.note LevelDebug $-                "Cached xpub " <> name <> ": " <> shown n <> " addresses added in " <> shown took <> " ms"-              gone <- cacheEvict m.redis m.keys m.config.cacheMax-              when (gone > 0) $ do-                tally m (.evictions) gone-                m.config.note LevelDebug ("Evicted " <> shown gone <> " xpubs from the cache")-            Just Abandoned -> do-              tally m (.abandoned) 1-              m.config.note LevelDebug ("Gave up caching xpub " <> name <> ": the fleet or the entry moved on")-            _ -> return ()---- | Bring the cache of extended public keys to the block that this member--- just moved the fleet tip to, with what the blocks in between did at the--- addresses that its entries have. The cache is emptied instead when the--- member cannot say what they did, when the chain was reorganised, which--- is rare and would need what the abandoned blocks did undone, or when--- the member has no view of the block.-advanceCache :: Member -> Maybe ByteString -> ByteString -> Maybe Between -> IO ()-advanceCache m seen target between = when (m.config.cacheMax > 0) $ do-  changes <- case (between, decode target) of-    (Just b, Right to) | not b.reorganised -> withViewAt m.views to $ \r -> do-      let touched = HashMap.toList (blockChanges r.net r.ctx b.added)-      flags <- cacheTracked m.redis m.keys (map fst touched)-      forM [x | (x, True) <- zip touched flags] $ \(a, change) -> do-        bal <- Q.getBalance r a-        return (a, bal, change)-    _ -> return Nothing-  let why = case (between, changes) of-        (Nothing, _) -> "this store cannot say what the blocks since the last tip did"-        (Just b, _) | b.reorganised -> "the chain was reorganised"-        (_, Nothing) -> "this store has no view of the block"-        _ -> "it was at another block than the last tip"-  cacheAdvance m.redis m.keys seen target changes >>= \case-    CacheEmptied -> wasEmptied m target why-    CacheMoved ->-      m.config.note LevelDebug $-        "The xpub cache is at block "-          <> blockName target-          <> ": "-          <> shown (maybe 0 length changes)-          <> " addresses changed"-    _ -> return ()+            Just Busy | tries > 0 -> threadDelay 25000 >> building (tries - 1)+            Just Abandoned | tries > 197 -> building (tries - 1)+            _ -> return (Just built)  -- | Count and say that this store emptied the cache, at a block and for a -- reason. It is information and not a warning: nothing is wrong with what--- the fleet answers, which comes from the database until entries are--- built again. Emptied at every block, though, the cache is of no use,--- and the log is where that shows.+-- the fleet answers, and entries are built again on demand. It happens+-- only when the pool starts again, or when a store cannot say what the+-- abandoned blocks of a reorganisation held. wasEmptied :: Member -> ByteString -> Text -> IO () wasEmptied m block why = do   tally m (.emptied) 1@@ -1074,31 +1247,16 @@ shown :: (Show a) => a -> Text shown = T.pack . show --- | See that the cache of extended public keys does not stay at another--- block than the fleet tip. It is there for a moment after every move of--- the tip, until the store that moved it has brought the cache along, and--- for good if that store stopped before it did. So a member that finds--- the cache behind the same tip at two turns in a row empties it, and it--- starts again at the tip. Every turn of a member runs this.-syncCache :: Member -> IO ()-syncCache m = when (m.config.cacheMax > 0) $ do-  -- how large the cache is, at every turn: it is one small read, and the-  -- number is the fleet's, the same from every store+-- | Tell StatsD how large the cache of extended public keys is, at every+-- turn: it is one small read, and the numbers are the fleet's, the same+-- from every store. The cache cannot lag behind the tip: it moves with it+-- ('moveTip').+cacheGauges :: Member -> IO ()+cacheGauges m =   forM_ m.metrics $ \x -> do     (entries, held) <- cacheSize m.redis m.keys     setGauge x.xpubs entries     setGauge x.addresses held-  tip <- fmap encode <$> readTVarIO m.tip-  at <- cacheTip m.redis m.keys-  before <- readTVarIO m.lagging-  case tip of-    Just t | at /= Just t -> do-      when (before == Just t) $-        cacheAdvance m.redis m.keys Nothing t Nothing >>= \case-          CacheEmptied -> wasEmptied m t "no store brought it to the fleet tip"-          _ -> return ()-      atomically (writeTVar m.lagging (Just t))-    _ -> atomically (writeTVar m.lagging Nothing)  -- | The shared pool of a fleet as it is at a tip, for the queries of -- "Haskoin.Store.Database.Query.Pooled". Every read throws v'TipMoved' if
src/Haskoin/Store/Fleet/Pool.hs view
@@ -8,7 +8,7 @@ -- -- The mempool that stores sharing one Redis answer from, as pure decisions: -- the second part of "A fleet that answers as one" in--- @docs\/layered-database.md@, after the fleet tip of "Haskoin.Store.Fleet".+-- @docs\/one-fleet.md@, after the fleet tip of "Haskoin.Store.Fleet". -- "Haskoin.Store.Fleet.Redis" carries it out. -- -- = The problem@@ -273,9 +273,9 @@         let leaving = Set.toList (descendants [s] p)          in (withdraw [s] p, replaced <> maybe [] (\w -> [(t, w) | t <- leaving]) (label o)) --- | What a transaction does at an address, for addresses of type @a@: the--- entries by address of "Haskoin.Store.Database.Mempool", which a store--- works out from the transaction alone and gives to the pool with it.+-- | What a transaction does at an address, for addresses of type @a@, which+-- a store works out from the transaction alone and gives to the pool with+-- it ('Haskoin.Store.Fleet.Member.poolTx'). data Entry a o   = -- | it spends from the address or pays to it     Touches !a
src/Haskoin/Store/Fleet/Redis.hs view
@@ -72,6 +72,7 @@     poolOffer,     poolWithdraw,     fleetAdvance,+    fleetCommit,     fleetRestart,     fleetRestarts,     FleetEvent (..),@@ -80,18 +81,30 @@     poolSpends,     poolBody,     poolRead,++    -- * What the fleet has seen+    fleetTrimSeen,+    fleetSeen,++    -- * Scripts+    Script,+    script,+    evalScript,   ) where +import Control.Monad.IO.Class (liftIO) import Data.ByteString (ByteString) import Data.ByteString qualified as B import Data.ByteString.Char8 qualified as C import Data.FileEmbed (embedFile, makeRelativeToProject)-import Data.Maybe (fromMaybe)+import Data.IORef (IORef, newIORef, readIORef, writeIORef)+import Data.Maybe (fromMaybe, isJust) import Data.Word (Word32, Word64)-import Database.Redis (Connection, Redis, Reply)+import Database.Redis (Connection, Redis, RedisResult, Reply (..)) import Database.Redis qualified as R import Haskoin.Store.Fleet (Time)+import System.IO.Unsafe (unsafePerformIO)  -- | The keys of a fleet in Redis, and the size of its blocks in bytes. data FleetKeys = FleetKeys@@ -137,7 +150,7 @@ -- tip and another store is live ('Haskoin.Store.Fleet.mayReport'). fleetReport :: Connection -> FleetKeys -> Maybe Time -> Time -> ByteString -> [ByteString] -> [ByteString] -> IO Bool fleetReport conn keys clock term store blocks views =-  (== 1) <$> run conn (R.eval reportScript (scriptKeys keys) args :: Redis (Either Reply Integer))+  (== 1) <$> run conn (evalScript reportScript (scriptKeys keys) args :: Redis (Either Reply Integer))   where     args = [store, num term, maybe "" num clock, num keys.size, B.concat blocks, B.concat views] @@ -145,7 +158,7 @@ -- the tip that the caller saw. fleetMove :: Connection -> FleetKeys -> Maybe Time -> Maybe ByteString -> IO Moved fleetMove conn keys clock seen =-  run conn (R.eval moveScript (scriptKeys keys) args) >>= \case+  run conn (evalScript moveScript (scriptKeys keys) args) >>= \case     ["moved", block] -> return (MovedTo block)     ["stays"] -> return Stays     ["raced"] -> return Raced@@ -247,7 +260,7 @@   [ByteString] ->   IO Offered poolOffer conn keys tsize seen tx unspents replaced =-  run conn (R.eval poolScript [] args) >>= \case+  run conn (evalScript poolScript [] args) >>= \case     ["taken"] -> return Taken     ["known"] -> return Known     "conflict" : rivals -> return (Conflict rivals)@@ -262,8 +275,37 @@ -- those that left. poolWithdraw :: Connection -> FleetKeys -> Int -> [ByteString] -> IO [ByteString] poolWithdraw conn keys tsize ts =-  run conn (R.eval poolScript [] ["withdraw", keys.prefix, num tsize, B.concat ts])+  run conn (evalScript poolScript [] ["withdraw", keys.prefix, num tsize, B.concat ts]) +-- | Keep the newest of the transactions that the fleet has seen, so+-- many. Gives how many are kept.+--+-- The fleet keeps, in one sorted set, each transaction that the pool has+-- decided on: each that entered it, that an offer found in it or in+-- conflict with it, and that left it unconfirmed, by the time at which it+-- was first seen. No store fetches one of them when a peer announces it+-- ('fleetSeen'). The pool's script adds them, with @ZADD NX@, in the step+-- that decides, so a transaction that a store received and has not offered+-- is not in the set: if the store stops first, another fetches it. In+-- Redis, the set is the same for every store and outlives a store that+-- restarts; in the memory of each store, as it was before, every store+-- fetched each transaction once, and again after a restart. The set is+-- trimmed here, when it has grown a hundredth past its bound, and not on+-- every addition.+fleetTrimSeen :: Connection -> FleetKeys -> Int -> IO Integer+fleetTrimSeen conn keys most =+  run conn (evalScript poolScript [] ["trim", keys.prefix, "0", num most])++-- | Of these transactions, which the fleet has seen ('fleetTrimSeen'), in one+-- round trip.+fleetSeen :: Connection -> FleetKeys -> [ByteString] -> IO [Bool]+fleetSeen _ _ [] = return []+fleetSeen conn keys ts =+  map isJust <$> run conn scores+  where+    scores :: Redis (Either Reply [Maybe ByteString])+    scores = R.sendRequest ("ZMSCORE" : keys.prefix <> "fleet:seen" : ts)+ -- | Move the tip to a block and bring the pool there, in one step: given -- the tip that the caller saw, the block, which must be the one that the -- live stores agree on, and what the blocks in between did.@@ -288,9 +330,33 @@   Maybe PoolChange ->   [ByteString] ->   IO Moved-fleetAdvance conn keys tsize clock seen target known =-  advance conn keys tsize clock seen target (maybe "1" (const "0") known) (fromMaybe unchanged known)+fleetAdvance conn keys tsize clock seen target known path =+  fleetCommit conn keys tsize clock seen target known path ""++-- | 'fleetAdvance' as the commit of a move of the tip with the cache of+-- extended public keys: given also the token of the move of the cache that+-- the store began ('Haskoin.Store.Fleet.XPub.cacheBegin'), whose pieces+-- marked the entries that it wrote. The cache is never emptied here: a+-- store that cannot say what the abandoned blocks held does not move the+-- tip. The pool, the tip and the cache change together: every marked entry gets+-- a new version and the cache is at the new tip, in the same script. The+-- answer is 'Raced' if another move of the cache is under way, or if the+-- store's was aborted.+fleetCommit ::+  Connection ->+  FleetKeys ->+  Int ->+  Maybe Time ->+  Maybe ByteString ->+  ByteString ->+  Maybe PoolChange ->+  [ByteString] ->+  ByteString ->+  IO Moved+fleetCommit conn keys tsize clock seen target known path token =+  advance conn keys tsize clock seen target how (fromMaybe unchanged known) (path, token)   where+    how = maybe "1" (const "0") known     unchanged = PoolChange [] [] [] [] Nothing []  -- | Move the tip to a block and empty the pool, for a store that has both@@ -306,7 +372,7 @@ -- count of 'fleetRestarts' went up. fleetRestart :: Connection -> FleetKeys -> Int -> Maybe Time -> Maybe ByteString -> ByteString -> IO Moved fleetRestart conn keys tsize clock seen target =-  advance conn keys tsize clock seen target "2" (PoolChange [] [] [] [] Nothing []) []+  advance conn keys tsize clock seen target "2" (PoolChange [] [] [] [] Nothing []) ([], "")  -- | How many times the pool was emptied to move the tip. A store that sees -- another number than at its last turn offers all of its mempool again:@@ -326,10 +392,10 @@   ByteString ->   ByteString ->   PoolChange ->-  [ByteString] ->+  ([ByteString], ByteString) ->   IO Moved-advance conn keys tsize clock seen target how change path =-  run conn (R.eval poolScript [] args) >>= \case+advance conn keys tsize clock seen target how change (path, token) =+  run conn (evalScript poolScript [] args) >>= \case     ["moved"] -> return (MovedTo target)     ["stays"] -> return Stays     ["raced"] -> return Raced@@ -341,14 +407,14 @@         <> [B.concat change.gone, B.concat change.confirmed, B.concat change.spent, B.concat change.lost]         <> [num (length change.returned)]         <> concatMap txArgs change.returned-        <> [B.concat path]+        <> [B.concat path, token]  -- | Read from the pool as it is at a tip: given the size of a transaction -- id, the tip that the caller answers from, what to read and its arguments, -- as @read.lua@ takes them. 'Nothing' if the tip is another one by now. poolRead :: Connection -> FleetKeys -> Int -> Maybe ByteString -> ByteString -> [ByteString] -> IO (Maybe [ByteString]) poolRead conn keys tsize tip what args =-  run conn (R.eval readScript [] (what : keys.prefix : num tsize : maybe "" id tip : args)) >>= \case+  run conn (evalScript readScript [] (what : keys.prefix : num tsize : maybe "" id tip : args)) >>= \case     "ok" : rows -> return (Just rows)     ["raced"] -> return Nothing     other -> fail ("Unexpected answer of the read script: " <> show (take 1 other))@@ -431,21 +497,63 @@     Right x -> return x     Left e -> fail ("Redis error: " <> show e) +-- | A Lua script that Redis keeps compiled under its digest.+--+-- @EVAL@ sends the whole script with every call, and Redis hashes it every+-- time to find it compiled: for the scripts of the pool, several kilobytes+-- for a read of a few bytes. That was most of what Redis did for a simple+-- request, and Redis has one thread for the whole fleet. So a script is+-- loaded once, its digest kept here, and called by it with @EVALSHA@. A+-- Redis that lost its scripts, as one that restarted, says @NOSCRIPT@, and+-- the script is loaded again. The digest is the same on every Redis, so+-- one is kept for the program, not for each connection.+data Script = Script+  { source :: !ByteString,+    digest :: !(IORef (Maybe ByteString))+  }++-- | A script of this source. Only for a top-level binding marked NOINLINE,+-- so that there is one of each script.+script :: ByteString -> Script+script s = unsafePerformIO (Script s <$> newIORef Nothing)+{-# NOINLINE script #-}++-- | Run a script, as 'R.eval' does, by its digest.+evalScript :: (RedisResult a) => Script -> [ByteString] -> [ByteString] -> Redis (Either Reply a)+evalScript sc keys args =+  liftIO (readIORef sc.digest) >>= \case+    Nothing -> load+    Just sha ->+      R.evalsha sha keys args >>= \case+        Left (Error e) | "NOSCRIPT" `B.isPrefixOf` e -> load+        r -> return r+  where+    load =+      R.scriptLoad sc.source >>= \case+        Left e -> return (Left e)+        Right sha -> do+          liftIO (writeIORef sc.digest (Just sha))+          R.evalsha sha keys args+ -- | 'Haskoin.Store.Fleet.mayReport' and 'Haskoin.Store.Fleet.report' in one -- step.-reportScript :: ByteString-reportScript = $(makeRelativeToProject "src/Haskoin/Store/Fleet/lua/report.lua" >>= embedFile)+reportScript :: Script+reportScript = script $(makeRelativeToProject "src/Haskoin/Store/Fleet/lua/report.lua" >>= embedFile)+{-# NOINLINE reportScript #-}  -- | 'Haskoin.Store.Fleet.move'.-moveScript :: ByteString-moveScript = $(makeRelativeToProject "src/Haskoin/Store/Fleet/lua/move.lua" >>= embedFile)+moveScript :: Script+moveScript = script $(makeRelativeToProject "src/Haskoin/Store/Fleet/lua/move.lua" >>= embedFile)+{-# NOINLINE moveScript #-}  -- | What a store reads from the pool, each read with a check that the tip -- is the one that the store answers from.-readScript :: ByteString-readScript = $(makeRelativeToProject "src/Haskoin/Store/Fleet/lua/read.lua" >>= embedFile)+readScript :: Script+readScript = script $(makeRelativeToProject "src/Haskoin/Store/Fleet/lua/read.lua" >>= embedFile)+{-# NOINLINE readScript #-}  -- | 'Haskoin.Store.Fleet.Pool.offer', 'Haskoin.Store.Fleet.Pool.withdraw', -- and 'Haskoin.Store.Fleet.Pool.advance' with the move of the tip.-poolScript :: ByteString-poolScript = $(makeRelativeToProject "src/Haskoin/Store/Fleet/lua/pool.lua" >>= embedFile)+poolScript :: Script+poolScript = script $(makeRelativeToProject "src/Haskoin/Store/Fleet/lua/pool.lua" >>= embedFile)+{-# NOINLINE poolScript #-}
src/Haskoin/Store/Fleet/XPub.hs view
@@ -13,8 +13,8 @@ -- answers as a member of a fleet ("Haskoin.Store.Fleet.Member") keeps in -- the Redis that the fleet shares, so that a request about a wallet is one -- or two reads of Redis instead of a read of the database for each of its--- addresses. It is built on the shared pool, and described in "The cache--- of extended public keys of a fleet" in @docs\/layered-database.md@.+-- addresses. It is built on the shared pool, and described in+-- @docs\/one-fleet.md@ and @docs\/xpub-indexing.md@. -- -- = What an entry is --@@ -49,32 +49,48 @@ -- * __A read answers only where the cache is__, which is a move of the --   fleet tip and the block that it went to, and only while the tip is --   there. A block alone would not do: the tip can leave a block and come---   back to it, as when a block is reverted and mined again, and on the---   way the pool takes that block's transactions out of the entries and---   puts them back as unconfirmed. When the tip moves, the store that moved it---   brings the cache there, with what the blocks in between did to the---   addresses that entries have ('cacheAdvance'). Until it has, every read---   is a miss and the store answers from its database. If it never does, or---   the chain was reorganised, or the pool was emptied, the cache is---   emptied: the next move finds it at a block that it does not start from.+--   back to it, as when a block is reverted and mined again. --+-- * __The tip and the cache move in one commit.__ The store that moves the+--   tip first writes what the blocks in between did at the addresses that+--   entries have ('tipMoves'), in pieces that mark the entries they touch+--   ('cachePieces'); readers wait for a marked entry. Then one script moves+--   the pool, the fleet tip and the cache together, gives each marked entry+--   a new version and clears the marks ('Haskoin.Store.Fleet.Redis.fleetCommit').+--   No request finds the tip without its cache, or an entry half moved.+--+-- * __A reorganisation is undone address by address__ ('tipMoves'), from+--   the views of the database at the old tip and the new one, and never+--   empties the cache. Only a move that empties the pool, or one whose+--   store cannot say what the abandoned blocks held, empties it too.+-- -- * __The addresses of a key depend on which are used__, and the pool --   counts. So every read of balances runs the scan again over what the---   entry has ('window'), and is a miss unless the scan reads exactly the---   addresses of the entry. The other reads say how many addresses the---   caller found, and miss if the entry has another number.------ The alternative that this replaces, "Haskoin.Store.Cache", keeps one--- store's view of each key and repairs it from that store's events. It is--- still what a store that is not in a fleet uses.+--   entry has ('window'). An entry also has its lookahead, the next gap's+--   worth of addresses, indexed at build time ('lookahead'), so that a+--   payment that moves the gap is in the entry at once, through the pool,+--   and the entry still answers. A read misses only when the scan needs an+--   address that the entry lacks; when the lookahead runs low, the indexer+--   is asked to extend it, and reads go on meanwhile. The other reads say+--   how many addresses the entry has, and miss if it has another number. -- -- = What it costs -- -- An entry is built from the database by the store that missed, in the -- background, a few addresses at a time ('cacheBuild'); when a key gains--- addresses only those are read. Each block costs one question to Redis--- about the addresses that it touches, and a write for those that entries--- have.+-- addresses only those are read. Each move of the tip costs one question+-- to Redis about the addresses that its blocks touch, and a write for those+-- that entries have.+--+-- = Which entries stay+--+-- Every key that a request asks about is cached, and the cache evicts by+-- GreedyDual ('Haskoin.Store.XPub.greedyLua'): each+-- entry has what it cost to build, at the prices of the token design, a use+-- gives it the level of the cache plus that cost, and the entry of the+-- lowest priority goes first, the level rising to it ('cacheEvict'). A+-- wallet of 40,000 addresses outlives an empty one used as recently, and+-- either goes once nobody asks for it. module Haskoin.Store.Fleet.XPub   ( -- * Reading     Cached (..),@@ -88,25 +104,31 @@     -- * Writing     Built (..),     cacheBuild,+    lookahead,     cacheDrop,     cacheHave,-    CacheMoved (..),+    cacheEvict,+    cacheSize,++    -- * Moving the tip     AddressChange (..),     blockChanges,     cacheTracked,-    cacheAdvance,-    cacheEvict,-    cacheTip,-    cacheSize,+    Begun (..),+    cacheBegin,+    AddressMove (..),+    tipMoves,+    cachePieces,+    cacheAbort,      -- * Encoding     xpubId,     score,     position,-    nothingPool,   ) where +import Control.Concurrent (threadDelay) import Control.Monad (forM) import Data.Bits (shift, (.&.), (.|.)) import Data.ByteString (ByteString)@@ -129,7 +151,6 @@ import Database.Redis qualified as R import Haskoin   ( Address (..),-    Ctx,     Network,     OutPoint (..),     Tx (..),@@ -143,7 +164,7 @@     sha256,     txHash,   )-import Haskoin.Store.Common (Limits (..), StoreRead (..), scanXPubBals)+import Haskoin.Store.Common (Limits (..), StoreRead (..), nub', scanXPubBals) import Haskoin.Store.Data   ( Balance (..),     BlockRef (..),@@ -157,13 +178,12 @@     nullBalance,     zeroBalance,   )-import Haskoin.Store.Database.Mempool qualified as M import Haskoin.Store.Database.Query qualified as Q-import Haskoin.Store.Database.Query.Layered (From (..))-import Haskoin.Store.Database.Query.Pooled (PoolReads (..), pooledRead)-import Haskoin.Store.Database.Reader (DatabaseReader (..))+import Haskoin.Store.Database.Query.Pooled (From (..), PoolReads (..), applySummary, emptyPool, pooledRead)+import Haskoin.Store.Database.Reader (DatabaseReader (..), Derived (..), derivedAddrs) import Haskoin.Store.Database.Types (MemSummary (..))-import Haskoin.Store.Fleet.Redis (FleetKeys (..))+import Haskoin.Store.Fleet.Redis (FleetKeys (..), Script, evalScript, script)+import Haskoin.Store.XPub (greedyLua, indexCost)  ------------------------------------------------------------------------------- -- Encoding@@ -182,7 +202,7 @@ -- their bytes, which for hashes and outputs is the order of the database -- ('Haskoin.Store.Common.txRefKey', 'Haskoin.Store.Common.unspentKey'). ----- It is the score of "Haskoin.Store.Cache", as a whole number: height and+-- It is a whole number: height and -- position fit in the 53 bits that Redis keeps exactly. score :: BlockRef -> Int64 score BlockRef {height = h, position = p} = 0x001fffffffffffff - (h' .|. p')@@ -199,23 +219,6 @@   where     m = 0x001fffffffffffff - s --- | A pool that has nothing. The queries of--- "Haskoin.Store.Database.Query.Pooled" with it answer from the chain--- alone, which is how the chain's layer of an entry is read.-nothingPool :: PoolReads-nothingPool =-  PoolReads-    { summaries = return . map (const Nothing),-      addressTxs = \_ _ _ -> return [],-      addressOuts = \_ _ _ -> return [],-      spentOutputs = \_ ps -> return (map (const False) ps),-      bodies = return . map (const (Nothing, IntMap.empty)),-      spenders = return . map (const Nothing),-      page = \_ _ -> return [],-      startingWith = \_ -> return [],-      replaced = \_ -> return Nothing-    }- ------------------------------------------------------------------------------- -- Reading -------------------------------------------------------------------------------@@ -225,7 +228,7 @@   = -- | the answer, which is the one that the database gives at the fleet     -- tip with the pool on top     Hit !a-  | -- | the cache cannot answer: ask the database+  | -- | the cache cannot answer: ask the indexer     Miss   | -- | the fleet tip is another block than the one that the request     -- answers from: answer again from the new tip@@ -240,19 +243,25 @@     sizes :: !(Int, Int),     -- | where each address is in the key. It is worked out when it is     -- asked for, which only a list of unspent outputs does.-    paths :: HashMap Address [Word32]+    paths :: HashMap Address [Word32],+    -- | the version of the entry: every change of it stamps a new one+    version :: !ByteString,+    -- | whether the lookahead of the entry is less than half the gap in a+    -- chain: payments used it up, and the indexer is to extend it+    short :: !Bool   }   deriving (Eq, Show)  -- | The balances of a key at the fleet tip, with the pool applied, if the -- cache has the key with exactly the addresses that a scan of it reads -- now. Arguments: the tip that the request answers from, the initial gap--- and the gap of the store, and the time in milliseconds, by which the--- cache knows which keys are in use.-cachedEntry :: Connection -> FleetKeys -> ByteString -> Word32 -> Word32 -> Word64 -> XPubSpec -> IO (Cached Entry)-cachedEntry conn keys tip igap gap now spec =-  ask conn keys "bals" [tip, xpubId spec, num now] <&> \case-    Hit rows -> maybe Miss Hit (entry (mapMaybe row (groups 3 rows)))+-- and the gap of the store, and whether the read is a use of the key,+-- which renews its priority in the cache (GreedyDual).+cachedEntry :: Connection -> FleetKeys -> ByteString -> Word32 -> Word32 -> Bool -> XPubSpec -> IO (Cached Entry)+cachedEntry conn keys tip igap gap used spec =+  ask conn keys "bals" [tip, xpubId spec, if used then "1" else ""] <&> \case+    Hit (ver : rows) -> maybe Miss Hit (entry ver (mapMaybe row (groups 3 rows)))+    Hit [] -> Miss     Miss -> Miss     Moved -> Moved   where@@ -263,40 +272,51 @@       n <- either (const Nothing) Just (decode (B.drop 1 path)) :: Maybe Word32       let stored = if B.null bal then Nothing else either (const Nothing) Just (decode bal)           chain = fromMaybe (zeroBalance address) stored-      return (fromIntegral m, n, maybe chain (M.applySummary chain) (summary sums))+      return (fromIntegral m, n, maybe chain (applySummary chain) (summary sums))     row _ = Nothing     summary sums = case map number (C.words sums) of       [txs, paid, paidCount, spent, spentCount, both, bothCount] -> Just MemSummary {..}       _ -> Nothing-    entry rows = do+    entry version rows = do       balances <- window igap gap rows-      let count m = length [() | XPubBal {path = m' : _} <- balances, m' == m]+      -- the numbers of the entry, its lookahead included, which the other+      -- reads check+      let held m = length [() | (m', _, _) <- rows, m' == m]+          scanned m = length [() | XPubBal {path = m' : _} <- balances, m' == m]+          ahead m = held m - scanned m       return         Entry           { balances,-            sizes = (count 0, count 1),-            paths = HashMap.fromList [(b.balance.address, b.path) | b <- balances]+            sizes = (held 0, held 1),+            paths = HashMap.fromList [(b.balance.address, b.path) | b <- balances],+            version,+            short = any (\m -> ahead m < fromIntegral gap `div` 2) [0, 1]           }  -- | The balances of a key as a scan of it gives them, from the balances of -- the addresses that an entry has, each with the chain that it is in and--- its number there: if the scan, run over them, reads exactly those--- addresses. 'Nothing' if it would read one that the entry does not have,--- or would stop before one that it has.+-- its number there: if the scan, run over them, reads none that the entry+-- lacks, and every address of the entry that it does not read is unused.+-- 'Nothing' otherwise. -- -- This is what makes an entry safe against the gap. The addresses of a key -- are those up to the last one that is used and so many more, and a--- payment in the pool counts as use. An entry is built for the addresses--- of one moment; after a payment to one of its unused addresses the key--- has more, and after such a payment is dropped it has fewer. Either way--- this gives 'Nothing', and the store answers from the database and--- builds the entry again.+-- payment in the pool counts as use. An entry is built with the addresses+-- of one moment and its lookahead, the next gap's worth after them in each+-- chain ('lookahead'), indexed as the others are, so that the pool puts a+-- payment to one of them in the entry at once. After such a payment the+-- key has more addresses, and the entry still has them; after a payment+-- is dropped it has fewer, and the rest are unused. Only a key that needs+-- more than the entry has, or an entry with a used address past what the+-- scan reads, gives 'Nothing', and the entry is built again. window :: Word32 -> Word32 -> [(Word32, Word32, Balance)] -> Maybe [XPubBal] window igap gap rows   | not (contiguous ext && contiguous chg) = Nothing   | otherwise = do       found <- scanXPubBals (traverse (`HashMap.lookup` known)) igap gap (ext <> [beyond]) (chg <> [beyond])-      if length found == HashMap.size known then Just found else Nothing+      let read' = HashMap.fromList [(b.balance.address, ()) | b <- found]+          unread = [b | (_, _, b) <- rows, not (HashMap.member b.address read')]+      if all ((== 0) . (.txs)) unread then Just found else Nothing   where     chain m = sortOn fst [(n, b.address) | (m', n, b) <- rows, m' == m]     ext = chain 0@@ -314,9 +334,21 @@ -- | A page of the transactions of a cached key, newest first, from where -- the page starts, given the numbers of addresses that the caller has for -- the key ('sizes').-cachedTxs :: Connection -> FleetKeys -> ByteString -> XPubSpec -> (Int, Int) -> From -> Limits -> IO (Cached [TxRef])+cachedTxs ::+  Connection ->+  FleetKeys ->+  ByteString ->+  XPubSpec ->+  (Int, Int) ->+  From ->+  Limits ->+  IO (Cached (ByteString, [TxRef])) cachedTxs conn keys tip spec (n0, n1) from limits =-  ask conn keys "txs" (who <> place from encode <> [num limits.offset, num limits.limit]) <&> fmap refs+  ask conn keys "txs" (who <> place from encode <> [num limits.offset, num limits.limit]) <&> \case+    Hit (v : rows) -> Hit (v, refs rows)+    Hit [] -> Miss+    Miss -> Miss+    Moved -> Moved   where     who = [tip, xpubId spec, num n0, num n1]     refs rows =@@ -336,10 +368,11 @@   Entry ->   From ->   Limits ->-  IO (Cached [XPubUnspent])+  IO (Cached (ByteString, [XPubUnspent])) cachedUnspents conn keys tip spec e from limits =   ask conn keys "outs" (who <> place from first <> [num limits.offset, num limits.limit]) <&> \case-    Hit rows -> maybe Miss Hit (mapM output (groups 3 rows))+    Hit (v : rows) -> maybe Miss (Hit . (v,)) (mapM output (groups 3 rows))+    Hit [] -> Miss     Miss -> Miss     Moved -> Moved   where@@ -364,10 +397,10 @@     output _ = Nothing  -- | The number of transactions of a cached key.-cachedCount :: Connection -> FleetKeys -> ByteString -> XPubSpec -> (Int, Int) -> IO (Cached Word32)+cachedCount :: Connection -> FleetKeys -> ByteString -> XPubSpec -> (Int, Int) -> IO (Cached (ByteString, Word32)) cachedCount conn keys tip spec (n0, n1) =   ask conn keys "count" [tip, xpubId spec, num n0, num n1] <&> \case-    Hit [n] -> Hit (number n)+    Hit [v, n] -> Hit (v, number n)     Hit _ -> Miss     Miss -> Miss     Moved -> Moved@@ -382,13 +415,23 @@   InChain ref -> ["score", num (score ref), ""]  -- | Ask the script something that it answers with rows.+--+-- An entry that a move of the tip is writing ('cachePieces') is read when+-- the move is done: the read waits a few milliseconds at a time, for two+-- seconds at most, and then says that the tip moved, which has the request+-- answered again. ask :: Connection -> FleetKeys -> ByteString -> [ByteString] -> IO (Cached [ByteString])-ask conn keys what args =-  run conn (R.eval xpubScript [] (what : keys.prefix : "32" : args)) >>= \case-    "ok" : rows -> return (Hit rows)-    ["miss"] -> return Miss-    ["raced"] -> return Moved-    other -> fail ("Unexpected answer of the xpub script: " <> show (take 1 other))+ask conn keys what args = go (400 :: Int)+  where+    go tries =+      run conn (evalScript xpubScript [] (what : keys.prefix : "32" : args)) >>= \case+        "ok" : rows -> return (Hit rows)+        ["miss"] -> return Miss+        ["raced"] -> return Moved+        ["marked"]+          | tries > 0 -> threadDelay 5000 >> go (tries - 1)+          | otherwise -> return Moved+        other -> fail ("Unexpected answer of the xpub script: " <> show (take 1 other))  ------------------------------------------------------------------------------- -- Writing@@ -402,25 +445,31 @@   | -- | the fleet tip moved, the cache is not at the tip, or another store     -- wrote the entry meanwhile: nothing more was written     Abandoned+  | -- | a move of the tip is under way: the build is to be tried again+    -- when it is done+    Busy   deriving (Eq, Show)  -- | The numbers of addresses that the cache has for a key in each of its -- two chains, both zero if it does not have the key. cacheHave :: Connection -> FleetKeys -> XPubSpec -> IO (Int, Int) cacheHave conn keys spec =-  run conn (R.eval xpubScript [] ["have", keys.prefix, "32", xpubId spec]) >>= \case+  run conn (evalScript xpubScript [] ["have", keys.prefix, "32", xpubId spec]) >>= \case     [n0, n1] -> return (fromInteger n0, fromInteger n1)     other -> fail ("Unexpected answer of the xpub script: " <> show (other :: [Integer]))  -- | Delete the entry of a key. cacheDrop :: Connection -> FleetKeys -> XPubSpec -> IO () cacheDrop conn keys spec = do-  _ <- run conn (R.eval xpubScript [] ["drop", keys.prefix, "32", xpubId spec]) :: IO [ByteString]+  _ <- run conn (evalScript xpubScript [] ["drop", keys.prefix, "32", xpubId spec]) :: IO [ByteString]   return ()  -- | Build the entry of a key, or add to it the addresses that the key has -- gained: given the tip that the fleet is at, the database as it was at--- that block, the pool, and the time in milliseconds.+-- that block, and the pool. A build is for a request that missed, and is a+-- use of the key: its priority in the cache becomes the level plus what the+-- entry cost to read, its addresses, transactions and outputs at the+-- prices of 'Haskoin.Store.XPub.indexCost', added up over its writes. -- -- The addresses of the key are read as a request reads them, from the -- chain with the pool on top. An entry that has some of them keeps what it@@ -432,9 +481,9 @@ -- -- Each write is taken only while the fleet tip, the block of the cache -- and the addresses of the entry are what this store found.-cacheBuild :: Connection -> FleetKeys -> ByteString -> DatabaseReader -> PoolReads -> Word64 -> XPubSpec -> IO Built-cacheBuild conn keys tip r pool now spec = do-  bals <- (pooledRead r pool).xPubBals spec+cacheBuild :: Connection -> FleetKeys -> ByteString -> DatabaseReader -> PoolReads -> XPubSpec -> IO Built+cacheBuild conn keys tip r pool spec = do+  bals <- (pooledRead r pool).xPubBals spec >>= lookahead r pool spec   let wanted m = [b | b@XPubBal {path = m' : _} <- bals, m' == m]       (w0, w1) = (length (wanted 0), length (wanted 1))   (h0, h1) <- cacheHave conn keys spec@@ -445,7 +494,7 @@   let new = drop n0 (wanted 0) <> drop n1 (wanted 1)   go (n0, n1) 0 =<< mapM chainOf new   where-    chain = pooledRead r nothingPool+    chain = pooledRead r emptyPool     chainOf b = do       let a = b.balance.address       bal <- Q.getBalance r a@@ -461,12 +510,19 @@     go _ added [] = return (Built added)     go (n0, n1) added todo = do       let (now', later) = batch chunk todo+          cost =+            indexCost+              (length now')+              (sum [length txs | (_, _, txs, _) <- now'])+              (sum [length outs | (_, _, _, outs) <- now'])+              0           args =-            [tip, xpubId spec, num n0, num n1, num now, num (length now')]+            [tip, xpubId spec, num n0, num n1, num cost, num (length now')]               <> concatMap address now'           count m = length [() | (XPubBal {path = m' : _}, _, _, _) <- now', m' == m]-      run conn (R.eval xpubScript [] ("build" : keys.prefix : "32" : args)) >>= \case+      run conn (evalScript xpubScript [] ("build" : keys.prefix : "32" : args)) >>= \case         ["ok"] -> go (n0 + count 0, n1 + count 1) (added + length now') later+        ["busy"] -> return Busy         [why] | why `elem` ["raced", "behind", "changed"] -> return Abandoned         other -> fail ("Unexpected answer of the xpub script: " <> show (take 1 (other :: [ByteString])))     address (b, bal, txs, outs) =@@ -480,24 +536,29 @@       [m, n] -> runPut (putWord8 (fromIntegral m) >> putWord32be n)       _ -> "" +-- | The addresses of a key as a scan gives them, with its lookahead: in+-- each chain, the next gap's worth of addresses after them, up to the first+-- that is used. Those are indexed with the others, so that a payment in the+-- pool to one of them is in the entry at once, and the entry still answers+-- when the payment moves the gap ('window'). A used address past the gap+-- is not one that the key has, and stops the lookahead.+lookahead :: DatabaseReader -> PoolReads -> XPubSpec -> [XPubBal] -> IO [XPubBal]+lookahead r pool spec bals = do+  Derived ext chg <- derivedAddrs r spec+  more <- forM [(0, ext), (1, chg)] $ \(m, derived) -> do+    let have = length [() | XPubBal {path = m' : _} <- bals, m' == m]+        next = take (fromIntegral r.maxGap) (drop have derived)+    found <- (pooledRead r pool).getBalances (map snd next)+    return+      . takeWhile ((== 0) . (.balance.txs))+      $ zipWith (\(n, _) b -> XPubBal {path = [m, n], balance = b}) next found+  return (bals <> concat more)+ -- | About how many transactions and outputs one write of 'cacheBuild' -- carries. chunk :: Int chunk = 5000 --- | What became of an attempt to bring the cache to a block.-data CacheMoved-  = -- | the cache is at the block now, with what the blocks did-    CacheMoved-  | -- | it was there already-    CacheStays-  | -- | the fleet tip is not that block any more: whoever moves it next-    -- sees to the cache-    CacheRaced-  | -- | the cache was emptied and starts again at the block-    CacheEmptied-  deriving (Eq, Show)- -- | What blocks did at an address: the transactions that they added to -- it, the outputs that they paid to it, and its outputs that they spent. data AddressChange = AddressChange@@ -512,14 +573,14 @@  -- | What the transactions of some blocks, in their order, did at each -- address that they touch. An output that one of them pays and a later one--- spends is in both lists of its address: 'cacheAdvance' takes out what--- was spent after it puts in what was paid.-blockChanges :: Network -> Ctx -> [TxData] -> HashMap Address AddressChange-blockChanges net ctx tds = HashMap.fromListWith (flip (<>)) (concatMap one tds)+-- spends is in both lists of its address ('tipMoves' looks each up at the+-- new tip).+blockChanges :: Network -> [TxData] -> HashMap Address AddressChange+blockChanges net tds = HashMap.fromListWith (flip (<>)) (concatMap one tds)   where     one td =       let h = txHash td.tx-          address = either (const Nothing) Just . scriptToAddressBS net ctx+          address = either (const Nothing) Just . scriptToAddressBS net           ins =             [ (a, AddressChange {txs = [], paid = [], spent = [i.outpoint]})             | (n, i) <- zip [0 ..] td.tx.inputs,@@ -541,51 +602,133 @@   where     asked some =       map (== '1') . C.unpack-        <$> run conn (R.eval xpubScript [] ("tracked" : keys.prefix : "32" : map encode some))+        <$> run conn (evalScript xpubScript [] ("tracked" : keys.prefix : "32" : map encode some))     pieces [] = []     pieces xs = let (now, later) = splitAt 4000 xs in now : pieces later --- | Bring the cache from the block that it is taken to be at to the block--- that the fleet tip has moved to: given what the blocks in between did at--- the addresses that entries have, each with its balance in the chain at--- the new block, or 'Nothing' to empty the cache instead, as a store does--- that cannot say what they did.+-- | What became of an attempt to begin a move of the cache.+data Begun+  = -- | the move is under way, under its token: builds wait for it+    Begun+  | -- | another store is moving the tip+    Moving+  | -- | the fleet tip is not the one that the store saw+    BeginRaced+  deriving (Eq, Show)++-- | Begin a move of the tip with the cache, under a token, from the tip+-- that the store saw: from here until the commit ('fleetCommit' in+-- "Haskoin.Store.Fleet.Redis") or 'cacheAbort', builds wait, so that every+-- entry is either one that the move brings along or one built after it. A+-- move whose store wrote nothing for ten seconds is aborted by whoever+-- comes across it.+cacheBegin :: Connection -> FleetKeys -> ByteString -> ByteString -> IO Begun+cacheBegin conn keys token seen =+  run conn (evalScript xpubScript [] ["begin", keys.prefix, "32", token, seen]) >>= \case+    ["ok"] -> return Begun+    ["busy"] -> return Moving+    ["raced"] -> return BeginRaced+    other -> fail ("Unexpected answer of the xpub script: " <> show (other :: [ByteString]))++-- | End a move of the cache that will not be committed, dropping the+-- entries that its pieces wrote, to be built again.+cacheAbort :: Connection -> FleetKeys -> ByteString -> IO ()+cacheAbort conn keys token = do+  _ <- run conn (evalScript xpubScript [] ["abort", keys.prefix, "32", token]) :: IO [ByteString]+  return ()++-- | What a move of the tip does to the chain's layer of an address that+-- entries have: its balance in the chain at the new tip, the transactions+-- of the abandoned blocks that touch it, those of the new blocks with+-- where they are, and, of the outputs that either touches, those that are+-- not unspent in the chain at the new tip and those that are.+data AddressMove = AddressMove+  { address :: !Address,+    balance :: !(Maybe Balance),+    abandoned :: ![TxHash],+    added :: ![(TxHash, BlockRef)],+    gone :: ![OutPoint],+    unspent :: ![Unspent]+  }+  deriving (Eq, Show)++-- | What a move of the tip does at the addresses that entries have, worked+-- out from the view of the database at the new tip, the transactions of+-- the abandoned blocks as the view at the old tip had them, and those of+-- the new blocks, both in the order of their blocks. ----- The cache is also emptied when it is found at another block than the--- one given: a store moved the tip and did not get to bring the cache--- along, and what the blocks since did to it is not known here.-cacheAdvance ::-  Connection ->-  FleetKeys ->-  Maybe ByteString ->-  ByteString ->-  Maybe [(Address, Maybe Balance, AddressChange)] ->-  IO CacheMoved-cacheAdvance conn keys from to changes =-  run conn (R.eval xpubScript [] ("advance" : keys.prefix : "32" : args)) >>= \case-    ["moved"] -> return CacheMoved-    ["stays"] -> return CacheStays-    ["raced"] -> return CacheRaced-    ["emptied"] -> return CacheEmptied-    other -> fail ("Unexpected answer of the xpub script: " <> show (take 1 (other :: [ByteString])))+-- = Undone address by address+--+-- A reorganisation is undone at each address as it was done: the+-- transactions of the abandoned blocks come out, the new ones go in, and+-- every output that either touches is looked up at the new tip, so that+-- what the abandoned blocks paid goes, what they spent comes back unless+-- the new blocks spend it, and what the new blocks pay and do not spend+-- comes in. The balance is read at the new tip. Nothing else of an entry+-- changes, so no entry has to be emptied or built again.+tipMoves :: Connection -> FleetKeys -> DatabaseReader -> [TxData] -> [TxData] -> IO [AddressMove]+tipMoves conn keys r old new = do+  let undone = blockChanges r.net old+      done = blockChanges r.net new+      touched = HashMap.keys (HashMap.union (() <$ undone) (() <$ done))+  flags <- cacheTracked conn keys touched+  forM [a | (a, True) <- zip touched flags] $ \a -> do+    let none = AddressChange [] [] []+        u = HashMap.lookupDefault none a undone+        d = HashMap.lookupDefault none a done+        outs = nub' ([p | (p, _, _, _) <- u.paid <> d.paid] <> u.spent <> d.spent)+    bal <- Q.getBalance r a+    found <- forM outs $ \p -> (p,) <$> Q.getUnspent r p+    return+      AddressMove+        { address = a,+          balance = bal,+          abandoned = map fst u.txs,+          added = d.txs,+          gone = [p | (p, Nothing) <- found],+          unspent = [x | (_, Just x) <- found]+        }++-- | Write the moves of the addresses into their entries, a few at a time,+-- each piece marking the entries that it writes, which readers then wait+-- for until the commit. 'False' if the move was taken over or the tip+-- moved meanwhile: the caller aborts it.+cachePieces :: Connection -> FleetKeys -> ByteString -> ByteString -> [AddressMove] -> IO Bool+cachePieces conn keys token seen = go   where-    args =-      [fromMaybe "" from, to, maybe "1" (const "0") changes, num (maybe 0 length changes)]-        <> concatMap one (fromMaybe [] changes)-    one (a, bal, c) =-      [encode a, maybe "" encode (bal >>= real), num (length c.txs)]-        <> concat [[encode h, num (score ref)] | (h, ref) <- c.txs]-        <> [num (length c.paid)]-        <> concat [[encode p, num (score ref), num v, s] | (p, ref, v, s) <- c.paid]-        <> [num (length c.spent)]-        <> map encode c.spent+    go [] = return True+    go moves = do+      let (now', later) = batch chunk moves+          args = ["piece", keys.prefix, "32", token, seen, num (length now')] <> concatMap one now'+      run conn (evalScript xpubScript [] args) >>= \case+        ["ok"] -> go later+        ["raced"] -> return False+        other -> fail ("Unexpected answer of the xpub script: " <> show (other :: [ByteString]))+    weight x = 1 + length x.abandoned + length x.added + length x.gone + length x.unspent+    batch _ [] = ([], [])+    batch room (x : xs)+      | room <= 0 = ([], x : xs)+      | otherwise = let (now', later) = batch (room - weight x) xs in (x : now', later)+    one x =+      [encode x.address, maybe "" encode (x.balance >>= real), num (length x.abandoned)]+        <> map encode x.abandoned+        <> [num (length x.added)]+        <> concat [[encode h, num (score ref)] | (h, ref) <- x.added]+        <> [num (length x.gone)]+        <> map encode x.gone+        <> [num (length x.unspent)]+        <> concat [[encode o.outpoint, num (score o.block), num o.value, o.script] | o <- x.unspent]     real b = if nullBalance b then Nothing else Just b --- | Delete the least recently used entries past so many. Says how many it--- deleted.+-- | Delete entries past so many by GreedyDual, those of the lowest priority+-- first, raising the level to each ('Haskoin.Store.XPub.greedyLua'). A use+-- of an entry gave it the level plus its cost, so a large key outlives+-- small ones used as recently, and any key goes once it is not used for+-- long enough. Entries of a version that ordered them by last use are given+-- the level first, so they go first. Says how many it deleted. cacheEvict :: Connection -> FleetKeys -> Int -> IO Int cacheEvict conn keys most =-  fromInteger <$> run conn (R.eval xpubScript [] ["evict", keys.prefix, "32", num most])+  fromInteger <$> run conn (evalScript xpubScript [] ["evict", keys.prefix, "32", num most])  -- | How large the cache is: the number of its entries, one for each -- extended public key, and the number of addresses that they have between@@ -593,17 +736,10 @@ -- transactions of those addresses. cacheSize :: Connection -> FleetKeys -> IO (Int, Int) cacheSize conn keys =-  run conn (R.eval xpubScript [] ["size", keys.prefix, "32"]) >>= \case+  run conn (evalScript xpubScript [] ["size", keys.prefix, "32"]) >>= \case     [entries, addresses] -> return (fromInteger entries, fromInteger addresses)     other -> fail ("Unexpected answer of the xpub script: " <> show (other :: [Integer])) --- | The block that the cache is at, if it is where the fleet tip is: at--- the same block, by the same move of the tip.-cacheTip :: Connection -> FleetKeys -> IO (Maybe ByteString)-cacheTip conn keys =-  run conn (R.eval xpubScript [] ["where", keys.prefix, "32"]) <&> \block ->-    if B.null block then Nothing else Just block- ------------------------------------------------------------------------------- -- Helpers -------------------------------------------------------------------------------@@ -639,5 +775,6 @@     Left e -> fail ("Redis error: " <> show e)  -- | The script of the cache; see its header for what it takes and gives.-xpubScript :: ByteString-xpubScript = $(makeRelativeToProject "src/Haskoin/Store/Fleet/lua/xpub.lua" >>= embedFile)+xpubScript :: Script+xpubScript = script (greedyLua <> $(makeRelativeToProject "src/Haskoin/Store/Fleet/lua/xpub.lua" >>= embedFile))+{-# NOINLINE xpubScript #-}
src/Haskoin/Store/Fleet/lua/pool.lua view
@@ -24,6 +24,9 @@ -- -- 'withdraw': the transactions. Returns those that left. --+-- 'trim': how many of the transactions seen to keep. Returns how many are+-- kept.+-- -- 'advance': the tip that the store saw or '', the block to move to, the -- time in milliseconds or '' for the clock of Redis, the size of a block in -- bytes, '1' to empty the pool instead of changing it or '0', the height of@@ -33,9 +36,18 @@ -- four bytes, big-endian, the transactions of the abandoned blocks that the -- store has as replaced, each followed in the same way, the number of -- transactions that come back, those transactions, and the blocks that the--- tip moves through, oldest first, or ''. Returns {'moved'}, {'stays'} or--- {'raced'}.+-- tip moves through, oldest first, or '', the token of the move of the+-- cache that the store began (xpub.lua) or ''. Returns {'moved'}, {'stays'} or {'raced'}, also when the token is+-- not that of the move of the cache under way. --+-- The advance is the commit of a move of the tip: the pool, the tip and the+-- cache of extended public keys change in it together. The pieces of the+-- move (xpub.lua) wrote the chain's layer of the entries and marked them;+-- here the pool's layer changes with the pool, every marked entry gets a+-- new version, the marks go, and the cache is put at the new place of the+-- tip. The cache is emptied only when the pool is, when the store cannot+-- say what the abandoned blocks held, or when it was not at the tip.+-- -- What the new blocks replaced is kept: each transaction of the pool that -- leaves because it, or an ancestor, spent an output that a transaction of -- the blocks spends, with that transaction and the height of its block.@@ -111,8 +123,12 @@   lostval = prefix .. 'pool:lost',   lostby = prefix .. 'pool:lostby:',   lostat = prefix .. 'pool:lostat',+  seen = prefix .. 'fleet:seen',   of = prefix .. 'fx:of:',-  entry = prefix .. 'fx:e:'+  entry = prefix .. 'fx:e:',+  xtip = prefix .. 'fx:tip',+  moving = prefix .. 'fx:moving',+  marked = prefix .. 'fx:marked' }  local function split(s, n)@@ -133,6 +149,24 @@   redis.call('XADD', key.events, 'MAXLEN', '~', '10000', '*', 'k', kind, 'h', what) end +-- The transactions that the fleet has seen, by the time in milliseconds at+-- which they were first seen: each that entered the pool, that an offer+-- found in the pool or in conflict with it, and that left the pool+-- unconfirmed. No store fetches one of them when a peer announces it. Only+-- what the pool has decided on is seen: a transaction that a store+-- received and has not offered yet is not, so that if the store stops+-- before the offer, another fetches it. Not under 'pool:', so that it+-- outlives a pool that is started again, as the transactions do: peers do+-- not announce them again. 'trim' keeps the newest of them.+local seenAt+local function seen(t)+  if not seenAt then+    local time = redis.call('TIME')+    seenAt = time[1] * 1000 + math.floor(time[2] / 1000)+  end+  redis.call('ZADD', key.seen, 'NX', seenAt, t)+end+ local function member(t)   return redis.call('HEXISTS', key.spends, t) == 1 end@@ -174,6 +208,15 @@   return tx, at end +-- whether a member of a list of an entry is in the pool's layer, with a+-- score below zero: what the pool puts in or takes out of an entry is never+-- what the chain's layer has, which a move of the tip writes ahead of the+-- pool (xpub.lua)+local function pooled(z, m)+  local s = redis.call('ZSCORE', z, m)+  return s and tonumber(s) < 0+end+ local function add(a, field, by)   redis.call('HINCRBY', key.asum .. a, field, string.format('%.0f', by)) end@@ -198,8 +241,8 @@       end       for _, id in ipairs(watching) do         if sign > 0 then-          redis.call('ZADD', key.entry .. id .. ':txs', -time, t)-        else+          redis.call('ZADD', key.entry .. id .. ':txs', 'NX', -time, t)+        elseif pooled(key.entry .. id .. ':txs', t) then           redis.call('ZREM', key.entry .. id .. ':txs', t)         end       end@@ -226,9 +269,10 @@       end       for _, id in ipairs(watching) do         if sign > 0 then-          redis.call('ZADD', key.entry .. id .. ':outs', -time, o)-          redis.call('HSET', key.entry .. id .. ':oval', o, e[4] .. ' ' .. #a .. ' ' .. a .. e[5])-        else+          if redis.call('ZADD', key.entry .. id .. ':outs', 'NX', -time, o) == 1 then+            redis.call('HSET', key.entry .. id .. ':oval', o, e[4] .. ' ' .. #a .. ' ' .. a .. e[5])+          end+        elseif pooled(key.entry .. id .. ':outs', o) then           redis.call('ZREM', key.entry .. id .. ':outs', o)           redis.call('HDEL', key.entry .. id .. ':oval', o)         end@@ -241,6 +285,15 @@       end     end   end+  -- every entry with an address that the transaction touches has a new+  -- version, so that a request that read it before reads it again+  local ver = nil+  for a, _ in pairs(touched) do+    for _, id in ipairs(redis.call('SMEMBERS', key.of .. a)) do+      ver = ver or redis.call('INCR', prefix .. 'fleet:xver')+      redis.call('HSET', key.entry .. id .. ':meta', 'ver', ver)+    end+  end   -- an address that no transaction touches has no summary   if sign < 0 then     for a, _ in pairs(touched) do@@ -288,6 +341,7 @@   end   if not quiet then     emit('d', t)+    seen(t)   end   local kept = cmsgpack.unpack(redis.call('HGET', key.entries, t))   account(t, kept[1], kept[2], -1)@@ -325,6 +379,7 @@   end   replaceable(t)   emit('t', t)+  seen(t) end  local function withdraw(from)@@ -339,6 +394,17 @@   return withdraw(split(ARGV[4], tsize)) end +-- trimmed only when the set has grown a hundredth past its bound, so that+-- most calls cost a ZCARD+if what == 'trim' then+  local most, n = tonumber(ARGV[4]), redis.call('ZCARD', key.seen)+  if n > most + math.max(1, math.floor(most / 100)) then+    redis.call('ZREMRANGEBYRANK', key.seen, 0, n - most - 1)+    n = most+  end+  return n+end+ -- forget that these transactions were replaced local function unreplaced(ts)   for _, t in ipairs(ts) do@@ -373,6 +439,7 @@   local tx = read(7)   local t, inputs = tx.id, split(tx.inputs, osize)   if member(t) then+    seen(t)     return {'known'}   end   -- what it replaces leaves first, but only if it is taken@@ -386,6 +453,7 @@     end   end   if #rivals > 0 then+    seen(t)     table.insert(rivals, 1, 'conflict')     return rivals   end@@ -459,6 +527,34 @@   return {'raced'} end +-- the cache: the place of the tip before this move and after it, as+-- xpub.lua has them, and the move of the cache that the store began+local moves = tonumber(redis.call('GET', key.moves) or '0')+local before, after = moves .. '@' .. tip, (moves + 1) .. '@' .. target+local function token(at)+  return ARGV[at + 1] or ''+end+local function settle(wipe)+  local at = redis.call('GET', key.xtip)+  if wipe or (at and at ~= before) then+    for _, k in ipairs(redis.call('KEYS', prefix .. 'fx:*')) do+      redis.call('DEL', k)+    end+  else+    local ids = redis.call('SMEMBERS', key.marked)+    if #ids > 0 then+      local ver = redis.call('INCR', prefix .. 'fleet:xver')+      for _, id in ipairs(ids) do+        if redis.call('EXISTS', key.entry .. id .. ':meta') == 1 then+          redis.call('HSET', key.entry .. id .. ':meta', 'ver', ver)+        end+      end+    end+    redis.call('DEL', key.marked, key.moving)+  end+  redis.call('SET', key.xtip, after)+end+ -- a store that does not know the way from the old tip starts the pool -- again: if it is alone, or if the way is too long for any store if ARGV[8] == '1' or ARGV[8] == '2' then@@ -469,16 +565,31 @@   for _, k in ipairs(redis.call('KEYS', prefix .. 'pool:*')) do     redis.call('DEL', k)   end+  -- the entries have the pool's layer of the pool that is gone+  settle(true)   redis.call('SET', key.tip, target)   redis.call('INCR', key.moves)   emit('b', target)   return {'moved'} end +-- the move of the cache must be the one that the store began, and nobody+-- aborted it; with none begun, none may be under way+local at = 15+for _ = 1, tonumber(ARGV[14]) do+  local _, next = read(at)+  at = next+end+local mine = token(at)+local current = redis.call('GET', key.moving)+if (current and string.sub(current, 1, #mine + 1) ~= mine .. ' ') or (not current and mine ~= '') then+  return {'raced'}+end+ -- the abandoned blocks: what can come back does, and what spends the rest -- leaves. What a transaction that comes back replaced is not replaced any -- more.-local at = 15+at = 15 for _ = 1, tonumber(ARGV[14]) do   local tx   tx, at = read(at)@@ -551,6 +662,7 @@   end end +settle(false) redis.call('SET', key.tip, target) redis.call('INCR', key.moves) -- the blocks that the tip moved through, oldest first, if the store that
src/Haskoin/Store/Fleet/lua/xpub.lua view
@@ -1,6 +1,7 @@ -- The cache of extended public keys of a fleet: xpubScript in--- Haskoin.Store.Fleet.XPub, which says what an entry is and why. pool.lua--- keeps the part of each entry that comes from the shared mempool.+-- Haskoin.Store.Fleet.XPub, which says what an entry is and why, after+-- greedy.lua. pool.lua keeps the part of each entry that comes from the+-- shared mempool. -- -- ARGV[1] is what to do, ARGV[2] the prefix of the keys, and ARGV[3] the -- size of a transaction id in bytes. An output is the id of the transaction@@ -14,14 +15,19 @@ -- key, and the number of its addresses in each of its two chains as the -- caller has them. They answer {'raced'} if the fleet tip is another block, -- {'miss'} if the cache is not where the tip is, if it does not have the key,--- or if it has it with other addresses, and {'ok', ...} otherwise.+-- or if it has it with other addresses, {'marked'} while a move of the tip+-- is writing the entry in pieces ('piece'), and {'ok', version, ...}+-- otherwise. A reader that gets 'marked' waits a moment and asks again: it+-- never sees an entry half moved. ----- 'bals': the tip, the id, and the time in milliseconds, or '' not to note--- that the key was used. The numbers of addresses are not given, and not+-- 'bals': the tip, the id, and '1' to note that the key was used, or ''+-- not to. The numbers of addresses are not given, and not -- checked. For each address of the entry: the address, its path followed -- by its balance in the chain if it has one, and its seven sums in the -- pool separated by spaces or ''. A path is five bytes: the chain of the--- key that the address is in, and its number there.+-- key that the address is in, and its number there. The entry may have+-- more addresses than a scan of the key reads: those after them are its+-- lookahead, which Haskoin.Store.Fleet.XPub.window accepts when unused. -- -- 'txs', 'outs': the tip, the id, the two numbers, where to start, an -- offset and how many, 0 for all. Where to start is 'top' and two '';@@ -36,48 +42,73 @@ -- at: {n0, n1}. -- -- 'build': the tip, the id, the two numbers that the entry is to have--- before, the time in milliseconds, the number of addresses to add, and for+-- before, what the addresses added cost to read, in tokens, the number of+-- addresses to add, and for -- each: the address, its path, its balance in the chain or '', the number -- of its transactions in the chain, a transaction and a score for each, the -- number of its unspent outputs in the chain, and an output, a score, an -- amount and a script for each. What the pool has for the address is added -- from the pool. Returns {'ok'}; {'raced'} if the fleet tip is another -- block; {'behind'} if the cache is at another block; {'changed'} if the--- entry does not have those two numbers of addresses.+-- entry does not have those two numbers of addresses; {'busy'} while a+-- move of the tip is under way. -- -- 'drop': the id. Deletes the entry. -- -- 'tracked': addresses. A string with '1' for each address that an entry -- has and '0' for the others. ----- 'where': nothing. The block that the cache is at, if it is at the fleet--- tip as it stands, and '' otherwise.+-- = Moving the tip with the cache ----- 'advance': the block that the cache is taken to be at or '', the block to--- bring it to, '1' to empty it or '0', the number of addresses that the--- blocks in between changed, and for each: the address, its balance in--- the chain at the new block or '', the number of transactions that the--- blocks added to it, a transaction and a score for each, the number of--- outputs that they paid to it, an output, a score, an amount and a script--- for each, the number of its outputs that they spent, and those outputs.--- Returns {'moved'}; {'stays'} if the cache is at the new block already;--- {'raced'} if the fleet tip is not the new block; and {'emptied'} if it--- had entries and was asked to empty them, or was not where the tip was--- one move ago, in which case what happened since cannot be told.+-- A move of the fleet tip changes the chain's layer of the entries in the+-- same commit as the pool and the tip: the last step is the advance of+-- pool.lua, which takes the pool to the new block, stamps a new version on+-- every entry that the move wrote, and puts the cache at the new place of+-- the tip. What comes before it is written in pieces, each a short script,+-- so that a large block does not hold Redis for long: --+-- 'begin': a token of the move and the tip that the mover saw. Marks that+-- a move is under way: builds wait until it is done, so that every entry+-- is either among those that the move brings along or built after it.+-- Returns {'ok'}, {'raced'} if the fleet tip is another block, or {'busy'}+-- if another store is moving it.+--+-- 'piece': the token, the tip that the mover saw, the number of addresses,+-- and for each: the address, its balance in the chain at the new block or+-- '', the number of transactions of the abandoned blocks, those+-- transactions, the number of transactions of the new blocks, a+-- transaction and a score for each, the number of outputs that are no+-- longer unspent in the chain, those outputs, the number of outputs that+-- are unspent in the chain at the new block, and an output, a score, an+-- amount and a script for each. Every entry that has the address is marked+-- and changed: what is taken out is only what has a score of the chain, at+-- or above zero, so that the pool's layer stays as it is. Returns {'ok'} or+-- {'raced'}.+--+-- 'abort': the token. Drops the entries that the pieces of that move+-- marked, and ends the move: for a mover whose advance was refused.+--+-- A move whose mover went quiet for ten seconds is aborted by whoever finds+-- it: its marked entries are dropped, one at a time, to be built again.+-- -- 'size': nothing. The number of entries, and the number of addresses that -- they have between them: {entries, addresses}. ----- 'evict': how many entries to keep. Deletes the least recently used ones--- past that many, and says how many it deleted.+-- 'evict': how many entries to keep. Deletes those of the lowest priority+-- past that many, by GreedyDual, and says how many it deleted. local what, prefix, tsize = ARGV[1], ARGV[2], tonumber(ARGV[3]) local key = {   fleet = prefix .. 'fleet:tip',   tip = prefix .. 'fx:tip',+  -- times of last use, of versions before GreedyDual   lru = prefix .. 'fx:lru',+  prio = prefix .. 'fx:prio',+  level = prefix .. 'fx:level',   addrs = prefix .. 'fx:addrs',   of = prefix .. 'fx:of:',   entry = prefix .. 'fx:e:',+  moving = prefix .. 'fx:moving',+  marked = prefix .. 'fx:marked',   spender = prefix .. 'pool:spender',   atx = prefix .. 'pool:atx:',   aout = prefix .. 'pool:aout:',@@ -112,17 +143,42 @@     redis.call('DECRBY', key.addrs, #had)   end   redis.call('DEL', e .. 'addrs', e .. 'meta', e .. 'txs', e .. 'outs', e .. 'oval')+  redis.call('ZREM', key.prio, id)   redis.call('ZREM', key.lru, id) end -local function empty()-  for _, id in ipairs(redis.call('ZRANGE', key.lru, 0, -1)) do+-- the number of entries+local function entries()+  return redis.call('ZCARD', key.prio) + redis.call('ZCARD', key.lru)+end++local function now()+  local t = redis.call('TIME')+  return t[1] * 1000 + math.floor(t[2] / 1000)+end++-- end a move of the tip that did not finish: what its pieces wrote is+-- dropped, entry by entry+local function abort()+  for _, id in ipairs(redis.call('SMEMBERS', key.marked)) do     drop(id)   end-  -- whatever a script that failed half way left behind-  for _, k in ipairs(redis.call('KEYS', prefix .. 'fx:*')) do-    redis.call('DEL', k)+  redis.call('DEL', key.marked, key.moving)+end++-- the token of the move of the tip under way, if there is one; a move that+-- wrote nothing for ten seconds is aborted+local function moving()+  local v = redis.call('GET', key.moving)+  if not v then+    return nil   end+  local space = string.find(v, ' ', 1, true)+  if now() - tonumber(string.sub(v, space + 1)) > 10000 then+    abort()+    return nil+  end+  return string.sub(v, 1, space - 1) end  local function numbers(e)@@ -143,16 +199,10 @@   end   return table.concat(flags) elseif what == 'size' then-  return {redis.call('ZCARD', key.lru), tonumber(redis.call('GET', key.addrs) or '0')}+  return {entries(), tonumber(redis.call('GET', key.addrs) or '0')} elseif what == 'evict' then-  local over = redis.call('ZCARD', key.lru) - tonumber(ARGV[4])-  if over <= 0 then-    return 0-  end-  for _, id in ipairs(redis.call('ZRANGE', key.lru, 0, over - 1)) do-    drop(id)-  end-  return over+  greedy_migrate(key.lru, key.prio, key.level)+  return greedy_evict(key.prio, key.level, tonumber(ARGV[4]), drop) end  local fleet = redis.call('GET', key.fleet) or ''@@ -168,74 +218,89 @@ end local here = place(moves, fleet) -if what == 'where' then-  if (redis.call('GET', key.tip) or '') == here then-    return fleet+if what == 'abort' then+  if moving() == ARGV[4] then+    abort()   end-  return ''-end--if what == 'advance' then-  local from, to = ARGV[4], ARGV[5]-  if fleet ~= to then+  return {'ok'}+elseif what == 'begin' then+  if fleet ~= ARGV[5] then     return {'raced'}   end-  local at = redis.call('GET', key.tip) or ''-  if at == here then-    return {'stays'}+  local other = moving()+  if other and other ~= ARGV[4] then+    return {'busy'}   end-  -- it can be brought along only from the move before this one-  local held = redis.call('ZCARD', key.lru) > 0-  if held and (ARGV[6] == '1' or at ~= place(moves - 1, from)) then-    empty()-    redis.call('SET', key.tip, here)-    return {'emptied'}+  redis.call('SET', key.moving, ARGV[4] .. ' ' .. now())+  return {'ok'}+elseif what == 'piece' then+  if fleet ~= ARGV[5] or moving() ~= ARGV[4] then+    return {'raced'}   end-  local changed = tonumber(ARGV[7])-  local i = 8-  for _ = 1, changed do+  redis.call('SET', key.moving, ARGV[4] .. ' ' .. now())+  local ver = redis.call('INCR', prefix .. 'fleet:xver')+  -- take out what has a score of the chain, and leave the pool's layer+  local function chained(z, m)+    local s = redis.call('ZSCORE', z, m)+    return s and tonumber(s) >= 0+  end+  local i = 7+  for _ = 1, tonumber(ARGV[6]) do     local a, bal = ARGV[i], ARGV[i + 1]-    local ids = redis.call('SMEMBERS', key.of .. a)-    local txs, outs, spent = {}, {}, {}+    local old, new, gone, unspent = {}, {}, {}, {}     local n = tonumber(ARGV[i + 2])     i = i + 3     for _ = 1, n do-      txs[#txs + 1] = {ARGV[i], ARGV[i + 1]}-      i = i + 2+      old[#old + 1] = ARGV[i]+      i = i + 1     end     n = tonumber(ARGV[i])     i = i + 1     for _ = 1, n do-      outs[#outs + 1] = {ARGV[i], ARGV[i + 1], kept(a, ARGV[i + 2], ARGV[i + 3])}-      i = i + 4+      new[#new + 1] = {ARGV[i], ARGV[i + 1]}+      i = i + 2     end     n = tonumber(ARGV[i])     i = i + 1     for _ = 1, n do-      spent[#spent + 1] = ARGV[i]+      gone[#gone + 1] = ARGV[i]       i = i + 1     end-    for _, id in ipairs(ids) do+    n = tonumber(ARGV[i])+    i = i + 1+    for _ = 1, n do+      unspent[#unspent + 1] = {ARGV[i], ARGV[i + 1], kept(a, ARGV[i + 2], ARGV[i + 3])}+      i = i + 4+    end+    for _, id in ipairs(redis.call('SMEMBERS', key.of .. a)) do       local e = key.entry .. id .. ':'+      redis.call('SADD', key.marked, id)+      redis.call('HSET', e .. 'meta', 'ver', ver)       local row = redis.call('HGET', e .. 'addrs', a)       if row then         redis.call('HSET', e .. 'addrs', a, string.sub(row, 1, 5) .. bal)       end-      for _, t in ipairs(txs) do+      for _, t in ipairs(old) do+        if chained(e .. 'txs', t) then+          redis.call('ZREM', e .. 'txs', t)+        end+      end+      for _, t in ipairs(new) do         redis.call('ZADD', e .. 'txs', t[2], t[1])       end-      for _, o in ipairs(outs) do+      for _, o in ipairs(gone) do+        if chained(e .. 'outs', o) then+          redis.call('ZREM', e .. 'outs', o)+          redis.call('HDEL', e .. 'oval', o)+        end+      end+      for _, o in ipairs(unspent) do         redis.call('ZADD', e .. 'outs', o[2], o[1])         redis.call('HSET', e .. 'oval', o[1], o[3])       end-      for _, o in ipairs(spent) do-        redis.call('ZREM', e .. 'outs', o)-        redis.call('HDEL', e .. 'oval', o)-      end     end   end-  redis.call('SET', key.tip, here)-  return {'moved'}+  return {'ok'} end  -- everything else is for a store that answers from the fleet tip@@ -246,6 +311,9 @@ end  if what == 'build' then+  if moving() then+    return {'busy'}+  end   local at = redis.call('GET', key.tip)   if not at then     -- nothing is cached yet: the cache starts here@@ -257,7 +325,8 @@   if n0 ~= tonumber(ARGV[6]) or n1 ~= tonumber(ARGV[7]) then     return {'changed'}   end-  redis.call('ZADD', key.lru, ARGV[8], id)+  -- a build is for a request that missed: a use, at the cost so far+  greedy_use(key.prio, key.level, id, redis.call('HINCRBY', e .. 'meta', 'cost', ARGV[8]), false)   local i = 10   for _ = 1, tonumber(ARGV[9]) do     local a, path, bal = ARGV[i], ARGV[i + 1], ARGV[i + 2]@@ -295,7 +364,7 @@       redis.call('HSET', e .. 'oval', o, kept(a, string.sub(value, 1, space - 1), string.sub(value, space + 1)))     end   end-  redis.call('HSET', e .. 'meta', 'n0', n0, 'n1', n1)+  redis.call('HSET', e .. 'meta', 'n0', n0, 'n1', n1, 'ver', redis.call('INCR', prefix .. 'fleet:xver'))   redis.call('INCRBY', key.addrs, tonumber(ARGV[9]))   return {'ok'} end@@ -303,7 +372,15 @@ if (redis.call('GET', key.tip) or '') ~= here then   return {'miss'} end-local answer = {'ok'}+-- an entry that a move of the tip is writing is read when it is done+if redis.call('SISMEMBER', key.marked, id) == 1 then+  if moving() then+    return {'marked'}+  end+  return {'miss'}+end+-- every read gives the version of the entry, which each change stamps+local answer = {'ok', redis.call('HGET', e .. 'meta', 'ver') or '0'}  if what == 'bals' then   local addrs = redis.call('HGETALL', e .. 'addrs')@@ -311,7 +388,7 @@     return {'miss'}   end   if ARGV[6] ~= '' then-    redis.call('ZADD', key.lru, ARGV[6], id)+    greedy_use(key.prio, key.level, id, redis.call('HGET', e .. 'meta', 'cost'), false)   end   -- the pool has a summary for an address when one of its transactions   -- touches it, and those transactions are in the entry, with scores@@ -344,7 +421,8 @@ end  if what == 'count' then-  return {'ok', tostring(redis.call('ZCARD', e .. 'txs'))}+  answer[#answer + 1] = tostring(redis.call('ZCARD', e .. 'txs'))+  return answer end  -- 'txs' and 'outs': the place in the list where the page starts
src/Haskoin/Store/Logic.hs view
@@ -10,1005 +10,647 @@ -- Stability   : experimental -- Portability : POSIX ----- Rules to import and revert blocks and transactions in the layout of--- version 20 of the database, in which the mempool is written into the--- indices of the chain. The store no longer imports with them: it runs on--- the layered layout, with the rules of "Haskoin.Store.Logic.Layered".--- These are kept as the reference: the tests give the same random histories--- to both and compare every result and every answer, and that comparison is--- the evidence that the new layout does what the old one did.------ These functions keep every index consistent: transactions, unspent outputs,--- spenders, address transactions, address unspent outputs, balances and the--- mempool. Each takes the 'Writer' of a batch as its first argument and runs--- in 'IO'; the writer buffers all changes and 'runWriter' commits them in one--- atomic batch, so a failure leaves the database untouched.------ When a block is imported, transactions already in the mempool are confirmed,--- new ones are imported, and mempool transactions that conflict with the block--- are deleted together with their descendants. When a block is reverted, its--- transactions go back to the mempool and its coinbase is deleted.------ On networks with replace-by-fee, a new mempool transaction may replace the--- unconfirmed transactions it conflicts with. Elsewhere a conflicting mempool--- transaction is rejected as a double spend.--- = Why these rules------ The indices are only useful if they always agree with each other: a balance--- must equal the sum of the unspent outputs of its address, a spent output must--- name its spender, and a transaction must appear in the index of every address--- it touches. Every function here changes all of them together, inside one--- atomic batch, so the database moves from one consistent state to the next.------ The store does not validate scripts or consensus rules. It follows the chain--- with the most work, as reported by haskoin-node, and checks only what it needs--- to keep the indices consistent: that inputs exist and are unspent, and that a--- transaction does not spend more than it has. A block that fails these checks--- is not imported, and the block store disconnects the peer that sent it.------ The mempool is treated as provisional. A confirmed block always wins: mempool--- transactions that conflict with it are deleted together with everything that--- spends them. Between mempool transactions, replace-by-fee decides on networks--- that support it; elsewhere the first one seen stays.------ These rules write unconfirmed transactions into the same indices as the--- chain. "Haskoin.Store.Logic.Layered" has the rules for a layout that keeps--- them apart, with the same functions; the store will move to it when that--- layout replaces this one.-module Haskoin.Store.Logic-  ( ImportException (..),-    initBest,-    revertBlock,-    revertBlockAt,-    importBlock,-    newMempoolTx,-    deleteUnconfirmedTx,-  )-where--import Control.Monad-  ( forM,-    forM_,-    guard,-    unless,-    void,-    when,-    zipWithM_,-  )-import Data.ByteString qualified as B-import Data.Either (rights)-import Data.HashMap.Strict qualified as M-import Data.HashSet qualified as H-import Data.IntMap.Strict qualified as I-import Data.List (nub)-import Data.Maybe-  ( catMaybes,-    fromMaybe,-    isJust,-    isNothing,-  )-import Data.Serialize (encode)-import Data.String.Conversions (cs)-import Data.Text (Text)-import Data.Word (Word32, Word64)-import Haskoin-  ( Address,-    Block (..),-    BlockHash,-    BlockHeader (..),-    BlockNode (..),-    Ctx,-    Network (..),-    OutPoint (..),-    Tx (..),-    TxHash,-    TxIn (..),-    TxOut (..),-    blockHashToHex,-    computeSubsidy,-    eitherToMaybe,-    genesisBlock,-    genesisNode,-    headerHash,-    isGenesis,-    nullOutPoint,-    scriptToAddressBS,-    txHash,-    txHashToHex,-  )-import Haskoin.Node (LogLevel (..), logAt)-import Haskoin.Store.Common (nub', sortTxs)-import Haskoin.Store.Data-import Haskoin.Store.Database.Writer hiding (deleteSpender, insertSpender)-import UnliftIO (Exception, throwIO)---- | Log a message with the logger of the writer.-logDebug, logError :: Writer -> Text -> IO ()-logDebug wr = logAt LevelDebug (writerLogger wr)-logError wr = logAt LevelError (writerLogger wr)---- | Network of the database being written.-getNetwork :: Writer -> IO Network-getNetwork wr = return (writerNet wr)---- | Secp256k1 context of the database being written.-getCtx :: Writer -> IO Ctx-getCtx wr = return (writerCtx wr)---- | Data of a transaction that is in the database and not deleted.-getActiveTxData :: Writer -> TxHash -> IO (Maybe TxData)-getActiveTxData wr th =-  getTxData wr th >>= \case-    Just td | not td.deleted -> return (Just td)-    _ -> return Nothing---- | Balance of an address, or a zero balance if it was never used.-getDefaultBalance :: Writer -> Address -> IO Balance-getDefaultBalance wr a = fromMaybe (zeroBalance a) <$> getBalance wr a---- | Reasons why an import can fail.-data ImportException-  = -- | the block does not build on the best block-    PrevBlockNotBest-  | -- | the transaction spends an output that is unknown or already spent-    Orphan-  | -- | a coinbase transaction cannot go into the mempool-    UnexpectedCoinbase-  | -- | the database has no best block, or it cannot be read-    BestBlockNotFound-  | -- | only the best block can be reverted-    BlockNotBest-  | -- | a transaction that should be in the database is not-    TxNotFound-  | -- | the transaction conflicts with another and the network does not have-    -- replace-by-fee-    DoubleSpend-  | -- | a confirmed transaction cannot be deleted or replaced-    TxConfirmed-  | -- | the outputs of the transaction are worth more than its inputs-    InsufficientFunds-  | -- | the transaction spends the same output twice-    DuplicatePrevOutput-  | -- | a transaction cannot be deleted while another one spends it-    TxSpent-  deriving (Eq, Ord, Exception)--instance Show ImportException where-  show PrevBlockNotBest = "Previous block not best"-  show Orphan = "Orphan"-  show UnexpectedCoinbase = "Unexpected coinbase"-  show BestBlockNotFound = "Best block not found"-  show BlockNotBest = "Block not best"-  show TxNotFound = "Transaction not found"-  show DoubleSpend = "Double spend"-  show TxConfirmed = "Transaction confirmed"-  show InsufficientFunds = "Insufficient funds"-  show DuplicatePrevOutput = "Duplicate previous output"-  show TxSpent = "Transaction is spent"---- | Import the genesis block if the database has no best block yet.-initBest :: Writer -> IO ()-initBest wr = do-  logDebug wr "Initializing best block"-  net <- getNetwork wr-  m <- getBestBlock wr-  when (isNothing m) . void $ do-    logDebug wr "Importing Genesis block"-    importBlock wr (genesisBlock net) (genesisNode net)---- | Import a transaction into the mempool with the time it arrived. Returns--- 'False' if it is already known.------ Conflicting unconfirmed transactions are deleted together with their--- descendants if the network has replace-by-fee; otherwise the transaction is--- rejected with 'DoubleSpend'. Throws an 'ImportException' if the transaction is--- invalid, for example if it spends unknown outputs.-newMempoolTx :: Writer -> Tx -> UnixTime -> IO Bool-newMempoolTx wr tx w =-  getActiveTxData wr (txHash tx) >>= \case-    Just _ ->-      return False-    Nothing -> do-      -- Check everything before writing, so that a rejected transaction-      -- leaves the mempool as it was.-      replaced <- getConflicts wr True True tx-      checkNewTx wr (map txHash replaced) tx-      mapM_ (deleteSingleTx wr . txHash) replaced-      rbf <- isRBF wr (MemRef w) tx-      _ <- importTx wr (MemRef w) w rbf tx-      return True---- | Data of the best block. Throws 'BestBlockNotFound' if there is none.-bestBlockData :: Writer -> IO BlockData-bestBlockData wr = do-  h <--    getBestBlock wr >>= \case-      Nothing -> do-        logError wr "Best block unknown"-        throwIO BestBlockNotFound-      Just h -> return h-  getBlock wr h >>= \case-    Nothing -> do-      logError wr "Best block not found"-      throwIO BestBlockNotFound-    Just b -> return b---- | Revert the best block: make its parent the best block, mark it as not in the--- main chain, move its transactions back to the mempool, and delete its--- coinbase. Throws 'BlockNotBest' if the block is not the best block.-revertBlock :: Writer -> BlockHash -> IO ()-revertBlock wr = revertWith wr id---- | 'revertBlock', with the transactions put back at a time, the time of--- the revert, as the layered layout puts them back--- ("Haskoin.Store.Logic.Layered"): the tests compare the two.-revertBlockAt :: Writer -> UnixTime -> BlockHash -> IO ()-revertBlockAt wr now = revertWith wr (\td -> (td :: TxData) {timestamp = now})---- | Revert the best block, changing each transaction that goes back to the--- mempool first.-revertWith :: Writer -> (TxData -> TxData) -> BlockHash -> IO ()-revertWith wr restamp bh = do-  bd <--    bestBlockData wr >>= \b ->-      if headerHash b.header == bh-        then return b-        else do-          logError wr $-            "Cannot revert non-head block: " <> blockHashToHex bh-          throwIO BlockNotBest-  logDebug wr $-    "Obtained block data for " <> blockHashToHex bh-  tds <- mapM (getImportTxData wr) bd.txs-  logDebug wr $-    "Obtained import tx data for block " <> blockHashToHex bh-  setBest wr bd.header.prev-  logDebug wr $-    "Set parent as best block "-      <> blockHashToHex bd.header.prev-  insertBlock wr bd {main = False}-  logDebug wr $-    "Updated as not in main chain: " <> blockHashToHex bh-  forM_ (drop 1 tds) (unConfirmTx wr . restamp)-  logDebug wr $-    "Unconfirmed " <> cs (show (length tds)) <> " transactions"-  deleteConfirmedTx wr (txHash (tds !! 0).tx)-  logDebug wr $-    "Deleted coinbase: " <> txHashToHex (txHash (tds !! 0).tx)---- | Check that a block builds on the best block, or that it is the genesis block--- of an empty database.-checkNewBlock :: Writer -> Block -> BlockNode -> IO ()-checkNewBlock wr b n =-  getBestBlock wr >>= \case-    Nothing-      | isGenesis n -> return ()-      | otherwise -> do-          logError wr $-            "Cannot import non-genesis block: "-              <> blockHashToHex (headerHash b.header)-          throwIO BestBlockNotFound-    Just h-      | b.header.prev == h -> return ()-      | otherwise -> do-          logError wr $-            "Block does not build on head: "-              <> blockHashToHex (headerHash b.header)-          throwIO PrevBlockNotBest---- | Import the transactions of a block, in dependency order. Mempool transactions--- that conflict with the block are deleted first. Transactions already in the--- mempool are confirmed; others are imported.-importOrConfirm :: Writer -> BlockNode -> [Tx] -> IO [TxData]-importOrConfirm wr bn txns = do-  logDebug wr "Freeing outputs..."-  mapM_ (freeOutputs wr True False . snd) (reverse txs)-  logDebug wr "Outputs freed"-  mapM (uncurry action) txs-  where-    txs = sortTxs txns-    br i = BlockRef {height = bn.height, position = i}-    bn_time = fromIntegral $ bn.header.timestamp-    action i tx =-      testPresent wr tx >>= \case-        False -> import_it i tx-        True -> confirm_it i tx-    confirm_it i tx =-      getActiveTxData wr (txHash tx) >>= \case-        Just t -> do-          logDebug wr $-            "Confirming tx: "-              <> txHashToHex (txHash tx)-          -- with the time of the block, as the layered rules give it: see-          -- 'Haskoin.Store.Logic.Layered.importChainTx'. Version 20 kept-          -- the time at which the store first saw the transaction; these-          -- rules are what the tests hold the layered ones to, and have-          -- to agree with them on it.-          td <- confirmTx wr t (br i)-          let timed = (td :: TxData) {timestamp = bn_time}-          insertTx wr timed-          return timed-        Nothing -> do-          logError wr $-            "Cannot find tx to confirm: "-              <> txHashToHex (txHash tx)-          throwIO TxNotFound-    import_it i tx = do-      logDebug wr $-        "Importing tx: " <> txHashToHex (txHash tx)-      importTx wr (br i) bn_time False tx---- | Import a block that builds on the best block, and make it the best block.--- Returns its data and the data of its transactions.-importBlock :: Writer -> Block -> BlockNode -> IO (BlockData, [TxData])-importBlock wr b n = do-  logDebug wr $-    "Checking new block: "-      <> blockHashToHex (headerHash n.header)-  checkNewBlock wr b n-  logDebug wr "Passed check"-  net <- getNetwork wr-  let subsidy = computeSubsidy net n.height-  bs <- getBlocksAtHeight wr n.height-  logDebug wr $-    "Inserting block entries for: "-      <> blockHashToHex (headerHash n.header)-  setBlocksAtHeight-    wr-    (nub (headerHash n.header : bs))-    n.height-  setBest wr (headerHash n.header)-  tds <- importOrConfirm wr n b.txs-  let bd =-        BlockData-          { height = n.height,-            main = True,-            work = n.work,-            header = n.header,-            size = fromIntegral (B.length (encode b)),-            txs = map txHash b.txs,-            weight = if net.segWit then w else 0,-            subsidy = subsidy,-            fee = sum $ map txDataFee tds,-            outputs = ts_out_val-          }-  insertBlock wr bd-  logDebug wr $-    "Finished importing block: "-      <> blockHashToHex (headerHash n.header)-  return (bd, tds)-  where-    ts_out_val =-      sum $ map (sum . map (.value) . (.outputs)) (drop 1 b.txs)-    w =-      let f t = (t :: Tx) {witness = []}-          b' = (b :: Block) {txs = map f b.txs}-          x = B.length (encode b)-          s = B.length (encode b')-       in fromIntegral $ s * 3 + x---- | Check a transaction that will replace these ones. Outputs that they spend--- count as unspent, and outputs that they create as missing.-checkNewTx :: Writer -> [TxHash] -> Tx -> IO ()-checkNewTx wr replaced tx = do-  when (unique_inputs < length tx.inputs) $ do-    logError wr $-      "Transaction spends same output twice: "-        <> txHashToHex (txHash tx)-    throwIO DuplicatePrevOutput-  prevs <- mapM prev_output (prevOuts tx)-  when (any isNothing prevs) $ do-    logError wr $-      "Orphan: " <> txHashToHex (txHash tx)-    throwIO Orphan-  when (isCoinbaseTx tx) $ do-    logError wr $-      "Coinbase cannot be imported into mempool: "-        <> txHashToHex (txHash tx)-    throwIO UnexpectedCoinbase-  when (outputs > sum (map (.value) (catMaybes prevs))) $ do-    logError wr $-      "Insufficient funds for tx: " <> txHashToHex (txHash tx)-    throwIO InsufficientFunds-  where-    prev_output op-      | op.hash `elem` replaced = return Nothing-      | otherwise = do-          parent <- getActiveTxData wr op.hash-          return (parent >>= getTxOut op.index . (.tx))-    outputs = sum (map (.value) tx.outputs)-    unique_inputs = length (nub' (map (.outpoint) tx.inputs))---- | Unspent outputs spent by a transaction, or 'Nothing' for those that are missing.-getUnspentOutputs :: Writer -> Tx -> IO [Maybe Unspent]-getUnspentOutputs wr tx = mapM (getUnspent wr) (prevOuts tx)---- | Data of a new transaction, from its replace-by-fee flag, position, time and--- the outputs it spends.-prepareTxData :: Bool -> BlockRef -> Word64 -> Tx -> [Unspent] -> TxData-prepareTxData rbf br tt tx us =-  TxData-    { block = br,-      tx = tx,-      prevs = ps,-      deleted = False,-      rbf = rbf,-      timestamp = tt,-      spenders = I.empty-    }-  where-    mkprv Unspent {..} = Prev script value-    ps = I.fromList $ zip [0 ..] $ map mkprv us---- | Import a transaction at a position in the chain or the mempool. Throws 'Orphan' if it spends a missing output.-importTx ::-  Writer ->-  BlockRef ->-  -- | unix time-  Word64 ->-  -- | RBF-  Bool ->-  Tx ->-  IO TxData-importTx wr br tt rbf tx = do-  mus <- getUnspentOutputs wr tx-  us <- forM mus $ \case-    Nothing -> do-      logError wr $-        "Attempted to import a tx missing UTXO: "-          <> txHashToHex (txHash tx)-      throwIO Orphan-    Just u -> return u-  let td = prepareTxData rbf br tt tx us-  commitAddTx wr td-  return td---- | Move a confirmed transaction back to the mempool.-unConfirmTx :: Writer -> TxData -> IO TxData-unConfirmTx wr t = confTx wr t Nothing---- | Confirm a mempool transaction at a position in a block.-confirmTx :: Writer -> TxData -> BlockRef -> IO TxData-confirmTx wr t br = confTx wr t (Just br)---- | Move a transaction to a new position in the indices of all its addresses.-replaceAddressTx :: Writer -> TxData -> BlockRef -> IO ()-replaceAddressTx wr t new = do-  net <- getNetwork wr-  ctx <- getCtx wr-  forM_ (txDataAddresses net ctx t) $ \a -> do-    deleteAddrTx-      wr-      a-      TxRef-        { block = t.block,-          txid = txHash t.tx-        }-    insertAddrTx-      wr-      a-      TxRef-        { block = new,-          txid = txHash t.tx-        }---- | Move an unspent output to a new position in the chain, updating the indices--- and balance of its address.-adjustAddressOutput ::-  Writer ->-  OutPoint ->-  TxOut ->-  BlockRef ->-  BlockRef ->-  IO ()-adjustAddressOutput wr op o old new = do-  let pk = o.script-  getUnspent wr op >>= \case-    Nothing -> return ()-    Just u -> do-      unless (u.block == old) $-        error $-          "Existing unspent block bad for output: " <> show op-      replace_unspent pk-  where-    replace_unspent pk = do-      net <- getNetwork wr-      ctx <- getCtx wr-      let ma = eitherToMaybe (scriptToAddressBS net ctx pk)-      deleteUnspent wr op-      insertUnspent-        wr-        Unspent-          { block = new,-            outpoint = op,-            value = o.value,-            script = pk,-            address = ma-          }-      forM_ ma $ replace_addr_unspent pk-    replace_addr_unspent pk a = do-      deleteAddrUnspent-        wr-        a-        Unspent-          { block = old,-            outpoint = op,-            value = o.value,-            script = pk,-            address = Just a-          }-      insertAddrUnspent-        wr-        a-        Unspent-          { block = new,-            outpoint = op,-            value = o.value,-            script = pk,-            address = Just a-          }-      decreaseBalance wr (confirmed old) a o.value-      increaseBalance wr (confirmed new) a o.value---- | Confirm a transaction at a position in a block, or move it back to the mempool--- if no position is given.-confTx :: Writer -> TxData -> Maybe BlockRef -> IO TxData-confTx wr t mbr = do-  replaceAddressTx wr t new-  forM_ (zip [0 ..] t.tx.outputs) $ \(n, o) -> do-    let op = OutPoint (txHash t.tx) n-    adjustAddressOutput wr op o old new-  rbf <- isRBF wr new t.tx-  let td = (t :: TxData) {block = new, rbf = rbf}-  insertTx wr td-  updateMempool wr td-  updateSpendersRBF wr td-  return td-  where-    new = fromMaybe (MemRef t.timestamp) mbr-    old = t.block---- | Spenders inherit BIP125 signalling from unconfirmed parents only, so their--- flag can change when a parent is confirmed or unconfirmed.-updateSpendersRBF :: Writer -> TxData -> IO ()-updateSpendersRBF wr td =-  forM_ (nub' (map (.txid) (I.elems td.spenders))) $ \h ->-    getActiveTxData wr h >>= mapM_ update-  where-    update s = do-      rbf <- isRBF wr s.block s.tx-      when (rbf /= s.rbf) $ do-        let s' = (s :: TxData) {rbf = rbf}-        insertTx wr s'-        updateSpendersRBF wr s'---- | Delete the transactions that spend the same outputs as this one, with their--- descendants. The first flag refuses to delete confirmed transactions, and the--- second refuses any deletion on networks without replace-by-fee.-freeOutputs :: Writer -> Bool -> Bool -> Tx -> IO ()-freeOutputs wr memonly rbfcheck tx =-  getConflicts wr memonly rbfcheck tx >>= mapM_ (deleteSingleTx wr . txHash)---- | Transactions that spend the same outputs as this one, with their--- descendants, in an order in which they can be deleted.-getConflicts ::-  Writer ->-  -- | only delete transaction if unconfirmed-  Bool ->-  -- | only delete if network has RBF-  Bool ->-  Tx ->-  IO [Tx]-getConflicts wr memonly rbfcheck tx = do-  logDebug wr $-    "Getting conflicts for tx " <> txHashToHex (txHash tx) <> "..."-  let prevs = prevOuts tx-  unspents <- mapM (getUnspent wr) prevs-  let spents = [p | (p, Nothing) <- zip prevs unspents]-  spndrs <- catMaybes <$> mapM (getSpender wr) spents-  let txids = H.fromList $ filter (/= txHash tx) $ map (.txid) spndrs-  chains <- mapM (getChain wr memonly rbfcheck) (H.toList txids)-  return . reverse . map snd . sortTxs . M.elems $-    M.fromList [(txHash t, t) | t <- concat chains]---- | Delete a transaction and its descendants, confirmed or not.-deleteConfirmedTx :: Writer -> TxHash -> IO ()-deleteConfirmedTx wr = deleteTx wr False False---- | Delete an unconfirmed transaction and its descendants. If the flag is set and--- the network has no replace-by-fee, throw 'DoubleSpend' instead. Does nothing if--- the transaction is not found.-deleteUnconfirmedTx :: Writer -> Bool -> TxHash -> IO ()-deleteUnconfirmedTx wr rbfcheck th =-  getActiveTxData wr th >>= \case-    Just _ -> deleteTx wr True rbfcheck th-    Nothing ->-      logDebug wr $-        "Not found or already deleted: " <> txHashToHex th---- | Delete a transaction and its descendants.-deleteTx ::-  Writer ->-  -- | only delete transaction if unconfirmed-  Bool ->-  -- | only delete if network has RBF-  Bool ->-  TxHash ->-  IO ()-deleteTx wr memonly rbfcheck th = do-  chain <- getChain wr memonly rbfcheck th-  logDebug wr $-    "Deleting "-      <> cs (show (length chain))-      <> " txs from chain leading to "-      <> txHashToHex th-  mapM_ (\t -> let h = txHash t in deleteSingleTx wr h >> return h) chain---- | A transaction and all its descendants, in an order in which they can be--- deleted: spenders first.-getChain ::-  Writer ->-  -- | only delete transaction if unconfirmed-  Bool ->-  -- | only delete if network has RBF-  Bool ->-  TxHash ->-  IO [Tx]-getChain wr memonly rbfcheck th' = do-  logDebug wr $-    "Getting chain for tx " <> txHashToHex th'-  sort_clean <$> go M.empty (H.singleton th')-  where-    sort_clean = reverse . map snd . sortTxs-    get_tx th =-      getActiveTxData wr th >>= \case-        Nothing -> do-          logDebug wr $-            "Transaction not found: " <> txHashToHex th-          return Nothing-        Just td-          | memonly && confirmed td.block -> do-              logError wr $-                "Transaction already confirmed: "-                  <> txHashToHex th-              throwIO TxConfirmed-          | rbfcheck ->-              getNetwork wr >>= \net ->-                if net.replaceByFee-                  then return $ Just td-                  else do-                    logError wr $-                      "Double-spending transaction: "-                        <> txHashToHex th-                    throwIO DoubleSpend-          | otherwise -> return $ Just td-    go txm pdg = do-      let ths = filter (not . (`M.member` txm)) (H.toList pdg)-      tds <- catMaybes <$> mapM get_tx ths-      let txmn = M.fromList $ fmap (\d -> (txHash d.tx, d.tx)) tds-          spds = concatMap (map (.txid) . I.elems . (.spenders)) tds-          pdg' = H.fromList spds-          txm' = txmn <> txm-      if H.null pdg'-        then do-          logDebug wr $-            "Chain for tx "-              <> txHashToHex th'-              <> " contains "-              <> cs (show (M.size txm'))-              <> " txs"-          return $ M.elems txm'-        else go txm' pdg'---- | Delete a transaction that nothing spends. Throws 'TxSpent' otherwise.-deleteSingleTx :: Writer -> TxHash -> IO ()-deleteSingleTx wr th =-  getActiveTxData wr th >>= \case-    Nothing -> do-      logError wr $-        "Already deleted: " <> txHashToHex th-      throwIO TxNotFound-    Just td ->-      if I.null td.spenders-        then do-          logDebug wr $-            "Deleting tx: " <> txHashToHex th-          commitDelTx wr td-        else do-          logError wr $-            "Tried to delete spent tx: "-              <> txHashToHex th-          throwIO TxSpent---- | Remove a transaction from every index and mark it as deleted.-commitDelTx :: Writer -> TxData -> IO ()-commitDelTx wr = commitModTx wr False---- | Add a transaction to every index.-commitAddTx :: Writer -> TxData -> IO ()-commitAddTx wr = commitModTx wr True---- | Add a transaction to every index, or remove it if the flag is 'False'.-commitModTx :: Writer -> Bool -> TxData -> IO ()-commitModTx wr add tx_data = do-  net <- getNetwork wr-  ctx <- getCtx wr-  mapM_ mod_addr_tx (txDataAddresses net ctx td)-  mod_outputs-  mod_unspent-  insertTx wr td-  updateMempool wr td-  where-    tx = td.tx-    br = td.block-    td = (tx_data :: TxData) {deleted = not add}-    tx_ref = TxRef br (txHash tx)-    mod_addr_tx a-      | add = do-          insertAddrTx wr a tx_ref-          modAddressCount wr add a-      | otherwise = do-          deleteAddrTx wr a tx_ref-          modAddressCount wr add a-    mod_unspent-      | add = spendOutputs wr tx-      | otherwise = unspendOutputs wr tx-    mod_outputs-      | add = addOutputs wr br tx-      | otherwise = delOutputs wr br tx---- | Add a transaction to the mempool if it is unconfirmed, or remove it otherwise.-updateMempool :: Writer -> TxData -> IO ()-updateMempool wr td@TxData {deleted = True} =-  deleteFromMempool wr (txHash td.tx)-updateMempool wr td@TxData {block = MemRef t} =-  addToMempool wr (txHash td.tx) t-updateMempool wr td@TxData {block = BlockRef {}} =-  deleteFromMempool wr (txHash td.tx)---- | Mark the outputs spent by a transaction as spent.-spendOutputs :: Writer -> Tx -> IO ()-spendOutputs wr tx =-  zipWithM_ (spendOutput wr (txHash tx)) [0 ..] (prevOuts tx)---- | Add the outputs of a transaction as unspent.-addOutputs :: Writer -> BlockRef -> Tx -> IO ()-addOutputs wr br tx =-  zipWithM_ (addOutput wr br . OutPoint (txHash tx)) [0 ..] tx.outputs---- | Whether an unconfirmed transaction can be replaced by fee: it signals BIP125--- in one of its inputs, or it spends an unconfirmed transaction that can be--- replaced. Always 'False' for confirmed transactions and on networks without--- replace-by-fee.-isRBF ::-  Writer ->-  BlockRef ->-  Tx ->-  IO Bool-isRBF wr br tx-  | confirmed br = return False-  | otherwise =-      getNetwork wr >>= \net ->-        if net.replaceByFee-          then go-          else return False-  where-    go-      | any ((< 0xffffffff - 1) . (.sequence)) tx.inputs = return True-      | otherwise = carry_on-    carry_on =-      let hs = nub' $ map (.outpoint.hash) tx.inputs-          ck [] = return False-          ck (h : hs') =-            getActiveTxData wr h >>= \case-              Nothing -> return False-              Just t-                | confirmed t.block -> ck hs'-                | t.rbf -> return True-                | otherwise -> ck hs'-       in ck hs---- | Add an unspent output.-addOutput :: Writer -> BlockRef -> OutPoint -> TxOut -> IO ()-addOutput wr = modOutput wr True---- | Delete an output.-delOutput :: Writer -> BlockRef -> OutPoint -> TxOut -> IO ()-delOutput wr = modOutput wr False---- | Add an unspent output, or delete it if the flag is 'False', updating the indices and balance of its address.-modOutput ::-  Writer ->-  Bool ->-  BlockRef ->-  OutPoint ->-  TxOut ->-  IO ()-modOutput wr add br op o = do-  net <- getNetwork wr-  ctx <- getCtx wr-  let ma = eitherToMaybe (scriptToAddressBS net ctx o.script)-  mod_unspent ma-  forM_ ma $ \a -> do-    mod_addr_unspent a (u ma)-    modBalance wr (confirmed br) add a o.value-    modifyReceived wr a v-  where-    v-      | add = (+ o.value)-      | otherwise = subtract o.value-    u ma =-      Unspent-        { script = o.script,-          block = br,-          outpoint = op,-          value = o.value,-          address = ma-        }-    mod_unspent ma-      | add = insertUnspent wr (u ma)-      | otherwise = deleteUnspent wr op-    mod_addr_unspent-      | add = insertAddrUnspent wr-      | otherwise = deleteAddrUnspent wr---- | Delete the outputs of a transaction.-delOutputs :: Writer -> BlockRef -> Tx -> IO ()-delOutputs wr br tx =-  forM_ (zip [0 ..] tx.outputs) $ \(i, o) -> do-    let op = OutPoint (txHash tx) i-    delOutput wr br op o---- | Data of a transaction. Throws 'TxNotFound' if it is missing or deleted.-getImportTxData :: Writer -> TxHash -> IO TxData-getImportTxData wr th =-  getActiveTxData wr th >>= \case-    Nothing -> do-      logDebug wr $ "Tx not found: " <> txHashToHex th-      throwIO TxNotFound-    Just d -> return d---- | Output of a transaction at an index, or 'Nothing' if out of range.-getTxOut :: Word32 -> Tx -> Maybe TxOut-getTxOut i tx = do-  guard (fromIntegral i < length tx.outputs)-  return $ tx.outputs !! fromIntegral i---- | Record which input spends an output.-insertSpender :: Writer -> OutPoint -> Spender -> IO ()-insertSpender wr op s = do-  td <- getImportTxData wr op.hash-  let p = td.spenders-      p' = I.insert (fromIntegral op.index) s p-      td' = (td :: TxData) {spenders = p'}-  insertTx wr td'---- | Forget which input spends an output.-deleteSpender :: Writer -> OutPoint -> IO ()-deleteSpender wr op = do-  td <- getImportTxData wr op.hash-  let p = td.spenders-      p' = I.delete (fromIntegral op.index) p-      td' = (td :: TxData) {spenders = p'}-  insertTx wr td'---- | Spend an unspent output with an input of a transaction.-spendOutput :: Writer -> TxHash -> Word32 -> OutPoint -> IO ()-spendOutput wr th ix op = do-  u <--    getUnspent wr op >>= \case-      Just u -> return u-      Nothing -> error $ "Could not find UTXO to spend: " <> show op-  deleteUnspent wr op-  insertSpender wr op (Spender th ix)-  let pk = u.script-  net <- getNetwork wr-  ctx <- getCtx wr-  forM_ (scriptToAddressBS net ctx pk) $ \a -> do-    decreaseBalance wr (confirmed u.block) a u.value-    deleteAddrUnspent wr a u---- | Make the outputs spent by a transaction unspent again.-unspendOutputs :: Writer -> Tx -> IO ()-unspendOutputs wr = mapM_ (unspendOutput wr) . prevOuts---- | Make an output unspent again.-unspendOutput :: Writer -> OutPoint -> IO ()-unspendOutput wr op = do-  net <- getNetwork wr-  ctx <- getCtx wr-  t <--    getActiveTxData wr op.hash >>= \case-      Nothing -> error $ "Could not find tx data: " <> show op.hash-      Just t -> return t-  let o =-        fromMaybe-          (error ("Could not find output: " <> show op))-          (getTxOut op.index t.tx)-      m = eitherToMaybe (scriptToAddressBS net ctx o.script)-      u =-        Unspent-          { value = o.value,-            block = t.block,-            script = o.script,-            outpoint = op,-            address = m-          }-  deleteSpender wr op-  insertUnspent wr u-  forM_ m $ \a -> do-    insertAddrUnspent wr a u-    increaseBalance wr (confirmed u.block) a o.value---- | Change the total received by an address.-modifyReceived :: Writer -> Address -> (Word64 -> Word64) -> IO ()-modifyReceived wr a f = do-  b <- getDefaultBalance wr a-  setBalance wr b {received = f b.received}---- | Remove an output from the confirmed or unconfirmed balance of an address.-decreaseBalance :: Writer -> Bool -> Address -> Word64 -> IO ()-decreaseBalance wr conf = modBalance wr conf False---- | Add an output to the confirmed or unconfirmed balance of an address.-increaseBalance :: Writer -> Bool -> Address -> Word64 -> IO ()-increaseBalance wr conf = modBalance wr conf True---- | Add or remove an output from the confirmed or unconfirmed balance of an address, and its count of unspent outputs.-modBalance ::-  Writer ->-  -- | confirmed-  Bool ->-  -- | add-  Bool ->-  Address ->-  Word64 ->-  IO ()-modBalance wr conf add a val = do-  b <- getDefaultBalance wr a-  setBalance wr $ (g . f) b-  where-    g b = (b :: Balance) {utxo = m 1 b.utxo}-    f b-      | conf = (b :: Balance) {confirmed = m val b.confirmed}-      | otherwise = (b :: Balance) {unconfirmed = m val b.unconfirmed}-    m-      | add = (+)-      | otherwise = subtract---- | Increase or decrease the number of transactions of an address.-modAddressCount :: Writer -> Bool -> Address -> IO ()-modAddressCount wr add a = do-  b <- getDefaultBalance wr a-  setBalance wr b {txs = f b.txs}-  where-    f-      | add = (+ 1)-      | otherwise = subtract 1---- | Addresses of a list of outputs, without duplicates.-txOutAddrs :: Network -> Ctx -> [TxOut] -> [Address]-txOutAddrs net ctx =-  nub' . rights . map (scriptToAddressBS net ctx . (.script))---- | Addresses of the outputs spent by a transaction, without duplicates.-txInAddrs :: Network -> Ctx -> [Prev] -> [Address]-txInAddrs net ctx =-  nub' . rights . map (scriptToAddressBS net ctx . (.script))---- | Addresses that a transaction is indexed under. An output script counts--- only when it has an address on the network.-txDataAddresses :: Network -> Ctx -> TxData -> [Address]-txDataAddresses net ctx t =-  nub' $ txInAddrs net ctx prevs <> txOutAddrs net ctx outs-  where-    prevs = I.elems t.prevs-    outs = t.tx.outputs---- | Outputs spent by a transaction, without the null outpoint of a coinbase.-prevOuts :: Tx -> [OutPoint]-prevOuts tx = filter (/= nullOutPoint) (map (.outpoint) tx.inputs)---- | Whether a transaction is in the database and not deleted.-testPresent :: Writer -> Tx -> IO Bool-testPresent wr tx = isJust <$> getActiveTxData wr (txHash tx)+-- Rules to import and revert blocks. The database holds the chain and+-- nothing else: the mempool is the pool that the fleet shares in Redis+-- ("Haskoin.Store.Fleet.Pool"), so these rules never meet an unconfirmed+-- transaction. 'importBlock' and 'revertBlock' are the only functions that+-- write to the column families of the chain, and what they write depends+-- on the block and on those column families alone: the same bytes on every+-- store that has the same blocks.+--+-- A reverted block takes its transactions out of the chain. They are not+-- kept anywhere in the database: the member that moves the fleet tip reads+-- them from its view of the database at the old tip, and gives them back to+-- the pool ("Haskoin.Store.Fleet.Member").+--+-- = Importing a block in stages+--+-- 'importBlock' works in four stages, so that a full block takes a second+-- or two and not eighteen, as it did on a store of mainnet whose disk and+-- cores were mostly idle. What cost the time was thirty thousand reads of+-- the database, one after another from one thread, many of them of a key+-- that had been read already.+--+-- 1. __Prepare__ ('prepareBlock'), on all cores, from the block alone: the+--    hash of each transaction, the address of each output, the sizes. They+--    are carried along from there.+--+-- 2. __Read ahead__: the outputs that the block spends are read with+--    multi-gets on all cores into the read cache of the batch+--    ("Haskoin.Store.Database.Writer").+--+-- 3. __Apply__: the rules, one transaction after another, without a read of+--    the database between them.+--+-- 4. __Write__: the batch.+--+-- Since database version 23 the balances of the addresses of a block and+-- the spenders of the outputs that it spends are not read at all: the+-- block writes how they change, as merge records, and RocksDB works out+-- the values when they are read ('addChainTx',+-- "Haskoin.Store.Database.Merge").+--+-- = A mined block is not validated+--+-- A block that gets here was mined: it passed proof of work, and the node+-- that the store follows took it. The store takes it that its+-- transactions are valid: every output that the block spends is unspent+-- in the chain or made by a transaction of the block, and none is spent+-- twice. Checking that is the work of the node, and each check here was a+-- read of the database for every input of every block. So the import+-- reads what it needs to write the block, and nothing to check it:+--+-- * __No spender of the chain is read.__ A block writes its spenders as+--   merge records and never reads a group of them.+--+-- * __An output that the block makes is not read from the database.__ The+--   rules find it in the changes of the batch, as each transaction is+--   applied after those that it spends ('readAheadBlock').+--+-- * __Only a coinbase is looked up in the chain__, in case it is confirmed+--   already. That happened to two coinbases of 2010, and it is data, not+--   a check: the later one takes the place of the earlier. A transaction+--   with an input cannot be confirmed twice, since its input is spent.+--+-- What stays is what the store's own data needs: the unspent outputs that+-- the block spends, whose values, scripts and positions the balances and+-- the indexes are made from, and the order in which transactions are taken+-- ('sortTxs', needed on Bitcoin Cash, whose blocks are in the order of the+-- hashes).+--+-- A block that breaks the assumption is promised one thing: it is written+-- whole or not at all, since a batch that throws writes nothing+-- ("Haskoin.Store.Database.Writer"). It is still refused when the one+-- read that stays finds nothing: an output that does not exist, that the+-- chain has as spent, or that an earlier transaction of the block spent,+-- is not there to be read, and the block is an 'Orphan'. A transaction+-- that spends one output twice stops the batch too, when it comes to+-- spend what it just spent ('addChainTx'): a look in the changes of the+-- batch, in memory.+module Haskoin.Store.Logic+  ( ImportException (..),+    initBest,+    revertBlock,+    importBlock,++    -- * What a block needs worked out once+    PreparedBlock (..),+    PreparedTx (..),+    prepareBlock,+  )+where++import Control.DeepSeq (NFData, force)+import Control.Exception (Exception, evaluate, throwIO)+import Control.Monad (forM, forM_, guard, unless, void, when, zipWithM_)+import Data.ByteString qualified as B+import Data.Default (def)+import Data.Either (rights)+import Data.Functor ((<&>))+import Data.HashSet qualified as H+import Data.IntMap.Strict qualified as I+import Data.List (nub)+import Data.Maybe (catMaybes, fromMaybe, isNothing, mapMaybe)+import Data.Serialize (encode)+import Data.Set qualified as S+import Data.Text (Text)+import Data.Word (Word32, Word64)+import GHC.Generics (Generic)+import Haskoin+  ( Address,+    Block (..),+    BlockHash,+    BlockHeader (..),+    BlockNode (..),+    Network (..),+    OutPoint (..),+    Tx (..),+    TxHash (..),+    TxIn (..),+    TxOut (..),+    VarInt (..),+    blockHashToHex,+    computeSubsidy,+    doubleSHA256,+    eitherToMaybe,+    genesisBlock,+    genesisNode,+    headerHash,+    isGenesis,+    nullOutPoint,+    scriptToAddressBS,+    txHash,+    txHashToHex,+  )+import Haskoin.Node (LogLevel (..), logAt)+import Haskoin.Store.Common (nub', replacedKept, sortTxs, sortTxsWith)+import Haskoin.Store.Data+import Haskoin.Store.Database.Types (BalVal (..))+import Haskoin.Store.Database.Writer++-- | Reasons why an import can fail.+data ImportException+  = -- | the block does not build on the best block+    PrevBlockNotBest+  | -- | the transaction spends an output that is unknown or already spent+    Orphan+  | -- | the database has no best block, or it cannot be read+    BestBlockNotFound+  | -- | only the best block can be reverted+    BlockNotBest+  | -- | a transaction that should be in the database is not+    TxNotFound+  | -- | a transaction cannot be taken out while another one spends it+    TxSpent+  deriving (Eq, Ord, Exception)++instance Show ImportException where+  show PrevBlockNotBest = "Previous block not best"+  show Orphan = "Orphan"+  show BestBlockNotFound = "Best block not found"+  show BlockNotBest = "Block not best"+  show TxNotFound = "Transaction not found"+  show TxSpent = "Transaction is spent"++-------------------------------------------------------------------------------+-- Reading+-------------------------------------------------------------------------------++-- | Log a message with the logger of the writer.+logDebug, logError :: Writer -> Text -> IO ()+logDebug wr = logAt LevelDebug (writerLogger wr)+logError wr = logAt LevelError (writerLogger wr)++-- | Address of an output script, if it has one on the network.+scriptAddress :: Writer -> B.ByteString -> Maybe Address+scriptAddress wr = eitherToMaybe . scriptToAddressBS (writerNet wr)++-- | Data of a confirmed transaction, with the spenders of its outputs that+-- are in blocks. Version 20 kept the bodies of deleted transactions in the+-- same place; they do not count.+chainTx :: Writer -> TxHash -> IO (Maybe TxData)+chainTx wr h =+  getTxData wr h <&> \case+    Just td | not td.deleted -> Just td+    _ -> Nothing++-- | An output as an unspent output, given the position of its transaction.+unspentOutput :: Writer -> BlockRef -> OutPoint -> TxOut -> Unspent+unspentOutput wr br p o =+  Unspent+    { block = br,+      outpoint = p,+      value = o.value,+      script = o.script,+      address = scriptAddress wr o.script+    }++-- | Data of the best block. Throws 'BestBlockNotFound' if there is none.+bestBlockData :: Writer -> IO BlockData+bestBlockData wr = do+  h <-+    getBestBlock wr >>= \case+      Nothing -> do+        logError wr "Best block unknown"+        throwIO BestBlockNotFound+      Just h -> return h+  getBlock wr h >>= \case+    Nothing -> do+      logError wr "Best block not found"+      throwIO BestBlockNotFound+    Just b -> return b++-------------------------------------------------------------------------------+-- Blocks+-------------------------------------------------------------------------------++-- | Import the genesis block if the database has no best block yet.+initBest :: Writer -> IO ()+initBest wr = do+  logDebug wr "Initializing best block"+  let net = writerNet wr+  m <- getBestBlock wr+  when (isNothing m) . void $ do+    logDebug wr "Importing Genesis block"+    importBlock wr (genesisBlock net) (genesisNode net)++-- | Check that a block builds on the best block, or that it is the genesis block+-- of an empty database.+checkNewBlock :: Writer -> Block -> BlockNode -> IO ()+checkNewBlock wr b n =+  getBestBlock wr >>= \case+    Nothing+      | isGenesis n -> return ()+      | otherwise -> do+          logError wr $ "Cannot import non-genesis block: " <> blockHashToHex (headerHash b.header)+          throwIO BestBlockNotFound+    Just h+      | b.header.prev == h -> return ()+      | otherwise -> do+          logError wr $ "Block does not build on head: " <> blockHashToHex (headerHash b.header)+          throwIO PrevBlockNotBest++-- | Import a block that builds on the best block, and make it the best block.+-- Returns its data and the data of its transactions.+--+-- It works in the stages that the head of this module describes, and notes+-- how long each took ('noteBlockTimes').+importBlock :: Writer -> Block -> BlockNode -> IO (BlockData, [TxData])+importBlock wr b n = do+  checkNewBlock wr b n+  (pb, preparing) <- stopwatch (prepareBlock (writerNet wr) b)+  (kinds, reading) <- stopwatch (readAheadBlock wr pb)+  (done, applying) <- stopwatch (applyBlock wr b n pb)+  noteBlockTimes wr preparing reading kinds applying+  return done++-- | A transaction of a block with what its import needs of it, worked out+-- once, from the transaction alone.+data PreparedTx = PreparedTx+  { tx :: !Tx,+    -- | its hash+    txid :: !TxHash,+    -- | the outputs that it spends, in the order of its inputs: none for a+    -- coinbase+    spends :: ![OutPoint],+    -- | the address of each of its outputs, for those that have one+    paid :: ![Maybe Address],+    -- | bytes that it takes in a block, with its witness+    size :: !Int,+    -- | the same without its witness+    base :: !Int+  }+  deriving stock (Eq, Show, Generic)+  deriving anyclass (NFData)++-- | A block with what its import needs of it, worked out once.+data PreparedBlock = PreparedBlock+  { -- | its transactions, in its order+    txs :: ![PreparedTx],+    -- | the same, each after those that it spends, with its position in+    -- the block ('sortTxs')+    ordered :: ![(Word32, PreparedTx)],+    -- | bytes of the block+    size :: !Int,+    -- | weight of the block, as BIP 141 defines it+    weight :: !Int+  }+  deriving (Eq, Show)++-- | Work out what the import of a block needs of it, on all cores.+--+-- Each transaction is serialised once without its witness: the hash is+-- taken of those bytes, and their length is its part of the size of the+-- block. One with a witness is serialised once more for its length. The+-- size and the weight of the block are sums of those, with the header and+-- the count of transactions, instead of two more serialisations of the+-- whole block.+prepareBlock :: Network -> Block -> IO PreparedBlock+prepareBlock net b = do+  txs <- concat <$> onAllCores 2 16 (evaluate . force . map (prepareTx net)) b.txs+  let count = length txs+      overhead = B.length (encode b.header) + B.length (encode (VarInt (fromIntegral count)))+      base = overhead + sum (map (.base) txs)+      full = overhead + sum (map (.size) txs)+      ordered = sortTxsWith (.txid) (.tx) txs+  -- the sort is part of this stage+  _ <- evaluate (length ordered)+  return PreparedBlock {txs, ordered, size = full, weight = 3 * base + full}++-- | Work out what the import of a block needs of one of its transactions.+prepareTx :: Network -> Tx -> PreparedTx+prepareTx net tx =+  PreparedTx+    { tx,+      txid = TxHash (doubleSHA256 stripped),+      spends = prevOuts tx,+      paid = map (eitherToMaybe . scriptToAddressBS net . (.script)) tx.outputs,+      size = if null tx.witness then B.length stripped else B.length (encode tx),+      base = B.length stripped+    }+  where+    -- what 'txHash' hashes+    stripped = encode (tx :: Tx) {witness = []}++-- | Read ahead what the rules will read from the database for a block,+-- into the read cache of the batch: the outputs that the block spends and+-- does not make, as unspent outputs of the chain, with multi-gets on all+-- cores. Their values, scripts and positions are what the balances and the+-- indexes of the block are made from.+--+-- An output that a transaction of the block makes is not asked for. The+-- database does not have it, and the rules find it in the changes of the+-- batch, since each transaction is applied after those that it spends. On+-- Bitcoin mainnet in November 2026 that was 3,000 to 6,500 of the inputs+-- of a block.+--+-- A key that is missing here costs a read when the rules ask for it, and+-- nothing else.+--+-- The balances of the addresses of the block, and the groups of spenders+-- of the outputs that it spends, are not read: they are changed without+-- being read ('addChainTx'). It says how long the read took, as+-- t'ReadTimes'.+readAheadBlock :: Writer -> PreparedBlock -> IO ReadTimes+readAheadBlock wr pb+  | not (readsAhead wr) = return noReads+  | otherwise = do+      (_, unspentReads) <- stopwatch (readAheadUnspents wr outside)+      return ReadTimes {unspentReads, spenderReads = 0, balanceReads = 0}+  where+    made = H.fromList (map (.txid) pb.txs)+    -- the outputs that the block spends and that no transaction of the+    -- block makes+    outside = filter (not . (`H.member` made) . (.hash)) (concatMap (.spends) pb.txs)++-- | The rules of the import of a block, given what was worked out of it.+applyBlock :: Writer -> Block -> BlockNode -> PreparedBlock -> IO (BlockData, [TxData])+applyBlock wr b n pb = do+  let net = writerNet wr+      subsidy = computeSubsidy net n.height+  bs <- getBlocksAtHeight wr n.height+  setBlocksAtHeight wr (nub (headerHash n.header : bs)) n.height+  setBest wr (headerHash n.header)+  tds <- importTxs wr n pb.ordered+  when (n.height >= replacedKept) $+    getReplacedAt wr (n.height - replacedKept) >>= mapM_ (deleteReplaced wr)+  let bd =+        BlockData+          { height = n.height,+            main = True,+            work = n.work,+            header = n.header,+            size = fromIntegral pb.size,+            txs = map (.txid) pb.txs,+            weight = if net.segWit then fromIntegral pb.weight else 0,+            subsidy = subsidy,+            fee = sum $ map txDataFee tds,+            outputs = sum $ map (sum . map (.value) . (.outputs)) (drop 1 b.txs)+          }+  insertBlock wr bd+  logDebug wr $ "Finished importing block: " <> blockHashToHex (headerHash n.header)+  return (bd, tds)++-- | Import the transactions of a block, in dependency order.+importTxs :: Writer -> BlockNode -> [(Word32, PreparedTx)] -> IO [TxData]+importTxs wr bn = mapM (uncurry action)+  where+    br i = BlockRef {height = bn.height, position = i}+    blockTime = fromIntegral bn.header.timestamp+    action i p+      -- only a transaction without inputs can be in the chain already:+      -- the inputs of any other would be spent+      | null p.spends =+          chainTx wr p.txid >>= \case+            Just td -> moveChainTx wr td (br i) blockTime+            Nothing -> importChainTx wr (br i) blockTime p+      | otherwise = importChainTx wr (br i) blockTime p++-- | Import a transaction of a block into the chain, with the time of the+-- block. Throws 'Orphan' if an output that it spends is not unspent in the+-- chain. That is not a check of the block: the output is read for what the+-- transaction needs of it, and there is nothing to go on with when it is+-- not there.+--+-- = Why the time of the block+--+-- What the chain's column families hold is to follow from the chain and+-- from nothing else: the same bytes on every store that has the same+-- blocks, whatever each of them saw before. The time of a transaction was+-- the one thing in them that did not. A store kept the time at which it+-- first saw a transaction in its mempool when a block confirmed it, so two+-- stores had two times for it, a store that synced later had the time of+-- the block, and the stores of a fleet gave different answers about the+-- same confirmed transaction. The time of first sight is what a+-- transaction has while it is in the pool.+importChainTx :: Writer -> BlockRef -> Word64 -> PreparedTx -> IO TxData+importChainTx wr br time p = do+  us <- forM p.spends $ \o ->+    getUnspent wr o >>= \case+      Just u -> return u+      Nothing -> do+        logError wr $ "Attempted to import a tx missing UTXO: " <> txHashToHex p.txid+        throwIO Orphan+  let td = prepareTxData False br time p.tx us+  addChainTx wr p td us+  return td++-- | Give a confirmed transaction a new position in the chain: a block has a+-- transaction that is already confirmed. Only two coinbases of 2010 are, on+-- Bitcoin; the later one takes the place of the earlier, with the outputs+-- that are still unspent, as in the layout of version 20, and the time of+-- the later block.+moveChainTx :: Writer -> TxData -> BlockRef -> Word64 -> IO TxData+moveChainTx wr t new time = do+  forM_ (txDataAddresses wr t) $ \a -> do+    deleteAddrTx wr a TxRef {block = t.block, txid = h}+    insertAddrTx wr a TxRef {block = new, txid = h}+  forM_ (zip [0 ..] t.tx.outputs) $ \(i, _) ->+    getUnspent wr (OutPoint h i) >>= mapM_ move+  let td = (t :: TxData) {block = new, timestamp = time}+  insertTx wr td+  return td+  where+    h = txHash t.tx+    move u = do+      let u' = (u :: Unspent) {block = new}+      insertUnspent wr u'+      forM_ u.address $ \a -> deleteAddrUnspent wr a u >> insertAddrUnspent wr a u'++-- | Revert the best block: make its parent the best block, mark it as not in+-- the main chain, and take its transactions out of the chain, each after+-- the transactions of the block that spend it. Throws 'BlockNotBest' if the+-- block is not the best block.+--+-- Nothing is kept of the transactions: the pool that the fleet shares gets+-- them back from the member that moves the tip, which reads them from its+-- view of the database at the old tip ("Haskoin.Store.Fleet.Member").+revertBlock :: Writer -> BlockHash -> IO ()+revertBlock wr bh = do+  bd <-+    bestBlockData wr >>= \b ->+      if headerHash b.header == bh+        then return b+        else do+          logError wr $ "Cannot revert non-head block: " <> blockHashToHex bh+          throwIO BlockNotBest+  tds <- mapM inChain bd.txs+  setBest wr bd.header.prev+  insertBlock wr bd {main = False}+  -- what the block replaced is not replaced any more+  getReplacedAt wr bd.height >>= mapM_ (deleteReplaced wr)+  -- out of the chain, each after the transactions that spend it+  forM_ (reverse (sortTxs (map (.tx) tds))) (leave . txHash . snd)+  where+    inChain h =+      chainTx wr h >>= \case+        Just td -> return td+        Nothing -> do+          logDebug wr $ "Tx not found: " <> txHashToHex h+          throwIO TxNotFound+    -- take a transaction that nothing in the chain spends out of the chain+    leave h = do+      td <- inChain h+      spent <- getChainSpenders wr h (length td.tx.outputs)+      unless (I.null spent) $ do+        logError wr $ "Tried to revert spent tx: " <> txHashToHex h+        throwIO TxSpent+      removeChainTx wr td++-- The chain's column families+-------------------------------------------------------------------------------++-- | Data of a new transaction, from its replace-by-fee flag, position, time and+-- the outputs it spends.+prepareTxData :: Bool -> BlockRef -> Word64 -> Tx -> [Unspent] -> TxData+prepareTxData rbf br tt tx us =+  TxData+    { block = br,+      tx = tx,+      prevs = ps,+      deleted = False,+      rbf = rbf,+      timestamp = tt,+      spenders = I.empty+    }+  where+    mkprv Unspent {..} = Prev script value+    ps = I.fromList $ zip [0 ..] $ map mkprv us++-- | Change the balance of an address in the chain, by reading it and+-- writing it whole. An address whose balance goes back to nothing, as when+-- the only block that paid it is reverted, has none.+--+-- The import of a block does not go through here: it changes a balance+-- without reading it ('addChainTx'). This is for taking a block out.+modifyBalance :: Writer -> Address -> (Balance -> Balance) -> IO ()+modifyBalance wr a f = do+  b <- f . fromMaybe (zeroBalance a) <$> getBalance wr a+  if nullBalance b then deleteBalance wr a else setBalance wr b++-- | Subtract, or stop: no count or amount of the chain goes below zero.+less :: String -> Word64 -> Word64 -> Word64+less what x y+  | y > x = error $ "The " <> what <> " of an address would go below zero"+  | otherwise = x - y++-- | Add a confirmed transaction to every index of the chain, given what+-- was worked out of it and the outputs that it spends, which must be+-- unspent in the chain.+--+-- Its addresses are those worked out before: of its outputs, and of the+-- outputs that it spends, which came with them. Each is taken once, by+-- comparing them: hashing an address, as 'nub'' does, costs more than the+-- few comparisons that a transaction needs. Each output is asked for+-- again as it is spent, which the batch answers from memory: it is how a+-- transaction that spends an output twice is stopped. A mined block is+-- not validated, and this look is the one thing here that does nothing+-- else. It stays because it costs nothing that could be measured: the+-- benchmark took as long without it (@docs\/block-import.md@).+--+-- = Nothing is read to be changed+--+-- A balance is changed by its differences ('mergeBalance') and a spender+-- is added to its group ('mergeSpender') without reading either: the batch+-- writes them as merge records, and RocksDB works out the values when they+-- are read ("Haskoin.Store.Database.Merge"). Reading the balances and the+-- groups of spenders was most of the read ahead of a block on a store of+-- mainnet, and the import only read them to write them back.+--+-- Two things follow. A count or an amount of a balance is not checked+-- against going below zero here, as it is in 'removeChainTx', since that+-- takes the value: an output that is spent was unspent in the chain, so+-- its amount is in the balance. And a balance is never deleted here: the+-- count of transactions of every address that the transaction touches+-- goes up by one, so none of them comes to nothing. Taking a block out+-- reads each value and writes it whole ('removeChainTx').+addChainTx :: Writer -> PreparedTx -> TxData -> [Unspent] -> IO ()+addChainTx wr p td us = do+  forM_ (S.toList (S.fromList (mapMaybe (.address) us <> catMaybes p.paid))) $ \a -> do+    insertAddrTx wr a TxRef {block = td.block, txid = h}+    mergeBalance wr a (def :: BalVal) {txs = 1}+  forM_ (zip3 [0 ..] td.tx.outputs p.paid) $ \(i, o, address) -> do+    let u = Unspent {block = td.block, outpoint = OutPoint h i, value = o.value, script = o.script, address}+    insertUnspent wr u+    forM_ address $ \a -> do+      insertAddrUnspent wr a u+      mergeBalance wr a (def :: BalVal) {confirmed = o.value, utxo = 1, received = o.value}+  zipWithM_ spend [0 ..] p.spends+  insertTxOf wr h td+  where+    h = p.txid+    spend ix o = do+      u <-+        getUnspent wr o >>= \case+          Just u -> return u+          Nothing -> error $ "Could not find UTXO to spend: " <> show o+      deleteUnspent wr o+      -- under the key of the output: the record of the transaction that+      -- made it is written once, when it is confirmed, and is not read+      -- here either+      mergeSpender wr o (Spender h ix)+      forM_ u.address $ \a -> do+        deleteAddrUnspent wr a u+        -- taken off by adding the two's complement+        mergeBalance wr a (def :: BalVal) {confirmed = negate u.value, utxo = negate 1}++-- | Take a confirmed transaction that nothing in the chain spends out of+-- every index of the chain, and make the outputs that it spent unspent+-- again.+removeChainTx :: Writer -> TxData -> IO ()+removeChainTx wr td = do+  forM_ (txDataAddresses wr td) $ \a -> do+    deleteAddrTx wr a TxRef {block = td.block, txid = h}+    modifyBalance wr a $ \b -> (b :: Balance) {txs = less "count of transactions" b.txs 1}+  forM_ (zip [0 ..] td.tx.outputs) $ \(i, o) -> do+    let p = OutPoint h i+        u = unspentOutput wr td.block p o+    deleteUnspent wr p+    forM_ u.address $ \a -> do+      deleteAddrUnspent wr a u+      modifyBalance wr a $ \b ->+        (b :: Balance)+          { confirmed = less "confirmed balance" b.confirmed o.value,+            utxo = less "count of unspent outputs" b.utxo 1,+            received = less "amount received" b.received o.value+          }+  forM_ (prevOuts td.tx) $ \p -> do+    parent <-+      chainTx wr p.hash >>= \case+        Just t -> return t+        Nothing -> error $ "Could not find tx data: " <> show p.hash+    let o = fromMaybe (error ("Could not find output: " <> show p)) (getTxOut p.index parent.tx)+        u = unspentOutput wr parent.block p o+    deleteSpender wr p+    insertUnspent wr u+    forM_ u.address $ \a -> do+      insertAddrUnspent wr a u+      modifyBalance wr a $ \b -> (b :: Balance) {confirmed = b.confirmed + o.value, utxo = b.utxo + 1}+  deleteTxData wr h+  where+    h = txHash td.tx++-------------------------------------------------------------------------------+-- Transactions+-------------------------------------------------------------------------------++-- | Output of a transaction at an index, or 'Nothing' if out of range.+getTxOut :: Word32 -> Tx -> Maybe TxOut+getTxOut i tx = do+  guard (fromIntegral i < length tx.outputs)+  return $ tx.outputs !! fromIntegral i++-- | Addresses that a transaction is indexed under: those of its outputs and+-- of the outputs that it spends. An output script counts only when it has+-- an address on the network.+txDataAddresses :: Writer -> TxData -> [Address]+txDataAddresses wr t =+  nub' . rights $+    map (scriptToAddressBS net . (.script)) (I.elems t.prevs)+      <> map (scriptToAddressBS net . (.script)) t.tx.outputs+  where+    net = writerNet wr :: Network++-- | Outputs spent by a transaction, without the null outpoint of a coinbase.+prevOuts :: Tx -> [OutPoint]+prevOuts tx = filter (/= nullOutPoint) (map (.outpoint) tx.inputs)
− src/Haskoin/Store/Logic/Layered.hs
@@ -1,814 +0,0 @@-{-# LANGUAGE RecordWildCards #-}-{-# OPTIONS_GHC -Wno-ambiguous-fields #-}---- |--- Module      : Haskoin.Store.Logic.Layered--- Copyright   : (c) Haskoin Developers--- License     : MIT--- Maintainer  : jprupp@protonmail.ch--- Stability   : experimental--- Portability : POSIX------ Rules to import and revert blocks and transactions in the layered layout--- of the database (@docs\/layered-database.md@), in which the mempool is--- kept apart from the chain. These are the rules that the block store--- imports with. They do what "Haskoin.Store.Logic" did for the layout of--- version 20, with the same functions, the same results and the same--- errors, which is how the block store moved from one to the other, and--- how the tests compare the two.------ = Two sets of rules------ __The chain__ has one door. 'importBlock' and 'revertBlock' are the only--- functions here that write to the column families of the chain, and what--- they write depends on the block and on those column families alone. A--- block is imported the same way whether or not the store had seen its--- transactions before, as during the initial sync: every output it spends--- must be unspent in the chain, or the block is refused.------ __The mempool__ is a set of transactions, each with its keys in the--- mempool area ('mempoolRecords'). A transaction enters by having its keys--- written and leaves by having them deleted; nothing else is read or--- changed, and nothing is written to the chain. What is left of the rules--- is the checks: that the outputs a transaction spends exist and are--- unspent, in the chain or in the mempool, and what it replaces.------ The two meet in three places. A block takes the transactions it confirms--- out of the mempool, and deletes those that conflict with it. A reverted--- block gives its transactions back to the mempool. And a mempool--- transaction may spend outputs of the chain, which the chain goes on--- listing as unspent: only the mempool area says otherwise.------ = Same answers as version 20------ For any sequence of blocks and transactions, these rules leave the--- database that "Haskoin.Store.Logic" followed by--- "Haskoin.Store.Database.Migrate" leaves, key for key, and each step gives--- the same result. The tests check that for random sequences. There is one--- difference by design. A block that spends an output of an unconfirmed--- transaction is refused here, because that output is not in the chain;--- version 20 imports it, since its indices do not tell the two apart.-module Haskoin.Store.Logic.Layered-  ( initBest,-    revertBlock,-    importBlock,-    newMempoolTx,-    deleteUnconfirmedTx,-  )-where--import Control.Monad-  ( forM,-    forM_,-    guard,-    unless,-    void,-    when,-    zipWithM_,-  )-import Data.ByteString qualified as B-import Data.Either (rights)-import Data.Functor ((<&>))-import Data.HashMap.Strict qualified as M-import Data.HashSet qualified as H-import Data.IntMap.Strict qualified as I-import Data.List (nub)-import Data.Maybe-  ( catMaybes,-    fromMaybe,-    isJust,-    isNothing,-    mapMaybe,-  )-import Data.Serialize (encode)-import Data.Text (Text)-import Data.Word (Word32, Word64)-import Haskoin-  ( Address,-    Block (..),-    BlockHash,-    BlockHeader (..),-    BlockNode (..),-    Ctx,-    Network (..),-    OutPoint (..),-    Tx (..),-    TxHash,-    TxIn (..),-    TxOut (..),-    blockHashToHex,-    computeSubsidy,-    eitherToMaybe,-    genesisBlock,-    genesisNode,-    headerHash,-    isGenesis,-    nullOutPoint,-    scriptToAddressBS,-    txHash,-    txHashToHex,-  )-import Haskoin.Node (LogLevel (..), logAt)-import Haskoin.Store.Common (Replaced (..), nub', replacedKept, sortTxs)-import Haskoin.Store.Data-import Haskoin.Store.Database.Mempool (MemRecords (..), mempoolRecords)-import Haskoin.Store.Database.Types (MemAddrKey (..), MemAddrVal (..), MemSummary (..))-import Haskoin.Store.Database.Writer-import Haskoin.Store.Logic (ImportException (..))-import UnliftIO (throwIO)------------------------------------------------------------------------------------ Reading------------------------------------------------------------------------------------ | Log a message with the logger of the writer.--- What a peer sends and the store refuses (an orphan, a double spend, a--- transaction without funds) is logged for debugging: it is the ordinary--- traffic of a network, and nothing is wrong with the store. An error is--- kept for what should not happen whatever peers do: a block that does not--- fit the database.-logDebug, logError :: Writer -> Text -> IO ()-logDebug wr = logAt LevelDebug (writerLogger wr)-logError wr = logAt LevelError (writerLogger wr)---- | Address of an output script, if it has one on the network.-scriptAddress :: Writer -> B.ByteString -> Maybe Address-scriptAddress wr = eitherToMaybe . scriptToAddressBS (writerNet wr) (writerCtx wr)---- | Data of a confirmed transaction, with the spenders of its outputs that--- are in blocks. Version 20 kept the bodies of deleted transactions in the--- same place; they do not count.-chainTx :: Writer -> TxHash -> IO (Maybe TxData)-chainTx wr h =-  getTxData wr h <&> \case-    Just td | not td.deleted -> Just td-    _ -> Nothing---- | Data of a transaction that is in the mempool or in the chain, without--- the mempool spenders of its outputs.-activeBody :: Writer -> TxHash -> IO (Maybe TxData)-activeBody wr h = getMempoolTx wr h >>= maybe (chainTx wr h) (return . Just)---- | Data of a transaction that is in the mempool or in the chain, with every--- spender of its outputs, in blocks and in the mempool.-activeTx :: Writer -> TxHash -> IO (Maybe TxData)-activeTx wr h = activeBody wr h >>= mapM spent-  where-    spent td = do-      ms <- getMempoolSpenders wr h (length td.tx.outputs)-      -- those of blocks are under keys of their own, and only a confirmed-      -- transaction can have any-      cs <- if confirmed td.block then getChainSpenders wr h (length td.tx.outputs) else return I.empty-      return (td :: TxData) {spenders = I.union cs ms}---- | An output that a new mempool transaction may spend: one that the chain--- has as unspent, or one of a mempool transaction, that no mempool--- transaction spends.-spendable :: Writer -> OutPoint -> IO (Maybe Unspent)-spendable wr p =-  getMempoolSpender wr p >>= \case-    Just _ -> return Nothing-    Nothing ->-      getUnspent wr p >>= \case-        Just u -> return (Just u)-        Nothing -> (>>= output) <$> getMempoolTx wr p.hash-  where-    output td = unspentOutput wr td.block p <$> getTxOut p.index td.tx---- | An output as an unspent output, given the position of its transaction.-unspentOutput :: Writer -> BlockRef -> OutPoint -> TxOut -> Unspent-unspentOutput wr br p o =-  Unspent-    { block = br,-      outpoint = p,-      value = o.value,-      script = o.script,-      address = scriptAddress wr o.script-    }---- | Input that spends an output, in the mempool or in a block.-spenderOf :: Writer -> OutPoint -> IO (Maybe Spender)-spenderOf wr p = getMempoolSpender wr p >>= maybe (getChainSpender wr p) (return . Just)---- | Data of the best block. Throws 'BestBlockNotFound' if there is none.-bestBlockData :: Writer -> IO BlockData-bestBlockData wr = do-  h <--    getBestBlock wr >>= \case-      Nothing -> do-        logError wr "Best block unknown"-        throwIO BestBlockNotFound-      Just h -> return h-  getBlock wr h >>= \case-    Nothing -> do-      logError wr "Best block not found"-      throwIO BestBlockNotFound-    Just b -> return b------------------------------------------------------------------------------------ Blocks------------------------------------------------------------------------------------ | Import the genesis block if the database has no best block yet.-initBest :: Writer -> IO ()-initBest wr = do-  logDebug wr "Initializing best block"-  let net = writerNet wr-  m <- getBestBlock wr-  when (isNothing m) . void $ do-    logDebug wr "Importing Genesis block"-    importBlock wr (genesisBlock net) (genesisNode net)---- | Check that a block builds on the best block, or that it is the genesis block--- of an empty database.-checkNewBlock :: Writer -> Block -> BlockNode -> IO ()-checkNewBlock wr b n =-  getBestBlock wr >>= \case-    Nothing-      | isGenesis n -> return ()-      | otherwise -> do-          logError wr $ "Cannot import non-genesis block: " <> blockHashToHex (headerHash b.header)-          throwIO BestBlockNotFound-    Just h-      | b.header.prev == h -> return ()-      | otherwise -> do-          logError wr $ "Block does not build on head: " <> blockHashToHex (headerHash b.header)-          throwIO PrevBlockNotBest---- | Import a block that builds on the best block, and make it the best block.--- Returns its data and the data of its transactions.------ The block goes into the chain's column families as it would into those of--- a store with an empty mempool. Before that, the mempool gives way: the--- transactions of the block that were in it leave it, and the mempool--- transactions that spend an output that the block spends are deleted, with--- their descendants.-importBlock :: Writer -> Block -> BlockNode -> IO (BlockData, [TxData])-importBlock wr b n = do-  checkNewBlock wr b n-  let net = writerNet wr-      subsidy = computeSubsidy net n.height-  bs <- getBlocksAtHeight wr n.height-  setBlocksAtHeight wr (nub (headerHash n.header : bs)) n.height-  setBest wr (headerHash n.header)-  tds <- importTxs wr n b.txs-  -- forget what a block replaced 'replacedKept' blocks ago-  when (n.height >= replacedKept) $-    getReplacedAt wr (n.height - replacedKept) >>= mapM_ (deleteReplaced wr)-  let bd =-        BlockData-          { height = n.height,-            main = True,-            work = n.work,-            header = n.header,-            size = fromIntegral (B.length (encode b)),-            txs = map txHash b.txs,-            weight = if net.segWit then w else 0,-            subsidy = subsidy,-            fee = sum $ map txDataFee tds,-            outputs = ts_out_val-          }-  insertBlock wr bd-  logDebug wr $ "Finished importing block: " <> blockHashToHex (headerHash n.header)-  return (bd, tds)-  where-    ts_out_val =-      sum $ map (sum . map (.value) . (.outputs)) (drop 1 b.txs)-    w =-      let f t = (t :: Tx) {witness = []}-          b' = (b :: Block) {txs = map f b.txs}-          x = B.length (encode b)-          s = B.length (encode b')-       in fromIntegral $ s * 3 + x---- | Import the transactions of a block, in dependency order, after deleting--- the mempool transactions that conflict with them. What is kept of each--- one deleted ('Replaced') names the transaction of the block that it--- conflicts with, or that its unconfirmed ancestor conflicts with.-importTxs :: Writer -> BlockNode -> [Tx] -> IO [TxData]-importTxs wr bn txns = do-  mapM_ (freeOutputs . snd) (reverse txs)-  mapM (uncurry action) txs-  where-    txs = sortTxs txns-    br i = BlockRef {height = bn.height, position = i}-    blockTime = fromIntegral bn.header.timestamp-    freeOutputs tx =-      getConflicts wr True False tx >>= mapM_ (\gone -> do-        deleteSingleTx wr (txHash gone)-        insertReplaced wr (txHash gone) Replaced {by = txHash tx, height = bn.height})-    action i tx =-      getMempoolTx wr (txHash tx) >>= \case-        Just td -> do-          -- it leaves the mempool, and with it the time at which this-          -- store first saw it: in the chain its time is that of its-          -- block, like that of a transaction that was never in the-          -- mempool, so that the record is the same on every store-          removeMempoolTx wr td-          td' <- importChainTx wr (br i) blockTime tx-          updateSpendersRBF wr tx-          return td'-        Nothing ->-          chainTx wr (txHash tx) >>= \case-            Just td -> moveChainTx wr td (br i) blockTime-            Nothing -> importChainTx wr (br i) blockTime tx---- | Import a transaction of a block into the chain, with the time of the--- block. Throws 'Orphan' if an output that it spends is not unspent in the--- chain.------ = Why the time of the block------ What the chain's column families hold is to follow from the chain and--- from nothing else: the same bytes on every store that has the same--- blocks, whatever each of them saw before. The time of a transaction was--- the one thing in them that did not. A store kept the time at which it--- first saw a transaction in its mempool when a block confirmed it, so two--- stores had two times for it, a store that synced later had the time of--- the block, and the stores of a fleet gave different answers about the--- same confirmed transaction. The time of first sight is still what a--- transaction has while it is in the mempool, where it is the store's own,--- or the pool's in a fleet.-importChainTx :: Writer -> BlockRef -> Word64 -> Tx -> IO TxData-importChainTx wr br time tx = do-  us <- forM (prevOuts tx) $ \p ->-    getUnspent wr p >>= \case-      Just u -> return u-      Nothing -> do-        logError wr $ "Attempted to import a tx missing UTXO: " <> txHashToHex (txHash tx)-        throwIO Orphan-  let td = prepareTxData False br time tx us-  addChainTx wr td-  return td---- | Give a confirmed transaction a new position in the chain: a block has a--- transaction that is already confirmed. Only two coinbases of 2010 are, on--- Bitcoin; the later one takes the place of the earlier, with the outputs--- that are still unspent, as in the layout of version 20, and the time of--- the later block.-moveChainTx :: Writer -> TxData -> BlockRef -> Word64 -> IO TxData-moveChainTx wr t new time = do-  forM_ (txDataAddresses wr t) $ \a -> do-    deleteAddrTx wr a TxRef {block = t.block, txid = h}-    insertAddrTx wr a TxRef {block = new, txid = h}-  forM_ (zip [0 ..] t.tx.outputs) $ \(i, _) ->-    getUnspent wr (OutPoint h i) >>= mapM_ move-  let td = (t :: TxData) {block = new, timestamp = time}-  insertTx wr td-  return td-  where-    h = txHash t.tx-    move u = do-      let u' = (u :: Unspent) {block = new}-      insertUnspent wr u'-      forM_ u.address $ \a -> deleteAddrUnspent wr a u >> insertAddrUnspent wr a u'---- | Revert the best block: make its parent the best block, mark it as not in--- the main chain, take its transactions out of the chain and give them to--- the mempool at a time, the time of the revert, and delete its coinbase--- with whatever spends it. Throws 'BlockNotBest' if the block is not the--- best block.------ The transactions go back with the time that they are put back, not that--- of their block. The mempool deletes what is older than its timeout--- (@--mempool-timeout@), by that time: with the time of the block, a--- transaction of a block older than the timeout was deleted as soon as it--- came back, while the nodes kept it. Stores of a fleet give it the same--- time all the same, as the shared mempool keeps the time of the first--- store that offers it.-revertBlock :: Writer -> UnixTime -> BlockHash -> IO ()-revertBlock wr now bh = do-  bd <--    bestBlockData wr >>= \b ->-      if headerHash b.header == bh-        then return b-        else do-          logError wr $ "Cannot revert non-head block: " <> blockHashToHex bh-          throwIO BlockNotBest-  tds <- mapM inChain bd.txs-  setBest wr bd.header.prev-  insertBlock wr bd {main = False}-  -- what the block replaced is not replaced any more-  getReplacedAt wr bd.height >>= mapM_ (deleteReplaced wr)-  case tds of-    [] -> return ()-    coinbase : rest -> do-      let returned = M.fromList [(txHash td.tx, td) | td <- rest]-          -- the transactions that go back to the mempool, each after the-          -- transactions that it spends-          ordered = filter ((`M.member` returned) . txHash) (map snd (sortTxs (map (.tx) tds)))-      -- out of the chain, each after the transactions that spend it-      forM_ (reverse ordered) (leave . txHash)-      -- into the mempool, at the time of the revert. The coinbase is still-      -- in the chain: a transaction that spends it has all its parents-      -- while its flag is worked out, as in the layout of version 20-      forM_ (mapMaybe ((`M.lookup` returned) . txHash) ordered) $ \td -> do-        rbf <- isRBF wr td.tx-        addMempoolTx wr now (td :: TxData) {rbf = rbf, timestamp = now}-        updateSpendersRBF wr td.tx-      -- a coinbase cannot be in the mempool: it is deleted, and so is-      -- everything that spends it-      spenders <- getMempoolSpenders wr (txHash coinbase.tx) (length coinbase.tx.outputs)-      forM_ (nub' (map (.txid) (I.elems spenders))) $ \h ->-        getChain wr True False h >>= mapM_ (deleteSingleTx wr . txHash)-      -- and forgotten: nothing is kept of a transaction that leaves the-      -- store unconfirmed-      leave (txHash coinbase.tx)-  where-    inChain h =-      chainTx wr h >>= \case-        Just td -> return td-        Nothing -> do-          logDebug wr $ "Tx not found: " <> txHashToHex h-          throwIO TxNotFound-    -- take a transaction that nothing in the chain spends out of the chain-    leave h = do-      td <- inChain h-      spent <- getChainSpenders wr h (length td.tx.outputs)-      unless (I.null spent) $ do-        logError wr $ "Tried to revert spent tx: " <> txHashToHex h-        throwIO TxSpent-      removeChainTx wr td------------------------------------------------------------------------------------ The chain's column families------------------------------------------------------------------------------------ | Data of a new transaction, from its replace-by-fee flag, position, time and--- the outputs it spends.-prepareTxData :: Bool -> BlockRef -> Word64 -> Tx -> [Unspent] -> TxData-prepareTxData rbf br tt tx us =-  TxData-    { block = br,-      tx = tx,-      prevs = ps,-      deleted = False,-      rbf = rbf,-      timestamp = tt,-      spenders = I.empty-    }-  where-    mkprv Unspent {..} = Prev script value-    ps = I.fromList $ zip [0 ..] $ map mkprv us---- | Change the balance of an address in the chain. An address whose balance--- goes back to nothing, as when the only block that paid it is reverted, has--- none.-modifyBalance :: Writer -> Address -> (Balance -> Balance) -> IO ()-modifyBalance wr a f = do-  b <- f . fromMaybe (zeroBalance a) <$> getBalance wr a-  if nullBalance b then deleteBalance wr a else setBalance wr b---- | Subtract, or stop: no count or amount of the chain goes below zero.-less :: String -> Word64 -> Word64 -> Word64-less what x y-  | y > x = error $ "The " <> what <> " of an address would go below zero"-  | otherwise = x - y---- | Add a confirmed transaction to every index of the chain. Every output--- that it spends must be unspent in the chain.-addChainTx :: Writer -> TxData -> IO ()-addChainTx wr td = do-  forM_ (txDataAddresses wr td) $ \a -> do-    insertAddrTx wr a TxRef {block = td.block, txid = h}-    modifyBalance wr a $ \b -> (b :: Balance) {txs = b.txs + 1}-  forM_ (zip [0 ..] td.tx.outputs) $ \(i, o) -> do-    let u = unspentOutput wr td.block (OutPoint h i) o-    insertUnspent wr u-    forM_ u.address $ \a -> do-      insertAddrUnspent wr a u-      modifyBalance wr a $ \b ->-        (b :: Balance)-          { confirmed = b.confirmed + o.value,-            utxo = b.utxo + 1,-            received = b.received + o.value-          }-  zipWithM_ spend [0 ..] (prevOuts td.tx)-  insertTx wr td-  where-    h = txHash td.tx-    spend ix p = do-      u <--        getUnspent wr p >>= \case-          Just u -> return u-          Nothing -> error $ "Could not find UTXO to spend: " <> show p-      deleteUnspent wr p-      -- under the key of the output: the record of the transaction that-      -- made it is written once, when it is confirmed, and is not read-      -- here either-      insertSpender wr p (Spender h ix)-      forM_ u.address $ \a -> do-        deleteAddrUnspent wr a u-        modifyBalance wr a $ \b ->-          (b :: Balance)-            { confirmed = less "confirmed balance" b.confirmed u.value,-              utxo = less "count of unspent outputs" b.utxo 1-            }---- | Take a confirmed transaction that nothing in the chain spends out of--- every index of the chain, and make the outputs that it spent unspent--- again.-removeChainTx :: Writer -> TxData -> IO ()-removeChainTx wr td = do-  forM_ (txDataAddresses wr td) $ \a -> do-    deleteAddrTx wr a TxRef {block = td.block, txid = h}-    modifyBalance wr a $ \b -> (b :: Balance) {txs = less "count of transactions" b.txs 1}-  forM_ (zip [0 ..] td.tx.outputs) $ \(i, o) -> do-    let p = OutPoint h i-        u = unspentOutput wr td.block p o-    deleteUnspent wr p-    forM_ u.address $ \a -> do-      deleteAddrUnspent wr a u-      modifyBalance wr a $ \b ->-        (b :: Balance)-          { confirmed = less "confirmed balance" b.confirmed o.value,-            utxo = less "count of unspent outputs" b.utxo 1,-            received = less "amount received" b.received o.value-          }-  forM_ (prevOuts td.tx) $ \p -> do-    parent <--      chainTx wr p.hash >>= \case-        Just t -> return t-        Nothing -> error $ "Could not find tx data: " <> show p.hash-    let o = fromMaybe (error ("Could not find output: " <> show p)) (getTxOut p.index parent.tx)-        u = unspentOutput wr parent.block p o-    deleteSpender wr p-    insertUnspent wr u-    forM_ u.address $ \a -> do-      insertAddrUnspent wr a u-      modifyBalance wr a $ \b -> (b :: Balance) {confirmed = b.confirmed + o.value, utxo = b.utxo + 1}-  deleteTxData wr h-  where-    h = txHash td.tx------------------------------------------------------------------------------------ The mempool area------------------------------------------------------------------------------------ | Import a transaction into the mempool with the time it arrived. Returns--- 'False' if it is already known.------ Conflicting unconfirmed transactions are deleted together with their--- descendants if the network has replace-by-fee; otherwise the transaction is--- rejected with 'DoubleSpend'. Throws an 'ImportException' if the transaction is--- invalid, for example if it spends unknown outputs.-newMempoolTx :: Writer -> Tx -> UnixTime -> IO Bool-newMempoolTx wr tx w =-  activeBody wr (txHash tx) >>= \case-    Just _ -> return False-    Nothing -> do-      -- Check everything before writing, so that a rejected transaction-      -- leaves the mempool as it was.-      replaced <- getConflicts wr True True tx-      checkNewTx wr (map txHash replaced) tx-      mapM_ (deleteSingleTx wr . txHash) replaced-      rbf <- isRBF wr tx-      us <- forM (prevOuts tx) $ \p ->-        spendable wr p >>= \case-          Just u -> return u-          Nothing -> do-            logDebug wr $ "Attempted to import a tx missing UTXO: " <> txHashToHex (txHash tx)-            throwIO Orphan-      addMempoolTx wr w (prepareTxData rbf (MemRef w) w tx us)-      return True---- | Delete an unconfirmed transaction and its descendants. If the flag is set and--- the network has no replace-by-fee, throw 'DoubleSpend' instead. Does nothing if--- the transaction is not found.-deleteUnconfirmedTx :: Writer -> Bool -> TxHash -> IO ()-deleteUnconfirmedTx wr rbfcheck th =-  activeBody wr th >>= \case-    Just _ -> getChain wr True rbfcheck th >>= mapM_ (deleteSingleTx wr . txHash)-    Nothing -> logDebug wr $ "Not found or already deleted: " <> txHashToHex th---- | Write the keys of a transaction to the mempool area, with the time it--- entered the mempool. The body goes in at that mempool position and without--- spenders, whatever it had.-addMempoolTx :: Writer -> UnixTime -> TxData -> IO ()-addMempoolTx wr time td = do-  insertMempoolTx wr r.body-  forM_ r.spends (uncurry (insertMempoolSpender wr))-  forM_ r.addresses (uncurry (insertMempoolAddr wr))-  adjustSummaries wr (+) r-  addToMempool wr r.txid r.time-  where-    r = mempoolRecords (writerNet wr) (writerCtx wr) time td---- | Delete the keys of a mempool transaction, given its body, from the--- mempool area.-removeMempoolTx :: Writer -> TxData -> IO ()-removeMempoolTx wr td = do-  deleteMempoolTx wr r.txid-  forM_ r.spends (deleteMempoolSpender wr . fst)-  forM_ r.addresses (deleteMempoolAddr wr . fst)-  adjustSummaries wr (less "mempool summary") r-  deleteFromMempool wr r.txid-  where-    time = case td.block of-      MemRef t -> t-      BlockRef {} -> td.timestamp-    r = mempoolRecords (writerNet wr) (writerCtx wr) time td---- | Add what a transaction does at each address it touches to the summary--- of the address, or take it off again, given how to combine a number of--- the summary with one of the transaction.------ Everything is in the entries of the transaction but one thing: whether an--- output that it spends was made by the mempool, and whether an output that--- it makes is spent by the mempool, which is what 'both' counts. Each is a--- read by key of what another transaction wrote. An output is counted in--- 'both' while both of its transactions are in the mempool, so it is added--- by the second of them to arrive and taken off by the first to leave,--- whichever that is.-adjustSummaries :: Writer -> (Word64 -> Word64 -> Word64) -> MemRecords -> IO ()-adjustSummaries wr op r = do-  deltas <- forM r.addresses $ \case-    (MemAddrTxKey a _ _, _) -> return (a, MemSummary 1 0 0 0 0 0 0)-    (MemAddrSpentKey a p, MemAddrSpentVal v) -> do-      made <- inMempool wr p.hash-      return (a, if made then MemSummary 0 0 0 v 1 v 1 else MemSummary 0 0 0 v 1 0 0)-    (MemAddrOutKey a _ p, MemAddrOutVal v _) -> do-      taken <- isJust <$> getMempoolSpender wr p-      return (a, if taken then MemSummary 0 v 1 0 0 v 1 else MemSummary 0 v 1 0 0 0 0)-    (k, _) -> error ("Not an entry of a transaction: " <> show k)-  forM_ (M.toList (M.fromListWith plus deltas)) $ \(a, d) -> do-    old <--      getMempoolAddr wr (MemAddrSumKey a) <&> \case-        Just (MemAddrSumVal s) -> s-        _ -> zero-    let new = combine op old d-    if new == zero-      then deleteMempoolAddr wr (MemAddrSumKey a)-      else insertMempoolAddr wr (MemAddrSumKey a) (MemAddrSumVal new)-  where-    zero = MemSummary 0 0 0 0 0 0 0-    plus = combine (+)-    combine f x y =-      MemSummary-        { txs = f x.txs y.txs,-          paid = f x.paid y.paid,-          paidCount = f x.paidCount y.paidCount,-          spent = f x.spent y.spent,-          spentCount = f x.spentCount y.spentCount,-          both = f x.both y.both,-          bothCount = f x.bothCount y.bothCount-        }---- | Delete a mempool transaction that nothing spends. Nothing is kept of--- it. Throws 'TxSpent' if something spends it.------ Until version 22 its body was kept, marked as deleted, and the API--- returned it with that mark. Clients that did not read the mark took a--- replaced transaction for a payment, each store had its own set of them,--- which a fleet cannot answer from, and nothing ever removed them. What--- the store needs of them, not to fetch a transaction again that it just--- deleted, it remembers for a while in memory--- ("Haskoin.Store.BlockStore").-deleteSingleTx :: Writer -> TxHash -> IO ()-deleteSingleTx wr th =-  getMempoolTx wr th >>= \case-    Nothing -> do-      logDebug wr $ "Already deleted: " <> txHashToHex th-      throwIO TxNotFound-    Just td -> do-      spenders <- getMempoolSpenders wr th (length td.tx.outputs)-      if I.null spenders-        then do-          logDebug wr $ "Deleting tx: " <> txHashToHex th-          removeMempoolTx wr td-        else do-          logError wr $ "Tried to delete spent tx: " <> txHashToHex th-          throwIO TxSpent---- | Check a transaction that will replace these ones. Outputs that they spend--- count as unspent, and outputs that they create as missing.-checkNewTx :: Writer -> [TxHash] -> Tx -> IO ()-checkNewTx wr replaced tx = do-  when (unique_inputs < length tx.inputs) $ do-    logDebug wr $ "Transaction spends same output twice: " <> txHashToHex (txHash tx)-    throwIO DuplicatePrevOutput-  prevs <- mapM prev_output (prevOuts tx)-  when (any isNothing prevs) $ do-    logDebug wr $ "Orphan: " <> txHashToHex (txHash tx)-    throwIO Orphan-  when (isCoinbaseTx tx) $ do-    logDebug wr $ "Coinbase cannot be imported into mempool: " <> txHashToHex (txHash tx)-    throwIO UnexpectedCoinbase-  when (outputs > sum (map (.value) (catMaybes prevs))) $ do-    logDebug wr $ "Insufficient funds for tx: " <> txHashToHex (txHash tx)-    throwIO InsufficientFunds-  where-    prev_output op-      | op.hash `elem` replaced = return Nothing-      | otherwise = (>>= getTxOut op.index . (.tx)) <$> activeBody wr op.hash-    outputs = sum (map (.value) tx.outputs)-    unique_inputs = length (nub' (map (.outpoint) tx.inputs))---- | Transactions that spend the same outputs as this one, with their--- descendants, in an order in which they can be deleted. The first flag--- refuses confirmed transactions, and the second refuses any on networks--- without replace-by-fee.-getConflicts :: Writer -> Bool -> Bool -> Tx -> IO [Tx]-getConflicts wr memonly rbfcheck tx = do-  let prevs = prevOuts tx-  unspents <- mapM (spendable wr) prevs-  let spents = [p | (p, Nothing) <- zip prevs unspents]-  spndrs <- catMaybes <$> mapM (spenderOf wr) spents-  let txids = H.fromList $ filter (/= txHash tx) $ map (.txid) spndrs-  chains <- mapM (getChain wr memonly rbfcheck) (H.toList txids)-  return . reverse . map snd . sortTxs . M.elems $-    M.fromList [(txHash t, t) | t <- concat chains]---- | A transaction and all its descendants, in an order in which they can be--- deleted: spenders first.-getChain :: Writer -> Bool -> Bool -> TxHash -> IO [Tx]-getChain wr memonly rbfcheck th' =-  sort_clean <$> go M.empty (H.singleton th')-  where-    sort_clean = reverse . map snd . sortTxs-    get_tx th =-      activeTx wr th >>= \case-        Nothing -> do-          logDebug wr $ "Transaction not found: " <> txHashToHex th-          return Nothing-        Just td-          | memonly && confirmed td.block -> do-              logDebug wr $ "Transaction already confirmed: " <> txHashToHex th-              throwIO TxConfirmed-          | rbfcheck ->-              if (writerNet wr).replaceByFee-                then return $ Just td-                else do-                  logDebug wr $ "Double-spending transaction: " <> txHashToHex th-                  throwIO DoubleSpend-          | otherwise -> return $ Just td-    go txm pdg = do-      let ths = filter (not . (`M.member` txm)) (H.toList pdg)-      tds <- catMaybes <$> mapM get_tx ths-      let txmn = M.fromList $ fmap (\d -> (txHash d.tx, d.tx)) tds-          spds = concatMap (map (.txid) . I.elems . (.spenders)) tds-          pdg' = H.fromList spds-          txm' = txmn <> txm-      if H.null pdg'-        then return $ M.elems txm'-        else go txm' pdg'---- | Whether a mempool transaction can be replaced by fee: it signals BIP125--- in one of its inputs, or it spends an unconfirmed transaction that can be--- replaced. Always 'False' on networks without replace-by-fee.-isRBF :: Writer -> Tx -> IO Bool-isRBF wr tx-  | not (writerNet wr).replaceByFee = return False-  | any ((< 0xffffffff - 1) . (.sequence)) tx.inputs = return True-  | otherwise = ck (nub' $ map (.outpoint.hash) tx.inputs)-  where-    ck [] = return False-    ck (h : hs') =-      activeBody wr h >>= \case-        Nothing -> return False-        Just t-          | confirmed t.block -> ck hs'-          | t.rbf -> return True-          | otherwise -> ck hs'---- | Mempool transactions inherit BIP125 signalling from unconfirmed parents--- only, so their flag can change when a parent is confirmed or goes back to--- the mempool. Correct the flags of the mempool transactions that spend the--- outputs of a transaction, and of theirs in turn.-updateSpendersRBF :: Writer -> Tx -> IO ()-updateSpendersRBF wr tx = do-  spenders <- getMempoolSpenders wr (txHash tx) (length tx.outputs)-  forM_ (nub' (map (.txid) (I.elems spenders))) $ \h ->-    getMempoolTx wr h >>= mapM_ update-  where-    update s = do-      rbf <- isRBF wr s.tx-      when (rbf /= s.rbf) $ do-        insertMempoolTx wr (s :: TxData) {rbf = rbf}-        updateSpendersRBF wr s.tx------------------------------------------------------------------------------------ Transactions------------------------------------------------------------------------------------ | Output of a transaction at an index, or 'Nothing' if out of range.-getTxOut :: Word32 -> Tx -> Maybe TxOut-getTxOut i tx = do-  guard (fromIntegral i < length tx.outputs)-  return $ tx.outputs !! fromIntegral i---- | Addresses that a transaction is indexed under: those of its outputs and--- of the outputs that it spends. An output script counts only when it has--- an address on the network.-txDataAddresses :: Writer -> TxData -> [Address]-txDataAddresses wr t =-  nub' . rights $-    map (scriptToAddressBS net ctx . (.script)) (I.elems t.prevs)-      <> map (scriptToAddressBS net ctx . (.script)) t.tx.outputs-  where-    net = writerNet wr :: Network-    ctx = writerCtx wr :: Ctx---- | Outputs spent by a transaction, without the null outpoint of a coinbase.-prevOuts :: Tx -> [OutPoint]-prevOuts tx = filter (/= nullOutPoint) (map (.outpoint) tx.inputs)
src/Haskoin/Store/Manager.hs view
@@ -17,11 +17,14 @@ -- and stops them all together. module Haskoin.Store.Manager   ( StoreConfig (..),+    cachePrefix,     Store (..),     withStore,   ) where +import Control.Concurrent.Async (link, withAsync)+import Control.Concurrent.STM (STM, atomically) import Control.Monad (forM_, unless, when) import Data.ByteString (ByteString) import Data.ByteString.Char8 qualified as C@@ -32,19 +35,20 @@ import Database.Redis qualified as Redis import Haskoin   ( BlockHash (..),-    Ctx,+    Headers (..),     Inv (..),     InvType (..),     InvVector (..),     Message (..),     MessageCommand (..),-    Network,+    Network (..),     NetworkAddress (..),     NotFound (..),     Pong (..),     Reject (..),     TxHash (..),     VarString (..),+    headerHash,     sockToHostAddress,   ) import Haskoin.Node@@ -65,16 +69,17 @@ import Haskoin.Store.BlockStore   ( BlockStore,     BlockStoreConfig (..),+    blockStoreAnnouncedSTM,     blockStoreBlockSTM,     blockStoreHeadSTM,     blockStoreNotFoundSTM,     blockStorePeerConnectSTM,     blockStorePeerDisconnectSTM,+    blockStorePongSTM,     blockStoreTxHashSTM,     blockStoreTxSTM,     withBlockStore,   )-import Haskoin.Store.Cache (Cache, CacheConfig (..), cachePrefix, withCache) import Haskoin.Store.Common   ( StoreEvent (..),     loopForever,@@ -86,16 +91,19 @@     withLayeredReader,   ) import Haskoin.Store.Fleet.Member-  ( Member,+  ( Member (..),     MemberConfig (..),     memberCacheLoop,     memberEvents,     memberForget,     memberLoop,+    memberReceived,+    memberSeen,     newMember,     noteBest,   ) import Haskoin.Store.Fleet.Redis (FleetEvent (..))+import Haskoin.Store.XPub.Service (XPubConfig (..)) import NQE   ( Inbox,     Publisher,@@ -108,12 +116,6 @@ import Network.Socket (SockAddr (..)) import System.Metrics.StatsD import System.Random (randomIO)-import UnliftIO-  ( STM,-    atomically,-    link,-    withAsync,-  )  -- | Store mailboxes. Handles to the running components of a store, given by -- 'withStore'.@@ -126,10 +128,9 @@     block :: !BlockStore,     -- | database reader     db :: !DatabaseReader,-    -- | cache of extended public keys, if Redis is configured and the store-    -- is not a member of a fleet-    cache :: !(Maybe Cache),-    -- | the store as a member of a fleet, if it is one+    -- | the store as a member of its fleet, with the door to the fleet's+    -- cache of xpubs that the web server answers them from; 'Nothing'+    -- without Redis, when the store is unhealthy and answers 503     fleet :: !(Maybe Member),     -- | what the store tells its clients: new best blocks and changes of     -- the mempool. For a store alone they are its own, those that the@@ -141,8 +142,6 @@     pub :: !(Publisher StoreEvent),     -- | network constants     net :: !Network,-    -- | secp256k1 context-    ctx :: !Ctx,     -- | how the store opens a connection to a node, for those it makes     -- besides the ones of the peer manager     -- ("Haskoin.Store.Web.Publish")@@ -161,42 +160,34 @@     db :: !FilePath,     -- | network constants     net :: !Network,-    -- | Redis URL for the cache of extended public keys ("Haskoin.Store.Cache");-    -- 'Nothing' for no cache, an empty URL for the local Redis+    -- | Redis URL of the fleet: its tip, pool and cache of extended public+    -- keys ("Haskoin.Store.Fleet.Member"); an empty URL for the local Redis.+    -- A store always has one: 'Nothing' is for tests of the block store+    -- alone, which then syncs blocks but answers 503.     redis :: !(Maybe String),-    -- | Secp256k1 context-    ctx :: !Ctx,     -- | gap on extended public key with no transactions     initGap :: !Word32,     -- | gap for extended public keys     gap :: !Word32,-    -- | cache xpubs with at least this many used addresses-    redisMinAddrs :: !Int,-    -- | or with at least this many transactions in their addresses-    redisMinTxs :: !Int,-    -- | evict the least recently used xpubs from the cache past this many+    -- | most xpubs in the fleet's cache, at least 1, by GreedyDual: those+    -- that cost least to index again go first, of those not used for a+    -- while     redisMaxXPubs :: !Int,-    -- | do not index new mempool transactions-    noMempool :: !Bool,-    -- | wipe mempool when starting-    wipeMempool :: !Bool,-    -- | answer as one of a fleet of stores that share the Redis of 'redis':-    -- every store of the fleet then gives the same answer to the same-    -- request, from the block that they agree on and the mempool that they-    -- share ("Haskoin.Store.Fleet.Member"). The cache of extended public-    -- keys of a store alone ("Haskoin.Store.Cache") is not used then: its-    -- entries follow each store's own database, which is what a fleet does-    -- not answer from. A fleet has one of its own-    -- ("Haskoin.Store.Fleet.XPub"), under the same three settings.-    fleet :: !Bool,-    -- | sync mempool from peers-    syncMempool :: !Bool,+    -- | the indexers of xpubs ("Haskoin.Store.XPub.Service");+    -- 'Haskoin.Store.XPub.Service.defaultXPubConfig' for one thread a core+    xpubs :: !XPubConfig,     -- | blocks to request from the syncing peer in one batch     blockWindow :: !Int,     -- | delete mempool transactions older than this number of days     mempoolTimeout :: !Int,     -- | disconnect peer if message not received for this many seconds     peerTimeout :: !NominalDiffTime,+    -- | ask another peer for the blocks that the syncing peer has not+    -- delivered for this many seconds+    blockStall :: !NominalDiffTime,+    -- | a peer is asked for blocks if it answers a ping within this long,+    -- or if none does+    blockPing :: !NominalDiffTime,     -- | disconnect peer if it has been connected this long     maxPeerLife :: !NominalDiffTime,     -- | connect to peers using the function 'Haskoin.Node.withConnection'@@ -205,8 +196,12 @@     stats :: !(Maybe Stats),     -- | Bloom filters on the column families that are read by key     bloom :: !Bool,-    -- | RocksDB max_open_files ('Nothing' = unlimited: every SST file's index-    -- and filter blocks stay resident for the life of the process)+    -- | RocksDB max_open_files ('Nothing' = unlimited, which is what a+    -- store with a block cache wants: its index and filter blocks are in+    -- the cache, an open file costs a few kilobytes, and a file that is+    -- not open has to be opened for every lookup that lands in it. Without+    -- a block cache every open file keeps its index and filter in memory,+    -- and the limit is what bounds them)     maxOpenFiles :: !(Maybe Int),     -- | bytes of RocksDB memtables across all column families ('Nothing' =     -- unlimited: each column family keeps up to two 64 MiB memtables)@@ -214,6 +209,11 @@     -- | bytes of write-ahead log after which the column families holding the     -- oldest log file are flushed ('Nothing' = RocksDB default)     maxWalSize :: !(Maybe Int),+    -- | threads that RocksDB may use at once for flushes and compactions+    -- together ('Nothing' = RocksDB default: two, one of each). More of+    -- them shorten a backlog of compactions where the disk and the cores+    -- have room, and take both from the requests where they have not+    backgroundJobs :: !(Maybe Int),     -- | bytes of RocksDB block cache shared by all column families, which     -- also holds their index and filter blocks ('Nothing' = RocksDB default:     -- 32 MiB per column family, index and filter blocks outside it)@@ -242,7 +242,7 @@             withBlockStore (blockStoreCfg cfg node pub db member) $ \block_store ->               withAsync (nodeForwarder block_store pub node_sub) $ \fwd -> do                 link fwd-                withStoreFleet cfg pub member . withStoreCache cfg db node.chain pub $ \cache ->+                withStoreFleet cfg pub member $                   withFleetEvents cfg pub member $ \events ->                     action $                       Store@@ -250,22 +250,24 @@                           chain = node.chain,                           block = block_store,                           db = db,-                          cache = cache,                           fleet = member,                           events = events,                           pub = pub,                           net = cfg.net,-                          ctx = cfg.ctx,                           connect = cfg.connect                         } +-- | What every key of the fleet in Redis starts with.+cachePrefix :: Network -> ByteString+cachePrefix net = "hs2:" <> C.pack net.name <> ":"+ -- | The store as a member of a fleet, if the configuration says that it is -- one and has a Redis to share: under a name of its own, with a lease of -- ten seconds, views of its last six blocks, and the store's limit on the -- age of unconfirmed transactions. storeMember :: StoreConfig -> DatabaseReader -> IO (Maybe Member)-storeMember cfg db = case (cfg.fleet, cfg.redis) of-  (True, Just url) -> do+storeMember cfg db = case cfg.redis of+  Just url -> do     info <-       if null url         then return Redis.defaultConnectInfo@@ -280,15 +282,15 @@             lease = 10000,             depth = 6,             expiry = fromIntegral cfg.mempoolTimeout * 24 * 3600,-            cacheMax = cfg.redisMaxXPubs,-            cacheMinAddrs = cfg.redisMinAddrs,-            cacheMinTxs = cfg.redisMinTxs,+            cacheMax = max 1 cfg.redisMaxXPubs,+            indexer = cfg.xpubs.indexer,             stats = cfg.stats,             note = \level -> logAt level (withSource "Fleet" cfg.logger)           }         conn         db-  _ -> return Nothing+  -- without Redis the store syncs blocks, but is unhealthy and answers 503+  Nothing -> return Nothing  -- | Take the store's turns as a member of its fleet while an action runs, -- if it is one: every two seconds, and whenever the best block changes.@@ -330,33 +332,6 @@     hash :: (Serialize x) => ByteString -> Maybe x     hash = either (const Nothing) Just . decode --- | Run the cache while an action runs, if Redis is configured.-withStoreCache ::-  StoreConfig ->-  DatabaseReader ->-  Chain ->-  Publisher StoreEvent ->-  (Maybe Cache -> IO a) ->-  IO a-withStoreCache cfg db chain pub action =-  case cfg.redis of-    Nothing -> action Nothing-    Just _ | cfg.fleet -> action Nothing-    Just url ->-      withCache-        CacheConfig-          { url = url,-            minAddrs = cfg.redisMinAddrs,-            minTxs = cfg.redisMinTxs,-            maxXPubs = cfg.redisMaxXPubs,-            stats = cfg.stats-          }-        db-        chain-        pub-        (withSource "Cache" cfg.logger)-        (action . Just)- -- | Open the database described by the configuration, bring it to the -- layout that the store runs on ('upgradeDatabase'), and run an action with -- its reader. A database of version 20 has its mempool moved to an area of@@ -370,7 +345,6 @@   stats <- mapM createDataMetrics cfg.stats   withLayeredReader     cfg.net-    cfg.ctx     cfg.initGap     cfg.gap     cfg.db@@ -378,6 +352,7 @@     cfg.maxOpenFiles     cfg.writeBufferSize     cfg.maxWalSize+    cfg.backgroundJobs     cfg.blockCache     stats     $ \db -> do@@ -405,14 +380,14 @@       pub = pub,       db = db,       net = cfg.net,-      ctx = cfg.ctx,-      noMempool = cfg.noMempool,-      wipeMempool = cfg.wipeMempool,       onBest = maybe (\_ -> return ()) noteBest member,-      syncMempool = cfg.syncMempool,+      onTx = maybe (\_ _ -> return ()) memberReceived member,+      seen = maybe (\_ -> return mempty) memberSeen member,       blockWindow = cfg.blockWindow,       mempoolTimeout = cfg.mempoolTimeout,       peerTimeout = cfg.peerTimeout,+      blockStall = cfg.blockStall,+      blockPing = cfg.blockPing,       stats = cfg.stats,       logger = withSource "BlockStore" cfg.logger     }@@ -471,8 +446,13 @@       blockStorePeerDisconnectSTM p b     PeerMessage p msg ->       case msg of-        MPong (Pong n) ->+        MPong (Pong n) -> do           publishSTM (StorePeerPong p n) pub+          blockStorePongSTM p n b+        -- the headers of a message are a chain: the last says what the+        -- peer has+        MHeaders (Headers hs)+          | (h, _) : _ <- reverse hs -> blockStoreAnnouncedSTM p [headerHash h] b         MBlock block ->           blockStoreBlockSTM p block b         MTx tx ->@@ -490,7 +470,13 @@                 | InvVector t h <- is,                   t == InvTx || t == InvWitnessTx                 ]+              blocks =+                [ BlockHash h+                | InvVector t h <- is,+                  t == InvBlock || t == InvWitnessBlock+                ]           publishSTM (StoreTxAnnounce p txs) pub+          unless (null blocks) $ blockStoreAnnouncedSTM p blocks b           unless (null txs) $ blockStoreTxHashSTM p txs b         -- A reject that names no transaction hash cannot be attributed to a         -- transaction and falls through to the catch-all below.
src/Haskoin/Store/Outage.hs view
@@ -32,12 +32,13 @@   ) where +import Control.Concurrent.STM (TVar, atomically, newTVarIO, swapTVar) import Control.Monad (when) import Data.Maybe (isJust) import Data.String.Conversions (cs) import Data.Text (Text) import Haskoin.Node (LogLevel (..))-import UnliftIO (TVar, atomically, newTVarIO, swapTVar, tryAny)+import Haskoin.Store.Exception (tryAny)  -- | Whether the last attempt at something failed. newtype Outage = Outage (TVar Bool)
src/Haskoin/Store/Runtime.hs view
@@ -34,19 +34,19 @@   ) where +import Control.Concurrent (threadDelay) import Control.Monad (forever, unless) import GHC.Stats (GCDetails (..), RTSStats (..), getRTSStats, getRTSStatsEnabled) import Haskoin.Node (Logger, logInfo) import System.Metrics.StatsD-  ( Stats,-    StatCounter,+  ( StatCounter,     StatGauge,+    Stats,     incrementCounter,     newStatCounter,     newStatGauge,     setGauge,   )-import UnliftIO.Concurrent (threadDelay)  -- | The metrics of the collector. data Metrics = Metrics
src/Haskoin/Store/Web.hs view
@@ -41,3785 +41,4257 @@ --   queue. A client that falls too far behind is disconnected rather than allowed --   to make the server buffer events without limit. ----- * __Compatibility endpoints ease migration.__ The @\/blockchain\/@ endpoints---   follow the Blockchain.info API so that existing wallets can switch to a---   self-hosted server with few changes.------ * __Handlers are plain functions.__ A handler takes the 'WebState' and---   the request ('Req'), and returns a response in 'IO'. The state holds the---   queries to answer with in 'query' ('webQuery': the database, with the---   cache in front of the queries about xpubs); @nocache=true@ is the same---   handler given a state whose queries read only the database ('cacheOr').---   The state is made for each request, around one view of the database,---   so that everything a request reads is of one moment; a store that is a---   member of a fleet takes the view at the block that its fleet answers---   from, with the mempool that the fleet shares on top, and has the---   fleet's cache of xpubs in front of the queries about them---   ("Haskoin.Store.Fleet.Member").---   Routes, parameters and the passing of a request to the next route are---   in "Haskoin.Store.Web.Router", a few dozen lines over WAI. A handler---   stops early by throwing an 'Except', which becomes the response---   ('guarded'). There is no web framework and no monad transformer: a---   framework runs every handler in a stack of its own, and a reader monad---   in it would hide the state.-module Haskoin.Store.Web-  ( -- * Web-    WebConfig (..),-    Except (..),-    WebLimits (..),-    runWeb,-    runWebSocket,--    -- * Request parsing-    decodeTxBody,--    -- * Logging-    logIt,-    ailments,-  )-where--import Conduit-  ( ConduitT,-    await,-    concatMapC,-    concatMapMC,-    dropC,-    dropWhileC,-    headC,-    lengthC,-    mapC,-    runConduit,-    sinkList,-    takeC,-    takeWhileC,-    yield,-    (.|),-  )-import Control.Applicative ((<|>))-import Control.Lens ((&), (.~), (?~), (^.))-import Control.Monad-  ( filterM,-    forM_,-    forever,-    guard,-    join,-    unless,-    void,-    when,-    (<=<),-  )-import Control.Monad.Trans (lift)-import Data.Aeson-  ( Encoding,-    ToJSON (..),-    (.=),-  )-import Data.Aeson.KeyMap qualified as KeyMap-import Data.Aeson qualified as A-import Data.Aeson.Encoding-  ( encodingToLazyByteString,-    fromEncoding,-    list,-    unsafeToEncoding,-  )-import Data.Aeson.Types (parseMaybe)-import Data.ByteString.Base16 (decodeBase16, parseBase16)-import Data.ByteString (ByteString)-import Data.ByteString.Builder (Builder, lazyByteString)-import Data.ByteString.Char8 qualified as C-import Data.ByteString.Lazy qualified as L-import Data.Bytes.Get-import Data.Bytes.Put-import Data.Bytes.Serial-import Data.Char (isSpace)-import Data.Default (Default (..))-import Data.HashMap.Strict (HashMap)-import Data.HashMap.Strict qualified as HashMap-import Data.HashSet (HashSet)-import Data.HashSet qualified as HashSet-import Data.Int (Int64)-import Data.List qualified-import Data.List (intersperse, nub)-import Data.Maybe-  ( catMaybes,-    fromJust,-    fromMaybe,-    isJust,-    listToMaybe,-    mapMaybe,-    maybeToList,-  )-import Data.Proxy (Proxy (..))-import Data.Serialize (Serialize, decode, encode)-import Data.String (fromString)-import Data.String.Conversions (cs)-import Data.Text (Text)-import Data.Text qualified as T-import Data.Text.Encoding qualified as T-import Data.Text.Lazy qualified as TL-import Data.Text.Lazy.Encoding qualified as TL-import Data.Time.Clock.POSIX (getPOSIXTime)-import Data.Word (Word32, Word64)-import Database.RocksDB-  ( Property (..),-    getProperty,-  )-import Haskoin.Address-import Haskoin.Block qualified as H-import Haskoin.Crypto.Keys-import Haskoin.Network-import Haskoin.Node-  ( Chain,-    Logger,-    OnlinePeer (..),-    PeerMgr,-    chainGetAncestor,-    chainGetBest,-    getPeers,-    logAt,-    logDebug,-    logError,-    logInfo,-    logWarn,-    withSource,-  )-import Haskoin.Script-import Haskoin.Store.BlockStore-import Haskoin.Store.Cache-import Haskoin.Store.Common hiding-  ( blockAtOrAfter,-    blockAtOrAfterMTP,-    blockAtOrBefore,-    getActiveBlock,-    getActiveTxData,-    getDefaultBalance,-    getNumTransaction,-    getTransaction,-    getTransactions,-  )-import Haskoin.Store.Common qualified as C-import Haskoin.Store.Data-import Haskoin.Store.Database.Query qualified as Q-import Haskoin.Store.Database.Query.Layered qualified as L-import Haskoin.Store.Database.Reader-import Haskoin.Store.Fleet.Member-  ( Member (..),-    TipMoved (..),-    answeringCached,-    memberAlive,-    memberEvents,-    memberTip,-  )-import Haskoin.Store.Fleet.Redis (FleetEvent (..))-import Haskoin.Store.Manager-import Haskoin.Store.Web.Docs (swaggerPage, swaggerYaml)-import Haskoin.Store.Web.Events-import Haskoin.Store.Web.Pieces (Piece, around, hexOf, listPieces, piece, pieced, piecesResponse)-import Haskoin.Store.Web.Publish qualified as Publish-import Haskoin.Store.Web.Router hiding (header)-import Haskoin.Store.WebCommon-import Haskoin.Transaction-import Haskoin.Util-import NQE-  ( subscribe,-    unsubscribe,-  )-import Network.HTTP.Types-  ( ResponseHeaders,-    Status (..),-    hContentType,-    methodGet,-    methodPost,-    parseQuery,-    renderStdMethod,-    status200,-    status202,-    status203,-    status204,-    status400,-    status404,-    status409,-    status410,-    status413,-    status500,-    status503,-    statusIsClientError,-    statusIsServerError,-  )-import Network.Socket (Socket)-import Network.Wai-  ( Application,-    Middleware,-    Request (..),-    RequestBodyLength (..),-    Response,-    getRequestBodyChunk,-    mapResponseHeaders,-    modifyResponse,-    responseBuilder,-    responseLBS,-    responseStatus,-    responseStream,-    setRequestBodyChunks,-  )-import Network.Wai.Handler.Warp-  ( defaultSettings,-    runSettings,-    runSettingsSocket,-    setHost,-    setPort,-  )-import Network.Wai.Handler.WarpTLS (runTLS, runTLSSocket, tlsSettings)-import Network.Wai.Handler.WebSockets (websocketsOr)-import Network.Wai.Middleware.Autohead (autohead)-import Network.Wai.Middleware.Gzip (gzip)-import Network.Wai.Middleware.Gzip qualified as Gzip-import Network.WebSockets-  ( acceptRequest,-    defaultConnectionOptions,-    pendingRequest,-    rejectRequestWith,-    requestPath,-    sendTextData,-  )-import Network.WebSockets qualified as WebSockets-import Network.WebSockets.Connection.PingPong-  ( PingPongOptions (..),-    defaultPingPongOptions,-    withPingPong,-  )-import Network.Wreq qualified as Wreq-import Network.Wreq.Session as Wreq (Session)-import Network.Wreq.Session qualified as Wreq.Session-import Numeric (showFFloat)-import Numeric.Natural (Natural)-import System.Metrics.StatsD-import Text.Read (readMaybe)-import UnliftIO-  ( TVar,-    atomically,-    bracket,-    bracket_,-    catch,-    catchAny,-    checkSTM,-    Exception,-    fromException,-    handleAny,-    link,-    atomicModifyIORef',-    newIORef,-    newTQueueIO,-    newTVarIO,-    onException,-    race_,-    readIORef,-    readTQueue,-    readTVar,-    readTVarIO,-    registerDelay,-    stateTVar,-    throwIO,-    timeout,-    try,-    tryAny,-    withAsync,-    writeIORef,-    writeTQueue,-    writeTVar,-  )-import UnliftIO.Concurrent (getNumCapabilities, threadDelay)---- | Limits of the API. A value of @0@ means no limit.-data WebLimits = WebLimits-  { -- | maximum number of items in a list response-    maxItemCount :: !Word32,-    -- | maximum number of full transactions in a list response-    maxFullItemCount :: !Word32,-    -- | maximum offset of a paged query-    maxOffset :: !Word32,-    -- | number of items in a list response when no limit is given-    defItemCount :: !Word32,-    -- | consecutive unused addresses after which xpub derivation stops-    xpubGap :: !Word32,-    -- | unused addresses to look at for an xpub with no transactions-    xpubGapInit :: !Word32,-    -- | seconds without a new transaction before the store is reported-    -- unhealthy-    txTimeout :: !Word64,-    -- | seconds without a new block before the store is reported unhealthy-    blockTimeout :: !Word64,-    -- | seconds that a request may take before the log has a warning-    -- about it ('logIt')-    slowRequest :: !Word32-  }-  deriving (Eq, Show)---- | 200000 items, 5000 full transactions, offset up to 50000, 100 items by--- default, xpub gaps of 32 and 20, transaction timeout of 30 minutes,--- block timeout of four hours, and a warning for requests of five seconds.-instance Default WebLimits where-  def =-    WebLimits-      { maxItemCount = 200000,-        maxFullItemCount = 5000,-        maxOffset = 50000,-        defItemCount = 100,-        xpubGap = 32,-        xpubGapInit = 20,-        txTimeout = 3600 `div` 2,-        blockTimeout = 4 * 3600,-        slowRequest = 5-      }---- | Configuration of the web server.-data WebConfig = WebConfig-  { -- | address to listen on, such as @*@ for all-    host :: !String,-    -- | port to listen on-    port :: !Int,-    -- | running store to serve-    store :: !Store,-    -- | blocks that the database may lag behind the headers before the store-    -- is reported unhealthy-    maxLaggingBlocks :: !Int,-    -- | transactions that may be waiting to be imported before the store is-    -- reported unhealthy-    maxPendingTxs :: !Int,-    -- | fewest peers before the store is reported unhealthy-    minPeers :: !Int,-    -- | limits of the API-    limits :: !WebLimits,-    -- | version reported by the health check-    version :: !String,-    -- | the mempool is not indexed, so do not report it in health checks-    noMempool :: !Bool,-    -- | StatsD client, if metrics are enabled-    stats :: !(Maybe Stats),-    -- | microseconds between updates of the price ticker-    tickerRefresh :: !Int,-    -- | URL of the price ticker service-    tickerURL :: !String,-    -- | URL of the historical price service-    priceHistoryURL :: !String,-    -- | disable the xpub endpoints-    noXPub :: !Bool,-    -- | disable the Blockchain.info compatible endpoints-    noBlockchainInfo :: !Bool,-    -- | seconds between health checks; there is one at each new block-    -- too-    healthCheckInterval :: !Int,-    -- | log every request, at the level of information ('logIt')-    accessLog :: !Bool,-    -- | URL of the JSON-RPC interface of a full node, with its user and-    -- password, to publish transactions through ('sendNode'); 'Nothing'-    -- to publish through peers only-    nodeRpc :: !(Maybe String),-    -- | a certificate and its private key, in files in PEM format, to-    -- serve HTTPS instead of HTTP; 'Nothing' for HTTP, which is the-    -- default-    tls :: !(Maybe (FilePath, FilePath)),-    -- | log messages here-    logger :: !Logger-  }---- | State shared by all request handlers.-data WebState = WebState-  { -- | configuration-    config :: !WebConfig,-    -- | latest prices by currency code-    ticker :: !(TVar (HashMap Text BinfoTicker)),-    -- | StatsD metrics, if enabled-    metrics :: !(Maybe WebMetrics),-    -- | HTTP session for price requests-    session :: !Wreq.Session,-    -- | result of the latest health check-    health :: !(TVar HealthCheck),-    -- | the queries to answer with, the cache included ('webQuery'): those-    -- of one view of the database, taken for the request-    query :: !StoreRead,-    -- | the same queries without the cache, on the same view-    direct :: !StoreRead,-    -- | whether Redis answered the last time it was asked; always 'True'-    -- for a store without a cache-    cacheUp :: !(TVar Bool),-    -- | the events that the clients of @\/events@ read ('feedEvents')-    eventLog :: !EventLog,-    -- | the transactions that were handed over and that the store asks-    -- for until it has them ("Haskoin.Store.Web.Publish")-    asking :: !Publish.Asking-  }---- | StatsD metrics of the web server: a timing for each kind of request,--- under @web.\<name\>@, error counters and a gauge of connected event--- clients.-data WebMetrics = WebMetrics-  { -- Addresses-    addressTx :: !StatTiming,-    addressTxFull :: !StatTiming,-    addressBalance :: !StatTiming,-    addressUnspent :: !StatTiming,-    xpub :: !StatTiming,-    xpubDelete :: !StatTiming,-    xpubTxFull :: !StatTiming,-    xpubTx :: !StatTiming,-    xpubBalance :: !StatTiming,-    xpubUnspent :: !StatTiming,-    -- Transactions-    tx :: !StatTiming,-    txRaw :: !StatTiming,-    txAfter :: !StatTiming,-    txBlock :: !StatTiming,-    txBlockRaw :: !StatTiming,-    txPost :: !StatTiming,-    mempool :: !StatTiming,-    -- Blocks-    block :: !StatTiming,-    blockRaw :: !StatTiming,-    -- Blockchain-    binfoMultiaddr :: !StatTiming,-    binfoBalance :: !StatTiming,-    binfoAddressRaw :: !StatTiming,-    binfoUnspent :: !StatTiming,-    binfoTxRaw :: !StatTiming,-    binfoBlock :: !StatTiming,-    binfoBlockHeight :: !StatTiming,-    binfoBlockLatest :: !StatTiming,-    binfoBlockRaw :: !StatTiming,-    binfoMempool :: !StatTiming,-    binfoExportHistory :: !StatTiming,-    -- Blockchain /q endpoints-    binfoQaddresstohash :: !StatTiming,-    binfoQhashtoaddress :: !StatTiming,-    binfoQaddrpubkey :: !StatTiming,-    binfoQpubkeyaddr :: !StatTiming,-    binfoQhashpubkey :: !StatTiming,-    binfoQgetblockcount :: !StatTiming,-    binfoQlatesthash :: !StatTiming,-    binfoQbcperblock :: !StatTiming,-    binfoQtxtotalbtcoutput :: !StatTiming,-    binfoQtxtotalbtcinput :: !StatTiming,-    binfoQtxfee :: !StatTiming,-    binfoQtxresult :: !StatTiming,-    binfoQgetreceivedbyaddress :: !StatTiming,-    binfoQgetsentbyaddress :: !StatTiming,-    binfoQaddressbalance :: !StatTiming,-    binfoQaddressfirstseen :: !StatTiming,-    -- Errors-    serverErrors :: StatCounter,-    clientErrors :: StatCounter,-    -- Others-    health :: !StatTiming,-    peers :: !StatTiming,-    db :: !StatTiming,-    events :: !StatGauge-  }---- | Register the web server metrics with a StatsD client.-createMetrics :: Stats -> IO WebMetrics-createMetrics s = do-  -- Addresses-  addressTx <- d "address_transactions"-  addressTxFull <- d "address_transactions_full"-  addressBalance <- d "address_balance"-  addressUnspent <- d "address_unspent"-  xpub <- d "xpub"-  xpubDelete <- d "xpub_delete"-  xpubTxFull <- d "xpub_transactions_full"-  xpubTx <- d "xpub_transactions"-  xpubBalance <- d "xpub_balances"-  xpubUnspent <- d "xpub_unspent"--  -- Transactions-  tx <- d "transaction"-  txRaw <- d "transaction_raw"-  txAfter <- d "transaction_after"-  txPost <- d "transaction_post"-  txBlock <- d "transactions_block"-  txBlockRaw <- d "transactions_block_raw"-  mempool <- d "mempool"--  -- Blocks-  block <- d "block"-  blockRaw <- d "block_raw"--  -- Blockchain-  binfoMultiaddr <- d "blockchain_multiaddr"-  binfoBalance <- d "blockchain_balance"-  binfoAddressRaw <- d "blockchain_rawaddr"-  binfoUnspent <- d "blockchain_unspent"-  binfoTxRaw <- d "blockchain_rawtx"-  binfoBlock <- d "blockchain_blocks"-  binfoBlockHeight <- d "blockchain_block_height"-  binfoBlockLatest <- d "blockchain_latestblock"-  binfoBlockRaw <- d "blockchain_rawblock"-  binfoMempool <- d "blockchain_mempool"-  binfoExportHistory <- d "blockchain_export_history"--  -- Blockchain /q endpoints-  binfoQaddresstohash <- d "blockchain_q_addresstohash"-  binfoQhashtoaddress <- d "blockchain_q_hashtoaddress"-  binfoQaddrpubkey <- d "blockckhain_q_addrpubkey"-  binfoQpubkeyaddr <- d "blockchain_q_pubkeyaddr"-  binfoQhashpubkey <- d "blockchain_q_hashpubkey"-  binfoQgetblockcount <- d "blockchain_q_getblockcount"-  binfoQlatesthash <- d "blockchain_q_latesthash"-  binfoQbcperblock <- d "blockchain_q_bcperblock"-  binfoQtxtotalbtcoutput <- d "blockchain_q_txtotalbtcoutput"-  binfoQtxtotalbtcinput <- d "blockchain_q_txtotalbtcinput"-  binfoQtxfee <- d "blockchain_q_txfee"-  binfoQtxresult <- d "blockchain_q_txresult"-  binfoQgetreceivedbyaddress <- d "blockchain_q_getreceivedbyaddress"-  binfoQgetsentbyaddress <- d "blockchain_q_getsentbyaddress"-  binfoQaddressbalance <- d "blockchain_q_addressbalance"-  binfoQaddressfirstseen <- d "blockchain_q_addressfirstseen"--  -- Errors-  serverErrors <- c "server_errors"-  clientErrors <- c "client_errors"--  -- Others-  health <- d "health"-  peers <- d "peers"-  db <- d "dbstats"--  events <- g "events_connected"-  return WebMetrics {..}-  where-    d x = newStatTiming s ("web." <> x) 10-    g x = newStatGauge s ("web." <> x) 0-    c x = newStatCounter s ("web." <> x) 10---- | Increase a gauge while an action runs.-withGaugeIO :: StatGauge -> IO a -> IO a-withGaugeIO g = bracket_ (incrementGauge g 1) (decrementGauge g 1)---- | Increase a gauge of the web metrics while a handler runs.-withGaugeIncrease ::-  WebState ->-  (WebMetrics -> StatGauge) ->-  IO a ->-  IO a-withGaugeIncrease st gf go =-  case st.metrics of-    Nothing -> go-    Just m -> withGaugeIO (gf m) go---- | Record how long a handler takes in a timing of the web metrics.-withMetrics ::-  WebState ->-  (WebMetrics -> StatTiming) ->-  IO a ->-  IO a-withMetrics st df go =-  case st.metrics of-    Nothing -> go-    Just m -> bracket time (stop m) (const go)-  where-    time = round . (* 1000) <$> getPOSIXTime-    stop m t1 = do-      t2 <- time-      addTiming (df m) (t2 - t1)------------------------ Path Handlers -------------------------- | Run the web server forever on the host and port of the configuration.--- Also starts a thread that updates prices, unless the Blockchain.info--- endpoints are disabled, and a thread that runs health checks.------ = HTTPS------ With a certificate and a key in 'tls', the server speaks HTTPS on that--- port, and nothing else: TLS 1.2 and 1.3, and HTTP\/2 for clients that--- ask for it. It is off by default. A public service usually sits behind--- a proxy that has the certificate, limits requests and spreads them over--- servers; this is for a server by itself, and for links between machines--- that should not be in the clear. The files are read when the server--- starts, so a renewed certificate takes a restart.-runWeb :: WebConfig -> IO ()-runWeb config = serveWeb config $ case config.tls of-  Nothing -> runSettings settings-  Just (certificate, key) -> runTLS (tlsSettings certificate key) settings-  where-    settings = setPort config.port . setHost (fromString config.host) $ defaultSettings---- | Like 'runWeb', on a socket that already listens; the host and port of the--- configuration are ignored. Binding port 0 and passing the socket gives a--- free port with no window in which another program can take it, which--- finding a free port first and binding it later has.-runWebSocket :: WebConfig -> Socket -> IO ()-runWebSocket config sock = serveWeb config $ case config.tls of-  Nothing -> runSettingsSocket defaultSettings sock-  Just (certificate, key) -> runTLSSocket (tlsSettings certificate key) defaultSettings sock---- | Run the web server with a function of Warp that serves an application.--- The application is the router ("Haskoin.Store.Web.Router") with the--- routes of 'handlePaths', behind the middleware for WebSocket events, for--- logs and for the header that lets any origin make requests.-serveWeb :: WebConfig -> (Application -> IO ()) -> IO ()-serveWeb config serve = do-  ticker <- newTVarIO HashMap.empty-  metrics <- mapM createMetrics config.stats-  session <- Wreq.Session.newAPISession-  first <- runHealthCheck-  health <- newTVarIO first-  -- A member of a fleet takes no turn before its store is known to be-  -- well. A member is made ready to take turns, since a store without a-  -- web server has nothing else to say when; with one, a store that had-  -- just started took a turn before its first check, reported the blocks-  -- that it had then, and with nobody else there moved the fleet to them.-  forM_ config.store.fleet $ \member -> atomically (writeTVar member.healthy (isOK first))-  cacheUp <- cacheCheck >>= newTVarIO-  eventLog <- newEventLog-  asking <- Publish.newAsking-  let db = config.store.db-      stateOf q = WebState {query = webQuery config.store cacheUp q, direct = q, ..}-      routesOf s = map (guarded s) (concatMap withPost (handlePaths s config))-      -- outside of a request, each query has a view of its own; a request-      -- has one view for all that it reads, which is let go when its-      -- response is sent-      st = stateOf (L.standaloneRead db)-      local answer = L.withView db (answer . routesOf . stateOf . L.layeredRead)-      scoped answer = case config.store.fleet of-        Nothing -> local answer-        Just member ->-          readTVarIO cacheUp >>= \case-            True -> shared member (3 :: Int) False answer-            -- without Redis there is no fleet to answer with-            False -> local answer-      -- as a member of a fleet: from the view at the fleet tip, with the-      -- pool that the fleet shares on top. The tip is taken to be the one-      -- that the member read at its last turn, which saves asking Redis-      -- for it: the first read of the pool checks it, and a request that-      -- reads nothing from the pool checks it when its handler is done. If-      -- it is not the tip any more, the request is answered again, from the-      -- tip that Redis has.-      shared member tries fresh answer = do-        known <- if fresh then return Nothing else fmap encode <$> readTVarIO member.tip-        asked <- case known of-          Just tip -> return (Just (Just tip))-          Nothing -> timeout 2000000 (memberTip member) `catchAny` \_ -> return Nothing-        case asked of-          Nothing -> atomically (writeTVar cacheUp False) >> local answer-          -- no store has reported yet-          Just Nothing -> local answer-          Just (Just tip) -> do-            handled <- newIORef False-            let marked confirm r = r {handler = \req -> r.handler req <* confirm <* writeIORef handled True} :: Route-                -- extended public keys through the fleet's cache of-                -- them, unless a request asks for the database-                inFleet plain cached confirm =-                  answer (map (marked confirm) (routesOf WebState {query = cached, direct = plain, ..}))-            try (answeringCached member tip inFleet) >>= \case-              Right (Just sent) -> return sent-              Right Nothing-                -- a tip from the member's last turn of which there is no-                -- view may not be the tip any more-                | not fresh && tries > 0 -> shared member (tries - 1) True answer-                | otherwise -> behind answer-              Left TipMoved ->-                -- the tip moved while the handler was reading: answer-                -- again from the new one, unless the response is on its way-                readIORef handled >>= \case-                  True -> throwIO TipMoved-                  False-                    | tries > 0 -> shared member (tries - 1) True answer-                    | otherwise -> behind answer-      behind answer = answer [r {handler = behindResponse st} :: Route | r <- routes]-      -- the stream of events never ends, and reads nothing from the-      -- database: answered like the rest it would hold a view, and the-      -- data under it, for as long as its client listens-      app request-        | pathInfo request == ["events"] = routerScoped routes ($ routes) notFound request-        | otherwise = routerScoped routes scoped notFound request-      routes = routesOf st-      notFound _ NoRoute = errorResponse st ThingNotFound-      notFound _ (Unknown ks) =-        errorResponse st . UserError $-          "Unknown parameter: " <> T.unpack (T.intercalate ", " ks)-  saidWell <- newIORef Nothing-  saidRedis <- newIORef True-  pinged <- newIORef 0-  -- set when the store has a new best block, for the health check to run-  -- at once-  newBlock <- newTVarIO False-  withAsync (priceUpdater session ticker) $ \_a1 ->-    withAsync (healthCheckLoop health cacheUp saidWell saidRedis pinged newBlock) $ \_a2 ->-      withAsync (feedEvents st) $ \a3 ->-        withAsync (askingLoop st) $ \_a4 ->-          link a3 >> do-            let logged = logIt config.logger config.accessLog config.limits.slowRequest-            serve . webSocketEvents st . logged . anyOrigin . preflight . autohead . compressed . cappedBody st $ app-  where-    priceUpdater session =-      unless config.noBlockchainInfo-        . price-          (withSource "Price" config.logger)-          config.store.net-          session-          config.tickerURL-          config.tickerRefresh-    runHealthCheck = healthCheck config-    cacheCheck = case (config.store.cache, config.store.fleet) of-      (Just cache, _) -> cacheAlive cache-      (_, Just member) -> memberAlive member-      _ -> return True-    -- The check is run every second by default, and at once when the-    -- store has a new best block, which is when its answer changes most:-    -- /health answers from the last result, and a load balancer or a-    -- fleet that acts on a result of half a minute ago sends requests to-    -- a server that fell behind. A check reads a handful of keys, so it-    -- costs little to run often; but a store that is catching up imports-    -- thousands of small blocks a second, and a check for each would be-    -- work for nothing, so there is a tenth of a second between two.-    healthCheckLoop v up saidWell saidRedis pinged newBlock =-      bracket (subscribe config.store.pub (heard newBlock)) (unsubscribe config.store.pub) $ \_ ->-        loopForever $ do-          threadDelay 100000-          due <- registerDelay (config.healthCheckInterval * 1000 * 1000)-          atomically $ do-            block <- readTVar newBlock-            time <- readTVar due-            checkSTM (block || time)-            writeTVar newBlock False-          h <- runHealthCheck-          atomically (writeTVar v h)-          tellHealth saidWell h-          -- a store whose checks fail stops taking its turns as a member-          -- of its fleet, so that it does not hold the others back-          forM_ config.store.fleet $ \member -> atomically (writeTVar member.healthy (isOK h))-          -- Redis is asked less often, every five seconds: it is the one-          -- part of the check that leaves the program, and waits up to-          -- two seconds for a Redis that is away. Requests find out by-          -- themselves that Redis is away; this is how the store finds-          -- out that it is back.-          now <- getPOSIXTime-          last' <- readIORef pinged-          when (now - last' >= 5) $ do-            writeIORef pinged now-            redis <- cacheCheck-            atomically (writeTVar up redis)-            tellRedis saidRedis redis-    heard newBlock = \case-      StoreBestBlock _ -> writeTVar newBlock True-      _ -> return ()-    -- A change of health is logged, and nothing while it stays as it was:-    -- what /health answers is otherwise only known to whoever asks it. A-    -- store is not healthy while it syncs, which is no cause for alarm,-    -- so only a store that was healthy gets a warning.-    health' = withSource "Health" config.logger-    tellHealth said h = do-      before <- readIORef said-      writeIORef said (Just (isOK h))-      case (before, isOK h) of-        (Just True, False) -> logWarn health' ("No longer healthy: " <> ailments h)-        (Nothing, False) -> logInfo health' ("Not healthy yet: " <> ailments h)-        (Just False, True) -> logInfo health' "Healthy again"-        (Nothing, True) -> logInfo health' "Healthy"-        _ -> return ()-    -- the same for the Redis that a store with a cache answers through;-    -- /health says 203 without it-    tellRedis said redis = do-      before <- readIORef said-      writeIORef said redis-      case (before, redis) of-        (True, False) -> logWarn health' "Redis does not answer: answering from the database"-        (False, True) -> logInfo health' "Redis answers again"-        _ -> return ()-    -- what a browser asks before a request from another site that is not-    -- of the plainest kind: which methods and headers it may use. Without-    -- an answer it does not send the request.-    preflight inner req respond-      | requestMethod req == "OPTIONS" =-          respond $-            responseLBS-              status204-              [ ("Allow", allowed),-                ("Access-Control-Allow-Methods", allowed),-                ("Access-Control-Allow-Headers", "Accept, Content-Type"),-                ("Access-Control-Max-Age", "86400")-              ]-              ""-      | otherwise = inner req respond-    allowed = "GET, HEAD, POST, DELETE, OPTIONS"-    -- answers are compressed for a client that says it reads gzip: JSON-    -- shrinks several times over. The events are a stream that has to-    -- leave as it is written, and are left alone.-    compressed inner req-      | pathInfo req == ["events"] = inner req-      | otherwise = gzip Gzip.defaultGzipSettings inner req-    -- any origin may ask, and whether the answer is compressed depends on-    -- what the client said it reads, which a cache between the client and-    -- the server has to know: it must not give gzip to a client that-    -- cannot read it.-    anyOrigin = modifyResponse (mapResponseHeaders varied)-    varied headers =-      [("Access-Control-Allow-Origin", "*"), ("Vary", "Accept-Encoding")]-        <> filter ((/= "Vary") . fst) headers---- | Answer the errors of the handler of a route: an 'Except' with its--- status and JSON, any other error as a server error, after logging it. A--- request passed on to the next route goes on.-guarded :: WebState -> Route -> Route-guarded st r = r {handler = \req -> r.handler req `catchAny` answer req}-  where-    answer req e-      | Just Next <- fromException e = throwIO Next-      | Just TipMoved <- fromException e = throwIO TipMoved-      | Just x <- fromException e = errorResponse st x-      | Just g <- fromException e = goneResponse st g-      | otherwise = do-          logError st.config.logger $ fmtReq req.wai <> ": " <> cs (show e)-          errorResponse st ServerError---- | A transaction that a block replaced ('Replaced'), asked for by its id:--- answered 410, Gone, with the transaction that replaced it and the height--- of its block, where one that never was is 404.------ The errors of the API are those of haskoin-store-data, whose client--- reads them. This one is not among them yet, so it is answered here, in--- the same form: the client takes it for an error it does not know.-data Gone = Gone !TxHash !Replaced-  deriving (Show)--instance Exception Gone---- | The answer to 'Gone', counted with the errors of clients.-goneResponse :: WebState -> Gone -> IO Response-goneResponse st (Gone txid r) = do-  forM_ st.metrics $ \m -> incrementCounter m.clientErrors 1-  return . responseLBS status410 [(hContentType, "application/json")] . A.encode $-    A.object-      [ "error" .= ("replaced" :: Text),-        "message"-          .= T.unwords-            [ "Transaction",-              txHashToHex txid,-              "was replaced by",-              txHashToHex r.by,-              "in the block at height",-              T.pack (show r.height)-            ],-        "replaced-by" .= r.by,-        "height" .= r.height-      ]---- | Not found: 'Gone' if a block replaced the transaction, else 404.-notFoundTx :: WebState -> TxHash -> IO a-notFoundTx st txid = st.query.getReplaced txid >>= maybe (raise st ThingNotFound) (throwIO . Gone txid)---- | What a store that is a member of a fleet answers when it has no view of--- the block that the fleet answers from: 503, so that a load balancer sends--- the request to another store. Any answer of its own could differ from--- those of the other stores, which is what a fleet is there to prevent.-behindResponse :: WebState -> Req -> IO Response-behindResponse st _ = errorResponse st BehindFleet---- | The response to an error, in JSON, counted in the metrics.-errorResponse :: WebState -> Except -> IO Response-errorResponse st e = do-  forM_ st.metrics $ \m ->-    mapM_ (`incrementCounter` 1) $-      if-        | statusIsClientError (errStatus e) -> Just m.clientErrors-        | statusIsServerError (errStatus e) -> Just m.serverErrors-        | otherwise -> Nothing-  return $ responseLBS (errStatus e) ((hContentType, "application/json") : again) (A.encode e)-  where-    -- a server behind its fleet catches up within a block or is taken out-    -- of rotation: a client or a load balancer that reads this header-    -- tries again in a second, and elsewhere meanwhile-    again = [("Retry-After", "1") | e == BehindFleet]---- | Historical prices of the coin of the network in a currency at several times. Returns an empty list on failure.-getRates ::-  Logger ->-  Network ->-  Wreq.Session ->-  String ->-  Text ->-  [Word64] ->-  IO [BinfoRate]-getRates logger net session url currency times = do-  handleAny err $ do-    r <--      Wreq.asJSON-        =<< Wreq.Session.postWith opts session url body-    return $ r ^. Wreq.responseBody-  where-    err _ = do-      logWarn logger "Could not get historic prices"-      return []-    body = toJSON times-    base =-      Wreq.defaults-        & Wreq.param "base" .~ [T.toUpper $ T.pack net.name]-    opts = base & Wreq.param "quote" .~ [currency]---- | Update the prices of the coin of the network periodically, for BTC and BCH only.-price ::-  Logger ->-  Network ->-  Wreq.Session ->-  String ->-  Int ->-  TVar (HashMap Text BinfoTicker) ->-  IO ()-price logger net session url pget v = forM_ purl $ \u -> loopForever $ do-  let err e = logWarn logger $ "Could not get prices: " <> cs (show e)-  handleAny err $ do-    r <- Wreq.asJSON =<< Wreq.Session.get session u-    atomically . writeTVar v $ r ^. Wreq.responseBody-  threadDelay pget-  where-    purl = case code of-      Nothing -> Nothing-      Just x -> Just (url <> "?base=" <> x)-      where-        code-          | net == btc = Just "btc"-          | net == bch = Just "bch"-          | otherwise = Nothing---- | Read no more of the body of a transaction to publish than a--- transaction can be long: twice 'Publish.maxTxSize' in hexadecimal, and--- some room for white space. More is answered 413, by its length when the--- client gave it, and else as soon as that much was read.------ The router reads the body whole before it knows what to do with it.--- Without this, anyone could have the server hold a body of any size.--- Other requests with a body are forms of parameters, and are not capped--- here.-cappedBody :: WebState -> Middleware-cappedBody st app request respond-  | requestMethod request == methodPost,-    filter (not . T.null) (pathInfo request) == ["transactions"] =-      case requestBodyLength request of-        KnownLength n | n > fromIntegral most -> refuse-        _ -> do-          left <- newIORef most-          let chunk = do-                bytes <- getRequestBodyChunk request-                now <- atomicModifyIORef' left (\l -> let l' = l - C.length bytes in (l', l'))-                when (now < 0) (throwIO RequestTooLarge)-                return bytes-          app (setRequestBodyChunks chunk request) respond `catch` \case-            RequestTooLarge -> refuse-            e -> throwIO e-  | otherwise = app request respond-  where-    most = maxPublishBody st.config.store.net-    refuse = errorResponse st RequestTooLarge >>= respond---- | The most bytes of the body of @POST \/transactions@.-maxPublishBody :: Network -> Int-maxPublishBody net = 2 * Publish.maxTxSize net + 4096---- | Stop handling the request, and answer with an error ('guarded').-raise :: WebState -> Except -> IO a-raise _ = throwIO---- | HTTP status of an error.-errStatus :: Except -> Status-errStatus ThingNotFound = status404-errStatus BadRequest = status400-errStatus UserError {} = status400-errStatus StringError {} = status400-errStatus ServerError = status500-errStatus TxIndexConflict {} = status409-errStatus ServerTimeout = status500-errStatus RequestTooLarge = status413-errStatus BehindFleet = status503---- | A route, and with it the same route under @POST@ if it takes a list in--- a parameter: addresses, blocks, heights or transactions.------ A list in the query string has to fit in a URL, and proxies cut those--- off at a few thousand characters: some hundred addresses. The router--- reads the parameters of a form in the body as it reads those of the--- query string, so under @POST@ the same parameters go in the body, with--- @Content-Type: application\/x-www-form-urlencoded@, and have no such--- limit. @\/transactions@ is left out: @POST@ there publishes a--- transaction.-withPost :: Route -> [Route]-withPost r-  | r.method == methodGet && takesList && r.pattern /= [Left "transactions"] = [r, r {method = methodPost}]-  | otherwise = [r]-  where-    takesList = any (`elem` r.accepts) ["addresses", "blocks", "heights", "txids"]---- | The parameters of @\/events@ that say what a client hears about from--- the start ('eventsOpen').-eventParams :: [Text]-eventParams = ["quiet", "addresses", "xpub", "derive", "filter", "functions", "tweak", "flags"]---- | The routes of every endpoint, except those disabled in the configuration.-handlePaths :: WebState -> WebConfig -> [Route]-handlePaths st cfg =-  [ marshalled ["notx"] (\r -> GetBlockBest <$> dflt r) (handleBlockBest st),-    marshalledList ["notx"] (\r -> GetBlockHeight <$> cap r <*> dflt r) (handleBlockHeight st),-    marshalled ["notx"] (\r -> GetBlockTime <$> cap r <*> dflt r) (handleBlockTime st),-    marshalled ["notx"] (\r -> GetBlockMTP <$> cap r <*> dflt r) (handleBlockMTP st),-    marshalled ["notx"] (\r -> GetBlock <$> cap r <*> dflt r) (handleBlock st),-    marshalled [] (\r -> GetTx <$> cap r) (handleTx st),-    pieces (.txRaw) [] (\r -> GetTxRaw <$> cap r) (handleTxRaw st),-    pathPosted [] (\r -> PostTx <$> parseTxBody st r) (handlePostTx st),-    plainList limits (\r -> GetAddrTxs <$> cap r <*> lim r) (handleAddrTxs st),-    marshalled [] (\r -> GetAddrBalance <$> cap r) (handleAddrBalance st),-    marshalledList limits (\r -> GetAddrUnspent <$> cap r <*> lim r) (handleAddrUnspent st),-    plainList [] (\_ -> return GetPeers) (handlePeers st),-    pathHealth st,-    plainList ["limit", "offset"] (\r -> GetMempool <$> opt r <*> off r) (handleMempool st),-    marshalledList ["blocks", "notx"] (\r -> GetBlocks <$> need r <*> dflt r) (handleBlocks st),-    pieces (.blockRaw) [] (\r -> GetBlockRaw <$> cap r) (handleBlockRaw st),-    pieces (.blockRaw) [] (\_ -> return GetBlockBestRaw) (handleBlockBestRaw st),-    marshalledList ["notx"] (\r -> GetBlockLatest <$> dflt r) (handleBlockLatest st),-    marshalledList ["heights", "notx"] (\r -> GetBlockHeights <$> need r <*> dflt r) (handleBlockHeights st),-    pieces (.blockRaw) [] (\r -> GetBlockHeightRaw <$> cap r) (handleBlockHeightRaw st),-    pieces (.blockRaw) [] (\r -> GetBlockTimeRaw <$> cap r) (handleBlockTimeRaw st),-    pieces (.blockRaw) [] (\r -> GetBlockMTPRaw <$> cap r) (handleBlockMTPRaw st),-    marshalledList ["txids"] (\r -> GetTxs <$> need r) (handleTxs st),-    pieces (.txRaw) ["txids"] (\r -> GetTxsRaw <$> need r) (handleTxsRaw st),-    marshalledPieces (.txBlock) [] (\r -> GetTxsBlock <$> cap r) (handleTxsBlock st),-    pieces (.txBlockRaw) [] (\r -> GetTxsBlockRaw <$> cap r) (handleTxsBlockRaw st),-    plain [] (\r -> GetTxAfter <$> cap r <*> cap r) (handleTxAfter st),-    plainList ("addresses" : limits) (\r -> GetAddrsTxs <$> need r <*> lim r) (handleAddrsTxs st),-    marshalledPieces (.addressTxFull) limits (\r -> GetAddrTxsFull <$> cap r <*> lim r) (handleAddrTxsFull st),-    marshalledPieces-      (.addressTxFull)-      ("addresses" : limits)-      (\r -> GetAddrsTxsFull <$> need r <*> lim r)-      (handleAddrsTxsFull st),-    marshalledList ["addresses"] (\r -> GetAddrsBalance <$> need r) (handleAddrsBalance st),-    marshalledList ("addresses" : limits) (\r -> GetAddrsUnspent <$> need r <*> lim r) (handleAddrsUnspent st),-    get "/events" ("after" : eventParams) (handleEvents st),-    get "/dbstats" [] (handleDbStats st),-    -- the reference of the API, as a page and as the file that it shows-    get "/" [] (\_ -> file "text/html; charset=utf-8" swaggerPage),-    get "/swagger.yaml" [] (\_ -> file "application/yaml" swaggerYaml)-  ]-    <> (if cfg.noXPub then [] else xpubRoutes)-    <> (if cfg.noBlockchainInfo then [] else binfoRoutes)-  where-    file kind bytes = return (responseLBS status200 [(hContentType, kind)] (L.fromStrict bytes))-    net = cfg.store.net-    -- the parameters that routes accept, beyond those of each route-    limits = ["limit", "offset", "height"]-    active = ["active", "activeP2SH", "activeBech32"]-    -- the Blockchain.info endpoints accept format=json ('jsonOnly')-    multiaddrParams =-      ["format", "onlyShow", "n", "offset", "noCompact", "no_compact", "currency", "cashaddr", "txidindex", "filter"]-        <> active-    rawaddrParams = ["format", "derive", "limit", "offset", "txidindex"]-    balanceParams = "format" : "cashaddr" : active-    unspentParams = "format" : "limit" : "confirmations" : "txidindex" : active-    historyParams = "format" : "start" : "end" : "currency" : active-    -- the parameters of a request-    cap, need :: (Param a) => Req -> IO a-    cap = paramCapture st-    need = paramRequired st-    dflt :: (Default a, Param a) => Req -> IO a-    dflt = paramDef st-    opt :: (Param a) => Req -> IO (Maybe a)-    opt = paramOptional st-    lim = parseLimits st-    off = parseOffset st-    -- routes by how they encode their responses: in the JSON of the-    -- network, as a list of those, or in plain JSON-    marshalled ::-      (ApiResource a b, MarshalJSON Network b) => [Text] -> (Req -> IO a) -> (a -> IO b) -> Route-    marshalled ps p h = pathCompact ps p h (marshalEncoding net)-    marshalledList ::-      (ApiResource a (SerialList b), MarshalJSON Network b) => [Text] -> (Req -> IO a) -> (a -> IO [b]) -> Route-    marshalledList ps p h =-      pathCompact ps p (fmap SerialList . h) (list (marshalEncoding net) . (.get))-    -- routes of what is answered while it is read: a list of items in-    -- the JSON of the network, or pieces that the handler made-    marshalledPieces ::-      (ApiResource a (SerialList b), MarshalJSON Network b) =>-      (WebMetrics -> StatTiming) ->-      [Text] ->-      (Req -> IO a) ->-      (a -> IO [IO [b]]) ->-      Route-    marshalledPieces df ps p h = pieces df ps p (listPieces (marshalEncoding net) <=< h)-    pieces ::-      (ApiResource a b) =>-      (WebMetrics -> StatTiming) ->-      [Text] ->-      (Req -> IO a) ->-      (a -> IO [Piece]) ->-      Route-    pieces = pathPieces st-    plain :: (ApiResource a b, ToJSON b) => [Text] -> (Req -> IO a) -> (a -> IO b) -> Route-    plain ps p h = pathCompact ps p h toEncoding-    plainList ::-      (ApiResource a (SerialList b), ToJSON b) => [Text] -> (Req -> IO a) -> (a -> IO [b]) -> Route-    plainList ps p h = plain ps p (fmap SerialList . h)-    xpubRoutes =-      [ plain ["derive", "nocache"] (\r -> GetXPub <$> cap r <*> dflt r <*> dflt r) (handleXPub st),-        plainList-          ("derive" : "nocache" : limits)-          (\r -> GetXPubTxs <$> cap r <*> dflt r <*> lim r <*> dflt r)-          (handleXPubTxs st),-        marshalledPieces-          (.xpubTxFull)-          ("derive" : "nocache" : limits)-          (\r -> GetXPubTxsFull <$> cap r <*> dflt r <*> lim r <*> dflt r)-          (handleXPubTxsFull st),-        marshalledList-          ["derive", "nocache"]-          (\r -> GetXPubBalances <$> cap r <*> dflt r <*> dflt r)-          (handleXPubBalances st),-        marshalledList-          ("derive" : "nocache" : limits)-          (\r -> GetXPubUnspent <$> cap r <*> dflt r <*> lim r <*> dflt r)-          (handleXPubUnspent st),-        plain ["derive"] (\r -> DelCachedXPub <$> cap r <*> dflt r) (handleDelXPub st)-      ]-    binfoRoutes =-      [ post "/blockchain/multiaddr" multiaddrParams (jsonOnly (handleMultiAddr st)),-        get "/blockchain/multiaddr" multiaddrParams (jsonOnly (handleMultiAddr st)),-        get "/blockchain/balance" balanceParams (jsonOnly (handleShortBal st)),-        post "/blockchain/balance" balanceParams (jsonOnly (handleShortBal st)),-        get "/blockchain/rawaddr/:addr" rawaddrParams (jsonOnly (handleRawAddr st)),-        get "/blockchain/address/:addr" rawaddrParams (jsonOnly (handleRawAddr st)),-        get "/blockchain/xpub/:addr" rawaddrParams (jsonOnly (handleRawAddr st)),-        post "/blockchain/unspent" unspentParams (jsonOnly (handleBinfoUnspent st)),-        get "/blockchain/unspent" unspentParams (jsonOnly (handleBinfoUnspent st)),-        get "/blockchain/rawtx/:txid" ["format", "txidindex"] (handleBinfoTx st),-        get "/blockchain/rawblock/:block" ["format", "txidindex"] (handleBinfoBlock st),-        get "/blockchain/latestblock" ["format", "txidindex"] (jsonOnly (handleBinfoLatest st)),-        get-          "/blockchain/unconfirmed-transactions"-          ["format", "limit", "offset", "txidindex"]-          (jsonOnly (handleBinfoMempool st)),-        get "/blockchain/block-height/:height" ["format", "txidindex"] (jsonOnly (handleBinfoBlockHeight st)),-        get "/blockchain/blocks/:milliseconds" ["format"] (jsonOnly (handleBinfoBlocksDay st)),-        get "/blockchain/export-history" historyParams (jsonOnly (handleBinfoHistory st)),-        post "/blockchain/export-history" historyParams (jsonOnly (handleBinfoHistory st)),-        get "/blockchain/q/addresstohash/:addr" [] (handleBinfoAddrToHash st),-        get "/blockchain/q/hashtoaddress/:hash" [] (handleBinfoHashToAddr st),-        get "/blockchain/q/addrpubkey/:pubkey" [] (handleBinfoAddrPubkey st),-        get "/blockchain/q/pubkeyaddr/:addr" [] (handleBinfoPubKeyAddr st),-        get "/blockchain/q/hashpubkey/:pubkey" [] (handleBinfoHashPubkey st),-        get "/blockchain/q/getblockcount" [] (handleBinfoGetBlockCount st),-        get "/blockchain/q/latesthash" [] (handleBinfoLatestHash st),-        get "/blockchain/q/bcperblock" [] (handleBinfoSubsidy st),-        get "/blockchain/q/txtotalbtcoutput/:txid" [] (handleBinfoTotalOut st),-        get "/blockchain/q/txtotalbtcinput/:txid" [] (handleBinfoTotalInput st),-        get "/blockchain/q/txfee/:txid" [] (handleBinfoTxFees st),-        get "/blockchain/q/txresult/:txid/:addr" [] (handleBinfoTxResult st),-        get "/blockchain/q/getreceivedbyaddress/:addr" [] (handleBinfoReceived st),-        get "/blockchain/q/getsentbyaddress/:addr" [] (handleBinfoSent st),-        get "/blockchain/q/addressbalance/:addr" [] (handleBinfoAddrBalance st),-        get "/blockchain/q/addressfirstseen/:addr" [] (handleFirstSeen st)-      ]---- | A handler of the Blockchain.info endpoints that answer in JSON only. They--- accept @format=json@, which clients written for Blockchain.info often--- send, and refuse any other format rather than answering in JSON anyway.-jsonOnly :: (Req -> IO Response) -> Req -> IO Response-jsonOnly h req = do-  paramOr req json "format" ()-  h req-  where-    json "json" = Just ()-    json _ = Nothing---- | The route of an API resource: parse the request, run the handler, and--- encode the response as JSON.-pathCompact :: (ApiResource a b) => [Text] -> (Req -> IO a) -> (a -> IO b) -> (b -> Encoding) -> Route-pathCompact ps parser action encJson =-  route (renderStdMethod (resourceMethod proxy)) (capturePath proxy) ps $ \req -> do-    res <- action =<< parser req-    return $ responseLBS status200 jsonType (encodingToLazyByteString (encJson res))-  where-    toProxy :: (Req -> IO a) -> Proxy a-    toProxy = const Proxy-    proxy = toProxy parser---- | The route of an API resource whose handler chooses the status of its--- answer: @POST \/transactions@, which answers 202 for a transaction that--- it handed over.-pathPosted :: (ApiResource a b) => [Text] -> (Req -> IO a) -> (a -> IO (Status, Encoding)) -> Route-pathPosted ps parser action =-  route (renderStdMethod (resourceMethod proxy)) (capturePath proxy) ps $ \req -> do-    (status, res) <- action =<< parser req-    return $ responseLBS status jsonType (encodingToLazyByteString res)-  where-    toProxy :: (Req -> IO a) -> Proxy a-    toProxy = const Proxy-    proxy = toProxy parser---- | The route of an API resource that is answered while it is read, as--- "Haskoin.Store.Web.Pieces" explains: the handler gives the bytes of the--- response in pieces.-pathPieces ::-  (ApiResource a b) =>-  WebState ->-  (WebMetrics -> StatTiming) ->-  [Text] ->-  (Req -> IO a) ->-  (a -> IO [Piece]) ->-  Route-pathPieces st df ps parser action =-  route (renderStdMethod (resourceMethod proxy)) (capturePath proxy) ps $ \req ->-    inPieces st df jsonType (action =<< parser req)-  where-    toProxy :: (Req -> IO a) -> Proxy a-    toProxy = const Proxy-    proxy = toProxy parser---- | A response in pieces, timed. The timing counts the whole response, the--- time that the client takes to read it included.-inPieces :: WebState -> (WebMetrics -> StatTiming) -> ResponseHeaders -> IO [Piece] -> IO Response-inPieces st df headers pieces = do-  began <- now-  let done = forM_ st.metrics $ \m -> now >>= \ended -> addTiming (df m) (ended - began)-  n <- encoders-  (pieces >>= piecesResponse n headers done) `onException` done-  where-    now = round . (* 1000) <$> getPOSIXTime :: IO Int---- | How many pieces of a response are read and encoded at once: one for--- each core, so that a long answer is encoded on all of them, but no more--- than eight, which bounds what a response holds to some nine pieces, and--- leaves cores to other requests on a large machine.-encoders :: IO Int-encoders = min 8 <$> getNumCapabilities---- | The headers of a response in JSON.-jsonType :: ResponseHeaders-jsonType = [(hContentType, "application/json")]---- | Respond with JSON.-streamEncoding :: Encoding -> IO Response-streamEncoding e =-  return $-    responseLBS status200 [(hContentType, "application/json; charset=utf-8")] (encodingToLazyByteString e)---- | Respond with text.-textResponse :: TL.Text -> IO Response-textResponse t =-  return $-    responseLBS status200 [(hContentType, "text/plain; charset=utf-8")] (TL.encodeUtf8 t)---- GET Block / GET Blocks ------ | Handler for @GET \/block\/:block@: a block by hash.-handleBlock ::-  WebState ->-  GetBlock ->-  IO BlockData-handleBlock st (GetBlock h (NoTx noTx)) = withMetrics st (.block) $ do-  st.query.getBlock h >>= \case-    Nothing ->-      raise st ThingNotFound-    Just b -> do-      return $ pruneTx noTx b---- | Blocks by hash, without duplicates, leaving out unknown ones.-getBlocks ::-  WebState ->-  [H.BlockHash] ->-  Bool ->-  IO [BlockData]-getBlocks st hs notx =-  (pruneTx notx <$>) . catMaybes <$> mapM (st.query.getBlock) (nub hs)---- | Handler for @GET \/blocks?blocks=…@: several blocks by hash.-handleBlocks ::-  WebState ->-  GetBlocks ->-  IO [BlockData]-handleBlocks st (GetBlocks hs (NoTx notx)) =-  withMetrics st (.block) $-    getBlocks st hs notx---- | Keep only the coinbase in the list of transactions of a block, if the flag is 'True'.-pruneTx :: Bool -> BlockData -> BlockData-pruneTx False b = b-pruneTx True BlockData {..} = BlockData {txs = take 1 txs, ..}---- GET BlockRaw ------ | Handler for @GET \/block\/:block\/raw@: a block by hash, serialized as in the peer-to-peer protocol.-handleBlockRaw :: WebState -> GetBlockRaw -> IO [Piece]-handleBlockRaw st (GetBlockRaw h) = rawResult <$> getRawBlock st h---- | The pieces of a block by hash, serialized, in hexadecimal. Responds--- with an error if not found.-getRawBlock :: WebState -> H.BlockHash -> IO [Piece]-getRawBlock st h =-  st.query.getBlock h >>= maybe (raise st ThingNotFound) (return . rawBlock st.query)---- | The pieces of a block serialized as in the peer-to-peer protocol, in--- hexadecimal: its header and the number of its transactions, then its--- transactions as they are read. The first piece has transactions too, so--- that a response does not begin before something of them was read.-rawBlock :: StoreRead -> BlockData -> [Piece]-rawBlock q b =-  around (hexOf b.header <> hexOf (VarInt (fromIntegral (length b.txs)))) mempty $-    map piece (pieced pieceSize (fmap (foldMap rawHex . catMaybes) . q.getManyTxBytes) b.txs)---- | The hex of bytes as stored, such as a transaction in the format of the--- network: read as they are, and written with the table of haskoin-core's--- 'hexBuilder'.-rawHex :: ByteString -> Builder-rawHex = hexBuilder . L.fromStrict---- | The pieces of the JSON of a 'RawResult', from those of the hexadecimal.-rawResult :: [Piece] -> [Piece]-rawResult = around "{\"result\":\"" "\"}"---- | The pieces of the JSON of a 'RawResultList', from those of the--- hexadecimal of each of its items.-rawResultList :: [[Piece]] -> [Piece]-rawResultList items =-  around "[" "]" (concat [around (comma <> "\"") "\"" item | (comma, item) <- zip ("" : repeat ",") items])---- GET BlockBest / BlockBestRaw ------ | Handler for @GET \/block\/best@: the best block.-handleBlockBest ::-  WebState ->-  GetBlockBest ->-  IO BlockData-handleBlockBest st (GetBlockBest (NoTx notx)) =-  withMetrics st (.block) $-    (bestBlock st.query) >>= maybe (raise st ThingNotFound) (return . pruneTx notx)---- | Handler for @GET \/block\/best\/raw@: the best block, serialized.-handleBlockBestRaw :: WebState -> GetBlockBestRaw -> IO [Piece]-handleBlockBestRaw st _ =-  st.query.getBestBlock >>= \case-    Nothing -> raise st ThingNotFound-    Just h -> rawResult <$> getRawBlock st h---- GET BlockLatest ------ | Handler for @GET \/block\/latest@: the best block and its ancestors, up to 100 blocks.-handleBlockLatest ::-  WebState ->-  GetBlockLatest ->-  IO [BlockData]-handleBlockLatest st (GetBlockLatest (NoTx noTx)) =-  withMetrics st (.block) $-    st.query.getBestBlock >>= maybe (raise st ThingNotFound) (go [] <=< st.query.getBlock)-  where-    go acc Nothing = return $ reverse acc-    go acc (Just b)-      | b.height <= 0 = return $ reverse acc-      | length acc == 99 = return . reverse $ pruneTx noTx b : acc-      | otherwise = go (pruneTx noTx b : acc) =<< st.query.getBlock b.header.prev---- GET BlockHeight / BlockHeights / BlockHeightRaw ------ | Handler for @GET \/block\/height\/:height@: the blocks at a height.-handleBlockHeight ::-  WebState ->-  GetBlockHeight ->-  IO [BlockData]-handleBlockHeight st (GetBlockHeight h (NoTx notx)) =-  withMetrics st (.block) $-    st.query.getBlocksAtHeight (fromIntegral h)-      >>= (\hs -> getBlocks st hs notx)---- | Handler for @GET \/block\/heights?heights=…@: the blocks at several heights.-handleBlockHeights ::-  WebState ->-  GetBlockHeights ->-  IO [BlockData]-handleBlockHeights st (GetBlockHeights (HeightsParam heights) (NoTx notx)) =-  withMetrics st (.block) $-    mapM (st.query.getBlocksAtHeight . fromIntegral) heights-      >>= (\hs -> getBlocks st hs notx) . concat---- | Handler for @GET \/block\/height\/:height\/raw@: the blocks at a height, serialized.-handleBlockHeightRaw :: WebState -> GetBlockHeightRaw -> IO [Piece]-handleBlockHeightRaw st (GetBlockHeightRaw h) =-  fmap rawResultList $-    mapM (getRawBlock st) =<< st.query.getBlocksAtHeight (fromIntegral h)---- GET BlockTime / BlockTimeRaw ------ | Handler for @GET \/block\/time\/:time@: the last block with a timestamp at or before a time.-handleBlockTime ::-  WebState ->-  GetBlockTime ->-  IO BlockData-handleBlockTime st (GetBlockTime (TimeParam t) (NoTx notx)) =-  withMetrics st (.block) $ do-    let ch = st.config.store.chain-    C.blockAtOrBefore st.query ch t >>= \case-      Nothing -> raise st ThingNotFound-      Just b -> return $ pruneTx notx b---- | Handler for @GET \/block\/mtp\/:time@: the first block with a median time past at or after a time.-handleBlockMTP ::-  WebState ->-  GetBlockMTP ->-  IO BlockData-handleBlockMTP st (GetBlockMTP (TimeParam t) (NoTx notx)) =-  withMetrics st (.block) $ do-    let ch = st.config.store.chain-    C.blockAtOrAfterMTP st.query ch t >>= \case-      Nothing -> raise st ThingNotFound-      Just b -> return $ pruneTx notx b---- | Handler for @GET \/block\/time\/:time\/raw@: the last block with a timestamp at or before a time, serialized.-handleBlockTimeRaw :: WebState -> GetBlockTimeRaw -> IO [Piece]-handleBlockTimeRaw st (GetBlockTimeRaw (TimeParam t)) =-  C.blockAtOrBefore st.query st.config.store.chain t >>= \case-    Nothing -> raise st ThingNotFound-    Just b -> return (rawResult (rawBlock st.query b))---- | Handler for @GET \/block\/mtp\/:time\/raw@: the first block with a median time--- past at or after a time, serialized.-handleBlockMTPRaw :: WebState -> GetBlockMTPRaw -> IO [Piece]-handleBlockMTPRaw st (GetBlockMTPRaw (TimeParam t)) =-  C.blockAtOrAfterMTP st.query st.config.store.chain t >>= \case-    Nothing -> raise st ThingNotFound-    Just b -> return (rawResult (rawBlock st.query b))---- GET Transactions ------ | Handler for @GET \/transaction\/:txid@: a transaction.-handleTx :: WebState -> GetTx -> IO Transaction-handleTx st (GetTx txid) =-  withMetrics st (.tx) $-    C.getTransaction st.query txid >>= maybe (notFoundTx st txid) return---- | Handler for @GET \/transactions?txids=…@: several transactions.-handleTxs ::-  WebState ->-  GetTxs ->-  IO [Transaction]-handleTxs st (GetTxs txids) =-  withMetrics st (.tx) $-    catMaybes <$> C.getTransactions st.query (nub txids)---- | Handler for @GET \/transaction\/:txid\/raw@: a transaction, serialized.-handleTxRaw ::-  WebState ->-  GetTxRaw ->-  IO [Piece]-handleTxRaw st (GetTxRaw txid) =-  st.query.getManyTxBytes [txid] >>= \case-    [Just bytes] -> return (rawResult [piece (return (rawHex bytes))])-    _ -> notFoundTx st txid---- | Handler for @GET \/transactions\/raw?txids=…@: several transactions, serialized. The client names--- them, so they are all read before the answer begins, in one piece.-handleTxsRaw ::-  WebState ->-  GetTxsRaw ->-  IO [Piece]-handleTxsRaw st (GetTxsRaw txids) = do-  found <- catMaybes <$> st.query.getManyTxBytes (nub txids)-  return [piece (return (listOf (map quoted found)))]-  where-    listOf items = "[" <> mconcat (intersperse "," items) <> "]"-    quoted bytes = "\"" <> rawHex bytes <> "\""---- | The hashes of the transactions of a block. Responds with an error if--- not found.-getTxsBlock :: WebState -> H.BlockHash -> IO [TxHash]-getTxsBlock st h = st.query.getBlock h >>= maybe (raise st ThingNotFound) (return . (.txs))---- | Handler for @GET \/transactions\/block\/:block@: the transactions of a block, in pieces.-handleTxsBlock :: WebState -> GetTxsBlock -> IO [IO [Transaction]]-handleTxsBlock st (GetTxsBlock h) = transactionPieces st.query <$> getTxsBlock st h---- | Handler for @GET \/transactions\/block\/:block\/raw@: the transactions of a block, serialized, in pieces.-handleTxsBlockRaw :: WebState -> GetTxsBlockRaw -> IO [Piece]-handleTxsBlockRaw st (GetTxsBlockRaw h) =-  listPieces hex . pieced pieceSize (fmap catMaybes . st.query.getManyTxBytes) =<< getTxsBlock st h-  where-    hex bytes = unsafeToEncoding ("\"" <> rawHex bytes <> "\"")---- GET TransactionAfterHeight ------ | Handler for @GET \/transaction\/:txid\/after\/:height@: whether any ancestor--- of a transaction is a coinbase confirmed after a height.-handleTxAfter ::-  WebState ->-  GetTxAfter ->-  IO (GenericResult (Maybe Bool))-handleTxAfter st (GetTxAfter txid height) =-  withMetrics st (.txAfter) $-    GenericResult . fst <$> (cbAfterHeight st.query (fromIntegral height) txid)---- | Check if any of the ancestors of this transaction is a coinbase after the--- specified height. Returns 'Nothing' if answer cannot be computed before--- hitting limits.-cbAfterHeight ::-  StoreRead ->-  H.BlockHeight ->-  TxHash ->-  IO (Maybe Bool, Int)-cbAfterHeight q height txid =-  inputs n HashSet.empty HashSet.empty [txid]-  where-    n = 10000-    inputs 0 _ _ [] = return (Nothing, 10000)-    inputs i is ns [] =-      let is' = HashSet.union is ns-          ns' = HashSet.empty-          ts = HashSet.toList (HashSet.difference ns is)-       in case ts of-            [] -> return (Just False, n - i)-            _ -> inputs i is' ns' ts-    inputs i is ns (t : ts) =-      C.getTransaction q t >>= \case-        Nothing -> return (Nothing, n - i)-        Just tx-          | height_check tx ->-              if cb_check tx-                then return (Just True, n - i + 1)-                else-                  let ns' = HashSet.union (ins tx) ns-                   in inputs (i - 1) is ns' ts-          | otherwise -> inputs (i - 1) is ns ts-    cb_check = any isCoinbase . (.inputs)-    ins = HashSet.fromList . map (.outpoint.hash) . (.inputs)-    height_check tx =-      case tx.block of-        BlockRef h _ -> h > height-        MemRef _ -> True---- POST Transaction ------ | Handler for @POST \/transactions@: check the transaction and hand it--- over ("Haskoin.Store.Web.Publish"). The answer is 202 with its id and a--- message once it was handed over, over the RPC of a node or to the peers,--- and 200 with its id for one that the store has in a block. What the--- store or a node refuses is 400 with the reason; no peer is 500.-handlePostTx :: WebState -> PostTx -> IO (Status, Encoding)-handlePostTx st (PostTx tx) =-  withMetrics st (.txPost) $-    Publish.publish (publishing st) tx >>= \case-      Publish.InBlock -> return (status200, toEncoding (TxId txid))-      Publish.ToNode -> handed "A full node took the transaction."-      Publish.ToPeers n ->-        handed . T.unwords $-          [ "Handed over to",-            T.pack (show n),-            if n == 1 then "peer." else "peers.",-            "The network decides whether to take it."-          ]-      Publish.Invalid why -> raise st (UserError (show why))-      Publish.Refused why ->-        raise st (UserError ("A node refused the transaction: " <> T.unpack why))-      Publish.Unverified i ->-        raise st . UserError $-          "This server cannot verify the signature of input "-            <> show i-            <> ", and sends to its peers only what it has verified"-      Publish.NoPeers -> raise st ServerError-  where-    txid = txHash tx-    handed message = return (status202, A.pairs ("txid" .= txid <> "message" .= (message :: Text)))---- | The chain as the store has it, for the checks of a transaction to--- publish: the height of the best block, and the median of the times of--- the last eleven blocks.-chainRules :: WebState -> IO Publish.Rules-chainRules st = do-  best <- st.query.getBestBlock >>= maybe (return Nothing) st.query.getBlock-  times <- maybe (return []) (back (11 :: Int)) best-  return-    Publish.Rules-      { net = own.net,-        inputs = \tx -> judgeInputs own.net own.ctx tx . map Just,-        height = maybe 0 (.height) best,-        medianTime = case Data.List.sort times of-          [] -> 0-          sorted -> fromIntegral (sorted !! (length sorted `div` 2))-      }-  where-    own = st.config.store-    back n b-      | n <= 1 = return [b.header.timestamp]-      | otherwise =-          (b.header.timestamp :) <$> (st.query.getBlock b.header.prev >>= maybe (return []) (back (n - 1)))---- | Give a transaction to a full node over its JSON-RPC interface--- (@sendrawtransaction@).------ The node runs every script and holds the transaction to its own rules--- and its own mempool, says why it refuses, and does not close the--- connections of the store over what it refuses, as it may over a--- transaction that a peer sends it. A transaction that the node takes--- comes back to the store as any other: the node announces it on the--- connection that the store syncs from.------ It is an option because the store needs nothing of a node but to be--- its peer, and this needs an account on one. A node that does not--- answer, or answers what cannot be read, is 'Publish.Unavailable': the--- transaction then goes to the peers, if the store verified it. A--- transaction that the node has already was sent.-sendNode :: WebState -> String -> Tx -> IO Publish.Direct-sendNode st url tx =-  tryAny (Wreq.Session.postWith opts st.session url body) >>= \case-    Left e -> unheard (cs (show e))-    Right r -> case A.decode (r ^. Wreq.responseBody) >>= answer of-      Nothing -> unheard ("HTTP " <> cs (show (r ^. Wreq.responseStatus . Wreq.statusCode)))-      Just Nothing -> return Publish.Sent-      Just (Just why)-        | "already" `T.isInfixOf` why -> return Publish.Sent-        | otherwise -> return (Publish.Declined why)-  where-    opts = Wreq.defaults & Wreq.checkResponse ?~ (\_ _ -> return ())-    body =-      A.object-        [ "jsonrpc" .= ("1.0" :: Text),-          "id" .= ("haskoin-store" :: Text),-          "method" .= ("sendrawtransaction" :: Text),-          "params" .= [encodeHex (runPutS (serialize tx))]-        ]-    -- the error of the answer if it has one, nothing if it has a result-    answer v = do-      A.Object top <- Just v-      case KeyMap.lookup "error" top of-        Just (A.Object e) | Just (A.String why) <- KeyMap.lookup "message" e -> Just (Just why)-        Just A.Null | KeyMap.member "result" top -> Just Nothing-        Nothing | KeyMap.member "result" top -> Just Nothing-        _ -> Nothing-    unheard why = do-      logWarn-        (withSource "Publish" st.config.logger)-        ("The node did not take a transaction over RPC, which goes to a peer if it is verified: " <> why)-      return Publish.Unavailable---- | What "Haskoin.Store.Web.Publish" needs, for a request.------ The peers are those of the store that take transactions. A transaction--- that was handed over is asked for again after the answer--- ('askingLoop'), unless the store keeps no mempool.-publishing :: WebState -> Publish.Publishing-publishing st =-  Publish.Publishing-    { rules = chainRules st,-      direct = maybe (\_ -> return Publish.Unavailable) (sendNode st) st.config.nodeRpc,-      holds = fmap heldOf . st.query.getTxData,-      coin = Publish.coinOf st.query,-      peers = map (.address) . filter (maybe False (.relay) . (.version)) <$> getPeers own.peerMgr,-      courier =-        Publish.deliver-          Publish.Courier-            { net = own.net,-              connect = own.connect,-              patience = 10,-              logger = withSource "Courier" st.config.logger-            },-      fetch = if st.config.noMempool then \_ -> return () else Publish.askFor st.asking,-      note = \level -> logAt level (withSource "Publish" st.config.logger)-    }-  where-    own = st.config.store-    heldOf = \case-      Just td-        | td.deleted -> Publish.Absent-        | confirmed td.block -> Publish.Confirmed-        | otherwise -> Publish.Pending-      Nothing -> Publish.Absent---- | Ask for the transactions that were handed over until the store has--- them ("Haskoin.Store.Web.Publish".'Publish.Asking'): as if each peer had--- announced them, so that the block store asks one of them. A failure is--- logged and the asking goes on.-askingLoop :: WebState -> IO ()-askingLoop st = forever $ do-  tryAny (Publish.askLoop st.asking present ask) >>= \case-    Left e -> logWarn (withSource "Publish" st.config.logger) ("Asking for transactions: " <> cs (show e))-    Right () -> return ()-  threadDelay 1000000-  where-    own = st.config.store-    present h = L.withView own.db $ \v -> maybe False (not . (.deleted)) <$> (L.layeredRead v).getTxData h-    ask h = getPeers own.peerMgr >>= mapM_ (\op -> blockStoreTxHash op.mailbox [h] own.block)---- GET Mempool / Events ------ | Handler for @GET \/mempool@: hashes of mempool transactions, newest first.-handleMempool ::-  WebState ->-  GetMempool ->-  IO [TxHash]-handleMempool st (GetMempool limitM (OffsetParam o)) =-  withMetrics st (.mempool) $ do-    WebLimits {..} <- pure st.config.limits-    let wl' = WebLimits {maxItemCount = 0, ..}-        l = Limits (validateLimit wl' False limitM) (fromIntegral o) Nothing-    map snd <$> st.query.getMempoolPage l---- | Middleware that serves @\/events@ over a WebSocket: each new block and--- each transaction that enters or leaves the mempool is a message, from the--- log of events ("Haskoin.Store.Web.Events").------ The client may send the commands of t'EventCommand' to choose the--- transactions that it hears about. Each side speaks JSON in text frames;--- a binary frame is a command that cannot be read. A command that the server cannot read--- or cannot do closes the connection with the reason, as a node does with--- an SPV client: there are no answers to wait for.------ @?after=@ with the hash of the last event heard takes up from there--- ('eventsStart'). Clients that do not answer pings, or that fall off the--- start of the log, are disconnected.-webSocketEvents :: WebState -> Middleware-webSocketEvents s =-  websocketsOr defaultConnectionOptions (wrap . events)-  where-    pingPong = defaultPingPongOptions {pingInterval = 30, pongTimeout = 60}-    gauge = (.events) <$> s.metrics-    net = s.config.store.net-    ctx = s.config.store.ctx-    gap = s.query.maxGap-    wrap f = case gauge of-      Nothing -> f-      Just g -> withGaugeIO g f-    events pending = do-      let request = pendingRequest pending-          (path, query) = C.break (== '?') (requestPath request)-          asked = parseQuery query-      if path /= "/events"-        then turnAway pending ThingNotFound-        else-          eventsOpen s (\name -> T.decodeUtf8Lenient <$> join (lookup (T.encodeUtf8 name) asked)) >>= \case-            Left e -> turnAway pending e-            Right (start, watch) -> do-              conn <- acceptRequest pending-              -- websockets only sees pongs while reading, which the-              -- thread that hears the commands does-              withPingPong pingPong conn $ \c ->-                race_ (hear c watch) (tell c watch start) `catch` gone-    -- a client that leaves is not an error to report-    gone :: WebSockets.ConnectionException -> IO ()-    gone _ = return ()-    -- the commands of the client, until one cannot be done-    hear c watch = do-      message <- WebSockets.receiveDataMessage c-      let read' = case message of-            WebSockets.Text bytes _ -> A.decode bytes >>= parseMaybe (unmarshalValue (net, ctx))-            WebSockets.Binary _ -> Nothing-      case read' of-        Nothing -> refuse c "Could not read the command"-        Just cmd ->-          try (command ctx gap s.query.xPubBals watch cmd) >>= \case-            Right (Right ()) -> hear c watch-            Right (Left why) -> refuse c why-            Left (_ :: Except) -> refuse c "Could not do the command"-    -- 1008: the message is against what the server allows-    refuse c why = WebSockets.sendCloseCode c 1008 (T.pack why)-    -- the events from a place in the log on, those that the client chose-    tell c watch from =-      atomically (readEvents s.eventLog from) >>= \case-        Nothing -> WebSockets.sendClose c ("Event queue full" :: Text)-        Just entries -> do-          forM_ entries $ \e -> do-            wanted <- atomically (stateTVar watch (relevant ctx gap e.seen))-            when wanted $ sendTextData c (A.encode e.event)-          tell c watch (from + fromIntegral (length entries))-    turnAway pending e =-      rejectRequestWith-        pending-        WebSockets.defaultRejectRequest-          { WebSockets.rejectBody = L.toStrict (A.encode e),-            WebSockets.rejectCode = statusCode (errStatus e),-            WebSockets.rejectMessage = statusMessage (errStatus e),-            WebSockets.rejectHeaders = [("Content-Type", "application/json")]-          }---- | Handler for @GET \/events@: stream of new blocks and of transactions--- that enter or leave the mempool, one per line in JSON, until the client--- falls off the start of the log.--- @after@ takes up from an event ('eventsStart').-handleEvents :: WebState -> Req -> IO Response-handleEvents st req = do-  let asked name = lookup name req.params-  (start, watch) <- eventsOpen st asked >>= either (raise st) return-  return . responseStream status200 [(hContentType, "application/json")] $ \write flush' ->-    let send from = do-          flush'-          -- ends the response when the client falls too far behind-          atomically (readEvents st.eventLog from) >>= \case-            Nothing -> return ()-            Just entries -> do-              forM_ entries $ \e -> do-                wanted <- atomically (stateTVar watch (relevant own.ctx st.query.maxGap e.seen))-                when wanted (write (serial e.event))-              send (from + fromIntegral (length entries))-     in withGaugeIncrease st (.events) (send start)-  where-    serial e = fromEncoding (toEncoding e) <> "\n"-    own = st.config.store---- | Where a client of the events starts reading the log, and what it hears--- about from the start, as the parameters of its request say: 'eventsStart'--- for @after@, and for the rest a choice made of the same commands that a--- WebSocket client sends later.------ A WebSocket client could only choose once it was connected, and until its--- first command was read it was sent every transaction of a busy mempool.--- With these the choice is there before the first event:------ * @quiet=true@: no transactions, only blocks, until something is watched;--- * @addresses@: a list of addresses to watch, separated by commas;--- * @xpub@ and @derive@: an extended public key and the types of its---   addresses, one or several separated by commas, legacy if left out;--- * @filter@, @functions@, @tweak@, @flags@: a Bloom filter, in hex, with---   its numbers.------ Any of them but @quiet=false@ makes the client a filtered one. The HTTP--- stream takes them too, and it is the only way that it has to choose.--- The place in the log is taken first, so that nothing that happens while--- a key is read from the database is left out.-eventsOpen :: WebState -> (Text -> Maybe Text) -> IO (Either Except (Word64, TVar Watch))-eventsOpen st asked =-  eventsStart st (asked "after") >>= \case-    Left e -> return (Left e)-    Right start -> case commands of-      Left why -> return (Left (UserError why))-      Right cmds -> do-        watch <- newTVarIO everything-        let run [] = return (Right (start, watch))-            run (cmd : rest) =-              command own.ctx st.query.maxGap st.query.xPubBals watch cmd >>= \case-                Left why -> return (Left (UserError why))-                Right () -> run rest-        run cmds-  where-    own = st.config.store-    commands = do-      quiet <- maybe (Right False) (need "quiet" . readBool) (asked "quiet")-      addresses <- mapM (need "addresses" . textToAddr own.net) (listed "addresses")-      derivs <- mapM (need "derive" . textToDeriveType) (listed "derive")-      keys <- mapM (need "xpub" . xPubImport own.net own.ctx) (maybe [] pure (asked "xpub"))-      filters <- case asked "filter" of-        Nothing -> Right []-        Just hex -> do-          let number name = maybe (Right 0) (need name . (readNum :: Text -> Maybe Word32)) (asked name)-          functions <- number "functions"-          tweak <- number "tweak"-          flags <- number "flags"-          let json =-                A.object-                  [ "type" A..= A.String "filterload",-                    "filter" A..= hex,-                    "functions" A..= functions,-                    "tweak" A..= tweak,-                    "flags" A..= flags-                  ]-          pure <$> need "filter" (parseMaybe (unmarshalValue (own.net, own.ctx)) json)-      when (null keys && not (null derivs)) (Left "Parameter derive needs xpub")-      let types = if null derivs then [DeriveNormal] else derivs-      return $-        [EventNone | quiet]-          <> [EventWatch addresses | not (null addresses)]-          <> [EventWatchXPub XPubSpec {key, deriv} | key <- keys, deriv <- types]-          <> filters-    listed name = maybe [] (filter (not . T.null) . T.splitOn ",") (asked name)-    need :: Text -> Maybe a -> Either String a-    need name = maybe (Left ("Could not read parameter " <> T.unpack name)) Right---- | Where a client of the events starts reading the log: at the end, or--- after the event that it says it heard last, named by the hash of its--- block or transaction. 404 when the log has no event with that hash: the--- client cannot know what it missed, and has to read what it follows again--- and listen from the end.-eventsStart :: WebState -> Maybe Text -> IO (Either Except Word64)-eventsStart st = \case-  Nothing -> Right <$> atomically (eventsEnd st.eventLog)-  Just hex -> case (H.hexToBlockHash hex, hexToTxHash hex) of-    (Nothing, Nothing) -> return (Left (UserError "The parameter after is not a hash"))-    (block, tx) -> maybe (Left ThingNotFound) Right <$> atomically (eventsAfter st.eventLog block tx)---- | Write what happens into the log of events for as long as the server--- runs, each transaction that enters with what the database has of it--- ('seenOf'), for the clients that choose what they hear about.------ A store alone writes what it publishes. A member of a fleet writes what--- the fleet tells ('memberEvents'), from the start of what Redis keeps, so--- that the logs of all the servers of a fleet hold the same events and a--- client can take up on any of them.-feedEvents :: WebState -> IO ()-feedEvents st = case own.fleet of-  Just member -> memberEvents member True (mapM_ happened . fleet)-  Nothing -> do-    queue <- newTQueueIO-    let listen = mapM_ (writeTQueue queue) . local-    bracket (subscribe own.pub listen) (unsubscribe own.pub) $ \_ ->-      forever (atomically (readTQueue queue) >>= happened)-  where-    own = st.config.store-    local = \case-      StoreBestBlock b -> Just (Left b)-      StoreMempoolNew t -> Just (Right (t, True))-      StoreMempoolDelete t -> Just (Right (t, False))-      _ -> Nothing-    fleet = \case-      FleetBlock h -> Left <$> hash h-      FleetEntered h -> Right . (,True) <$> hash h-      FleetLeft h -> Right . (,False) <$> hash h-    hash :: (Serialize x) => C.ByteString -> Maybe x-    hash = either (const Nothing) Just . decode-    happened = \case-      Left b -> atomically (appendEvent st.eventLog (BlockArrived b))-      Right (t, False) -> atomically (appendEvent st.eventLog (TxLeft t))-      Right (t, True) -> do-        -- a transaction that cannot be read is told to every client-        known <- st.direct.getTxData t `catchAny` \_ -> return Nothing-        atomically (appendEvent st.eventLog (TxEntered t (seenOf own.net own.ctx <$> known)))---- GET Address Transactions ------ | Handler for @GET \/address\/:address\/transactions@: transactions of an address.-handleAddrTxs ::-  WebState ->-  GetAddrTxs ->-  IO [TxRef]-handleAddrTxs st (GetAddrTxs addr pLimits) =-  withMetrics st (.addressTx) $-    st.query.getAddressTxs addr =<< paramToLimits st False pLimits---- | Handler for @GET \/address\/transactions?addresses=…@: transactions of several addresses.-handleAddrsTxs ::-  WebState ->-  GetAddrsTxs ->-  IO [TxRef]-handleAddrsTxs st (GetAddrsTxs addrs pLimits) =-  withMetrics st (.addressTx) $-    st.query.getAddressesTxs addrs =<< paramToLimits st False pLimits---- | Handler for @GET \/address\/:address\/transactions\/full@: full transactions of an address, in the pieces--- of 'pathPieces'. Which transactions they are is read at once; their--- bodies are read piece by piece.-handleAddrTxsFull ::-  WebState ->-  GetAddrTxsFull ->-  IO [IO [Transaction]]-handleAddrTxsFull st (GetAddrTxsFull addr pLimits) = do-  txs <- st.query.getAddressTxs addr =<< paramToLimits st True pLimits-  return (transactionPieces st.query (map (.txid) txs))---- | Handler for @GET \/address\/transactions\/full?addresses=…@: full transactions of several addresses, in--- the pieces of 'pathPieces'.-handleAddrsTxsFull ::-  WebState ->-  GetAddrsTxsFull ->-  IO [IO [Transaction]]-handleAddrsTxsFull st (GetAddrsTxsFull addrs pLimits) = do-  txs <- st.query.getAddressesTxs addrs =<< paramToLimits st True pLimits-  return (transactionPieces st.query (map (.txid) txs))---- | Transactions by their hashes, in pieces of 'pieceSize' that are each--- read with one multi-get. The hashes are those that the database gave,--- of a block or of addresses, so the transactions are there; one that is--- gone by the time its piece is read is left out, as it is when the list--- is read at once.-transactionPieces :: StoreRead -> [TxHash] -> [IO [Transaction]]-transactionPieces q = pieced pieceSize (fmap catMaybes . C.getTransactions q)---- | Transactions in a piece of a list answered piece by piece. With 20 the--- first byte leaves a millisecond or two sooner and with 500 some ten--- later; the time of the whole response is the same with all three.-pieceSize :: Int-pieceSize = 100---- | Handler for @GET \/address\/:address\/balance@: balance of an address.-handleAddrBalance :: WebState -> GetAddrBalance -> IO Balance-handleAddrBalance st (GetAddrBalance addr) =-  withMetrics st (.addressBalance) $ C.getDefaultBalance st.query addr---- | Handler for @GET \/address\/balances?addresses=…@: balances of several addresses.-handleAddrsBalance ::-  WebState ->-  GetAddrsBalance ->-  IO [Balance]-handleAddrsBalance st (GetAddrsBalance addrs) =-  withMetrics st (.addressBalance) $ st.query.getBalances addrs---- | Handler for @GET \/address\/:address\/unspent@: unspent outputs of an address.-handleAddrUnspent ::-  WebState ->-  GetAddrUnspent ->-  IO [Unspent]-handleAddrUnspent st (GetAddrUnspent addr pLimits) =-  withMetrics st (.addressUnspent) $-    st.query.getAddressUnspents addr =<< paramToLimits st False pLimits---- | Handler for @GET \/address\/unspent?addresses=…@: unspent outputs of several addresses.-handleAddrsUnspent ::-  WebState ->-  GetAddrsUnspent ->-  IO [Unspent]-handleAddrsUnspent st (GetAddrsUnspent addrs pLimits) =-  withMetrics st (.addressUnspent) $-    st.query.getAddressesUnspents addrs =<< paramToLimits st False pLimits---- GET XPubs ------ | Handler for @GET \/xpub\/:xpub@: summary of an extended public key.-handleXPub ::-  WebState ->-  GetXPub ->-  IO XPubSummary-handleXPub st (GetXPub xpub deriv (NoCache noCache)) =-  withMetrics st (.xpub) $-    let xs = XPubSpec xpub deriv-     in xPubSummary xs <$> xPubBalsChecked st noCache xs---- | Handler for @DELETE \/xpub\/:xpub@: remove an extended public key from the cache. Returns 'True' if it was cached.-handleDelXPub ::-  WebState ->-  DelCachedXPub ->-  IO (GenericResult Bool)-handleDelXPub st (DelCachedXPub xpub deriv) =-  withMetrics st (.xpubDelete) $ do-    let c = st.config.store.cache-    r <- maybe (return False) (`cacheDelete` XPubSpec xpub deriv) c-    return $ GenericResult r---- | Balances of the addresses of an extended public key ('xPubBals'), from--- the cache unless told not to, refusing a wrong kind of address--- ('checkDerive').-xPubBalsChecked :: WebState -> Bool -> XPubSpec -> IO [XPubBal]-xPubBalsChecked st noCache xs = do-  bals <- (cacheOr noCache st).query.xPubBals xs-  checkDerive st xs bals-  return bals---- | Answer with an error if an extended public key has no transactions with--- the kind of address asked for, but has some with another kind: the request--- named the wrong kind.------ An xpub does not record which kind of address its wallet derives, and the--- same key gives different addresses for each kind, so a request with the--- wrong @derive@ would otherwise get an empty wallet: a valid answer, and a--- wrong one, that a wallet would show its user as a zero balance. An error--- says what went wrong and how to fix it.------ The check costs nothing for a wallet that has transactions. For one that--- looks unused, it reads the balances of the first addresses of each other--- kind, as many as the store reads to decide that an xpub is unused, in one--- read per kind. A wallet unused with every kind is a new wallet, and gets its--- empty answer. Networks without SegWit have one kind only, and no check.-checkDerive :: WebState -> XPubSpec -> [XPubBal] -> IO ()-checkDerive st xs bals =-  when (net.segWit && all (\b -> b.balance.txs == 0) bals) $ do-    used <- filterM usedAs others-    forM_ (take 1 used) $ \d ->-      raise st . UserError $-        unwords-          [ "This xpub has no transactions with derive=" <> name xs.deriv <> ",",-            "but has some with derive=" <> name d <> ";",-            "ask with derive=" <> name d <> "."-          ]-  where-    net = st.config.store.net-    ctx = st.config.store.ctx-    others = filter (/= xs.deriv) [DeriveNormal, DeriveP2SH, DeriveP2WPKH]-    name = T.unpack . deriveTypeToText-    usedAs d = do-      let external = pubSubKey ctx xs.key 0-          addrs =-            map snd . take (fromIntegral st.query.initialGap) $-              deriveAddresses (deriveFunction ctx d) external 0-      any (\b -> b.txs > 0) <$> st.query.getBalances addrs---- | Transactions of an extended public key, optionally without the cache.-getXPubTxs ::-  WebState ->-  XPubKey ->-  DeriveType ->-  LimitsParam ->-  Bool ->-  IO [TxRef]-getXPubTxs st xpub deriv plimits nocache = do-  limits <- paramToLimits st False plimits-  let xs = XPubSpec xpub deriv-      st' = cacheOr nocache st-  xbals <- xPubBalsChecked st nocache xs-  st'.query.xPubTxs xs xbals limits---- | Handler for @GET \/xpub\/:xpub\/transactions@: transactions of an extended public key.-handleXPubTxs ::-  WebState ->-  GetXPubTxs ->-  IO [TxRef]-handleXPubTxs st (GetXPubTxs xpub deriv plimits (NoCache nocache)) =-  withMetrics st (.xpubTx) $-    getXPubTxs st xpub deriv plimits nocache---- | Handler for @GET \/xpub\/:xpub\/transactions\/full@: full transactions of an extended public key, in pieces.-handleXPubTxsFull :: WebState -> GetXPubTxsFull -> IO [IO [Transaction]]-handleXPubTxsFull st (GetXPubTxsFull xpub deriv plimits (NoCache nocache)) =-  transactionPieces (cacheOr nocache st).query . map (.txid)-    <$> getXPubTxs st xpub deriv plimits nocache---- | Handler for @GET \/xpub\/:xpub\/balances@: balances of the used addresses of an extended public key.-handleXPubBalances ::-  WebState ->-  GetXPubBalances ->-  IO [XPubBal]-handleXPubBalances st (GetXPubBalances xpub deriv (NoCache noCache)) =-  withMetrics st (.xpubBalance) $-    xPubBalsChecked st noCache (XPubSpec xpub deriv)---- | Handler for @GET \/xpub\/:xpub\/unspent@: unspent outputs of an extended public key.-handleXPubUnspent ::-  WebState ->-  GetXPubUnspent ->-  IO [XPubUnspent]-handleXPubUnspent st (GetXPubUnspent xpub deriv pLimits (NoCache noCache)) =-  withMetrics st (.xpubUnspent) $ do-    limits <- paramToLimits st False pLimits-    let xspec = XPubSpec xpub deriv-        st' = cacheOr noCache st-    xbals <- xPubBalsChecked st noCache xspec-    st'.query.xPubUnspents xspec xbals limits-------------------------------------------- Blockchain.info API Compatibility ---------------------------------------------- | Currency symbol of the coin of a network, in the Blockchain.info format.-netBinfoSymbol :: Network -> BinfoSymbol-netBinfoSymbol net-  | net == btc =-      BinfoSymbol-        { code = "BTC",-          symbol = "BTC",-          name = "Bitcoin",-          conversion = 100 * 1000 * 1000,-          after = True,-          local = False-        }-  | net == bch =-      BinfoSymbol-        { code = "BCH",-          symbol = "BCH",-          name = "Bitcoin Cash",-          conversion = 100 * 1000 * 1000,-          after = True,-          local = False-        }-  | otherwise =-      BinfoSymbol-        { code = "XTS",-          symbol = "¤",-          name = "Test",-          conversion = 100 * 1000 * 1000,-          after = False,-          local = False-        }---- | Currency symbol of a fiat currency from its price ticker.-binfoTickerToSymbol :: Text -> BinfoTicker -> BinfoSymbol-binfoTickerToSymbol code t =-  BinfoSymbol-    { code,-      symbol = t.symbol,-      name,-      conversion = 100 * 1000 * 1000 / t.fifteen, -- sat/usd-      after = False,-      local = True-    }-  where-    name = case code of-      "EUR" -> "Euro"-      "USD" -> "U.S. dollar"-      "GBP" -> "British pound"-      x -> x---- | Addresses and xpubs in a request parameter, separated by pipes or commas.-getBinfoAddrsParam ::-  WebState ->-  Req ->-  Text ->-  IO (HashSet BinfoAddr)-getBinfoAddrsParam st req name = do-  let net = st.config.store.net-  let ctx = st.config.store.ctx-  p <- fromMaybe "" <$> param req name-  if T.null p-    then return HashSet.empty-    else case parseBinfoAddr net ctx p of-      Nothing -> raise st (UserError "invalid address")-      Just xs -> return $ HashSet.fromList xs---- | Xpubs and addresses in the @active@, @activeP2SH@ and @activeBech32@--- parameters. Xpubs in each parameter derive legacy, nested SegWit and native--- SegWit addresses.-getBinfoActive ::-  WebState ->-  Req ->-  IO (HashSet XPubSpec, HashSet Address)-getBinfoActive st req = do-  active <- getBinfoAddrsParam st req "active"-  p2sh <- getBinfoAddrsParam st req "activeP2SH"-  bech32 <- getBinfoAddrsParam st req "activeBech32"-  let xspec d b = (`XPubSpec` d) <$> xpub b-      xspecs =-        HashSet.fromList $-          concat-            [ mapMaybe (xspec DeriveNormal) (HashSet.toList active),-              mapMaybe (xspec DeriveP2SH) (HashSet.toList p2sh),-              mapMaybe (xspec DeriveP2WPKH) (HashSet.toList bech32)-            ]-      addrs = HashSet.fromList $ mapMaybe addr $ HashSet.toList active-  return (xspecs, addrs)-  where-    addr (BinfoAddr a) = Just a-    addr (BinfoXpub _) = Nothing-    xpub (BinfoXpub x) = Just x-    xpub (BinfoAddr _) = Nothing---- | A kind of address to derive, as in the @derive@ parameter of the xpub--- endpoints: SegWit kinds only on networks with SegWit.-readDerive :: Network -> Text -> Maybe DeriveType-readDerive net t = do-  d <- textToDeriveType t-  guard (net.segWit || d == DeriveNormal)-  return d---- | Whether to identify transactions by short number, unless @txidindex@ is false.-getNumTxId :: Req -> IO Bool-getNumTxId req = not <$> paramOr req readBool "txidindex" False---- | Height of the best header.-getChainHeight :: WebState -> IO H.BlockHeight-getChainHeight st =-  fmap (.height) $ chainGetBest =<< pure st.config.store.chain---- | Handler for @GET\/POST \/blockchain\/unspent@: unspent outputs of addresses and xpubs, as Blockchain.info.-handleBinfoUnspent :: WebState -> Req -> IO Response-handleBinfoUnspent st req =-  withMetrics st (.binfoUnspent) $ do-    (xspecs, addrs) <- getBinfoActive st req-    numtxid <- getNumTxId req-    limit <- get_limit-    min_conf <- get_min_conf-    let net = st.config.store.net-    let ctx = st.config.store.ctx-    height <- getChainHeight st-    let mn u = min_conf > u.confirmations-    xbals <- getXBals st.query xspecs-    bus <--      runConduit $-        getBinfoUnspents st.query numtxid height xbals xspecs addrs-          .| (dropWhileC mn >> takeC limit .| sinkList)-    streamEncoding (marshalEncoding (net, ctx) (BinfoUnspents bus))-  where-    get_limit = min 1000 . fromIntegral <$> paramOr req readNat "limit" 250-    get_min_conf = fromIntegral <$> paramOr req readNat "confirmations" 0---- | Stream the unspent outputs of xpubs and addresses, newest first, in the--- Blockchain.info format.-getBinfoUnspents ::-  StoreRead ->-  Bool ->-  H.BlockHeight ->-  HashMap XPubSpec [XPubBal] ->-  HashSet XPubSpec ->-  HashSet Address ->-  ConduitT () BinfoUnspent IO ()-getBinfoUnspents q numtxid height xbals xspecs addrs = do-  cs' <- conduits-  joinDescStreams (unspentKey . fst) cs' .| mapC (uncurry binfo)-  where-    binfo u xp =-      let conf = case u.block of-            MemRef _ -> 0-            -- an output from the cache may be in a block that the chain of-            -- this store does not have yet: it is confirmed at least once-            BlockRef h _ -> if h > height then 1 else height - h + 1-       in BinfoUnspent-            { txid = u.outpoint.hash,-              index = u.outpoint.index,-              script = u.script,-              value = u.value,-              confirmations = fromIntegral conf,-              txidx = encodeBinfoTxId numtxid u.outpoint.hash,-              xpub = xp-            }-    conduits = (<>) <$> xconduits <*> pure acounduits-    xconduits = lift $ do-      let f x (XPubUnspent u p) =-            let path = toSoft (listToPath p)-                xp = BinfoXPubPath x.key <$> path-             in (u, xp)-          g x = do-            let h = q.xPubUnspents x (xBals x xbals)-                l = let Limits {..} = def in Limits {limit = 16, ..}-            return $ streamThings h Nothing l .| mapC (f x)-      mapM g (HashSet.toList xspecs)-    acounduits =-      let f u = (u, Nothing)-          h = q.getAddressUnspents-          l = let Limits {..} = def in Limits {limit = 16, ..}-          g a = streamThings (h a) Nothing l .| mapC f-       in map g (HashSet.toList addrs)---- | Balances of the used addresses of each xpub.-getXBals ::-  StoreRead ->-  HashSet XPubSpec ->-  IO (HashMap XPubSpec [XPubBal])-getXBals q =-  fmap HashMap.fromList . mapM f . HashSet.toList-  where-    f x = (x,) . filter (not . nullBalance . (.balance)) <$> q.xPubBals x---- | Balances of an xpub, or none if missing.-xBals :: XPubSpec -> HashMap XPubSpec [XPubBal] -> [XPubBal]-xBals = HashMap.findWithDefault []---- | Stream the references to the transactions of xpubs and addresses, newest--- first, without duplicates. Each xpub and address is read page by page, only--- as far as the consumer needs, in pages of the given size: about as many as--- the consumer is expected to take, as every page is a query of its own.-binfoTxRefs ::-  StoreRead ->-  Word32 ->-  HashMap XPubSpec [XPubBal] ->-  HashSet XPubSpec ->-  HashSet Address ->-  ConduitT () TxRef IO ()-binfoTxRefs q pageSize xbals sxspecs saddrs =-  joinDescStreams txRefKey $-    map xpub_c (HashSet.toList sxspecs) <> map addr_c (HashSet.toList saddrs)-  where-    l = Limits {limit = pageSize, offset = 0, start = Nothing}-    xpub_c x = streamThings (q.xPubTxs x (xBals x xbals)) (Just (.txid)) l-    addr_c a = streamThings (q.getAddressTxs a) (Just (.txid)) l---- | Stream the transactions of xpubs and addresses, newest first, in the--- Blockchain.info format, with the running balance of the given addresses after--- each, keeping only those that pass the filter.-getBinfoTxs ::-  StoreRead ->-  Word32 -> -- page size of the references, as for 'binfoTxRefs'-  HashMap XPubSpec [XPubBal] -> -- xpub balances-  HashMap Address (Maybe BinfoXPubPath) -> -- address book-  HashSet XPubSpec -> -- show xpubs-  HashSet Address -> -- show addrs-  HashSet Address -> -- balance addresses-  BinfoFilter ->-  Bool -> -- numtxid-  Bool -> -- prune outputs-  Int64 -> -- starting balance-  ConduitT () BinfoTx IO ()-getBinfoTxs-  q-  pageSize-  xbals-  abook-  sxspecs-  saddrs-  baddrs-  bfilter-  numtxid-  prune-  bal =-    binfoTxRefs q pageSize xbals sxspecs saddrs .| bodies .| go bal-    where-      binfo_tx = toBinfoTx numtxid abook prune-      compute_bal_change t =-        let ins = map (.output) t.inputs-            out = t.outputs-            f b BinfoTxOutput {..} =-              let val = fromIntegral value-               in case address of-                    Nothing -> 0-                    Just a-                      | a `HashSet.member` baddrs ->-                          if b then val else negate val-                      | otherwise -> 0-         in sum $ map (f False) ins <> map (f True) out-      -- the bodies, read a page at a time, which is one round trip to Redis-      -- or one multi-get of the database instead of one per transaction-      bodies = do-        refs <- takeC (fromIntegral pageSize) .| sinkList-        unless (null refs) $ do-          lift (C.getTransactions q (map (.txid) refs)) >>= mapM_ yield-          bodies-      go b =-        await >>= \case-          Nothing -> return ()-          Just Nothing -> go b-          Just (Just x) -> do-            let a = binfo_tx b x-                b' = b - compute_bal_change a-                c = isJust a.blockHeight-                d = fst (fromJust a.balance)-                r = d + fromIntegral a.fee-            case bfilter of-              BinfoFilterAll ->-                yield a >> go b'-              BinfoFilterSent-                | 0 > r -> yield a >> go b'-                | otherwise -> go b'-              BinfoFilterReceived-                | r > 0 -> yield a >> go b'-                | otherwise -> go b'-              BinfoFilterMoved-                | r == 0 -> yield a >> go b'-                | otherwise -> go b'-              BinfoFilterConfirmed-                | c -> yield a >> go b'-                | otherwise -> go b'-              BinfoFilterMempool-                | c -> return ()-                | otherwise -> yield a >> go b'---- | Page size to read the references of the transactions in, to show this--- many after this offset: all of them in one page if they fit in 1000.-pageFor :: Int -> Int -> Word32-pageFor offset n = fromIntegral (max 16 (min 1000 (offset + n)))---- | Whether to show addresses in CashAddr format (@cashaddr@).-getCashAddr :: Req -> IO Bool-getCashAddr req = paramOr req readBool "cashaddr" False---- | Parse an address. Responds with not found if invalid.-getAddress :: WebState -> Text -> IO Address-getAddress st txt = do-  let net = st.config.store.net-  case textToAddr net txt of-    Nothing -> raise st ThingNotFound-    Just a -> return a---- | Address or xpub in a path parameter. Passes to the next route if invalid.-getBinfoAddr :: WebState -> Req -> Text -> IO BinfoAddr-getBinfoAddr st req param' = do-  txt <- capture req param'-  let net = st.config.store.net-  let ctx = st.config.store.ctx-  let addr = BinfoAddr <$> textToAddr net txt-      xpub = BinfoXpub <$> xPubImport net ctx txt-  maybe next return (addr <|> xpub)---- | Handler for @GET\/POST \/blockchain\/export-history@: daily history of the--- balance of addresses and xpubs with prices, as Blockchain.info.-handleBinfoHistory :: WebState -> Req -> IO Response-handleBinfoHistory st req =-  withMetrics st (.binfoExportHistory) $ do-    (xspecs, addrs) <- getBinfoActive st req-    (startM, endM) <- get_dates-    (code, price') <- getPrice st req-    xbals <- (getXBals st.query xspecs)-    let xaddrs = HashSet.fromList $ concatMap (map get_addr) (HashMap.elems xbals)-        aaddrs = xaddrs <> addrs-        cur = price'.fifteen-        cs' = conduits (HashMap.toList xbals) addrs endM-    txs <--      runConduit $-        joinDescStreams txRefKey cs'-          .| takeWhileC (is_newer startM)-          .| concatMapMC get_transaction-          .| sinkList-    let times = map (\Transaction {..} -> timestamp) txs-    let net = st.config.store.net-    let url = st.config.priceHistoryURL-    let session = st.session-    let logger = st.config.logger-    rates <- map (.price) <$> (getRates logger net session url code times)-    let hs = zipWith (convert cur aaddrs) txs (rates <> repeat 0.0)-    streamEncoding $ toEncoding hs-  where-    is_newer-      (Just BlockData {height = bh})-      TxRef {block = BlockRef {height = th}} =-        bh <= th-    is_newer Nothing TxRef {} = True-    is_newer _ _ = undefined-    get_addr = (.balance.address)-    get_transaction TxRef {txid = h} =-      C.getTransaction st.query h-    convert cur addrs tx rate =-      let ins = tx.inputs-          outs = tx.outputs-          fins = filter (input_addr addrs) ins-          fouts = filter (output_addr addrs) outs-          vin = fromIntegral . sum $ map (.value) fins-          vout = fromIntegral . sum $ map (.value) fouts-          fee = tx.fee-          v = vout - vin-          t = tx.timestamp-          h = txHash $ transactionData tx-       in toBinfoHistory v t rate cur fee h-    input_addr addrs' StoreInput {address = Just a} =-      a `HashSet.member` addrs'-    input_addr _ _ = False-    output_addr addrs' StoreOutput {address = Just a} =-      a `HashSet.member` addrs'-    output_addr _ _ = False-    get_dates = do-      BinfoDate start <- requireParam st req parseBinfoDate "start"-      BinfoDate end' <- requireParam st req parseBinfoDate "end"-      let end = end' + 24 * 60 * 60-      let ch = st.config.store.chain-      startM <- C.blockAtOrAfter st.query ch start-      endM <- C.blockAtOrBefore st.query ch end-      return (startM, endM)-    conduits xpubs addrs endM =-      map (uncurry (xpub_c endM)) xpubs-        <> map (addr_c endM) (HashSet.toList addrs)-    lim endM =-      Limits-        { limit = 16,-          offset = 0,-          start = AtBlock . (.height) <$> endM-        }-    addr_c endM a =-      streamThings (st.query.getAddressTxs a) (Just (.txid)) (lim endM)-    xpub_c endM x bs =-      streamThings (st.query.xPubTxs x bs) (Just (.txid)) (lim endM)---- | Currency code in the @currency@ parameter, USD by default, in any case,--- and its latest price. A currency that the price service does not list is--- refused. Until the server has prices, which it may never have if the--- service cannot be reached, every currency has a price of zero, rather than--- every request being refused.-getPrice :: WebState -> Req -> IO (Text, BinfoTicker)-getPrice st req = do-  prices <- readTVarIO st.ticker-  let known c-        | HashMap.null prices = Just (c, def)-        | otherwise = (c,) <$> HashMap.lookup c prices-  paramOr req (known . T.toUpper) "currency" ("USD", fromMaybe def (HashMap.lookup "USD" prices))---- | Symbol of the currency in the @currency@ parameter.-getSymbol :: WebState -> Req -> IO BinfoSymbol-getSymbol st req = uncurry binfoTickerToSymbol <$> getPrice st req---- | Handler for @GET \/blockchain\/blocks\/:milliseconds@: the blocks of the day--- before a time in milliseconds, as Blockchain.info.-handleBinfoBlocksDay :: WebState -> Req -> IO Response-handleBinfoBlocksDay st req =-  withMetrics st (.binfoBlock) $ do-    t <- min h . (`div` 1000) <$> captureWith req readNum "milliseconds"-    let ch = st.config.store.chain-    m <- C.blockAtOrBefore st.query ch t-    bs <- go (d t) m-    streamEncoding $ toEncoding $ map toBinfoBlockInfo bs-  where-    h = fromIntegral (maxBound :: H.Timestamp)-    d = subtract (24 * 3600)-    go _ Nothing = return []-    go t (Just b)-      | b.header.timestamp <= fromIntegral t =-          return []-      | otherwise = do-          b' <- st.query.getBlock b.header.prev-          (b :) <$> go t b'---- | Handler for @GET\/POST \/blockchain\/multiaddr@: balances and transactions of--- addresses and xpubs, as Blockchain.info.-handleMultiAddr :: WebState -> Req -> IO Response-handleMultiAddr st req =-  withMetrics st (.binfoMultiaddr) $ do-    (addrs', _, saddrs, sxpubs, xspecs) <- get_addrs-    numtxid <- getNumTxId req-    cashaddr <- getCashAddr req-    local' <- getSymbol st req-    offset <- getBinfoOffset st req-    n <- getBinfoCount st req "n"-    prune <- get_prune-    fltr <- get_filter-    xbals <- (getXBals st.query xspecs)-    xtxns <- get_xpub_tx_count xbals xspecs-    let sxbals = only_show_xbals sxpubs xbals-        xabals = compute_xabals xbals-        addrs = addrs' `HashSet.difference` HashMap.keysSet xabals-    abals <- get_abals addrs-    let sxspecs = only_show_xspecs sxpubs xspecs-        sxabals = compute_xabals sxbals-        sabals = only_show_abals saddrs abals-        sallbals = sabals <> sxabals-        sbal = compute_bal sallbals-        allbals = abals <> xabals-        abook = compute_abook addrs xbals-        sxaddrs = compute_xaddrs sxbals-        salladdrs = saddrs <> sxaddrs-        bal = compute_bal allbals-        ibal = fromIntegral sbal-    ftxs <--      runConduit $-        getBinfoTxs-          st.query-          (pageFor offset n)-          xbals-          abook-          sxspecs-          saddrs-          salladdrs-          fltr-          numtxid-          prune-          ibal-          .| (dropC offset >> takeC n .| sinkList)-    -- counted without reading the transactions, which takes seconds for-    -- xpubs with tens of thousands of them: one xpub has its own count, from-    -- the cache when it is cached; for several xpubs and addresses, the-    -- database counts the distinct transactions of all their addresses-    ntx <- case (HashSet.toList sxspecs, HashSet.null saddrs) of-      ([x], True) -> return (HashMap.findWithDefault 0 x xtxns)-      _ -> fromIntegral <$> st.direct.getAddressesTxCount (HashSet.toList salladdrs)-    nfiltered <- case fltr of-      BinfoFilterAll -> return ntx-      _ ->-        runConduit $-          getBinfoTxs-            st.query-            1000-            xbals-            abook-            sxspecs-            saddrs-            salladdrs-            fltr-            numtxid-            prune-            ibal-            .| lengthC-    let net = st.config.store.net-    let ctx = st.config.store.ctx-    best <- get_best_block-    peers <- get_peers-    let baddrs = toBinfoAddrs sabals sxbals xtxns-        abaddrs = toBinfoAddrs abals xbals xtxns-        recv = sum $ map (.received) abaddrs-        sent' = sum $ map (.sent) abaddrs-        wallet =-          BinfoWallet-            { balance = bal,-              txs = ntx,-              filtered = nfiltered,-              received = recv,-              sent = sent'-            }-        coin = netBinfoSymbol net-    let block =-          BinfoBlockInfo-            { hash = H.headerHash best.header,-              height = best.height,-              timestamp = best.header.timestamp,-              index = best.height-            }-    let info =-          BinfoInfo-            { connected = peers,-              conversion = 100 * 1000 * 1000,-              fiat = local',-              crypto = coin,-              head = block-            }-    let multiaddr =-          BinfoMultiAddr-            { addresses = baddrs,-              wallet = wallet,-              txs = ftxs,-              info = info,-              recommendFee = True,-              cashAddr = cashaddr-            }-    streamEncoding $ marshalEncoding (net, ctx) multiaddr-  where-    get_xpub_tx_count xbals =-      fmap HashMap.fromList-        . mapM-          ( \x ->-              (x,)-                . fromIntegral-                <$> st.query.xPubTxCount x (xBals x xbals)-          )-        . HashSet.toList-    get_filter = paramOr req parseBinfoFilter "filter" BinfoFilterAll-    get_best_block =-      st.query.getBestBlock >>= \case-        Nothing -> raise st ThingNotFound-        Just bh ->-          st.query.getBlock bh >>= \case-            Nothing -> raise st ThingNotFound-            Just b -> return b-    get_prune = do-      noCompact <- paramOr req readBool "no_compact" False-      not <$> paramOr req readBool "noCompact" noCompact-    only_show_xbals sxpubs =-      HashMap.filterWithKey (\k _ -> k.key `HashSet.member` sxpubs)-    only_show_xspecs sxpubs =-      HashSet.filter (\k -> k.key `HashSet.member` sxpubs)-    only_show_abals saddrs =-      HashMap.filterWithKey (\k _ -> k `HashSet.member` saddrs)-    addr (BinfoAddr a) = Just a-    addr (BinfoXpub _) = Nothing-    xpub (BinfoXpub x) = Just x-    xpub (BinfoAddr _) = Nothing-    get_addrs = do-      (xspecs, addrs) <- getBinfoActive st req-      sh <- getBinfoAddrsParam st req "onlyShow"-      let xpubs = HashSet.map (.key) xspecs-          actives =-            HashSet.map BinfoAddr addrs-              <> HashSet.map BinfoXpub xpubs-          sh' = if HashSet.null sh then actives else sh-          saddrs = HashSet.fromList $ mapMaybe addr $ HashSet.toList sh'-          sxpubs = HashSet.fromList $ mapMaybe xpub $ HashSet.toList sh'-      return (addrs, xpubs, saddrs, sxpubs, xspecs)-    get_abals =-      let f b = (b.address, b)-          g = HashMap.fromList . map f-       in fmap g . st.query.getBalances . HashSet.toList-    get_peers = do-      ps <- getPeersInformation st.config.store.peerMgr-      return (fromIntegral (length ps))-    compute_xabals =-      let f b = (b.balance.address, b.balance)-       in HashMap.fromList . concatMap (map f) . HashMap.elems-    compute_bal =-      let f b = b.confirmed + b.unconfirmed-       in sum . map f . HashMap.elems-    compute_abook addrs xbals =-      let f xs xb =-            let a = xb.balance.address-                e = error "lions and tigers and bears"-                s = toSoft (listToPath xb.path)-             in (a, Just (BinfoXPubPath xs.key (fromMaybe e s)))-          amap =-            HashMap.map (const Nothing) $-              HashSet.toMap addrs-          xmap =-            HashMap.fromList-              . concatMap (uncurry (map . f))-              $ HashMap.toList xbals-       in amap <> xmap-    compute_xaddrs =-      let f = map (.balance.address)-       in HashSet.fromList . concatMap f . HashMap.elems---- | Number of items requested in a parameter, within the limits.-getBinfoCount :: WebState -> Req -> Text -> IO Int-getBinfoCount st req str = do-  let d = st.config.limits.defItemCount-  let x = st.config.limits.maxFullItemCount-  i <- fromIntegral <$> paramOr req readNat str (fromIntegral d)-  return (fromIntegral (if x == 0 then i else min x i) :: Int)---- | Offset requested in the @offset@ parameter. Responds with an error if above the maximum.-getBinfoOffset ::-  WebState ->-  Req ->-  IO Int-getBinfoOffset st req = do-  let x = st.config.limits.maxOffset-  o <- fromIntegral <$> paramOr req readNat "offset" 0-  when (o > x) $-    raise st $-      UserError $-        "offset exceeded: " <> show o <> " > " <> show x-  return (fromIntegral o :: Int)---- | Handler for @GET \/blockchain\/rawaddr\/:addr@: balance and transactions of an--- address or xpub, as Blockchain.info; also at @\/blockchain\/address\/:addr@--- and @\/blockchain\/xpub\/:addr@.-handleRawAddr :: WebState -> Req -> IO Response-handleRawAddr st req =-  withMetrics st (.binfoAddressRaw) $-    getBinfoAddr st req "addr" >>= \case-      BinfoAddr addr -> do_addr addr-      BinfoXpub xpub -> do_xpub xpub-  where-    do_xpub xpub = do-      numtxid <- getNumTxId req-      derive <- paramOr req (readDerive st.config.store.net) "derive" DeriveNormal-      let xspec = XPubSpec xpub derive-      n <- getBinfoCount st req "limit"-      off <- getBinfoOffset st req-      xbals <- (getXBals st.query (HashSet.singleton xspec))-      checkDerive st xspec (xBals xspec xbals)-      let net = st.config.store.net-      let summary = xPubSummary xspec (xBals xspec xbals)-          abook = compute_abook xpub (xBals xspec xbals)-          xspecs = HashSet.singleton xspec-          saddrs = HashSet.empty-          baddrs = HashMap.keysSet abook-          bfilter = BinfoFilterAll-          amnt = summary.confirmed + summary.unconfirmed-      txs <--        runConduit $-          getBinfoTxs-            st.query-            (pageFor off n)-            xbals-            abook-            xspecs-            saddrs-            baddrs-            bfilter-            numtxid-            False-            (fromIntegral amnt)-            .| (dropC off >> takeC n .| sinkList)-      -- n_tx is every transaction of the xpub, as for an address, not the-      -- number on this page-      ntx <- st.query.xPubTxCount xspec (xBals xspec xbals)-      let ra =-            BinfoRawAddr-              { address = BinfoXpub xpub,-                balance = amnt,-                ntx = fromIntegral ntx,-                utxo = summary.utxo,-                received = summary.received,-                sent = fromIntegral summary.received - fromIntegral amnt,-                txs = txs-              }-      let ctx = st.config.store.ctx-      streamEncoding $ marshalEncoding (net, ctx) ra-    compute_abook xpub xbals =-      let f xb =-            let a = xb.balance.address-                e = error "black hole swallows all your code"-                s = toSoft $ listToPath xb.path-                m = fromMaybe e s-             in (a, Just (BinfoXPubPath xpub m))-       in HashMap.fromList $ map f xbals-    do_addr addr = do-      numtxid <- getNumTxId req-      n <- getBinfoCount st req "limit"-      off <- getBinfoOffset st req-      bal <- fromMaybe (zeroBalance addr) <$> st.query.getBalance addr-      let net = st.config.store.net-      let abook = HashMap.singleton addr Nothing-          xspecs = HashSet.empty-          saddrs = HashSet.singleton addr-          bfilter = BinfoFilterAll-          balance = bal.confirmed + bal.unconfirmed-      txs <--        runConduit $-          getBinfoTxs-            st.query-            (pageFor off n)-            HashMap.empty-            abook-            xspecs-            saddrs-            saddrs-            bfilter-            numtxid-            False-            (fromIntegral balance)-            .| (dropC off >> takeC n .| sinkList)-      let ra =-            BinfoRawAddr-              { address = BinfoAddr addr,-                balance,-                ntx = bal.txs,-                utxo = bal.utxo,-                received = bal.received,-                sent = fromIntegral bal.received - fromIntegral balance,-                txs-              }-      let ctx = st.config.store.ctx-      streamEncoding $ marshalEncoding (net, ctx) ra---- | Handler for @GET \/blockchain\/q\/getreceivedbyaddress\/:addr@: total received by an address.-handleBinfoReceived :: WebState -> Req -> IO Response-handleBinfoReceived st req =-  withMetrics st (.binfoQgetreceivedbyaddress) $ do-    a <- getAddress st =<< capture req "addr"-    b <- fromMaybe (zeroBalance a) <$> st.query.getBalance a-    textResponse $ cs $ show b.received---- | Handler for @GET \/blockchain\/q\/getsentbyaddress\/:addr@: total sent by an address.-handleBinfoSent :: WebState -> Req -> IO Response-handleBinfoSent st req =-  withMetrics st (.binfoQgetsentbyaddress) $ do-    a <- getAddress st =<< capture req "addr"-    b <- fromMaybe (zeroBalance a) <$> st.query.getBalance a-    textResponse $ cs $ show $ b.received - b.confirmed - b.unconfirmed---- | Handler for @GET \/blockchain\/q\/addressbalance\/:addr@: balance of an address.-handleBinfoAddrBalance :: WebState -> Req -> IO Response-handleBinfoAddrBalance st req =-  withMetrics st (.binfoQaddressbalance) $ do-    a <- getAddress st =<< capture req "addr"-    b <- fromMaybe (zeroBalance a) <$> st.query.getBalance a-    textResponse $ cs $ show $ b.confirmed + b.unconfirmed---- | Handler for @GET \/blockchain\/q\/addressfirstseen\/:addr@: time of the first--- block with a transaction of an address.-handleFirstSeen :: WebState -> Req -> IO Response-handleFirstSeen st req =-  withMetrics st (.binfoQaddressfirstseen) $ do-    a <- getAddress st =<< capture req "addr"-    let ch = st.config.store.chain-    bb <- chainGetBest ch-    let top = bb.height-        bot = 0-    i <- go ch bb a bot top-    textResponse $ cs $ show i-  where-    go ch bb a bot top = do-      let mid = bot + (top - bot) `div` 2-          n = top - bot < 2-      x <- hasone a bot-      y <- hasone a mid-      z <- hasone a top-      if-        | x -> getblocktime ch bb bot-        | n -> getblocktime ch bb top-        | y -> go ch bb a bot mid-        | z -> go ch bb a mid top-        | otherwise -> return 0-    getblocktime ch bb h =-      (chainGetAncestor ch h bb) >>= \case-        Just b -> return b.header.timestamp-        Nothing -> do-          let logger = st.config.logger-          logError logger $-            "Could not get ancestor at height "-              <> cs (show h)-              <> " for block "-              <> H.blockHashToHex (H.headerHash bb.header)-          error "Block ancestor retreival error"-    hasone a h = do-      let l = Limits 1 0 (Just (AtBlock h))-      not . null <$> st.query.getAddressTxs a l---- | Handler for @GET\/POST \/blockchain\/balance@: short balances of addresses and xpubs, as Blockchain.info.-handleShortBal :: WebState -> Req -> IO Response-handleShortBal st req =-  withMetrics st (.binfoBalance) $ do-    (xspecs, addrs) <- getBinfoActive st req-    cashaddr <- getCashAddr req-    let net = st.config.store.net-    abals <- catMaybes <$> mapM (getabal net cashaddr) (HashSet.toList addrs)-    xbals <- mapM (getxbal net) (HashSet.toList xspecs)-    let res = HashMap.fromList (abals <> xbals)-    streamEncoding $ toEncoding res-  where-    shorten bal =-      BinfoShortBal-        { final = bal.confirmed + bal.unconfirmed,-          ntx = bal.txs,-          received = bal.received-        }-    getabal net cashaddr a =-      let net' =-            if-              | cashaddr -> net-              | net == bch -> btc-              | net == bchTest -> btcTest-              | net == bchTest4 -> btcTest-              | otherwise -> net-       in case addrToText net' a of-            Nothing -> return Nothing-            Just a' ->-              st.query.getBalance a >>= \case-                Nothing -> return $ Just (a', shorten (zeroBalance a))-                Just b -> return $ Just (a', shorten b)-    getxbal net xpub = do-      xbals <- st.query.xPubBals xpub-      ntx <- fromIntegral <$> st.query.xPubTxCount xpub xbals-      let val = sum $ map (.balance.confirmed) xbals-          zro = sum $ map (.balance.unconfirmed) xbals-          exs = [x | x@XPubBal {path = 0 : _} <- xbals]-          final = val + zro-          received = sum $ map (.balance.received) exs-          sbl = BinfoShortBal {final, ntx, received}-      let ctx = st.config.store.ctx-      return (xPubExport net ctx xpub.key, sbl)---- | Whether to respond with hex (@format=hex@).-getBinfoHex :: Req -> IO Bool-getBinfoHex req = paramOr req format "format" False-  where-    format "hex" = Just True-    format "json" = Just False-    format _ = Nothing---- | Handler for @GET \/blockchain\/block-height\/:height@: the blocks at a height, as Blockchain.info, in--- pieces.-handleBinfoBlockHeight :: WebState -> Req -> IO Response-handleBinfoBlockHeight st req =-  inPieces st (.binfoBlockHeight) binfoType $ do-    numtxid <- getNumTxId req-    height <- captureWith req readNum "height"-    bs <- fmap catMaybes $ st.query.getBlocksAtHeight height >>= mapM (st.query.getBlock)-    blocks <- mapM (binfoBlockPieces st numtxid) bs-    return . around "{\"blocks\":[" "]}" $-      concat [around comma mempty block | (comma, block) <- zip ("" : repeat ",") blocks]---- | The pieces of a block as Blockchain.info, with its transactions as--- they are read.------ The block is encoded without its transactions, which are the last thing--- in it, and they are put where the empty list was. Should the encoding of--- haskoin-store-data come to end another way, the block is read and--- encoded whole, as it was before.-binfoBlockPieces :: WebState -> Bool -> BlockData -> IO [Piece]-binfoBlockPieces st numtxid b = do-  nhs <- fmap catMaybes $ st.query.getBlocksAtHeight (b.height + 1) >>= mapM (st.query.getBlock)-  let h x = H.headerHash x.header-      nxt = [h n | n <- nhs, n.header.prev == h b]-      own = (net, ctx)-      txs = transactionPieces st.query b.txs-      -- as 'toBinfoBlock' makes it, with the number of the transactions-      -- that are not in it yet-      bare =-        BinfoBlock-          { hash = h b,-            version = b.header.version,-            prev = b.header.prev,-            merkle = b.header.merkle,-            timestamp = b.header.timestamp,-            bits = b.header.bits,-            next = nxt,-            fee = b.fee,-            nonce = b.header.nonce,-            ntx = fromIntegral (length b.txs),-            size = b.size,-            index = b.height,-            main = b.main,-            height = b.height,-            weight = b.weight,-            txs = []-          }-      bytes = encodingToLazyByteString (marshalEncoding own bare)-      (before, after) = L.splitAt (L.length bytes - 3) bytes-  if after == "[]}"-    then-      around (lazyByteString before) "}"-        <$> listPieces (marshalEncoding own . toBinfoTxSimple numtxid) txs-    else do-      whole <- map (toBinfoTxSimple numtxid) . concat <$> sequence txs-      return [piece (return (fromEncoding (marshalEncoding own (toBinfoBlock b whole nxt))))]-  where-    net = st.config.store.net-    ctx = st.config.store.ctx---- | The headers of a response in JSON of the Blockchain.info endpoints.-binfoType :: ResponseHeaders-binfoType = [(hContentType, "application/json; charset=utf-8")]---- | Handler for @GET \/blockchain\/latestblock@: the best block, as Blockchain.info.-handleBinfoLatest :: WebState -> Req -> IO Response-handleBinfoLatest st req =-  withMetrics st (.binfoBlockLatest) $ do-    numtxid <- getNumTxId req-    b <- (bestBlock st.query) >>= maybe (raise st ThingNotFound) return-    streamEncoding $-      toEncoding-        BinfoHeader-          { hash = H.headerHash b.header,-            timestamp = b.header.timestamp,-            index = b.height,-            height = b.height,-            txids = map (encodeBinfoTxId numtxid) b.txs-          }---- | Handler for @GET \/blockchain\/rawblock\/:block@: a block with its transactions, as Blockchain.info, in--- pieces.-handleBinfoBlock :: WebState -> Req -> IO Response-handleBinfoBlock st req = do-  numtxid <- getNumTxId req-  hex <- getBinfoHex req-  let kind = if hex then [(hContentType, "text/plain; charset=utf-8")] else binfoType-  inPieces st (.binfoBlockRaw) kind $ do-    bh <--      captureWith req parseBinfoBlockId "block" >>= \case-        BinfoBlockHash bh -> return bh-        BinfoBlockIndex i -> st.query.getBlocksAtHeight i >>= maybe (raise st ThingNotFound) return . listToMaybe-    b <- maybe (raise st ThingNotFound) return =<< st.query.getBlock bh-    if hex then return (rawBlock st.query b) else binfoBlockPieces st numtxid b---- | Transaction by hash or short number. Fails with 'TxIndexConflict' if a short number matches several.-getBinfoTx ::-  WebState ->-  BinfoTxId ->-  IO (Either Except Transaction)-getBinfoTx st txid = do-  tx <- case txid of-    BinfoTxIdHash h -> maybeToList <$> C.getTransaction st.query h-    BinfoTxIdIndex i -> C.getNumTransaction st.query i-  case tx of-    [t] -> return $ Right t-    [] -> return $ Left ThingNotFound-    ts ->-      let tids = map (.txid) ts-       in return $ Left (TxIndexConflict tids)---- | Handler for @GET \/blockchain\/rawtx\/:txid@: a transaction, as Blockchain.info.-handleBinfoTx :: WebState -> Req -> IO Response-handleBinfoTx st req =-  withMetrics st (.binfoTxRaw) $ do-    numtxid <- getNumTxId req-    hex <- getBinfoHex req-    txid <- captureWith req parseBinfoTxId "txid"-    tx <--      getBinfoTx st txid >>= \case-        Right t -> return t-        Left e -> raise st e-    if hex then hx tx else js numtxid tx-  where-    js numtxid t = do-      let net = st.config.store.net-      let ctx = st.config.store.ctx-      streamEncoding $ marshalEncoding (net, ctx) $ toBinfoTxSimple numtxid t-    -- the bytes as stored, of the transaction that the id names-    hx t =-      st.query.getManyTxBytes [t.txid] >>= \case-        [Just bytes] -> return (responseBuilder status200 textType (rawHex bytes))-        _ -> raise st ThingNotFound-    textType = [(hContentType, "text/plain; charset=utf-8")]---- | Handler for @GET \/blockchain\/q\/txtotalbtcoutput\/:txid@: total value of the outputs of a transaction.-handleBinfoTotalOut :: WebState -> Req -> IO Response-handleBinfoTotalOut st req =-  withMetrics st (.binfoQtxtotalbtcoutput) $ do-    txid <- captureWith req parseBinfoTxId "txid"-    tx <--      getBinfoTx st txid >>= \case-        Right t -> return t-        Left e -> raise st e-    textResponse $ cs $ show $ sum $ map (.value) tx.outputs---- | Handler for @GET \/blockchain\/q\/txfee\/:txid@: fee of a transaction.-handleBinfoTxFees :: WebState -> Req -> IO Response-handleBinfoTxFees st req =-  withMetrics st (.binfoQtxfee) $ do-    txid <- captureWith req parseBinfoTxId "txid"-    tx <--      getBinfoTx st txid >>= \case-        Right t -> return t-        Left e -> raise st e-    let i = sum $ map (.value) $ filter f tx.inputs-        o = sum $ map (.value) tx.outputs-    textResponse . cs . show $ i - o-  where-    f StoreInput {} = True-    f StoreCoinbase {} = False---- | Handler for @GET \/blockchain\/q\/txresult\/:txid\/:addr@: net amount that a transaction sends to an address.-handleBinfoTxResult :: WebState -> Req -> IO Response-handleBinfoTxResult st req =-  withMetrics st (.binfoQtxresult) $ do-    txid <- captureWith req parseBinfoTxId "txid"-    addr <- getAddress st =<< capture req "addr"-    tx <--      getBinfoTx st txid >>= \case-        Right t -> return t-        Left e -> raise st e-    let i = toInteger $ sum $ map (.value) $ filter (f addr) tx.inputs-        o = toInteger $ sum $ map (.value) $ filter (g addr) tx.outputs-    textResponse $ cs $ show $ o - i-  where-    f addr StoreInput {address = Just a} = a == addr-    f _ _ = False-    g addr StoreOutput {address = Just a} = a == addr-    g _ _ = False---- | Handler for @GET \/blockchain\/q\/txtotalbtcinput\/:txid@: total value of the inputs of a transaction.-handleBinfoTotalInput :: WebState -> Req -> IO Response-handleBinfoTotalInput st req =-  withMetrics st (.binfoQtxtotalbtcinput) $ do-    txid <- captureWith req parseBinfoTxId "txid"-    tx <--      getBinfoTx st txid >>= \case-        Right t -> return t-        Left e -> raise st e-    textResponse $ cs $ show $ sum $ map (.value) $ filter f $ tx.inputs-  where-    f StoreInput {} = True-    f StoreCoinbase {} = False---- | Handler for @GET \/blockchain\/unconfirmed-transactions@: mempool transactions, as Blockchain.info.-handleBinfoMempool :: WebState -> Req -> IO Response-handleBinfoMempool st req =-  withMetrics st (.binfoMempool) $ do-    numtxid <- getNumTxId req-    offset <- getBinfoOffset st req-    n <- getBinfoCount st req "limit"-    -- a limit of 0 would mean no limit to st.query.getMempoolPage-    txids <--      if n <= 0-        then return []-        else-          map snd-            <$> st.query.getMempoolPage (Limits (fromIntegral n) (fromIntegral offset) Nothing)-    txs <- catMaybes <$> C.getTransactions st.query txids-    let net = st.config.store.net-    let mem = BinfoMempool $ map (toBinfoTxSimple numtxid) txs-    let ctx = st.config.store.ctx-    streamEncoding $ marshalEncoding (net, ctx) mem---- | Handler for @GET \/blockchain\/q\/getblockcount@: height of the best header.-handleBinfoGetBlockCount :: WebState -> Req -> IO Response-handleBinfoGetBlockCount st _ =-  withMetrics st (.binfoQgetblockcount) $ do-    let ch = st.config.store.chain-    bn <- chainGetBest ch-    textResponse $ TL.pack $ show bn.height---- | Handler for @GET \/blockchain\/q\/latesthash@: hash of the best header.-handleBinfoLatestHash :: WebState -> Req -> IO Response-handleBinfoLatestHash st _ =-  withMetrics st (.binfoQlatesthash) $ do-    let ch = st.config.store.chain-    bn <- chainGetBest ch-    textResponse $ TL.fromStrict $ H.blockHashToHex $ H.headerHash bn.header---- | Handler for @GET \/blockchain\/q\/bcperblock@: block subsidy at the next height.-handleBinfoSubsidy :: WebState -> Req -> IO Response-handleBinfoSubsidy st _ =-  withMetrics st (.binfoQbcperblock) $ do-    let ch = st.config.store.chain-    let net = st.config.store.net-    bn <- chainGetBest ch-    textResponse $-      cs $-        show $-          (/ (100 * 1000 * 1000 :: Double)) $-            fromIntegral $-              H.computeSubsidy net (bn.height + 1)---- | Handler for @GET \/blockchain\/q\/addresstohash\/:addr@: hash160 of an address in hex.-handleBinfoAddrToHash :: WebState -> Req -> IO Response-handleBinfoAddrToHash st req =-  withMetrics st (.binfoQaddresstohash) $ do-    addr <- getAddress st =<< capture req "addr"-    if-      | Just h <- addressHash160 addr ->-          textResponse $ encodeHexLazy $ runPutL $ serialize h-      | Just h <- addressHash256 addr ->-          textResponse $ encodeHexLazy $ runPutL $ serialize h-      | otherwise ->-          textResponse ""---- | Handler for @GET \/blockchain\/q\/hashtoaddress\/:hash@: pay-to-public-key-hash address of a hash160.-handleBinfoHashToAddr :: WebState -> Req -> IO Response-handleBinfoHashToAddr st req =-  withMetrics st (.binfoQhashtoaddress) $ do-    bs <- maybe next return . decodeHex =<< capture req "hash"-    let net = st.config.store.net-    hash <- either (const next) return $ decode bs-    addr <- maybe next return $ addrToText net $ PubKeyAddress hash-    textResponse $ TL.fromStrict addr---- | Handler for @GET \/blockchain\/q\/addrpubkey\/:pubkey@: address of a public key.-handleBinfoAddrPubkey :: WebState -> Req -> IO Response-handleBinfoAddrPubkey st req =-  withMetrics st (.binfoQaddrpubkey) $ do-    hex <- capture req "pubkey"-    let ctx = st.config.store.ctx-    pubkey <--      maybe next (return . pubKeyAddr ctx) $-        eitherToMaybe . unmarshal ctx =<< decodeHex hex-    let net = st.config.store.net-    case addrToText net pubkey of-      Nothing -> raise st ThingNotFound-      Just a -> textResponse $ TL.fromStrict a---- | Handler for @GET \/blockchain\/q\/pubkeyaddr\/:addr@: public key of an--- address, found in a transaction input that spends from it.-handleBinfoPubKeyAddr :: WebState -> Req -> IO Response-handleBinfoPubKeyAddr st req =-  withMetrics st (.binfoQpubkeyaddr) $ do-    addr <- getAddress st =<< capture req "addr"-    mi <- strm addr-    i <- case mi of-      Nothing -> raise st ThingNotFound-      Just i -> return i-    pk <- case extr addr i of-      Left e -> raise st $ UserError e-      Right t -> return t-    textResponse $ encodeHexLazy $ L.fromStrict pk-  where-    strm addr = do-      runConduit $ do-        let l = Limits {limit = 8, offset = 0, start = Nothing}-        streamThings (st.query.getAddressTxs addr) (Just (.txid)) l-          .| concatMapMC (C.getTransaction st.query . (.txid))-          .| concatMapC (filter (inp addr) . (.inputs))-          .| headC-    inp addr StoreInput {address = Just a} = a == addr-    inp _ _ = False-    extr addr StoreInput {script, pkscript, witness} = do-      Script sig <- decode script-      Script pks <- decode pkscript-      case addr of-        PubKeyAddress {} ->-          case sig of-            [OP_PUSHDATA _ _, OP_PUSHDATA pub _] ->-              Right pub-            [OP_PUSHDATA _ _] ->-              case pks of-                [OP_PUSHDATA pub _, OP_CHECKSIG] ->-                  Right pub-                _ -> Left "Could not parse scriptPubKey"-            _ -> Left "Could not parse scriptSig"-        WitnessPubKeyAddress {} ->-          case witness of-            [_, pub] -> return pub-            _ -> Left "Could not parse scriptPubKey"-        _ -> Left "Address does not have public key"-    extr _ _ = Left "Incorrect input type"---- | Handler for @GET \/blockchain\/q\/hashpubkey\/:pubkey@: hash160 of a public key in hex.-handleBinfoHashPubkey :: WebState -> Req -> IO Response-handleBinfoHashPubkey st req =-  withMetrics st (.binfoQhashpubkey) $ do-    let ctx = st.config.store.ctx-    pkm <--      fmap-        (eitherToMaybe . unmarshal ctx <=< decodeHex)-        (capture req "pubkey")-    addr <- case pkm of-      Nothing -> raise st $ UserError "Could not decode public key"-      Just pk -> return $ pubKeyAddr ctx pk-    if-      | Just h <- addressHash160 addr ->-          textResponse $ encodeHexLazy $ runPutL $ serialize h-      | Just h <- addressHash256 addr ->-          textResponse $ encodeHexLazy $ runPutL $ serialize h-      | otherwise ->-          textResponse ""---- GET Network Information ------ | Handler for @GET \/peers@: connected peers.-handlePeers ::-  WebState ->-  GetPeers ->-  IO [PeerInfo]-handlePeers st _ =-  withMetrics st (.peers) $ do-    getPeersInformation =<< pure st.config.store.peerMgr---- | Obtain information about connected peers from peer manager process.-getPeersInformation ::-  PeerMgr -> IO [PeerInfo]-getPeersInformation mgr =-  mapMaybe toInfo <$> (getPeers mgr)-  where-    toInfo op = do-      ver <- op.version-      return-        PeerInfo-          { userAgent = ver.userAgent.get,-            address = show op.address,-            -- Signed, as the peer sent it. See 'PeerInfo'.-            version = ver.version,-            services = ver.services,-            relay = ver.relay-          }---- | The route of @GET \/health@. Its status says in one number what a load--- balancer needs to know:------ * 200: every check passes;------ * 203: every check passes, but Redis does not answer, so the store---   answers from its own database and mempool alone;------ * 503: a check fails.------ = Why 203 for a store without its Redis------ Such a store is not broken: its answers are right for what it has seen. A--- load balancer should go on sending it requests, which it does for any--- status in the 200s, so the status cannot be an error. But the answers are--- no longer those of the fleet: they do not come from the copy that the--- stores share. That is what 203, Non-Authoritative Information, says of a--- response, and a monitor that expects exactly 200 will notice it. 207 was--- the other candidate, for a body with a status for each part; it promises--- a format that this body does not have. The response also names what is--- missing in a @Haskoin-Degraded@ header.------ = The check of Redis in the body------ The body of a store that has a cache has one more check than the others,--- 'Haskoin.Store.Data.redis', which says whether Redis answers. It does not--- count towards the @ok@ of the whole, which stays the answer to "is this--- store fit to serve".-pathHealth :: WebState -> Route-pathHealth st =-  route (renderStdMethod (resourceMethod proxy)) (capturePath proxy) [] $ \_ -> do-    h <- handleHealth st GetHealth-    let (status, extra)-          | not (isOK h) = (status503, [("Retry-After", "5")])-          | h.redis == Just False = (status203, [("Haskoin-Degraded", "redis")])-          | otherwise = (status200, [])-    return $-      responseLBS status ((hContentType, "application/json") : extra) (encodingToLazyByteString (toEncoding h))-  where-    proxy = Proxy :: Proxy GetHealth---- | Handler for @GET \/health@: health of the store.-handleHealth ::-  WebState ->-  GetHealth ->-  IO HealthCheck-handleHealth st _ =-  withMetrics st (.health) $ do-    h <- readTVarIO st.health-    up <- readTVarIO st.cacheUp-    -- a store without a cache has no Redis to be without-    let HealthCheck {blocks, lastBlock, lastTx, pendingTxs, peers, network, version, time} = h-    let shares = void st.config.store.cache <|> void st.config.store.fleet-    return HealthCheck {redis = up <$ shares, ..}---- | Heights of the best header and best block, and the maximum lag allowed.-blockHealthCheck :: WebConfig -> IO BlockHealth-blockHealthCheck cfg = do-  let ch = cfg.store.chain-      db = cfg.store.db-  headers <- (.height) <$> chainGetBest ch-  blocks <--    Q.getBestBlock db >>= \case-      Nothing -> return 0-      Just h -> maybe 0 (.height) <$> Q.getBlock db h-  return-    BlockHealth-      { headers,-        blocks,-        max = fromIntegral cfg.maxLaggingBlocks-      }---- | Age of the best header and the maximum allowed.-lastBlockHealthCheck :: Chain -> WebLimits -> IO TimeHealth-lastBlockHealthCheck ch WebLimits {blockTimeout} = do-  n <- round <$> getPOSIXTime-  t <- fromIntegral . (.header.timestamp) <$> (chainGetBest ch)-  return-    TimeHealth-      { age = n - t,-        max = fromIntegral blockTimeout-      }---- | Age of the newest mempool transaction or best header, and the maximum allowed.-lastTxHealthCheck :: WebConfig -> IO TimeHealth-lastTxHealthCheck WebConfig {noMempool, store = store', limits} = do-  n <- round <$> getPOSIXTime-  b <- fromIntegral . (.header.timestamp) <$> (chainGetBest ch)-  t <--    Q.getMempoolPage store'.db (Limits 1 0 Nothing) >>= \case-      t : _ ->-        let x = fromIntegral $ fst t-         in return $ max x b-      [] -> return b-  return-    TimeHealth-      { age = n - t,-        max = fromIntegral to-      }-  where-    ch = store'.chain-    to =-      if noMempool-        then limits.blockTimeout-        else limits.txTimeout---- | Transactions waiting to be imported and the maximum allowed.-pendingTxsHealthCheck :: WebConfig -> IO MaxHealth-pendingTxsHealthCheck cfg = do-  n <- blockStorePendingTxs cfg.store.block-  return-    MaxHealth-      { max = fromIntegral cfg.maxPendingTxs,-        count = fromIntegral n-      }---- | Online peers and the minimum required.-peerHealthCheck :: WebConfig -> IO CountHealth-peerHealthCheck cfg = do-  count <- fromIntegral . length <$> (getPeers cfg.store.peerMgr)-  return CountHealth {min = fromIntegral cfg.minPeers, count}---- | Run every health check.-healthCheck :: WebConfig -> IO HealthCheck-healthCheck cfg = do-  blocks <- blockHealthCheck cfg-  lastBlock <- lastBlockHealthCheck cfg.store.chain cfg.limits-  lastTx <- lastTxHealthCheck cfg-  pendingTxs <- pendingTxsHealthCheck cfg-  peers <- peerHealthCheck cfg-  time <- round <$> getPOSIXTime-  return-    HealthCheck-      { network = cfg.store.net.name,-        version = cfg.version,-        -- filled in when the check is served, from what the last request-        -- or ping found-        redis = Nothing,-        ..-      }---- | Handler for @GET \/dbstats@: RocksDB memory use and statistics, as text.-handleDbStats :: WebState -> Req -> IO Response-handleDbStats st _ =-  withMetrics st (.db) $ do-    let db = st.config.store.db.db-    memory <- (memoryReport db)-    statsM <- (getProperty db Stats)-    textResponse . cs $ memory <> maybe "Could not get stats" cs statsM---------------------------- Parameter Parsing ------------------------------ | Returns @Nothing@ if the parameter is not supplied. Raises an exception on--- parse failure.-paramOptional :: forall a. (Param a) => WebState -> Req -> IO (Maybe a)-paramOptional st req = do-  let net = st.config.store.net-  let ctx = st.config.store.ctx-  param req label >>= \case-    Nothing -> return Nothing -- Parameter was not supplied-    Just t -> Just <$> readParam (parseParam net ctx . T.splitOn ",") label t-  where-    label = proxyLabel (Proxy :: Proxy a)---- | Raises an exception if the parameter is not supplied-paramRequired :: forall a. (Param a) => WebState -> Req -> IO a-paramRequired st req = maybe (raise st err) return =<< paramOptional st req-  where-    label = T.unpack (proxyLabel (Proxy :: Proxy a))-    err = UserError $ "Missing parameter " <> label---- | Returns the default value of a parameter if it is not supplied. Raises an--- exception on parse failure.-paramDef :: (Default a, Param a) => WebState -> Req -> IO a-paramDef st req = fromMaybe def <$> paramOptional st req---- | Does not raise exceptions. Passes the request to the next route ('next') if the parameter is--- not supplied or if parsing fails.-paramCapture :: forall a. (Param a) => WebState -> Req -> IO a-paramCapture st req = do-  let net = st.config.store.net-  let ctx = st.config.store.ctx-  p <- capture req label-  maybe next return (parseParam net ctx (T.splitOn "," p))-  where-    label = proxyLabel (Proxy :: Proxy a)---- | Read the transaction that a request carries in its body.-parseTxBody :: WebState -> Req -> IO Tx-parseTxBody st req = do-  let net = st.config.store.net-  case decodeTxBody net (L.toStrict req.body) of-    Left _ -> raise st $ UserError "Failed to parse request body"-    Right tx -> return tx---- | Read a transaction from a request body. The body is the transaction in--- hex. White space around and inside it is ignored.------ The body must hold one transaction and nothing else. Bytes after the--- transaction are refused, as Bitcoin Core does for a raw transaction.------ The transaction must be in the format that peers of the network read. A--- network without SegWit, such as Bitcoin Cash, has the format without--- witness only. Its nodes would not read the SegWit format, so the store--- does not take it either.-decodeTxBody :: Network -> C.ByteString -> Either String Tx-decodeTxBody net body =-  case parseBase16 (C.filter (not . isSpace) body) of-    Right b16 -> fromBytes (decodeBase16 b16)-    Left e -> Left (cs e)-  where-    fromBytes = runGetS $ do-      tx <- getNetworkTx net-      done <- isEmpty-      unless done $ fail "Trailing bytes after transaction"-      return tx---- | Offset in the @offset@ parameter. Responds with an error if above the maximum.-parseOffset :: WebState -> Req -> IO OffsetParam-parseOffset st req = do-  res@(OffsetParam o) <- paramDef st req-  let limits = st.config.limits-  when (limits.maxOffset > 0 && fromIntegral o > limits.maxOffset) $-    raise st . UserError $-      "offset exceeded: " <> show o <> " > " <> show limits.maxOffset-  return res---- | The best block, if there is one.-bestBlock :: StoreRead -> IO (Maybe BlockData)-bestBlock q = q.getBestBlock >>= maybe (return Nothing) q.getBlock---- | Turn a start parameter into a position: a transaction, the block of a block--- hash, a height, or the last block at or before a time.-parseStart ::-  WebState ->-  Maybe StartParam ->-  IO (Maybe Start)-parseStart _ Nothing = return Nothing-parseStart st (Just s) = case s of-  -- a transaction if there is one with that hash, else a block-  StartParamHash {hash = h} ->-    st.query.getTxData (TxHash h) >>= \case-      Just _ -> return (Just (AtTx (TxHash h)))-      Nothing -> fmap (AtBlock . (.height)) <$> st.query.getBlock (H.BlockHash h)-  StartParamHeight {height = h} -> return (Just (AtBlock (fromIntegral h)))-  StartParamTime {time = q} ->-    fmap (AtBlock . (.height)) <$> C.blockAtOrBefore st.query st.config.store.chain q---- | Limit, offset and start in the parameters of a request.-parseLimits :: WebState -> Req -> IO LimitsParam-parseLimits st req = LimitsParam <$> paramOptional st req <*> parseOffset st req <*> paramOptional st req---- | Pagination of a query from request parameters, within the limits for full or short items.-paramToLimits ::-  WebState ->-  Bool ->-  LimitsParam ->-  IO Limits-paramToLimits st full (LimitsParam limitM o startM) = do-  let wl = st.config.limits-  Limits (validateLimit wl full limitM) (fromIntegral o) <$> parseStart st startM---- | Limit of a query: the requested one or the default, but no more than the maximum.-validateLimit :: WebLimits -> Bool -> Maybe LimitParam -> Word32-validateLimit wl full limitM =-  f m $ maybe d (fromIntegral . (.get)) limitM-  where-    m-      | full && wl.maxFullItemCount > 0 = wl.maxFullItemCount-      | otherwise = wl.maxItemCount-    d = wl.defItemCount-    f a 0 = a-    f 0 b = b-    f a b = min a b-------------------- Utilities ---------------------- | The queries that the web server answers with: those of the database,--- with the Redis cache, if there is one, in front of the queries about xpubs--- and of the bodies of transactions. An xpub that the cache does not have is--- read from the database, and cached for next time if it is large enough.------ When Redis does not answer, every query is answered from the database, as--- if the store had no cache. The variable says whether Redis answered the--- last time: a query that fails clears it, so that the next ones do not wait--- for Redis in turn, and the health check sets it again when Redis is back.-webQuery :: Store -> TVar Bool -> StoreRead -> StoreRead-webQuery s up base = case s.cache of-  Nothing -> base-  Just c ->-    (base :: StoreRead)-      { getTxData = fmap listToMaybe' . txData c . pure,-        getManyTxData = txData c,-        xPubBals = \x ->-          cached (cacheXPubBals c x) >>= \case-            Just bals -> return bals-            Nothing -> do-              bals <- base.xPubBals x-              _ <- cached (Just <$> cacheRequest c x bals)-              return bals,-        xPubUnspents = \x bals l ->-          cached (cacheXPubUnspents c x bals l) >>= maybe (base.xPubUnspents x bals l) return,-        xPubTxs = \x bals l ->-          cached (cacheXPubTxs c x l) >>= maybe (base.xPubTxs x bals l) return,-        xPubTxCount = \x bals ->-          cached (cacheXPubTxCount c x) >>= maybe (base.xPubTxCount x bals) return-      }-  where-    -- ask Redis, unless it is down, and take a failure, or no answer within-    -- two seconds, for "not cached"-    cached :: IO (Maybe a) -> IO (Maybe a)-    cached ask =-      readTVarIO up >>= \case-        False -> return Nothing-        True ->-          (timeout 2000000 ask `catchAny` \_ -> return Nothing) >>= \case-            Just answer -> return answer-            Nothing -> atomically (writeTVar up False) >> return Nothing-    -- bodies from Redis when the cache has them, so that the transactions-    -- of a cached xpub are served in full even before this store has-    -- imported them, and from the database otherwise-    txData _ [] = return []-    txData c ths = do-      found <- cached (Just <$> cacheTxData c ths)-      let fromCache = fromMaybe (map (const Nothing) ths) found-          missing = [h | (h, Nothing) <- zip ths fromCache]-      fromDB <- HashMap.fromList . zip missing <$> base.getManyTxData missing-      -- deleted ones are left out, wherever they come from-      return $-        zipWith-          (\h m -> notDeleted (maybe (fromMaybe Nothing (HashMap.lookup h fromDB)) Just m))-          ths-          fromCache---- | The state of the web server, answering without the cache if the flag is--- 'True', as @nocache=true@ asks.-cacheOr :: Bool -> WebState -> WebState-cacheOr False st = st-cacheOr True st = (st :: WebState) {query = st.direct}---- | The only element of a list of one optional value.-listToMaybe' :: [Maybe a] -> Maybe a-listToMaybe' [x] = x-listToMaybe' _ = Nothing---- | Middleware that logs every request with its response status. Only an--- error of the server is logged as one: a request that the server could--- not answer although nothing was wrong with it (500).------ A request that the client got wrong (400, 404) is the client's error,--- and a public server gets them all day. A 503 is the server saying that--- it is not ready, which its health check and its fleet already report.--- Both are logged for debugging, as successes are; logged as errors they--- hid the ones that are.------ A request that took a number of seconds or more to answer gets a--- warning, with how long it took (0 for never). That is the one thing--- about a single request that says something of the server: a query that--- reads far more than it should, or a database that has become slow. The--- time runs until the last byte of the response is handed to the--- connection. A long answer is sent while it is read--- ("Haskoin.Store.Web.Pieces"), so the time that its status took to make--- says little: the reads that matter come after it. The price is that a--- client on a slow line, or a proxy that reads slowly, can make a request--- slow too. A response cut short, by the client leaving or by a failure,--- is logged with what it took until then. The stream of events, whose--- response never ends, is left out.------ With the access log on, every request is logged as information instead--- of for debugging, with the address of the client and the time it took.--- It is off by default: a public server has a proxy in front that keeps--- such a log, and knows the address of the client, where the server sees--- that of the proxy unless the proxy passes it on in @X-Forwarded-For@.--- It is for a server that is run without one.-logIt :: Logger -> Bool -> Word32 -> Middleware-logIt logger access slow app req respond = do-  start <- getPOSIXTime-  app req $ \res -> do-    sent <- respond res `onException` report start res True-    report start res False-    return sent-  where-    report start res cut = do-      took <- subtract start <$> getPOSIXTime-      let s = responseStatus res-          msg = fmtReq req <> ": " <> fmtStatus s <> if cut then ", cut short" else ""-          seconds = cs (showFFloat (Just 1) (realToFrac took :: Double) "")-          millis = cs (showFFloat (Just 1) (realToFrac took * 1000 :: Double) "")-          client = case lookup "X-Forwarded-For" (requestHeaders req) of-            Just given -> T.strip (T.takeWhile (/= ',') (T.decodeUtf8Lenient given))-            Nothing -> cs (show (remoteHost req))-      if-        | statusIsServerError s && s /= status503 -> logError logger msg-        | access -> logInfo logger (client <> " " <> msg <> ", " <> millis <> " ms")-        | otherwise -> logDebug logger msg+-- * __Long requests take turns.__ Only so many long requests, lists,+--   whole blocks and xpubs, are answered at once, as many as the cores keep+--   up with ("Haskoin.Store.Web.Admission", "Haskoin.Store.Web.Congestion");+--   the next ones wait in a short line, and are refused with 429 when it+--   stops moving. A store with more to do than its cores can do still+--   answers the simple requests at once.+--+-- * __Compatibility endpoints ease migration.__ The @\/blockchain\/@ endpoints+--   follow the Blockchain.info API so that existing wallets can switch to a+--   self-hosted server with few changes.+--+-- * __Handlers are plain functions.__ A handler takes the 'WebState' and+--   the request ('Req'), and returns a response in 'IO'. The state holds the+--   queries to answer with in 'query' ('xpubQuery': the database, with the+--   door to the fleet's cache of xpubs in front of the queries about+--   them, which asks the indexer for an xpub that it does not have).+--   The state is made for each request, around one view of the database,+--   so that everything a request reads is of one moment; a store that is a+--   member of a fleet takes the view at the block that its fleet answers+--   from, with the mempool that the fleet shares on top, and has the+--   fleet's cache of xpubs in front of the queries about them+--   ("Haskoin.Store.Fleet.Member").+--   Routes, parameters and the passing of a request to the next route are+--   in "Haskoin.Store.Web.Router", a few dozen lines over WAI. A handler+--   stops early by throwing an 'Except', which becomes the response+--   ('guarded'). There is no web framework and no monad transformer: a+--   framework runs every handler in a stack of its own, and a reader monad+--   in it would hide the state.+module Haskoin.Store.Web+  ( -- * Web+    WebConfig (..),+    Except (..),+    WebLimits (..),+    runWeb,+    runWebSocket,++    -- * Request parsing+    decodeTxBody,++    -- * Logging+    logIt,+    ailments,++    -- * Xpubs+    Unindexed (..),+    xpubQuery,+    askXPubs,+    unindexedResponseOf,+  )+where++import Conduit+  ( ConduitT,+    await,+    concatMapC,+    concatMapMC,+    dropC,+    dropWhileC,+    headC,+    lengthC,+    mapC,+    runConduit,+    sinkList,+    takeC,+    takeWhileC,+    yield,+    (.|),+  )+import Control.Applicative ((<|>))+import Control.Concurrent (getNumCapabilities, threadDelay)+import Control.Concurrent.Async (link, mapConcurrently, race_, withAsync)+import Control.Concurrent.STM+  ( TVar,+    atomically,+    check,+    newTQueueIO,+    newTVarIO,+    readTQueue,+    readTVar,+    readTVarIO,+    registerDelay,+    retry,+    stateTVar,+    writeTQueue,+    writeTVar,+  )+import Control.Exception (Exception, catch, evaluate, fromException, throwIO, try)+import Control.Lens ((&), (.~), (<&>), (?~), (^.))+import Control.Monad+  ( filterM,+    forM,+    forM_,+    forever,+    guard,+    join,+    unless,+    when,+    (<=<),+  )+import Control.Monad.Trans (lift)+import Data.Aeson+  ( Encoding,+    ToJSON (..),+    (.=),+  )+import Data.Aeson qualified as A+import Data.Aeson.Encoding+  ( encodingToLazyByteString,+    fromEncoding,+    list,+    unsafeToEncoding,+  )+import Data.Aeson.KeyMap qualified as KeyMap+import Data.Aeson.Types (parseMaybe)+import Data.ByteString (ByteString)+import Data.ByteString.Base16 (decodeBase16, parseBase16)+import Data.ByteString.Builder (Builder, lazyByteString)+import Data.ByteString.Char8 qualified as C+import Data.ByteString.Lazy qualified as L+import Data.Bytes.Get+import Data.Bytes.Put+import Data.Bytes.Serial+import Data.Char (isSpace)+import Data.Default (Default (..))+import Data.Foldable qualified as Foldable+import Data.HashMap.Strict (HashMap)+import Data.HashMap.Strict qualified as HashMap+import Data.HashSet (HashSet)+import Data.HashSet qualified as HashSet+import Data.IORef (IORef, atomicModifyIORef', newIORef, readIORef, writeIORef)+import Data.Int (Int64)+import Data.List (intersperse, nub)+import Data.List qualified+import Data.Maybe+  ( catMaybes,+    fromJust,+    fromMaybe,+    isJust,+    listToMaybe,+    mapMaybe,+    maybeToList,+  )+import Data.Proxy (Proxy (..))+import Data.Serialize (Serialize, decode, encode)+import Data.String (fromString)+import Data.String.Conversions (cs)+import Data.Text (Text)+import Data.Text qualified as T+import Data.Text.Encoding qualified as T+import Data.Text.Lazy qualified as TL+import Data.Text.Lazy.Encoding qualified as TL+import Data.Time.Clock (getCurrentTime)+import Data.Time.Clock.POSIX (POSIXTime, getPOSIXTime)+import Data.Time.Format (defaultTimeLocale, formatTime)+import Data.Word (Word32, Word64)+import Database.RocksDB+  ( Property (..),+    getProperty,+  )+import Haskoin.Address+import Haskoin.Block qualified as H+import Haskoin.Crypto.Keys+import Haskoin.Network+import Haskoin.Node+  ( Chain,+    LogLevel (..),+    Logger (..),+    OnlinePeer (..),+    PeerMgr,+    chainGetAncestor,+    chainGetBest,+    getPeers,+    logAt,+    logDebug,+    logError,+    logInfo,+    logWarn,+    withSource,+  )+import Haskoin.Script+import Haskoin.Store.BlockStore+import Haskoin.Store.Common hiding+  ( blockAtOrAfter,+    blockAtOrAfterMTP,+    blockAtOrBefore,+    getActiveBlock,+    getActiveTxData,+    getDefaultBalance,+    getNumTransaction,+    getTransaction,+    getTransactions,+  )+import Haskoin.Store.Common qualified as C+import Haskoin.Store.Data+import Haskoin.Store.Database.Query qualified as Q+import Haskoin.Store.Database.Query.Pooled (chainRead)+import Haskoin.Store.Database.Reader+import Haskoin.Store.Exception (bracket, bracket_, catchAny, finally, handleAny, onException, tryAny)+import Haskoin.Store.Fleet.Member+  ( Member (..),+    TipMoved (..),+    answeringCached,+    memberAlive,+    memberEvents,+    memberNewest,+    memberPoolBody,+    memberReceived,+    memberTip,+  )+import Haskoin.Store.Fleet.Redis (FleetEvent (..))+import Haskoin.Store.Manager+import Haskoin.Store.Web.Admission+  ( Admission,+    Aside (..),+    Load (..),+    Refusal (..),+    Report (..),+    Settings (..),+    admittedAside,+    aside,+    load,+    newAdmission,+    regulate,+  )+import Haskoin.Store.Web.Congestion (snooze)+import Haskoin.Store.Web.Docs (swaggerPage, swaggerYaml)+import Haskoin.Store.Web.Events+import Haskoin.Store.Web.Pieces+  ( Piece,+    around,+    hexOf,+    listPieces,+    piece,+    pieceOf,+    pieced,+    piecesResponse,+  )+import Haskoin.Store.Web.Publish qualified as Publish+import Haskoin.Store.Web.Router hiding (header)+import Haskoin.Store.WebCommon+import Haskoin.Store.XPub (EntryMoved (..), XPubReads (..))+import Haskoin.Store.XPub.Service+  ( Answer (..),+    XPubs,+    retryAfter,+    unpinned,+    xpubAsk,+  )+import Haskoin.Store.XPub.Service qualified as Door+import Haskoin.Transaction+import Haskoin.Util+import NQE+  ( subscribe,+    unsubscribe,+  )+import Network.HTTP.Types+  ( ResponseHeaders,+    Status (..),+    hContentType,+    methodGet,+    methodPost,+    parseQuery,+    renderStdMethod,+    status200,+    status202,+    status204,+    status400,+    status404,+    status409,+    status410,+    status413,+    status429,+    status500,+    status503,+    statusIsClientError,+    statusIsServerError,+  )+import Network.Socket (Socket)+import Network.Wai+  ( Application,+    Middleware,+    Request (..),+    RequestBodyLength (..),+    Response,+    getRequestBodyChunk,+    mapResponseHeaders,+    modifyResponse,+    responseBuilder,+    responseHeaders,+    responseLBS,+    responseStatus,+    responseStream,+    setRequestBodyChunks,+  )+import Network.Wai.Handler.Warp+  ( defaultSettings,+    runSettings,+    runSettingsSocket,+    setHost,+    setPort,+  )+import Network.Wai.Handler.WarpTLS (runTLS, runTLSSocket, tlsSettings)+import Network.Wai.Handler.WebSockets (websocketsOr)+import Network.Wai.Middleware.Autohead (autohead)+import Network.Wai.Middleware.Gzip (gzip)+import Network.Wai.Middleware.Gzip qualified as Gzip+import Network.WebSockets+  ( acceptRequest,+    defaultConnectionOptions,+    pendingRequest,+    rejectRequestWith,+    requestPath,+    sendTextData,+  )+import Network.WebSockets qualified as WebSockets+import Network.WebSockets.Connection.PingPong+  ( PingPongOptions (..),+    defaultPingPongOptions,+    withPingPong,+  )+import Network.Wreq qualified as Wreq+import Network.Wreq.Session as Wreq (Session)+import Network.Wreq.Session qualified as Wreq.Session+import Numeric (showFFloat)+import Numeric.Natural (Natural)+import System.Metrics.StatsD+import System.Timeout (timeout)+import Text.Read (readMaybe)++-- | Limits of the API. A value of @0@ means no limit.+data WebLimits = WebLimits+  { -- | maximum number of items in a list response+    maxItemCount :: !Word32,+    -- | maximum number of full transactions in a list response+    maxFullItemCount :: !Word32,+    -- | maximum offset of a paged query+    maxOffset :: !Word32,+    -- | number of items in a list response when no limit is given+    defItemCount :: !Word32,+    -- | consecutive unused addresses after which xpub derivation stops+    xpubGap :: !Word32,+    -- | unused addresses to look at for an xpub with no transactions+    xpubGapInit :: !Word32,+    -- | seconds without a new transaction before the store is reported+    -- unhealthy+    txTimeout :: !Word64,+    -- | seconds without a new block before the store is reported unhealthy+    blockTimeout :: !Word64,+    -- | seconds that a request may take before the log has a warning+    -- about it ('logIt')+    slowRequest :: !Word32,+    -- | most long requests answered at once: the ceiling of the limit+    -- that adapts to how busy the cores are+    -- ("Haskoin.Store.Web.Admission"); 0 is no admission control at all+    maxRequests :: !Word32,+    -- | fewest long requests answered at once: the floor of that limit,+    -- where it starts; at least one, and at most 'maxRequests'+    minRequests :: !Word32,+    -- | milliseconds that a thread ready to run may wait for its turn+    -- before the limit comes down ("Haskoin.Store.Web.Congestion")+    targetDelay :: !Word32,+    -- | requests that may wait for a slot; more are refused+    requestQueue :: !Word32,+    -- | seconds that a request may wait for a slot while the line moves+    requestWait :: !Word32+  }+  deriving (Eq, Show)++-- | 200000 items, 5000 full transactions, offset up to 50000, 100 items by+-- default, xpub gaps of 32 and 20, transaction timeout of 30 minutes,+-- block timeout of four hours, and a warning for requests of five seconds.+-- No admission control, which the executable turns on with limits from the+-- number of cores; with it, a target of 25 ms, and 64 requests that may+-- wait, for a second.+instance Default WebLimits where+  def =+    WebLimits+      { maxItemCount = 200000,+        maxFullItemCount = 5000,+        maxOffset = 50000,+        defItemCount = 100,+        xpubGap = 32,+        xpubGapInit = 20,+        txTimeout = 3600 `div` 2,+        blockTimeout = 4 * 3600,+        slowRequest = 5,+        maxRequests = 0,+        minRequests = 1,+        targetDelay = 25,+        requestQueue = 64,+        requestWait = 1+      }++-- | Configuration of the web server.+data WebConfig = WebConfig+  { -- | address to listen on, such as @*@ for all+    host :: !String,+    -- | port to listen on+    port :: !Int,+    -- | running store to serve+    store :: !Store,+    -- | blocks that the database may lag behind the headers before the store+    -- is reported unhealthy+    maxLaggingBlocks :: !Int,+    -- | transactions that may be waiting to be imported before the store is+    -- reported unhealthy+    maxPendingTxs :: !Int,+    -- | fewest peers before the store is reported unhealthy+    minPeers :: !Int,+    -- | limits of the API+    limits :: !WebLimits,+    -- | version reported by the health check+    version :: !String,+    -- | StatsD client, if metrics are enabled+    stats :: !(Maybe Stats),+    -- | microseconds between updates of the price ticker+    tickerRefresh :: !Int,+    -- | URL of the price ticker service+    tickerURL :: !String,+    -- | URL of the historical price service+    priceHistoryURL :: !String,+    -- | disable the Blockchain.info compatible endpoints+    noBlockchainInfo :: !Bool,+    -- | seconds between health checks; there is one at each new block+    -- too+    healthCheckInterval :: !Int,+    -- | log every request, at the level of information ('logIt')+    accessLog :: !Bool,+    -- | URL of the JSON-RPC interface of a full node, with its user and+    -- password, to publish transactions through ('sendNode'); 'Nothing'+    -- to publish through peers only+    nodeRpc :: !(Maybe String),+    -- | a certificate and its private key, in files in PEM format, to+    -- serve HTTPS instead of HTTP; 'Nothing' for HTTP, which is the+    -- default+    tls :: !(Maybe (FilePath, FilePath)),+    -- | log messages here+    logger :: !Logger+  }++-- | State shared by all request handlers.+data WebState = WebState+  { -- | configuration+    config :: !WebConfig,+    -- | latest prices by currency code+    ticker :: !(TVar (HashMap Text BinfoTicker)),+    -- | StatsD metrics, if enabled+    metrics :: !(Maybe WebMetrics),+    -- | HTTP session for price requests+    session :: !Wreq.Session,+    -- | result of the latest health check+    health :: !(TVar HealthCheck),+    -- | the queries to answer with, the cache included ('xpubQuery'): those+    -- of one view of the database, taken for the request+    query :: !StoreRead,+    -- | the same queries without the cache, on the same view+    direct :: !StoreRead,+    -- | how the request steps aside from its slot while it waits for the+    -- indexer of xpubs ("Haskoin.Store.Web.Admission")+    aside :: !Aside,+    -- | whether Redis answered the last time it was asked+    cacheUp :: !(TVar Bool),+    -- | the events that the clients of @\/events@ read ('feedEvents')+    eventLog :: !EventLog,+    -- | the transactions that were handed over and that the store asks+    -- for until it has them ("Haskoin.Store.Web.Publish")+    asking :: !Publish.Asking,+    -- | the limit on the long requests answered at once+    admission :: !Admission,+    -- | the reads of the caches of xpubs for this request, each xpub of one+    -- version, and the check that ends it ('xpubSession'); outside of a+    -- request, reads of no request+    reading :: !Door.Session+  }++-- | StatsD metrics of the web server: a timing for each kind of request,+-- under @web.\<name\>@, error counters, a gauge of connected event+-- clients, and the metrics of admission control, under @web.cap.*@+-- ("Haskoin.Store.Web.Admission").+data WebMetrics = WebMetrics+  { -- Addresses+    addressTx :: !StatTiming,+    addressTxFull :: !StatTiming,+    addressBalance :: !StatTiming,+    addressUnspent :: !StatTiming,+    xpub :: !StatTiming,+    xpubDelete :: !StatTiming,+    xpubTxFull :: !StatTiming,+    xpubTx :: !StatTiming,+    xpubBalance :: !StatTiming,+    xpubUnspent :: !StatTiming,+    -- Transactions+    tx :: !StatTiming,+    txRaw :: !StatTiming,+    txAfter :: !StatTiming,+    txBlock :: !StatTiming,+    txBlockRaw :: !StatTiming,+    txPost :: !StatTiming,+    mempool :: !StatTiming,+    -- Blocks+    block :: !StatTiming,+    blockRaw :: !StatTiming,+    -- Blockchain+    binfoMultiaddr :: !StatTiming,+    binfoBalance :: !StatTiming,+    binfoAddressRaw :: !StatTiming,+    binfoUnspent :: !StatTiming,+    binfoTxRaw :: !StatTiming,+    binfoBlock :: !StatTiming,+    binfoBlockHeight :: !StatTiming,+    binfoBlockLatest :: !StatTiming,+    binfoBlockRaw :: !StatTiming,+    binfoMempool :: !StatTiming,+    binfoExportHistory :: !StatTiming,+    -- Blockchain /q endpoints+    binfoQaddresstohash :: !StatTiming,+    binfoQhashtoaddress :: !StatTiming,+    binfoQaddrpubkey :: !StatTiming,+    binfoQpubkeyaddr :: !StatTiming,+    binfoQhashpubkey :: !StatTiming,+    binfoQgetblockcount :: !StatTiming,+    binfoQlatesthash :: !StatTiming,+    binfoQbcperblock :: !StatTiming,+    binfoQtxtotalbtcoutput :: !StatTiming,+    binfoQtxtotalbtcinput :: !StatTiming,+    binfoQtxfee :: !StatTiming,+    binfoQtxresult :: !StatTiming,+    binfoQgetreceivedbyaddress :: !StatTiming,+    binfoQgetsentbyaddress :: !StatTiming,+    binfoQaddressbalance :: !StatTiming,+    binfoQaddressfirstseen :: !StatTiming,+    -- Errors+    serverErrors :: StatCounter,+    clientErrors :: StatCounter,+    -- | requests answered again because an entry of a cache that they read+    -- changed meanwhile (@web.reruns@, 'xpubSession')+    reruns :: StatCounter,+    -- Admission+    refused :: StatCounter,+    running :: !StatGauge,+    waiting :: !StatGauge,+    capLimit :: !StatGauge,+    capDelay :: !StatGauge,+    capLateness :: !StatGauge,+    -- | the use of the cores that the controller sees, in hundredths+    capUsed :: !StatGauge,+    -- | the round trip that the controller sees, in microseconds+    capRoundTrip :: !StatGauge,+    capOverloaded :: !StatGauge,+    capSojourn :: !StatTiming,+    refusedFull :: !StatCounter,+    refusedWaited :: !StatCounter,+    refusedCodel :: !StatCounter,+    -- Others+    health :: !StatTiming,+    peers :: !StatTiming,+    db :: !StatTiming,+    events :: !StatGauge+  }++-- | Register the web server metrics with a StatsD client.+createMetrics :: Stats -> IO WebMetrics+createMetrics s = do+  -- Addresses+  addressTx <- d "address_transactions"+  addressTxFull <- d "address_transactions_full"+  addressBalance <- d "address_balance"+  addressUnspent <- d "address_unspent"+  xpub <- d "xpub"+  xpubDelete <- d "xpub_delete"+  xpubTxFull <- d "xpub_transactions_full"+  xpubTx <- d "xpub_transactions"+  xpubBalance <- d "xpub_balances"+  xpubUnspent <- d "xpub_unspent"++  -- Transactions+  tx <- d "transaction"+  txRaw <- d "transaction_raw"+  txAfter <- d "transaction_after"+  txPost <- d "transaction_post"+  txBlock <- d "transactions_block"+  txBlockRaw <- d "transactions_block_raw"+  mempool <- d "mempool"++  -- Blocks+  block <- d "block"+  blockRaw <- d "block_raw"++  -- Blockchain+  binfoMultiaddr <- d "blockchain_multiaddr"+  binfoBalance <- d "blockchain_balance"+  binfoAddressRaw <- d "blockchain_rawaddr"+  binfoUnspent <- d "blockchain_unspent"+  binfoTxRaw <- d "blockchain_rawtx"+  binfoBlock <- d "blockchain_blocks"+  binfoBlockHeight <- d "blockchain_block_height"+  binfoBlockLatest <- d "blockchain_latestblock"+  binfoBlockRaw <- d "blockchain_rawblock"+  binfoMempool <- d "blockchain_mempool"+  binfoExportHistory <- d "blockchain_export_history"++  -- Blockchain /q endpoints+  binfoQaddresstohash <- d "blockchain_q_addresstohash"+  binfoQhashtoaddress <- d "blockchain_q_hashtoaddress"+  binfoQaddrpubkey <- d "blockckhain_q_addrpubkey"+  binfoQpubkeyaddr <- d "blockchain_q_pubkeyaddr"+  binfoQhashpubkey <- d "blockchain_q_hashpubkey"+  binfoQgetblockcount <- d "blockchain_q_getblockcount"+  binfoQlatesthash <- d "blockchain_q_latesthash"+  binfoQbcperblock <- d "blockchain_q_bcperblock"+  binfoQtxtotalbtcoutput <- d "blockchain_q_txtotalbtcoutput"+  binfoQtxtotalbtcinput <- d "blockchain_q_txtotalbtcinput"+  binfoQtxfee <- d "blockchain_q_txfee"+  binfoQtxresult <- d "blockchain_q_txresult"+  binfoQgetreceivedbyaddress <- d "blockchain_q_getreceivedbyaddress"+  binfoQgetsentbyaddress <- d "blockchain_q_getsentbyaddress"+  binfoQaddressbalance <- d "blockchain_q_addressbalance"+  binfoQaddressfirstseen <- d "blockchain_q_addressfirstseen"++  -- Errors+  serverErrors <- c "server_errors"+  clientErrors <- c "client_errors"+  reruns <- c "reruns"+  refused <- newStatCounter s "web.requests_refused" 1+  running <- g "requests_running"+  waiting <- g "requests_waiting"+  capLimit <- g "cap.limit"+  capDelay <- g "cap.delay_us"+  capLateness <- g "cap.lateness_us"+  capUsed <- g "cap.used"+  capRoundTrip <- g "cap.round_trip_us"+  capOverloaded <- g "cap.overloaded"+  capSojourn <- newStatTiming s "web.cap.sojourn" 1+  refusedFull <- newStatCounter s "web.cap.refused_full" 1+  refusedWaited <- newStatCounter s "web.cap.refused_waited" 1+  refusedCodel <- newStatCounter s "web.cap.refused_codel" 1++  -- Others+  health <- d "health"+  peers <- d "peers"+  db <- d "dbstats"++  events <- g "events_connected"+  return WebMetrics {..}+  where+    d x = newStatTiming s ("web." <> x) 10+    g x = newStatGauge s ("web." <> x) 0+    c x = newStatCounter s ("web." <> x) 10++-- | Increase a gauge while an action runs.+withGaugeIO :: StatGauge -> IO a -> IO a+withGaugeIO g = bracket_ (incrementGauge g 1) (decrementGauge g 1)++-- | Increase a gauge of the web metrics while a handler runs.+withGaugeIncrease ::+  WebState ->+  (WebMetrics -> StatGauge) ->+  IO a ->+  IO a+withGaugeIncrease st gf go =+  case st.metrics of+    Nothing -> go+    Just m -> withGaugeIO (gf m) go++-- | Record how long a handler takes in a timing of the web metrics.+withMetrics ::+  WebState ->+  (WebMetrics -> StatTiming) ->+  IO a ->+  IO a+withMetrics st df go =+  case st.metrics of+    Nothing -> go+    Just m -> bracket time (stop m) (const go)+  where+    time = round . (* 1000) <$> getPOSIXTime+    stop m t1 = do+      t2 <- time+      addTiming (df m) (t2 - t1)++-------------------+-- Path Handlers --+-------------------++-- | Run the web server forever on the host and port of the configuration.+-- Also starts a thread that updates prices, unless the Blockchain.info+-- endpoints are disabled, and a thread that runs health checks.+--+-- = HTTPS+--+-- With a certificate and a key in 'tls', the server speaks HTTPS on that+-- port, and nothing else: TLS 1.2 and 1.3, and HTTP\/2 for clients that+-- ask for it. It is off by default. A public service usually sits behind+-- a proxy that has the certificate, limits requests and spreads them over+-- servers; this is for a server by itself, and for links between machines+-- that should not be in the clear. The files are read when the server+-- starts, so a renewed certificate takes a restart.+runWeb :: WebConfig -> IO ()+runWeb config = serveWeb config $ case config.tls of+  Nothing -> runSettings settings+  Just (certificate, key) -> runTLS (tlsSettings certificate key) settings+  where+    settings = setPort config.port . setHost (fromString config.host) $ defaultSettings++-- | Like 'runWeb', on a socket that already listens; the host and port of the+-- configuration are ignored. Binding port 0 and passing the socket gives a+-- free port with no window in which another program can take it, which+-- finding a free port first and binding it later has.+runWebSocket :: WebConfig -> Socket -> IO ()+runWebSocket config sock = serveWeb config $ case config.tls of+  Nothing -> runSettingsSocket defaultSettings sock+  Just (certificate, key) -> runTLSSocket (tlsSettings certificate key) defaultSettings sock++-- | Run the web server with a function of Warp that serves an application.+-- The application is the router ("Haskoin.Store.Web.Router") with the+-- routes of 'handlePaths', behind the middleware for WebSocket events, for+-- logs and for the header that lets any origin make requests.+serveWeb :: WebConfig -> (Application -> IO ()) -> IO ()+serveWeb config serve = do+  ticker <- newTVarIO HashMap.empty+  metrics <- mapM createMetrics config.stats+  session <- Wreq.Session.newAPISession+  first <- runHealthCheck+  health <- newTVarIO first+  -- A member of a fleet takes no turn before its store is known to be+  -- well. A member is made ready to take turns, since a store without a+  -- web server has nothing else to say when; with one, a store that had+  -- just started took a turn before its first check, reported the blocks+  -- that it had then, and with nobody else there moved the fleet to them.+  forM_ config.store.fleet $ \member -> atomically (writeTVar member.healthy (isOK first.blocks && isOK first.peers))+  cacheUp <- cacheCheck >>= newTVarIO+  eventLog <- newEventLog+  asking <- Publish.newAsking+  admission <-+    newAdmission+      Settings+        { ceiling = fromIntegral config.limits.maxRequests,+          floor = fromIntegral config.limits.minRequests,+          target = fromIntegral config.limits.targetDelay * 1000,+          room = fromIntegral config.limits.requestQueue,+          patience = fromIntegral config.limits.requestWait * 1000000+        }+  let db = config.store.db+      -- the door to the fleet's cache of xpubs; a store without Redis has+      -- none, and answers about no xpub+      xpubDoor = (.door) <$> config.store.fleet+      stateOf out sess q =+        WebState+          { query = xpubQuery xpubDoor out sess q,+            direct = q,+            reading = sess,+            aside = out,+            ..+          }+      routesOf s = map (guarded s . settled s) (concatMap withPost (handlePaths s config))+      -- outside of a request, each query has a view of its own; a request+      -- has one view for all that it reads, which is let go when its+      -- response is sent+      st = stateOf (Aside id) unpinned (chainRead db)+      -- every store is a member of a fleet: without Redis, or without a+      -- view of the fleet tip, it is unhealthy and answers 503+      scoped out answer = case config.store.fleet of+        Nothing -> behind answer+        Just member ->+          readTVarIO cacheUp >>= \case+            True -> shared member out (3 :: Int) False answer+            False -> behind answer+      -- as a member of a fleet: from the view at the fleet tip, with the+      -- pool that the fleet shares on top. The tip is taken to be the one+      -- that the member read at its last turn, which saves asking Redis+      -- for it: the first read of the pool checks it, and a request that+      -- reads nothing from the pool checks it when its handler is done,+      -- unless a request saw it in Redis a moment ago ('answeringAt'). If+      -- it is not the tip any more, the request is answered again, from the+      -- tip that Redis has.+      shared member out tries fresh answer = do+        known <- if fresh then return Nothing else fmap encode <$> readTVarIO member.tip+        asked <- case known of+          Just tip -> return (Just (Just tip))+          Nothing -> timeout 2000000 (memberTip member) `catchAny` \_ -> return Nothing+        case asked of+          Nothing -> atomically (writeTVar cacheUp False) >> behind answer+          -- no store has reported yet+          Just Nothing -> behind answer+          Just (Just tip) -> do+            handled <- newIORef False+            let marked confirm r = r {handler = \req -> r.handler req <* confirm <* writeIORef handled True} :: Route+                -- extended public keys through the fleet's cache of+                -- them, unless a request asks for the database+                inFleet plain sess confirm =+                  answer (map (marked confirm) (routesOf (stateOf out sess plain)))+            try (try (answeringCached member tip inFleet)) >>= \case+              -- an entry of the cache changed or was evicted while the+              -- request read it: answered again, from the start+              Left EntryMoved+                | tries > 0 -> do+                    forM_ metrics (\x -> incrementCounter x.reruns 1)+                    shared member out (tries - 1) False answer+                | otherwise -> answer [r {handler = \_ -> unindexedResponse st JobBroke} :: Route | r <- routes]+              Right r -> case r of+                Right (Just sent) -> return sent+                Right Nothing+                  -- a tip from the member's last turn of which there is no+                  -- view may not be the tip any more+                  | not fresh && tries > 0 -> shared member out (tries - 1) True answer+                  | otherwise -> behind answer+                Left TipMoved ->+                  -- the tip moved while the handler was reading: answer+                  -- again from the new one, unless the response is on its way+                  readIORef handled >>= \case+                    True -> throwIO TipMoved+                    False+                      | tries > 0 -> shared member out (tries - 1) True answer+                      | otherwise -> behind answer+      behind answer = answer [r {handler = behindResponse st} :: Route | r <- routes]+      routes = routesOf st+  -- the metrics of each long route, by its name+  capStats <- case config.stats of+    Nothing -> return HashMap.empty+    Just s ->+      fmap HashMap.fromList . forM (nub [routeName r | r <- routes, r.cost == Long]) $ \n ->+        (n,) <$> createCapStats s n+  let app request = routerScoped routes capped notFound request+      -- a light route reads nothing from the database, and is answered at+      -- once, with no view of it: the stream of events, which never ends,+      -- would hold one for as long as its client listens. A long one is+      -- answered in a slot, or told that the server is full.+      capped r answer = case r.cost of+        Light -> answer routes+        Short -> scoped (Aside id) answer+        Long -> case (metrics, HashMap.lookup (routeName r) capStats) of+          (Just w, Just m) -> do+            asked <- millis+            let refused why = do+                  incrementCounter m.refused 1+                  incrementCounter (refusedBy w why) 1+                  answer full+            admittedAside admission refused $ \out -> do+              entered <- millis+              addTiming m.waited (entered - asked)+              addTiming w.capSojourn (entered - asked)+              bracket_+                (atomicModifyIORef' m.inSlot (\n -> (n + 1, ())))+                (atomicModifyIORef' m.inSlot (\n -> (n - 1, ())))+                (scoped out answer)+                `finally` (millis >>= \left -> addTiming m.held (left - entered))+          _ -> admittedAside admission (const (answer full)) (`scoped` answer)+      refusedBy w = \case+        LineFull -> w.refusedFull+        WaitedTooLong -> w.refusedWaited+        Overloaded -> w.refusedCodel+      millis = round . (* 1000) <$> getPOSIXTime :: IO Int+      full = [r {handler = busyResponse st} :: Route | r <- routes]+      notFound _ NoRoute = errorResponse st ThingNotFound+      notFound _ (Unknown ks) =+        errorResponse st . UserError $+          "Unknown parameter: " <> T.unpack (T.intercalate ", " ks)+  saidWell <- newIORef Nothing+  saidRedis <- newIORef True+  pinged <- newIORef 0+  -- set when the store has a new best block, for the health check to run+  -- at once+  newBlock <- newTVarIO False+  date <- httpDate >>= newIORef+  withAsync (forever (snooze 500_000 >> httpDate >>= writeIORef date)) $ \_a0 ->+    withAsync (priceUpdater session ticker) $ \_a1 ->+      withAsync (healthCheckLoop health cacheUp saidWell saidRedis pinged newBlock) $ \_a2 ->+        withAsync (admissionLoop metrics capStats admission) $ \_a5 ->+          withAsync (regulate admission (reportControl metrics)) $ \_a6 ->+            withAsync (feedEvents st) $ \a3 ->+              withAsync (askingLoop st) $ \_a4 ->+                withAsync (refillLoop st) $ \_a7 ->+                  link a3 >> do+                    let logged = logIt config.logger config.accessLog config.limits.slowRequest+                        outer =+                          webSocketEvents st . logged . dated date . anyOrigin . preflight . autohead . compressed+                    serve . outer . cappedBody st $ app+  where+    priceUpdater session =+      unless config.noBlockchainInfo+        . price+          (withSource "Price" config.logger)+          config.store.net+          session+          config.tickerURL+          config.tickerRefresh+    runHealthCheck = healthCheck config+    cacheCheck = maybe (return False) memberAlive config.store.fleet+    -- The check is run every second by default, and at once when the+    -- store has a new best block, which is when its answer changes most:+    -- /health answers from the last result, and a load balancer or a+    -- fleet that acts on a result of half a minute ago sends requests to+    -- a server that fell behind. A check reads a handful of keys, so it+    -- costs little to run often; but a store that is catching up imports+    -- thousands of small blocks a second, and a check for each would be+    -- work for nothing, so there is a tenth of a second between two.+    healthCheckLoop v up saidWell saidRedis pinged newBlock =+      bracket (subscribe config.store.pub (heard newBlock)) (unsubscribe config.store.pub) $ \_ ->+        loopForever $ do+          threadDelay 100000+          due <- registerDelay (config.healthCheckInterval * 1000 * 1000)+          atomically $ do+            block <- readTVar newBlock+            time <- readTVar due+            check (block || time)+            writeTVar newBlock False+          h <- runHealthCheck+          atomically (writeTVar v h)+          tellHealth saidWell h+          -- a store whose checks fail stops taking its turns as a member+          -- of its fleet, so that it does not hold the others back+          forM_ config.store.fleet $ \member -> atomically (writeTVar member.healthy (isOK h.blocks && isOK h.peers))+          -- Redis is asked less often, every five seconds: it is the one+          -- part of the check that leaves the program, and waits up to+          -- two seconds for a Redis that is away. Requests find out by+          -- themselves that Redis is away; this is how the store finds+          -- out that it is back.+          now <- getPOSIXTime+          last' <- readIORef pinged+          when (now - last' >= 5) $ do+            writeIORef pinged now+            redis <- cacheCheck+            atomically (writeTVar up redis)+            tellRedis saidRedis redis+    heard newBlock = \case+      StoreBestBlock _ -> writeTVar newBlock True+      _ -> return ()+    -- A change of health is logged, and nothing while it stays as it was:+    -- what /health answers is otherwise only known to whoever asks it. A+    -- store is not healthy while it syncs, which is no cause for alarm,+    -- so only a store that was healthy gets a warning.+    health' = withSource "Health" config.logger+    tellHealth said h = do+      before <- readIORef said+      writeIORef said (Just (isOK h))+      case (before, isOK h) of+        (Just True, False) -> logWarn health' ("No longer healthy: " <> ailments h)+        (Nothing, False) -> logInfo health' ("Not healthy yet: " <> ailments h)+        (Just False, True) -> logInfo health' "Healthy again"+        (Nothing, True) -> logInfo health' "Healthy"+        _ -> return ()+    -- the same for the Redis that a store with a cache answers through;+    -- /health says 503 without it+    tellRedis said redis = do+      before <- readIORef said+      writeIORef said redis+      case (before, redis) of+        (True, False) -> logWarn health' "Redis does not answer: answering from the database"+        (False, True) -> logInfo health' "Redis answers again"+        _ -> return ()+    -- what a browser asks before a request from another site that is not+    -- of the plainest kind: which methods and headers it may use. Without+    -- an answer it does not send the request.+    preflight inner req respond+      | requestMethod req == "OPTIONS" =+          respond $+            responseLBS+              status204+              [ ("Allow", allowed),+                ("Access-Control-Allow-Methods", allowed),+                ("Access-Control-Allow-Headers", "Accept, Content-Type"),+                ("Access-Control-Max-Age", "86400")+              ]+              ""+      | otherwise = inner req respond+    allowed = "GET, HEAD, POST, DELETE, OPTIONS"+    -- answers are compressed for a client that says it reads gzip: JSON+    -- shrinks several times over. The events are a stream that has to+    -- leave as it is written, and are left alone.+    compressed inner req+      | pathInfo req == ["events"] = inner req+      | otherwise = gzip Gzip.defaultGzipSettings inner req+    -- any origin may ask, and whether the answer is compressed depends on+    -- what the client said it reads, which a cache between the client and+    -- the server has to know: it must not give gzip to a client that+    -- cannot read it.+    anyOrigin = modifyResponse (mapResponseHeaders varied)+    varied headers =+      [("Access-Control-Allow-Origin", "*"), ("Vary", "Accept-Encoding")]+        <> filter ((/= "Vary") . fst) headers++-- | Give every response the @Date@ header from the store's own clock+-- ('httpDate'), so that Warp does not add its own.+--+-- Warp keeps the date in a value that it makes again once a second and+-- does not compute until a response needs it: the first response of the+-- second computes it, and the others wait for it. When that thread is+-- stopped in the middle, at the end of its time slice on a busy core,+-- every response waits until it runs again: an eventlog of an overload+-- showed 58 responses wait up to 70 ms at once on it. This date is made+-- whole by a thread of its own, twice a second, and responses only read+-- it.+dated :: IORef C.ByteString -> Middleware+dated ref app req respond =+  app req $ \res -> readIORef ref >>= \d -> respond (mapResponseHeaders (("Date", d) :) res)++-- | The time now as the @Date@ header of HTTP has it, computed whole.+httpDate :: IO C.ByteString+httpDate = getCurrentTime >>= evaluate . C.pack . formatTime defaultTimeLocale "%a, %d %b %Y %H:%M:%S GMT"++-- | Answer the errors of the handler of a route: an 'Except' with its+-- status and JSON, any other+-- error as a server error, after logging it. A request passed on to the next+-- route goes on.+guarded :: WebState -> Route -> Route+guarded st r = r {handler = \req -> r.handler req `catchAny` answer req}+  where+    answer req e+      | Just Next <- fromException e = throwIO Next+      | Just TipMoved <- fromException e = throwIO TipMoved+      | Just EntryMoved <- fromException e = throwIO EntryMoved+      | Just u <- fromException e = unindexedResponse st u+      | Just x <- fromException e = errorResponse st x+      | Just g <- fromException e = goneResponse st g+      | otherwise = do+          logError st.config.logger $ fmtReq req.wai <> ": " <> cs (show e)+          errorResponse st ServerError++-- | A transaction that a block replaced ('Replaced'), asked for by its id:+-- answered 410, Gone, with the transaction that replaced it and the height+-- of its block, where one that never was is 404.+--+-- The errors of the API are those of haskoin-store-data, whose client+-- reads them. This one is not among them yet, so it is answered here, in+-- the same form: the client takes it for an error it does not know.+data Gone = Gone !TxHash !Replaced+  deriving (Show)++instance Exception Gone++-- | The answer to 'Gone', counted with the errors of clients.+goneResponse :: WebState -> Gone -> IO Response+goneResponse st (Gone txid r) = do+  forM_ st.metrics $ \m -> incrementCounter m.clientErrors 1+  return . responseLBS status410 [(hContentType, "application/json")] . A.encode $+    A.object+      [ "error" .= ("replaced" :: Text),+        "message"+          .= T.unwords+            [ "Transaction",+              txHashToHex txid,+              "was replaced by",+              txHashToHex r.by,+              "in the block at height",+              T.pack (show r.height)+            ],+        "replaced-by" .= r.by,+        "height" .= r.height+      ]++-- | Not found: 'Gone' if a block replaced the transaction, else 404.+notFoundTx :: WebState -> TxHash -> IO a+notFoundTx st txid = st.query.getReplaced txid >>= maybe (raise st ThingNotFound) (throwIO . Gone txid)++-- | What a store that is a member of a fleet answers when it has no view of+-- the block that the fleet answers from: 503, so that a load balancer sends+-- the request to another store. Any answer of its own could differ from+-- those of the other stores, which is what a fleet is there to prevent.+behindResponse :: WebState -> Req -> IO Response+behindResponse st _ = errorResponse st BehindFleet++-- | What a server that is full answers ("Haskoin.Store.Web.Admission"): 429+-- with @{"error": "server-busy"}@, and a header that says to try again in a+-- second: as an xpub that the indexer cannot have soon, and as nginx's+-- @limit_conn_status 429@. Clients and proxies take a 429 with+-- @Retry-After@ to mean "come back later"; a load generator seen on a+-- production store retried a 503 at once, and the refusals became a storm.+-- It is not counted as an error of the server: nothing is wrong with it,+-- and the refusals have a metric of their own.+busyResponse :: WebState -> Req -> IO Response+busyResponse st _ = do+  forM_ st.metrics $ \m -> incrementCounter m.refused 1+  return $+    responseLBS+      status429+      [(hContentType, "application/json"), ("Retry-After", "1")]+      "{\"error\":\"server-busy\"}"++-- | Report how full the server is to StatsD, once a second: in all, and+-- the slots that each long route holds.+admissionLoop :: Maybe WebMetrics -> HashMap Text CapStats -> Admission -> IO ()+admissionLoop Nothing _ _ = return ()+admissionLoop (Just m) capStats admission = forever $ do+  full <- load admission+  setGauge m.running full.busy+  setGauge m.waiting full.waiting+  forM_ capStats $ \c -> readIORef c.inSlot >>= setGauge c.running+  threadDelay 1000000++-- | Report a step of the controller of the limit to StatsD+-- ("Haskoin.Store.Web.Congestion"): the limit, the 90th percentile of the+-- delay of the scheduler without the pauses of GHC's collector, which the+-- limit follows, and with them, in microseconds, and whether the line is+-- overloaded, as 1 or 0.+reportControl :: Maybe WebMetrics -> Report -> IO ()+reportControl Nothing _ = return ()+reportControl (Just m) r = do+  setGauge m.capLimit r.limit+  setGauge m.capDelay r.delay+  setGauge m.capLateness r.lateness+  setGauge m.capUsed r.used+  setGauge m.capRoundTrip r.roundTrip+  setGauge m.capOverloaded (if r.overloaded then 1 else 0)++-- | A route whose handler, once it has its response and before the response+-- is sent, checks that every cached xpub that the request read is still of+-- the version that it read ('xpubSession'): if one is not, it throws+-- v'EntryMoved', and the request is answered again. The routes of xpubs read+-- the caches only before they respond.+settled :: WebState -> Route -> Route+settled st r = r {handler = \req -> r.handler req <* st.reading.settle} :: Route++-- | The metrics of one long route under the limit on requests+-- ("Haskoin.Store.Web.Admission"), under @web.cap.\<route\>.*@.+--+-- The totals of 'WebMetrics' say that the server is full, and not with+-- what: which routes hold the slots and for how long, which wait, and+-- which are refused. They also tell a route that is slow from+-- one that waits: the timing of a request ('withMetrics') starts when its+-- handler does, after the wait.+data CapStats = CapStats+  { -- | requests refused+    refused :: !StatCounter,+    -- | milliseconds that a request waited for its slot; zero for most+    waited :: !StatTiming,+    -- | milliseconds that a request held its slot, to the last byte sent+    held :: !StatTiming,+    -- | slots held now+    running :: !StatGauge,+    -- | slots held now, which 'admissionLoop' reports: a gauge that each+    -- request moved up and down would drift with every datagram lost+    inSlot :: !(IORef Int)+  }++-- | Register the metrics of a long route. Every sample is sent: long+-- requests are few, and one in ten of them would say little.+createCapStats :: Stats -> Text -> IO CapStats+createCapStats s n =+  CapStats+    <$> newStatCounter s (key "refused") 1+    <*> newStatTiming s (key "waited") 1+    <*> newStatTiming s (key "held") 1+    <*> newStatGauge s (key "running") 0+    <*> newIORef 0+  where+    key k = T.unpack ("web.cap." <> n <> "." <> k)++-- | The name of a route in the metrics: the fixed segments of its path,+-- so that @\/address\/:address\/transactions\/full@ is+-- @address_transactions_full@. Routes that differ only in their method,+-- or in what they capture, share a name and their metrics. The route keeps+-- it, made once, as every request looks up its kind by it.+routeName :: Route -> Text+routeName r = r.name++-- | Mark a route as one that answers from memory alone, to be answered+-- even by a server that is full.+lightly :: Route -> Route+lightly r = r {cost = Light}++-- | Mark a route as one whose answer may be long: a list, or all of+-- something. A server with too many of those to answer makes the next ones+-- wait ("Haskoin.Store.Web.Admission").+long :: Route -> Route+long r = r {cost = Long}++-- | Mark a route as one that looks up one thing, where the kind of its+-- response would have made it long.+short :: Route -> Route+short r = r {cost = Short}++-- | The response to an error, in JSON, counted in the metrics.+errorResponse :: WebState -> Except -> IO Response+errorResponse st e = do+  forM_ st.metrics $ \m ->+    mapM_ (`incrementCounter` 1) $+      if+        | statusIsClientError (errStatus e) -> Just m.clientErrors+        | statusIsServerError (errStatus e) -> Just m.serverErrors+        | otherwise -> Nothing+  return $ responseLBS (errStatus e) ((hContentType, "application/json") : again) (A.encode e)+  where+    -- a server behind its fleet catches up within a block or is taken out+    -- of rotation: a client or a load balancer that reads this header+    -- tries again in a second, and elsewhere meanwhile+    again = [("Retry-After", "1") | e == BehindFleet]++-- | Historical prices of the coin of the network in a currency at several times. Returns an empty list on failure.+getRates ::+  Logger ->+  Network ->+  Wreq.Session ->+  String ->+  Text ->+  [Word64] ->+  IO [BinfoRate]+getRates logger net session url currency times = do+  handleAny err $ do+    r <-+      Wreq.asJSON+        =<< Wreq.Session.postWith opts session url body+    return $ r ^. Wreq.responseBody+  where+    err _ = do+      logWarn logger "Could not get historic prices"+      return []+    body = toJSON times+    base =+      Wreq.defaults+        & Wreq.param "base" .~ [T.toUpper $ T.pack net.name]+    opts = base & Wreq.param "quote" .~ [currency]++-- | Update the prices of the coin of the network periodically, for BTC and BCH only.+price ::+  Logger ->+  Network ->+  Wreq.Session ->+  String ->+  Int ->+  TVar (HashMap Text BinfoTicker) ->+  IO ()+price logger net session url pget v = forM_ purl $ \u -> loopForever $ do+  let err e = logWarn logger $ "Could not get prices: " <> cs (show e)+  handleAny err $ do+    r <- Wreq.asJSON =<< Wreq.Session.get session u+    atomically . writeTVar v $ r ^. Wreq.responseBody+  threadDelay pget+  where+    purl = case code of+      Nothing -> Nothing+      Just x -> Just (url <> "?base=" <> x)+      where+        code+          | net == btc = Just "btc"+          | net == bch = Just "bch"+          | otherwise = Nothing++-- | Read no more of the body of a transaction to publish than a+-- transaction can be long: twice 'Publish.maxTxSize' in hexadecimal, and+-- some room for white space. More is answered 413, by its length when the+-- client gave it, and else as soon as that much was read.+--+-- The router reads the body whole before it knows what to do with it.+-- Without this, anyone could have the server hold a body of any size.+-- Other requests with a body are forms of parameters, and are not capped+-- here.+cappedBody :: WebState -> Middleware+cappedBody st app request respond+  | requestMethod request == methodPost,+    filter (not . T.null) (pathInfo request) == ["transactions"] =+      case requestBodyLength request of+        KnownLength n | n > fromIntegral most -> refuse+        _ -> do+          left <- newIORef most+          let chunk = do+                bytes <- getRequestBodyChunk request+                now <- atomicModifyIORef' left (\l -> let l' = l - C.length bytes in (l', l'))+                when (now < 0) (throwIO RequestTooLarge)+                return bytes+          app (setRequestBodyChunks chunk request) respond `catch` \case+            RequestTooLarge -> refuse+            e -> throwIO e+  | otherwise = app request respond+  where+    most = maxPublishBody st.config.store.net+    refuse = errorResponse st RequestTooLarge >>= respond++-- | The most bytes of the body of @POST \/transactions@.+maxPublishBody :: Network -> Int+maxPublishBody net = 2 * Publish.maxTxSize net + 4096++-- | Stop handling the request, and answer with an error ('guarded').+raise :: WebState -> Except -> IO a+raise _ = throwIO++-- | HTTP status of an error.+errStatus :: Except -> Status+errStatus ThingNotFound = status404+errStatus BadRequest = status400+errStatus UserError {} = status400+errStatus StringError {} = status400+errStatus ServerError = status500+errStatus TxIndexConflict {} = status409+errStatus ServerTimeout = status500+errStatus RequestTooLarge = status413+errStatus BehindFleet = status503++-- | A route, and with it the same route under @POST@ if it takes a list in+-- a parameter: addresses, blocks, heights or transactions.+--+-- A list in the query string has to fit in a URL, and proxies cut those+-- off at a few thousand characters: some hundred addresses. The router+-- reads the parameters of a form in the body as it reads those of the+-- query string, so under @POST@ the same parameters go in the body, with+-- @Content-Type: application\/x-www-form-urlencoded@, and have no such+-- limit. @\/transactions@ is left out: @POST@ there publishes a+-- transaction.+withPost :: Route -> [Route]+withPost r+  | r.method == methodGet && takesList && r.pattern /= [Left "transactions"] = [r, r {method = methodPost}]+  | otherwise = [r]+  where+    takesList = any (`elem` r.accepts) ["addresses", "blocks", "heights", "txids"]++-- | The parameters of @\/events@ that say what a client hears about from+-- the start ('eventsOpen').+eventParams :: [Text]+eventParams = ["quiet", "addresses", "xpub", "derive", "filter", "functions", "tweak", "flags"]++-- | The routes of every endpoint, except those disabled in the configuration.+handlePaths :: WebState -> WebConfig -> [Route]+handlePaths st cfg =+  [ marshalled ["notx"] (\r -> GetBlockBest <$> dflt r) (handleBlockBest st),+    short (marshalledList ["notx"] (\r -> GetBlockHeight <$> cap r <*> dflt r) (handleBlockHeight st)),+    marshalled ["notx"] (\r -> GetBlockTime <$> cap r <*> dflt r) (handleBlockTime st),+    marshalled ["notx"] (\r -> GetBlockMTP <$> cap r <*> dflt r) (handleBlockMTP st),+    marshalled ["notx"] (\r -> GetBlock <$> cap r <*> dflt r) (handleBlock st),+    marshalled [] (\r -> GetTx <$> cap r) (handleTx st),+    short (pieces (.txRaw) [] (\r -> GetTxRaw <$> cap r) (handleTxRaw st)),+    pathPosted [] (\r -> PostTx <$> parseTxBody st r) (handlePostTx st),+    plainList limits (\r -> GetAddrTxs <$> cap r <*> lim r) (handleAddrTxs st),+    marshalled [] (\r -> GetAddrBalance <$> cap r) (handleAddrBalance st),+    marshalledList limits (\r -> GetAddrUnspent <$> cap r <*> lim r) (handleAddrUnspent st),+    lightly (plainList [] (\_ -> return GetPeers) (handlePeers st)),+    lightly (pathHealth st),+    plainList ["limit", "offset"] (\r -> GetMempool <$> opt r <*> off r) (handleMempool st),+    marshalledList ["blocks", "notx"] (\r -> GetBlocks <$> need r <*> dflt r) (handleBlocks st),+    pieces (.blockRaw) [] (\r -> GetBlockRaw <$> cap r) (handleBlockRaw st),+    pieces (.blockRaw) [] (\_ -> return GetBlockBestRaw) (handleBlockBestRaw st),+    marshalledList ["notx"] (\r -> GetBlockLatest <$> dflt r) (handleBlockLatest st),+    marshalledList ["heights", "notx"] (\r -> GetBlockHeights <$> need r <*> dflt r) (handleBlockHeights st),+    pieces (.blockRaw) [] (\r -> GetBlockHeightRaw <$> cap r) (handleBlockHeightRaw st),+    pieces (.blockRaw) [] (\r -> GetBlockTimeRaw <$> cap r) (handleBlockTimeRaw st),+    pieces (.blockRaw) [] (\r -> GetBlockMTPRaw <$> cap r) (handleBlockMTPRaw st),+    marshalledList ["txids"] (\r -> GetTxs <$> need r) (handleTxs st),+    pieces (.txRaw) ["txids"] (\r -> GetTxsRaw <$> need r) (handleTxsRaw st),+    marshalledPieces (.txBlock) [] (\r -> GetTxsBlock <$> cap r) (handleTxsBlock st),+    pieces (.txBlockRaw) [] (\r -> GetTxsBlockRaw <$> cap r) (handleTxsBlockRaw st),+    long (plain [] (\r -> GetTxAfter <$> cap r <*> cap r) (handleTxAfter st)),+    plainList ("addresses" : limits) (\r -> GetAddrsTxs <$> need r <*> lim r) (handleAddrsTxs st),+    marshalledPieces (.addressTxFull) limits (\r -> GetAddrTxsFull <$> cap r <*> lim r) (handleAddrTxsFull st),+    marshalledPieces+      (.addressTxFull)+      ("addresses" : limits)+      (\r -> GetAddrsTxsFull <$> need r <*> lim r)+      (handleAddrsTxsFull st),+    marshalledList ["addresses"] (\r -> GetAddrsBalance <$> need r) (handleAddrsBalance st),+    marshalledList ("addresses" : limits) (\r -> GetAddrsUnspent <$> need r <*> lim r) (handleAddrsUnspent st),+    lightly (get "/events" ("after" : eventParams) (handleEvents st)),+    lightly (get "/dbstats" [] (handleDbStats st)),+    -- the reference of the API, as a page and as the file that it shows+    lightly (get "/" [] (\_ -> file "text/html; charset=utf-8" swaggerPage)),+    lightly (get "/swagger.yaml" [] (\_ -> file "application/yaml" swaggerYaml))+  ]+    <> xpubRoutes+    <> (if cfg.noBlockchainInfo then [] else binfoRoutes)+  where+    file kind bytes = return (responseLBS status200 [(hContentType, kind)] (L.fromStrict bytes))+    net = cfg.store.net+    -- the parameters that routes accept, beyond those of each route+    limits = ["limit", "offset", "height"]+    active = ["active", "activeP2SH", "activeBech32"]+    -- the Blockchain.info endpoints accept format=json ('jsonOnly')+    multiaddrParams =+      ["format", "onlyShow", "n", "offset", "noCompact", "no_compact", "currency", "cashaddr", "txidindex", "filter"]+        <> active+    rawaddrParams = ["format", "derive", "limit", "offset", "txidindex"]+    balanceParams = "format" : "cashaddr" : active+    unspentParams = "format" : "limit" : "confirmations" : "txidindex" : active+    historyParams = "format" : "start" : "end" : "currency" : active+    -- the parameters of a request+    cap, need :: (Param a) => Req -> IO a+    cap = paramCapture st+    need = paramRequired st+    dflt :: (Default a, Param a) => Req -> IO a+    dflt = paramDef st+    opt :: (Param a) => Req -> IO (Maybe a)+    opt = paramOptional st+    lim = parseLimits st+    off = parseOffset st+    -- routes by how they encode their responses: in the JSON of the+    -- network, as a list of those, or in plain JSON. A list is a long+    -- route, as is one answered in pieces; the few that are not say so.+    marshalled ::+      (ApiResource a b, MarshalJSON Network b) => [Text] -> (Req -> IO a) -> (a -> IO b) -> Route+    marshalled ps p h = pathCompact ps p h (marshalEncoding net)+    marshalledList ::+      (ApiResource a (SerialList b), MarshalJSON Network b) =>+      [Text] ->+      (Req -> IO a) ->+      (a -> IO [b]) ->+      Route+    marshalledList ps p h =+      long (pathCompact ps p (fmap SerialList . h) (list (marshalEncoding net) . (.get)))+    -- routes of what is answered while it is read: a list of items in+    -- the JSON of the network, or pieces that the handler made+    marshalledPieces ::+      (ApiResource a (SerialList b), MarshalJSON Network b) =>+      (WebMetrics -> StatTiming) ->+      [Text] ->+      (Req -> IO a) ->+      (a -> IO [(Int, IO [b])]) ->+      Route+    marshalledPieces df ps p h = pieces df ps p (listPieces (marshalEncoding net) <=< h)+    -- pieces that the handler made+    pieces ::+      (ApiResource a b) =>+      (WebMetrics -> StatTiming) ->+      [Text] ->+      (Req -> IO a) ->+      (a -> IO [Piece]) ->+      Route+    pieces df ps p h = long (pathPieces st df ps p h)+    plain :: (ApiResource a b, ToJSON b) => [Text] -> (Req -> IO a) -> (a -> IO b) -> Route+    plain ps p h = pathCompact ps p h toEncoding+    plainList ::+      (ApiResource a (SerialList b), ToJSON b) => [Text] -> (Req -> IO a) -> (a -> IO [b]) -> Route+    plainList ps p h = long (plain ps p (fmap SerialList . h))+    -- @nocache@ is taken and ignored: the cache always answers, and older+    -- clients that still send it are not refused+    cached = pure (NoCache False)+    xpubRoutes =+      [ long (plain ["derive", "nocache"] (\r -> GetXPub <$> cap r <*> dflt r <*> cached) (handleXPub st)),+        plainList+          ("derive" : "nocache" : limits)+          (\r -> GetXPubTxs <$> cap r <*> dflt r <*> lim r <*> cached)+          (handleXPubTxs st),+        marshalledPieces+          (.xpubTxFull)+          ("derive" : "nocache" : limits)+          (\r -> GetXPubTxsFull <$> cap r <*> dflt r <*> lim r <*> cached)+          (handleXPubTxsFull st),+        marshalledList+          ["derive", "nocache"]+          (\r -> GetXPubBalances <$> cap r <*> dflt r <*> cached)+          (handleXPubBalances st),+        marshalledList+          ("derive" : "nocache" : limits)+          (\r -> GetXPubUnspent <$> cap r <*> dflt r <*> lim r <*> cached)+          (handleXPubUnspent st),+        plain ["derive"] (\r -> DelCachedXPub <$> cap r <*> dflt r) (handleDelXPub st)+      ]+    -- the routes of Blockchain.info take their parameters themselves+    binfoRoutes =+      [ long (post "/blockchain/multiaddr" multiaddrParams (jsonOnly (handleMultiAddr st))),+        long (get "/blockchain/multiaddr" multiaddrParams (jsonOnly (handleMultiAddr st))),+        long (get "/blockchain/balance" balanceParams (jsonOnly (handleShortBal st))),+        long (post "/blockchain/balance" balanceParams (jsonOnly (handleShortBal st))),+        long (get "/blockchain/rawaddr/:addr" rawaddrParams (jsonOnly (handleRawAddr st))),+        long (get "/blockchain/address/:addr" rawaddrParams (jsonOnly (handleRawAddr st))),+        long (get "/blockchain/xpub/:addr" rawaddrParams (jsonOnly (handleRawAddr st))),+        long (post "/blockchain/unspent" unspentParams (jsonOnly (handleBinfoUnspent st))),+        long (get "/blockchain/unspent" unspentParams (jsonOnly (handleBinfoUnspent st))),+        get "/blockchain/rawtx/:txid" ["format", "txidindex"] (handleBinfoTx st),+        long (get "/blockchain/rawblock/:block" ["format", "txidindex"] (handleBinfoBlock st)),+        get "/blockchain/latestblock" ["format", "txidindex"] (jsonOnly (handleBinfoLatest st)),+        long $+          get+            "/blockchain/unconfirmed-transactions"+            ["format", "limit", "offset", "txidindex"]+            (jsonOnly (handleBinfoMempool st)),+        long $+          get+            "/blockchain/block-height/:height"+            ["format", "txidindex"]+            (jsonOnly (handleBinfoBlockHeight st)),+        long (get "/blockchain/blocks/:milliseconds" ["format"] (jsonOnly (handleBinfoBlocksDay st))),+        long (get "/blockchain/export-history" historyParams (jsonOnly (handleBinfoHistory st))),+        long (post "/blockchain/export-history" historyParams (jsonOnly (handleBinfoHistory st))),+        get "/blockchain/q/addresstohash/:addr" [] (handleBinfoAddrToHash st),+        get "/blockchain/q/hashtoaddress/:hash" [] (handleBinfoHashToAddr st),+        get "/blockchain/q/addrpubkey/:pubkey" [] (handleBinfoAddrPubkey st),+        get "/blockchain/q/pubkeyaddr/:addr" [] (handleBinfoPubKeyAddr st),+        get "/blockchain/q/hashpubkey/:pubkey" [] (handleBinfoHashPubkey st),+        get "/blockchain/q/getblockcount" [] (handleBinfoGetBlockCount st),+        get "/blockchain/q/latesthash" [] (handleBinfoLatestHash st),+        get "/blockchain/q/bcperblock" [] (handleBinfoSubsidy st),+        get "/blockchain/q/txtotalbtcoutput/:txid" [] (handleBinfoTotalOut st),+        get "/blockchain/q/txtotalbtcinput/:txid" [] (handleBinfoTotalInput st),+        get "/blockchain/q/txfee/:txid" [] (handleBinfoTxFees st),+        get "/blockchain/q/txresult/:txid/:addr" [] (handleBinfoTxResult st),+        long (get "/blockchain/q/getreceivedbyaddress/:addr" [] (handleBinfoReceived st)),+        long (get "/blockchain/q/getsentbyaddress/:addr" [] (handleBinfoSent st)),+        get "/blockchain/q/addressbalance/:addr" [] (handleBinfoAddrBalance st),+        long (get "/blockchain/q/addressfirstseen/:addr" [] (handleFirstSeen st))+      ]++-- | A handler of the Blockchain.info endpoints that answer in JSON only. They+-- accept @format=json@, which clients written for Blockchain.info often+-- send, and refuse any other format rather than answering in JSON anyway.+jsonOnly :: (Req -> IO Response) -> Req -> IO Response+jsonOnly h req = do+  paramOr req json "format" ()+  h req+  where+    json "json" = Just ()+    json _ = Nothing++-- | The route of an API resource: parse the request, run the handler, and encode the+-- response as JSON, as it is sent ('lazyResponse').+pathCompact ::+  (ApiResource a b) => [Text] -> (Req -> IO a) -> (a -> IO b) -> (b -> Encoding) -> Route+pathCompact ps parser action encJson =+  route (renderStdMethod (resourceMethod proxy)) (capturePath proxy) ps $ \req -> do+    asked <- parser req+    res <- action asked+    lazyResponse status200 jsonType (encodingToLazyByteString (encJson res))+  where+    toProxy :: (Req -> IO a) -> Proxy a+    toProxy = const Proxy+    proxy = toProxy parser++-- | The route of an API resource whose handler chooses the status of its+-- answer: @POST \/transactions@, which answers 202 for a transaction that+-- it handed over.+pathPosted ::+  (ApiResource a b) => [Text] -> (Req -> IO a) -> (a -> IO (Status, Encoding)) -> Route+pathPosted ps parser action =+  route (renderStdMethod (resourceMethod proxy)) (capturePath proxy) ps $ \req -> do+    asked <- parser req+    (status, res) <- action asked+    lazyResponse status jsonType (encodingToLazyByteString res)+  where+    toProxy :: (Req -> IO a) -> Proxy a+    toProxy = const Proxy+    proxy = toProxy parser++-- | The route of an API resource that is answered while it is read, as+-- "Haskoin.Store.Web.Pieces" explains: the handler gives the bytes of the+-- response in pieces.+pathPieces ::+  (ApiResource a b) =>+  WebState ->+  (WebMetrics -> StatTiming) ->+  [Text] ->+  (Req -> IO a) ->+  (a -> IO [Piece]) ->+  Route+pathPieces st df ps parser action =+  route (renderStdMethod (resourceMethod proxy)) (capturePath proxy) ps $ \req ->+    inPieces st df jsonType $ do+      asked <- parser req+      action asked+  where+    toProxy :: (Req -> IO a) -> Proxy a+    toProxy = const Proxy+    proxy = toProxy parser++-- | A response in pieces, timed. The timing counts the whole response, the+-- time that the client takes to read it included.+inPieces :: WebState -> (WebMetrics -> StatTiming) -> ResponseHeaders -> IO [Piece] -> IO Response+inPieces st df headers pieces = do+  began <- now+  let done = forM_ st.metrics $ \m -> now >>= \ended -> addTiming (df m) (ended - began)+  n <- encoders+  (pieces >>= piecesResponse n headers done) `onException` done+  where+    now = round . (* 1000) <$> getPOSIXTime :: IO Int++-- | How many pieces of a response are read and encoded at once: one for+-- each core, so that a long answer is encoded on all of them, but no more+-- than eight, which bounds what a response holds to some nine pieces, and+-- leaves cores to other requests on a large machine.+encoders :: IO Int+encoders = min 8 <$> getNumCapabilities++-- | The headers of a response in JSON.+jsonType :: ResponseHeaders+jsonType = [(hContentType, "application/json")]++-- | Respond with JSON, encoded as it is sent ('lazyResponse').+streamEncoding :: Encoding -> IO Response+streamEncoding e =+  lazyResponse status200 [(hContentType, "application/json; charset=utf-8")] (encodingToLazyByteString e)++-- | A response of bytes made as they are sent, such as JSON from its+-- encoding. Warp writes a lazy byte string a chunk at a time, as the+-- encoding makes them, so a long answer is never all in memory at once.+lazyResponse :: Status -> ResponseHeaders -> L.ByteString -> IO Response+lazyResponse status headers bytes = return (responseLBS status headers bytes)++-- | Respond with text.+textResponse :: TL.Text -> IO Response+textResponse t =+  return $+    responseLBS status200 [(hContentType, "text/plain; charset=utf-8")] (TL.encodeUtf8 t)++-- GET Block / GET Blocks --++-- | Handler for @GET \/block\/:block@: a block by hash.+handleBlock ::+  WebState ->+  GetBlock ->+  IO BlockData+handleBlock st (GetBlock h (NoTx noTx)) = withMetrics st (.block) $ do+  st.query.getBlock h >>= \case+    Nothing ->+      raise st ThingNotFound+    Just b -> do+      return $ pruneTx noTx b++-- | Blocks by hash, without duplicates, leaving out unknown ones.+getBlocks ::+  WebState ->+  [H.BlockHash] ->+  Bool ->+  IO [BlockData]+getBlocks st hs notx =+  (pruneTx notx <$>) . catMaybes <$> mapM (st.query.getBlock) (nub hs)++-- | Handler for @GET \/blocks?blocks=…@: several blocks by hash.+handleBlocks ::+  WebState ->+  GetBlocks ->+  IO [BlockData]+handleBlocks st (GetBlocks hs (NoTx notx)) =+  withMetrics st (.block) $+    getBlocks st hs notx++-- | Keep only the coinbase in the list of transactions of a block, if the flag is 'True'.+pruneTx :: Bool -> BlockData -> BlockData+pruneTx False b = b+pruneTx True BlockData {..} = BlockData {txs = take 1 txs, ..}++-- GET BlockRaw --++-- | Handler for @GET \/block\/:block\/raw@: a block by hash, serialized as in the peer-to-peer protocol.+handleBlockRaw :: WebState -> GetBlockRaw -> IO [Piece]+handleBlockRaw st (GetBlockRaw h) = rawResult <$> getRawBlock st h++-- | The pieces of a block by hash, serialized, in hexadecimal. Responds+-- with an error if not found.+getRawBlock :: WebState -> H.BlockHash -> IO [Piece]+getRawBlock st h =+  st.query.getBlock h >>= maybe (raise st ThingNotFound) (return . rawBlock st.query)++-- | The pieces of a block serialized as in the peer-to-peer protocol, in+-- hexadecimal: its header and the number of its transactions, then its+-- transactions as they are read. The first piece has transactions too, so+-- that a response does not begin before something of them was read.+rawBlock :: StoreRead -> BlockData -> [Piece]+rawBlock q b =+  around (hexOf b.header <> hexOf (VarInt (fromIntegral (length b.txs)))) mempty $+    map (uncurry pieceOf) (pieced pieceSize (fmap (foldMap rawHex . catMaybes) . q.getManyTxBytes) b.txs)++-- | The hex of bytes as stored, such as a transaction in the format of the+-- network: read as they are, and written with the table of haskoin-core's+-- 'hexBuilder'.+rawHex :: ByteString -> Builder+rawHex = hexBuilder . L.fromStrict++-- | The pieces of the JSON of a 'RawResult', from those of the hexadecimal.+rawResult :: [Piece] -> [Piece]+rawResult = around "{\"result\":\"" "\"}"++-- | The pieces of the JSON of a 'RawResultList', from those of the+-- hexadecimal of each of its items.+rawResultList :: [[Piece]] -> [Piece]+rawResultList items =+  around "[" "]" (concat [around (comma <> "\"") "\"" item | (comma, item) <- zip ("" : repeat ",") items])++-- GET BlockBest / BlockBestRaw --++-- | Handler for @GET \/block\/best@: the best block.+handleBlockBest ::+  WebState ->+  GetBlockBest ->+  IO BlockData+handleBlockBest st (GetBlockBest (NoTx notx)) =+  withMetrics st (.block) $+    (bestBlock st.query) >>= maybe (raise st ThingNotFound) (return . pruneTx notx)++-- | Handler for @GET \/block\/best\/raw@: the best block, serialized.+handleBlockBestRaw :: WebState -> GetBlockBestRaw -> IO [Piece]+handleBlockBestRaw st _ =+  st.query.getBestBlock >>= \case+    Nothing -> raise st ThingNotFound+    Just h -> rawResult <$> getRawBlock st h++-- GET BlockLatest --++-- | Handler for @GET \/block\/latest@: the best block and its ancestors, up to 100 blocks.+handleBlockLatest ::+  WebState ->+  GetBlockLatest ->+  IO [BlockData]+handleBlockLatest st (GetBlockLatest (NoTx noTx)) =+  withMetrics st (.block) $+    st.query.getBestBlock >>= maybe (raise st ThingNotFound) (go [] <=< st.query.getBlock)+  where+    go acc Nothing = return $ reverse acc+    go acc (Just b)+      | b.height <= 0 = return $ reverse acc+      | length acc == 99 = return . reverse $ pruneTx noTx b : acc+      | otherwise = go (pruneTx noTx b : acc) =<< st.query.getBlock b.header.prev++-- GET BlockHeight / BlockHeights / BlockHeightRaw --++-- | Handler for @GET \/block\/height\/:height@: the blocks at a height.+handleBlockHeight ::+  WebState ->+  GetBlockHeight ->+  IO [BlockData]+handleBlockHeight st (GetBlockHeight h (NoTx notx)) =+  withMetrics st (.block) $+    st.query.getBlocksAtHeight (fromIntegral h)+      >>= (\hs -> getBlocks st hs notx)++-- | Handler for @GET \/block\/heights?heights=…@: the blocks at several heights.+handleBlockHeights ::+  WebState ->+  GetBlockHeights ->+  IO [BlockData]+handleBlockHeights st (GetBlockHeights (HeightsParam heights) (NoTx notx)) =+  withMetrics st (.block) $+    mapM (st.query.getBlocksAtHeight . fromIntegral) heights+      >>= (\hs -> getBlocks st hs notx) . concat++-- | Handler for @GET \/block\/height\/:height\/raw@: the blocks at a height, serialized.+handleBlockHeightRaw :: WebState -> GetBlockHeightRaw -> IO [Piece]+handleBlockHeightRaw st (GetBlockHeightRaw h) =+  fmap rawResultList $+    mapM (getRawBlock st) =<< st.query.getBlocksAtHeight (fromIntegral h)++-- GET BlockTime / BlockTimeRaw --++-- | Handler for @GET \/block\/time\/:time@: the last block with a timestamp at or before a time.+handleBlockTime ::+  WebState ->+  GetBlockTime ->+  IO BlockData+handleBlockTime st (GetBlockTime (TimeParam t) (NoTx notx)) =+  withMetrics st (.block) $ do+    let ch = st.config.store.chain+    C.blockAtOrBefore st.query ch t >>= \case+      Nothing -> raise st ThingNotFound+      Just b -> return $ pruneTx notx b++-- | Handler for @GET \/block\/mtp\/:time@: the first block with a median time past at or after a time.+handleBlockMTP ::+  WebState ->+  GetBlockMTP ->+  IO BlockData+handleBlockMTP st (GetBlockMTP (TimeParam t) (NoTx notx)) =+  withMetrics st (.block) $ do+    let ch = st.config.store.chain+    C.blockAtOrAfterMTP st.query ch t >>= \case+      Nothing -> raise st ThingNotFound+      Just b -> return $ pruneTx notx b++-- | Handler for @GET \/block\/time\/:time\/raw@: the last block with a timestamp at or before a time, serialized.+handleBlockTimeRaw :: WebState -> GetBlockTimeRaw -> IO [Piece]+handleBlockTimeRaw st (GetBlockTimeRaw (TimeParam t)) =+  C.blockAtOrBefore st.query st.config.store.chain t >>= \case+    Nothing -> raise st ThingNotFound+    Just b -> return (rawResult (rawBlock st.query b))++-- | Handler for @GET \/block\/mtp\/:time\/raw@: the first block with a median time+-- past at or after a time, serialized.+handleBlockMTPRaw :: WebState -> GetBlockMTPRaw -> IO [Piece]+handleBlockMTPRaw st (GetBlockMTPRaw (TimeParam t)) =+  C.blockAtOrAfterMTP st.query st.config.store.chain t >>= \case+    Nothing -> raise st ThingNotFound+    Just b -> return (rawResult (rawBlock st.query b))++-- GET Transactions --++-- | Handler for @GET \/transaction\/:txid@: a transaction.+handleTx :: WebState -> GetTx -> IO Transaction+handleTx st (GetTx txid) =+  withMetrics st (.tx) $+    C.getTransaction st.query txid >>= maybe (notFoundTx st txid) return++-- | Handler for @GET \/transactions?txids=…@: several transactions.+handleTxs ::+  WebState ->+  GetTxs ->+  IO [Transaction]+handleTxs st (GetTxs txids) =+  withMetrics st (.tx) $+    catMaybes <$> C.getTransactions st.query (nub txids)++-- | Handler for @GET \/transaction\/:txid\/raw@: a transaction, serialized.+handleTxRaw ::+  WebState ->+  GetTxRaw ->+  IO [Piece]+handleTxRaw st (GetTxRaw txid) =+  st.query.getManyTxBytes [txid] >>= \case+    [Just bytes] -> return (rawResult [pieceOf 1 (return (rawHex bytes))])+    _ -> notFoundTx st txid++-- | Handler for @GET \/transactions\/raw?txids=…@: several transactions, serialized. The client names+-- them, so they are all read before the answer begins, in one piece.+handleTxsRaw ::+  WebState ->+  GetTxsRaw ->+  IO [Piece]+handleTxsRaw st (GetTxsRaw txids) = do+  found <- catMaybes <$> st.query.getManyTxBytes (nub txids)+  return [piece (return (listOf (map quoted found)))]+  where+    listOf items = "[" <> mconcat (intersperse "," items) <> "]"+    quoted bytes = "\"" <> rawHex bytes <> "\""++-- | The hashes of the transactions of a block. Responds with an error if+-- not found.+getTxsBlock :: WebState -> H.BlockHash -> IO [TxHash]+getTxsBlock st h = st.query.getBlock h >>= maybe (raise st ThingNotFound) (return . (.txs))++-- | Handler for @GET \/transactions\/block\/:block@: the transactions of a block, in pieces.+handleTxsBlock :: WebState -> GetTxsBlock -> IO [(Int, IO [Transaction])]+handleTxsBlock st (GetTxsBlock h) = transactionPieces st.query <$> getTxsBlock st h++-- | Handler for @GET \/transactions\/block\/:block\/raw@: the transactions of a block, serialized, in pieces.+handleTxsBlockRaw :: WebState -> GetTxsBlockRaw -> IO [Piece]+handleTxsBlockRaw st (GetTxsBlockRaw h) =+  listPieces hex . pieced pieceSize (fmap catMaybes . st.query.getManyTxBytes) =<< getTxsBlock st h+  where+    hex bytes = unsafeToEncoding ("\"" <> rawHex bytes <> "\"")++-- GET TransactionAfterHeight --++-- | Handler for @GET \/transaction\/:txid\/after\/:height@: whether any ancestor+-- of a transaction is a coinbase confirmed after a height.+handleTxAfter ::+  WebState ->+  GetTxAfter ->+  IO (GenericResult (Maybe Bool))+handleTxAfter st (GetTxAfter txid height) =+  withMetrics st (.txAfter) $+    GenericResult . fst <$> (cbAfterHeight st.query (fromIntegral height) txid)++-- | Check if any of the ancestors of this transaction is a coinbase after the+-- specified height. Returns 'Nothing' if answer cannot be computed before+-- hitting limits.+cbAfterHeight ::+  StoreRead ->+  H.BlockHeight ->+  TxHash ->+  IO (Maybe Bool, Int)+cbAfterHeight q height txid =+  inputs n HashSet.empty HashSet.empty [txid]+  where+    n = 10000+    inputs 0 _ _ [] = return (Nothing, 10000)+    inputs i is ns [] =+      let is' = HashSet.union is ns+          ns' = HashSet.empty+          ts = HashSet.toList (HashSet.difference ns is)+       in case ts of+            [] -> return (Just False, n - i)+            _ -> inputs i is' ns' ts+    inputs i is ns (t : ts) =+      C.getTransaction q t >>= \case+        Nothing -> return (Nothing, n - i)+        Just tx+          | height_check tx ->+              if cb_check tx+                then return (Just True, n - i + 1)+                else+                  let ns' = HashSet.union (ins tx) ns+                   in inputs (i - 1) is ns' ts+          | otherwise -> inputs (i - 1) is ns ts+    cb_check = any isCoinbase . (.inputs)+    ins = HashSet.fromList . map (.outpoint.hash) . (.inputs)+    height_check tx =+      case tx.block of+        BlockRef h _ -> h > height+        MemRef _ -> True++-- POST Transaction --++-- | Handler for @POST \/transactions@: check the transaction and hand it+-- over ("Haskoin.Store.Web.Publish"). The answer is 202 with its id and a+-- message once it was handed over, over the RPC of a node or to the peers,+-- and 200 with its id for one that the store has in a block. What the+-- store or a node refuses is 400 with the reason; no peer is 500.+handlePostTx :: WebState -> PostTx -> IO (Status, Encoding)+handlePostTx st (PostTx tx) =+  withMetrics st (.txPost) $+    Publish.publish (publishing st) tx >>= \case+      Publish.InBlock -> return (status200, toEncoding (TxId txid))+      Publish.ToNode -> handed "A full node took the transaction."+      Publish.ToPeers n ->+        handed . T.unwords $+          [ "Handed over to",+            T.pack (show n),+            if n == 1 then "peer." else "peers.",+            "The network decides whether to take it."+          ]+      Publish.Invalid why -> raise st (UserError (show why))+      Publish.Refused why ->+        raise st (UserError ("A node refused the transaction: " <> T.unpack why))+      Publish.Unverified i ->+        raise st . UserError $+          "This server cannot verify the signature of input "+            <> show i+            <> ", and sends to its peers only what it has verified"+      Publish.NoPeers -> raise st ServerError+  where+    txid = txHash tx+    handed message = return (status202, A.pairs ("txid" .= txid <> "message" .= (message :: Text)))++-- | The chain as the store has it, for the checks of a transaction to+-- publish: the height of the best block, and the median of the times of+-- the last eleven blocks.+chainRules :: WebState -> IO Publish.Rules+chainRules st = do+  best <- st.query.getBestBlock >>= maybe (return Nothing) st.query.getBlock+  times <- maybe (return []) (back (11 :: Int)) best+  return+    Publish.Rules+      { net = own.net,+        inputs = \tx -> judgeInputs own.net tx . map Just,+        height = maybe 0 (.height) best,+        medianTime = case Data.List.sort times of+          [] -> 0+          sorted -> fromIntegral (sorted !! (length sorted `div` 2))+      }+  where+    own = st.config.store+    back n b+      | n <= 1 = return [b.header.timestamp]+      | otherwise =+          (b.header.timestamp :) <$> (st.query.getBlock b.header.prev >>= maybe (return []) (back (n - 1)))++-- | Give a transaction to a full node over its JSON-RPC interface+-- (@sendrawtransaction@).+--+-- The node runs every script and holds the transaction to its own rules+-- and its own mempool, says why it refuses, and does not close the+-- connections of the store over what it refuses, as it may over a+-- transaction that a peer sends it. A transaction that the node takes+-- comes back to the store as any other: the node announces it on the+-- connection that the store syncs from.+--+-- It is an option because the store needs nothing of a node but to be+-- its peer, and this needs an account on one. A node that does not+-- answer, or answers what cannot be read, is 'Publish.Unavailable': the+-- transaction then goes to the peers, if the store verified it. A+-- transaction that the node has already was sent.+sendNode :: WebState -> String -> Tx -> IO Publish.Direct+sendNode st url tx =+  tryAny (Wreq.Session.postWith opts st.session url body) >>= \case+    Left e -> unheard (cs (show e))+    Right r -> case A.decode (r ^. Wreq.responseBody) >>= answer of+      Nothing -> unheard ("HTTP " <> cs (show (r ^. Wreq.responseStatus . Wreq.statusCode)))+      Just Nothing -> return Publish.Sent+      Just (Just why)+        | "already" `T.isInfixOf` why -> return Publish.Sent+        | otherwise -> return (Publish.Declined why)+  where+    opts = Wreq.defaults & Wreq.checkResponse ?~ (\_ _ -> return ())+    body =+      A.object+        [ "jsonrpc" .= ("1.0" :: Text),+          "id" .= ("haskoin-store" :: Text),+          "method" .= ("sendrawtransaction" :: Text),+          "params" .= [encodeHex (runPutS (serialize tx))]+        ]+    -- the error of the answer if it has one, nothing if it has a result+    answer v = do+      A.Object top <- Just v+      case KeyMap.lookup "error" top of+        Just (A.Object e) | Just (A.String why) <- KeyMap.lookup "message" e -> Just (Just why)+        Just A.Null | KeyMap.member "result" top -> Just Nothing+        Nothing | KeyMap.member "result" top -> Just Nothing+        _ -> Nothing+    unheard why = do+      logWarn+        (withSource "Publish" st.config.logger)+        ("The node did not take a transaction over RPC, which goes to a peer if it is verified: " <> why)+      return Publish.Unavailable++-- | What "Haskoin.Store.Web.Publish" needs, for a request.+--+-- The peers are those of the store that take transactions. A transaction+-- that was handed over is asked for again after the answer+-- ('askingLoop'), unless the store keeps no mempool.+publishing :: WebState -> Publish.Publishing+publishing st =+  Publish.Publishing+    { rules = chainRules st,+      direct = maybe (\_ -> return Publish.Unavailable) (sendNode st) st.config.nodeRpc,+      holds = fmap heldOf . st.query.getTxData,+      coin = Publish.coinOf st.query,+      peers = map (.address) . filter (maybe False (.relay) . (.version)) <$> getPeers own.peerMgr,+      courier =+        Publish.deliver+          Publish.Courier+            { net = own.net,+              connect = own.connect,+              patience = 10,+              logger = withSource "Courier" st.config.logger+            },+      fetch = Publish.askFor st.asking,+      note = \level -> logAt level (withSource "Publish" st.config.logger)+    }+  where+    own = st.config.store+    heldOf = \case+      Just td+        | td.deleted -> Publish.Absent+        | confirmed td.block -> Publish.Confirmed+        | otherwise -> Publish.Pending+      Nothing -> Publish.Absent++-- | Ask for the transactions that were handed over until the store has+-- them ("Haskoin.Store.Web.Publish".'Publish.Asking'): as if each peer had+-- announced them, so that the block store asks one of them. A failure is+-- logged and the asking goes on.+askingLoop :: WebState -> IO ()+askingLoop st = forever $ do+  tryAny (Publish.askLoop st.asking present ask) >>= \case+    Left e -> logWarn (withSource "Publish" st.config.logger) ("Asking for transactions: " <> cs (show e))+    Right () -> return ()+  threadDelay 1000000+  where+    own = st.config.store+    -- in the chain, or in the pool+    present h =+      (chainRead own.db).getTxData h >>= \case+        Just _ -> return True+        Nothing -> maybe (return False) (\m -> isJust <$> memberPoolBody m h) own.fleet+    ask h = getPeers own.peerMgr >>= mapM_ (\op -> blockStoreTxHash op.mailbox [h] own.block)++-- | Fill the pool again each time the member finds that Redis lost its+-- data ('Haskoin.Store.Fleet.Member.noteEpoch'): ask every peer for its+-- mempool ('blockStoreRefill'), and with the RPC of the store's node+-- (@--node-rpc@), read the node's whole mempool, 'refillBatch'+-- transactions every two seconds, a turn of the member, each given to the+-- member as if a peer had sent it ('memberReceived'). The peers alone may+-- not answer: a node answers a request for its mempool only from a peer+-- that it allows to use bloom filters.+refillLoop :: WebState -> IO ()+refillLoop st = forM_ own.fleet $ \m -> go m 0+  where+    own = st.config.store+    say = withSource "Fleet" st.config.logger+    go m seen = do+      n <- atomically $ do+        n <- readTVar m.refills+        when (n == seen) retry+        return n+      blockStoreRefill own.block+      forM_ st.config.nodeRpc $ \url ->+        tryAny (refillFromNode st m url) >>= \case+          Left e -> logWarn say ("Could not read the mempool of the node over RPC: " <> cs (show e))+          Right k -> logInfo say ("Gave the pool " <> cs (show k) <> " transactions of the node's mempool")+      go m n++-- | How many transactions 'refillLoop' reads from the node at a time.+refillBatch :: Int+refillBatch = 2000++-- | Give a member every transaction of the mempool of a node, read over its+-- RPC (@getrawmempool@, then @getrawtransaction@ in batches of+-- 'refillBatch'). Says how many it gave.+refillFromNode :: WebState -> Member -> String -> IO Int+refillFromNode st m url = do+  ids <-+    nodeCalls st url [("getrawmempool", [])] <&> \case+      [Just (A.Array a)] -> [h | A.String h <- Foldable.toList a]+      _ -> []+  fmap sum . forM (batches ids) $ \hs -> do+    now <- fromIntegral . (floor :: POSIXTime -> Integer) <$> getPOSIXTime+    found <- nodeCalls st url [("getrawtransaction", [A.String h]) | h <- hs]+    let txs = [tx | Just (A.String hex) <- found, Just bytes <- [decodeHex hex], Right tx <- [decode bytes]]+    mapM_ (memberReceived m now) txs+    threadDelay 2000000+    return (length txs)+  where+    batches xs+      | null xs = []+      | otherwise = take refillBatch xs : batches (drop refillBatch xs)++-- | The results of calls to the JSON-RPC of a node, made in one batch, in+-- their order: 'Nothing' for one that failed.+nodeCalls :: WebState -> String -> [(Text, [A.Value])] -> IO [Maybe A.Value]+nodeCalls st url calls = do+  r <- Wreq.Session.postWith opts st.session url (A.toJSON (zipWith call [0 :: Int ..] calls))+  let answers = case A.decode (r ^. Wreq.responseBody) of+        Just (A.Array a) ->+          HashMap.fromList+            [ (i, v)+            | A.Object o <- Foldable.toList a,+              Just (A.String i) <- [KeyMap.lookup "id" o],+              Just v <- [KeyMap.lookup "result" o],+              v /= A.Null+            ]+        _ -> HashMap.empty+  return [HashMap.lookup (T.pack (show i)) answers | i <- [0 .. length calls - 1]]+  where+    opts = Wreq.defaults & Wreq.checkResponse ?~ (\_ _ -> return ())+    call i (method, params) =+      A.object+        [ "jsonrpc" .= ("1.0" :: Text),+          "id" .= T.pack (show i),+          "method" .= method,+          "params" .= params+        ]++-- GET Mempool / Events --++-- | Handler for @GET \/mempool@: hashes of mempool transactions, newest first.+handleMempool ::+  WebState ->+  GetMempool ->+  IO [TxHash]+handleMempool st (GetMempool limitM (OffsetParam o)) =+  withMetrics st (.mempool) $ do+    WebLimits {..} <- pure st.config.limits+    let wl' = WebLimits {maxItemCount = 0, ..}+        l = Limits (validateLimit wl' False limitM) (fromIntegral o) Nothing+    map snd <$> st.query.getMempoolPage l++-- | Middleware that serves @\/events@ over a WebSocket: each new block and+-- each transaction that enters or leaves the mempool is a message, from the+-- log of events ("Haskoin.Store.Web.Events").+--+-- The client may send the commands of t'EventCommand' to choose the+-- transactions that it hears about. Each side speaks JSON in text frames;+-- a binary frame is a command that cannot be read. A command that the server cannot read+-- or cannot do closes the connection with the reason, as a node does with+-- an SPV client: there are no answers to wait for.+--+-- @?after=@ with the hash of the last event heard takes up from there+-- ('eventsStart'). Clients that do not answer pings, or that fall off the+-- start of the log, are disconnected.+webSocketEvents :: WebState -> Middleware+webSocketEvents s =+  websocketsOr defaultConnectionOptions (wrap . events)+  where+    pingPong = defaultPingPongOptions {pingInterval = 30, pongTimeout = 60}+    gauge = (.events) <$> s.metrics+    net = s.config.store.net+    gap = s.query.maxGap+    wrap f = case gauge of+      Nothing -> f+      Just g -> withGaugeIO g f+    events pending = do+      let request = pendingRequest pending+          (path, query) = C.break (== '?') (requestPath request)+          asked = parseQuery query+      if path /= "/events"+        then turnAway pending ThingNotFound+        else+          try (eventsOpen s (\name -> T.decodeUtf8Lenient <$> join (lookup (T.encodeUtf8 name) asked))) >>= \case+            Left u -> turnBusy pending u+            Right (Left e) -> turnAway pending e+            Right (Right (start, watch)) -> do+              conn <- acceptRequest pending+              -- websockets only sees pongs while reading, which the+              -- thread that hears the commands does+              withPingPong pingPong conn $ \c ->+                race_ (hear c watch) (tell c watch start) `catch` gone+    -- a client that leaves is not an error to report+    gone :: WebSockets.ConnectionException -> IO ()+    gone _ = return ()+    -- the commands of the client, until one cannot be done+    hear c watch = do+      message <- WebSockets.receiveDataMessage c+      let read' = case message of+            WebSockets.Text bytes _ -> A.decode bytes >>= parseMaybe (unmarshalValue net)+            WebSockets.Binary _ -> Nothing+      case read' of+        Nothing -> refuse c "Could not read the command"+        Just cmd ->+          try (try (command gap (watching s) watch cmd)) >>= \case+            Right (Right (Right ())) -> hear c watch+            Right (Right (Left why)) -> refuse c why+            Right (Left (_ :: Except)) -> refuse c "Could not do the command"+            Left u -> busy c u+    -- 1008: the message is against what the server allows+    refuse c why = WebSockets.sendCloseCode c 1008 (T.pack why)+    -- 1013: try again later, as 429 or 503 says to a request, for an xpub+    -- that the indexer refused or could not read+    busy c u = WebSockets.sendCloseCode c 1013 (T.pack ("Busy: try again in " <> show (secondsOf u) <> " s"))+    secondsOf = \case+      NotYet t -> retryAfter t+      JobBroke -> 1+    -- the 429 or 503 of 'unindexedResponse', before the websocket opens+    turnBusy pending u =+      let r = unindexedResponseOf u+       in rejectRequestWith+            pending+            WebSockets.defaultRejectRequest+              { WebSockets.rejectBody = case u of+                  NotYet _ -> "{\"error\":\"server-busy\"}"+                  JobBroke -> L.toStrict (A.encode ServerError),+                WebSockets.rejectCode = statusCode (responseStatus r),+                WebSockets.rejectMessage = statusMessage (responseStatus r),+                WebSockets.rejectHeaders = responseHeaders r+              }+    -- the events from a place in the log on, those that the client chose+    tell c watch from =+      atomically (readEvents s.eventLog from) >>= \case+        Nothing -> WebSockets.sendClose c ("Event queue full" :: Text)+        Just entries -> do+          forM_ entries $ \e -> do+            wanted <- atomically (stateTVar watch (relevant gap e.seen))+            when wanted $ sendTextData c (A.encode e.event)+          tell c watch (from + fromIntegral (length entries))+    turnAway pending e =+      rejectRequestWith+        pending+        WebSockets.defaultRejectRequest+          { WebSockets.rejectBody = L.toStrict (A.encode e),+            WebSockets.rejectCode = statusCode (errStatus e),+            WebSockets.rejectMessage = statusMessage (errStatus e),+            WebSockets.rejectHeaders = [("Content-Type", "application/json")]+          }++-- | Handler for @GET \/events@: stream of new blocks and of transactions+-- that enter or leave the mempool, one per line in JSON, until the client+-- falls off the start of the log.+-- @after@ takes up from an event ('eventsStart').+handleEvents :: WebState -> Req -> IO Response+handleEvents st req = do+  let asked name = lookup name req.params+  (start, watch) <- eventsOpen st asked >>= either (raise st) return+  return . responseStream status200 [(hContentType, "application/json")] $ \write flush' ->+    let send from = do+          flush'+          -- ends the response when the client falls too far behind+          atomically (readEvents st.eventLog from) >>= \case+            Nothing -> return ()+            Just entries -> do+              forM_ entries $ \e -> do+                wanted <- atomically (stateTVar watch (relevant st.query.maxGap e.seen))+                when wanted (write (serial e.event))+              send (from + fromIntegral (length entries))+     in withGaugeIncrease st (.events) (send start)+  where+    serial e = fromEncoding (toEncoding e) <> "\n"++-- | Where a client of the events starts reading the log, and what it hears+-- about from the start, as the parameters of its request say: 'eventsStart'+-- for @after@, and for the rest a choice made of the same commands that a+-- WebSocket client sends later.+--+-- A WebSocket client could only choose once it was connected, and until its+-- first command was read it was sent every transaction of a busy mempool.+-- With these the choice is there before the first event:+--+-- * @quiet=true@: no transactions, only blocks, until something is watched;+-- * @addresses@: a list of addresses to watch, separated by commas;+-- * @xpub@ and @derive@: an extended public key and the types of its+--   addresses, one or several separated by commas, legacy if left out;+-- * @filter@, @functions@, @tweak@, @flags@: a Bloom filter, in hex, with+--   its numbers.+--+-- Any of them but @quiet=false@ makes the client a filtered one. The HTTP+-- stream takes them too, and it is the only way that it has to choose.+-- The place in the log is taken first, so that nothing that happens while+-- a key is read from the database is left out.+eventsOpen :: WebState -> (Text -> Maybe Text) -> IO (Either Except (Word64, TVar Watch))+eventsOpen st asked =+  eventsStart st (asked "after") >>= \case+    Left e -> return (Left e)+    Right start -> case commands of+      Left why -> return (Left (UserError why))+      Right cmds -> do+        watch <- newTVarIO everything+        let run [] = return (Right (start, watch))+            run (cmd : rest) =+              command st.query.maxGap (watching st) watch cmd >>= \case+                Left why -> return (Left (UserError why))+                Right () -> run rest+        run cmds+  where+    own = st.config.store+    commands = do+      quiet <- maybe (Right False) (need "quiet" . readBool) (asked "quiet")+      addresses <- mapM (need "addresses" . textToAddr own.net) (listed "addresses")+      derivs <- mapM (need "derive" . textToDeriveType) (listed "derive")+      keys <- mapM (need "xpub" . xPubImport own.net) (maybe [] pure (asked "xpub"))+      filters <- case asked "filter" of+        Nothing -> Right []+        Just hex -> do+          let number name = maybe (Right 0) (need name . (readNum :: Text -> Maybe Word32)) (asked name)+          functions <- number "functions"+          tweak <- number "tweak"+          flags <- number "flags"+          let json =+                A.object+                  [ "type" A..= A.String "filterload",+                    "filter" A..= hex,+                    "functions" A..= functions,+                    "tweak" A..= tweak,+                    "flags" A..= flags+                  ]+          pure <$> need "filter" (parseMaybe (unmarshalValue own.net) json)+      when (null keys && not (null derivs)) (Left "Parameter derive needs xpub")+      let types = if null derivs then [DeriveNormal] else derivs+      return $+        [EventNone | quiet]+          <> [EventWatch addresses | not (null addresses)]+          <> [EventWatchXPub XPubSpec {key, deriv} | key <- keys, deriv <- types]+          <> filters+    listed name = maybe [] (filter (not . T.null) . T.splitOn ",") (asked name)+    need :: Text -> Maybe a -> Either String a+    need name = maybe (Left ("Could not read parameter " <> T.unpack name)) Right++-- | Where a client of the events starts reading the log: at the end, or+-- after the event that it says it heard last, named by the hash of its+-- block or transaction. 404 when the log has no event with that hash: the+-- client cannot know what it missed, and has to read what it follows again+-- and listen from the end.+eventsStart :: WebState -> Maybe Text -> IO (Either Except Word64)+eventsStart st = \case+  Nothing -> Right <$> atomically (eventsEnd st.eventLog)+  Just hex -> case (H.hexToBlockHash hex, hexToTxHash hex) of+    (Nothing, Nothing) -> return (Left (UserError "The parameter after is not a hash"))+    (block, tx) -> maybe (Left ThingNotFound) Right <$> atomically (eventsAfter st.eventLog block tx)++-- | Write what happens into the log of events for as long as the server+-- runs, each transaction that enters with what the database has of it+-- ('seenOf'), for the clients that choose what they hear about.+--+-- A store alone writes what it publishes. A member of a fleet writes what+-- the fleet tells ('memberEvents'), from the start of what Redis keeps, so+-- that the logs of all the servers of a fleet hold the same events and a+-- client can take up on any of them.+feedEvents :: WebState -> IO ()+feedEvents st = case own.fleet of+  Just member -> memberEvents member True (mapM_ happened . fleet)+  Nothing -> do+    queue <- newTQueueIO+    let listen = mapM_ (writeTQueue queue) . local+    bracket (subscribe own.pub listen) (unsubscribe own.pub) $ \_ ->+      forever (atomically (readTQueue queue) >>= happened)+  where+    own = st.config.store+    local = \case+      StoreBestBlock b -> Just (Left b)+      StoreMempoolNew t -> Just (Right (t, True))+      StoreMempoolDelete t -> Just (Right (t, False))+      _ -> Nothing+    fleet = \case+      FleetBlock h -> Left <$> hash h+      FleetEntered h -> Right . (,True) <$> hash h+      FleetLeft h -> Right . (,False) <$> hash h+    hash :: (Serialize x) => C.ByteString -> Maybe x+    hash = either (const Nothing) Just . decode+    happened = \case+      Left b -> atomically (appendEvent st.eventLog (BlockArrived b))+      Right (t, False) -> atomically (appendEvent st.eventLog (TxLeft t))+      Right (t, True) -> do+        -- a transaction that cannot be read is told to every client+        known <- maybe (st.direct.getTxData t) (`memberPoolBody` t) own.fleet `catchAny` \_ -> return Nothing+        atomically (appendEvent st.eventLog (TxEntered t (seenOf own.net <$> known)))++-- GET Address Transactions --++-- | Handler for @GET \/address\/:address\/transactions@: transactions of an address.+handleAddrTxs ::+  WebState ->+  GetAddrTxs ->+  IO [TxRef]+handleAddrTxs st (GetAddrTxs addr pLimits) =+  withMetrics st (.addressTx) $+    st.query.getAddressTxs addr =<< paramToLimits st False pLimits++-- | Handler for @GET \/address\/transactions?addresses=…@: transactions of several addresses.+handleAddrsTxs ::+  WebState ->+  GetAddrsTxs ->+  IO [TxRef]+handleAddrsTxs st (GetAddrsTxs addrs pLimits) =+  withMetrics st (.addressTx) $+    st.query.getAddressesTxs addrs =<< paramToLimits st False pLimits++-- | Handler for @GET \/address\/:address\/transactions\/full@: full transactions of an address, in the pieces+-- of 'pathPieces'. Which transactions they are is read at once; their+-- bodies are read piece by piece.+handleAddrTxsFull ::+  WebState ->+  GetAddrTxsFull ->+  IO [(Int, IO [Transaction])]+handleAddrTxsFull st (GetAddrTxsFull addr pLimits) = do+  txs <- st.query.getAddressTxs addr =<< paramToLimits st True pLimits+  return (transactionPieces st.query (map (.txid) txs))++-- | Handler for @GET \/address\/transactions\/full?addresses=…@: full transactions of several addresses, in+-- the pieces of 'pathPieces'.+handleAddrsTxsFull ::+  WebState ->+  GetAddrsTxsFull ->+  IO [(Int, IO [Transaction])]+handleAddrsTxsFull st (GetAddrsTxsFull addrs pLimits) = do+  txs <- st.query.getAddressesTxs addrs =<< paramToLimits st True pLimits+  return (transactionPieces st.query (map (.txid) txs))++-- | Transactions by their hashes, in pieces of 'pieceSize' that are each+-- read with one multi-get. The hashes are those that the database gave,+-- of a block or of addresses, so the transactions are there; one that is+-- gone by the time its piece is read is left out, as it is when the list+-- is read at once.+transactionPieces :: StoreRead -> [TxHash] -> [(Int, IO [Transaction])]+transactionPieces q = pieced pieceSize (fmap catMaybes . C.getTransactions q)++-- | Transactions in a piece of a list answered piece by piece. With 20 the+-- first byte leaves a millisecond or two sooner and with 500 some ten+-- later; the time of the whole response is the same with all three.+pieceSize :: Int+pieceSize = 100++-- | Handler for @GET \/address\/:address\/balance@: balance of an address.+handleAddrBalance :: WebState -> GetAddrBalance -> IO Balance+handleAddrBalance st (GetAddrBalance addr) =+  withMetrics st (.addressBalance) $ C.getDefaultBalance st.query addr++-- | Handler for @GET \/address\/balances?addresses=…@: balances of several addresses.+handleAddrsBalance ::+  WebState ->+  GetAddrsBalance ->+  IO [Balance]+handleAddrsBalance st (GetAddrsBalance addrs) =+  withMetrics st (.addressBalance) $ st.query.getBalances addrs++-- | Handler for @GET \/address\/:address\/unspent@: unspent outputs of an address.+handleAddrUnspent ::+  WebState ->+  GetAddrUnspent ->+  IO [Unspent]+handleAddrUnspent st (GetAddrUnspent addr pLimits) =+  withMetrics st (.addressUnspent) $+    st.query.getAddressUnspents addr =<< paramToLimits st False pLimits++-- | Handler for @GET \/address\/unspent?addresses=…@: unspent outputs of several addresses.+handleAddrsUnspent ::+  WebState ->+  GetAddrsUnspent ->+  IO [Unspent]+handleAddrsUnspent st (GetAddrsUnspent addrs pLimits) =+  withMetrics st (.addressUnspent) $+    st.query.getAddressesUnspents addrs =<< paramToLimits st False pLimits++-- GET XPubs --++-- | Handler for @GET \/xpub\/:xpub@: summary of an extended public key.+handleXPub ::+  WebState ->+  GetXPub ->+  IO XPubSummary+handleXPub st (GetXPub xpub deriv _) =+  withMetrics st (.xpub) $+    let xs = XPubSpec xpub deriv+     in xPubSummary xs <$> xPubBalsChecked st xs++-- | Handler for @DELETE \/xpub\/:xpub@: remove an extended public key from+-- the fleet's cache. Returns 'True' if the store has the cache.+handleDelXPub ::+  WebState ->+  DelCachedXPub ->+  IO (GenericResult Bool)+handleDelXPub st (DelCachedXPub xpub deriv) =+  withMetrics st (.xpubDelete) $+    GenericResult <$> case st.config.store.fleet of+      Just member -> Door.xpubDelete member.door (XPubSpec xpub deriv)+      Nothing -> return False++-- | Balances of the addresses of an extended public key ('xPubBals'), from+-- the cache, refusing a wrong kind of address ('checkDerive').+xPubBalsChecked :: WebState -> XPubSpec -> IO [XPubBal]+xPubBalsChecked st xs = do+  bals <- st.query.xPubBals xs+  checkDerive st xs bals+  return bals++-- | Answer with an error if an extended public key has no transactions with+-- the kind of address asked for, but has some with another kind: the request+-- named the wrong kind.+--+-- An xpub does not record which kind of address its wallet derives, and the+-- same key gives different addresses for each kind, so a request with the+-- wrong @derive@ would otherwise get an empty wallet: a valid answer, and a+-- wrong one, that a wallet would show its user as a zero balance. An error+-- says what went wrong and how to fix it.+--+-- The check costs nothing for a wallet that has transactions. For one that+-- looks unused, it reads the balances of the first addresses of each other+-- kind, as many as the store reads to decide that an xpub is unused, in one+-- read per kind. A wallet unused with every kind is a new wallet, and gets its+-- empty answer. Networks without SegWit have one kind only, and no check.+checkDerive :: WebState -> XPubSpec -> [XPubBal] -> IO ()+checkDerive st xs bals =+  when (net.segWit && all (\b -> b.balance.txs == 0) bals) $ do+    used <- filterM usedAs others+    forM_ (take 1 used) $ \d ->+      raise st . UserError $+        unwords+          [ "This xpub has no transactions with derive=" <> name xs.deriv <> ",",+            "but has some with derive=" <> name d <> ";",+            "ask with derive=" <> name d <> "."+          ]+  where+    net = st.config.store.net+    others = filter (/= xs.deriv) [DeriveNormal, DeriveP2SH, DeriveP2WPKH]+    name = T.unpack . deriveTypeToText+    usedAs d = do+      let external = pubSubKey xs.key 0+          addrs =+            map snd . take (fromIntegral st.query.initialGap) $+              deriveAddresses (deriveFunction d) external 0+      any (\b -> b.txs > 0) <$> st.query.getBalances addrs++-- | Transactions of an extended public key, from the cache.+getXPubTxs ::+  WebState ->+  XPubKey ->+  DeriveType ->+  LimitsParam ->+  IO [TxRef]+getXPubTxs st xpub deriv plimits = do+  limits <- paramToLimits st False plimits+  let xs = XPubSpec xpub deriv+  xbals <- xPubBalsChecked st xs+  st.query.xPubTxs xs xbals limits++-- | Handler for @GET \/xpub\/:xpub\/transactions@: transactions of an extended public key.+handleXPubTxs ::+  WebState ->+  GetXPubTxs ->+  IO [TxRef]+handleXPubTxs st (GetXPubTxs xpub deriv plimits _) =+  withMetrics st (.xpubTx) $+    getXPubTxs st xpub deriv plimits++-- | Handler for @GET \/xpub\/:xpub\/transactions\/full@: full transactions of an extended public key, in pieces.+handleXPubTxsFull :: WebState -> GetXPubTxsFull -> IO [(Int, IO [Transaction])]+handleXPubTxsFull st (GetXPubTxsFull xpub deriv plimits _) =+  transactionPieces st.query . map (.txid)+    <$> getXPubTxs st xpub deriv plimits++-- | Handler for @GET \/xpub\/:xpub\/balances@: balances of the used addresses of an extended public key.+handleXPubBalances ::+  WebState ->+  GetXPubBalances ->+  IO [XPubBal]+handleXPubBalances st (GetXPubBalances xpub deriv _) =+  withMetrics st (.xpubBalance) $+    xPubBalsChecked st (XPubSpec xpub deriv)++-- | Handler for @GET \/xpub\/:xpub\/unspent@: unspent outputs of an extended public key.+handleXPubUnspent ::+  WebState ->+  GetXPubUnspent ->+  IO [XPubUnspent]+handleXPubUnspent st (GetXPubUnspent xpub deriv pLimits _) =+  withMetrics st (.xpubUnspent) $ do+    limits <- paramToLimits st False pLimits+    let xspec = XPubSpec xpub deriv+    xbals <- xPubBalsChecked st xspec+    st.query.xPubUnspents xspec xbals limits++---------------------------------------+-- Blockchain.info API Compatibility --+---------------------------------------++-- | Currency symbol of the coin of a network, in the Blockchain.info format.+netBinfoSymbol :: Network -> BinfoSymbol+netBinfoSymbol net+  | net == btc =+      BinfoSymbol+        { code = "BTC",+          symbol = "BTC",+          name = "Bitcoin",+          conversion = 100 * 1000 * 1000,+          after = True,+          local = False+        }+  | net == bch =+      BinfoSymbol+        { code = "BCH",+          symbol = "BCH",+          name = "Bitcoin Cash",+          conversion = 100 * 1000 * 1000,+          after = True,+          local = False+        }+  | otherwise =+      BinfoSymbol+        { code = "XTS",+          symbol = "¤",+          name = "Test",+          conversion = 100 * 1000 * 1000,+          after = False,+          local = False+        }++-- | Currency symbol of a fiat currency from its price ticker.+binfoTickerToSymbol :: Text -> BinfoTicker -> BinfoSymbol+binfoTickerToSymbol code t =+  BinfoSymbol+    { code,+      symbol = t.symbol,+      name,+      conversion = 100 * 1000 * 1000 / t.fifteen, -- sat/usd+      after = False,+      local = True+    }+  where+    name = case code of+      "EUR" -> "Euro"+      "USD" -> "U.S. dollar"+      "GBP" -> "British pound"+      x -> x++-- | Addresses and xpubs in a request parameter, separated by pipes or commas.+getBinfoAddrsParam ::+  WebState ->+  Req ->+  Text ->+  IO (HashSet BinfoAddr)+getBinfoAddrsParam st req name = do+  let net = st.config.store.net+  p <- fromMaybe "" <$> param req name+  if T.null p+    then return HashSet.empty+    else case parseBinfoAddr net p of+      Nothing -> raise st (UserError "invalid address")+      Just xs -> return $ HashSet.fromList xs++-- | Xpubs and addresses in the @active@, @activeP2SH@ and @activeBech32@+-- parameters. Xpubs in each parameter derive legacy, nested SegWit and native+-- SegWit addresses.+getBinfoActive ::+  WebState ->+  Req ->+  IO (HashSet XPubSpec, HashSet Address)+getBinfoActive st req = do+  active <- getBinfoAddrsParam st req "active"+  p2sh <- getBinfoAddrsParam st req "activeP2SH"+  bech32 <- getBinfoAddrsParam st req "activeBech32"+  let xspec d b = (`XPubSpec` d) <$> xpub b+      xspecs =+        HashSet.fromList $+          concat+            [ mapMaybe (xspec DeriveNormal) (HashSet.toList active),+              mapMaybe (xspec DeriveP2SH) (HashSet.toList p2sh),+              mapMaybe (xspec DeriveP2WPKH) (HashSet.toList bech32)+            ]+      addrs = HashSet.fromList $ mapMaybe addr $ HashSet.toList active+  return (xspecs, addrs)+  where+    addr (BinfoAddr a) = Just a+    addr (BinfoXpub _) = Nothing+    xpub (BinfoXpub x) = Just x+    xpub (BinfoAddr _) = Nothing++-- | A kind of address to derive, as in the @derive@ parameter of the xpub+-- endpoints: SegWit kinds only on networks with SegWit.+readDerive :: Network -> Text -> Maybe DeriveType+readDerive net t = do+  d <- textToDeriveType t+  guard (net.segWit || d == DeriveNormal)+  return d++-- | Whether to identify transactions by short number, unless @txidindex@ is false.+getNumTxId :: Req -> IO Bool+getNumTxId req = not <$> paramOr req readBool "txidindex" False++-- | Height of the best header.+getChainHeight :: WebState -> IO H.BlockHeight+getChainHeight st =+  fmap (.height) $ chainGetBest =<< pure st.config.store.chain++-- | Handler for @GET\/POST \/blockchain\/unspent@: unspent outputs of addresses and xpubs, as Blockchain.info.+handleBinfoUnspent :: WebState -> Req -> IO Response+handleBinfoUnspent st req =+  withMetrics st (.binfoUnspent) $ do+    (xspecs, addrs) <- getBinfoActive st req+    numtxid <- getNumTxId req+    limit <- get_limit+    min_conf <- get_min_conf+    let net = st.config.store.net+    height <- getChainHeight st+    let mn u = min_conf > u.confirmations+    xbals <- getXBals st xspecs+    bus <-+      runConduit $+        getBinfoUnspents st.query numtxid height xbals xspecs addrs+          .| (dropWhileC mn >> takeC limit .| sinkList)+    streamEncoding (marshalEncoding net (BinfoUnspents bus))+  where+    get_limit = min 1000 . fromIntegral <$> paramOr req readNat "limit" 250+    get_min_conf = fromIntegral <$> paramOr req readNat "confirmations" 0++-- | Stream the unspent outputs of xpubs and addresses, newest first, in the+-- Blockchain.info format.+getBinfoUnspents ::+  StoreRead ->+  Bool ->+  H.BlockHeight ->+  HashMap XPubSpec [XPubBal] ->+  HashSet XPubSpec ->+  HashSet Address ->+  ConduitT () BinfoUnspent IO ()+getBinfoUnspents q numtxid height xbals xspecs addrs = do+  cs' <- conduits+  joinDescStreams (unspentKey . fst) cs' .| mapC (uncurry binfo)+  where+    binfo u xp =+      let conf = case u.block of+            MemRef _ -> 0+            -- an output from the cache may be in a block that the chain of+            -- this store does not have yet: it is confirmed at least once+            BlockRef h _ -> if h > height then 1 else height - h + 1+       in BinfoUnspent+            { txid = u.outpoint.hash,+              index = u.outpoint.index,+              script = u.script,+              value = u.value,+              confirmations = fromIntegral conf,+              txidx = encodeBinfoTxId numtxid u.outpoint.hash,+              xpub = xp+            }+    conduits = (<>) <$> xconduits <*> pure acounduits+    xconduits = lift $ do+      let f x (XPubUnspent u p) =+            let path = toSoft (listToPath p)+                xp = BinfoXPubPath x.key <$> path+             in (u, xp)+          g x = do+            let h = q.xPubUnspents x (xBals x xbals)+                l = let Limits {..} = def in Limits {limit = 16, ..}+            return $ streamThings h Nothing l .| mapC (f x)+      mapM g (HashSet.toList xspecs)+    acounduits =+      let f u = (u, Nothing)+          h = q.getAddressUnspents+          l = let Limits {..} = def in Limits {limit = 16, ..}+          g a = streamThings (h a) Nothing l .| mapC f+       in map g (HashSet.toList addrs)++-- | Balances of the used addresses of each xpub of a request: those that+-- the cache has, and those that it does not have once the indexer has read+-- them, all asked for at once ('askXPubs'). If any is refused, the request+-- is, with the latest @Retry-After@ of them.+getXBals ::+  WebState ->+  HashSet XPubSpec ->+  IO (HashMap XPubSpec [XPubBal])+getXBals st xspecs = do+  found <- askXPubs ((.door) <$> st.config.store.fleet) st.aside st.reading.reads (HashSet.toList xspecs)+  bals <- forM found $ \(x, cached) -> (x,) <$> maybe (st.query.xPubBals x) return cached+  return (HashMap.fromList [(x, filter (not . nullBalance . (.balance)) b) | (x, b) <- bals])++-- | Balances of an xpub, or none if missing.+xBals :: XPubSpec -> HashMap XPubSpec [XPubBal] -> [XPubBal]+xBals = HashMap.findWithDefault []++-- | Stream the references to the transactions of xpubs and addresses, newest+-- first, without duplicates. Each xpub and address is read page by page, only+-- as far as the consumer needs, in pages of the given size: about as many as+-- the consumer is expected to take, as every page is a query of its own.+binfoTxRefs ::+  StoreRead ->+  Word32 ->+  HashMap XPubSpec [XPubBal] ->+  HashSet XPubSpec ->+  HashSet Address ->+  ConduitT () TxRef IO ()+binfoTxRefs q pageSize xbals sxspecs saddrs =+  joinDescStreams txRefKey $+    map xpub_c (HashSet.toList sxspecs) <> map addr_c (HashSet.toList saddrs)+  where+    l = Limits {limit = pageSize, offset = 0, start = Nothing}+    xpub_c x = streamThings (q.xPubTxs x (xBals x xbals)) (Just (.txid)) l+    addr_c a = streamThings (q.getAddressTxs a) (Just (.txid)) l++-- | Stream the transactions of xpubs and addresses, newest first, in the+-- Blockchain.info format, with the running balance of the given addresses after+-- each, keeping only those that pass the filter.+getBinfoTxs ::+  StoreRead ->+  Word32 -> -- page size of the references, as for 'binfoTxRefs'+  HashMap XPubSpec [XPubBal] -> -- xpub balances+  HashMap Address (Maybe BinfoXPubPath) -> -- address book+  HashSet XPubSpec -> -- show xpubs+  HashSet Address -> -- show addrs+  HashSet Address -> -- balance addresses+  BinfoFilter ->+  Bool -> -- numtxid+  Bool -> -- prune outputs+  Int64 -> -- starting balance+  ConduitT () BinfoTx IO ()+getBinfoTxs+  q+  pageSize+  xbals+  abook+  sxspecs+  saddrs+  baddrs+  bfilter+  numtxid+  prune+  bal =+    binfoTxRefs q pageSize xbals sxspecs saddrs .| bodies .| go bal+    where+      binfo_tx = toBinfoTx numtxid abook prune+      compute_bal_change t =+        let ins = map (.output) t.inputs+            out = t.outputs+            f b BinfoTxOutput {..} =+              let val = fromIntegral value+               in case address of+                    Nothing -> 0+                    Just a+                      | a `HashSet.member` baddrs ->+                          if b then val else negate val+                      | otherwise -> 0+         in sum $ map (f False) ins <> map (f True) out+      -- the bodies, read a page at a time, which is one round trip to Redis+      -- or one multi-get of the database instead of one per transaction+      bodies = do+        refs <- takeC (fromIntegral pageSize) .| sinkList+        unless (null refs) $ do+          lift (C.getTransactions q (map (.txid) refs)) >>= mapM_ yield+          bodies+      go b =+        await >>= \case+          Nothing -> return ()+          Just Nothing -> go b+          Just (Just x) -> do+            let a = binfo_tx b x+                b' = b - compute_bal_change a+                c = isJust a.blockHeight+                d = fst (fromJust a.balance)+                r = d + fromIntegral a.fee+            case bfilter of+              BinfoFilterAll ->+                yield a >> go b'+              BinfoFilterSent+                | 0 > r -> yield a >> go b'+                | otherwise -> go b'+              BinfoFilterReceived+                | r > 0 -> yield a >> go b'+                | otherwise -> go b'+              BinfoFilterMoved+                | r == 0 -> yield a >> go b'+                | otherwise -> go b'+              BinfoFilterConfirmed+                | c -> yield a >> go b'+                | otherwise -> go b'+              BinfoFilterMempool+                | c -> return ()+                | otherwise -> yield a >> go b'++-- | Page size to read the references of the transactions in, to show this+-- many after this offset: all of them in one page if they fit in 1000.+pageFor :: Int -> Int -> Word32+pageFor offset n = fromIntegral (max 16 (min 1000 (offset + n)))++-- | Whether to show addresses in CashAddr format (@cashaddr@).+getCashAddr :: Req -> IO Bool+getCashAddr req = paramOr req readBool "cashaddr" False++-- | Parse an address. Responds with not found if invalid.+getAddress :: WebState -> Text -> IO Address+getAddress st txt = do+  let net = st.config.store.net+  case textToAddr net txt of+    Nothing -> raise st ThingNotFound+    Just a -> return a++-- | Address or xpub in a path parameter. Passes to the next route if invalid.+getBinfoAddr :: WebState -> Req -> Text -> IO BinfoAddr+getBinfoAddr st req param' = do+  txt <- capture req param'+  let net = st.config.store.net+  let addr = BinfoAddr <$> textToAddr net txt+      xpub = BinfoXpub <$> xPubImport net txt+  maybe next return (addr <|> xpub)++-- | Handler for @GET\/POST \/blockchain\/export-history@: daily history of the+-- balance of addresses and xpubs with prices, as Blockchain.info.+handleBinfoHistory :: WebState -> Req -> IO Response+handleBinfoHistory st req =+  withMetrics st (.binfoExportHistory) $ do+    (xspecs, addrs) <- getBinfoActive st req+    (startM, endM) <- get_dates+    (code, price') <- getPrice st req+    xbals <- getXBals st xspecs+    let xaddrs = HashSet.fromList $ concatMap (map get_addr) (HashMap.elems xbals)+        aaddrs = xaddrs <> addrs+        cur = price'.fifteen+        cs' = conduits (HashMap.toList xbals) addrs endM+    txs <-+      runConduit $+        joinDescStreams txRefKey cs'+          .| takeWhileC (is_newer startM)+          .| concatMapMC get_transaction+          .| sinkList+    let times = map (\Transaction {..} -> timestamp) txs+    let net = st.config.store.net+    let url = st.config.priceHistoryURL+    let session = st.session+    let logger = st.config.logger+    rates <- map (.price) <$> (getRates logger net session url code times)+    let hs = zipWith (convert cur aaddrs) txs (rates <> repeat 0.0)+    streamEncoding $ toEncoding hs+  where+    is_newer+      (Just BlockData {height = bh})+      TxRef {block = BlockRef {height = th}} =+        bh <= th+    is_newer Nothing TxRef {} = True+    is_newer _ _ = undefined+    get_addr = (.balance.address)+    get_transaction TxRef {txid = h} =+      C.getTransaction st.query h+    convert cur addrs tx rate =+      let ins = tx.inputs+          outs = tx.outputs+          fins = filter (input_addr addrs) ins+          fouts = filter (output_addr addrs) outs+          vin = fromIntegral . sum $ map (.value) fins+          vout = fromIntegral . sum $ map (.value) fouts+          fee = tx.fee+          v = vout - vin+          t = tx.timestamp+          h = txHash $ transactionData tx+       in toBinfoHistory v t rate cur fee h+    input_addr addrs' StoreInput {address = Just a} =+      a `HashSet.member` addrs'+    input_addr _ _ = False+    output_addr addrs' StoreOutput {address = Just a} =+      a `HashSet.member` addrs'+    output_addr _ _ = False+    get_dates = do+      BinfoDate start <- requireParam st req parseBinfoDate "start"+      BinfoDate end' <- requireParam st req parseBinfoDate "end"+      let end = end' + 24 * 60 * 60+      let ch = st.config.store.chain+      startM <- C.blockAtOrAfter st.query ch start+      endM <- C.blockAtOrBefore st.query ch end+      return (startM, endM)+    conduits xpubs addrs endM =+      map (uncurry (xpub_c endM)) xpubs+        <> map (addr_c endM) (HashSet.toList addrs)+    lim endM =+      Limits+        { limit = 16,+          offset = 0,+          start = AtBlock . (.height) <$> endM+        }+    addr_c endM a =+      streamThings (st.query.getAddressTxs a) (Just (.txid)) (lim endM)+    xpub_c endM x bs =+      streamThings (st.query.xPubTxs x bs) (Just (.txid)) (lim endM)++-- | Currency code in the @currency@ parameter, USD by default, in any case,+-- and its latest price. A currency that the price service does not list is+-- refused. Until the server has prices, which it may never have if the+-- service cannot be reached, every currency has a price of zero, rather than+-- every request being refused.+getPrice :: WebState -> Req -> IO (Text, BinfoTicker)+getPrice st req = do+  prices <- readTVarIO st.ticker+  let known c+        | HashMap.null prices = Just (c, def)+        | otherwise = (c,) <$> HashMap.lookup c prices+  paramOr req (known . T.toUpper) "currency" ("USD", fromMaybe def (HashMap.lookup "USD" prices))++-- | Symbol of the currency in the @currency@ parameter.+getSymbol :: WebState -> Req -> IO BinfoSymbol+getSymbol st req = uncurry binfoTickerToSymbol <$> getPrice st req++-- | Handler for @GET \/blockchain\/blocks\/:milliseconds@: the blocks of the day+-- before a time in milliseconds, as Blockchain.info.+handleBinfoBlocksDay :: WebState -> Req -> IO Response+handleBinfoBlocksDay st req =+  withMetrics st (.binfoBlock) $ do+    t <- min h . (`div` 1000) <$> captureWith req readNum "milliseconds"+    let ch = st.config.store.chain+    m <- C.blockAtOrBefore st.query ch t+    bs <- go (d t) m+    streamEncoding $ toEncoding $ map toBinfoBlockInfo bs+  where+    h = fromIntegral (maxBound :: H.Timestamp)+    d = subtract (24 * 3600)+    go _ Nothing = return []+    go t (Just b)+      | b.header.timestamp <= fromIntegral t =+          return []+      | otherwise = do+          b' <- st.query.getBlock b.header.prev+          (b :) <$> go t b'++-- | Handler for @GET\/POST \/blockchain\/multiaddr@: balances and transactions of+-- addresses and xpubs, as Blockchain.info.+handleMultiAddr :: WebState -> Req -> IO Response+handleMultiAddr st req =+  withMetrics st (.binfoMultiaddr) $ do+    (addrs', _, saddrs, sxpubs, xspecs) <- get_addrs+    numtxid <- getNumTxId req+    cashaddr <- getCashAddr req+    local' <- getSymbol st req+    offset <- getBinfoOffset st req+    n <- getBinfoCount st req "n"+    prune <- get_prune+    fltr <- get_filter+    xbals <- getXBals st xspecs+    xtxns <- get_xpub_tx_count xbals xspecs+    let sxbals = only_show_xbals sxpubs xbals+        xabals = compute_xabals xbals+        addrs = addrs' `HashSet.difference` HashMap.keysSet xabals+    abals <- get_abals addrs+    let sxspecs = only_show_xspecs sxpubs xspecs+        sxabals = compute_xabals sxbals+        sabals = only_show_abals saddrs abals+        sallbals = sabals <> sxabals+        sbal = compute_bal sallbals+        allbals = abals <> xabals+        abook = compute_abook addrs xbals+        sxaddrs = compute_xaddrs sxbals+        salladdrs = saddrs <> sxaddrs+        bal = compute_bal allbals+        ibal = fromIntegral sbal+    ftxs <-+      runConduit $+        getBinfoTxs+          st.query+          (pageFor offset n)+          xbals+          abook+          sxspecs+          saddrs+          salladdrs+          fltr+          numtxid+          prune+          ibal+          .| (dropC offset >> takeC n .| sinkList)+    -- counted without reading the transactions, which takes seconds for+    -- xpubs with tens of thousands of them: one xpub has its own count, from+    -- the cache when it is cached; for several xpubs and addresses, the+    -- database counts the distinct transactions of all their addresses+    ntx <- case (HashSet.toList sxspecs, HashSet.null saddrs) of+      ([x], True) -> return (HashMap.findWithDefault 0 x xtxns)+      _ -> fromIntegral <$> st.direct.getAddressesTxCount (HashSet.toList salladdrs)+    nfiltered <- case fltr of+      BinfoFilterAll -> return ntx+      _ ->+        runConduit $+          getBinfoTxs+            st.query+            1000+            xbals+            abook+            sxspecs+            saddrs+            salladdrs+            fltr+            numtxid+            prune+            ibal+            .| lengthC+    let net = st.config.store.net+    best <- get_best_block+    peers <- get_peers+    let baddrs = toBinfoAddrs sabals sxbals xtxns+        abaddrs = toBinfoAddrs abals xbals xtxns+        recv = sum $ map (.received) abaddrs+        sent' = sum $ map (.sent) abaddrs+        wallet =+          BinfoWallet+            { balance = bal,+              txs = ntx,+              filtered = nfiltered,+              received = recv,+              sent = sent'+            }+        coin = netBinfoSymbol net+    let block =+          BinfoBlockInfo+            { hash = H.headerHash best.header,+              height = best.height,+              timestamp = best.header.timestamp,+              index = best.height+            }+    let info =+          BinfoInfo+            { connected = peers,+              conversion = 100 * 1000 * 1000,+              fiat = local',+              crypto = coin,+              head = block+            }+    let multiaddr =+          BinfoMultiAddr+            { addresses = baddrs,+              wallet = wallet,+              txs = ftxs,+              info = info,+              recommendFee = True,+              cashAddr = cashaddr+            }+    streamEncoding $ marshalEncoding net multiaddr+  where+    get_xpub_tx_count xbals =+      fmap HashMap.fromList+        . mapM+          ( \x ->+              (x,)+                . fromIntegral+                <$> st.query.xPubTxCount x (xBals x xbals)+          )+        . HashSet.toList+    get_filter = paramOr req parseBinfoFilter "filter" BinfoFilterAll+    get_best_block =+      st.query.getBestBlock >>= \case+        Nothing -> raise st ThingNotFound+        Just bh ->+          st.query.getBlock bh >>= \case+            Nothing -> raise st ThingNotFound+            Just b -> return b+    get_prune = do+      noCompact <- paramOr req readBool "no_compact" False+      not <$> paramOr req readBool "noCompact" noCompact+    only_show_xbals sxpubs =+      HashMap.filterWithKey (\k _ -> k.key `HashSet.member` sxpubs)+    only_show_xspecs sxpubs =+      HashSet.filter (\k -> k.key `HashSet.member` sxpubs)+    only_show_abals saddrs =+      HashMap.filterWithKey (\k _ -> k `HashSet.member` saddrs)+    addr (BinfoAddr a) = Just a+    addr (BinfoXpub _) = Nothing+    xpub (BinfoXpub x) = Just x+    xpub (BinfoAddr _) = Nothing+    get_addrs = do+      (xspecs, addrs) <- getBinfoActive st req+      sh <- getBinfoAddrsParam st req "onlyShow"+      let xpubs = HashSet.map (.key) xspecs+          actives =+            HashSet.map BinfoAddr addrs+              <> HashSet.map BinfoXpub xpubs+          sh' = if HashSet.null sh then actives else sh+          saddrs = HashSet.fromList $ mapMaybe addr $ HashSet.toList sh'+          sxpubs = HashSet.fromList $ mapMaybe xpub $ HashSet.toList sh'+      return (addrs, xpubs, saddrs, sxpubs, xspecs)+    get_abals =+      let f b = (b.address, b)+          g = HashMap.fromList . map f+       in fmap g . st.query.getBalances . HashSet.toList+    get_peers = do+      ps <- getPeersInformation st.config.store.peerMgr+      return (fromIntegral (length ps))+    compute_xabals =+      let f b = (b.balance.address, b.balance)+       in HashMap.fromList . concatMap (map f) . HashMap.elems+    compute_bal =+      let f b = b.confirmed + b.unconfirmed+       in sum . map f . HashMap.elems+    compute_abook addrs xbals =+      let f xs xb =+            let a = xb.balance.address+                e = error "lions and tigers and bears"+                s = toSoft (listToPath xb.path)+             in (a, Just (BinfoXPubPath xs.key (fromMaybe e s)))+          amap =+            HashMap.map (const Nothing) $+              HashSet.toMap addrs+          xmap =+            HashMap.fromList+              . concatMap (uncurry (map . f))+              $ HashMap.toList xbals+       in amap <> xmap+    compute_xaddrs =+      let f = map (.balance.address)+       in HashSet.fromList . concatMap f . HashMap.elems++-- | Number of items requested in a parameter, within the limits.+getBinfoCount :: WebState -> Req -> Text -> IO Int+getBinfoCount st req str = do+  let d = st.config.limits.defItemCount+  let x = st.config.limits.maxFullItemCount+  i <- fromIntegral <$> paramOr req readNat str (fromIntegral d)+  return (fromIntegral (if x == 0 then i else min x i) :: Int)++-- | Offset requested in the @offset@ parameter. Responds with an error if above the maximum.+getBinfoOffset ::+  WebState ->+  Req ->+  IO Int+getBinfoOffset st req = do+  let x = st.config.limits.maxOffset+  o <- fromIntegral <$> paramOr req readNat "offset" 0+  when (o > x) $+    raise st $+      UserError $+        "offset exceeded: " <> show o <> " > " <> show x+  return (fromIntegral o :: Int)++-- | Handler for @GET \/blockchain\/rawaddr\/:addr@: balance and transactions of an+-- address or xpub, as Blockchain.info; also at @\/blockchain\/address\/:addr@+-- and @\/blockchain\/xpub\/:addr@.+handleRawAddr :: WebState -> Req -> IO Response+handleRawAddr st req =+  withMetrics st (.binfoAddressRaw) $+    getBinfoAddr st req "addr" >>= \case+      BinfoAddr addr -> do_addr addr+      BinfoXpub xpub -> do_xpub xpub+  where+    do_xpub xpub = do+      numtxid <- getNumTxId req+      derive <- paramOr req (readDerive st.config.store.net) "derive" DeriveNormal+      let xspec = XPubSpec xpub derive+      n <- getBinfoCount st req "limit"+      off <- getBinfoOffset st req+      xbals <- getXBals st (HashSet.singleton xspec)+      checkDerive st xspec (xBals xspec xbals)+      let net = st.config.store.net+      let summary = xPubSummary xspec (xBals xspec xbals)+          abook = compute_abook xpub (xBals xspec xbals)+          xspecs = HashSet.singleton xspec+          saddrs = HashSet.empty+          baddrs = HashMap.keysSet abook+          bfilter = BinfoFilterAll+          amnt = summary.confirmed + summary.unconfirmed+      txs <-+        runConduit $+          getBinfoTxs+            st.query+            (pageFor off n)+            xbals+            abook+            xspecs+            saddrs+            baddrs+            bfilter+            numtxid+            False+            (fromIntegral amnt)+            .| (dropC off >> takeC n .| sinkList)+      -- n_tx is every transaction of the xpub, as for an address, not the+      -- number on this page+      ntx <- st.query.xPubTxCount xspec (xBals xspec xbals)+      let ra =+            BinfoRawAddr+              { address = BinfoXpub xpub,+                balance = amnt,+                ntx = fromIntegral ntx,+                utxo = summary.utxo,+                received = summary.received,+                sent = fromIntegral summary.received - fromIntegral amnt,+                txs = txs+              }+      streamEncoding $ marshalEncoding net ra+    compute_abook xpub xbals =+      let f xb =+            let a = xb.balance.address+                e = error "black hole swallows all your code"+                s = toSoft $ listToPath xb.path+                m = fromMaybe e s+             in (a, Just (BinfoXPubPath xpub m))+       in HashMap.fromList $ map f xbals+    do_addr addr = do+      numtxid <- getNumTxId req+      n <- getBinfoCount st req "limit"+      off <- getBinfoOffset st req+      bal <- fromMaybe (zeroBalance addr) <$> st.query.getBalance addr+      let net = st.config.store.net+      let abook = HashMap.singleton addr Nothing+          xspecs = HashSet.empty+          saddrs = HashSet.singleton addr+          bfilter = BinfoFilterAll+          balance = bal.confirmed + bal.unconfirmed+      txs <-+        runConduit $+          getBinfoTxs+            st.query+            (pageFor off n)+            HashMap.empty+            abook+            xspecs+            saddrs+            saddrs+            bfilter+            numtxid+            False+            (fromIntegral balance)+            .| (dropC off >> takeC n .| sinkList)+      let ra =+            BinfoRawAddr+              { address = BinfoAddr addr,+                balance,+                ntx = bal.txs,+                utxo = bal.utxo,+                received = bal.received,+                sent = fromIntegral bal.received - fromIntegral balance,+                txs+              }+      streamEncoding $ marshalEncoding net ra++-- | Handler for @GET \/blockchain\/q\/getreceivedbyaddress\/:addr@: total received by an address.+handleBinfoReceived :: WebState -> Req -> IO Response+handleBinfoReceived st req =+  withMetrics st (.binfoQgetreceivedbyaddress) $ do+    a <- getAddress st =<< capture req "addr"+    b <- fromMaybe (zeroBalance a) <$> st.query.getBalance a+    textResponse $ cs $ show b.received++-- | Handler for @GET \/blockchain\/q\/getsentbyaddress\/:addr@: total sent by an address.+handleBinfoSent :: WebState -> Req -> IO Response+handleBinfoSent st req =+  withMetrics st (.binfoQgetsentbyaddress) $ do+    a <- getAddress st =<< capture req "addr"+    b <- fromMaybe (zeroBalance a) <$> st.query.getBalance a+    textResponse $ cs $ show $ b.received - b.confirmed - b.unconfirmed++-- | Handler for @GET \/blockchain\/q\/addressbalance\/:addr@: balance of an address.+handleBinfoAddrBalance :: WebState -> Req -> IO Response+handleBinfoAddrBalance st req =+  withMetrics st (.binfoQaddressbalance) $ do+    a <- getAddress st =<< capture req "addr"+    b <- fromMaybe (zeroBalance a) <$> st.query.getBalance a+    textResponse $ cs $ show $ b.confirmed + b.unconfirmed++-- | Handler for @GET \/blockchain\/q\/addressfirstseen\/:addr@: time of the first+-- block with a transaction of an address.+handleFirstSeen :: WebState -> Req -> IO Response+handleFirstSeen st req =+  withMetrics st (.binfoQaddressfirstseen) $ do+    a <- getAddress st =<< capture req "addr"+    let ch = st.config.store.chain+    bb <- chainGetBest ch+    let top = bb.height+        bot = 0+    i <- go ch bb a bot top+    textResponse $ cs $ show i+  where+    go ch bb a bot top = do+      let mid = bot + (top - bot) `div` 2+          n = top - bot < 2+      x <- hasone a bot+      y <- hasone a mid+      z <- hasone a top+      if+        | x -> getblocktime ch bb bot+        | n -> getblocktime ch bb top+        | y -> go ch bb a bot mid+        | z -> go ch bb a mid top+        | otherwise -> return 0+    getblocktime ch bb h =+      (chainGetAncestor ch h bb) >>= \case+        Just b -> return b.header.timestamp+        Nothing -> do+          let logger = st.config.logger+          logError logger $+            "Could not get ancestor at height "+              <> cs (show h)+              <> " for block "+              <> H.blockHashToHex (H.headerHash bb.header)+          error "Block ancestor retreival error"+    hasone a h = do+      let l = Limits 1 0 (Just (AtBlock h))+      not . null <$> st.query.getAddressTxs a l++-- | Handler for @GET\/POST \/blockchain\/balance@: short balances of addresses and xpubs, as Blockchain.info.+handleShortBal :: WebState -> Req -> IO Response+handleShortBal st req =+  withMetrics st (.binfoBalance) $ do+    (xspecs, addrs) <- getBinfoActive st req+    cashaddr <- getCashAddr req+    let net = st.config.store.net+    abals <- catMaybes <$> mapM (getabal net cashaddr) (HashSet.toList addrs)+    found <- askXPubs ((.door) <$> st.config.store.fleet) st.aside st.reading.reads (HashSet.toList xspecs)+    xbals <- mapM (getxbal net) found+    let res = HashMap.fromList (abals <> xbals)+    streamEncoding $ toEncoding res+  where+    shorten bal =+      BinfoShortBal+        { final = bal.confirmed + bal.unconfirmed,+          ntx = bal.txs,+          received = bal.received+        }+    getabal net cashaddr a =+      let net' =+            if+              | cashaddr -> net+              | net == bch -> btc+              | net == bchTest -> btcTest+              | net == bchTest4 -> btcTest+              | otherwise -> net+       in case addrToText net' a of+            Nothing -> return Nothing+            Just a' ->+              st.query.getBalance a >>= \case+                Nothing -> return $ Just (a', shorten (zeroBalance a))+                Just b -> return $ Just (a', shorten b)+    getxbal net (xpub, cached) = do+      xbals <- maybe (st.query.xPubBals xpub) return cached+      ntx <- fromIntegral <$> st.query.xPubTxCount xpub xbals+      let val = sum $ map (.balance.confirmed) xbals+          zro = sum $ map (.balance.unconfirmed) xbals+          exs = [x | x@XPubBal {path = 0 : _} <- xbals]+          final = val + zro+          received = sum $ map (.balance.received) exs+          sbl = BinfoShortBal {final, ntx, received}+      return (xPubExport net xpub.key, sbl)++-- | Whether to respond with hex (@format=hex@).+getBinfoHex :: Req -> IO Bool+getBinfoHex req = paramOr req format "format" False+  where+    format "hex" = Just True+    format "json" = Just False+    format _ = Nothing++-- | Handler for @GET \/blockchain\/block-height\/:height@: the blocks at a height, as Blockchain.info, in+-- pieces.+handleBinfoBlockHeight :: WebState -> Req -> IO Response+handleBinfoBlockHeight st req =+  inPieces st (.binfoBlockHeight) binfoType $ do+    numtxid <- getNumTxId req+    height <- captureWith req readNum "height"+    bs <- fmap catMaybes $ st.query.getBlocksAtHeight height >>= mapM (st.query.getBlock)+    blocks <- mapM (binfoBlockPieces st numtxid) bs+    return . around "{\"blocks\":[" "]}" $+      concat [around comma mempty block | (comma, block) <- zip ("" : repeat ",") blocks]++-- | The pieces of a block as Blockchain.info, with its transactions as+-- they are read.+--+-- The block is encoded without its transactions, which are the last thing+-- in it, and they are put where the empty list was. Should the encoding of+-- haskoin-store-data come to end another way, the block is read and+-- encoded whole, as it was before.+binfoBlockPieces :: WebState -> Bool -> BlockData -> IO [Piece]+binfoBlockPieces st numtxid b = do+  nhs <- fmap catMaybes $ st.query.getBlocksAtHeight (b.height + 1) >>= mapM (st.query.getBlock)+  let h x = H.headerHash x.header+      nxt = [h n | n <- nhs, n.header.prev == h b]+      txs = transactionPieces st.query b.txs+      -- as 'toBinfoBlock' makes it, with the number of the transactions+      -- that are not in it yet+      bare =+        BinfoBlock+          { hash = h b,+            version = b.header.version,+            prev = b.header.prev,+            merkle = b.header.merkle,+            timestamp = b.header.timestamp,+            bits = b.header.bits,+            next = nxt,+            fee = b.fee,+            nonce = b.header.nonce,+            ntx = fromIntegral (length b.txs),+            size = b.size,+            index = b.height,+            main = b.main,+            height = b.height,+            weight = b.weight,+            txs = []+          }+      bytes = encodingToLazyByteString (marshalEncoding net bare)+      (before, after) = L.splitAt (L.length bytes - 3) bytes+  if after == "[]}"+    then+      around (lazyByteString before) "}"+        <$> listPieces (marshalEncoding net . toBinfoTxSimple numtxid) txs+    else do+      whole <- map (toBinfoTxSimple numtxid) . concat <$> mapM snd txs+      return [piece (return (fromEncoding (marshalEncoding net (toBinfoBlock b whole nxt))))]+  where+    net = st.config.store.net++-- | The headers of a response in JSON of the Blockchain.info endpoints.+binfoType :: ResponseHeaders+binfoType = [(hContentType, "application/json; charset=utf-8")]++-- | Handler for @GET \/blockchain\/latestblock@: the best block, as Blockchain.info.+handleBinfoLatest :: WebState -> Req -> IO Response+handleBinfoLatest st req =+  withMetrics st (.binfoBlockLatest) $ do+    numtxid <- getNumTxId req+    b <- (bestBlock st.query) >>= maybe (raise st ThingNotFound) return+    streamEncoding $+      toEncoding+        BinfoHeader+          { hash = H.headerHash b.header,+            timestamp = b.header.timestamp,+            index = b.height,+            height = b.height,+            txids = map (encodeBinfoTxId numtxid) b.txs+          }++-- | Handler for @GET \/blockchain\/rawblock\/:block@: a block with its transactions, as Blockchain.info, in+-- pieces.+handleBinfoBlock :: WebState -> Req -> IO Response+handleBinfoBlock st req = do+  numtxid <- getNumTxId req+  hex <- getBinfoHex req+  let kind = if hex then [(hContentType, "text/plain; charset=utf-8")] else binfoType+  inPieces st (.binfoBlockRaw) kind $ do+    bh <-+      captureWith req parseBinfoBlockId "block" >>= \case+        BinfoBlockHash bh -> return bh+        BinfoBlockIndex i -> st.query.getBlocksAtHeight i >>= maybe (raise st ThingNotFound) return . listToMaybe+    b <- maybe (raise st ThingNotFound) return =<< st.query.getBlock bh+    if hex then return (rawBlock st.query b) else binfoBlockPieces st numtxid b++-- | Transaction by hash or short number. Fails with 'TxIndexConflict' if a short number matches several.+getBinfoTx ::+  WebState ->+  BinfoTxId ->+  IO (Either Except Transaction)+getBinfoTx st txid = do+  tx <- case txid of+    BinfoTxIdHash h -> maybeToList <$> C.getTransaction st.query h+    BinfoTxIdIndex i -> C.getNumTransaction st.query i+  case tx of+    [t] -> return $ Right t+    [] -> return $ Left ThingNotFound+    ts ->+      let tids = map (.txid) ts+       in return $ Left (TxIndexConflict tids)++-- | Handler for @GET \/blockchain\/rawtx\/:txid@: a transaction, as Blockchain.info.+handleBinfoTx :: WebState -> Req -> IO Response+handleBinfoTx st req =+  withMetrics st (.binfoTxRaw) $ do+    numtxid <- getNumTxId req+    hex <- getBinfoHex req+    txid <- captureWith req parseBinfoTxId "txid"+    tx <-+      getBinfoTx st txid >>= \case+        Right t -> return t+        Left e -> raise st e+    if hex then hx tx else js numtxid tx+  where+    js numtxid t = do+      let net = st.config.store.net+      streamEncoding $ marshalEncoding net $ toBinfoTxSimple numtxid t+    -- the bytes as stored, of the transaction that the id names+    hx t =+      st.query.getManyTxBytes [t.txid] >>= \case+        [Just bytes] -> return (responseBuilder status200 textType (rawHex bytes))+        _ -> raise st ThingNotFound+    textType = [(hContentType, "text/plain; charset=utf-8")]++-- | Handler for @GET \/blockchain\/q\/txtotalbtcoutput\/:txid@: total value of the outputs of a transaction.+handleBinfoTotalOut :: WebState -> Req -> IO Response+handleBinfoTotalOut st req =+  withMetrics st (.binfoQtxtotalbtcoutput) $ do+    txid <- captureWith req parseBinfoTxId "txid"+    tx <-+      getBinfoTx st txid >>= \case+        Right t -> return t+        Left e -> raise st e+    textResponse $ cs $ show $ sum $ map (.value) tx.outputs++-- | Handler for @GET \/blockchain\/q\/txfee\/:txid@: fee of a transaction.+handleBinfoTxFees :: WebState -> Req -> IO Response+handleBinfoTxFees st req =+  withMetrics st (.binfoQtxfee) $ do+    txid <- captureWith req parseBinfoTxId "txid"+    tx <-+      getBinfoTx st txid >>= \case+        Right t -> return t+        Left e -> raise st e+    let i = sum $ map (.value) $ filter f tx.inputs+        o = sum $ map (.value) tx.outputs+    textResponse . cs . show $ i - o+  where+    f StoreInput {} = True+    f StoreCoinbase {} = False++-- | Handler for @GET \/blockchain\/q\/txresult\/:txid\/:addr@: net amount that a transaction sends to an address.+handleBinfoTxResult :: WebState -> Req -> IO Response+handleBinfoTxResult st req =+  withMetrics st (.binfoQtxresult) $ do+    txid <- captureWith req parseBinfoTxId "txid"+    addr <- getAddress st =<< capture req "addr"+    tx <-+      getBinfoTx st txid >>= \case+        Right t -> return t+        Left e -> raise st e+    let i = toInteger $ sum $ map (.value) $ filter (f addr) tx.inputs+        o = toInteger $ sum $ map (.value) $ filter (g addr) tx.outputs+    textResponse $ cs $ show $ o - i+  where+    f addr StoreInput {address = Just a} = a == addr+    f _ _ = False+    g addr StoreOutput {address = Just a} = a == addr+    g _ _ = False++-- | Handler for @GET \/blockchain\/q\/txtotalbtcinput\/:txid@: total value of the inputs of a transaction.+handleBinfoTotalInput :: WebState -> Req -> IO Response+handleBinfoTotalInput st req =+  withMetrics st (.binfoQtxtotalbtcinput) $ do+    txid <- captureWith req parseBinfoTxId "txid"+    tx <-+      getBinfoTx st txid >>= \case+        Right t -> return t+        Left e -> raise st e+    textResponse $ cs $ show $ sum $ map (.value) $ filter f $ tx.inputs+  where+    f StoreInput {} = True+    f StoreCoinbase {} = False++-- | Handler for @GET \/blockchain\/unconfirmed-transactions@: mempool transactions, as Blockchain.info.+handleBinfoMempool :: WebState -> Req -> IO Response+handleBinfoMempool st req =+  withMetrics st (.binfoMempool) $ do+    numtxid <- getNumTxId req+    offset <- getBinfoOffset st req+    n <- getBinfoCount st req "limit"+    -- a limit of 0 would mean no limit to st.query.getMempoolPage+    txids <-+      if n <= 0+        then return []+        else+          map snd+            <$> st.query.getMempoolPage (Limits (fromIntegral n) (fromIntegral offset) Nothing)+    txs <- catMaybes <$> C.getTransactions st.query txids+    let net = st.config.store.net+    let mem = BinfoMempool $ map (toBinfoTxSimple numtxid) txs+    streamEncoding $ marshalEncoding net mem++-- | Handler for @GET \/blockchain\/q\/getblockcount@: height of the best header.+handleBinfoGetBlockCount :: WebState -> Req -> IO Response+handleBinfoGetBlockCount st _ =+  withMetrics st (.binfoQgetblockcount) $ do+    let ch = st.config.store.chain+    bn <- chainGetBest ch+    textResponse $ TL.pack $ show bn.height++-- | Handler for @GET \/blockchain\/q\/latesthash@: hash of the best header.+handleBinfoLatestHash :: WebState -> Req -> IO Response+handleBinfoLatestHash st _ =+  withMetrics st (.binfoQlatesthash) $ do+    let ch = st.config.store.chain+    bn <- chainGetBest ch+    textResponse $ TL.fromStrict $ H.blockHashToHex $ H.headerHash bn.header++-- | Handler for @GET \/blockchain\/q\/bcperblock@: block subsidy at the next height.+handleBinfoSubsidy :: WebState -> Req -> IO Response+handleBinfoSubsidy st _ =+  withMetrics st (.binfoQbcperblock) $ do+    let ch = st.config.store.chain+    let net = st.config.store.net+    bn <- chainGetBest ch+    textResponse $+      cs $+        show $+          (/ (100 * 1000 * 1000 :: Double)) $+            fromIntegral $+              H.computeSubsidy net (bn.height + 1)++-- | Handler for @GET \/blockchain\/q\/addresstohash\/:addr@: hash160 of an address in hex.+handleBinfoAddrToHash :: WebState -> Req -> IO Response+handleBinfoAddrToHash st req =+  withMetrics st (.binfoQaddresstohash) $ do+    addr <- getAddress st =<< capture req "addr"+    if+      | Just h <- addressHash160 addr ->+          textResponse $ encodeHexLazy $ runPutL $ serialize h+      | Just h <- addressHash256 addr ->+          textResponse $ encodeHexLazy $ runPutL $ serialize h+      | otherwise ->+          textResponse ""++-- | Handler for @GET \/blockchain\/q\/hashtoaddress\/:hash@: pay-to-public-key-hash address of a hash160.+handleBinfoHashToAddr :: WebState -> Req -> IO Response+handleBinfoHashToAddr st req =+  withMetrics st (.binfoQhashtoaddress) $ do+    bs <- maybe next return . decodeHex =<< capture req "hash"+    let net = st.config.store.net+    hash <- either (const next) return $ decode bs+    addr <- maybe next return $ addrToText net $ PubKeyAddress hash+    textResponse $ TL.fromStrict addr++-- | Handler for @GET \/blockchain\/q\/addrpubkey\/:pubkey@: address of a public key.+handleBinfoAddrPubkey :: WebState -> Req -> IO Response+handleBinfoAddrPubkey st req =+  withMetrics st (.binfoQaddrpubkey) $ do+    hex <- capture req "pubkey"+    pubkey <-+      maybe next (return . pubKeyAddr) $+        eitherToMaybe . decodeSerial =<< decodeHex hex+    let net = st.config.store.net+    case addrToText net pubkey of+      Nothing -> raise st ThingNotFound+      Just a -> textResponse $ TL.fromStrict a++-- | Handler for @GET \/blockchain\/q\/pubkeyaddr\/:addr@: public key of an+-- address, found in a transaction input that spends from it.+handleBinfoPubKeyAddr :: WebState -> Req -> IO Response+handleBinfoPubKeyAddr st req =+  withMetrics st (.binfoQpubkeyaddr) $ do+    addr <- getAddress st =<< capture req "addr"+    mi <- strm addr+    i <- case mi of+      Nothing -> raise st ThingNotFound+      Just i -> return i+    pk <- case extr addr i of+      Left e -> raise st $ UserError e+      Right t -> return t+    textResponse $ encodeHexLazy $ L.fromStrict pk+  where+    strm addr = do+      runConduit $ do+        let l = Limits {limit = 8, offset = 0, start = Nothing}+        streamThings (st.query.getAddressTxs addr) (Just (.txid)) l+          .| concatMapMC (C.getTransaction st.query . (.txid))+          .| concatMapC (filter (inp addr) . (.inputs))+          .| headC+    inp addr StoreInput {address = Just a} = a == addr+    inp _ _ = False+    extr addr StoreInput {script, pkscript, witness} = do+      Script sig <- decode script+      Script pks <- decode pkscript+      case addr of+        PubKeyAddress {} ->+          case sig of+            [OP_PUSHDATA _ _, OP_PUSHDATA pub _] ->+              Right pub+            [OP_PUSHDATA _ _] ->+              case pks of+                [OP_PUSHDATA pub _, OP_CHECKSIG] ->+                  Right pub+                _ -> Left "Could not parse scriptPubKey"+            _ -> Left "Could not parse scriptSig"+        WitnessPubKeyAddress {} ->+          case witness of+            [_, pub] -> return pub+            _ -> Left "Could not parse scriptPubKey"+        _ -> Left "Address does not have public key"+    extr _ _ = Left "Incorrect input type"++-- | Handler for @GET \/blockchain\/q\/hashpubkey\/:pubkey@: hash160 of a public key in hex.+handleBinfoHashPubkey :: WebState -> Req -> IO Response+handleBinfoHashPubkey st req =+  withMetrics st (.binfoQhashpubkey) $ do+    pkm <-+      fmap+        (eitherToMaybe . decodeSerial <=< decodeHex)+        (capture req "pubkey")+    addr <- case pkm of+      Nothing -> raise st $ UserError "Could not decode public key"+      Just pk -> return $ pubKeyAddr pk+    if+      | Just h <- addressHash160 addr ->+          textResponse $ encodeHexLazy $ runPutL $ serialize h+      | Just h <- addressHash256 addr ->+          textResponse $ encodeHexLazy $ runPutL $ serialize h+      | otherwise ->+          textResponse ""++-- GET Network Information --++-- | Handler for @GET \/peers@: connected peers.+handlePeers ::+  WebState ->+  GetPeers ->+  IO [PeerInfo]+handlePeers st _ =+  withMetrics st (.peers) $ do+    getPeersInformation =<< pure st.config.store.peerMgr++-- | Obtain information about connected peers from peer manager process.+getPeersInformation ::+  PeerMgr -> IO [PeerInfo]+getPeersInformation mgr =+  mapMaybe toInfo <$> (getPeers mgr)+  where+    toInfo op = do+      ver <- op.version+      return+        PeerInfo+          { userAgent = ver.userAgent.get,+            address = show op.address,+            -- Signed, as the peer sent it. See 'PeerInfo'.+            version = ver.version,+            services = ver.services,+            relay = ver.relay+          }++-- | The route of @GET \/health@. Its status says in one number what a load+-- balancer needs to know:+--+-- * 200: every check passes, Redis answering;+--+-- * 503: a check fails, or Redis does not answer.+--+-- = Why 503 for a store without its Redis+--+-- Redis holds the mempool and the cache of xpubs that every store answers+-- from (@docs\/one-fleet.md@). Without it a store knows no mempool and+-- answers no query about an xpub, so the load balancer must stop sending+-- it requests. The response names what is missing in a @Haskoin-Degraded@+-- header, and the body says whether Redis answers in+-- 'Haskoin.Store.Data.redis'.+pathHealth :: WebState -> Route+pathHealth st =+  route (renderStdMethod (resourceMethod proxy)) (capturePath proxy) [] $ \_ -> do+    h <- handleHealth st GetHealth+    let (status, extra)+          | not (isOK h) = (status503, [("Retry-After", "5")])+          | h.redis == Just False = (status503, [("Retry-After", "5"), ("Haskoin-Degraded", "redis")])+          | otherwise = (status200, [])+    return $+      responseLBS status ((hContentType, "application/json") : extra) (encodingToLazyByteString (toEncoding h))+  where+    proxy = Proxy :: Proxy GetHealth++-- | Handler for @GET \/health@: health of the store.+handleHealth ::+  WebState ->+  GetHealth ->+  IO HealthCheck+handleHealth st _ =+  withMetrics st (.health) $ do+    h <- readTVarIO st.health+    up <- readTVarIO st.cacheUp+    -- a store without a cache has no Redis to be without+    let HealthCheck {blocks, lastBlock, lastTx, pendingTxs, peers, network, version, time} = h+    return HealthCheck {redis = Just up, ..}++-- | Heights of the best header and best block, and the maximum lag allowed.+blockHealthCheck :: WebConfig -> IO BlockHealth+blockHealthCheck cfg = do+  let ch = cfg.store.chain+      db = cfg.store.db+  headers <- (.height) <$> chainGetBest ch+  blocks <-+    Q.getBestBlock db >>= \case+      Nothing -> return 0+      Just h -> maybe 0 (.height) <$> Q.getBlock db h+  return+    BlockHealth+      { headers,+        blocks,+        max = fromIntegral cfg.maxLaggingBlocks+      }++-- | Age of the best header and the maximum allowed.+lastBlockHealthCheck :: Chain -> WebLimits -> IO TimeHealth+lastBlockHealthCheck ch WebLimits {blockTimeout} = do+  n <- round <$> getPOSIXTime+  t <- fromIntegral . (.header.timestamp) <$> (chainGetBest ch)+  return+    TimeHealth+      { age = n - t,+        max = fromIntegral blockTimeout+      }++-- | Age of the newest mempool transaction or best header, and the maximum allowed.+lastTxHealthCheck :: WebConfig -> IO TimeHealth+lastTxHealthCheck WebConfig {store = store', limits} = do+  n <- round <$> getPOSIXTime+  b <- fromIntegral . (.header.timestamp) <$> (chainGetBest ch)+  t <-+    maybe (return Nothing) memberNewest store'.fleet >>= \case+      Just x -> return (max (fromIntegral x) b)+      Nothing -> return b+  return+    TimeHealth+      { age = n - t,+        max = fromIntegral limits.txTimeout+      }+  where+    ch = store'.chain++-- | Transactions waiting to be imported and the maximum allowed.+pendingTxsHealthCheck :: WebConfig -> IO MaxHealth+pendingTxsHealthCheck cfg = do+  n <- blockStorePendingTxs cfg.store.block+  return+    MaxHealth+      { max = fromIntegral cfg.maxPendingTxs,+        count = fromIntegral n+      }++-- | Online peers and the minimum required.+peerHealthCheck :: WebConfig -> IO CountHealth+peerHealthCheck cfg = do+  count <- fromIntegral . length <$> (getPeers cfg.store.peerMgr)+  return CountHealth {min = fromIntegral cfg.minPeers, count}++-- | Run every health check.+healthCheck :: WebConfig -> IO HealthCheck+healthCheck cfg = do+  blocks <- blockHealthCheck cfg+  lastBlock <- lastBlockHealthCheck cfg.store.chain cfg.limits+  lastTx <- lastTxHealthCheck cfg+  pendingTxs <- pendingTxsHealthCheck cfg+  peers <- peerHealthCheck cfg+  time <- round <$> getPOSIXTime+  return+    HealthCheck+      { network = cfg.store.net.name,+        version = cfg.version,+        -- filled in when the check is served, from what the last request+        -- or ping found+        redis = Nothing,+        ..+      }++-- | Handler for @GET \/dbstats@: RocksDB memory use and statistics, as text.+handleDbStats :: WebState -> Req -> IO Response+handleDbStats st _ =+  withMetrics st (.db) $ do+    let db = st.config.store.db.db+    memory <- (memoryReport db)+    statsM <- (getProperty db Stats)+    textResponse . cs $ memory <> maybe "Could not get stats" cs statsM++-----------------------+-- Parameter Parsing --+-----------------------++-- | Returns @Nothing@ if the parameter is not supplied. Raises an exception on+-- parse failure.+paramOptional :: forall a. (Param a) => WebState -> Req -> IO (Maybe a)+paramOptional st req = do+  let net = st.config.store.net+  param req label >>= \case+    Nothing -> return Nothing -- Parameter was not supplied+    Just t -> Just <$> readParam (parseParam net . T.splitOn ",") label t+  where+    label = proxyLabel (Proxy :: Proxy a)++-- | Raises an exception if the parameter is not supplied+paramRequired :: forall a. (Param a) => WebState -> Req -> IO a+paramRequired st req = maybe (raise st err) return =<< paramOptional st req+  where+    label = T.unpack (proxyLabel (Proxy :: Proxy a))+    err = UserError $ "Missing parameter " <> label++-- | Returns the default value of a parameter if it is not supplied. Raises an+-- exception on parse failure.+paramDef :: (Default a, Param a) => WebState -> Req -> IO a+paramDef st req = fromMaybe def <$> paramOptional st req++-- | Does not raise exceptions. Passes the request to the next route ('next') if the parameter is+-- not supplied or if parsing fails.+paramCapture :: forall a. (Param a) => WebState -> Req -> IO a+paramCapture st req = do+  let net = st.config.store.net+  p <- capture req label+  maybe next return (parseParam net (T.splitOn "," p))+  where+    label = proxyLabel (Proxy :: Proxy a)++-- | Read the transaction that a request carries in its body.+parseTxBody :: WebState -> Req -> IO Tx+parseTxBody st req = do+  let net = st.config.store.net+  case decodeTxBody net (L.toStrict req.body) of+    Left _ -> raise st $ UserError "Failed to parse request body"+    Right tx -> return tx++-- | Read a transaction from a request body. The body is the transaction in+-- hex. White space around and inside it is ignored.+--+-- The body must hold one transaction and nothing else. Bytes after the+-- transaction are refused, as Bitcoin Core does for a raw transaction.+--+-- The transaction must be in the format that peers of the network read. A+-- network without SegWit, such as Bitcoin Cash, has the format without+-- witness only. Its nodes would not read the SegWit format, so the store+-- does not take it either.+decodeTxBody :: Network -> C.ByteString -> Either String Tx+decodeTxBody net body =+  case parseBase16 (C.filter (not . isSpace) body) of+    Right b16 -> fromBytes (decodeBase16 b16)+    Left e -> Left (cs e)+  where+    fromBytes = runGetS $ do+      tx <- getNetworkTx net+      done <- isEmpty+      unless done $ fail "Trailing bytes after transaction"+      return tx++-- | Offset in the @offset@ parameter. Responds with an error if above the maximum.+parseOffset :: WebState -> Req -> IO OffsetParam+parseOffset st req = do+  res@(OffsetParam o) <- paramDef st req+  let limits = st.config.limits+  when (limits.maxOffset > 0 && fromIntegral o > limits.maxOffset) $+    raise st . UserError $+      "offset exceeded: " <> show o <> " > " <> show limits.maxOffset+  return res++-- | The best block, if there is one.+bestBlock :: StoreRead -> IO (Maybe BlockData)+bestBlock q = q.getBestBlock >>= maybe (return Nothing) q.getBlock++-- | Turn a start parameter into a position: a transaction, the block of a block+-- hash, a height, or the last block at or before a time.+parseStart ::+  WebState ->+  Maybe StartParam ->+  IO (Maybe Start)+parseStart _ Nothing = return Nothing+parseStart st (Just s) = case s of+  -- a transaction if there is one with that hash, else a block+  StartParamHash {hash = h} ->+    st.query.getTxData (TxHash h) >>= \case+      Just _ -> return (Just (AtTx (TxHash h)))+      Nothing -> fmap (AtBlock . (.height)) <$> st.query.getBlock (H.BlockHash h)+  StartParamHeight {height = h} -> return (Just (AtBlock (fromIntegral h)))+  StartParamTime {time = q} ->+    fmap (AtBlock . (.height)) <$> C.blockAtOrBefore st.query st.config.store.chain q++-- | Limit, offset and start in the parameters of a request.+parseLimits :: WebState -> Req -> IO LimitsParam+parseLimits st req = LimitsParam <$> paramOptional st req <*> parseOffset st req <*> paramOptional st req++-- | Pagination of a query from request parameters, within the limits for full or short items.+paramToLimits ::+  WebState ->+  Bool ->+  LimitsParam ->+  IO Limits+paramToLimits st full (LimitsParam limitM o startM) = do+  let wl = st.config.limits+  Limits (validateLimit wl full limitM) (fromIntegral o) <$> parseStart st startM++-- | Limit of a query: the requested one or the default, but no more than the maximum.+validateLimit :: WebLimits -> Bool -> Maybe LimitParam -> Word32+validateLimit wl full limitM =+  f m $ maybe d (fromIntegral . (.get)) limitM+  where+    m+      | full && wl.maxFullItemCount > 0 = wl.maxFullItemCount+      | otherwise = wl.maxItemCount+    d = wl.defItemCount+    f a 0 = a+    f 0 b = b+    f a b = min a b++---------------+-- Utilities --+---------------++-- | Why a request about xpubs cannot be answered yet: the indexer refused+-- it, saying in how many microseconds to come back (429), or the job of+-- one of its xpubs failed (503). Thrown by the queries of 'xpubQuery', and+-- answered by 'unindexedResponse'.+data Unindexed+  = NotYet !Int+  | JobBroke+  deriving (Eq, Show)++instance Exception Unindexed++-- | The queries that the web server answers with: those of the database,+-- with the door to the fleet's cache of xpubs in front of the queries about+-- xpubs and of the bodies of transactions (@docs\/xpub-indexing.md@).+--+-- * __The balances of an xpub__ come from the cache. On a miss the indexer+--   is asked for the xpub ('xpubAsk'), and the request waits for it, up to+--   the wait of the indexer: once it is indexed, the cache answers;+--   refused, the request is, with 429 and @Retry-After@; if its job failed,+--   with 503. An xpub evicted before the request read it is asked for once+--   more, and then the request has 503.+-- * __Its transactions, outputs and count__ come from the cache too, of+--   the version of the balances. One that is not there, because the entry+--   was evicted since, has the request answered again (v'EntryMoved').+-- * __Bodies of transactions__ come from Redis when it has them, and from+--   the database otherwise.+--+-- The database never answers a query about an xpub (decision of+-- 2026-11-17): every answer is of one version of an entry. There is no+-- setting that turns the cache off. A store without Redis has no door+-- ('Nothing'), and answers 503 to every query about an xpub.+xpubQuery :: Maybe XPubs -> Aside -> Door.Session -> StoreRead -> StoreRead+xpubQuery gate out sess base =+  (base :: StoreRead)+    { getTxData = fmap listToMaybe' . txData . pure,+      getManyTxData = txData,+      xPubBals = balances,+      xPubUnspents = \x bals l -> cached (r.unspents x bals l),+      xPubTxs = \x _ l -> cached (r.transactions x l),+      xPubTxCount = \x _ -> cached (r.count x)+    }+  where+    r = sess.reads+    cached :: IO (Maybe a) -> IO a+    cached ask = ask >>= maybe (throwIO EntryMoved) return+    balances x = r.balances x >>= maybe (asked x (2 :: Int)) return+    -- once indexed, the cache, read at one version as every read of the+    -- request ('xpubSession'); an xpub evicted at once is asked for again+    asked x tries = case gate of+      Nothing -> throwIO JobBroke+      Just xs -> do+        a <- aside out (xpubAsk xs x)+        unindexed [a]+        r.balances x >>= \case+          Just bals -> return bals+          Nothing+            | tries > 1 -> asked x (tries - 1)+            | otherwise -> throwIO JobBroke+    txData [] = return []+    txData ths = do+      fromCache <- r.bodies ths+      let missing = [h | (h, Nothing) <- zip ths fromCache]+      fromDB <- HashMap.fromList . zip missing <$> base.getManyTxData missing+      -- deleted ones are left out, wherever they come from, and the wallets+      -- that the request read say what their entries say of each+      r.wallet . map notDeleted $+        zipWith+          (\h m -> maybe (fromMaybe Nothing (HashMap.lookup h fromDB)) Just m)+          ths+          fromCache++-- | Throw what the answers of the indexer about the xpubs of a request say,+-- if they do not all say that the cache has them now: the latest+-- @Retry-After@ of those refused, or that a job failed.+unindexed :: [Answer] -> IO ()+unindexed answers+  | not (null waits) = throwIO (NotYet (maximum waits))+  | Failed `elem` answers = throwIO JobBroke+  | otherwise = return ()+  where+    waits = [t | Refused _ t <- answers]++-- | The balances of some xpubs from the cache, with their misses all asked+-- for at once, concurrently, and read from the cache once indexed; with+-- 'Nothing' for a miss evicted at once, which the request reads through+-- its queries. If the indexer refused any, this throws, with the latest+-- @Retry-After@ of them ('unindexed'); a store without Redis throws that+-- the job failed (503).+askXPubs :: Maybe XPubs -> Aside -> XPubReads -> [XPubSpec] -> IO [(XPubSpec, Maybe [XPubBal])]+askXPubs gate out r specs = do+  found <- forM specs $ \x -> (x,) <$> r.balances x+  let misses = [x | (x, Nothing) <- found]+  answers <- case (misses, gate) of+    ([], _) -> return []+    (_, Nothing) -> throwIO JobBroke+    (_, Just xs) -> aside out (mapConcurrently (xpubAsk xs) misses)+  unindexed answers+  -- the misses that were indexed, read at one version, as every read of+  -- the request+  indexed <- HashMap.fromList <$> forM misses (\x -> (x,) <$> r.balances x)+  return [(x, maybe (join (HashMap.lookup x indexed)) Just b) | (x, b) <- found]++-- | The balances of an xpub for the filters of @\/events@, which ask the+-- indexer for an xpub that the cache does not have, as every query does.+-- A filter is not made inside a request, so it reads the cache in a+-- session of its own, at the fleet tip, taken again if the tip moves. A+-- refusal turns the HTTP stream away with 429, as any request, and the+-- websocket with 1013, try again later; so does a store that has no view+-- of the fleet tip, or no Redis.+watching :: WebState -> XPubSpec -> IO [XPubBal]+watching st x = go (3 :: Int)+  where+    go n = case st.config.store.fleet of+      Nothing -> throwIO JobBroke+      Just m ->+        memberTip m >>= \case+          Nothing -> throwIO JobBroke+          Just tip ->+            try (answeringCached m tip (balances m)) >>= \case+              Right (Just bals) -> return bals+              Right Nothing | n > 1 -> go (n - 1)+              Left TipMoved | n > 1 -> go (n - 1)+              _ -> throwIO JobBroke+    balances m q sess _ = (xpubQuery (Just m.door) (Aside id) sess q).xPubBals x <* sess.settle++-- | The answer to a request about xpubs that cannot be answered yet: 429+-- with @{"error":"server-busy"}@ and @Retry-After@ in whole seconds, the+-- time that the job is expected to take, or 503 with @Retry-After: 1@ for a job that+-- failed.+unindexedResponse :: WebState -> Unindexed -> IO Response+unindexedResponse st u = do+  forM_ st.metrics $ \m -> case u of+    NotYet _ -> incrementCounter m.refused 1+    JobBroke -> incrementCounter m.serverErrors 1+  return (unindexedResponseOf u)++-- | The response of 'unindexedResponse'.+unindexedResponseOf :: Unindexed -> Response+unindexedResponseOf = \case+  NotYet t ->+    responseLBS+      status429+      [(hContentType, "application/json"), ("Retry-After", C.pack (show (retryAfter t)))]+      "{\"error\":\"server-busy\"}"+  JobBroke ->+    responseLBS+      status503+      [(hContentType, "application/json"), ("Retry-After", "1")]+      (A.encode ServerError)++-- | The only element of a list of one optional value.+listToMaybe' :: [Maybe a] -> Maybe a+listToMaybe' [x] = x+listToMaybe' _ = Nothing++-- | Middleware that logs every request with its response status. Only an+-- error of the server is logged as one: a request that the server could+-- not answer although nothing was wrong with it (500).+--+-- A request that the client got wrong (400, 404) is the client's error,+-- and a public server gets them all day. A 503 is the server saying that+-- it is not ready, which its health check and its fleet already report.+-- Both are logged for debugging, as successes are; logged as errors they+-- hid the ones that are.+--+-- A request that took a number of seconds or more to answer gets a+-- warning, with how long it took (0 for never). That is the one thing+-- about a single request that says something of the server: a query that+-- reads far more than it should, or a database that has become slow. The+-- time runs until the last byte of the response is handed to the+-- connection. A long answer is sent while it is read+-- ("Haskoin.Store.Web.Pieces"), so the time that its status took to make+-- says little: the reads that matter come after it. The price is that a+-- client on a slow line, or a proxy that reads slowly, can make a request+-- slow too. A response cut short, by the client leaving or by a failure,+-- is logged with what it took until then. The stream of events, whose+-- response never ends, is left out.+--+-- With the access log on, every request is logged as information instead+-- of for debugging, with the address of the client and the time it took.+-- It is off by default: a public server has a proxy in front that keeps+-- such a log, and knows the address of the client, where the server sees+-- that of the proxy unless the proxy passes it on in @X-Forwarded-For@.+-- It is for a server that is run without one.+logIt :: Logger -> Bool -> Word32 -> Middleware+logIt logger access slow app req respond = do+  start <- getPOSIXTime+  app req $ \res -> do+    sent <- respond res `onException` report start res True+    report start res False+    return sent+  where+    report start res cut = do+      took <- subtract start <$> getPOSIXTime+      let s = responseStatus res+          msg = fmtReq req <> ": " <> fmtStatus s <> if cut then ", cut short" else ""+          seconds = cs (showFFloat (Just 1) (realToFrac took :: Double) "")+          millis = cs (showFFloat (Just 1) (realToFrac took * 1000 :: Double) "")+          client = case lookup "X-Forwarded-For" (requestHeaders req) of+            Just given -> T.strip (T.takeWhile (/= ',') (T.decodeUtf8Lenient given))+            Nothing -> cs (show (remoteHost req))+      if+        | statusIsServerError s && s /= status503 -> logError logger msg+        | access -> logInfo logger (client <> " " <> msg <> ", " <> millis <> " ms")+        | logger.level <= LevelDebug -> logDebug logger msg+        | otherwise -> return ()       when (slow > 0 && took >= fromIntegral slow && pathInfo req /= ["events"]) $         logWarn logger ("Slow request, " <> seconds <> " seconds: " <> msg) 
+ src/Haskoin/Store/Web/Admission.hs view
@@ -0,0 +1,667 @@+{-# LANGUAGE LambdaCase #-}+{-# LANGUAGE MultiWayIf #-}+{-# LANGUAGE OverloadedRecordDot #-}+{-# LANGUAGE RankNTypes #-}++-- |+-- Module      : Haskoin.Store.Web.Admission+-- Copyright   : (c) Haskoin Developers+-- License     : MIT+-- Maintainer  : jprupp@protonmail.ch+-- Stability   : experimental+-- Portability : POSIX+--+-- A limit on the long requests that are answered at once, so that a server+-- with more work than it can do stays quick at what is quick. The limit+-- moves with how busy the cores are ("Haskoin.Store.Web.Congestion"); the+-- requests over it wait in a line that refuses them when it stops moving+-- (CoDel). @docs/congestion-control.md@ explains the design for a reader+-- new to congestion control.+--+-- The web server answers each request in a thread of its own, and GHC gives+-- every thread a turn, with no priorities. A server that is asked for more+-- than its cores can do therefore slows down for everyone at once: each+-- request takes longer, so more of them are in flight, so each gets fewer+-- turns. Measured on a store of mainnet with 56 connections asking for long+-- lists, on a machine with no core to spare, @\/health@, which only reads+-- memory, took over a second, as did a path that does not exist. Nothing was+-- wrong with either: they waited for their turn behind everything else.+--+-- Here only so many long requests are answered at once: those for a list,+-- for all of a block, for an extended public key. Each holds a slot, and+-- the controller of "Haskoin.Store.Web.Congestion" sets how many slots+-- there are ('regulate'), between a floor and a ceiling. The next requests+-- wait in line, in the order that they came, and are answered as slots come+-- free. A request that the line cannot take is refused, and its client is+-- told to try again, which costs the server nearly nothing. The work in+-- flight stays within what the cores can do, a request that is let in is+-- answered at the speed of a server that is not overloaded, and a load+-- balancer hears at once that this server is full.+--+-- = What is limited+--+-- Each route has a cost ('Haskoin.Store.Web.Router.Cost'), and only the+-- long ones take a slot. Those that answer from memory, like the health+-- check, and those that look up one thing by its key, like a transaction or+-- a balance, are answered at once: they are what tells an operator, or a+-- load balancer, how the server is, and what most clients ask for most.+--+-- The first version of this capped everything but the routes that answer+-- from memory, with two slots a core. Measured on stores of mainnet and+-- testnet under the load above, the health check went from 1.65 seconds+-- to 0.03 at the 95th percentile, and the server answered a third of the+-- requests that it had: a transaction asked for by its id waited in the+-- same line as a block of twenty seconds, and eight requests in flight+-- left the machine idle, since most of the time of a request goes in+-- RocksDB, on a thread of its own, often waiting for the disk.+--+-- = The line: CoDel+--+-- A line is good when it absorbs a burst and empties again, and bad when it+-- stands: every request waits, and none is the better for it. CoDel, as+-- folly and Facebook's servers use it, tells the two apart by the shortest+-- wait of the requests let in during an interval of 100 ms. A line that+-- emptied even once in the interval let someone in at once; one whose+-- shortest wait was over 5 ms stood all along, and is /overloaded/ until+-- the end of the next interval. While it is overloaded, a request that+-- finds no free slot is refused at once, and one that has waited more than+-- 10 ms is refused: when its slot comes, which goes to the next, or before,+-- at the next request or sweep ('sweep'). The line drains in a moment, and+-- the clients that it refuses learn so at once, rather than after waiting+-- for nothing.+--+-- CoDel holds its fire while the controller is free to raise the limit,+-- with the cores under 90% and the limit under its ceiling ('rising'). A+-- line that stands then waits for the limit to catch up with the load, as+-- it does after a quiet spell, and is not a sign that the store is behind.+-- Refused, its clients would wait a second each (@Retry-After@): with the+-- database on a slow disk, sixteen clients that asked one after the other+-- were refused about twice a second, each time losing a second of a+-- client, while the limit climbed.+--+-- While the line is not overloaded, a request waits until it is given a+-- slot, or for at most a limit ('patience'), and a line already holding so+-- many requests ('room') takes no more. Both are bounds for safety, which+-- CoDel should leave untouched.+--+-- A request that waits sets no timer of its own: those that have waited+-- too long are at the front of the line, and are refused there at each+-- request, and ten times a second ('regulate'). Every timer of the program,+-- @threadDelay@ and @registerDelay@ included, goes through one thread of+-- GHC, its timer manager, and with a timer for each request in line, on+-- four cores kept busy, that thread was found blocked for seconds on a+-- thunk that a request had begun to evaluate when it lost its core: no+-- timer of the program fired meanwhile, the health check's included.+--+-- An interval ends at the first request, or sweep, after its end, and one+-- in which nobody was let in counts the wait of the first in line so far:+-- the next one in will have waited at least that long, and a line whose+-- slots are all held by requests of many seconds stands as surely as one+-- that moves slowly.+--+-- = Alternatives+--+-- * /Priorities/ would let the short requests pass the long ones. GHC's+--   scheduler has none, and a queue of our own in front of it would be this+--   module with more than one line.+--+-- * /A limit in the proxy/, on connections or on the rate of requests,+--   protects the server from one client and belongs there too. It does not+--   know what a request costs, nor how many cores the server has.+--+-- * /Refusing at once/, with no line, turns every burst into errors that a+--   wait of a few milliseconds would have absorbed. /A fixed wait/, the+--   five seconds of the first version, keeps every request of a line that+--   stands waiting for its whole length before refusing it.+--+-- = A slot goes from hand to hand+--+-- A request that ends gives its slot to the first in line, who alone is+-- woken, and the slot counts as taken from then on. The first version let+-- the slot come free and had every request in line watch for it: each+-- change woke them all, to find that all but one had to wait again, and a+-- slot that had come free stayed empty until the first in line was given a+-- core. Measured with four slots and sixty requests in line on four cores+-- that forty other threads kept busy, the slots were 0.2 full on average+-- and answered 1,345 requests a second; handing them over, they are always+-- taken and answer 1,457. The gain is small because what a waiting request+-- waits for, once it has its slot, is still its turn on a core, which GHC+-- gives to every thread alike.+--+-- That turn is the price of a limit. With requests that compute for 10 ms,+-- on four cores (@bench/Admission.hs@), a fixed limit of sixteen answered+-- 2% fewer than no limit, and the limit that adapts, at about six slots,+-- 8% fewer: a slot handed to a thread that has yet to run is idle until it+-- does, and fewer slots leave less to fill the gap.+--+-- When the controller cuts the limit, the requests in flight run to their+-- end, and the slots that they leave are not handed on until fewer run than+-- the limit. When it raises the limit, the new slot goes to the first in+-- line at once.+--+-- = A slot is held until the response is sent+--+-- A long response is read and encoded while it is sent+-- ("Haskoin.Store.Web.Pieces"), so the work of a request ends with its last+-- byte, and that is when its slot comes free. The price is that a client+-- that reads slowly holds a slot while it does. A proxy in front that+-- buffers responses, as nginx does by default, takes them at the speed of+-- the server; a server that faces slow clients by itself should have a+-- higher floor.+module Haskoin.Store.Web.Admission+  ( -- * Admission+    Admission,+    Settings (..),+    newAdmission,+    admitted,+    admittedAside,+    Aside (..),+    aside,+    Refusal (..),++    -- * The controller+    regulate,+    Report (..),+    setLimit,+    setClimbing,++    -- * How full the server is+    Load (..),+    Refusals (..),+    load,++    -- * Time, for tests and tools+    newAdmissionWith,+    sweep,+    interval,+    sojournTarget,+  )+where++import Control.Concurrent (getNumCapabilities)+import Control.Concurrent.STM+  ( STM,+    TVar,+    atomically,+    modifyTVar',+    newTVar,+    newTVarIO,+    readTVar,+    retry,+    writeTVar,+  )+import Control.Exception (finally, mask, onException)+import Control.Monad (replicateM_, unless, when)+import Data.IORef (newIORef, readIORef, writeIORef)+import Data.Sequence (Seq, ViewL (..), (|>))+import Data.Sequence qualified as Seq+import GHC.Clock (getMonotonicTimeNSec)+import Haskoin.Store.Web.Congestion+  ( Control (..),+    Delays (..),+    Observation (..),+    Window (..),+    control,+    cpuTime,+    delays,+    rising,+    snooze,+    window,+    withProbes,+  )++-- | How the long requests are let in.+data Settings = Settings+  { -- | most requests answered at once, the ceiling of the limit; none or+    -- fewer is no admission control at all+    ceiling :: !Int,+    -- | fewest: the floor of the limit, where it starts; at least one,+    -- and at most the ceiling+    floor :: !Int,+    -- | microseconds that a thread ready to run may wait for its turn+    -- before the limit comes down ("Haskoin.Store.Web.Congestion")+    target :: !Int,+    -- | requests that may wait in line+    room :: !Int,+    -- | microseconds that a request may wait while the line is not+    -- overloaded+    patience :: !Int+  }+  deriving stock (Eq, Show)++-- | The requests being answered and those that wait, and the limits of both.+data Admission = Admission+  { settings :: !Settings,+    -- | now, in nanoseconds of a monotonic clock; the tests have one that+    -- they move by hand, to test CoDel without waiting for it+    clock :: !(IO Int),+    -- | requests answered at once, which the controller sets+    limit :: !(TVar Int),+    -- | slots taken: by a request being answered, or given to one that+    -- waited and has yet to run; more than the limit after a cut+    running :: !(TVar Int),+    -- | the requests that wait, first come first+    line :: !(TVar (Seq Waiter)),+    -- | whether every slot was taken, or a request waited or was refused,+    -- since the last step of the controller+    saturated :: !(TVar Bool),+    codel :: !(TVar Codel),+    -- | whether the controller is free to raise the limit, as of its last+    -- step; CoDel does not refuse while it is+    climbing :: !(TVar Bool),+    refusals :: !(TVar Refusals),+    -- | nanoseconds that the slots given back since the last step of the+    -- controller were held, in all, and how many+    held :: !(TVar (Int, Int))+  }++-- | A request in line: when it came, and where it is told what became of+-- it.+data Waiter = Waiter+  { arrived :: !Int,+    verdict :: !(TVar Verdict)+  }++-- | What became of a request in line: still waiting, given a slot, refused+-- by CoDel, or refused for waiting too long.+data Verdict = Waiting | Given | Shed | Expired++-- | The state of CoDel.+data Codel = Codel+  { -- | when the interval ends, in nanoseconds+    ends :: !Int,+    -- | the shortest wait of the requests let in during the interval, in+    -- nanoseconds; 'maxBound' while there is none+    least :: !Int,+    -- | whether the line stood during the last interval+    overloaded :: !Bool+  }++-- | Why a request was refused.+data Refusal+  = -- | the line already held as many requests as it may+    LineFull+  | -- | it waited as long as a request may+    WaitedTooLong+  | -- | the line is overloaded (CoDel): it found no free slot, or it had+    -- waited more than twice 'sojournTarget'+    Overloaded+  deriving stock (Eq, Show)++-- | Requests refused since the start, by why.+data Refusals = Refusals+  { full :: !Int,+    waited :: !Int,+    shed :: !Int+  }+  deriving stock (Eq, Show)++-- | Nanoseconds of an interval of CoDel: 100 ms, as in CoDel and folly.+-- It should be about the time that a request takes, which for the long+-- requests of a store is tens to hundreds of milliseconds.+interval :: Int+interval = 100_000_000++-- | Nanoseconds that requests may wait in a line that works: 5 ms, as in+-- CoDel and folly. While the line is overloaded a request that waited+-- twice as long is refused.+sojournTarget :: Int+sojournTarget = 5_000_000++-- | Intervals of CoDel between two steps of the controller: half a second,+-- which holds fifty sleeps of each probe, and a few long requests.+stepIntervals :: Int+stepIntervals = 5++-- | Admission with these settings, on the monotonic clock of the system.+newAdmission :: Settings -> IO Admission+newAdmission = newAdmissionWith (fromIntegral <$> getMonotonicTimeNSec)++-- | Admission with these settings, on this clock, which gives nanoseconds.+-- The limit starts at the floor.+newAdmissionWith :: IO Int -> Settings -> IO Admission+newAdmissionWith clock given = do+  t <- clock+  Admission s clock+    <$> newTVarIO s.floor+    <*> newTVarIO 0+    <*> newTVarIO Seq.empty+    <*> newTVarIO False+    <*> newTVarIO Codel {ends = t + interval, least = maxBound, overloaded = False}+    <*> newTVarIO False+    <*> newTVarIO (Refusals 0 0 0)+    <*> newTVarIO (0, 0)+  where+    s =+      given+        { floor = max 1 (min given.floor given.ceiling),+          room = max 0 given.room,+          patience = max 0 given.patience+        }++-- | What a request is told when it asks for a slot.+data Entry = Enter | Refuse !Refusal | Wait !Waiter++-- | Answer a request in a slot, waiting in line for one if need be; or, if+-- it is refused, do the first action instead, which should cost little: it+-- runs without a slot. The slot is given back when the second action ends,+-- however it ends, and a request that is stopped while it waits leaves the+-- line.+admitted :: Admission -> (Refusal -> IO a) -> IO a -> IO a+admitted a refused = admittedAside a refused . const++-- | 'admitted', with an action that may step 'aside' from its slot.+admittedAside :: Admission -> (Refusal -> IO a) -> (Aside -> IO a) -> IO a+admittedAside a refused action+  | a.settings.ceiling <= 0 = action (Aside id)+  | otherwise = mask $ \restore -> do+      let answer t = do+            since <- newIORef t+            let out = do+                  t' <- a.clock+                  s <- readIORef since+                  atomically (hold a (t' - s) >> release a t')+                back = do+                  a.clock >>= writeIORef since+                  atomically (modifyTVar' a.running (+ 1))+                step = Aside $ \act -> mask $ \inner -> out >> (inner act `finally` back)+            restore (action step) `finally` out+      t <- a.clock+      atomically (entry a t) >>= \case+        Enter -> answer t+        Refuse why -> restore (refused why)+        Wait w ->+          restore (await w) `onException` (a.clock >>= atomically . abandon a w) >>= \case+            Nothing -> a.clock >>= answer+            Just why -> restore (refused why)++-- | How a request in a slot steps aside from it ('aside').+newtype Aside = Aside (forall b. IO b -> IO b)++-- | Give the slot back while the action runs, and take one again when it+-- ends, past the limit if need be, without waiting in line. For a wait+-- that uses no core and that something else bounds: a request for an xpub+-- that is not in the cache waits for the indexer of xpubs, whose threads+-- bound that work. Holding the slot meanwhile would leave a core idle, and+-- a few such waits of seconds would hold every slot, the requests in line+-- waiting behind them for nothing.+aside :: Aside -> IO b -> IO b+aside (Aside f) = f++-- | A slot was held so many nanoseconds.+hold :: Admission -> Int -> STM ()+hold a ns = modifyTVar' a.held (\(total, n) -> (total + max 0 ns, n + 1))++-- | Ask for a slot at the given time.+entry :: Admission -> Int -> STM Entry+entry a t = do+  tick a t+  n <- readTVar a.limit+  busy <- readTVar a.running+  -- nobody waits unless every slot is taken: 'release' and 'setLimit'+  -- see to it+  if busy < n+    then do+      writeTVar a.running (busy + 1)+      when (busy + 1 >= n) (saturate a)+      seen a 0+      return Enter+    else do+      saturate a+      c <- readTVar a.codel+      waiting <- readTVar a.line+      if+        | c.overloaded -> Refuse Overloaded <$ refuse a Overloaded+        | Seq.length waiting >= a.settings.room -> Refuse LineFull <$ refuse a LineFull+        | otherwise -> do+            w <- Waiter t <$> newTVar Waiting+            Wait w <$ writeTVar a.line (waiting |> w)++-- | Wait to be given a slot, or to be refused: 'Nothing' once given one.+await :: Waiter -> IO (Maybe Refusal)+await w =+  atomically $+    readTVar w.verdict >>= \case+      Waiting -> retry+      Given -> return Nothing+      Shed -> return (Just Overloaded)+      Expired -> return (Just WaitedTooLong)++-- | Stopped while waiting: out of the line, or, if a slot had just been+-- given, the slot goes to the next.+abandon :: Admission -> Waiter -> Int -> STM ()+abandon a w t =+  readTVar w.verdict >>= \case+    Given -> release a t+    _ -> leave a w++-- | Out of the line.+leave :: Admission -> Waiter -> STM ()+leave a w = modifyTVar' a.line (Seq.filter ((/= w.verdict) . (.verdict)))++-- | A request ends: its slot goes on, unless more run than the limit.+release :: Admission -> Int -> STM ()+release a t = do+  tick a t+  n <- readTVar a.limit+  busy <- readTVar a.running+  if busy > n then writeTVar a.running (busy - 1) else handOff a t++-- | The slot of a request that ends, or a new one, goes to the first in+-- line, who alone is woken; while the line is overloaded, to the first who+-- has not waited too long, the others being refused; or it comes free.+handOff :: Admission -> Int -> STM ()+handOff a t = do+  c <- readTVar a.codel+  (Seq.viewl <$> readTVar a.line) >>= \case+    EmptyL -> modifyTVar' a.running (subtract 1)+    w :< rest -> do+      writeTVar a.line rest+      -- the clock of another thread may have been read a moment after+      -- this one+      let sojourn = max 0 (t - w.arrived)+      seen a sojourn+      if c.overloaded && sojourn > 2 * sojournTarget+        then writeTVar w.verdict Shed >> refuse a Overloaded >> handOff a t+        else writeTVar w.verdict Given++-- | A request was let in after waiting so many nanoseconds.+seen :: Admission -> Int -> STM ()+seen a sojourn = do+  c <- readTVar a.codel+  when (sojourn < c.least) (writeTVar a.codel c {least = sojourn})++-- | End the interval of CoDel if its time has come, and refuse those that+-- have waited too long: for the line, or, while it is overloaded, for+-- CoDel.+tick :: Admission -> Int -> STM ()+tick a t = do+  c <- readTVar a.codel+  when (t >= c.ends) $ do+    shortest <-+      if c.least < maxBound+        then return c.least+        else+          -- nobody was let in: the next one will have waited at least as+          -- long as the first in line has+          readTVar a.line >>= \l -> return $ case Seq.viewl l of+            w :< _ -> max 0 (t - w.arrived)+            EmptyL -> 0+    up <- readTVar a.climbing+    writeTVar a.codel Codel {ends = t + interval, least = maxBound, overloaded = shortest > sojournTarget && not up}+  readTVar a.codel >>= front . (.overloaded)+  where+    -- the line is in the order of arrival, so those who waited longest+    -- are at its front+    front overloaded =+      (Seq.viewl <$> readTVar a.line) >>= \case+        w :< rest+          | waited w >= a.settings.patience * 1000 -> out w rest Expired WaitedTooLong >> front overloaded+          | overloaded && waited w > 2 * sojournTarget -> do+              seen a (waited w)+              out w rest Shed Overloaded+              front overloaded+        _ -> return ()+    waited w = max 0 (t - w.arrived)+    out w rest verdict why = writeTVar a.line rest >> writeTVar w.verdict verdict >> refuse a why++-- | End the interval of CoDel if its time has come, and refuse those that+-- have waited too long, as every request does, and 'regulate' ten times a+-- second.+sweep :: Admission -> IO ()+sweep a = a.clock >>= atomically . tick a++-- | Every slot is taken, or a request could not have one at once.+saturate :: Admission -> STM ()+saturate a = readTVar a.saturated >>= \s -> unless s (writeTVar a.saturated True)++-- | Count a refusal.+refuse :: Admission -> Refusal -> STM ()+refuse a why = modifyTVar' a.refusals $ \r -> case why of+  LineFull -> r {full = r.full + 1}+  WaitedTooLong -> r {waited = r.waited + 1}+  Overloaded -> r {shed = r.shed + 1}++-- | Set the limit, as the controller does. Slots that it adds go to those+-- in line at once. The limit is kept between the floor and the ceiling.+setLimit :: Admission -> Int -> IO ()+setLimit a n = a.clock >>= atomically . resize a n++-- | Say whether the limit is free to rise, as the controller does after+-- each step. While it is, CoDel does not refuse: a line that stands is+-- waiting for the limit to grow.+setClimbing :: Admission -> Bool -> IO ()+setClimbing a = atomically . writeTVar a.climbing++-- | Set the limit at the given time.+resize :: Admission -> Int -> Int -> STM ()+resize a n t = do+  tick a t+  old <- readTVar a.limit+  when (m /= old) (writeTVar a.limit m)+  fill+  where+    m = max a.settings.floor (min a.settings.ceiling n)+    fill = do+      busy <- readTVar a.running+      waiting <- readTVar a.line+      when (busy < m && not (Seq.null waiting)) $+        modifyTVar' a.running (+ 1) >> handOff a t >> fill++-- | What a step of the controller did.+data Report = Report+  { -- | the limit after the step+    limit :: !Int,+    -- | the 90th percentile of the probes' lateness during the step,+    -- without the pauses of GHC's collector, in microseconds: what the+    -- controller follows+    delay :: !Int,+    -- | the same with the collector's pauses, in microseconds+    lateness :: !Int,+    -- | the use of the cores, smoothed over about two seconds, in+    -- hundredths of the capabilities+    used :: !Int,+    -- | microseconds that a request held its slot, smoothed+    roundTrip :: !Int,+    -- | whether the line is overloaded+    overloaded :: !Bool+  }+  deriving stock (Eq, Show)++-- | Run the controller and its probes, a step every half second for as long+-- as it runs, giving each t'Report' to the function, and 'sweep' every+-- interval of CoDel in between. It returns at once when admission control+-- is off, and starts no probe.+--+-- The use of the cores is the time of the cores that the program used, all+-- of its threads included, over that of its capabilities, smoothed with a+-- time constant of two seconds: long enough to see through the bursts of a+-- few requests, short enough to follow a change of load. The round trip+-- is the time that a slot is held, smoothed with a gain of a quarter, as+-- TCP smooths its round trip.+regulate :: Admission -> (Report -> IO ()) -> IO ()+regulate a report+  | a.settings.ceiling <= 0 = return ()+  | otherwise = withProbes $ \probes -> do+      caps <- getNumCapabilities+      t0 <- a.clock+      c0 <- cpuTime+      let go w used rtt before cpuBefore = do+            replicateM_ (stepIntervals - 1) (snooze (interval `div` 1000) >> sweep a)+            snooze (interval `div` 1000)+            d <- delays probes+            t <- a.clock+            cpu <- cpuTime+            let dt = max 1 (t - before)+                now = fromIntegral (cpu - cpuBefore) / fromIntegral (dt * caps) :: Double+                used' = used + (1 - exp (negate (fromIntegral dt / 2e9))) * (now - used)+            (w', rtt', r) <- atomically (step d t dt used' rtt w)+            report r+            go w' used' rtt' t cpu+      go (window settings) 0 0 t0 c0+  where+    settings = Control {floor = a.settings.floor, ceiling = a.settings.ceiling, target = a.settings.target}+    step d t dt used rtt w = do+      n <- readTVar a.limit+      busy <- readTVar a.running+      full <- readTVar a.saturated+      writeTVar a.saturated False+      (total, ended) <- readTVar a.held+      writeTVar a.held (0, 0)+      let rtt'+            | ended > 0 = rtt + (fromIntegral (total `div` ended) / 1000 - rtt) / 4 :: Double+            | otherwise = rtt+          seenNow =+            Observation+              { delay = d.scheduling,+                used = round (100 * used),+                saturated = full || busy >= n,+                roundTrip = round rtt',+                elapsed = dt `div` 1000+              }+          -- from the limit set by hand since the step before, if it was+          from+            | n == w.limit = w+            | otherwise = (window settings) {limit = n, level = fromIntegral n}+          w' = control settings seenNow from+      resize a w'.limit t+      writeTVar a.climbing (rising settings seenNow w')+      c <- readTVar a.codel+      return+        ( w',+          rtt',+          Report+            { limit = w'.limit,+              delay = d.scheduling,+              lateness = d.raw,+              used = round (100 * used),+              roundTrip = round rtt',+              overloaded = c.overloaded+            }+        )++-- | How full the server is.+data Load = Load+  { -- | requests that may be answered at once+    limit :: !Int,+    -- | requests being answered in a slot+    busy :: !Int,+    -- | requests waiting for a slot+    waiting :: !Int,+    -- | whether the line is overloaded (CoDel)+    overloaded :: !Bool,+    -- | requests refused since the start+    refused :: !Refusals+  }+  deriving stock (Eq, Show)++-- | How full the server is now.+load :: Admission -> IO Load+load a = atomically $ do+  c <- readTVar a.codel+  Load+    <$> readTVar a.limit+    <*> readTVar a.running+    <*> (Seq.length <$> readTVar a.line)+    <*> pure c.overloaded+    <*> readTVar a.refusals
+ src/Haskoin/Store/Web/Congestion.hs view
@@ -0,0 +1,473 @@+-- |+-- Module      : Haskoin.Store.Web.Congestion+-- Copyright   : (c) Haskoin Developers+-- License     : MIT+-- Maintainer  : jprupp@protonmail.ch+-- Stability   : experimental+-- Portability : POSIX+--+-- How late GHC's scheduler runs a thread that is ready, measured by probes+-- on each capability ('withProbes', 'delays'), and the controller that+-- follows it ('control'): a limit on how many long requests run at once+-- ("Haskoin.Store.Web.Admission"), cut by a quarter when the cores do not+-- keep up and raised while they have room, as TCP CUBIC does with its+-- window.+-- @docs/congestion-control.md@ explains the signal and the controller for a+-- reader new to congestion control.+--+-- No fixed number is right. A request that waits for RocksDB on the disk+-- gives its core back while it waits, in a safe foreign call, and the disk+-- of a production store answered 31,000 reads a second with 32 of them in+-- flight: such requests want many slots. A request that computes wants+-- about one a core, and more of them only make every thread, the health+-- check's included, wait longer for its turn. A cap of four a core was+-- wrong for both, and nobody retunes it as the requests change.+--+-- = The signal: how late a thread that the network woke runs+--+-- A probe on each capability waits for a bell, rung every 10 ms from+-- outside GHC, the way a connection waits for its socket, and notes how+-- late it ran ('withProbes'). GHC's manager of input and output wakes it,+-- and it goes to the back of its capability's queue of threads that are+-- ready, each of which runs for up to GHC's timeslice of 20 ms; when the+-- manager itself is in that queue rather than waiting in epoll, its turn+-- comes first. So the lateness is the length of that queue in time, as a+-- request that the network woke finds it. It rises only when the+-- capabilities have more to do than they can, whatever the requests cost;+-- a thread that waits for the disk in a safe foreign call is not in the+-- queue and does not raise it; and it is what a health check, or any short+-- request, waits before it runs.+--+-- = Without the pauses of the collector+--+-- A probe that is due while GHC's collector has stopped every thread wakes+-- only when the collection ends, late by as long as it lasted. Counted as+-- it is, that lateness says that the cores are behind when they are not:+-- the collection stops the idle cores as much as the busy ones, and fewer+-- slots would not make it shorter. So each wake is also counted+-- without the time that the collector took between the probe's sleep and+-- its wake, as GHC reports it ('getRTSStats', with @+RTS -T@), and+-- 'delays' gives both percentiles, with the time the collector took in+-- all. The limit follows the one without.+-- A collection that falls in the sleep and ends before the probe is due is+-- taken out too, which counts the wake as a little earlier than it was.+-- Without the statistics of GHC both are the same.+--+-- The latency of the requests themselves is not the signal, which rules out+-- controllers such as TCP Vegas, Netflix's Gradient2 or BBR, which compare+-- the latency of a request with the least seen. The long routes differ by a+-- factor of a thousand in cost, with the route and the parameters, so a+-- slower request says nothing about the server: it may just be a bigger+-- one.+--+-- = The control: cut once a round trip, grow as TCP CUBIC does+--+-- Every half second the controller takes the 90th percentile of the+-- probes' lateness without the collector's pauses, and the use of the+-- cores, smoothed over about two seconds ('cpuTime'), and moves the limit+-- ('control'):+--+-- * __Over the target__ (25 ms by default) __with the cores used at 90%__:+--   the limit is cut by a quarter, at most once a round trip, the time that+--   a request holds its slot, smoothed. A cut takes effect as requests end,+--   and a second cut before it has would answer one congestion twice, as+--   TCP cuts its window once a round trip.+--+-- * __Under the target, every slot used, and the cores used under 90%__:+--   the limit grows along the cubic curve of TCP CUBIC, concave back+--   toward the level that it was last cut from, flat near it, and convex+--   past it. It finds again at once the level that worked, and searches+--   carefully around it. The time on the curve runs only while the limit+--   may grow, so a limit held back for a while does not leap when it may+--   grow again.+--+-- The limit stays between a floor, two a core by default, and a ceiling.+-- A floor equal to the ceiling is a fixed cap.+--+-- The use of the cores is what keeps the limit from costing throughput, and+-- from growing when it is no use. An earlier controller added one slot a+-- step while the probes were under the target, and cut whenever they were+-- over it and the last cut had taken effect. On two cores of a production+-- store it answered a third fewer requests than no limit to sixteen+-- clients that asked one after the other: each cut left cores idle, and+-- one slot a step took seconds to fill them again. A server whose cores+-- are used at 90% gains nothing from more slots, and one whose cores have+-- room loses requests it could have answered while the limit climbs back.+--+-- For the same reason the delay alone does not cut. Requests that wait for+-- the disk come back to the cores in bursts, and the probes then read over+-- the target for a moment while the cores are idle the rest of the time.+-- With the database on a slow disk and two cores used at 63%, cutting on+-- such bursts held the limit under the number of clients, and the line+-- refused requests: sixteen clients that asked one after the other got up+-- to a sixth fewer answers than with no limit. A store that is really behind uses its+-- cores to the full, so the controller asks for both signals.+--+-- The target cannot be much lower than GHC's timeslice: a thread that+-- wakes behind one other that computes waits up to 20 ms, which is normal+-- scheduling and not congestion. Measured with requests that compute, on+-- four cores (@bench/Admission.hs@), a target of 10 ms held the limit at+-- the floor, where the probes still read 17 ms; 50 ms let a light request+-- wait 83 ms at the 99th percentile, where 25 ms had it at 53.+--+-- = Not done here+--+-- * The requests for one thing by its key (@Short@, in+--   "Haskoin.Store.Web.Router") are not limited, and are answered before+--   the long ones. The same control could hold them too, with a limit of+--   their own.+--+-- * Linux reports how long tasks waited for the disk (pressure stall+--   information, @\/proc\/pressure\/io@). It would tell a disk that is the+--   limit from one with room, which the scheduler's delay cannot see, and+--   could be a second signal.+module Haskoin.Store.Web.Congestion+  ( -- * The controller+    Control (..),+    Observation (..),+    Window (..),+    window,+    control,+    rising,+    cpuTime,++    -- * The probes+    Probe,+    withProbes,+    Delays (..),+    delays,+    lateness,+    weighted,+    percentile,+    snooze,+  )+where++import Control.Concurrent (getNumCapabilities, threadWaitRead)+import Control.Concurrent.Async (withAsyncOn)+import Control.Monad (forM, forever, replicateM, void, when)+import Data.IORef (IORef, atomicModifyIORef', newIORef)+import Data.Int (Int64)+import Data.List (sort, sortOn)+import Data.Maybe (listToMaybe)+import Foreign.C.Types (CInt (..), CUInt (..))+import Foreign.Marshal.Alloc (alloca)+import Foreign.Marshal.Array (allocaArray, peekArray)+import Foreign.Ptr (Ptr, nullPtr)+import Foreign.Storable (peek)+import GHC.Clock (getMonotonicTimeNSec)+import GHC.Conc (closeFdWith)+import GHC.Stats (RTSStats (..), getRTSStats, getRTSStatsEnabled)+import Haskoin.Store.Exception (bracket)+import System.CPUTime (getCPUTime)+import System.Posix.Types (Fd (..))++-- | The settings of the controller.+data Control = Control+  { -- | fewest slots+    floor :: !Int,+    -- | most slots+    ceiling :: !Int,+    -- | microseconds of lateness above which the limit comes down+    target :: !Int+  }+  deriving stock (Eq, Show)++-- | What the controller saw in a step.+data Observation = Observation+  { -- | the 90th percentile of the probes' lateness, without the pauses of+    -- the collector, in microseconds+    delay :: !Int,+    -- | the use of the cores, smoothed, in hundredths of all of them+    used :: !Int,+    -- | whether every slot was taken at some point, or a request waited+    -- or was refused+    saturated :: !Bool,+    -- | microseconds that a request holds its slot, smoothed: the round+    -- trip, once a cut has taken effect+    roundTrip :: !Int,+    -- | microseconds since the step before+    elapsed :: !Int+  }+  deriving stock (Eq, Show)++-- | The state of the controller between steps.+data Window = Window+  { -- | the limit: the level, rounded down, between the floor and the+    -- ceiling+    limit :: !Int,+    -- | the number of slots before rounding+    level :: !Double,+    -- | the level that the limit was last cut from, where the curve is+    -- flat+    peak :: !Double,+    -- | seconds on the curve, which run only while the limit may grow+    climbed :: !Double,+    -- | seconds from the start of the curve to its flat point+    knee :: !Double,+    -- | microseconds since the last cut+    sinceCut :: !Int+  }+  deriving stock (Eq, Show)++-- | The window at the start: at the floor, on the convex part of the curve.+window :: Control -> Window+window c = Window {limit = n, level = x, peak = x, climbed = 0, knee = 0, sinceCut = maxBound `div` 2}+  where+    n = lowest c+    x = fromIntegral n++-- | The floor, giving way to a ceiling below it.+lowest :: Control -> Int+lowest c = max 1 (min c.floor c.ceiling)++-- | What is kept of the level at a cut: three quarters.+backoff :: Double+backoff = 0.75++-- | How fast the curve rises away from its flat point, in slots a cubed+-- second, as in TCP CUBIC: back to the level of a cut from 16 slots in two+-- seconds, from 64 in three and a half.+cubic :: Double+cubic = 0.4++-- | Hundredths of the cores in use from which the limit does not grow, and+-- under which it is not cut.+busyCores :: Int+busyCores = 90++-- | The window after a step. Its limit is always between the floor and the+-- ceiling, the floor giving way to a ceiling below it.+control :: Control -> Observation -> Window -> Window+control c o w+  | o.delay > c.target && o.used >= busyCores && since >= o.roundTrip && w.limit > lowest c = cut+  | o.delay < c.target && o.saturated && o.used < busyCores = grow+  | otherwise = w {sinceCut = since}+  where+    since = min (maxBound `div` 2) (w.sinceCut + max 0 o.elapsed)+    -- a cut below the peak of the last one is a sign that the room is+    -- shrinking: the new peak is set lower still, as CUBIC does, so that+    -- the limit leaves room sooner+    cut =+      let top = if w.level < w.peak then w.level * (1 + backoff) / 2 else w.level+          low = max (fromIntegral (lowest c)) (w.level * backoff)+       in settle w {level = low, peak = top, climbed = 0, knee = cbrt ((top - low) / cubic), sinceCut = 0}+    grow =+      let t = w.climbed + fromIntegral (max 0 o.elapsed) / 1e6+          curve = cubic * (t - w.knee) ^ (3 :: Int) + w.peak+       in settle w {level = max w.level (min (fromIntegral c.ceiling) curve), climbed = t, sinceCut = since}+    settle x = x {limit = max (lowest c) (min c.ceiling (floor x.level))}+    cbrt x = if x <= 0 then 0 else x ** (1 / 3)++-- | Whether the limit is free to rise after a step: the cores have room and+-- it is under the ceiling. A line that stands while it is shows that the+-- limit lags behind the load, not that the store is behind: the limit+-- will grow, and refusing the requests that wait for it would cost+-- answers that the store could give ("Haskoin.Store.Web.Admission").+rising :: Control -> Observation -> Window -> Bool+rising c o w = o.used < busyCores && w.limit < c.ceiling++-- | Nanoseconds of the cores that the program has used so far, in all of+-- its threads, those of the foreign calls included.+cpuTime :: IO Int+cpuTime = (`div` 1000) . fromIntegral <$> getCPUTime++-- | The probe of a capability ('withProbes').+data Probe = Probe+  { wakes :: !(IORef Wakes),+    -- | nanoseconds that GHC's collector has stopped the program, so far+    collector :: !(IO Int)+  }++-- | What a probe noted since it was last asked.+data Wakes = Wakes+  { -- | when the probe last ran, in nanoseconds of the monotonic clock+    slept :: !Int,+    -- | the collector's time when the probe last ran, in nanoseconds+    paused :: !Int,+    -- | the time up to which the samples count, in nanoseconds: the last+    -- wake, or the last time that a late sleep was counted so far+    counted :: !Int,+    -- | each wake: microseconds late, microseconds late without the+    -- collector's pauses, and microseconds that it counts for+    samples :: ![(Int, Int, Int)]+  }++-- | Nanoseconds that GHC's collector has stopped the program so far, if+-- GHC keeps its statistics (@+RTS -T@); zero always otherwise.+collectorTime :: IO (IO Int)+collectorTime =+  getRTSStatsEnabled >>= \case+    True -> return (fromIntegral . (.gc_elapsed_ns) <$> getRTSStats)+    False -> return (return 0)++-- | Microseconds that a probe sleeps: a hundred samples a second from each+-- capability, and a thread that wakes that often costs nothing to notice.+nap :: Int+nap = 10000++-- | Nanoseconds of the monotonic clock.+nanos :: IO Int+nanos = fromIntegral <$> getMonotonicTimeNSec++-- | Sleep so many microseconds in the system, in a safe foreign call, which+-- gives the capability back while it sleeps.+foreign import ccall safe "usleep" c_usleep :: CUInt -> IO CInt++-- | Sleep so many microseconds, up to a second, with no timer of GHC's+-- timer manager ('withProbes' says why), and run again as soon as the+-- thread that runs on the capability ends its timeslice.+snooze :: Int -> IO ()+snooze us = void (c_usleep (fromIntegral (max 0 (min 999999 us))))++-- | The bell of the probes, in C (@cbits/haskoin_store_probe.c@).+data Bell++foreign import ccall unsafe "haskoin_probes_start"+  c_probesStart :: CInt -> CInt -> CInt -> Ptr CInt -> IO (Ptr Bell)++foreign import ccall safe "haskoin_probes_stop"+  c_probesStop :: Ptr Bell -> IO ()++foreign import ccall unsafe "haskoin_probe_take"+  c_probeTake :: Ptr Bell -> CInt -> Ptr Int64 -> Ptr Int64 -> IO CInt++foreign import ccall unsafe "close"+  c_close :: CInt -> IO CInt++-- | Run a probe on each capability while the action runs: a thread of the+-- capability that waits for a bell, rung every 10 ms by a thread of the+-- system, and notes how long after the bell it ran.+--+-- The probe waits as a connection waits for its socket: the bell writes+-- to an eventfd of the probe's own, which the probe waits for in+-- 'threadWaitRead'. GHC's manager of input and output for the capability+-- sees it in epoll and wakes the probe, which goes to the back of the+-- capability's queue of threads that are ready. So the wait counts what+-- a request that the network woke waits before it runs: the manager's+-- turn, when it is not already waiting in epoll but in the queue itself,+-- and the probe's own turn after it.+--+-- An earlier probe slept in a safe foreign call and woke a second thread+-- through an 'Control.Concurrent.MVar'. GHC lets a thread that returns+-- from a foreign call run first, so it counted the second thread's turn+-- and not the manager's: at overload it read what a connection waits once+-- the manager has woken it (40 ms at the 90th percentile, against 38 for+-- the connections), where this probe reads 86 ms with the manager's turn.+-- At moderate load the two read alike. The bell is outside GHC, as the+-- sleep was: every timer of a program goes through one thread of GHC, its+-- timer manager, which a busy capability can hold back, and a probe that+-- used one would measure that.+withProbes :: ([Probe] -> IO a) -> IO a+withProbes act = do+  n <- getNumCapabilities+  counted <- getRTSStatsEnabled+  t <- nanos+  gc <- collectorTime+  g <- gc+  ps <- replicateM n ((`Probe` gc) <$> newIORef (Wakes t g t []))+  bracket (start n counted) stop $ \(bell, fds) ->+    let listening [] = act ps+        listening ((i, p, fd) : rest) = withAsyncOn i (listen bell i p fd) $ \_ -> listening rest+     in listening (zip3 [0 ..] ps fds)+  where+    start n counted = allocaArray n $ \arr -> do+      bell <- c_probesStart (fromIntegral n) (fromIntegral nap) (if counted then 1 else 0) arr+      when (bell == nullPtr) $ ioError (userError "Haskoin.Store.Web.Congestion: no eventfd for the probes")+      fds <- map Fd <$> peekArray n arr+      return (bell, fds)+    -- the probes have stopped waiting by then; the eventfds are closed+    -- through GHC's manager, which may still have them+    stop (bell, fds) = do+      c_probesStop bell+      mapM_ (closeFdWith (\(Fd fd) -> void (c_close fd))) fds+    listen bell i p fd = alloca $ \wp -> alloca $ \gp -> forever $ do+      threadWaitRead fd+      t <- nanos+      g <- p.collector+      had <- c_probeTake bell (fromIntegral (i :: Int)) wp gp+      when (had /= 0) $ do+        due <- fromIntegral <$> peek wp+        before <- fromIntegral <$> peek gp+        atomicModifyIORef' p.wakes $ \w ->+          let sample =+                ( (t - due) `div` 1000,+                  (t - due - (g - before)) `div` 1000,+                  (t - w.counted) `div` 1000+                )+           in (Wakes t g t (sample : w.samples), ())++-- | Microseconds late, at the given time, of a probe that last ran at the+-- other; zero or less is not late.+overdue :: Int -> Int -> Int+overdue slept t = (t - slept) `div` 1000 - nap++-- | What the probes saw since they were last asked ('delays').+data Delays = Delays+  { -- | the 90th percentile of how late they woke, in microseconds+    raw :: !Int,+    -- | the same with the pauses of GHC's collector taken out of each wake:+    -- how late the scheduler alone made them+    scheduling :: !Int,+    -- | nanoseconds that the collector has stopped the program so far, as+    -- GHC counts them; zero when it keeps no statistics+    collected :: !Int+  }+  deriving stock (Eq, Show)++-- | The 90th percentile of how late the probes woke since the last call,+-- each wake counting for the time since the one before it ('delays'), with+-- the pauses of the collector counted as lateness.+lateness :: [Probe] -> IO Int+lateness ps = (.raw) <$> delays ps++-- | The 90th percentiles of how late the probes woke since the last call,+-- with the pauses of GHC's collector and without them, each wake counting+-- for the time since the one before it, and the collector's time so far.+--+-- Counting wakes would leave out the capabilities that matter most: a+-- probe on a capability with a long queue wakes seldom, and one on an idle+-- capability a hundred times a second. Weighed by time, every capability+-- counts alike, and the percentile is of the delay that a thread woken at+-- any moment, on any capability, would have. A probe that is still asleep+-- past its time counts with how late it is so far, for the time since it+-- was last counted: on capabilities so busy that no probe woke, the delay+-- would otherwise have no samples.+delays :: [Probe] -> IO Delays+delays ps = do+  t <- nanos+  g <- maybe (return 0) (.collector) (listToMaybe ps)+  xs <- forM ps $ \p ->+    atomicModifyIORef' p.wakes $ \w ->+      let late = overdue w.slept t+          sample = (late, late - (g - w.paused) `div` 1000, (t - w.counted) `div` 1000)+       in if late > 0+            then (w {counted = t, samples = []}, sample : w.samples)+            else (w {samples = []}, w.samples)+  let all' = concat xs+  return+    Delays+      { raw = weighted 90 [(x, w) | (x, _, w) <- all'],+        scheduling = weighted 90 [(x, w) | (_, x, w) <- all'],+        collected = g+      }++-- | The value that so many hundredths of the weight is at or under, each+-- value counting for its weight; zero for no weight.+weighted :: Int -> [(Int, Int)] -> Int+weighted p xs = go 0 (sortOn fst xs)+  where+    total = sum (map snd xs)+    go _ [] = 0+    go below ((x, w) : rest)+      | 100 * (below + w) >= p * total = x+      | otherwise = go (below + w) rest++-- | The value that so many hundredths of the values are at or under, by the+-- nearest rank; zero for no values.+percentile :: Int -> [Int] -> Int+percentile _ [] = 0+percentile p xs = sort xs !! max 0 (min (n - 1) (rank - 1))+  where+    n = length xs+    rank = (p * n + 99) `div` 100
src/Haskoin/Store/Web/Events.hs view
@@ -117,7 +117,6 @@   ( Address,     BlockHash,     BloomFilter,-    Ctx,     Network,     Tx (..),     TxHash,@@ -151,11 +150,11 @@   deriving (Eq, Show)  -- | A transaction as the log keeps it.-seenOf :: Network -> Ctx -> TxData -> Seen-seenOf net ctx td =+seenOf :: Network -> TxData -> Seen+seenOf net td =   Seen     { tx = td.tx,-      addresses = HashSet.fromList (rights (map (scriptToAddressBS net ctx) scripts)),+      addresses = HashSet.fromList (rights (map (scriptToAddressBS net) scripts)),       size = 256 + sum (map B.length scripts) + sum (map (sum . map B.length) td.tx.witness)     }   where@@ -308,22 +307,22 @@ -- -- A client that has not chosen is told of everything, and every client is -- told of a block and of a transaction that the server does not know.-relevant :: Ctx -> Word32 -> Maybe Seen -> Watch -> (Bool, Watch)-relevant _ _ Nothing w = (True, w)-relevant ctx gap (Just s) w+relevant :: Word32 -> Maybe Seen -> Watch -> (Bool, Watch)+relevant _ Nothing w = (True, w)+relevant gap (Just s) w   | not w.filtered = (True, w)   | otherwise = (named || not (null hits) || matched, foldl' moved w hits)   where     touched = HashSet.toList s.addresses     named = any (`HashSet.member` w.plain) touched     hits = mapMaybe (`HashMap.lookup` w.derived) touched-    matched = maybe False (\f -> isJust (bloomRelevantUpdate ctx f s.tx)) w.bloom-    moved now (spec, chain, index) = extend ctx spec chain (index + 1 + gap) now+    matched = maybe False (\f -> isJust (bloomRelevantUpdate f s.tx)) w.bloom+    moved now (spec, chain, index) = extend spec chain (index + 1 + gap) now  -- | Derive the addresses of a watched key on one chain up to a number of -- them, if there are fewer.-extend :: Ctx -> XPubSpec -> Word32 -> Word32 -> Watch -> Watch-extend ctx spec chain upto w = case HashMap.lookup spec w.keys of+extend :: XPubSpec -> Word32 -> Word32 -> Watch -> Watch+extend spec chain upto w = case HashMap.lookup spec w.keys of   Just (external, change)     | upto > have ->         w@@ -332,14 +331,14 @@           }     where       have = if chain == 0 then external else change-      derive = deriveFunction ctx spec.deriv (pubSubKey ctx spec.key chain)+      derive = deriveFunction spec.deriv (pubSubKey spec.key chain)   _ -> w  -- | Do what a client asked with a command, and say why not when it cannot -- be done, in which case the connection is to be closed. The function -- reads the balances of an extended public key from the database.-command :: Ctx -> Word32 -> (XPubSpec -> IO [XPubBal]) -> TVar Watch -> EventCommand -> IO (Either String ())-command ctx gap balances var = \case+command :: Word32 -> (XPubSpec -> IO [XPubBal]) -> TVar Watch -> EventCommand -> IO (Either String ())+command gap balances var = \case   EventWatch as -> change $ \w ->     let plain = foldl' (flip HashSet.insert) w.plain as      in if HashSet.size plain > maxWatchAddresses@@ -357,7 +356,7 @@         | HashMap.size w.keys >= maxWatchXPubs -> Left "Too many extended public keys"         | otherwise ->             let named = (chosen w) {keys = HashMap.insert spec (0, 0) w.keys}-             in Right (extend ctx spec 1 (upto 1) (extend ctx spec 0 (upto 0) named))+             in Right (extend spec 1 (upto 1) (extend spec 0 (upto 0) named))   EventUnwatchXPub spec -> change $ \w ->     Right       (chosen w)
src/Haskoin/Store/Web/Pieces.hs view
@@ -81,6 +81,7 @@     -- * Pieces     Piece,     piece,+    pieceOf,     pieced,     listPieces,     around,@@ -88,33 +89,41 @@   ) where -import Control.Exception (evaluate)+import Control.Concurrent.Async (asyncWithUnmask, cancel, wait)+import Control.Exception (evaluate, mask_) import Data.Aeson.Encoding (Encoding, fromEncoding) import Data.ByteString.Builder (Builder, lazyByteString, toLazyByteString) import Data.ByteString.Lazy qualified as L import Data.Bytes.Put (runPutL) import Data.Bytes.Serial (Serial (..))-import Haskoin.Util (hexBuilder) import Data.IORef (modifyIORef', newIORef, readIORef, writeIORef) import Data.List (intersperse) import Data.Sequence (Seq (..), (|>))+import Haskoin.Store.Exception (finally)+import Haskoin.Util (hexBuilder) import Network.HTTP.Types (ResponseHeaders, status200) import Network.Wai (Response, responseStream)-import UnliftIO (asyncWithUnmask, cancel, finally, mask_, wait) --- | A piece of a response. Its outer action reads what the piece is made of--- and encodes it to bytes, and may run at the same time as those of other--- pieces. The action that it gives runs in order, right before the piece is--- written, and gives the bytes to write. It should be cheap: it is there--- for what depends on the pieces before, such as a comma.-newtype Piece = Piece (IO (IO Builder))+-- | A piece of a response, with how many items it holds, known before it is+-- read. Its outer action reads what the piece is made of and encodes it to+-- bytes, and may run at the same time as those of other pieces; it gives+-- how many bytes it made. The action that it gives runs in order, right+-- before the piece is written, and gives the bytes to write. It should be+-- cheap: it is there for what depends on the pieces before, such as a+-- comma.+data Piece = Piece !Int (IO (Int, IO Builder)) --- | A piece from an action that reads its bytes. They are encoded in the--- thread that reads them, so the action that it gives only hands them over.+-- | A piece from an action that reads its bytes, with no items to pay for+-- in advance. They are encoded in the thread that reads them, so the+-- action that it gives only hands them over. piece :: IO Builder -> Piece-piece act = Piece $ do+piece = pieceOf 0++-- | A piece of so many items from an action that reads their bytes.+pieceOf :: Int -> IO Builder -> Piece+pieceOf n act = Piece n $ do   bytes <- rendered =<< act-  return (return (lazyByteString bytes))+  return (fromIntegral (L.length bytes), return (lazyByteString bytes))  -- | The bytes of a builder, all of them made, so that the work of making -- them is done where this is run.@@ -133,11 +142,11 @@ piecesResponse n headers done pieces = do   (first, rest) <- case pieces of     [] -> return (mempty, [])-    Piece p : more -> do-      bytes <- p >>= id+    Piece _ p : more -> do+      bytes <- p >>= snd       return (bytes, more)   return . responseStream status200 headers $ \write flush ->-    let send (Piece p) = p+    let send (Piece _ p) = snd <$> p         out next = next >>= write >> flush      in (write first >> flush >> ahead n (map send rest) out) `finally` done @@ -165,9 +174,10 @@   (mapM_ start first >> go later) `finally` (readIORef running >>= mapM_ cancel)  -- | Cut a list of keys into pieces of a size, each read with the action--- given: the pieces of items that are read by key.-pieced :: Int -> ([k] -> IO b) -> [k] -> [IO b]-pieced size fetch = map fetch . cut+-- given, with how many keys it has: the pieces of items that are read by+-- key.+pieced :: Int -> ([k] -> IO b) -> [k] -> [(Int, IO b)]+pieced size fetch = map (\ks -> (length ks, fetch ks)) . cut   where     cut [] = []     cut ks = let (part, more) = splitAt (max 1 size) ks in part : cut more@@ -177,13 +187,13 @@ -- The items of each piece are encoded where it is read; whether a comma -- comes before them, which depends on whether an item came in the pieces -- before, is settled in order, as 'piecesResponse' runs the pieces.-listPieces :: (b -> Encoding) -> [IO [b]] -> IO [Piece]+listPieces :: (b -> Encoding) -> [(Int, IO [b])] -> IO [Piece] listPieces enc pieces = do   begun <- newIORef False-  let encoded part = Piece $ do+  let encoded (n, part) = Piece n $ do         items <- part         bytes <- rendered (mconcat (intersperse "," (map (fromEncoding . enc) items)))-        return $ do+        return . (fromIntegral (L.length bytes),) $ do           was <- readIORef begun           let has = not (null items)           writeIORef begun (was || has)@@ -201,7 +211,7 @@     ending [] = []     ending [final] = [wrap mempty close final]     ending (p : more) = p : ending more-    wrap before after (Piece p) = Piece (fmap (fmap (\b -> before <> b <> after)) p)+    wrap before after (Piece n p) = Piece n (fmap (fmap (fmap (\b -> before <> b <> after))) p)  -- | The serialization of a value, in hexadecimal. hexOf :: (Serial a) => a -> Builder
src/Haskoin/Store/Web/Publish.hs view
@@ -113,6 +113,9 @@   ) where +import Control.Concurrent (threadDelay)+import Control.Concurrent.Async (mapConcurrently, race, waitCatch, withAsync)+import Control.Concurrent.STM (TVar, atomically, modifyTVar', newTVarIO, readTVarIO, stateTVar, writeTVar) import Control.Monad (forM_, forever, unless, when) import Data.Bits (shiftL, (.&.), (.|.)) import Data.ByteString (ByteString)@@ -172,21 +175,7 @@ import NQE (newMailbox, receive, withPublisher, withSubscription) import Network.Socket (SockAddr (..)) import System.Random (randomIO)-import UnliftIO-  ( TVar,-    atomically,-    mapConcurrently,-    modifyTVar',-    newTVarIO,-    race,-    readTVarIO,-    stateTVar,-    timeout,-    waitCatch,-    withAsync,-    writeTVar,-  )-import UnliftIO.Concurrent (threadDelay)+import System.Timeout (timeout)  -- | Whether the store has a transaction, and where. data Held
src/Haskoin/Store/Web/Router.hs view
@@ -44,6 +44,7 @@ module Haskoin.Store.Web.Router   ( -- * Routes     Route (..),+    Cost (..),     Miss (..),     Undeclared (..),     route,@@ -96,15 +97,48 @@     pattern :: ![Either Text Text],     -- | the names of the parameters it accepts in the query string or a form     accepts :: ![Text],-    handler :: Req -> IO Response+    handler :: Req -> IO Response,+    -- | what answering it may cost; 'route' makes routes that are 'Short'+    cost :: !Cost,+    -- | the fixed segments of its path, joined by underscores, as the+    -- metrics and the kinds of request name it; made the first time that it+    -- is asked for, and kept+    name :: Text   } +-- | What a request to a route may cost the server, roughly. A 'Light' or+-- 'Short' route is answered at once; only so many 'Long' ones run at once+-- ("Haskoin.Store.Web.Admission"), and the next ones wait their turn.+--+-- It is the route that has a cost, and not the request: a list may turn out+-- to have one item, and is 'Long' all the same. Telling by the request+-- would mean reading what it asks for before deciding to, which is the+-- work that a full server is trying not to do.+data Cost+  = -- | answered from memory alone, like a health check+    Light+  | -- | one thing looked up by its key: a transaction, a balance, a block+    Short+  | -- | as much as its parameters ask for: a list, the transactions of a+    -- block, the addresses of an extended public key+    Long+  deriving stock (Eq, Show)+ -- | A route for a method, a path such as @\/block\/:block@, where a segment -- that starts with a colon captures what is there under its name, and the -- names of the parameters it accepts. route :: Method -> Text -> [Text] -> (Req -> IO Response) -> Route-route m path = Route m [segment t | t <- T.splitOn "/" path, not (T.null t)]+route m path accepts handler =+  Route+    { method = m,+      pattern = segments,+      accepts = accepts,+      handler = handler,+      cost = Short,+      name = T.intercalate "_" [l | Left l <- segments]+    }   where+    segments = [segment t | t <- T.splitOn "/" path, not (T.null t)]     segment t = maybe (Left t) Right (T.stripPrefix ":" t)  -- | A route for @GET@.@@ -147,7 +181,7 @@ -- request is read whole first, and parsed as a form if it is one; handlers -- that stream a response still stream it. router :: [Route] -> (Req -> Miss -> IO Response) -> Application-router routes = routerScoped routes ($ routes)+router routes = routerScoped routes (\_ answer -> answer routes)  -- | 'router' for handlers that need something that lasts as long as one -- request does, such as a view of a database that must be the same for@@ -155,8 +189,9 @@ -- -- The routes are given twice. The first list is used for its methods, paths -- and parameters only, to find the route of a request. The second argument--- runs an action with the same routes, in the same order, made for the one--- request: the handler that answers is taken from those. The response is+-- is told which route matched, and runs an action with the same routes, in+-- the same order, made for the one request: the handler that answers is+-- taken from those. The response is -- sent inside that action, so that a handler that streams its response can -- go on reading while it is sent. --@@ -166,7 +201,7 @@ -- many requests do. routerScoped ::   [Route] ->-  (([Route] -> IO (Maybe ResponseReceived)) -> IO (Maybe ResponseReceived)) ->+  (Route -> ([Route] -> IO (Maybe ResponseReceived)) -> IO (Maybe ResponseReceived)) ->   (Req -> Miss -> IO Response) ->   Application routerScoped routes scoped fallback request respond = do@@ -185,7 +220,7 @@         | r.method == requestMethod request,           Just cs <- match r.pattern path =             case nub [k | (k, _) <- params, k `notElem` r.accepts] of-              [] -> scoped (answer i (req cs r.accepts)) >>= maybe (go unknown rs) return+              [] -> scoped r (answer i (req cs r.accepts)) >>= maybe (go unknown rs) return               ks -> go (unknown <|> Just ks) rs         | otherwise = go unknown rs   go Nothing (zip [0 :: Int ..] routes)
+ src/Haskoin/Store/XPub.hs view
@@ -0,0 +1,206 @@+{-# LANGUAGE MultiWayIf #-}+{-# LANGUAGE TemplateHaskell #-}++-- |+-- Module      : Haskoin.Store.XPub+-- Copyright   : (c) Haskoin Developers+-- License     : MIT+-- Maintainer  : jprupp@protonmail.ch+-- Stability   : experimental+-- Portability : POSIX+--+-- Extended public keys (xpubs) answered from a cache and indexed in the+-- background: what the parts of that design share. @docs\/xpub-indexing.md@+-- explains the design; this module holds its vocabulary.+--+-- An xpub with many used addresses is slow to read from the database: each+-- address is derived and read, then its transactions or outputs. So no+-- request reads an xpub from the database itself. It reads the xpub from a+-- cache, and on a miss it hands the xpub to the indexer+-- ("Haskoin.Store.XPub.Indexer"), which reads it in the background and+-- writes it to the cache, while the request waits or is told to come back.+--+-- The cache is the fleet's, in Redis ("Haskoin.Store.Fleet.XPub"), shared+-- by every store, and it cannot be turned off: its size is all there is to+-- set. It gives the reads of a t'XPubReads', each of one version of an+-- entry. It keeps the xpubs that cost most to index again and were used+-- most recently (GreedyDual, 'greedyLua'), and counts that cost in the+-- units of the prices below: they rank entries, and charge nobody. A job+-- settles and tells its size through a t'Budget'.+module Haskoin.Store.XPub+  ( -- * Reading a cache+    XPubReads (..),+    EntryMoved (..),++    -- * Paying for a job+    Budget (..),+    noBudget,++    -- * Prices+    addressPrice,+    entryPrice,+    bodyPrice,+    indexCost,++    -- * Keeping up with the store+    inSync,+    txAddresses,++    -- * Eviction in Redis+    greedyLua,+  )+where++import Control.Exception (Exception)+import Data.ByteString (ByteString)+import Data.Either (rights)+import Data.FileEmbed (embedFile, makeRelativeToProject)+import Data.IntMap.Strict qualified as IntMap+import Data.List (nub)+import Data.Word (Word32)+import Haskoin+  ( Address,+    BlockNode (..),+    Network,+    Tx (..),+    TxHash,+    TxOut (..),+    headerHash,+    scriptToAddressBS,+  )+import Haskoin.Node (Chain, chainGetBest)+import Haskoin.Store.Common (Limits (..))+import Haskoin.Store.Data+  ( Prev (..),+    TxData (..),+    TxRef (..),+    XPubBal (..),+    XPubSpec,+    XPubUnspent,+  )+import Haskoin.Store.Database.Query qualified as Q+import Haskoin.Store.Database.Reader (DatabaseReader (..))++-- | What the web server reads from the cache of xpubs. Each+-- read is 'Nothing' when the cache does not have the xpub, and marks it as+-- used when it does.+data XPubReads = XPubReads+  { -- | balances of the addresses up to the gap+    balances :: XPubSpec -> IO (Maybe [XPubBal]),+    -- | a page of transactions, newest first, as the database gives it+    transactions :: XPubSpec -> Limits -> IO (Maybe [TxRef]),+    -- | number of transactions+    count :: XPubSpec -> IO (Maybe Word32),+    -- | a page of unspent outputs, newest first, given the balances, which+    -- the Redis cache needs for the paths of the addresses+    unspents :: XPubSpec -> [XPubBal] -> Limits -> IO (Maybe [XPubUnspent]),+    -- | bodies of transactions, with their spenders, in the order of the+    -- hashes; 'Nothing' for those the cache does not have+    bodies :: [TxHash] -> IO [Maybe TxData],+    -- | the bodies of transactions as the entries that the reads read have+    -- them, wherever the bodies come from: the confirmation of each+    -- transaction of an entry, and who spends each output of its wallet, at+    -- the version that the request read; the rest as the bodies have it+    -- (@docs\/xpub-indexing.md@, Exact reads)+    wallet :: [Maybe TxData] -> IO [Maybe TxData]+  }++-- | A request read an entry of a cache that changed before the request was+-- done: a write landed between two of its reads, or the entry went. The+-- reads of one request are of one version of each xpub, so the web server+-- answers it again from the start, before anything of it was sent+-- ('Haskoin.Store.Fleet.Member.fleetSession', @docs\/xpub-indexing.md@,+-- Exact reads).+data EntryMoved = EntryMoved+  deriving (Eq, Show)++instance Exception EntryMoved++-- | What a job of the indexer calls as it goes. 'settle' is called at the+-- safe points of a job, where it holds no iterator of the database: between+-- batches of addresses, between calls into the database. 'finish' is+-- called when the job is done, however it ends. Every job of the store gets+-- 'noBudget', in which only 'sized', set by the indexer, does anything:+-- the jobs are not paced. The seam is kept so that a job could be paced+-- or measured without changing the indexer.+data Budget = Budget+  { settle :: IO (),+    finish :: IO (),+    -- | the job tells how many transactions it has to read, once it knows:+    -- the indexer tells from it when the job should be done+    -- ('Haskoin.Store.XPub.Indexer.joinEstimate')+    sized :: Int -> IO ()+  }++-- | A budget that measures nothing and never waits.+noBudget :: Budget+noBudget = Budget {settle = return (), finish = return (), sized = \_ -> return ()}++-- | The cost of an address of an xpub: derived, 25, and its balance read+-- by key, 10. First guesses all, in the same arbitrary units.+addressPrice :: Int+addressPrice = 35++-- | The cost of an entry of a scan: a transaction of an address's history,+-- or one of its unspent outputs.+entryPrice :: Int+entryPrice = 2++-- | The cost of the body of a transaction: read by key, 10, and decoded, 5.+bodyPrice :: Int+bodyPrice = 15++-- | What it costs to index an xpub from nothing: its addresses,+-- its transactions and its unspent outputs, and the bodies of the+-- transactions if the cache keeps them. It is the cost that eviction weighs+-- ('greedyLua'): what the cache would pay to read the+-- xpub again.+--+-- It is worked out from what was read rather than from the time the job+-- took, which would count the load of the store with the size of the xpub.+indexCost ::+  -- | addresses+  Int ->+  -- | transactions+  Int ->+  -- | unspent outputs+  Int ->+  -- | bodies+  Int ->+  Int+indexCost addrs txs utxos bods =+  addressPrice * addrs + entryPrice * (txs + utxos) + bodyPrice * bods++-- | Whether the database has the best header of the node. A store that is+-- not in sync takes no job of the indexer: it would write what knows less+-- than the database of a store that is, and its upkeep would chase every+-- block that it imports while it catches up.+inSync :: DatabaseReader -> Chain -> IO Bool+inSync db chain =+  Q.getBestBlock db >>= \case+    Nothing -> return False+    Just bb -> do+      cb <- chainGetBest chain+      return (cb.height > 0 && headerHash cb.header == bb)++-- | Addresses that a transaction spends from or pays to: those whose cached+-- xpubs it changes.+txAddresses :: Network -> TxData -> [Address]+txAddresses net td =+  nub . rights $+    map (scriptToAddressBS net . (.script)) (IntMap.elems td.prevs)+      <> map (scriptToAddressBS net . (.script)) td.tx.outputs++-- | GreedyDual for the caches of xpubs in Redis, as Lua functions that the+-- scripts which evict or mark a use start with: @greedy_use@, which gives+-- an entry the level plus its cost as its priority, @greedy_evict@, which+-- evicts the lowest priorities past a number and raises the level to each,+-- and @greedy_migrate@, which gives the level to the entries of a sorted+-- set that held times of last use, so that they go first. The cache of+-- the fleet ("Haskoin.Store.Fleet.XPub") runs it in front of its scripts.+-- Between+-- equal priorities Redis goes by the id of the xpub. The file is+-- @src\/Haskoin\/Store\/XPub\/lua\/greedy.lua@; a script that uses it is+-- run with it in front, as @cat greedy.lua script.lua@ for @redis-cli@.+greedyLua :: ByteString+greedyLua = $(makeRelativeToProject "src/Haskoin/Store/XPub/lua/greedy.lua" >>= embedFile)
+ src/Haskoin/Store/XPub/Indexer.hs view
@@ -0,0 +1,1029 @@+{-# LANGUAGE MultiWayIf #-}++-- |+-- Module      : Haskoin.Store.XPub.Indexer+-- Copyright   : (c) Haskoin Developers+-- License     : MIT+-- Maintainer  : jprupp@protonmail.ch+-- Stability   : experimental+-- Portability : POSIX+--+-- The indexer of xpubs: threads that read xpubs from the database into a+-- cache in the background, so that no request reads an xpub itself+-- (@docs\/xpub-indexing.md@, The indexer).+--+-- A request for an xpub that the cache does not have hands it to the+-- indexer ('request'), and either waits for the job to be done, up to a+-- set time, or is told when to come back. The work goes on either way, so+-- the client that comes back finds the xpub in the cache. The queue can+-- also take upkeep, behind the jobs that requests wait for ('upkeep'); the+-- store queues none, as a move of the fleet tip updates the cache in Redis+-- itself ("Haskoin.Store.Fleet.Member").+--+-- = The queue+--+-- First in, first out, with no key in it twice, in two parts: the jobs that+-- a request asked for, then upkeep. A request for a key that is already+-- queued or running joins that job; one for a key that is queued for+-- upkeep moves it to the end of the first part. Upkeep for a key that is+-- queued adds to that job ('Semigroup' of the payload); for a key whose job+-- is running, it queues the key again for when the job is done, since that+-- job may have read the database before the change. A key never runs in two+-- threads at once, so two jobs of one xpub never race to write it.+--+-- = Deciding+--+-- When a request does not find its key in the queue, the store estimates+-- when a new job would be done ('estimate'): (place ÷ threads + 1) × the+-- median time of recent jobs. That is rough, since the size of an xpub is+-- not known until it is read, but it only has to tell soon from later.+-- 'decide' then picks one of three, a pure function of the estimate, the+-- wait, the horizon, the room in the queue and the lock:+--+-- * __wait__: queue and lock the key, or join its job, and wait for it;+-- * __refuse and keep the job__: queue and lock it, and answer 429 with+--   @Retry-After@ from the estimate;+-- * __refuse and take nothing__: beyond the horizon, with a full queue, or+--   with the key locked by another store of the fleet, which says when it+--   expects to be done.+--+-- The lock ("Haskoin.Store.XPub.Lock") is taken when the key is queued,+-- not when the job starts, renewed while the store holds the job, and let+-- go when the job is done. A store that loses the race for it takes+-- nothing.+--+-- = Waiting without timers+--+-- A request that waits sets no timer: it waits on a variable of its own,+-- which the job sets when it is done, and a sweeper thread, which sleeps in+-- the system ten times a second, refuses the requests whose time is up+-- ('sweep'). Every timer of a program, @threadDelay@, @registerDelay@ and+-- @timeout@ included, goes through one thread of GHC, its timer manager,+-- which was seen stuck for seconds under load ("Haskoin.Store.Web.Admission"+-- says how). Sleeping in a safe foreign call does not depend on it.+--+-- = What is a parameter+--+-- The job itself, a t'Runner': reading an xpub and writing it to Redis+-- ("Haskoin.Store.Fleet.XPub"); and+-- a t'Budget' per job, which it settles at its safe points and tells its+-- size through, from which the indexer estimates when it will be done.+module Haskoin.Store.XPub.Indexer+  ( -- * Settings+    Settings (..),+    defaultSettings,++    -- * The jobs+    Runner (..),+    plainRunner,++    -- * The indexer+    Indexer,+    newIndexer,+    newIndexerWith,+    withIndexer,+    runIndexer,++    -- * Asking for keys+    Answer (..),+    Refusal (..),+    request,+    requestWithin,+    prefetch,+    upkeep,+    retryAfter,++    -- * Deciding+    Holder (..),+    Situation (..),+    Decision (..),+    decide,+    estimate,+    median,++    -- * Watching+    IndexerStats (..),+    Counts (..),+    Refusals (..),+    indexerStats,+    indexerIdle,++    -- * Paying++    -- * Steps+    sweep,+    renewHeld,+    runNext,+    snooze,+  )+where++import Control.Concurrent (getNumCapabilities)+import Control.Concurrent.Async (link, replicateConcurrently_, withAsync)+import Control.Concurrent.STM+  ( STM,+    TVar,+    atomically,+    modifyTVar',+    newTVar,+    newTVarIO,+    readTVar,+    retry,+    writeTVar,+  )+import Control.Exception (SomeException)+import Control.Monad (forM_, replicateM_, unless, void, when)+import Data.HashMap.Strict (HashMap)+import Data.HashMap.Strict qualified as HashMap+import Data.Hashable (Hashable)+import Data.List (sort)+import Data.Map.Strict (Map)+import Data.Map.Strict qualified as Map+import Data.Maybe (fromMaybe, maybeToList)+import Data.Sequence (Seq, ViewL (..), (|>))+import Data.Sequence qualified as Seq+import Foreign.C.Types (CInt (..), CUInt (..))+import GHC.Clock (getMonotonicTimeNSec)+import Haskoin.Store.Common (loopForever)+import Haskoin.Store.Exception (catchAny, finally, tryAny)+import Haskoin.Store.XPub (Budget (..), noBudget)+import Haskoin.Store.XPub.Lock (Lock (..), Taken (..))++-------------------------------------------------------------------------------+-- Settings+-------------------------------------------------------------------------------++-- | Settings of the indexer. Times are in microseconds.+data Settings = Settings+  { -- | threads that run jobs; zero or less for one per core. They spend+    -- much of their time waiting for RocksDB, so one per core keeps the+    -- disk busy+    workers :: !Int,+    -- | how long a request may wait for its job+    wait :: !Int,+    -- | beyond this estimate, the store takes no new job+    horizon :: !Int,+    -- | most jobs that requests asked for in the queue; upkeep has no limit+    -- but the number of cached xpubs, since no key is queued twice+    room :: !Int,+    -- | the median time of a job before any has been timed+    guess :: !Int+  }+  deriving (Eq, Show)++-- | One thread per core, a wait of ten seconds, a horizon of thirty, room+-- for a thousand jobs, and a first guess of a tenth of a second a job.+defaultSettings :: Settings+defaultSettings =+  Settings+    { workers = 0,+      wait = 10000000,+      horizon = 30000000,+      room = 1000,+      guess = 100000+    }++-- | How many of the latest jobs that requests asked for make the median.+recentJobs :: Int+recentJobs = 31++-------------------------------------------------------------------------------+-- The jobs+-------------------------------------------------------------------------------++-- | What the indexer does with a key, and with the payload that came with+-- it: whatever the cache needs to know, such as the changes that upkeep is+-- for. Payloads of one key are joined with '<>'.+--+-- The fields are lazy, so that a cache can make a runner of itself before+-- it is built ('System.IO.fixIO').+data Runner k p = Runner+  { -- | read the key and write it to the cache, paying from the budget;+    -- it throws if it fails. The requests that wait read the cache once it+    -- is done, as every request does: nothing goes from the job to them+    index :: Budget -> k -> p -> IO (),+    -- | the budget of a job+    budget :: k -> IO Budget,+    -- | whether the store may take jobs: a store that is not in sync with+    -- its node takes none, since it would write what knows less than the+    -- others+    ready :: IO Bool,+    -- | a job that will not run, because the store is not ready: its+    -- payload is given back+    abandon :: k -> p -> IO (),+    -- | another store holds the lock of the key, and expects to be done in+    -- so many microseconds: the payload of the job that was not queued is+    -- given back+    blocked :: k -> p -> Int -> IO (),+    -- | a job threw, or its lock could not be taken+    failed :: k -> p -> SomeException -> IO (),+    -- | called every time the locks are renewed, with the jobs that this+    -- store holds, queued or running+    renew :: [(k, p)] -> IO ()+  }++-- | A runner that only runs the job: it is always ready, pays nothing, and+-- has nothing to give back.+plainRunner :: (Budget -> k -> p -> IO ()) -> Runner k p+plainRunner f =+  Runner+    { index = f,+      budget = \_ -> return noBudget,+      ready = return True,+      abandon = \_ _ -> return (),+      blocked = \_ _ _ -> return (),+      failed = \_ _ _ -> return (),+      renew = \_ -> return ()+    }++-- | Where a job is.+data Stage p+  = -- | a request or upkeep reserved it, and is taking its lock+    Pending+  | -- | in the queue+    Queued+  | -- | being run, since a time if known, with upkeep that came meanwhile,+    -- to run again once it is done+    Running !(Maybe Int) !(Maybe p)+  | -- | done, with what came of it, while its lock is let go, with upkeep+    -- that came meanwhile, to queue again once the lock is gone+    Finishing !Answer !(Maybe p)++-- | A job of the indexer.+data Job p = Job+  { payload :: !p,+    -- | whether a request asked for it: then it is in the first part+    asked :: !Bool,+    -- | the requests that wait for it+    waiters :: ![TVar Verdict],+    stage :: !(Stage p)+  }++-- | The queue: the keys of each part, in order, and every job, queued,+-- pending or running.+data Queue k p = Queue+  { first :: !(Seq k),+    second :: !(Seq k),+    jobs :: !(HashMap k (Job p))+  }++-- | What a waiting request was told, if anything yet.+data Verdict = Waiting | Answered !Answer++-------------------------------------------------------------------------------+-- Answers and decisions+-------------------------------------------------------------------------------++-- | What became of a request for a key.+data Answer+  = -- | the job wrote the key to the cache: the request reads it there+    Indexed+  | -- | refused, with how many microseconds to wait before asking again+    Refused !Refusal !Int+  | -- | the job failed, or its lock could not be taken+    Failed+  deriving (Eq, Show)++-- | Why a request was refused.+data Refusal+  = -- | another store of the fleet holds the lock of the key+    OtherStore+  | -- | the queue has no room for another job+    QueueFull+  | -- | a new job would be done later than the horizon+    BeyondHorizon+  | -- | the job would be done later than the request may wait; it goes on+    TooLong+  | -- | the request waited as long as it may; the job goes on+    WaitedOut+  | -- | the store is not in sync with its node, and takes no job+    NotInSync+  deriving (Eq, Show, Enum, Bounded)++-- | Seconds for @Retry-After@ from microseconds: rounded up, at least one.+retryAfter :: Int -> Int+retryAfter us = max 1 ((us + 999999) `div` 1000000)++-- | Who has the job of a key.+data Holder+  = -- | nobody+    Nobody+  | -- | this store, queued or running+    ThisStore+  | -- | another store of the fleet, which expects to be done in so many+    -- microseconds+    AnotherStore !Int+  deriving (Eq, Show)++-- | What a decision is made from. Times are in microseconds. The fields+-- are named apart from those of t'Settings', so that an update of either is+-- never ambiguous.+data Situation = Situation+  { -- | when the job would be done: a new one, or the one of this store+    expected :: !Int,+    -- | how long the request may wait: the wait of the settings, or none+    patience :: !Int,+    -- | the horizon: beyond this estimate, take no new job+    reach :: !Int,+    -- | whether the queue has room for a new job+    hasRoom :: !Bool,+    holder :: !Holder+  }+  deriving (Eq, Show)++-- | What to do with a request for a key that the cache does not have.+data Decision+  = -- | queue and lock a new job, or join the job of this store, and wait+    Wait+  | -- | queue and lock a new job, or keep the one of this store, and+    -- refuse with this @Retry-After@ in microseconds: the estimate+    RefuseKeep !Int+  | -- | take nothing, and refuse with this @Retry-After@: from the lock of+    -- another store, or the horizon+    RefuseNothing !Refusal !Int+  deriving (Eq, Show)++-- | The decision of @docs\/xpub-indexing.md@ (What a request for an xpub+-- does):+--+-- +------------------------+---------------------------+------------------------------------++-- | Holder                 | Estimate                  | Decision                           |+-- +========================+===========================+====================================++-- | another store          | any                       | take nothing, retry from its lock  |+-- +------------------------+---------------------------+------------------------------------++-- | this store             | within the wait           | join and wait                      |+-- +------------------------+---------------------------+------------------------------------++-- | this store             | beyond the wait           | keep the job, retry from estimate  |+-- +------------------------+---------------------------+------------------------------------++-- | nobody, no room        | any                       | take nothing, retry at the horizon |+-- +------------------------+---------------------------+------------------------------------++-- | nobody                 | beyond the horizon        | take nothing, retry at the horizon |+-- +------------------------+---------------------------+------------------------------------++-- | nobody                 | within the wait           | queue, lock and wait               |+-- +------------------------+---------------------------+------------------------------------++-- | nobody                 | within the horizon        | queue, lock, retry from estimate   |+-- +------------------------+---------------------------+------------------------------------++--+-- A store that takes nothing tells its client to come back at the horizon:+-- by then this store, or another store of the fleet, may have room. In a+-- fleet the estimate of a job that a store holds is the one in its lock in+-- Redis, which every store reads, and the door decides from it before it+-- asks ('Haskoin.Store.XPub.Service.xpubAsk', 'requestWithin').+decide :: Situation -> Decision+decide s = case s.holder of+  AnotherStore t -> RefuseNothing OtherStore (max 0 t)+  ThisStore+    | s.expected <= s.patience -> Wait+    | otherwise -> RefuseKeep s.expected+  Nobody+    | not s.hasRoom -> RefuseNothing QueueFull s.reach+    | s.expected > s.reach -> RefuseNothing BeyondHorizon s.reach+    | s.expected <= s.patience -> Wait+    | otherwise -> RefuseKeep s.expected++-- | When a job at this place in the queue should be done, in the unit of+-- the median: the jobs ahead of it, shared between the threads, and then+-- its own, @(place ÷ threads + 1) × median@.+estimate :: Int -> Int -> Int -> Int+estimate med threads place = med + place * med `div` max 1 threads++-- | The median of some times, or the guess if there are none.+median :: Int -> Seq Int -> Int+median guess xs+  | Seq.null xs = guess+  | otherwise = sort (foldr (:) [] xs) !! (Seq.length xs `div` 2)++-------------------------------------------------------------------------------+-- The indexer+-------------------------------------------------------------------------------++-- | Requests refused since the start, by why.+data Refusals = Refusals+  { otherStore :: !Int,+    queueFull :: !Int,+    beyondHorizon :: !Int,+    tooLong :: !Int,+    waitedOut :: !Int,+    notInSync :: !Int+  }+  deriving (Eq, Show)++-- | What the indexer has done since the start.+data Counts = Counts+  { -- | jobs done+    done :: !Int,+    -- | jobs that threw+    errors :: !Int,+    -- | jobs dropped because the store was not ready+    dropped :: !Int,+    -- | requests answered after waiting+    answered :: !Int,+    -- | locks found taken by another store when they were renewed+    lost :: !Int,+    -- | requests refused, by why+    refused :: !Refusals,+    -- | prefetches that queued a job or found one: they are not refusals,+    -- since nobody waits for them+    prefetched :: !Int,+    -- | prefetches that took nothing: beyond the horizon, with a full queue,+    -- with the key locked by another store, or with the store not ready+    declined :: !Int+  }+  deriving (Eq, Show)++-- | The indexer of keys of type @k@, with payloads of type @p@.+data Indexer k p = Indexer+  { settings :: !Settings,+    -- | microseconds of a monotonic clock; the tests have one that they+    -- move by hand+    clock :: !(IO Int),+    lock :: !(Lock k),+    runner :: Runner k p,+    queue :: !(TVar (Queue k p)),+    -- | the requests that wait, by when their time is up and a serial+    -- number, with their keys+    deadlines :: !(TVar (Map (Int, Int) (k, TVar Verdict))),+    serial :: !(TVar Int),+    -- | microseconds of the latest jobs that requests asked for+    times :: !(TVar (Seq Int)),+    -- | the size of each running job, in transactions, that told it+    sizes :: !(TVar (HashMap k Int)),+    -- | microseconds per thousand transactions of the latest jobs that+    -- told their size+    rates :: !(TVar (Seq Int)),+    counts :: !(TVar Counts)+  }++-- | An indexer on the monotonic clock of the system. Nothing runs until+-- 'runIndexer' does, or use 'withIndexer'.+newIndexer :: Settings -> Lock k -> Runner k p -> IO (Indexer k p)+newIndexer = newIndexerWith ((`div` 1000) . fromIntegral <$> getMonotonicTimeNSec)++-- | An indexer on this clock, which gives microseconds.+newIndexerWith :: IO Int -> Settings -> Lock k -> Runner k p -> IO (Indexer k p)+newIndexerWith clock s lock runner = do+  n <- if s.workers > 0 then return s.workers else getNumCapabilities+  Indexer s {workers = n, room = max 0 s.room, wait = max 0 s.wait} clock lock runner+    <$> newTVarIO (Queue Seq.empty Seq.empty HashMap.empty)+    <*> newTVarIO Map.empty+    <*> newTVarIO 0+    <*> newTVarIO Seq.empty+    <*> newTVarIO HashMap.empty+    <*> newTVarIO Seq.empty+    <*> newTVarIO (Counts 0 0 0 0 0 (Refusals 0 0 0 0 0 0) 0 0)++-- | Run an indexer while an action runs: its threads, its sweeper and the+-- renewal of its locks.+withIndexer :: (Hashable k) => Settings -> Lock k -> Runner k p -> (Indexer k p -> IO a) -> IO a+withIndexer s lock runner action = do+  i <- newIndexer s lock runner+  withAsync (runIndexer i) $ \a -> link a >> action i++-- | Run the threads of an indexer, its sweeper and the renewal of its+-- locks, forever.+runIndexer :: (Hashable k) => Indexer k p -> IO ()+runIndexer i =+  withAsync sweeper $ \a1 ->+    withAsync renewer $ \a2 -> do+      link a1+      link a2+      replicateConcurrently_ i.settings.workers (loopForever (next i >>= run i))+  where+    sweeper = loopForever (snooze 100000 >> sweep i)+    renewer = loopForever (nap i.lock.every >> renewHeld i)++-- | Sleep so many microseconds, up to a second, in the system: a safe+-- foreign call, which gives the capability back while it sleeps, and sets+-- no timer of GHC ('Haskoin.Store.Web.Congestion.snooze' is the same).+snooze :: Int -> IO ()+snooze us = void (c_usleep (fromIntegral (max 0 (min 999999 us))))++foreign import ccall safe "usleep" c_usleep :: CUInt -> IO CInt++-- | Sleep so many microseconds, a tenth of a second at a time.+nap :: Int -> IO ()+nap us = replicateM_ (max 1 (us `div` 100000)) (snooze 100000)++-------------------------------------------------------------------------------+-- Asking+-------------------------------------------------------------------------------++-- | What a request does after it has looked at the queue.+data Step+  = -- | it has its answer+    Done !Answer+  | -- | it waits for a job+    Await !(Int, Int) !(TVar Verdict)+  | -- | it reserved a new job, and takes its lock, then waits, or answers+    Take !Decision !(Maybe ((Int, Int), TVar Verdict)) !Int++-- | Ask for a key that the cache does not have, as a request does: join+-- its job, or queue a new one, and wait for it, or be refused ('decide').+-- 'Indexed' means that the cache has the key now, and the request reads it+-- there, as every request does. Each refusal is counted by why, in a+-- t'Refusals'.+request :: (Hashable k, Semigroup p) => Indexer k p -> k -> p -> IO Answer+request i = asking False i i.settings.wait++-- | 'request', for a door that decided from the estimate in the lock of+-- the key, which every store of a fleet reads, that the request waits: it+-- joins a job of this store and waits, whatever this store's own estimate,+-- so that every store gives the same answer from the one estimate. It+-- waits at most so many microseconds, and no longer than the wait of the+-- settings, as for a request that waited for another store already and+-- takes the job over when that store let go of it. A new job is decided as+-- 'request' decides it.+requestWithin :: (Hashable k, Semigroup p) => Indexer k p -> Int -> k -> p -> IO Answer+requestWithin i = asking True i++-- | 'request' and 'requestWithin'.+asking :: (Hashable k, Semigroup p) => Bool -> Indexer k p -> Int -> k -> p -> IO Answer+asking decided i wait k p = do+  a <- ask decided i (max 0 (min wait i.settings.wait)) k p+  case a of+    Refused why _ -> atomically (refuse i why)+    _ -> return ()+  return a++-- | Ask for a key without waiting: queue it if a request could, and answer+-- at once. For a request that was answered from elsewhere and wants the key+-- in the cache for next time. Nobody waits for the answer, so it is not a+-- refusal when it says to come back: it counts as 'prefetched' if a job is+-- queued for the key, and as 'declined' if not.+prefetch :: (Hashable k, Semigroup p) => Indexer k p -> k -> p -> IO Answer+prefetch i k p = do+  a <- ask False i 0 k p+  atomically . modifyTVar' i.counts $ \c -> case a of+    Refused TooLong _ -> c {prefetched = c.prefetched + 1}+    Refused _ _ -> c {declined = c.declined + 1}+    _ -> c+  return a++-- | Ask for a key, waiting so many microseconds at most. It counts nothing:+-- 'request' and 'prefetch' count what they were told.+ask :: (Hashable k, Semigroup p) => Bool -> Indexer k p -> Int -> k -> p -> IO Answer+ask decided i wait k p =+  i.runner.ready >>= \case+    False -> return (Refused NotInSync i.settings.horizon)+    True -> do+      now <- i.clock+      step <- atomically $ do+        q <- readTVar i.queue+        med <- median i.settings.guess <$> readTVar i.times+        expected <- expectedOf i k+        case HashMap.lookup k q.jobs of+          Just Job {stage = Finishing a _} -> return (Done a)+          Just job -> do+            let est = joinEstimate i.settings.workers q med expected now k job+            case if decided && wait > 0 then Wait else decision est True ThisStore of+              Wait -> do+                (key, v) <- waiter i k (now + wait)+                joinJob i k p (Just v)+                return (Await key v)+              RefuseKeep e -> do+                joinJob i k p Nothing+                return (Done (Refused TooLong e))+              RefuseNothing why e -> return (Done (Refused why e))+          Nothing -> do+            let place = Seq.length q.first+                est = estimate med i.settings.workers place+            case decision est (place < i.settings.room) Nobody of+              RefuseNothing why e -> return (Done (Refused why e))+              d -> do+                w <- if d == Wait then Just <$> waiter i k (now + wait) else return Nothing+                let job = Job {payload = p, asked = True, waiters = maybeToList (snd <$> w), stage = Pending}+                writeTVar i.queue q {jobs = HashMap.insert k job q.jobs}+                return (Take d w est)+      case step of+        Done a -> return a+        Await key v -> await i key v+        Take d w est ->+          acquireFor i k est >>= \case+            Just a -> do+              forM_ w (unwait i . fst)+              return a+            Nothing -> case (d, w) of+              (Wait, Just (key, v)) -> await i key v+              (RefuseKeep e, _) -> return (Refused TooLong e)+              _ -> return Failed+  where+    -- a request that may not wait does not, even for a job due at once+    decision est room holder =+      case decide Situation {expected = est, patience = wait, reach = i.settings.horizon, hasRoom = room, holder} of+        Wait | wait <= 0 -> RefuseKeep est+        d -> d++-- | Queue a key for upkeep, behind the jobs that requests asked for, or+-- add the payload to its job. The payload is given back to the runner if+-- the store is not ready ('abandon'), if another store holds the lock+-- ('blocked'), or if the lock cannot be taken ('failed').+upkeep :: (Hashable k, Semigroup p) => Indexer k p -> k -> p -> IO ()+upkeep i k p =+  i.runner.ready >>= \case+    False -> do+      atomically (modifyTVar' i.counts (\c -> c {dropped = c.dropped + 1}))+      i.runner.abandon k p+    True -> do+      reserved <- atomically $ do+        q <- readTVar i.queue+        med <- median i.settings.guess <$> readTVar i.times+        case HashMap.lookup k q.jobs of+          Just job -> do+            let job' = case job.stage of+                  Running t again -> job {stage = Running t (Just (maybe p (<> p) again))}+                  Finishing a again -> job {stage = Finishing a (Just (maybe p (<> p) again))}+                  _ -> job {payload = job.payload <> p}+            writeTVar i.queue q {jobs = HashMap.insert k job' q.jobs}+            return Nothing+          Nothing -> do+            let job = Job {payload = p, asked = False, waiters = [], stage = Pending}+                place = Seq.length q.first + Seq.length q.second+            writeTVar i.queue q {jobs = HashMap.insert k job q.jobs}+            return (Just (estimate med i.settings.workers place))+      forM_ reserved (void . acquireFor i k)++-- | Take the lock of a reserved job and queue it; or, if another store+-- holds the lock or it cannot be taken, drop the reservation, answer the+-- requests that joined it, and give its payload back. 'Nothing' once the+-- job is queued.+acquireFor :: (Hashable k) => Indexer k p -> k -> Int -> IO (Maybe Answer)+acquireFor i k est =+  tryAny (i.lock.acquire k est) >>= \case+    Right Took -> do+      atomically . modifyTVar' i.queue $ \q -> case HashMap.lookup k q.jobs of+        Just job@Job {stage = Pending}+          | job.asked -> q {first = q.first |> k, jobs = HashMap.insert k job {stage = Queued} q.jobs}+          | otherwise -> q {second = q.second |> k, jobs = HashMap.insert k job {stage = Queued} q.jobs}+        _ -> q+      return Nothing+    Right (Held t) -> do+      let a = case decide (Situation 0 0 0 False (AnotherStore t)) of+            RefuseNothing why e -> Refused why e+            _ -> Refused OtherStore (max 0 t)+      gone <- atomically (unreserve a)+      forM_ gone $ \j -> i.runner.blocked k j.payload t `catchAny` \_ -> return ()+      return (Just a)+    Left e -> do+      gone <- atomically (unreserve Failed)+      forM_ gone $ \j -> i.runner.failed k j.payload e `catchAny` \_ -> return ()+      return (Just Failed)+  where+    unreserve a = do+      q <- readTVar i.queue+      case HashMap.lookup k q.jobs of+        Just job@Job {stage = Pending} -> do+          writeTVar i.queue q {jobs = HashMap.delete k q.jobs}+          mapM_ (answer a) job.waiters+          return (Just job)+        _ -> return Nothing++-- | A request joins the job of a key: a queued job of upkeep moves to the+-- end of the first part, and takes the payload of the request; a running+-- job takes nothing but the waiter, since what the request adds would only+-- be read after the job is done.+joinJob :: (Hashable k, Semigroup p) => Indexer k p -> k -> p -> Maybe (TVar Verdict) -> STM ()+joinJob i k p v = do+  q <- readTVar i.queue+  forM_ (HashMap.lookup k q.jobs) $ \job -> do+    let ws = maybeToList v <> job.waiters+        joined = job {payload = job.payload <> p, asked = True, waiters = ws}+    case job.stage of+      Running {} -> writeTVar i.queue q {jobs = HashMap.insert k job {waiters = ws} q.jobs}+      Queued+        | not job.asked ->+            writeTVar+              i.queue+              q+                { first = q.first |> k,+                  second = Seq.filter (/= k) q.second,+                  jobs = HashMap.insert k joined q.jobs+                }+      _ -> writeTVar i.queue q {jobs = HashMap.insert k joined q.jobs}++-- | When the job of a key that a request joins should be done: a queued+-- one from its place, the place at the end of the first part for upkeep,+-- which the request moves there; a running one from what it has left.+--+-- What a running job has left is what its size says it takes, at the rate+-- of the latest jobs that told theirs, less what it has run; without a+-- size, the median less what it has run; and once it has run longer than+-- either, the median again. It never grows with the time that the job has+-- run: an estimate that did, as long again as the job had run, sent clients+-- back at about twice its length.+joinEstimate :: (Hashable k) => Int -> Queue k p -> Int -> Maybe Int -> Int -> k -> Job p -> Int+joinEstimate threads q med expected now k job = case job.stage of+  Running (Just t) _ ->+    let e = max 0 (now - t)+     in case expected of+          Just total | total > e -> total - e+          Nothing | med > e -> med - e+          _ -> med+  Running Nothing _ -> fromMaybe med expected+  Finishing {} -> 0+  Queued | job.asked -> estimate med threads (fromMaybe (Seq.length q.first) (Seq.elemIndexL k q.first))+  _ -> estimate med threads (Seq.length q.first)++-- | What the running job of a key is expected to take in all, from the+-- size that it told and the rate of the latest jobs that told theirs; or+-- 'Nothing' if it told none.+expectedOf :: (Hashable k) => Indexer k p -> k -> STM (Maybe Int)+expectedOf i k = ($ k) <$> expectations i++-- | 'expectedOf' for every key at once.+expectations :: (Hashable k) => Indexer k p -> STM (k -> Maybe Int)+expectations i = do+  sizes <- readTVar i.sizes+  rate <- median rateGuess <$> readTVar i.rates+  return $ \k -> (\n -> n * rate `div` 1000) <$> HashMap.lookup k sizes++-- | Microseconds per thousand transactions of a job, before any job has+-- told its size: 100 microseconds a transaction, about twice what the+-- Redis cache takes for a large wallet.+rateGuess :: Int+rateGuess = 100000++-- | A new waiter, whose time is up at the given time.+waiter :: Indexer k p -> k -> Int -> STM ((Int, Int), TVar Verdict)+waiter i k deadline = do+  n <- readTVar i.serial+  writeTVar i.serial (n + 1)+  v <- newTVar Waiting+  modifyTVar' i.deadlines (Map.insert (deadline, n) (k, v))+  return ((deadline, n), v)++-- | Forget a waiter.+unwait :: Indexer k p -> (Int, Int) -> IO ()+unwait i key = atomically (modifyTVar' i.deadlines (Map.delete key))++-- | Wait for a verdict.+await :: Indexer k p -> (Int, Int) -> TVar Verdict -> IO Answer+await i key v = do+  a <-+    atomically+      ( readTVar v >>= \case+          Waiting -> retry+          Answered a -> return a+      )+      `finally` unwait i key+  case a of+    Indexed -> atomically (modifyTVar' i.counts (\c -> c {answered = c.answered + 1}))+    _ -> return ()+  return a++-- | Tell a waiting request, unless it was told already.+answer :: Answer -> TVar Verdict -> STM ()+answer a v =+  readTVar v >>= \case+    Waiting -> writeTVar v (Answered a)+    _ -> return ()++-- | Count a refusal.+refuse :: Indexer k p -> Refusal -> STM ()+refuse i why = modifyTVar' i.counts $ \c ->+  let r = c.refused+   in c+        { refused = case why of+            OtherStore -> r {otherStore = r.otherStore + 1}+            QueueFull -> r {queueFull = r.queueFull + 1}+            BeyondHorizon -> r {beyondHorizon = r.beyondHorizon + 1}+            TooLong -> r {tooLong = r.tooLong + 1}+            WaitedOut -> r {waitedOut = r.waitedOut + 1}+            NotInSync -> r {notInSync = r.notInSync + 1}+        }++-------------------------------------------------------------------------------+-- Running+-------------------------------------------------------------------------------++-- | Take the next job: the first that a request asked for, or the first of+-- upkeep. It blocks while there is none.+next :: (Hashable k) => Indexer k p -> IO (k, Job p)+next i = atomically (readTVar i.queue >>= pick)+  where+    pick q = do+      (k, q') <- case (Seq.viewl q.first, Seq.viewl q.second) of+        (k :< rest, _) -> return (k, q {first = rest})+        (EmptyL, k :< rest) -> return (k, q {second = rest})+        (EmptyL, EmptyL) -> retry+      case HashMap.lookup k q'.jobs of+        Just job -> do+          writeTVar i.queue q' {jobs = HashMap.insert k job {stage = Running Nothing Nothing} q'.jobs}+          return (k, job)+        -- not possible: every key of the queue has a job+        Nothing -> pick q'++-- | Run the next job in this thread, if there is one: for tests, which run+-- jobs one by one without the threads of 'runIndexer'. Returns whether+-- there was one.+runNext :: (Hashable k) => Indexer k p -> IO Bool+runNext i = do+  q <- atomically (readTVar i.queue)+  if Seq.null q.first && Seq.null q.second+    then return False+    else next i >>= run i >> return True++-- | Run a job, then let go of its lock and tell the requests that wait.+run :: (Hashable k) => Indexer k p -> (k, Job p) -> IO ()+run i (k, job) =+  i.runner.ready >>= \case+    False -> do+      atomically (modifyTVar' i.counts (\c -> c {dropped = c.dropped + 1}))+      i.runner.abandon k job.payload `catchAny` \_ -> return ()+      finish i k (Refused NotInSync i.settings.horizon)+    True -> do+      t0 <- i.clock+      atomically . modifyTVar' i.queue $ \q ->+        q {jobs = HashMap.adjust (\j -> j {stage = started t0 j.stage}) k q.jobs}+      -- the estimate in the lock, which the other stores read, from now+      -- that the job runs, and again once it tells its size+      publish i k+      r <- tryAny $ do+        b <- i.runner.budget k+        let told n = do+              atomically (modifyTVar' i.sizes (HashMap.insert k (max 0 n)))+              publish i k+              b.sized n+        i.runner.index b {sized = told} k job.payload `finally` b.finish+      t1 <- i.clock+      size <- atomically $ do+        sz <- HashMap.lookup k <$> readTVar i.sizes+        modifyTVar' i.sizes (HashMap.delete k)+        return sz+      case r of+        Right () -> do+          atomically $ do+            modifyTVar' i.counts (\c -> c {done = c.done + 1})+            when job.asked $ modifyTVar' i.times (\ts -> Seq.take recentJobs ((t1 - t0) Seq.<| ts))+            forM_ size $ \n ->+              when (n > 0) $+                modifyTVar' i.rates (\rs -> Seq.take recentJobs (((t1 - t0) * 1000 `div` n) Seq.<| rs))+          finish i k Indexed+        Left e -> do+          atomically (modifyTVar' i.counts (\c -> c {errors = c.errors + 1}))+          i.runner.failed k job.payload e `catchAny` \_ -> return ()+          finish i k Failed+  where+    started t = \case+      Running _ again -> Running (Just t) again+      s -> s++-- | A job is done: tell the requests that wait, at once; then queue its key+-- again if upkeep came while it ran, keeping the lock, or let go of the+-- lock, then of the job.+--+-- The job stays, finishing, while its lock is let go, which is a call to+-- Redis: a request that comes meanwhile has its answer at once, and upkeep+-- that comes meanwhile is queued again once the lock is gone, taking it+-- anew. Without that, either would have found the lock of this store, and+-- taken it for that of another. The requests that waited do not wait for+-- that call: what they read was written before it.+finish :: (Hashable k) => Indexer k p -> k -> Answer -> IO ()+finish i k a = do+  kept <- atomically $ do+    q <- readTVar i.queue+    case HashMap.lookup k q.jobs of+      Nothing -> return (Just [])+      Just job -> case job.stage of+        Running _ (Just p) -> do+          let job' = Job {payload = p, asked = False, waiters = [], stage = Queued}+          writeTVar i.queue q {second = q.second |> k, jobs = HashMap.insert k job' q.jobs}+          return (Just job.waiters)+        _ -> do+          mapM_ (answer a) job.waiters+          writeTVar i.queue q {jobs = HashMap.insert k job {waiters = [], stage = Finishing a Nothing} q.jobs}+          return Nothing+  case kept of+    Just ws -> atomically (mapM_ (answer a) ws)+    Nothing -> do+      i.lock.release k `catchAny` \_ -> return ()+      again <- atomically $ do+        q <- readTVar i.queue+        med <- median i.settings.guess <$> readTVar i.times+        case HashMap.lookup k q.jobs of+          Just job -> do+            mapM_ (answer a) job.waiters+            case job.stage of+              Finishing _ (Just p) -> do+                let job' = Job {payload = p, asked = False, waiters = [], stage = Pending}+                    place = Seq.length q.first + Seq.length q.second+                writeTVar i.queue q {jobs = HashMap.insert k job' q.jobs}+                return (Just (estimate med i.settings.workers place))+              _ -> do+                writeTVar i.queue q {jobs = HashMap.delete k q.jobs}+                return Nothing+          Nothing -> return Nothing+      forM_ again (void . acquireFor i k)++-- | Refuse the requests whose time is up, with the estimate of their jobs:+-- what the sweeper does ten times a second. Each request counts its own+-- refusal ('request').+sweep :: (Hashable k) => Indexer k p -> IO ()+sweep i = do+  now <- i.clock+  atomically $ do+    ds <- readTVar i.deadlines+    let (due, rest) = Map.spanAntitone (\(t, _) -> t <= now) ds+    unless (Map.null due) $ do+      writeTVar i.deadlines rest+      q <- readTVar i.queue+      med <- median i.settings.guess <$> readTVar i.times+      forM_ due $ \(k, v) ->+        readTVar v >>= \case+          Waiting -> do+            expected <- expectedOf i k+            let e = maybe med (joinEstimate i.settings.workers q med expected now k) (HashMap.lookup k q.jobs)+            writeTVar v (Answered (Refused WaitedOut e))+          Answered _ -> return ()++-- | Put the estimate of the job of a key in its lock now, rather than at+-- the next renewal: when it starts to run, and when it tells its size. Every+-- store of a fleet decides from the estimate in the lock, and gives the+-- same @Retry-After@ from it.+publish :: (Hashable k) => Indexer k p -> k -> IO ()+publish i k = do+  now <- i.clock+  est <- atomically $ do+    q <- readTVar i.queue+    med <- median i.settings.guess <$> readTVar i.times+    expected <- expectedOf i k+    return (joinEstimate i.settings.workers q med expected now k <$> HashMap.lookup k q.jobs)+  forM_ est $ \e -> void (i.lock.renew [(k, e)]) `catchAny` \_ -> return ()++-- | Renew the locks of the jobs that this store holds, queued or running,+-- each with its estimate, and tell the runner which they are: what the+-- indexer does every so often ('every' of the lock).+renewHeld :: (Hashable k) => Indexer k p -> IO ()+renewHeld i = do+  now <- i.clock+  (held, payloads) <- atomically $ do+    q <- readTVar i.queue+    med <- median i.settings.guess <$> readTVar i.times+    expecting <- expectations i+    let threads = i.settings.workers+        places = HashMap.fromList (zip (foldr (:) [] (q.first <> q.second)) [0 :: Int ..])+        estimateOf k job = case job.stage of+          Running {} -> joinEstimate threads q med (expecting k) now k job+          _ -> estimate med threads (HashMap.findWithDefault 0 k places)+        mine = [(k, job) | (k, job) <- HashMap.toList q.jobs, holding job.stage]+    return+      ( [(k, estimateOf k job) | (k, job) <- mine],+        concat [(k, job.payload) : [(k, p) | Running _ (Just p) <- [job.stage]] | (k, job) <- mine]+      )+  n <- i.lock.renew held `catchAny` \_ -> return 0+  when (n > 0) $ atomically (modifyTVar' i.counts (\c -> c {lost = c.lost + n}))+  i.runner.renew payloads `catchAny` \_ -> return ()+  where+    -- a pending job has not taken its lock yet, and a finishing one is+    -- letting go of it+    holding = \case+      Pending -> False+      Finishing {} -> False+      _ -> True++-------------------------------------------------------------------------------+-- Watching+-------------------------------------------------------------------------------++-- | What the indexer holds and has done.+data IndexerStats = IndexerStats+  { -- | jobs in the first part of the queue, which requests asked for+    forRequests :: !Int,+    -- | jobs of upkeep in the queue+    forUpkeep :: !Int,+    -- | jobs running+    running :: !Int,+    -- | requests waiting for a job+    waiting :: !Int,+    -- | what it has done since the start+    counts :: !Counts,+    -- | the median time of the latest jobs that requests asked for, in+    -- microseconds+    median :: !Int,+    -- | threads that run jobs+    threads :: !Int+  }+  deriving (Eq, Show)++-- | Whether the indexer has nothing to do: no job queued, waiting for its+-- lock, running or letting go of its lock. For tests, and for a store that+-- wants to know that its upkeep has caught up.+indexerIdle :: Indexer k p -> IO Bool+indexerIdle i = HashMap.null . (.jobs) <$> atomically (readTVar i.queue)++-- | What the indexer holds and has done now.+indexerStats :: Indexer k p -> IO IndexerStats+indexerStats i = atomically $ do+  q <- readTVar i.queue+  ds <- readTVar i.deadlines+  ts <- readTVar i.times+  c <- readTVar i.counts+  return+    IndexerStats+      { forRequests = Seq.length q.first,+        forUpkeep = Seq.length q.second,+        running = length [() | Job {stage = Running {}} <- HashMap.elems q.jobs],+        waiting = Map.size ds,+        counts = c,+        median = median i.settings.guess ts,+        threads = i.settings.workers+      }
+ src/Haskoin/Store/XPub/Lock.hs view
@@ -0,0 +1,208 @@+{-# LANGUAGE TemplateHaskell #-}++-- |+-- Module      : Haskoin.Store.XPub.Lock+-- Copyright   : (c) Haskoin Developers+-- License     : MIT+-- Maintainer  : jprupp@protonmail.ch+-- Stability   : experimental+-- Portability : POSIX+--+-- One store of a fleet indexes an xpub at a time: a lock per xpub in the+-- Redis that the stores share, taken by the indexer+-- ("Haskoin.Store.XPub.Indexer") when it queues the xpub and let go when+-- the job has written the entry (@docs\/xpub-indexing.md@, In a fleet).+--+-- = Why+--+-- A load balancer spreads a burst of requests for one xpub over every+-- store of a fleet. Without a lock, each store reads the xpub from its own+-- database, for each request that comes before the first of them has+-- cached it. With it, the first store to queue the xpub reads it, once, and+-- the others tell their clients when to come back: the value of the lock+-- says when its holder expects to be done.+--+-- = The key+--+-- @\<prefix\>idx:\<id\>@, with the prefix of the fleet and the id of+-- 'Haskoin.Store.Fleet.XPub.xpubId', holding @\<token\> \<done\> \<store\>@: a random+-- token of the holder, when it expects the job to be done in milliseconds+-- of the clock of Redis, and the name of the store. The scripts read the+-- clock of Redis, so that stores whose clocks differ agree on the time.+--+-- * __Taken when the xpub is queued__ ('acquire'), with @SET NX PX@, not+--   when the job starts, so that the other stores stop at once. A store+--   that loses the race is told who holds it and until when.+-- * __It lasts 30 seconds, renewed every ten__ ('renew') for every job the+--   store holds, with a new estimate of when it will be done. A store that+--   dies lets its locks go within half a minute.+-- * __Let go by its holder only__ ('release'): a script deletes the key if+--   it still holds the holder's token, so a store whose lock expired and+--   was taken by another does not delete the other's.+--+-- The lock is for efficiency, not for the safety of the cache: the scripts+-- of "Haskoin.Store.Fleet.XPub" keep an entry right whoever writes it, and a store+-- that has lost its lock, because it could not renew it in time, writes+-- what it read as safely as any other.+--+-- It replaces the lock of the old cache of a store alone that was held for the time+-- of a reload only (@lock:\<id\>@, two minutes, not renewed): taken that+-- late, it let several stores read the same xpub for requests that came at+-- the same time.+--+-- A store without Redis has no fleet to share xpubs with: 'noLock' always+-- gives the lock.+module Haskoin.Store.XPub.Lock+  ( Lock (..),+    Taken (..),+    noLock,+    LockConfig (..),+    defaultLockConfig,+    redisLock,+    lockKey,+  )+where++import Control.Monad (forM, void)+import Data.ByteString (ByteString)+import Data.ByteString.Char8 qualified as C8+import Data.FileEmbed (embedFile, makeRelativeToProject)+import Data.HashMap.Strict qualified as HashMap+import Data.IORef (atomicModifyIORef', newIORef, readIORef)+import Data.Maybe (catMaybes)+import Data.Word (Word64)+import Database.Redis (Connection, Redis, Reply)+import Database.Redis qualified as R+import System.Random (randomIO)+import Text.Read (readMaybe)++-- | The lock of a key, in a fleet or not. Times are in microseconds from+-- now.+data Lock k = Lock+  { -- | take the lock of a key, saying in how long the job should be done+    acquire :: k -> Int -> IO Taken,+    -- | renew the locks of these keys that this store holds, each with a+    -- new estimate; returns how many it no longer held+    renew :: [(k, Int)] -> IO Int,+    -- | let go of the lock of a key, if this store holds it+    release :: k -> IO (),+    -- | how long until the holder of the lock of a key expects to be done,+    -- and its name, if the key is locked+    holder :: k -> IO (Maybe (Int, ByteString)),+    -- | how often to renew the locks held+    every :: !Int+  }++-- | What came of trying to take a lock.+data Taken+  = -- | this store holds it now+    Took+  | -- | another store holds it, and expects to be done in so many+    -- microseconds, zero or less if it is late+    Held !Int+  deriving (Eq, Show)++-- | The lock of a store alone: always given, never held by another.+noLock :: Lock k+noLock =+  Lock+    { acquire = \_ _ -> return Took,+      renew = \_ -> return 0,+      release = \_ -> return (),+      holder = \_ -> return Nothing,+      every = 10000000+    }++-- | Settings of the lock in Redis.+data LockConfig = LockConfig+  { -- | prefix of the keys: the prefix of the fleet's keys+    prefix :: !ByteString,+    -- | name of this store, in the value of the lock, for those who look+    name :: !ByteString,+    -- | milliseconds that a lock lasts if it is not renewed; it is renewed+    -- three times as often+    ttl :: !Int+  }++-- | A lock of 30 seconds with these prefix and name.+defaultLockConfig :: ByteString -> ByteString -> LockConfig+defaultLockConfig prefix name = LockConfig {prefix, name, ttl = 30000}++-- | The key of the lock of an xpub, by the id of the xpub+-- ('Haskoin.Store.Fleet.XPub.xpubId').+lockKey :: ByteString -> ByteString -> ByteString+lockKey prefix i = prefix <> "idx:" <> i++-- | Locks in Redis, for keys with these ids. The store keeps the token of+-- each lock it holds, to renew and let go of it.+redisLock :: LockConfig -> Connection -> (k -> ByteString) -> IO (Lock k)+redisLock cfg conn ident = do+  tokens <- newIORef HashMap.empty+  let key = lockKey cfg.prefix . ident+      ms us = num (max 0 us `div` 1000)+      acquire k est = do+        token <- num <$> (randomIO :: IO Word64)+        r <- run (R.eval takeScript [key k] [token, ms est, cfg.name, num cfg.ttl])+        case r of+          ["ok"] -> do+            atomicModifyIORef' tokens (\m -> (HashMap.insert (ident k) token m, ()))+            return Took+          "held" : left : _ -> return (Held (maybe 0 (* 1000) (readMaybe (C8.unpack left))))+          _ -> fail ("Unexpected answer of Redis to the lock of an xpub: " <> show r)+      renew ks = do+        m <- readIORef tokens+        let held = catMaybes [(\t -> (key k, t, ms est)) <$> HashMap.lookup (ident k) m | (k, est) <- ks]+        fmap sum . forM (chunks held) $ \ch ->+          fromInteger+            <$> run+              ( R.eval+                  renewScript+                  [k | (k, _, _) <- ch]+                  (num cfg.ttl : concat [[t, e] | (_, t, e) <- ch])+              )+      release k = do+        t <- atomicModifyIORef' tokens (\m -> (HashMap.delete (ident k) m, HashMap.lookup (ident k) m))+        case t of+          Nothing -> return ()+          Just token -> void (run (R.eval releaseScript [key k] [token] :: Redis (Either Reply Integer)))+      holder k =+        run (R.eval peekScript [key k] []) >>= \case+          [left, who] | Just n <- readMaybe (C8.unpack left) -> return (Just (n * 1000, who))+          _ -> return Nothing+  return Lock {acquire, renew, release, holder, every = cfg.ttl * 1000 `div` 3}+  where+    run :: Redis (Either Reply a) -> IO a+    run f =+      R.runRedis conn f >>= \case+        Right x -> return x+        Left e -> fail ("Redis error: " <> show e)+    chunks [] = []+    chunks xs = let (a, b) = splitAt 500 xs in a : chunks b++num :: (Integral a) => a -> ByteString+num = C8.pack . show . toInteger++-- | Take a lock. Keys: the lock. Arguments: the token, milliseconds until+-- the job should be done, the name of the store, milliseconds that the lock+-- lasts. Returns @{\"ok\"}@ if it was taken, or @{\"held\", left, name}@:+-- milliseconds until the holder expects to be done, by the clock of Redis,+-- and its name.+takeScript :: ByteString+takeScript = $(makeRelativeToProject "src/Haskoin/Store/XPub/lua/take.lua" >>= embedFile)++-- | Renew locks. Keys: the locks. Arguments: milliseconds that a lock+-- lasts, then for each key its token and milliseconds until its job should+-- be done. A lock whose key no longer holds the token is not touched.+-- Returns how many were not held.+renewScript :: ByteString+renewScript = $(makeRelativeToProject "src/Haskoin/Store/XPub/lua/renew.lua" >>= embedFile)++-- | Delete a lock if it holds the token. Keys: the lock. Arguments: the+-- token. Returns 1 if it was deleted.+releaseScript :: ByteString+releaseScript = $(makeRelativeToProject "src/Haskoin/Store/XPub/lua/release.lua" >>= embedFile)++-- | Read a lock. Keys: the lock. Returns nothing if it is not held, or+-- milliseconds until its holder expects to be done, and its name.+peekScript :: ByteString+peekScript = $(makeRelativeToProject "src/Haskoin/Store/XPub/lua/peek.lua" >>= embedFile)
+ src/Haskoin/Store/XPub/Service.hs view
@@ -0,0 +1,136 @@+-- |+-- Module      : Haskoin.Store.XPub.Service+-- Copyright   : (c) Haskoin Developers+-- License     : MIT+-- Maintainer  : jprupp@protonmail.ch+-- Stability   : experimental+-- Portability : POSIX+--+-- The door to the cache of xpubs of the fleet ("Haskoin.Store.Fleet.XPub"),+-- for the web server (@docs\/xpub-indexing.md@). Every answer about an xpub+-- comes from the cache. A miss asks the indexer, which one store of the+-- fleet runs for the xpub, holding its dibs in Redis with the estimate of+-- when it will be done: a job expected past the wait is refused at once,+-- with the same @Retry-After@ on every store; one expected within it is+-- waited for, until the entry is announced or the wait runs out (429).+module Haskoin.Store.XPub.Service+  ( XPubConfig (..),+    defaultXPubConfig,+    Backend (..),+    XPubs (..),+    Session (..),+    unpinned,+    xpubAsk,+    xpubDelete,+    Answer (..),+    Refusal (..),+    retryAfter,+  )+where++import Data.Maybe (fromMaybe)+import GHC.Clock (getMonotonicTimeNSec)+import Haskoin.Store.Data (XPubSpec)+import Haskoin.Store.XPub (XPubReads (..))+import Haskoin.Store.XPub.Indexer (Answer (..), Refusal (..), Settings (..), defaultSettings, retryAfter)++-- | The settings of the indexer of xpubs.+newtype XPubConfig = XPubConfig {indexer :: Settings}+  deriving (Eq, Show)++defaultXPubConfig :: XPubConfig+defaultXPubConfig = XPubConfig {indexer = defaultSettings}++-- | The cache, as the door sees it.+data Backend = Backend+  { -- | ask its indexer for an xpub, waiting so many microseconds at most+    ask :: XPubSpec -> Int -> IO Answer,+    -- | wait at most so many microseconds for the job of another store+    await :: XPubSpec -> Int -> IO Bool,+    -- | in how many microseconds the job of an xpub is expected to be done,+    -- from the one estimate in its lock+    eta :: XPubSpec -> IO (Maybe Int),+    delete :: XPubSpec -> IO Bool+  }++-- | The door: the cache of the fleet, and how long a request waits. There+-- is always a cache behind it: a store has no way to answer about an xpub+-- but from the cache, and no setting turns it off; its size is all that+-- is set.+data XPubs = XPubs+  { backend :: !Backend,+    patience :: !Int+  }++-- | The reads of the cache for one request, each xpub at one version, and+-- what to run once the request has its answer.+data Session = Session+  { reads :: XPubReads,+    settle :: IO ()+  }++-- | No reads: every read is a miss. Outside of a request, and for a store+-- without Redis, which answers about no xpub.+unpinned :: Session+unpinned =+  Session+    { reads =+        XPubReads+          { balances = \_ -> return Nothing,+            transactions = \_ _ -> return Nothing,+            count = \_ -> return Nothing,+            unspents = \_ _ _ -> return Nothing,+            bodies = return . map (const Nothing),+            wallet = return+          },+      settle = return ()+    }++xpubDelete :: XPubs -> XPubSpec -> IO Bool+xpubDelete xs x = xs.backend.delete x++-- | Ask for an xpub that the cache does not have: refused at once if the+-- one estimate in its lock is past the wait, else queued or joined, and+-- waited for; a job of another store is waited for through the cache.+xpubAsk :: XPubs -> XPubSpec -> IO Answer+xpubAsk xs x = do+  start <- clockUs+  b.eta x >>= \case+    Just t | t > xs.patience -> return (Refused TooLong t)+    _ ->+      b.ask x xs.patience >>= \case+        Refused OtherStore t -> awaitOther xs b x start (start + t)+        Refused WaitedOut t -> Refused WaitedOut . fromMaybe t <$> b.eta x+        a -> return a+  where+    b = xs.backend++-- | Wait for the job of another store, which holds the dibs; every second+-- without the entry ask again, which takes the dibs if their holder died.+awaitOther :: XPubs -> Backend -> XPubSpec -> Int -> Int -> IO Answer+awaitOther xs b x start due0 = do+  now0 <- clockUs+  if due0 - now0 > xs.patience - (now0 - start)+    then return (Refused OtherStore (due0 - now0))+    else go due0+  where+    go due = do+      now <- clockUs+      let left = xs.patience - (now - start)+      if left <= 0+        then Refused OtherStore . max 1000000 . fromMaybe (due - now) <$> b.eta x+        else+          b.await x (min left 1000000) >>= \case+            True -> return Indexed+            False -> do+              now' <- clockUs+              let left' = xs.patience - (now' - start)+              if left' <= 0+                then go due+                else+                  b.ask x left' >>= \case+                    Refused OtherStore t -> go (now' + t)+                    a -> return a++clockUs :: IO Int+clockUs = (`div` 1000) . fromIntegral <$> getMonotonicTimeNSec
+ src/Haskoin/Store/XPub/lua/greedy.lua view
@@ -0,0 +1,35 @@+-- GreedyDual for the caches of xpubs in Redis: greedyLua in Haskoin.Store.XPub,+-- put before the scripts that use it. A use: priority = level + cost.+local function greedy_use(prio, level, id, cost, only_new)+  local p = tonumber(redis.call('GET', level) or '0') + (tonumber(cost) or 0)+  if only_new then+    redis.call('ZADD', prio, 'NX', p, id)+  else+    redis.call('ZADD', prio, p, id)+  end+end+-- evict the lowest priority past max, raising the level; drop deletes an entry+local function greedy_evict(prio, level, max, drop)+  local n = 0+  while redis.call('ZCARD', prio) > max do+    local low = redis.call('ZRANGE', prio, 0, 0, 'WITHSCORES')+    local id, score = low[1], tonumber(low[2])+    if score > tonumber(redis.call('GET', level) or '0') then+      redis.call('SET', level, string.format('%.0f', score))+    end+    drop(id)+    redis.call('ZREM', prio, id)+    n = n + 1+  end+  return n+end+-- entries scored by the time of their last use: the level, so they go first+local function greedy_migrate(old, prio, level)+  if redis.call('EXISTS', old) == 1 then+    local lv = tonumber(redis.call('GET', level) or '0')+    for _, id in ipairs(redis.call('ZRANGE', old, 0, -1)) do+      redis.call('ZADD', prio, 'NX', lv, id)+    end+    redis.call('DEL', old)+  end+end
+ src/Haskoin/Store/XPub/lua/peek.lua view
@@ -0,0 +1,8 @@+-- Who holds a lock, and until when: peekScript in Haskoin.Store.XPub.Lock.+local v = redis.call('GET', KEYS[1])+if not v then return {} end+local t = redis.call('TIME')+local now = t[1] * 1000 + math.floor(t[2] / 1000)+local _, _, at, who = string.find(v, '^%S+ (%S+) (.*)$')+if not at then return {'0', ''} end+return {string.format('%.0f', tonumber(at) - now), who}
+ src/Haskoin/Store/XPub/lua/release.lua view
@@ -0,0 +1,6 @@+-- Let go of a lock if this store still holds it: releaseScript in Haskoin.Store.XPub.Lock.+local v = redis.call('GET', KEYS[1])+if v and string.sub(v, 1, #ARGV[1] + 1) == ARGV[1] .. ' ' then+  return redis.call('DEL', KEYS[1])+end+return 0
+ src/Haskoin/Store/XPub/lua/renew.lua view
@@ -0,0 +1,16 @@+-- Renew the locks that a store holds: renewScript in Haskoin.Store.XPub.Lock.+local t = redis.call('TIME')+local now = t[1] * 1000 + math.floor(t[2] / 1000)+local lost = 0+for i, k in ipairs(KEYS) do+  local token, est = ARGV[2 * i], tonumber(ARGV[2 * i + 1])+  local v = redis.call('GET', k)+  if v and string.sub(v, 1, #token + 1) == token .. ' ' then+    local _, _, who = string.find(v, '^%S+ %S+ (.*)$')+    local done = string.format('%.0f', now + est)+    redis.call('SET', k, token .. ' ' .. done .. ' ' .. (who or ''), 'PX', ARGV[1])+  else+    lost = lost + 1+  end+end+return lost
+ src/Haskoin/Store/XPub/lua/take.lua view
@@ -0,0 +1,11 @@+-- Take the lock of an xpub, to index it: takeScript in Haskoin.Store.XPub.Lock.+local t = redis.call('TIME')+local now = t[1] * 1000 + math.floor(t[2] / 1000)+local done = string.format('%.0f', now + tonumber(ARGV[2]))+if redis.call('SET', KEYS[1], ARGV[1] .. ' ' .. done .. ' ' .. ARGV[3], 'NX', 'PX', ARGV[4]) then+  return {'ok'}+end+local v = redis.call('GET', KEYS[1])+local _, _, at, who = string.find(v or '', '^%S+ (%S+) (.*)$')+if not at then return {'held', '0', ''} end+return {'held', string.format('%.0f', tonumber(at) - now), who}
stress/Main.hs view
@@ -46,6 +46,9 @@ --   killed run left. module Main (main) where +import Control.Concurrent (threadDelay)+import Control.Concurrent.Async (withAsync)+import Control.Concurrent.MVar (MVar, newMVar, withMVar) import Control.Concurrent.STM (TVar, atomically, newTVarIO, readTVarIO, writeTVar) import Control.DeepSeq (force) import Control.Exception (evaluate)@@ -74,6 +77,7 @@ import Data.Word (Word64) import Database.Redis qualified as Redis import Haskoin hiding (Coin, Key)+import Haskoin.Store.Exception (bracket, tryAny) import Haskoin.Store.Integration.Image (testImage, valkeyBin, wrkBin) import Haskoin.Store.Integration.Node import Haskoin.Store.Integration.Podman@@ -83,14 +87,12 @@ import Network.Wreq qualified as Wreq import Numeric (readHex) import Options.Applicative+import System.Directory (createDirectoryIfMissing, makeAbsolute) import System.Exit (ExitCode (..)) import System.IO (Handle, IOMode (..), hFlush, hPutStrLn, stderr, withFile) import System.Process import System.Random.Stateful (IOGenM, StdGen, mkStdGen, newIOGenM, uniformRM) import Text.Read (readMaybe)-import UnliftIO (MVar, bracket, newMVar, tryAny, withAsync, withMVar)-import UnliftIO.Concurrent (threadDelay)-import UnliftIO.Directory (createDirectoryIfMissing, makeAbsolute)  ------------------------------------------------------------------------------- -- Options@@ -147,20 +149,19 @@   podmanUsable >>= either fail return   say "Building or finding the test image"   image <- testImage-  withContext $ \ctx ->-    withContainer (valkey image) $ \rc -> do-      redisURL <- ("redis://127.0.0.1:" <>) . show <$> hostPort rc 6379-      waitRedis redisURL-      withContainer (nodeSpec o image) $ \nc -> do-        rpcPort <- hostPort nc 18443-        p2p <- hostPort nc 18444-        let node = Node {kind = BitcoinCore, url = "http://127.0.0.1:" <> show rpcPort <> "/", p2p = p2p}-        waitRpc node-        withContainer (wrkSpec image outDir) $ \wc ->-          withStatsd $ \statsPort stats ->-            withRamDir $ \dir ->-              withServer o outDir dir redisURL p2p statsPort $ \srv ->-                stress o {out = outDir} ctx node redisURL srv stats wc+  withContainer (valkey image) $ \rc -> do+    redisURL <- ("redis://127.0.0.1:" <>) . show <$> hostPort rc 6379+    waitRedis redisURL+    withContainer (nodeSpec o image) $ \nc -> do+      rpcPort <- hostPort nc 18443+      p2p <- hostPort nc 18444+      let node = Node {kind = BitcoinCore, url = "http://127.0.0.1:" <> show rpcPort <> "/", p2p = p2p}+      waitRpc node+      withContainer (wrkSpec image outDir) $ \wc ->+        withStatsd $ \statsPort stats ->+          withRamDir $ \dir ->+            withServer o outDir dir redisURL p2p statsPort $ \srv ->+              stress o {out = outDir} node redisURL srv stats wc  -- | Valkey with room for the bodies of every transaction. valkey :: String -> ContainerSpec@@ -244,8 +245,6 @@             "--redis",             "--redis-url",             redisURL,-            "--redis-min-addrs",-            "10",             "--xpub-gap",             show gap,             "--xpub-gap-init",@@ -319,20 +318,20 @@   }  -- | The wallets, derived from a fixed seed, and the key of the miner.-makeWallets :: Ctx -> Opts -> ([Wallet], Key)-makeWallets ctx o = ([wallet (fromIntegral i) | i <- [0 .. o.wallets - 1]], keyOf (hardSubKey ctx root 9999) 0 0)+makeWallets :: Opts -> ([Wallet], Key)+makeWallets o = ([wallet (fromIntegral i) | i <- [0 .. o.wallets - 1]], keyOf (hardSubKey root 9999) 0 0)   where     root = makeXPrvKey (C8.pack ("haskoin-store-stress-" <> show o.seed))     wallet i =-      let w = hardSubKey ctx root i+      let w = hardSubKey root i        in Wallet-            { xpub = xPubExport btcRegTest ctx (deriveXPubKey ctx w),+            { xpub = xPubExport btcRegTest (deriveXPubKey w),               receive = Seq.fromList [keyOf w 0 j | j <- [0 .. fromIntegral o.addresses - 1]],               change = Seq.fromList [keyOf w 1 j | j <- [0 .. fromIntegral o.addresses - 1]]             }     keyOf w c j =-      let k = prvSubKey ctx (prvSubKey ctx w c) j-          a = xPubAddr ctx (deriveXPubKey ctx k)+      let k = prvSubKey (prvSubKey w c) j+          a = xPubAddr (deriveXPubKey k)        in Key {script = fromMaybe (error "no script") (addressToOutput a), secret = k.key}  -- | Address of a key, as text.@@ -342,20 +341,19 @@   _ -> error "not a key hash"  -- | Sign a transaction that spends coins.-sign :: Ctx -> Tx -> [Coin] -> Tx-sign ctx tx coins =+sign :: Tx -> [Coin] -> Tx+sign tx coins =   either error id $     signTx       btcRegTest-      ctx       tx       [SigInput c.key.script c.value c.outpoint sigHashAll Nothing | c <- coins]       [c.key.secret | c <- coins]  -- | A payment from a coin to another wallet, with change back to the wallet -- of the coin. Gives the transaction and its new coins.-pay :: Ctx -> IOGenM StdGen -> [Wallet] -> Coin -> IO (Tx, [Coin])-pay ctx g ws c = do+pay :: IOGenM StdGen -> [Wallet] -> Coin -> IO (Tx, [Coin])+pay g ws c = do   w' <- (\i -> (c.wallet + i) `mod` length ws) <$> uniformRM (1, max 1 (length ws - 1)) g   let to = ws !! w'       from = ws !! c.wallet@@ -367,7 +365,7 @@       outs         | c.value >= amount + fee + 10000 = [(r.script, amount), (ch.script, c.value - amount - fee)]         | otherwise = [(r.script, c.value - fee)]-      tx = sign ctx (buildTx ctx [c.outpoint] outs) [c]+      tx = sign (buildTx [c.outpoint] outs) [c]       h = txHash tx       owners = [(w', r), (c.wallet, ch)]       coins = [Coin (OutPoint h i) v w k | (i, (_, v), (w, k)) <- zip3 [0 ..] outs owners]@@ -433,10 +431,10 @@ -- The stress test ------------------------------------------------------------------------------- -stress :: Opts -> Ctx -> Node -> String -> Server -> IORef Stats -> Container -> IO ()-stress o ctx node redisURL srv stats wc = do+stress :: Opts -> Node -> String -> Server -> IORef Stats -> Container -> IO ()+stress o node redisURL srv stats wc = do   g <- newIOGenM (mkStdGen o.seed)-  let (ws, miner) = makeWallets ctx o+  let (ws, miner) = makeWallets o       minerAddr = keyAddress miner       perFanOut = 1000       fanOuts = max 1 ((o.blockTxs * 3 `div` 2 + perFanOut - 1) `div` perFanOut)@@ -461,7 +459,7 @@         each = (v - 200000) `div` fromIntegral n         -- every receiving address of every wallet, in turn         targets = take n (cycle [(wi, Seq.index w.receive j) | j <- [0 .. o.addresses - 1], (wi, w) <- zip [0 ..] ws])-        tx = sign ctx (buildTx ctx [op] [(k.script, each) | (_, k) <- targets]) [Coin op v (-1) miner]+        tx = sign (buildTx [op] [(k.script, each) | (_, k) <- targets]) [Coin op v (-1) miner]     errs <- sendRaw node [tx]     forM_ errs $ mapM_ (\e -> fail ("The node refused a funding transaction: " <> show e))     return [Coin (OutPoint (txHash tx) i) each wi k | (i, (wi, k)) <- zip [0 ..] targets]@@ -484,7 +482,7 @@       withAsync (sampler start node srv redisURL stats stop sh) $ \_ ->         withAsync (wrkAll o ws srv wc start stop wlock) $ \_ -> do           (sent, refused) <- withFile (o.out <> "/rounds.jsonl") WriteMode $ \rh ->-            rounds o ctx g node ws minerAddr start rh (Seq.fromList coins0)+            rounds o g node ws minerAddr start rh (Seq.fromList coins0)           say "Waiting for the server to catch up"           t0 <- getCurrentTime           waitSynced node srv@@ -511,8 +509,8 @@  -- | Make the transactions, a block at a time. Gives how many were sent and -- how many the node refused.-rounds :: Opts -> Ctx -> IOGenM StdGen -> Node -> [Wallet] -> Text -> UTCTime -> Handle -> Seq Coin -> IO (Int, Int)-rounds o ctx g node ws minerAddr start rh = go 0 (0, 0)+rounds :: Opts -> IOGenM StdGen -> Node -> [Wallet] -> Text -> UTCTime -> Handle -> Seq Coin -> IO (Int, Int)+rounds o g node ws minerAddr start rh = go 0 (0, 0)   where     total = o.txs     go :: Int -> (Int, Int) -> Seq Coin -> IO (Int, Int)@@ -523,7 +521,7 @@               (spend, rest) = Seq.splitAt n pool           when (n == 0) $ fail "No coins left to spend"           t0 <- getCurrentTime-          made <- mapM (pay ctx g ws) (toList spend)+          made <- mapM (pay g ws) (toList spend)           -- sign now, not when the transactions are sent           mapM_ (evaluate . force . fst) made           t1 <- getCurrentTime
swagger.yaml view
@@ -11,7 +11,7 @@   - url: './'  info:-  version: &version 1.15.1+  version: &version 1.16.0   title: Haskoin   contact:     name: JP Rupp@@ -131,13 +131,13 @@     | 404 | `not-found-or-invalid-arg` | The thing does not exist, or a value in the path is not valid. |     | 409 | `multiple-tx-index` | A short transaction number matches several transactions. |     | 410 | `replaced` | A transaction asked for by its id was replaced by one that a block confirmed; `replaced-by` and `height` say which and where. |+    | 429 | `server-busy` | A wallet asked for is not in the cache yet, and the server cannot have it there soon. Try again after the seconds of its `Retry-After` header. Any endpoint of wallets may answer it; `/events` only for an `xpub` to watch. |+    | 429 | `server-busy` | The server has more long requests (lists, whole blocks, wallets) than it can answer now. Try again after a second, or ask another server. |     | 500 | `server-error` | Something went wrong on the server; it is logged there. |+    | 503 | `server-error` | Reading a wallet into the cache failed; ask again after a second. |     | 503 | | From `/health`: the server is not ready. |     | 503 | `behind-fleet` | The server is one of a fleet and does not have the block that the fleet answers from; another server of the fleet does. | -    `/health` also answers 203 for a server that works but cannot reach its-    Redis; see there.-     A value in the path that does not parse, such as a misspelt address, gives     404 rather than 400. Paths are matched by the shape of what is in them, and     a segment that does not parse may belong to another endpoint (`/block/best`@@ -216,12 +216,21 @@     of any kind is new, and gets an empty answer. The check only runs for     wallets that look empty, so it costs nothing for wallets in use. -    Servers with a Redis cache keep large wallets ready: their first query reads-    the database and caches the result, and later ones are answered from memory.-    A cached answer is always the same as the database's; `nocache=true` skips-    the cache. The servers of a fleet share one such cache, and each answers-    from it what it would answer from its database at the block that the-    fleet answers from.+    Servers keep wallets ready in a cache, in the Redis that the servers of+    a fleet share. A wallet that is not in the cache is read+    into it first, once, by one server: the queries that come meanwhile, to+    this server or to another that shares the Redis, wait for that one+    reading, ten seconds at most by default, and are then answered with it.+    When the wallet is not expected to be ready within that time, because+    many wallets are waiting or this one is large, or when the wait runs out,+    the server answers 429 with `Retry-After`: the seconds after which the+    server that reads it expects to have it. Every server that shares the+    Redis gives the same number, from that one estimate. The reading goes on+    meanwhile, so ask again then. A cached answer is always the same as the+    database's at some moment, never parts of two. The caches keep the wallets used most recently and those that take+    longest to read. The servers of a fleet share one cache in Redis, and each+    answers from it what it would answer from its database at the block that+    the fleet answers from.      ## Live updates @@ -291,7 +300,15 @@     - **`Vary: Accept-Encoding`** is on every answer, so that a cache keeps       compressed and plain answers apart.     - **`Retry-After`** is on a 503: one second from a server that is behind-      its fleet, five from a health check that fails.+      its fleet or failed to read a wallet into its cache, and five from a+      health check that fails. It is on a 429 too: one second from a server+      that is too busy; for a wallet that is not in the cache yet, it is when+      the server expects to have it there, from the wallets ahead of it and+      how long a wallet takes to read; from a server that another one is+      reading it for, when that one expects to be done.+    - **A busy server makes long requests wait** for a moment, and refuses+      with 429 those that it cannot start soon. A request that it started+      is answered in full.     - **The reference itself** is at the root of every server: `/` is this       page, for the version that the server runs, and `/swagger.yaml` the file       behind it.@@ -333,34 +350,16 @@         still catching up with the network answers 503, so a load balancer stops         sending it traffic until its answers are current. -        A server that has a Redis cache and cannot reach it answers **203** when-        every check passes, with the header `Haskoin-Degraded: redis` and-        `"redis": {"ok": false}` in the body. It goes on-        answering every request, from its own database: the answers are right for-        what this server has seen, but they are no longer the ones that the servers-        sharing that Redis give together, and requests about large wallets are-        slower. A load balancer that takes any status in the 200s for healthy keeps-        sending it traffic, which is what it should do; a monitor that expects 200-        sees that something is wrong.+        A server that cannot reach its Redis answers **503** too, with the+        header `Haskoin-Degraded: redis` and `"redis": {"ok": false}` in the+        body: Redis holds the mempool and the cache of extended public keys+        that every server of the fleet answers from.          The checks are run in the background every 30 seconds (unless the operator         changed it), and this endpoint returns the latest result. It does no work of         its own, so it is safe to call as often as a load balancer likes.       operationId: healthCheck       responses:-        '203':-          description: |-            Every check passes, but the server cannot reach its Redis and answers-            from its own database alone. The body is that of 200.-          headers:-            Haskoin-Degraded:-              description: What the server is doing without; `redis`.-              schema:-                type: string-          content:-            application/json:-              schema:-                $ref: '#/components/schemas/HealthCheck'         '200':           description: success           content:@@ -2565,6 +2564,13 @@         - **No answers.** As between a node and an SPV client, nothing is sent           back. A message that the server cannot read or will not do closes           the connection with code 1008 and the reason.+        - **Wallets come from the cache.** An `xpub` to watch that the server+          does not have in its cache yet is read into it first, as for+          `/xpub/{xpub}`. If that cannot be done soon, the stream, or the+          opening of the websocket, is answered 429 with `Retry-After`, or+          503 if reading the wallet failed; a message to watch a key on an open+          websocket closes it with code 1013, to try again later, with the+          seconds to wait in the reason.         - When the server does not know the transaction of an event, which           happens when one leaves the mempool long after it entered, the event           goes to every client.@@ -2590,6 +2596,10 @@             application/json:               schema:                 $ref: '#/components/schemas/Error'+        '429':+          $ref: '#/components/responses/walletNotReady'+        '503':+          $ref: '#/components/responses/walletFailed'    '/peers':     get:@@ -2644,7 +2654,6 @@       parameters:         - $ref: '#/components/parameters/xpubParam'         - $ref: '#/components/parameters/xpubDerive'-        - $ref: '#/components/parameters/nocacheParam'       responses:         '200':           description: Information about extended public key.@@ -2681,19 +2690,23 @@             application/json:               schema:                 $ref: '#/components/schemas/Error'+        '429':+          $ref: '#/components/responses/walletNotReady'+        '503':+          $ref: '#/components/responses/walletFailed'     delete:       tags:         - Address       operationId: deleteCachedXPub       summary: Delete Cached Extended Key       description: |-        Remove a wallet from the Redis cache of this server, if it has one. The-        answer says whether it was cached.+        Remove a wallet from the cache of the fleet, in Redis.+        The answer says whether the cache had it.          There is rarely a reason to call this: cached wallets are kept up to date-        with every block and transaction, and the least recently used ones are-        evicted when the cache is full. It is there to free memory, or to rule out-        the cache when looking into a problem.+        with every block and transaction, and when a cache is full it evicts the+        wallets that were used least recently and are quickest to read again. It is+        there to free memory, or to rule out the cache when looking into a problem.       parameters:         - $ref: '#/components/parameters/xpubParam'         - $ref: '#/components/parameters/xpubDerive'@@ -2733,7 +2746,6 @@         - $ref: '#/components/parameters/heightParam'         - $ref: '#/components/parameters/limitParam'         - $ref: '#/components/parameters/offsetParam'-        - $ref: '#/components/parameters/nocacheParam'       responses:         '200':           description: List of transaction objects.@@ -2972,6 +2984,10 @@             application/json:               schema:                 $ref: '#/components/schemas/Error'+        '429':+          $ref: '#/components/responses/walletNotReady'+        '503':+          $ref: '#/components/responses/walletFailed'    '/xpub/{xpub}/transactions':     get:@@ -2989,7 +3005,6 @@         - $ref: '#/components/parameters/heightParam'         - $ref: '#/components/parameters/limitParam'         - $ref: '#/components/parameters/offsetParam'-        - $ref: '#/components/parameters/nocacheParam'       responses:         '200':           description: List of transactions.@@ -3032,6 +3047,10 @@             application/json:               schema:                 $ref: '#/components/schemas/Error'+        '429':+          $ref: '#/components/responses/walletNotReady'+        '503':+          $ref: '#/components/responses/walletFailed'    '/xpub/{xpub}/balances':     get:@@ -3047,7 +3066,6 @@       parameters:         - $ref: '#/components/parameters/xpubParam'         - $ref: '#/components/parameters/xpubDerive'-        - $ref: '#/components/parameters/nocacheParam'       responses:         '200':           description: Balances only for used addresses.@@ -3108,6 +3126,10 @@             application/json:               schema:                 $ref: '#/components/schemas/Error'+        '429':+          $ref: '#/components/responses/walletNotReady'+        '503':+          $ref: '#/components/responses/walletFailed'    '/xpub/{xpub}/unspent':     get:@@ -3124,7 +3146,6 @@         - $ref: '#/components/parameters/heightParam'         - $ref: '#/components/parameters/limitParam'         - $ref: '#/components/parameters/offsetParam'-        - $ref: '#/components/parameters/nocacheParam'       responses:         '200':           description: List of unspent output objects.@@ -3191,6 +3212,10 @@             application/json:               schema:                 $ref: '#/components/schemas/Error'+        '429':+          $ref: '#/components/responses/walletNotReady'+        '503':+          $ref: '#/components/responses/walletFailed'    '/blockchain/balance':     get:@@ -3229,6 +3254,10 @@             application/json:               schema:                 $ref: '#/components/schemas/Error'+        '429':+          $ref: '#/components/responses/walletNotReady'+        '503':+          $ref: '#/components/responses/walletFailed'     post:       tags:         - Blockchain@@ -3264,6 +3293,10 @@             application/json:               schema:                 $ref: '#/components/schemas/Error'+        '429':+          $ref: '#/components/responses/walletNotReady'+        '503':+          $ref: '#/components/responses/walletFailed'    '/blockchain/rawaddr/{address}':     get:@@ -3427,6 +3460,10 @@             application/json:               schema:                 $ref: '#/components/schemas/Error'+        '429':+          $ref: '#/components/responses/walletNotReady'+        '503':+          $ref: '#/components/responses/walletFailed'    '/blockchain/unconfirmed-transactions':     get:@@ -3595,6 +3632,10 @@             application/json:               schema:                 $ref: '#/components/schemas/Error'+        '429':+          $ref: '#/components/responses/walletNotReady'+        '503':+          $ref: '#/components/responses/walletFailed'     post:       tags:         - Blockchain@@ -3894,6 +3935,10 @@             application/json:               schema:                 $ref: '#/components/schemas/Error'+        '429':+          $ref: '#/components/responses/walletNotReady'+        '503':+          $ref: '#/components/responses/walletFailed'    '/blockchain/rawblock/{block}':     get:@@ -5162,6 +5207,10 @@             application/json:               schema:                 $ref: '#/components/schemas/Error'+        '429':+          $ref: '#/components/responses/walletNotReady'+        '503':+          $ref: '#/components/responses/walletFailed'     post:       tags:         - Blockchain@@ -5223,6 +5272,10 @@             application/json:               schema:                 $ref: '#/components/schemas/Error'+        '429':+          $ref: '#/components/responses/walletNotReady'+        '503':+          $ref: '#/components/responses/walletFailed'    '/blockchain/export-history':     get:@@ -5279,6 +5332,10 @@             application/json:               schema:                 $ref: '#/components/schemas/Error'+        '429':+          $ref: '#/components/responses/walletNotReady'+        '503':+          $ref: '#/components/responses/walletFailed'     post:       tags:         - Blockchain@@ -5325,6 +5382,10 @@             application/json:               schema:                 $ref: '#/components/schemas/Error'+        '429':+          $ref: '#/components/responses/walletNotReady'+        '503':+          $ref: '#/components/responses/walletFailed'    '/blockchain/q/addresstohash/{address}':     get:@@ -7154,6 +7215,45 @@           type: boolean           example: true +  responses:++    walletNotReady:+      description: |+        A wallet asked for is not in the cache yet, and cannot be there soon: many+        wallets are waiting to be read, or this one is large; another server that+        shares the cache may be reading it, which this one waits for as long as it+        would for its own. Unless its queue is full, or it is not in sync with the+        network, the server goes on reading it. Ask again after `Retry-After`+        seconds, when the server expects to have it, or ask another server. A+        request for several wallets is refused if one of them is not ready, with+        the longest wait of those that are not.+      headers:+        Retry-After:+          description: Seconds after which to ask again.+          schema:+            type: integer+      content:+        application/json:+          schema:+            $ref: '#/components/schemas/Error'+          example:+            error: server-busy++    walletFailed:+      description: Reading a wallet into the cache failed. Ask again after a second.+      headers:+        Retry-After:+          description: Always one second.+          schema:+            type: integer+      content:+        application/json:+          schema:+            $ref: '#/components/schemas/Error'+          example:+            error: server-error+            message: Server error+   parameters:      pubKeyParam:@@ -7246,19 +7346,6 @@       required: false       schema:         type: integer--    nocacheParam:-      name: nocache-      in: query-      description: |-        Answer from the database of this server, without reading the Redis cache and-        without caching the wallet. The answer is the same; this is for comparing,-        and for servers whose cache is shared with others that may be ahead of or-        behind this one. A server of a fleet still answers from the block and the-        mempool that the fleet shares, only without its cache of wallets.-      required: false-      schema:-        type: boolean      notxParam:       name: notx
test/Haskoin/Store/BlockStore/SyncSpec.hs view
@@ -1,3 +1,6 @@+{-# LANGUAGE LambdaCase #-}+{-# LANGUAGE OverloadedRecordDot #-}+ -- | The state machine of the block download, driven by random sequences of -- events, with its rules checked after every step. module Haskoin.Store.BlockStore.SyncSpec (spec) where@@ -13,12 +16,14 @@ import Test.Hspec.QuickCheck (prop) import Test.QuickCheck --- | Something that happens to the block store, from one of three peers.+-- | Something that happens to the block store, from one of four peers. data Event   = Start Int+  | Pass Int   | Release Int   | Request [Int]-  | Import Int+  | -- | a peer sends a block, which is imported if it is accepted+    Deliver Int Int   | Forget   | Wait Int   deriving stock (Show)@@ -27,14 +32,15 @@   arbitrary =     frequency       [ (2, Start <$> peer),+        (2, Pass <$> peer),         (1, Release <$> peer),         (3, Request <$> listOf block),-        (3, Import <$> block),+        (4, Deliver <$> peer <*> block),         (1, pure Forget),         (2, Wait <$> choose (0, 30))       ]     where-      peer = choose (0, 2)+      peer = choose (0, 3)       block = choose (0, 9)  -- | A state and the time it is at.@@ -46,66 +52,137 @@ t0 :: UTCTime t0 = UTCTime (fromGregorian 2026 11 9) 0 +at :: Int -> UTCTime+at secs = addUTCTime (fromIntegral secs) t0+ -- | Apply an event as the block store does: imports only what 'accept' -- takes. step :: World -> Event -> World step (now, s) = \case   Start p -> (now, fromMaybe s (start now p s))+  Pass p -> (now, snd (pass now p s))   Release p -> (now, snd (release p s))   Request bs -> (now, request (map hash bs) s)-  Import b -> case syncingPeer s of-    Just p | accept p (hash b) s == Requested -> (now, imported now (hash b) s)-    _ -> (now, s)+  Deliver p b -> case accept p (hash b) s of+    (Requested, s') -> (now, imported now (hash b) s')+    (_, s') -> (now, s')   Forget -> (now, forgetRequests s)   Wait secs -> (addUTCTime (fromIntegral secs) now, s)  -- | Every state that a sequence of events goes through. run :: [Event] -> [World]-run = scanl step (t0, Idle)+run = scanl step (t0, idle t0) -timeout :: NominalDiffTime+-- | Each state with the event that follows it and the state after.+steps :: [Event] -> [(World, Event, World)]+steps es = zip3 (run es) es (drop 1 (run es))++timeout, stall :: NominalDiffTime timeout = 60+stall = 10 +-- | What a peer owes, as the syncing peer or as one passed over.+owed :: Int -> Sync Int -> Maybe [BlockHash]+owed p s = case [a.owes | a <- maybe [] pure s.current <> s.passed, a.peer == p] of+  [] -> Nothing+  os -> Just (concat os)+ spec :: Spec spec = describe "Block download" $ do-  prop "never has a block outstanding twice" $ \es ->+  prop "never wants a block twice" $ \es ->     all (\(_, s) -> nub (outstanding s) == outstanding s) (run es) -  prop "has a syncing peer only after starting, and keeps it until released" $ \es ->+  prop "has each peer once, and no peer passed over that owes nothing" $ \es ->     and+      [ nub peers == peers && all (not . null . (.owes)) s.passed+      | (_, s) <- run es,+        let peers = maybe [] pure (syncingPeer s) <> passedPeers s+      ]++  prop "changes its syncing peer only by starting, passing it over or releasing it" $ \es ->+    and       [ case e of-          Start _ -> syncingPeer s' == syncingPeer s || isNothing (syncingPeer s)-          Release p-            | syncingPeer s == Just p -> syncingPeer s' == Nothing+          Start p -> syncingPeer s' == if isNothing (syncingPeer s) then Just p else syncingPeer s+          Pass p -> syncingPeer s' == (p <$ syncingPeer s)+          Release p | syncingPeer s == Just p -> isNothing (syncingPeer s')           _ -> syncingPeer s' == syncingPeer s-      | ((_, s), e, (_, s')) <- zip3 (run es) es (drop 1 (run es))+      | ((_, s), e, (_, s')) <- steps es       ] -  prop "accepts a block only from the syncing peer, and only if outstanding" $ \es ->-    forAll (choose (0, 2)) $ \peer -> forAll (hash <$> choose (0, 9)) $ \h ->+  prop "keeps a peer that is passed over owed its blocks" $ \es ->+    and+      [ owed old s' == owed old s+      | ((_, s), Pass p, (_, s')) <- steps es,+        Just old <- [syncingPeer s],+        old /= p,+        owed old s /= Just []+      ]++  prop "accepts a block only from a peer that was asked for it, as often as it was asked" $ \es ->+    forAll (choose (0, 3)) $ \peer -> forAll (hash <$> choose (0, 9)) $ \h ->       let (_, s) = last (run es)-       in accept peer h s === case syncingPeer s of-            Just q-              | q /= peer -> NotSyncingPeer-              | h `elem` outstanding s -> Requested-              | otherwise -> NotRequested-            Nothing -> NotSyncingPeer+          (verdict, s') = accept peer h s+          copies = length . filter (== h) . fromMaybe []+       in conjoin+            [ verdict === case owed peer s of+                Nothing -> NotSyncingPeer+                Just os+                  | h `notElem` os -> NotRequested+                  | h `elem` outstanding s -> Requested+                  | otherwise -> Late,+              -- the peer owes one copy fewer: it may send a block as many+              -- times as it was asked for it, as after a reorganisation+              counterexample "still owed" $+                verdict `notElem` [Requested, Late] || copies (owed peer s') == copies (owed peer s) - 1+            ] +  prop "imports a block from whichever peer delivers it first" $ \es ->+    and+      [ hash b `notElem` outstanding s'+      | ((_, s), Deliver p b, (_, s')) <- steps es,+        fst (accept p (hash b) s) == Requested+      ]+   prop "keeps the requested blocks in the order they were asked for" $ \es bs ->     let (_, s) = last (run es)         s' = request (map hash bs) s      in case syncingPeer s of-          Nothing -> outstanding s' === []+          Nothing -> outstanding s' === outstanding s           Just _ -> take (length (outstanding s)) (outstanding s') === outstanding s -  prop "times out only a peer that delivered nothing for longer than the timeout" $ \es ->+  prop "asks the next syncing peer for all that is wanted" $ \es ->     and-      [ case s of-          Idle -> isNothing (timedOut timeout now s)-          Syncing d -> (timedOut timeout now s == Just d.peer) == (now `diffUTCTime` d.time > timeout)+      [ fmap snd asked == Just (outstanding s) && all (`elem` fromMaybe [] (owed p s')) (outstanding s)+      | ((now, s), Pass p, _) <- steps es,+        Just old <- [syncingPeer s],+        old /= p,+        let (asked, s') = pass now p s+      ]++  prop "passes over a peer only when blocks are wanted and none came within its patience" $ \es ->+    and+      [ stalled stall timeout now s+          == if not (null (outstanding s)) && now `diffUTCTime` s.time > patience stall timeout s+            then syncingPeer s+            else Nothing       | (now, s) <- run es       ] +  prop "is patient for the stall at least, and for the timeout at most" $ \es ->+    and [patience stall timeout s >= stall && patience stall timeout s <= timeout | (_, s) <- run es]++  prop "times out only a syncing peer when nothing was imported for longer than the timeout" $ \es ->+    and+      [ timedOut timeout now s == if now `diffUTCTime` s.time > timeout then syncingPeer s else Nothing+      | (now, s) <- run es+      ]++  prop "expires the peers passed over that owe blocks for longer than the timeout" $ \es ->+    and+      [ expired timeout now s == [a.peer | a <- s.passed, now `diffUTCTime` a.since > timeout]+      | (now, s) <- run es+      ]+   prop "asks for more blocks exactly when the window has room" $ \es (Positive window) ->     let (_, s) = last (run es)      in wantsMore window s === case syncingPeer s of@@ -121,9 +198,96 @@       needsMoreBlocks 1 0 `shouldBe` True       needsMoreBlocks 3 0 `shouldBe` True -  it "resets the clock of the peer when it delivers a block" $ do-    let s = Syncing (Download (0 :: Int) t0 [])-        later = addUTCTime 50 t0-        s' = imported later (hash 1) (request [hash 1] s)-    timedOut timeout (addUTCTime 100 t0) s `shouldBe` Just 0-    timedOut timeout (addUTCTime 100 t0) s' `shouldBe` Nothing+  it "resets the clock when a block is imported" $ do+    let s = fromMaybe (idle t0) (start t0 (0 :: Int) (idle t0))+        s' = imported (at 50) (hash 1) (request [hash 1] s)+    timedOut timeout (at 100) s `shouldBe` Just 0+    timedOut timeout (at 100) s' `shouldBe` Nothing++  describe "a peer that does not deliver" $ do+    -- peer 0 is asked for block 1 and stays silent+    let asked = request [hash 1] (fromMaybe (idle t0) (start t0 (0 :: Int) (idle t0)))+    it "is passed over after the stall, and not before" $ do+      stalled stall timeout (at 10) asked `shouldBe` Nothing+      stalled stall timeout (at 11) asked `shouldBe` Just 0+    it "is not passed over when nothing is wanted" $+      stalled stall timeout (at 1000) (fromMaybe (idle t0) (start t0 (0 :: Int) (idle t0))) `shouldBe` Nothing+    it "has its block taken if it delivers before the next peer" $ do+      let (told, s) = pass (at 11) 1 asked+      told `shouldBe` Just (0, [hash 1])+      syncingPeer s `shouldBe` Just 1+      passedPeers s `shouldBe` [0]+      fst (accept 0 (hash 1) s) `shouldBe` Requested+    it "has its block dropped, and is not at fault, if the next peer delivered first" $ do+      let (_, s) = pass (at 11) 1 asked+          (first, s') = accept 1 (hash 1) s+          s'' = imported (at 12) (hash 1) s'+      first `shouldBe` Requested+      fst (accept 0 (hash 1) s'') `shouldBe` Late+    it "gives the next peers twice as long each time, up to the timeout" $ do+      let (_, s1) = pass (at 11) 1 asked+          (_, s2) = pass (at 32) 2 s1+          (_, s3) = pass (at 73) 3 s2+      map (patience stall timeout) [asked, s1, s2, s3] `shouldBe` [10, 20, 40, 60]+      -- peer 1 was asked at 11 seconds, and has 20 from then+      stalled stall timeout (at 31) s1 `shouldBe` Nothing+      stalled stall timeout (at 32) s1 `shouldBe` Just 1+    it "goes back to the first stall when a block is imported" $ do+      let (_, s1) = pass (at 11) 1 asked+          s2 = imported (at 12) (hash 1) (snd (accept 1 (hash 1) s1))+      patience stall timeout s2 `shouldBe` 10+    it "is disconnected if it still owes its block after the timeout" $ do+      let (_, s) = pass (at 11) 1 asked+      -- silent since the start, not since it was passed over+      expired timeout (at 60) s `shouldBe` []+      expired timeout (at 61) s `shouldBe` [0]+    it "is forgotten when it disconnects" $ do+      let (_, s) = pass (at 11) 1 asked+          (wasSyncing, s') = release 0 s+      wasSyncing `shouldBe` False+      passedPeers s' `shouldBe` []+      fst (accept 0 (hash 1) s') `shouldBe` NotSyncingPeer+    it "still owes its blocks if it becomes the syncing peer again" $ do+      let (_, s1) = pass (at 11) 1 asked+          (_, s2) = release 1 s1+          s3 = fromMaybe s2 (start (at 20) 0 s2)+      passedPeers s3 `shouldBe` []+      -- the block is not wanted any more: the request went with peer 1+      fst (accept 0 (hash 1) s3) `shouldBe` Late++  describe "Looking for a peer to ask" $ do+    let wait = 5 :: NominalDiffTime+        (Just (n, _), searching) = ping (at 0) 7 [0, 1] (idle t0)+    it "pings the peers once, all with the same nonce" $ do+      n `shouldBe` 7+      probing searching `shouldBe` True+      -- peer 2 announces the block later: it is pinged, the others not again+      let (more, s) = ping (at 1) 9 [0, 1, 2] searching+      more `shouldBe` Just (7, [2])+      fst (ping (at 2) 9 [0, 1, 2] s) `shouldBe` Nothing+    it "chooses the first peer that answers in time" $ do+      let (chosen, s) = pong wait (at 1) 1 7 searching+      chosen `shouldBe` True+      -- the search goes on with the rest until that peer is made the syncing peer+      probing s `shouldBe` True+      probing <$> start (at 1) 1 s `shouldBe` Just False+    it "takes no answer to another ping, nor from a peer that was not pinged" $ do+      pong wait (at 1) 1 8 searching `shouldBe` (False, searching)+      pong wait (at 1) 3 7 searching `shouldBe` (False, searching)+    it "does not choose a peer that answers late while another may be in time" $ do+      let (_, s1) = ping (at 3) 9 [2] searching+          (chosen, s2) = pong wait (at 6) 0 7 s1+      chosen `shouldBe` False+      -- peer 1 is late too, and peer 2 has until 8 seconds+      overdue wait (at 6) s2 `shouldBe` []+      fst (pong wait (at 7) 2 7 s2) `shouldBe` True+    it "asks a late peer when there is no other option, one that answered first" $ do+      overdue wait (at 5) searching `shouldBe` []+      overdue wait (at 6) searching `shouldBe` [0, 1]+      let (_, s) = pong wait (at 7) 1 7 searching+      overdue wait (at 7) s `shouldBe` [1, 0]+    it "ends when the last peer pinged is gone" $ do+      let (_, s1) = release 0 searching+      probing s1 `shouldBe` True+      probing (skip 1 s1) `shouldBe` False+      probing (settle searching) `shouldBe` False
− test/Haskoin/Store/Cache/ProtocolSpec.hs
@@ -1,158 +0,0 @@--- | The protocol of stores sharing one Redis, against a model of Redis:--- several stores catch up with a chain in which every block changes one--- cached xpub, in a random interleaving, and decide with the functions of--- "Haskoin.Store.Cache.Protocol" what to do at each step.-module Haskoin.Store.Cache.ProtocolSpec (spec) where--import Data.IntMap.Strict (IntMap)-import Data.IntMap.Strict qualified as IntMap-import Haskoin.Store.Cache.Protocol-import Test.Hspec-import Test.Hspec.QuickCheck (prop)-import Test.QuickCheck---- | The claim of the change of a block.-data Key = Absent | BusyBy Int | DoneKey-  deriving stock (Eq, Show)---- | Redis: the height the entry is stamped with, and the claims.-data Redis = Redis-  { stamp :: !(Maybe Integer),-    claims :: !(IntMap Key)-  }-  deriving stock (Show)---- | A store: its height, the change it holds, and changes to claim again.-data Store = Store-  { height :: !Int,-    holding :: !(Maybe Int),-    retry :: ![Int]-  }---- | What happened.-data Outcome = Outcome-  { -- | the stamp written by each write, by the store and change it was for-    writes :: ![(Int, Int, Integer)],-    -- | claims given up-    releases :: !Int,-    redis :: !Redis,-    -- | whether every store reached the tip with nothing left to do-    finished :: !Bool-  }-  deriving stock (Show)---- | Try to claim the change of a block, and do what 'afterClaim' says.-tryClaim :: Int -> Int -> Redis -> Store -> (Redis, Store)-tryClaim s i r st =-  let found = case IntMap.findWithDefault Absent i r.claims of-        Absent -> Claimed-        DoneKey -> Done-        BusyBy _ -> Busy-   in case afterClaim found of-        ReloadNow -> (r {claims = IntMap.insert i (BusyBy s) r.claims}, st {holding = Just i})-        Skip -> (r, st)-        ClaimLater -> (r, st {retry = st.retry <> [i]})---- | A step of a store, if it can take one: reload for the change it holds,--- claim a change again, or import the next block and claim its change.-steps :: Int -> Int -> Outcome -> Store -> [(Outcome, Store)]-steps tip s o st = case st.holding of-  Just i ->-    let h = toInteger st.height-        -- the entry has a transaction the store has not seen if it is-        -- stamped with a later block, as each block adds one-        unknown = maybe False (> h) o.redis.stamp-        reload = if mayReplace o.redis.stamp (fromIntegral st.height) unknown then Reloaded else Behind-        r = o.redis-     in case afterReload reload of-          Finish ->-            [ ( o-                  { writes = o.writes <> [(s, i, h)],-                    redis = r {stamp = Just h, claims = IntMap.insert i DoneKey r.claims}-                  },-                st {holding = Nothing}-              )-            ]-          Release ->-            [ ( o {releases = o.releases + 1, redis = r {claims = IntMap.insert i Absent r.claims}},-                st {holding = Nothing}-              )-            ]-          Hold -> []-  Nothing ->-    [ let (r, st') = tryClaim s i o.redis st {retry = rest} in (o {redis = r}, st')-    | i : rest <- [st.retry]-    ]-      <> [ let (r, st') = tryClaim s (st.height + 1) o.redis st {height = st.height + 1}-            in (o {redis = r}, st')-         | st.height < tip-         ]---- | Run stores that start at the given heights until none can take a step,--- picking the next step at random. The changes of the blocks up to the--- lowest store were applied before.-simulate :: Int -> [Int] -> Gen Outcome-simulate tip starts =-  simulateFrom-    (Redis (Just (toInteger (minimum starts))) (IntMap.fromList [(i, DoneKey) | i <- [1 .. minimum starts]]))-    tip-    starts---- | Like 'simulate', from a state of Redis.-simulateFrom :: Redis -> Int -> [Int] -> Gen Outcome-simulateFrom r0 tip starts = go (10000 :: Int) start0 (IntMap.fromList (zip [0 ..] stores0))-  where-    stores0 = [Store h Nothing [] | h <- starts]-    start0 = Outcome {writes = [], releases = 0, redis = r0, finished = False}-    go 0 o _ = return o-    go n o stores =-      case [(s, x) | (s, st) <- IntMap.toList stores, x <- steps tip s o st] of-        [] -> return o {finished = True}-        xs -> do-          (s, (o', st')) <- elements xs-          go (n - 1) o' (IntMap.insert s st' stores)---- | A small chain and up to four stores at random heights on it.-chain :: Gen (Int, [Int])-chain = do-  tip <- choose (1, 12)-  stores <- choose (1, 4)-  starts <- vectorOf stores (choose (0, tip))-  return (tip, starts)--spec :: Spec-spec = describe "Stores sharing Redis" $ do-  prop "all reach the tip with nothing left to do" $-    forAll chain $-      \(tip, starts) -> forAll (simulate tip starts) $ \o -> o.finished--  prop "end with the entry of the tip" $-    forAll chain $ \(tip, starts) -> forAll (simulate tip starts) $ \o ->-      o.redis.stamp === Just (toInteger tip)--  prop "never replace an entry with one that knows less" $-    forAll chain $ \(tip, starts) -> forAll (simulate tip starts) $ \o ->-      let stamps = [h | (_, _, h) <- o.writes]-       in and (zipWith (<=) stamps (drop 1 stamps))--  prop "write once per change, and again only for a change given up" $-    forAll chain $ \(tip, starts) -> forAll (simulate tip starts) $ \o ->-      length o.writes <= tip - minimum starts + o.releases--  prop "leave nothing to do for stores that catch up after the first" $-    forAll chain $ \(tip, starts) ->-      forAll (simulate tip [minimum starts]) $ \first ->-        forAll (simulateFrom first.redis tip starts) $ \rest ->-          rest.writes === []--  it "claims, skips or tries again as the claim says" $-    map afterClaim [minBound .. maxBound] `shouldBe` [ReloadNow, Skip, ClaimLater]--  it "finishes, releases or holds as the reload says" $-    map afterReload [minBound .. maxBound] `shouldBe` [Finish, Release, Hold]--  it "replaces an entry of the same height or lower, never a higher one" $ do-    mayReplace Nothing 5 False `shouldBe` True-    mayReplace (Just 5) 5 False `shouldBe` True-    mayReplace (Just 6) 5 False `shouldBe` False-    mayReplace (Just 4) 5 True `shouldBe` False
− test/Haskoin/Store/CacheSpec.hs
@@ -1,57 +0,0 @@-module Haskoin.Store.CacheSpec (spec) where--import Data.ByteString qualified as BS-import Data.ByteString.Char8 qualified as C8-import Data.List (nub, sort)-import Haskoin (bch, btc, deriveXPubKey, makeXPrvKey, prepareContext)-import Haskoin.Store.Cache (blockRefScore, cachePrefix, fullReloadAfter, fullReloadEvery, scoreBlockRef)-import Haskoin.Store.Data (BlockRef (..), DeriveType (..), XPubSpec (..))-import Test.Hspec (Spec, describe, it, shouldBe, shouldSatisfy)-import Test.Hspec.QuickCheck (prop)-import Test.QuickCheck (Gen, choose, forAll, listOf, oneof)--spec :: Spec-spec = do-  describe "Score for block reference" $ do-    prop "sorts newest first" $-      forAll arbitraryBlockRefs $ \ts ->-        let scores = map blockRefScore (sort ts)-         in sort scores == reverse scores-    prop "respects identity" $-      forAll arbitraryBlockRef $ \b ->-        scoreBlockRef (blockRefScore b) == b-  describe "Keys" $-    it "start with a prefix of their own for each network" $ do-      cachePrefix btc `shouldBe` "hs2:btc:"-      cachePrefix bch `shouldBe` "hs2:bch:"-      -- no key of the old cache can start like a key of this one, nor the-      -- other way round: its keys are these names, or one of the one-letter-      -- prefixes followed by a serialized xpub or address-      let old = ["head", "max", "mempool", "lock", "xlock-", "b", "t", "u", "a", "i"]-          p = cachePrefix btc-      filter (\o -> o `BS.isPrefixOf` p || p `BS.isPrefixOf` o) old `shouldBe` []--  prepareContext $ \ctx ->-    describe "Scheduled full reloads" $-      it "come every five to ten minutes, at intervals spread over xpubs" $ do-        let seed i = C8.pack ("seed of the test xpub " <> show i)-            xpub i = XPubSpec (deriveXPubKey ctx (makeXPrvKey (seed i))) DeriveNormal-            after = map (fullReloadAfter . xpub) [1 .. 200 :: Int]-            half = fullReloadEvery `div` 2-        after `shouldSatisfy` all (\a -> a >= half && a < fullReloadEvery)-        -- each of the five minutes of the range has some of them-        sort (nub [(a - half) `div` 60000 | a <- after]) `shouldBe` [0 .. 4]--arbitraryBlockRefs :: Gen [BlockRef]-arbitraryBlockRefs = listOf arbitraryBlockRef--arbitraryBlockRef :: Gen BlockRef-arbitraryBlockRef = oneof [b, m]-  where-    b = do-      h <- choose (0, 0x07ffffff)-      p <- choose (0, 0x03ffffff)-      return BlockRef {height = h, position = p}-    m = do-      t <- choose (0, 0x001fffffffffffff)-      return MemRef {timestamp = t}
+ test/Haskoin/Store/Database/MergeSpec.hs view
@@ -0,0 +1,331 @@+-- | The merge operators of the database against what they are meant to+-- do, said in Haskell.+--+-- The operators are C functions that RocksDB calls on its own threads+-- ("Haskoin.Store.Database.Merge"). They are tested twice. First by+-- calling them directly, as RocksDB would, with random values and merge+-- records: the result must be the bytes of the Haskell fold of the same,+-- records combined beforehand must do what they did one by one, and input+-- that is not of the format must be refused. Then through a database:+-- merge records, puts and deletes written in any order, in memtables and+-- in table files, must read as the same fold, before and after a+-- compaction.+module Haskoin.Store.Database.MergeSpec (spec) where++import Control.Monad (forM_)+import Data.ByteString (ByteString)+import Data.ByteString qualified as B+import Data.Default (def)+import Data.IntMap.Strict (IntMap)+import Data.IntMap.Strict qualified as IntMap+import Data.Map.Strict qualified as Map+import Data.Maybe (fromMaybe)+import Data.Serialize (encode)+import Data.Word (Word32, Word64)+import Database.RocksDB+  ( FullMerge,+    MergeOperator (..),+    PartialMerge,+    columnFamilies,+    compactRange,+    deleteCF,+    flush,+    mergeCF,+    putCF,+  )+import Database.RocksDB.Query (retrieveCF)+import Foreign.C.String (CString)+import Foreign.C.Types (CInt (..), CSize (..), CUChar (..))+import Foreign.Marshal.Alloc (alloca, free)+import Foreign.Marshal.Array (withArray)+import Foreign.Marshal.Utils (withMany)+import Foreign.Ptr (FunPtr, Ptr, nullPtr)+import Foreign.Storable (peek)+import Haskoin+  ( Address,+    TxHash (..),+    btcRegTest,+    eitherToMaybe,+    scriptToAddressBS,+    sha256,+  )+import Haskoin.Store.Data (Spender (..))+import Haskoin.Store.Database.Merge (balanceOperator, spenderOperator)+import Haskoin.Store.Database.Query qualified as Q+import Haskoin.Store.Database.Reader (DatabaseReader (..), balanceCF, spenderCF)+import Haskoin.Store.Database.Types+  ( BalKey (..),+    BalVal (..),+    SpenderKey (..),+    SpenderVal (..),+    addBalVal,+    valToBalance,+  )+import Haskoin.Store.Histories (openLayered)+import Haskoin.Store.Integration.Temp (withSystemTempDirectory)+import Test.Hspec+import Test.Hspec.QuickCheck (modifyMaxSuccess, prop)+import Test.QuickCheck+  ( Gen,+    arbitrary,+    choose,+    elements,+    forAll,+    frequency,+    ioProperty,+    listOf,+    listOf1,+    oneof,+    vectorOf,+  )++spec :: Spec+spec = describe "the merge operators of the database" $ do+  describe "the formats that they read and write" $ do+    it "are those of the values: forty bytes for a balance, a count and forty bytes for each spender" $ do+      encode (BalVal 1 2 3 4 0x0102030405060708)+        `shouldBe` B.pack (concat [[0, 0, 0, 0, 0, 0, 0, n] | n <- [1 .. 4]] <> [1 .. 8])+      let spender = Spender (TxHash (sha256 ("spender" :: ByteString))) 0x0a0b0c0d+          two = SpenderVal (IntMap.fromList [(0x01020304, spender), (2, spender)])+      encode two+        `shouldBe` B.concat+          [ B.pack [2],+            B.pack [0, 0, 0, 2],+            encode (sha256 ("spender" :: ByteString)),+            B.pack [0x0a, 0x0b, 0x0c, 0x0d],+            B.pack [1, 2, 3, 4],+            encode (sha256 ("spender" :: ByteString)),+            B.pack [0x0a, 0x0b, 0x0c, 0x0d]+          ]+      B.length (encode two) `shouldBe` 1 + 2 * 40++  describe "of balances" $ do+    prop "adds the differences to the balance, each number modulo 2^64, from zero if there is none" $+      forAll ((,) <$> oneof [pure Nothing, Just <$> genBalVal] <*> listOf genBalVal) $ \(existing, records) ->+        ioProperty $+          fullOf balanceOperator (encode <$> existing) (map encode records)+            `shouldReturn` Just (encode (foldl addBalVal (fromMaybe def existing) records))+    prop "makes one record of several, which does what they did" $+      forAll ((,) <$> oneof [pure Nothing, Just <$> genBalVal] <*> listOf1 (listOf1 genBalVal)) $+        \(existing, parts) -> ioProperty $ do+          combined <- mapM (partialOf balanceOperator . map encode) parts+          map (fmap B.length) combined `shouldBe` map (const (Just 40)) parts+          combined `shouldBe` map (Just . encode . foldl1 addBalVal) parts+          one <- fullOf balanceOperator (encode <$> existing) (map encode (concat parts))+          fullOf balanceOperator (encode <$> existing) [r | Just r <- combined] `shouldReturn` one+    it "takes a number down by its two's complement" $ do+      let had = BalVal {confirmed = 5000, unconfirmed = 0, utxo = 2, txs = 7, received = 9000}+          spent = (def :: BalVal) {confirmed = negate 3000, utxo = negate 1, txs = 1}+      fullOf balanceOperator (Just (encode had)) [encode spent]+        `shouldReturn` Just (encode BalVal {confirmed = 2000, unconfirmed = 0, utxo = 1, txs = 8, received = 9000})+    it "refuses what is not forty bytes" $ do+      let good = encode (def :: BalVal)+      fullOf balanceOperator (Just good) [good] `shouldReturn` Just good+      fullOf balanceOperator (Just (B.init good)) [good] `shouldReturn` Nothing+      fullOf balanceOperator (Just good) [good, good <> "\0"] `shouldReturn` Nothing+      fullOf balanceOperator Nothing [""] `shouldReturn` Nothing+      fullOf balanceOperator (Just "") [good] `shouldReturn` Nothing+      partialOf balanceOperator [good, B.take 8 good] `shouldReturn` Nothing++  describe "of groups of spenders" $ do+    prop "joins the records with the group, the newest winning for an output, in the order of the outputs" $+      forAll ((,) <$> oneof [pure Nothing, Just <$> genGroup] <*> listOf genGroup) $ \(existing, records) ->+        ioProperty $+          fullOf spenderOperator (group <$> existing) (map group records)+            `shouldReturn` Just (group (joined (fromMaybe IntMap.empty existing) records))+    prop "makes one record of several, which does what they did" $+      forAll ((,) <$> oneof [pure Nothing, Just <$> genGroup] <*> listOf1 (listOf1 genGroup)) $+        \(existing, parts) -> ioProperty $ do+          combined <- mapM (partialOf spenderOperator . map group) parts+          combined `shouldBe` map (Just . group . joined IntMap.empty) parts+          one <- fullOf spenderOperator (group <$> existing) (map group (concat parts))+          fullOf spenderOperator (group <$> existing) [r | Just r <- combined] `shouldReturn` one+    it "refuses a group whose length is not that of its count" $ do+      let good = group (IntMap.fromList [(1, spenderOf 1), (2, spenderOf 2)])+      fullOf spenderOperator (Just good) [good] `shouldReturn` Just good+      fullOf spenderOperator (Just (B.init good)) [good] `shouldReturn` Nothing+      fullOf spenderOperator (Just good) [good <> "\0"] `shouldReturn` Nothing+      fullOf spenderOperator Nothing [""] `shouldReturn` Nothing+      fullOf spenderOperator Nothing [B.pack [3] <> B.drop 1 good] `shouldReturn` Nothing+      partialOf spenderOperator [good, B.take 41 good] `shouldReturn` Nothing+    it "takes a group with no entries for what it is" $+      fullOf spenderOperator (Just "\0") ["\0"] `shouldReturn` Just "\0"++  describe "in a database" $+    modifyMaxSuccess (const 25) $ do+      prop "give the balances that the same writes give when each is read and written whole" $+        forAll (listOf (genStep genBalVal)) $ \steps -> ioProperty . withDB $ \db -> do+          let cf = balanceCF db.db+              wanted = model addBalVal def steps+              agree = forM_ [0 .. keys - 1] $ \k -> do+                let a = address (fromIntegral k)+                Q.getBalance db a `shouldReturn` (valToBalance a <$> Map.lookup k wanted)+                retrieveCF db.db cf (BalKey a) `shouldReturn` Map.lookup k wanted+          forM_ steps $ \case+            Whole k v -> putCF db.db cf (encode (BalKey (address (fromIntegral k)))) (encode v)+            Gone k -> deleteCF db.db cf (encode (BalKey (address (fromIntegral k))))+            Change k v -> mergeCF db.db cf (encode (BalKey (address (fromIntegral k)))) (encode v)+            ToFile -> flush db.db (Just cf)+          agree+          flush db.db (Just cf)+          agree+          compactRange db.db (Just cf) Nothing Nothing+          agree+      prop "give the groups of spenders that the same writes give when each is read and written whole" $+        forAll (listOf (genStep genGroup)) $ \steps -> ioProperty . withDB $ \db -> do+          let cf = spenderCF db.db+              wanted = model (flip IntMap.union) IntMap.empty steps+              -- the groups of one transaction, sixteen outputs apart, and+              -- one of another+              key k = if k == 0 then SpenderKey (made 0) 7 else SpenderKey (made 1) (fromIntegral k)+              agree = do+                forM_ [0 .. keys - 1] $ \k ->+                  retrieveCF db.db cf (key k) `shouldReturn` (SpenderVal <$> Map.lookup k wanted)+                -- and as the queries read them, with a scan of the+                -- transaction+                Q.getChainSpenders db (made 0) `shouldReturn` fromMaybe IntMap.empty (Map.lookup 0 wanted)+                Q.getChainSpenders db (made 1)+                  `shouldReturn` IntMap.unions [m | (k, m) <- Map.toList wanted, k /= 0]+          forM_ steps $ \case+            Whole k v -> putCF db.db cf (encode (key k)) (group v)+            Gone k -> deleteCF db.db cf (encode (key k))+            Change k v -> mergeCF db.db cf (encode (key k)) (group v)+            ToFile -> flush db.db (Just cf)+          agree+          flush db.db (Just cf)+          agree+          compactRange db.db (Just cf) Nothing Nothing+          agree+      it "has the two operators on their column families, and the one that keeps older stores out" $+        withDB $+          \db -> length (columnFamilies db.db) `shouldBe` 11++-------------------------------------------------------------------------------+-- Calling the operators as RocksDB does+-------------------------------------------------------------------------------++foreign import ccall "dynamic" callFull :: FunPtr FullMerge -> FullMerge++foreign import ccall "dynamic" callPartial :: FunPtr PartialMerge -> PartialMerge++-- | The full merge of an operator: the value that a key has, if any, and+-- its merge records, oldest first. 'Nothing' if the function says that it+-- failed, which it must say as RocksDB's C API reads it: with no bytes.+fullOf :: MergeOperator -> Maybe ByteString -> [ByteString] -> IO (Maybe ByteString)+fullOf op existing records = case op of+  MergeFunctions {fullMerge = function} ->+    operands records $ \pointers lengths count success size -> do+      let call (value, n) =+            callFull function nullPtr nullPtr 0 value (fromIntegral n) pointers lengths count success size+      maybe (call (nullPtr, 0 :: Int)) (`B.useAsCStringLen` call) existing+  UInt64Add -> fail "not an operator of the store"++-- | The partial merge of an operator: merge records, oldest first.+partialOf :: MergeOperator -> [ByteString] -> IO (Maybe ByteString)+partialOf op records = case op of+  MergeFunctions {partialMerge = function} ->+    operands records $ \pointers lengths count success size ->+      callPartial function nullPtr nullPtr 0 pointers lengths count success size+  UInt64Add -> fail "not an operator of the store"++-- | Hand merge records to a function as RocksDB's C API does, and read+-- what it returns: memory from malloc, a length, and whether it worked.+operands ::+  [ByteString] ->+  (Ptr CString -> Ptr CSize -> CInt -> Ptr CUChar -> Ptr CSize -> IO CString) ->+  IO (Maybe ByteString)+operands records call =+  withMany B.useAsCStringLen records $ \rows ->+    withArray (map fst rows) $ \pointers ->+      withArray (map (fromIntegral . snd) rows :: [CSize]) $ \lengths ->+        alloca $ \success ->+          alloca $ \size -> do+            bytes <- call pointers lengths (fromIntegral (length rows)) success size+            ok <- peek success+            n <- peek size+            if ok == 0+              then do+                -- RocksDB copies this many bytes before it looks at the flag+                (bytes, n) `shouldBe` (nullPtr, 0)+                return Nothing+              else do+                result <- B.packCStringLen (bytes, fromIntegral n)+                free bytes+                return (Just result)++-------------------------------------------------------------------------------+-- What they are meant to do+-------------------------------------------------------------------------------++-- | A group with these records joined to it, oldest first: a newer entry+-- takes the place of an older one for the same output.+joined :: IntMap Spender -> [IntMap Spender] -> IntMap Spender+joined = foldl (flip IntMap.union)++-- | The bytes of a group of spenders.+group :: IntMap Spender -> ByteString+group = encode . SpenderVal++-- | A write to one of a few keys: a whole value, a delete, a merge record,+-- or a flush of the memtable to a table file.+data Step v+  = Whole Int v+  | Gone Int+  | Change Int v+  | ToFile+  deriving (Show)++-- | The keys that the steps write to.+keys :: Int+keys = 4++genStep :: Gen v -> Gen (Step v)+genStep value =+  frequency+    [ (2, Whole <$> key <*> value),+      (1, Gone <$> key),+      (6, Change <$> key <*> value),+      (2, pure ToFile)+    ]+  where+    key = choose (0, keys - 1)++-- | What each key holds after the steps, given how a merge record changes+-- a value and the value that a key without one counts from.+model :: (v -> v -> v) -> v -> [Step v] -> Map.Map Int v+model merge zero = foldl step Map.empty+  where+    step m (Whole k v) = Map.insert k v m+    step m (Gone k) = Map.delete k m+    step m (Change k v) = Map.insert k (merge (Map.findWithDefault zero k m) v) m+    step m ToFile = m++-- | Numbers of any size, so that sums wrap around, and small ones.+genBalVal :: Gen BalVal+genBalVal = BalVal <$> number <*> number <*> number <*> number <*> number+  where+    number = oneof [arbitrary, fromIntegral <$> choose (0, 3 :: Int), negate . fromIntegral <$> choose (1, 3 :: Int)]++-- | A few spenders, of outputs that other groups often have too.+genGroup :: Gen (IntMap Spender)+genGroup = do+  n <- choose (0, 5)+  IntMap.fromList <$> vectorOf n ((,) <$> output <*> (spenderOf <$> arbitrary))+  where+    -- most in the sixteen outputs of a group, and some whose number takes+    -- all four bytes+    output = frequency [(8, choose (0, 15)), (1, elements [255, 256, 65536, 16777216, 2000000000])]++spenderOf :: Word32 -> Spender+spenderOf n = Spender (TxHash (sha256 (encode n))) (n `mod` 7)++made :: Word64 -> TxHash+made n = TxHash (sha256 (encode n))++address :: Word32 -> Address+address i =+  fromMaybe (error "no address") . eitherToMaybe . scriptToAddressBS btcRegTest $+    "\x76\xa9\x14" <> B.take 20 (encode (sha256 (encode i))) <> "\x88\xac"++withDB :: (DatabaseReader -> IO a) -> IO a+withDB action = withSystemTempDirectory "haskoin-store-merge-" $ \dir -> openLayered btcRegTest dir action
test/Haskoin/Store/Database/MigrateSpec.hs view
@@ -1,491 +1,99 @@-{-# LANGUAGE DeriveAnyClass #-}- module Haskoin.Store.Database.MigrateSpec (spec) where -import Control.Monad (foldM, forM_, void, when)+import Control.Exception (Exception (..))+import Control.Monad (forM, forM_, replicateM_, void, when) import Data.ByteString qualified as B-import Data.Either (isRight, rights)-import Data.IORef (modifyIORef', newIORef, readIORef)-import Data.IntMap.Strict qualified as IntMap-import Data.List (sort)-import Data.Map.Strict qualified as Map-import Data.Maybe (catMaybes, listToMaybe)-import Data.Serialize (decode, encode)-import Data.Text qualified as T-import Data.Word (Word16, Word32, Word64)-import Database.RocksDB (BatchOp (..))+import Data.List (isInfixOf)+import Data.Serialize (encode)+import Data.Word (Word32)+import Database.RocksDB (columnFamilies, put) import Database.RocksDB.Query (insert, insertOp, retrieve, writeBatch)-import Haskoin-  ( BlockHeader (..),-    Ctx,-    Network,-    OutPoint (..),-    Tx (..),-    TxHash (..),-    TxIn (..),-    TxOut (..),-    btcRegTest,-    headerHash,-    prepareContext,-    scriptToAddressBS,-    sha256,-    txHash,-  )-import Haskoin.Store.Data (Balance (..), BlockData (..), BlockRef (..), Spender (..), TxData (..))-import Haskoin.Store.Database.Mempool qualified as M+import Haskoin (TxHash, btcRegTest)+import Haskoin.Store.Data (UnixTime) import Haskoin.Store.Database.Migrate-import Haskoin.Store.Database.Query qualified as Q-import Haskoin.Store.Database.Query.Layered qualified as L-import Haskoin.Store.Database.Reader-  ( DatabaseReader (..),-    mempoolAddrCF,-    mempoolCF,-    mempoolInsertOps,-    spenderCF,-    txCF,-  )-import Haskoin.Store.Database.Types (MigrationKey (..), SpenderKey (..), SpenderVal (..), VersionKey (..))+import Haskoin.Store.Database.Reader (DatabaseReader (..))+import Haskoin.Store.Database.Types (MigrationKey (..), VersionKey (..)) import Haskoin.Store.Histories-import Haskoin.Store.TestBlocks (script)+import Haskoin.Store.Integration.Temp (withSystemTempDirectory)+import System.IO.Error (ioeGetErrorString) import Test.Hspec-import Test.Hspec.QuickCheck (modifyMaxSuccess)-import Test.QuickCheck (forAll, ioProperty, property, tabulate, vectorOf)-import UnliftIO (Exception, throwIO, try, withSystemTempDirectory)  spec :: Spec-spec = prepareContext $ \ctx -> describe "migration to the layered layout" $ do-  it "takes the mempool out of the chain's indices, keeping every answer" $-    withSystemTempDirectory "haskoin-store-migrate-" $ \dir -> do-      -- A coin of a block; a payment from it in the mempool, with change;-      -- and a transaction that spends that change, also in the mempool.-      (hist, old) <- mixedDatabase ctx dir $ \db hist0 -> do-        let payment = Tx 2 [TxIn (coinOf hist0) "" maxBound] [TxOut 3000 (p2pkh 1), TxOut 2000 script] [] 0-            child = Tx 2 [TxIn (OutPoint (txHash payment) 1) "" maxBound] [TxOut 1500 (p2pkh 2)] [] 0-        foldM (addTx db) hist0 [payment, child]-      (miner, one, two) <- case rights (map (scriptToAddressBS net ctx) [script, p2pkh 1, p2pkh 2]) of-        [a, b, c] -> return (a, b, c)-        _ -> fail "a script has no address"-      let numbers bs a = fmap (\b -> (b.confirmed, b.unconfirmed, b.utxo, b.txs, b.received)) (lookup a bs)-      -- the coin is spent in the mempool: it no longer counts as confirmed-      numbers old.balances miner `shouldBe` Just (0, 0, 0, 3, 5000002000)-      numbers old.balances one `shouldBe` Just (0, 3000, 1, 1, 3000)-      numbers old.balances two `shouldBe` Just (0, 1500, 1, 1, 1500)-      openLayered net ctx dir $ \db -> do-        migrateLayered db `shouldReturn` Migration {moved = 2, batches = 1, checked = 2, rewritten = 0}-        retrieve db.db VersionKey `shouldReturn` Just layeredVersion-        (retrieve db.db MigrationKey :: IO (Maybe Word32)) `shouldReturn` Nothing-        -- the chain has the coin again, whole and unspent-        chainBalance <- fmap (\b -> (b.confirmed, b.unconfirmed, b.utxo, b.txs, b.received)) <$> Q.getBalance db miner-        chainBalance `shouldBe` Just (5000000000, 0, 1, 1, 5000000000)-        Q.getBalance db one `shouldReturn` Nothing-        Q.getBalance db two `shouldReturn` Nothing-        -- and with the mempool on top, every answer is what it was-        -- the queries read a database of the version that the store runs on-        settle db-        layered db (ask hist) `shouldReturn` old-        -- a second run finds nothing to do-        migrateLayered db `shouldReturn` Migration {moved = 0, batches = 0, checked = 0, rewritten = 0}--  modifyMaxSuccess (const 50) $-    it "keeps every answer and leaves the chain as if the mempool had been wiped, for random histories" $-      property . forAll ((,) <$> vectorOf 45 anyNumber <*> vectorOf 12 anyNumber) $ \(seeds, last') -> ioProperty $ do-        withSystemTempDirectory "haskoin-store-migrate-" $ \dir -> do-          -- anything at first, then payments, so that the mempool has something-          (hist, old) <- mixedDatabase ctx dir $ \db hist0 -> foldM (step db) hist0 (seeds <> onlyPayments last')-          withSystemTempDirectory "haskoin-store-chain-" $ \dir' ->-            open20 net ctx dir' $ \fresh -> do-              -- the same blocks, in a database that never saw the mempool-              void (run fresh mixedRules.initBest) :: IO ()-              forM_ (reverse hist.chain) $ \(block, node) ->-                run fresh (\w -> mixedRules.importBlock w block node) >>= either (fail . show) (\_ -> return ())-              openLayered net ctx dir $ \db -> do-                done <- migrateLayeredWith defaultMigrationOptions {chunk = 3} db-                done.checked `shouldBe` length old.mempool-                done.moved `shouldBe` length old.mempool-                -- every answer is what it was-                -- the queries read a database of the version that the store runs on-                settle db-                layered db (ask hist) `shouldReturn` old-                -- the chain's column families are those of the fresh database,-                -- but for the times at which transactions were first seen-                migrated <- chainOf db-                expected <- chainOf fresh-                length expected.addressTxs `shouldSatisfy` (>= length hist.chain)-                migrated {firstSeen = []} `shouldBe` expected {firstSeen = []}-                -- the mempool area is the keys of its transactions, each worked-                -- out from that transaction alone: no more and no less-                moved <- catMaybes <$> mapM (M.getMempoolTx db . snd) old.mempool-                let records = [M.mempoolRecords net ctx time td | td <- moved, MemRef time <- [td.block]]-                length records `shouldBe` length old.mempool-                dump db (mempoolCF db.db) `shouldReturn` sort (concatMap (fst . M.recordEntries) records)-                dump db (mempoolAddrCF db.db) `shouldReturn` sort (M.addressArea records)-                -- without the mempool area, the answers are those of the chain-                void (M.wipeMempoolArea db)-                wiped <- layered db (ask hist)-                alone <- mixed fresh (ask hist)-                (wiped.balances, wiped.addressTxs, wiped.addressUnspents, wiped.unspents, wiped.mempool)-                  `shouldBe` (alone.balances, alone.addressTxs, alone.addressUnspents, alone.unspents, [])-          let inMempool = [td | (_, Just td) <- old.txs, not td.deleted, MemRef {} <- [td.block]]-              unconfirmed = map (txHash . (.tx)) inMempool-              chained = [() | td <- inMempool, i <- td.tx.inputs, i.outpoint.hash `elem` unconfirmed]-              bucket n-                | n == 0 = "none"-                | n <= 3 = "1 to 3"-                | n <= 9 = "4 to 9"-                | otherwise = "10 or more" :: String-          return-            . tabulate "mempool transactions" [bucket (length inMempool)]-            . tabulate "of them spending another" [bucket (length chained)]-            . tabulate "deleted transactions" [bucket (length [() | (_, Just td) <- old.txs, td.deleted])]-            $ tabulate "blocks" [bucket (length hist.chain)] True--  it "carries on after being interrupted, between any two writes" $-    forM_ [1 .. 6] $ \stopAt ->-      withSystemTempDirectory "haskoin-store-migrate-" $ \dir -> do-        (hist, old) <- mixedDatabase ctx dir $ \db hist0 -> foldM (addTx db) hist0 (payments (coinOf hist0))-        length old.mempool `shouldBe` 8-        -- one transaction in each write, and the plug pulled after a few-        let pull n = when (n == stopAt) (throwIO Interrupted)-            interrupted = defaultMigrationOptions {chunk = 1, afterBatch = pull}-        openLayered net ctx dir $ \db -> do-          try (migrateLayeredWith interrupted db) `shouldReturn` Left Interrupted-          -- a store of version 20 would refuse the database from here on-          retrieve db.db VersionKey `shouldReturn` Just layeredVersion-          (retrieve db.db MigrationKey :: IO (Maybe Word32)) `shouldReturn` Just 20-        openLayered net ctx dir $ \db -> do-          done <- migrateLayered db-          done.moved `shouldBe` length old.mempool - stopAt-          done.checked `shouldBe` length old.mempool-          (retrieve db.db MigrationKey :: IO (Maybe Word32)) `shouldReturn` Nothing-          -- the queries read a database of the version that the store runs on-          settle db-          layered db (ask hist) `shouldReturn` old--  it "refuses a database that it cannot handle, and leaves it untouched" $ do+spec = describe "the version of the database" $ do+  it "is the current one for a new database, and nothing is migrated" $     withSystemTempDirectory "haskoin-store-migrate-" $ \dir ->-      openLayered net ctx dir $ \db -> do-        migrateLayered db `shouldThrow` (== UnknownVersion Nothing)-        insert db.db VersionKey (19 :: Word32)-        migrateLayered db `shouldThrow` (== UnknownVersion (Just 19))-    withSystemTempDirectory "haskoin-store-migrate-" $ \dir -> do-      -- the mempool lists a transaction that has no body-      let lost = TxHash (sha256 ("lost" :: B.ByteString))-      _ <- mixedDatabase ctx dir $ \db hist0 -> do-        hist1 <- foldM (addTx db) hist0 (take 5 (payments (coinOf hist0)))-        writeBatch db.db (mempoolInsertOps db.db lost 7)-        return hist1-      openLayered net ctx dir $ \db -> do-        migrateLayered db `shouldThrow` \case-          Inconsistent _ -> True-          _ -> False-        retrieve db.db VersionKey `shouldReturn` Just (20 :: Word32)-        dump db (mempoolAddrCF db.db) `shouldReturn` []-        -- the mempool column family has its list and its index by time only-        written <- dump db (mempoolCF db.db)-        written `shouldSatisfy` all ((`elem` ["\x08", "\x0b"]) . B.take 1 . fst)+      openLayered net dir $ \db -> do+        upgradeDatabase (\_ -> return ()) db `shouldReturn` nothing+        retrieve db.db VersionKey `shouldReturn` Just currentVersion+        -- a second run finds nothing to do, and says nothing+        upgradeDatabase (\_ -> fail "nothing to say") db `shouldReturn` nothing -  it "brings a database of any version that the store knows to the layout it runs on" $ do-    -- a new database-    withSystemTempDirectory "haskoin-store-migrate-" $ \dir ->-      openLayered net ctx dir $ \db -> do-        upgradeDatabase (\_ -> return ()) db-          `shouldReturn` Migration {moved = 0, batches = 0, checked = 0, rewritten = 0}+  it "refuses a version that the store does not know, untouched" $+    forM_ [16, 17 :: Word32] $ \v ->+      withSystemTempDirectory "haskoin-store-migrate-" $ \dir ->+        openLayered net dir $ \db -> do+          insert db.db VersionKey v+          upgradeDatabase (\_ -> return ()) db `shouldThrow` (== UnknownVersion (Just v))+          retrieve db.db VersionKey `shouldReturn` Just v++  it "refuses the database of a newer store, and one whose version does not fit its marker, untouched" $+    withSystemTempDirectory "haskoin-store-merges-" $ \dir ->+      openLayered net dir $ \db -> do+        void (upgradeDatabase (\_ -> return ()) db)+        retrieve db.db VersionKey `shouldReturn` Just (23 :: Word32)+        insert db.db VersionKey (24 :: Word32)+        upgradeDatabase (\_ -> return ()) db `shouldThrow` (== UnknownVersion (Just 24))+        retrieve db.db VersionKey `shouldReturn` Just (24 :: Word32)+        displayException (UnknownVersion (Just 24)) `shouldSatisfy` ("newer haskoin-store" `isInfixOf`)+        -- no migration leaves a marker with the current version+        writeBatch db.db [insertOp VersionKey currentVersion, insertOp MigrationKey layeredVersion]+        upgradeDatabase (\_ -> return ()) db `shouldThrow` (== UnknownVersion (Just currentVersion))+        migrateRecords (\_ -> return ()) db `shouldThrow` (== UnknownVersion (Just currentVersion))         retrieve db.db VersionKey `shouldReturn` Just currentVersion-    -- one of version 20 with a mempool, and then the same one again-    withSystemTempDirectory "haskoin-store-migrate-" $ \dir -> do-      (hist, old) <- mixedDatabase ctx dir $ \db hist0 ->-        foldM (addTx db) hist0 (take 5 (payments (coinOf hist0)))-      forM_ [5, 0] $ \moved ->-        openLayered net ctx dir $ \db -> do++  it "takes a database of version 18 or 19 to the current one, reading its mempool where it is" $+    forM_ [18, 19 :: Word32] $ \v ->+      withSystemTempDirectory "haskoin-store-migrate-" $ \dir -> do+        openOlder net dir $ \db -> do+          insert db.db VersionKey v+          -- version 18 listed its mempool under one key, here empty+          when (v == 18) $ put db.db (B.singleton 0x07) (encode ([] :: [(UnixTime, TxHash)]))+        openLayered net dir $ \db -> do           done <- upgradeDatabase (\_ -> return ()) db-          done.moved `shouldBe` moved+          done.moved `shouldBe` 0           retrieve db.db VersionKey `shouldReturn` Just currentVersion           (retrieve db.db MigrationKey :: IO (Maybe Word32)) `shouldReturn` Nothing-          layered db (ask hist) `shouldReturn` old-          M.wrongSummaries db `shouldReturn` []-    -- one of version 19, which had no index of the mempool by time-    withSystemTempDirectory "haskoin-store-migrate-" $ \dir -> do-      (hist, old) <- mixedDatabase ctx dir $ \db hist0 ->-        foldM (addTx db) hist0 (take 3 (payments (coinOf hist0)))-      openLayered net ctx dir $ \db -> do-        index <- dump db (mempoolCF db.db)-        writeBatch db.db $-          insertOp VersionKey (19 :: Word32)-            : [DelCF (mempoolCF db.db) k | (k, _) <- index, B.take 1 k == "\x0b"]-        done <- upgradeDatabase (\_ -> return ()) db-        done.moved `shouldBe` 3-        layered db (ask hist) `shouldReturn` old-    -- and one of a version that it does not know-    withSystemTempDirectory "haskoin-store-migrate-" $ \dir ->-      openLayered net ctx dir $ \db -> do-        insert db.db VersionKey (17 :: Word32)-        upgradeDatabase (\_ -> return ()) db `shouldThrow` (== UnknownVersion (Just 17))--  describe "the time of a confirmed transaction" $ do-    it "is the time of its block, not of when the store first saw it" $-      withSystemTempDirectory "haskoin-store-times-" $ \dir ->-        openLayered net ctx dir $ \db -> do-          hist <- start layeredRules db-          let tx = Tx 2 [TxIn (coinOf hist) "" maxBound] [TxOut 4000 (p2pkh 1)] [] 0-              stored = do-                rows <- dump db (txCF db.db)-                return [td :: TxData | (_, v) <- rows, Right td <- [decode v], txHash td.tx == txHash tx]-          (.result) <$> apply layeredRules db (Pay tx 1000) `shouldReturn` Right "True"-          -- in the mempool it has the time at which it came-          fmap (.timestamp) <$> M.getMempoolTx db (txHash tx) `shouldReturn` Just 1000-          mined <- blockOn db [tx] >>= apply layeredRules db-          isRight mined.result `shouldBe` True-          best <- Q.getBestBlock db >>= maybe (fail "no best block") (Q.getBlock db)-          time <- maybe (fail "no block") (return . fromIntegral . (.header.timestamp)) best-          time `shouldNotBe` (1000 :: Word64)-          map (\td -> (td.block, td.timestamp)) <$> stored `shouldReturn` [(BlockRef 2 1, time)]-          wrongTimes db `shouldReturn` []-          -- and when the block is reverted it goes back to the mempool-          -- with the time of the revert, which the timeout of the mempool-          -- counts from-          reverted <- apply layeredRules db (Revert (headerHash (maybe (error "no block") (.header) best)) 7000)-          isRight reverted.result `shouldBe` True-          fmap (.timestamp) <$> M.getMempoolTx db (txHash tx) `shouldReturn` Just 7000-    modifyMaxSuccess (const 20) $-      it "is the time of its block, after any history" $-        -- whatever a store saw in its mempool before, and through blocks-        -- that are reverted and replaced-        property . forAll (vectorOf 60 anyNumber) $ \seeds -> ioProperty $-          withSystemTempDirectory "haskoin-store-times-" $ \dir ->-            openLayered net ctx dir $ \db -> withShadow net ctx $ \shadow -> do-              hist0 <- start layeredRules db-              void (foldM (walk shadow db) hist0 seeds)-              late <- wrongTimes db-              late `shouldBe` []-    modifyMaxSuccess (const 20) $-      it "is set by the migration to version 22, and nothing else is changed" $-        -- A database as the layered rules write it has the time of its-        -- block for every confirmed transaction. Some of them are given-        -- another time, as a store of version 21 that saw them in its-        -- mempool had, and the database is marked as of that version. The-        -- migration must give back the database as it was, byte for byte.-        property . forAll (vectorOf 40 anyNumber) $ \seeds -> ioProperty $-          withSystemTempDirectory "haskoin-store-times-" $ \dir ->-            openLayered net ctx dir $ \db -> withShadow net ctx $ \shadow -> do-              hist0 <- start layeredRules db-              void (foldM (walk shadow db) hist0 seeds)-              original <- dump db (txCF db.db)-              let confirmed = [(k, td :: TxData) | (k, v) <- original, Right td <- [decode v], inChain td]-                  -- every other one, and all of them when there are few-                  picked = [x | (i, x) <- zip [0 :: Int ..] confirmed, even i || length confirmed < 4]-              writeBatch db.db $-                insertOp VersionKey layeredVersion-                  : [PutCF (txCF db.db) k (encode (later td)) | (k, td) <- picked]-              wrongTimes db >>= (`shouldBe` length picked) . length-              done <- upgradeDatabase (\_ -> return ()) db-              done.rewritten `shouldBe` length picked-              retrieve db.db VersionKey `shouldReturn` Just currentVersion-              (retrieve db.db MigrationKey :: IO (Maybe Word32)) `shouldReturn` Nothing-              dump db (txCF db.db) `shouldReturn` original-              -- and again, it finds nothing to do-              upgradeDatabase (\_ -> return ()) db-                `shouldReturn` Migration {moved = 0, batches = 0, checked = 0, rewritten = 0}-    modifyMaxSuccess (const 20) $-      it "is in a record that is written once, and not when its outputs are spent" $-        -- After every step of a history, the record of each transaction-        -- that was confirmed before the step and still is, is the same-        -- bytes as before: blocks that spend its outputs, and blocks that-        -- are reverted, write keys of their own and leave it alone. No-        -- record has spenders in it.-        property . forAll (vectorOf 60 anyNumber) $ \seeds -> ioProperty $-          withSystemTempDirectory "haskoin-store-times-" $ \dir ->-            openLayered net ctx dir $ \db -> withShadow net ctx $ \shadow -> do-              hist0 <- start layeredRules db-              let records = do-                    rows <- dump db (txCF db.db)-                    return (Map.fromList [(k, v) | (k, v) <- rows, Right td <- [decode v], inChain td])-                  turn (hist, was) seed = do-                    hist' <- walk shadow db hist seed-                    now <- records-                    Map.filter id (Map.intersectionWith (/=) was now) `shouldBe` Map.empty-                    return (hist', now)-              first <- records-              (_, final) <- foldM turn (hist0, first) seeds-              [td.spenders | v <- Map.elems final, Right (td :: TxData) <- [decode v], not (IntMap.null td.spenders)]-                `shouldBe` []-    it "has the spenders of its outputs under a key for each sixteen of them" $-      withSystemTempDirectory "haskoin-store-times-" $ \dir ->-        openLayered net ctx dir $ \db -> do-          hist <- start layeredRules db-          -- forty outputs, of which three are spent, in three groups-          let wide = Tx 2 [TxIn (coinOf hist) "" maxBound] (replicate 40 (TxOut 1000 (p2pkh 1))) [] 0-              spend i = Tx 2 [TxIn (OutPoint (txHash wide) i) "" maxBound] [TxOut 500 (p2pkh 2)] [] 0-              spends = map spend [3, 17, 39]-              keysNow = do-                rows <- dump db (spenderCF db.db)-                return-                  [ (g, IntMap.keys m)-                  | (k, v) <- rows,-                    Right (SpenderKey h g) <- [decode k],-                    h == txHash wide,-                    Right (SpenderVal m) <- [decode v]-                  ]-              mine txs = do-                mined <- blockOn db txs >>= apply layeredRules db-                isRight mined.result `shouldBe` True-          mine [wide]-          written <- dump db (txCF db.db)-          keysNow `shouldReturn` []-          mine spends-          keysNow `shouldReturn` [(0, [3]), (1, [17]), (2, [39])]-          -- the record is as the first block wrote it-          now <- dump db (txCF db.db)-          [v | (k, v) <- now, k `elem` map fst written] `shouldBe` map snd written-          -- and the transaction is read with its spenders, each also by itself-          found <- L.withView db (`L.getTxData` txHash wide)-          let expected = IntMap.fromList [(fromIntegral i, Spender (txHash (spend i)) 0) | i <- [3, 17, 39]]-          fmap (.spenders) found `shouldBe` Just expected-          forM_ [3, 4, 17, 39] $ \i ->-            L.withView db (`L.getSpender` OutPoint (txHash wide) i)-              `shouldReturn` IntMap.lookup (fromIntegral i) expected-          -- a block that is reverted takes its spenders along, and a group-          -- with none left has no key-          best <- Q.getBestBlock db >>= maybe (fail "no best block") return-          reverted <- apply layeredRules db (Revert best 0)-          isRight reverted.result `shouldBe` True-          keysNow `shouldReturn` []-          Q.getChainSpenders db (txHash wide) `shouldReturn` IntMap.empty-          -- the transactions that spent them are back in the mempool, and-          -- the transaction is read with them as its spenders still-          fmap (.spenders) <$> L.withView db (`L.getTxData` txHash wide) `shouldReturn` Just expected-    modifyMaxSuccess (const 20) $-      it "has the spenders of its outputs moved to keys of their own by the migration" $-        -- A database as the layered rules write it has a key for each-        -- spent output. The spenders are put where version 21 had them,-        -- in the record of the transaction that made the output, the keys-        -- are deleted, and the database is marked as of that version. The-        -- migration must give back the records and the keys as they were.-        property . forAll (vectorOf 40 anyNumber) $ \seeds -> ioProperty $-          withSystemTempDirectory "haskoin-store-times-" $ \dir ->-            openLayered net ctx dir $ \db -> withShadow net ctx $ \shadow -> do-              hist0 <- start layeredRules db-              void (foldM (walk shadow db) hist0 seeds)-              records <- dump db (txCF db.db)-              keys <- dump db (spenderCF db.db)-              let spent =-                    [ (h, m :: IntMap.IntMap Spender)-                    | (k, v) <- keys,-                      Right (SpenderKey h _) <- [decode k],-                      Right (SpenderVal m) <- [decode v]-                    ]-                  inside td = IntMap.unions [m | (h, m) <- spent, h == txHash td.tx]-                  old (k, v) = case decode v of-                    Right td | inChain td -> (k, encode (embed td (inside td)))-                    _ -> (k, v)-              writeBatch db.db $-                insertOp VersionKey layeredVersion-                  : [DelCF (spenderCF db.db) k | (k, _) <- keys]-                    <> [PutCF (txCF db.db) k v | (k, v) <- map old records]-              done <- upgradeDatabase (\_ -> return ()) db-              -- every record that had a spender in it was written again-              done.rewritten `shouldBe` length [() | (k, v) <- records, old (k, v) /= (k, v)]-              dump db (txCF db.db) `shouldReturn` records-              dump db (spenderCF db.db) `shouldReturn` keys-    it "is set by a migration that carries on after it was interrupted" $-      withSystemTempDirectory "haskoin-store-times-" $ \dir ->-        openLayered net ctx dir $ \db -> withShadow net ctx $ \shadow -> do-          hist0 <- start layeredRules db-          -- payments, and then blocks, which confirm some of them-          void (foldM (walk shadow db) hist0 (onlyPayments [1 .. 6] <> [8 .. 11]))-          original <- dump db (txCF db.db)-          let confirmed = [(k, td :: TxData) | (k, v) <- original, Right td <- [decode v], inChain td]-          length confirmed `shouldSatisfy` (> 0)-          -- as a migration leaves it that stopped before its first change-          writeBatch db.db $-            [insertOp VersionKey currentVersion, insertOp MigrationKey layeredVersion]-              <> [PutCF (txCF db.db) k (encode ((td :: TxData) {timestamp = 7})) | (k, td) <- confirmed]-          said <- newIORef []-          done <- upgradeDatabase (\line -> modifyIORef' said (line :)) db-          done.rewritten `shouldBe` length confirmed-          dump db (txCF db.db) `shouldReturn` original-          (retrieve db.db MigrationKey :: IO (Maybe Word32)) `shouldReturn` Nothing-          readIORef said >>= (`shouldSatisfy` any ("Carrying on" `T.isPrefixOf`))-    it "is not guessed for a transaction whose block the database does not have" $-      withSystemTempDirectory "haskoin-store-times-" $ \dir ->-        openLayered net ctx dir $ \db -> do-          _ <- start layeredRules db-          rows <- dump db (txCF db.db)-          let confirmed = [(k, td :: TxData) | (k, v) <- rows, Right td <- [decode v], inChain td]-          (k, td) <- maybe (fail "no confirmed transaction") return (listToMaybe confirmed)-          writeBatch-            db.db-            [ insertOp VersionKey layeredVersion,-              PutCF (txCF db.db) k (encode ((td :: TxData) {block = BlockRef 5000 0}))-            ]-          upgradeDatabase (\_ -> return ()) db `shouldThrow` \case-            Inconsistent _ -> True-            _ -> False--net :: Network-net = btcRegTest---- | Choose one operation of a history from a database of version 20 that--- has gone through it ('withShadow'), and carry it out there and, under--- the layered rules, in a database of the layered layout.-walk :: DatabaseReader -> DatabaseReader -> History -> Word16 -> IO History-walk shadow db hist seed =-  plan shadow hist seed >>= \case-    (hist', Nothing) -> return hist'-    (hist', Just op) -> do-      void (apply mixedRules shadow op)-      record db hist' op <$> apply layeredRules db op---- | A transaction with a time that is not that of its block, and marked as--- one that can be replaced, which a confirmed transaction cannot be.-later :: TxData -> TxData-later td = (td :: TxData) {timestamp = td.timestamp + 12345, rbf = True}---- | Bring a database of version 21 to the version that the store runs on.-settle :: DatabaseReader -> IO ()-settle db = void (migrateRecords (\_ -> return ()) db)---- | A transaction with spenders of its outputs in its record.-embed :: TxData -> IntMap.IntMap Spender -> TxData-embed td inside = (td :: TxData) {spenders = inside}---- | Whether a transaction is in the chain.-inChain :: TxData -> Bool-inChain td = case td.block of-  BlockRef {} -> not td.deleted-  MemRef {} -> False---- | The confirmed transactions of a database whose time is not that of--- their block, each with both times.-wrongTimes :: DatabaseReader -> IO [(TxHash, Word64, Maybe Word64)]-wrongTimes db = do-  rows <- dump db (txCF db.db)-  fmap catMaybes . mapM check $ [td :: TxData | (_, v) <- rows, Right td <- [decode v], inChain td]-  where-    check td = do-      hashes <- Q.getBlocksAtHeight db td.block.height-      blocks <- catMaybes <$> mapM (Q.getBlock db) hashes-      let time = listToMaybe [fromIntegral b.header.timestamp | b <- blocks, b.main]-      return (if time == Just td.timestamp then Nothing else Just (txHash td.tx, td.timestamp, time))--data Interrupted = Interrupted-  deriving (Eq, Show, Exception)+          -- and a second run finds nothing to do+          upgradeDatabase (\_ -> return ()) db `shouldReturn` nothing --- | Make a database of version 20 in a directory: the genesis block, a block--- with a coin, and whatever the function does next. Gives the history, and--- the answers of the database to every query about it.-mixedDatabase ::-  Ctx ->-  FilePath ->-  (DatabaseReader -> History -> IO History) ->-  IO (History, Answers)-mixedDatabase ctx dir go =-  open20 net ctx dir $ \db -> do-    hist <- start mixedRules db >>= go db-    answers <- mixed db (ask hist)-    return (hist, answers)+  it "takes a database of version 22 with one write" $+    withSystemTempDirectory "haskoin-store-merges-" $ \dir -> do+      openOlder net dir $ \db -> do+        insert db.db VersionKey recordsVersion+        length (columnFamilies db.db) `shouldBe` 10+      openLayered net dir $ \db -> do+        upgradeDatabase (\_ -> return ()) db `shouldReturn` nothing+        retrieve db.db VersionKey `shouldReturn` Just currentVersion+        (retrieve db.db MigrationKey :: IO (Maybe Word32)) `shouldReturn` Nothing+        length (columnFamilies db.db) `shouldBe` 11 --- | Add a transaction to the mempool, by the rules of today's layout.-addTx :: DatabaseReader -> History -> Tx -> IO History-addTx db hist tx = record db hist op <$> apply mixedRules db op+  it "cannot be opened by a store that does not know the merge operators, which leaves its merge records be" $+    withSystemTempDirectory "haskoin-store-merges-" $ \dir -> do+      -- blocks with merge records, which are in the write-ahead log and+      -- in no table file when the database is closed+      written <- openLayered net dir $ \db -> do+        void (upgradeDatabase (\_ -> return ()) db)+        startChain db+        replicateM_ 5 (mine db [])+        contents db+      length [() | cf <- written, not (null cf)] `shouldSatisfy` (> 5)+      -- RocksDB would open the database without the merge operators,+      -- and drop those merge records: it does not get that far, for+      -- want of the column family that such a store does not know+      openOlder net dir (\_ -> return ()) `shouldThrow` \e ->+        "merge-operators" `isInfixOf` ioeGetErrorString e+      openLayered net dir contents `shouldReturn` written   where-    op = Pay tx hist.clock+    net = btcRegTest+    nothing = Migration {moved = 0, batches = 0, checked = 0, rewritten = 0}+    contents db = forM (columnFamilies db.db) (dump db)
− test/Haskoin/Store/Database/Query/LayeredSpec.hs
@@ -1,173 +0,0 @@-module Haskoin.Store.Database.Query.LayeredSpec (spec) where--import Control.Monad (foldM, forM_, void)-import Data.ByteString (ByteString)-import Data.Default (def)-import Data.List (sort)-import Haskoin-  ( Address,-    OutPoint (..),-    Tx (..),-    TxHash (..),-    TxIn (..),-    TxOut (..),-    bchRegTest,-    btcRegTest,-    prepareContext,-    scriptToAddressBS,-    sha256,-    txHash,-  )-import Haskoin.Store.Common (Limits (..), Start (..), StoreRead (..))-import Haskoin.Store.Data-  ( Balance (..),-    TxData (..),-    TxRef (..),-    Unspent (..),-    confirmed,-  )-import Haskoin.Store.Database.Migrate (upgradeDatabase)-import Haskoin.Store.Database.Query.Layered qualified as L-import Haskoin.Store.Database.Reader (DatabaseReader (..))-import Haskoin.Store.Histories-import Haskoin.Store.TestBlocks (coinbaseTx, script)-import Test.Hspec-import Test.Hspec.QuickCheck (modifyMaxSuccess)-import Test.QuickCheck (arbitrary, forAll, ioProperty, property, tabulate, vectorOf)-import Test.QuickCheck.Gen (unGen)-import Test.QuickCheck.Random (mkQCGen)-import UnliftIO (withSystemTempDirectory)--spec :: Spec-spec = prepareContext $ \ctx -> describe "queries of the layered layout" $ do-  forM_ [("Bitcoin", btcRegTest), ("Bitcoin Cash", bchRegTest)] $ \(name, net) ->-    modifyMaxSuccess (const 30) $-      it ("give the pages of today's queries on a migrated database, on " <> name) $-        property . forAll ((,,) <$> vectorOf 45 anyNumber <*> vectorOf 12 anyNumber <*> arbitrary) $-          \(seeds, last', seed) -> ioProperty $-            withSystemTempDirectory "haskoin-store-pages-" $ \dir -> do-              -- anything at first, then payments, so that the mempool has-              -- something; and questions about what came of it-              (asked, was, starts, fair) <- open20 net ctx dir $ \db -> do-                hist <- start mixedRules db >>= \hist0 -> foldM (step db) hist0 (seeds <> onlyPayments last')-                let asked = unGen (questions net ctx hist 150) (mkQCGen seed) 30-                was <- replies mixed db asked-                starts <- mixed db (\queries -> mapM (from queries) asked)-                -- pages that start at a deleted transaction are not-                -- compared: the layered layout does not know those-                fair <- mapM (startsKnown db) asked-                return (asked, was, starts, fair)-              now <- openLayered net ctx dir $ \db -> do-                void (upgradeDatabase (\_ -> return ()) db)-                replies layered db asked-              length now `shouldBe` length was-              let kept :: [x] -> [x]-                  kept xs = [x | (x, True) <- zip xs (fair <> fair)]-              take 2 (differences (kept (asked <> asked)) (kept was) (kept now)) `shouldBe` []-              let content reply = if size reply == 0 then "nothing in it" else "something in it"-                  pages = [place <> ", " <> content reply | (Just place, reply) <- zip starts was]-              return (tabulate "pages that start at" pages True)--  it "cut pages from a list that is part in the mempool and part in the chain" $-    withSystemTempDirectory "haskoin-store-pages-" $ \dir ->-      openLayered btcRegTest ctx dir $ \db -> do-        hist <- start layeredRules db-        -- a payment from the coin of the first block, with change to the-        -- miner, and a payment from that change, in the same second-        let one = Tx 2 [TxIn (coinOf hist) "" maxBound] [TxOut 3000 (p2pkh 1), TxOut 2000 script] [] 0-            two = Tx 2 [TxIn (OutPoint (txHash one) 1) "" maxBound] [TxOut 1500 (p2pkh 2)] [] 0-            coinbase = txHash (coinbaseTx 1)-            -- those of one second come in the order of their hashes-            unconfirmed = sort [txHash one, txHash two]-            never = TxHash (sha256 ("never made" :: ByteString))-            page limits = L.withView db $ \v -> map (.txid) <$> L.getAddressTxs v miner limits-            outputs = L.withView db $ \v -> map (.outpoint) <$> L.getAddressUnspents v miner def-            amounts = L.withView db $ \v -> fmap numbers <$> L.getBalance v miner-            numbers b = (b.confirmed, b.unconfirmed, b.utxo, b.txs, b.received)-            miner :: Address-            miner = either error id (scriptToAddressBS btcRegTest ctx script)-        forM_ [one, two] $ \tx ->-          (.result) <$> apply layeredRules db (Pay tx 1000) `shouldReturn` Right "True"-        page def `shouldReturn` unconfirmed <> [coinbase]-        page def {limit = 1} `shouldReturn` take 1 unconfirmed-        page def {limit = 2, offset = 1} `shouldReturn` drop 1 unconfirmed <> [coinbase]-        page def {offset = 2} `shouldReturn` [coinbase]-        -- from a mempool transaction: it, and what is older-        page def {start = Just (AtTx (unconfirmed !! 1))} `shouldReturn` drop 1 unconfirmed <> [coinbase]-        page def {start = Just (AtTx (unconfirmed !! 1)), offset = 1} `shouldReturn` [coinbase]-        -- from a confirmed transaction: nothing of the mempool-        page def {start = Just (AtTx coinbase)} `shouldReturn` [coinbase]-        -- from a height: nothing of the mempool either, even above the best-        -- block, as today-        page def {start = Just (AtBlock 5)} `shouldReturn` [coinbase]-        page def {start = Just (AtBlock 0)} `shouldReturn` []-        page def {start = Just (AtTx never)} `shouldReturn` []-        -- the mempool spends the coin, and the change that it made-        outputs `shouldReturn` []-        amounts `shouldReturn` Just (0, 0, 0, 3, 5000002000)-        -- a block confirms the first payment and not the second-        block <- blockOn db [one]-        (.removed) <$> apply layeredRules db block `shouldReturn` [txHash one]-        let reward = txHash (coinbaseTx 2)-        page def `shouldReturn` [txHash two, txHash one, reward, coinbase]-        page def {start = Just (AtTx (txHash one))} `shouldReturn` [txHash one, reward, coinbase]-        page def {start = Just (AtBlock 2), offset = 1, limit = 1} `shouldReturn` [reward]-        -- the change is in the chain now, and the mempool still spends it-        outputs `shouldReturn` [OutPoint reward 0]-        amounts `shouldReturn` Just (5000000000, 0, 1, 4, 10000002000)--  it "answer from one moment, whatever is written in the meantime" $-    withSystemTempDirectory "haskoin-store-pages-" $ \dir ->-      openLayered btcRegTest ctx dir $ \db -> do-        hist0 <- start layeredRules db-        let chained = payments (coinOf hist0)-            pay hist tx = record db hist (Pay tx hist.clock) <$> apply layeredRules db (Pay tx hist.clock)-        hist <- foldM pay hist0 (take 4 chained)-        let asked = unGen (questions btcRegTest ctx hist 100) (mkQCGen 7) 30-            everything queries = (,) <$> ask hist queries <*> mapM (answer queries) asked-        (early, during) <- L.withView db $ \v -> do-          early <- everything (L.layeredRead v)-          -- a block confirms two of the payments, and two more arrive-          block <- blockOn db (take 2 chained)-          mined <- apply layeredRules db block-          length mined.removed `shouldBe` 2-          forM_ (take 2 (drop 4 chained)) $ \tx ->-            (.result) <$> apply layeredRules db (Pay tx 2000) `shouldReturn` Right "True"-          during <- everything (L.layeredRead v)-          return (early, during)-        during `shouldBe` early-        -- a view taken now sees all of it-        late <- layered db everything-        length (fst late).mempool `shouldBe` 4-        (fst late).mempool `shouldNotBe` (fst early).mempool-        (fst late).balances `shouldNotBe` (fst early).balances-        take 1 (differences asked (snd early) (snd late)) `shouldNotBe` []---- | The answers of a database to some questions: asked with the gaps that--- the tests open their databases with, which reach every address of the--- wallet that is paid, and asked again with small gaps, under which the--- later ones are found only when earlier ones are used.-replies ::-  (DatabaseReader -> (StoreRead -> IO [Reply]) -> IO [Reply]) ->-  DatabaseReader ->-  [Question] ->-  IO [Reply]-replies with db asked = concat <$> mapM under [(db.initGap, db.maxGap), (2, 3)]-  where-    under (initial, gap) =-      with (db :: DatabaseReader) {initGap = initial, maxGap = gap} $ \queries ->-        mapM (answer queries) asked---- | Where the page of a question starts, for the questions that have a--- page, given the queries of today's layout.-from :: StoreRead -> Question -> IO (Maybe String)-from queries question = case startOf question of-  Nothing -> return Nothing-  Just Nothing -> return (Just "the top")-  Just (Just (AtBlock _)) -> return (Just "a height")-  Just (Just (AtTx h)) -> Just . maybe "a transaction that is not known" known <$> queries.getTxData h-  where-    known td-      | td.deleted = "a deleted transaction"-      | confirmed td.block = "a confirmed transaction"-      | otherwise = "a mempool transaction"
test/Haskoin/Store/Database/SlicesSpec.hs view
@@ -22,22 +22,22 @@  spec :: Spec spec = do-  ctx <- runIO createContext   modifyMaxSuccess (const 500) $ describe "stored values read in part" $ do     it "read a block as the full decoder does" $-      forAll arbitraryBlockData $ \b -> blockData (encode b) `shouldBe` Right b+      forAll arbitraryBlockData $+        \b -> blockData (encode b) `shouldBe` Right b     it "refuse a block that has fewer bytes than its count of transactions" $       forAll arbitraryBlockData $ \b ->         let bytes = encode b             cut = B.take (B.length bytes - 25) bytes          in not (null b.txs) ==> either (const True) (const False) (blockData cut)     it "find the transaction of a stored one, and whether it is deleted" $-      forAll (arbitraryTxData ctx) $ \td ->+      forAll arbitraryTxData $ \td ->         txBytes (encode td) `shouldBe` Right (runPutS (serialize td.tx), td.deleted)     it "give the bytes that building and serializing the transaction gave" $-      forAll (arbitraryTxData ctx) $ \td ->+      forAll arbitraryTxData $ \td ->         fmap fst (txBytes (encode td))-          `shouldBe` Right (runPutS (serialize (transactionData (toTransaction btc ctx td))))+          `shouldBe` Right (runPutS (serialize (transactionData (toTransaction btc td))))  arbitraryBlockData :: Gen BlockData arbitraryBlockData =@@ -53,11 +53,11 @@     <*> arbitrary     <*> arbitrary -arbitraryTxData :: Ctx -> Gen TxData-arbitraryTxData ctx =+arbitraryTxData :: Gen TxData+arbitraryTxData =   TxData     <$> oneof [MemRef <$> arbitrary, BlockRef <$> arbitrary <*> arbitrary]-    <*> oneof [arbitraryTx btc ctx, arbitraryWitnessTx btc ctx, arbitraryLegacyTx btc ctx]+    <*> oneof [arbitraryTx btc, arbitraryWitnessTx btc, arbitraryLegacyTx btc]     <*> (IntMap.fromList <$> listOf ((,) <$> choose (0, 1000) <*> (Prev <$> arbitraryBS <*> arbitrary)))     <*> arbitrary     <*> arbitrary
test/Haskoin/Store/Database/ViewsSpec.hs view
@@ -1,37 +1,34 @@ module Haskoin.Store.Database.ViewsSpec (spec) where -import Control.Monad (void) import Data.List (sort)-import Haskoin (BlockHash, btcRegTest, prepareContext)+import Haskoin (BlockHash, btcRegTest) import Haskoin.Store.Database.Query qualified as Q import Haskoin.Store.Database.Views import Haskoin.Store.Histories+import Haskoin.Store.Integration.Temp (withSystemTempDirectory) import Test.Hspec-import UnliftIO (withSystemTempDirectory)  spec :: Spec-spec = prepareContext $ \ctx -> describe "views of the database by block" $ do+spec = describe "views of the database by block" $ do   let withChain go =         withSystemTempDirectory "haskoin-store-views-" $ \dir ->-          open20 btcRegTest ctx dir $ \db -> do-            void (start mixedRules db)+          openLayered btcRegTest dir $ \db -> do+            startChain db             views <- newViews db             let best = Q.getBestBlock db >>= maybe (fail "no best block") return-                mine = do-                  block <- blockOn db []-                  void (apply mixedRules db block)-                  b <- best+                next = do+                  b <- mine db []                   takeView views b                   return b-            go db views best mine `shouldReturn` ()+            go db views best next `shouldReturn` ()             closeViews views    it "answer from the block that each was taken at, whatever came after" $-    withChain $ \db views best mine -> do+    withChain $ \db views best next -> do       first <- best       takeView views first-      second <- mine-      third <- mine+      second <- next+      third <- next       sort <$> heldViews views `shouldReturn` sort [first, second, third]       Q.getBestBlock db `shouldReturn` Just third       withViewAt views first Q.getBestBlock `shouldReturn` Just (Just first)@@ -40,27 +37,27 @@       (fmap (fmap (const ())) <$> withViewAt views first (`Q.getBlock` second)) `shouldReturn` Just Nothing    it "let go of the views that are not to be kept" $-    withChain $ \_ views best mine -> do+    withChain $ \_ views best next -> do       first <- best       takeView views first-      second <- mine+      second <- next       keepViews views [second]       heldViews views `shouldReturn` [second]       withViewAt views first Q.getBestBlock `shouldReturn` (Nothing :: Maybe (Maybe BlockHash))       withViewAt views second Q.getBestBlock `shouldReturn` Just (Just second)    it "keep a view that is being read until its reader is done" $-    withChain $ \_ views best mine -> do+    withChain $ \_ views best next -> do       first <- best       takeView views first-      second <- mine+      second <- next       result <- withViewAt views first $ \old -> do         keepViews views [second]         -- gone for anyone who asks now         heldViews views `shouldReturn` [second]         withViewAt views first Q.getBestBlock `shouldReturn` (Nothing :: Maybe (Maybe BlockHash))         -- and still there for the one that was reading, while blocks go on-        third <- mine+        third <- next         (,third) <$> Q.getBestBlock old       fmap fst result `shouldBe` Just (Just first)       -- the reader is done: the view is gone, and the others are as they were
+ test/Haskoin/Store/Database/WriterSpec.hs view
@@ -0,0 +1,339 @@+module Haskoin.Store.Database.WriterSpec (spec) where++import Control.Monad (forM_)+import Data.ByteString (ByteString)+import Data.ByteString qualified as B+import Data.Default (def)+import Data.IntMap.Strict qualified as IntMap+import Data.Maybe (fromMaybe)+import Data.Serialize (encode)+import Data.Word (Word32, Word64)+import Database.RocksDB (columnFamilies)+import Haskoin+  ( Address,+    OutPoint (..),+    Tx (..),+    TxHash (..),+    TxIn (..),+    TxOut (..),+    btcRegTest,+    eitherToMaybe,+    scriptToAddressBS,+    sha256,+    txHash,+  )+import Haskoin.Node (LogLevel (..), Logger (..))+import Haskoin.Store.Data+  ( Balance (..),+    BlockRef (..),+    Spender (..),+    TxData (..),+    TxRef (..),+    Unspent (..),+  )+import Haskoin.Store.Database.Query qualified as Q+import Haskoin.Store.Database.Reader (DatabaseReader (..))+import Haskoin.Store.Database.Types (BalVal (..))+import Haskoin.Store.Database.Writer+import Haskoin.Store.Histories (dump, openLayered)+import Haskoin.Store.Integration.Temp (withSystemTempDirectory)+import Test.Hspec++spec :: Spec+spec = describe "a batch of writes" $ do+  describe "its read cache" $ do+    it "never hides a change of the batch, whether the key was read ahead or not" $+      withDB $ \db -> do+        batch db $ \w -> insertUnspent w (coin 1) >> setBalance w (balanceOf 1 10)+        batch db $ \w -> do+          -- one that the database has, one that it has not+          readAheadUnspents w [(coin 1).outpoint, (coin 2).outpoint]+          readAheadBalances w [address 1, address 2]+          getUnspent w (coin 1).outpoint `shouldReturn` Just (coin 1)+          getUnspent w (coin 2).outpoint `shouldReturn` Nothing+          getBalance w (address 1) `shouldReturn` Just (balanceOf 1 10)+          getBalance w (address 2) `shouldReturn` Nothing+          deleteUnspent w (coin 1).outpoint+          insertUnspent w (coin 2)+          setBalance w (balanceOf 1 20)+          setBalance w (balanceOf 2 30)+          -- reading ahead again changes nothing either+          readAheadUnspents w [(coin 1).outpoint, (coin 2).outpoint]+          readAheadBalances w [address 1, address 2]+          getUnspent w (coin 1).outpoint `shouldReturn` Nothing+          getUnspent w (coin 2).outpoint `shouldReturn` Just (coin 2)+          getBalance w (address 1) `shouldReturn` Just (balanceOf 1 20)+          getBalance w (address 2) `shouldReturn` Just (balanceOf 2 30)+          -- a key that was not read ahead is read when asked for+          getUnspent w (coin 3).outpoint `shouldReturn` Nothing+          insertUnspent w (coin 3)+          getUnspent w (coin 3).outpoint `shouldReturn` Just (coin 3)+        Q.getUnspent db (coin 1).outpoint `shouldReturn` Nothing+        Q.getUnspent db (coin 2).outpoint `shouldReturn` Just (coin 2)+        Q.getUnspent db (coin 3).outpoint `shouldReturn` Just (coin 3)+        Q.getBalance db (address 1) `shouldReturn` Just (balanceOf 1 20)++    it "reads the groups of spenders ahead, and adds to them" $+      withDB $ \db -> do+        let made = TxHash (sha256 ("made" :: ByteString))+            spender i = Spender (TxHash (sha256 (encode i))) i+        batch db $ \w -> insertSpender w (OutPoint made 1) (spender 1)+        batch db $ \w -> do+          readAheadSpenders w [OutPoint made 2, OutPoint made 40]+          insertSpender w (OutPoint made 2) (spender 2)+          insertSpender w (OutPoint made 40) (spender 40)+          getChainSpender w (OutPoint made 1) `shouldReturn` Just (spender 1)+        Q.getChainSpenders db made+          `shouldReturn` IntMap.fromList [(1, spender 1), (2, spender 2), (40, spender 40)]++    it "gives the same answers without the cache" $+      withDB $ \db -> do+        batch db $ \w -> insertUnspent w (coin 1)+        batch db $ \w -> do+          let plain = withoutReadCache w+          readsAhead plain `shouldBe` False+          readAheadUnspents plain [(coin 1).outpoint]+          getUnspent plain (coin 1).outpoint `shouldReturn` Just (coin 1)+          -- a change made through one is seen through the other+          deleteUnspent plain (coin 1).outpoint+          getUnspent w (coin 1).outpoint `shouldReturn` Nothing++  describe "its merges: changes to values that it has not read" $ do+    it "adds differences to a balance that it has not read, and shows the sum to its own reads" $+      withDB $ \db -> do+        batch db $ \w -> setBalance w (balanceOf 1 10)+        batch db $ \w -> do+          mergeBalance w (address 1) (def :: BalVal) {confirmed = 5, txs = 1}+          mergeBalance w (address 1) (def :: BalVal) {confirmed = negate 3, utxo = 2}+          mergeBalance w (address 2) (def :: BalVal) {received = 7, txs = 1}+          -- the database is read for the value, and the differences added+          getBalance w (address 1) `shouldReturn` Just (numbers 1 12 3 2 10)+          getBalance w (address 2) `shouldReturn` Just (numbers 2 0 0 1 7)+          getBalance w (address 3) `shouldReturn` Nothing+          -- and the same through a writer that remembers no read+          getBalance (withoutReadCache w) (address 1) `shouldReturn` Just (numbers 1 12 3 2 10)+        Q.getBalance db (address 1) `shouldReturn` Just (numbers 1 12 3 2 10)+        Q.getBalance db (address 2) `shouldReturn` Just (numbers 2 0 0 1 7)+        Q.getBalance db (address 3) `shouldReturn` Nothing+        -- one batch after another, each without a read+        batch db $ \w -> mergeBalance w (address 2) (def :: BalVal) {received = 1}+        batch db $ \w -> mergeBalance w (address 2) (def :: BalVal) {received = 1}+        Q.getBalance db (address 2) `shouldReturn` Just (numbers 2 0 0 1 9)++    it "applies differences to a balance that it has whole, set or deleted" $+      withDB $ \db -> do+        batch db $ \w -> setBalance w (balanceOf 1 10) >> setBalance w (balanceOf 2 20)+        batch db $ \w -> do+          setBalance w (numbers 1 100 1 1 100)+          mergeBalance w (address 1) (def :: BalVal) {confirmed = 1}+          getBalance w (address 1) `shouldReturn` Just (numbers 1 101 1 1 100)+          deleteBalance w (address 2)+          mergeBalance w (address 2) (def :: BalVal) {confirmed = 4, txs = 1}+          getBalance w (address 2) `shouldReturn` Just (numbers 2 4 0 1 0)+        Q.getBalance db (address 1) `shouldReturn` Just (numbers 1 101 1 1 100)+        -- from zero, not from the 20 that the batch deleted+        Q.getBalance db (address 2) `shouldReturn` Just (numbers 2 4 0 1 0)++    it "lets a whole balance take the place of the differences, which whoever set it had read" $+      withDB $ \db -> do+        batch db $ \w -> setBalance w (balanceOf 1 10) >> setBalance w (balanceOf 2 20)+        batch db $ \w -> do+          mergeBalance w (address 1) (def :: BalVal) {confirmed = 5}+          mergeBalance w (address 2) (def :: BalVal) {confirmed = 5}+          -- as the revert of a block does: read, change, write whole+          Just b <- getBalance w (address 1)+          b.confirmed `shouldBe` 15+          setBalance w (b :: Balance) {confirmed = b.confirmed - 1}+          getBalance w (address 1) `shouldReturn` Just (numbers 1 14 1 1 10)+          deleteBalance w (address 2)+          getBalance w (address 2) `shouldReturn` Nothing+          -- and differences after that are applied to the whole value+          mergeBalance w (address 1) (def :: BalVal) {confirmed = 1}+          getBalance w (address 1) `shouldReturn` Just (numbers 1 15 1 1 10)+        Q.getBalance db (address 1) `shouldReturn` Just (numbers 1 15 1 1 10)+        Q.getBalance db (address 2) `shouldReturn` Nothing++    it "adds spenders to groups that it has not read, and shows them to each of its reads" $+      withDB $ \db -> do+        let made = TxHash (sha256 ("made" :: ByteString))+            other = TxHash (sha256 ("other" :: ByteString))+            spender i = Spender (TxHash (sha256 (encode i))) i+            out = OutPoint made+        batch db $ \w -> insertSpender w (out 1) (spender 1) >> insertSpender w (out 40) (spender 40)+        batch db $ \w -> do+          mergeSpender w (out 2) (spender 2)+          mergeSpender w (out 3) (spender 3)+          mergeSpender w (out 17) (spender 17)+          mergeSpender w (OutPoint other 0) (spender 9)+          -- one output at a time: what the database has, and what is added+          getChainSpender w (out 1) `shouldReturn` Just (spender 1)+          getChainSpender w (out 2) `shouldReturn` Just (spender 2)+          getChainSpender w (out 17) `shouldReturn` Just (spender 17)+          getChainSpender w (out 4) `shouldReturn` Nothing+          getChainSpender w (OutPoint other 0) `shouldReturn` Just (spender 9)+          getChainSpender (withoutReadCache w) (out 3) `shouldReturn` Just (spender 3)+          -- and all those of a transaction, over its three groups+          getChainSpenders w made 48+            `shouldReturn` IntMap.fromList [(i, spender (fromIntegral i)) | i <- [1, 2, 3, 17, 40]]+          getChainSpenders w other 1 `shouldReturn` IntMap.fromList [(0, spender 9)]+        Q.getChainSpenders db made+          `shouldReturn` IntMap.fromList [(i, spender (fromIntegral i)) | i <- [1, 2, 3, 17, 40]]+        Q.getChainSpender db (OutPoint other 0) `shouldReturn` Just (spender 9)+        -- the newer spender wins for an output that already has one+        batch db $ \w -> mergeSpender w (out 2) (spender 22)+        Q.getChainSpender db (out 2) `shouldReturn` Just (spender 22)++    it "puts a spender in a group that it has whole, and lets a whole group take the place of what it added" $+      withDB $ \db -> do+        let made = TxHash (sha256 ("made" :: ByteString))+            spender i = Spender (TxHash (sha256 (encode i))) i+            out = OutPoint made+            group is = IntMap.fromList [(i, spender (fromIntegral i)) | i <- is]+        batch db $ \w -> mapM_ (\i -> insertSpender w (out i) (spender i)) [1, 20, 35]+        batch db $ \w -> do+          -- added to, then read and written whole, as a revert does+          mergeSpender w (out 2) (spender 2)+          deleteSpender w (out 1)+          getChainSpenders w made 48 `shouldReturn` group [2, 20, 35]+          -- whole, then added to+          insertSpender w (out 3) (spender 3)+          mergeSpender w (out 4) (spender 4)+          getChainSpenders w made 48 `shouldReturn` group [2, 3, 4, 20, 35]+          -- a group that the batch empties is deleted, and what is added+          -- afterwards is all that it has+          deleteSpender w (out 20)+          getChainSpender w (out 20) `shouldReturn` Nothing+          mergeSpender w (out 21) (spender 21)+          getChainSpenders w made 48 `shouldReturn` group [2, 3, 4, 21, 35]+          -- added to, then a spender put in by reading the group: what+          -- is written whole has what the database had and what was added+          mergeSpender w (out 33) (spender 33)+          insertSpender w (out 34) (spender 34)+          getChainSpender w (out 33) `shouldReturn` Just (spender 33)+          getChainSpenders w made 48 `shouldReturn` group [2, 3, 4, 21, 33, 34, 35]+        Q.getChainSpenders db made `shouldReturn` group [2, 3, 4, 21, 33, 34, 35]++  it "writes a large batch, packed on all cores, as it writes the same changes a few at a time" $+    withDB $ \large ->+      withDB $ \small -> do+        -- Two thousand unspent outputs and balances and as many+        -- transactions, then a thousand transactions of three kilobytes,+        -- which fill the buffer of each core more than once. Among them,+        -- so that the records around them have to keep their place, one+        -- with a script too long to be copied into a shared buffer and one+        -- larger than such a buffer.+        let changes =+              concat+                [ [ \w -> insertUnspent w (coin i),+                    \w -> setBalance w (balanceOf i i),+                    \w -> insertAddrTx w (address i) (TxRef (BlockRef 7 i) (txHash (payment i))),+                    \w -> insertAddrUnspent w (address i) (coin i),+                    \w -> insertTx w (txData i (payment i))+                  ]+                | i <- [1 .. 2000]+                ]+                <> [\w -> insertTx w (txData 1 long), \w -> insertTx w (txData 2 big)]+                <> [\w -> insertTx w (txData i (wide i)) | i <- [1 .. 1000]]+                -- and merge records, for balances and for groups of+                -- spenders that have no value yet: two for each key, which+                -- one batch makes one record of and forty at a time two+                <> concat+                  [ [ \w -> mergeBalance w (address i) (def :: BalVal) {confirmed = fromIntegral i, txs = 1},+                      \w -> mergeSpender w (OutPoint (txHash (payment i)) (i `mod` 40)) (Spender (txHash (wide i)) i)+                    ]+                  | i <- [5001 .. 5600] <> [5001 .. 5600]+                  ]+        report <- snd <$> runWriterReport btcRegTest large quiet (\w -> mapM_ ($ w) changes)+        report.operations `shouldBe` 12202+        -- a few at a time, a batch is not packed: each operation is handed+        -- to RocksDB by itself, as all of them were before+        forM_ (pieces 40 changes) $ \some -> batch small (\w -> mapM_ ($ w) some)+        forM_ (zip (columnFamilies large.db) (columnFamilies small.db)) $ \(a, b) -> do+          written <- dump large a+          expected <- dump small b+          length written `shouldBe` length expected+          written `shouldBe` expected+        entries <- concat <$> mapM (dump large) (columnFamilies large.db)+        length entries `shouldBe` 12202+        -- every change set a key of its own, or merged into one that had+        -- no value, where the one merge record that the batch makes of+        -- the two is the value. So the bytes of the batch are those of the+        -- keys and values that the database now has+        report.bytes `shouldBe` sum [B.length k + B.length v | (k, v) <- entries]+        report.bytes `shouldSatisfy` (> 3600000)+        Q.getBalance large (address 5001) `shouldReturn` Just (numbers 5001 10002 0 2 0)++quiet :: Logger+quiet = Logger {level = LevelError, source = "", push = \_ -> return ()}++withDB :: (DatabaseReader -> IO a) -> IO a+withDB action = withSystemTempDirectory "haskoin-store-writer-" $ \dir -> openLayered btcRegTest dir action++batch :: DatabaseReader -> (Writer -> IO a) -> IO a+batch db = runWriter btcRegTest db quiet++address :: Word32 -> Address+address i = fromMaybe (error "no address") (eitherToMaybe (scriptToAddressBS btcRegTest (scriptOf i)))++scriptOf :: Word32 -> ByteString+scriptOf i = "\x76\xa9\x14" <> B.take 20 (encode (sha256 (encode i))) <> "\x88\xac"++-- | An unspent output of a number, paid to the address of that number.+coin :: Word32 -> Unspent+coin i =+  Unspent+    { block = BlockRef 7 i,+      outpoint = OutPoint (TxHash (sha256 (encode i))) (i `mod` 3),+      value = 1000 + fromIntegral i,+      script = scriptOf i,+      address = Just (address i)+    }++-- | A balance of an address by its numbers: confirmed, unspent outputs,+-- transactions and received.+numbers :: Word32 -> Word64 -> Word64 -> Word64 -> Word64 -> Balance+numbers i confirmed utxo txs received =+  Balance {address = address i, confirmed, unconfirmed = 0, utxo, txs, received}++balanceOf :: Word32 -> Word32 -> Balance+balanceOf i n =+  Balance+    { address = address i,+      confirmed = fromIntegral n,+      unconfirmed = 0,+      utxo = 1,+      txs = 1,+      received = fromIntegral n+    }++payment :: Word32 -> Tx+payment i = Tx 2 [TxIn (coin i).outpoint "" maxBound] [TxOut 900 (scriptOf (i + 1))] [] 0++-- | A transaction with an output script of twenty kilobytes.+long :: Tx+long = Tx 2 [TxIn (coin 1).outpoint "" maxBound] [TxOut 1 (B.replicate 20000 0x6a)] [] 0++-- | A transaction of three kilobytes with a number: short enough to be+-- copied into a shared buffer, and a few dozen of them fill one.+wide :: Word32 -> Tx+wide i = Tx 2 [TxIn (OutPoint (TxHash (sha256 (encode i))) 7) "" maxBound] [TxOut 1 (B.replicate 3000 0x6a)] [] 0++-- | A transaction of four hundred kilobytes, in outputs of a kilobyte.+big :: Tx+big = Tx 2 [TxIn (coin 2).outpoint "" maxBound] (replicate 400 (TxOut 1 (B.replicate 1000 0x6a))) [] 0++txData :: Word32 -> Tx -> TxData+txData i tx =+  TxData+    { block = BlockRef 7 i,+      tx = tx,+      prevs = mempty,+      deleted = False,+      rbf = False,+      timestamp = 1700000000,+      spenders = mempty+    }++pieces :: Int -> [a] -> [[a]]+pieces _ [] = []+pieces n xs = let (a, b) = splitAt n xs in a : pieces n b
+ test/Haskoin/Store/ExceptionSpec.hs view
@@ -0,0 +1,111 @@+-- | The handlers and cleanups of "Haskoin.Store.Exception": what stops a+-- thread from outside is not taken for an error, and a cleanup is not cut+-- short.+module Haskoin.Store.ExceptionSpec (spec) where++import Control.Concurrent (threadDelay)+import Control.Concurrent.Async (AsyncCancelled (..), async, cancel, waitCatch)+import Control.Concurrent.MVar (newEmptyMVar, putMVar, takeMVar)+import Control.Exception (ErrorCall (..), fromException, throwIO, try)+import Control.Monad (forM_, void)+import Data.IORef (modifyIORef', newIORef, readIORef, writeIORef)+import Haskoin.Store.Exception+import System.Timeout (timeout)+import Test.Hspec++spec :: Spec+spec = describe "exceptions" $ do+  describe "a handler for any error" $ do+    it "catches the errors of the code that it guards" $ do+      (broken `catchAny` \e -> return (show e)) `shouldReturn` "broken"+      handleAny (\e -> return (show e)) broken `shouldReturn` "broken"+      (either show id <$> tryAny broken) `shouldReturn` "broken"+      (either show id <$> tryAny (return "fine")) `shouldReturn` "fine"++    it "does not take a cancellation for an error" $+      forM_ guards $ \(name, guard) -> do+        started <- newEmptyMVar+        survived <- newIORef False+        a <- async $ do+          guard (putMVar started () >> threadDelay 10000000)+          writeIORef survived True+        takeMVar started+        cancel a+        ended <- waitCatch a+        (name, either fromException (const Nothing) ended) `shouldBe` (name, Just AsyncCancelled)+        (name,) <$> readIORef survived `shouldReturn` (name, False)++    it "does not take a timeout for an error" $+      forM_ guards $ \(name, guard) -> do+        caught <- newIORef (0 :: Int)+        -- what a loop that goes on after any error does, as those of the+        -- store do: were the timeout caught, this would return+        let step = guard (threadDelay 10000000) >> modifyIORef' caught (+ 1)+        (name,) <$> timeout 20000 step `shouldReturn` (name, Nothing)+        (name,) <$> readIORef caught `shouldReturn` (name, 0)++  describe "a cleanup" $ do+    it "runs to its end when the thread is stopped during it" $+      forM_ cleanups $ \(name, with) -> do+        cleaning <- newEmptyMVar+        finished <- newIORef False+        -- the cleanup waits, which is where an exception would get in+        let cleanup = putMVar cleaning () >> threadDelay 100000 >> writeIORef finished True+        a <- async (void (try @ErrorCall (with cleanup)))+        takeMVar cleaning+        -- returns when the thread has ended+        cancel a+        (name,) <$> readIORef finished `shouldReturn` (name, True)++    it "runs when the thread is stopped during the action" $ do+      let stopped with = do+            started <- newEmptyMVar+            cleaned <- newIORef False+            a <- async (with (putMVar started () >> threadDelay 10000000) (writeIORef cleaned True))+            takeMVar started+            cancel a+            ended <- waitCatch a+            either fromException (const Nothing) ended `shouldBe` Just AsyncCancelled+            readIORef cleaned `shouldReturn` True+      stopped (\action cleanup -> bracket (return ()) (const cleanup) (const action))+      stopped (\action cleanup -> bracket_ (return ()) cleanup action)+      stopped finally+      stopped onException++    it "runs after an action that worked, unless it is only for failures" $ do+      ran <- newIORef (0 :: Int)+      let cleanup = modifyIORef' ran (+ 1)+      bracket (return 'a') (const cleanup) return `shouldReturn` 'a'+      bracket_ (return ()) cleanup (return 'b') `shouldReturn` 'b'+      (return 'c' `finally` cleanup) `shouldReturn` 'c'+      readIORef ran `shouldReturn` 3+      (return 'd' `onException` cleanup) `shouldReturn` 'd'+      readIORef ran `shouldReturn` 3++    it "leaves the error of the action to be thrown when it fails too" $ do+      let failing = throwIO (ErrorCall "cleanup") :: IO ()+      try (bracket (return ()) (const failing) (const broken)) `shouldReturn` Left (ErrorCall "broken")+      try (bracket_ (return ()) failing broken) `shouldReturn` Left (ErrorCall "broken")+      try (broken `finally` failing) `shouldReturn` Left (ErrorCall "broken")+      try (broken `onException` failing) `shouldReturn` Left (ErrorCall "broken")+      -- with nothing else to say, its own error is thrown+      try (return () `finally` failing) `shouldReturn` Left (ErrorCall "cleanup")+      try (bracket_ (return ()) failing (return ())) `shouldReturn` Left (ErrorCall "cleanup")+  where+    broken = throwIO (ErrorCall "broken") :: IO String+    -- each way of going on after any error+    guards :: [(String, IO () -> IO ())]+    guards =+      [ ("catchAny", \action -> action `catchAny` \_ -> return ()),+        ("handleAny", handleAny (\_ -> return ())),+        ("tryAny", void . tryAny)+      ]+    -- each way of running a cleanup, reached without the thread being+    -- stopped: after an action that worked, or one that failed+    cleanups :: [(String, IO () -> IO ())]+    cleanups =+      [ ("bracket", \cleanup -> bracket (return ()) (const cleanup) return),+        ("bracket_", \cleanup -> bracket_ (return ()) cleanup (return ())),+        ("finally", \cleanup -> return () `finally` cleanup),+        ("onException", \cleanup -> void broken `onException` cleanup)+      ]
+ test/Haskoin/Store/Fleet/CacheSpec.hs view
@@ -0,0 +1,538 @@+-- | The fleet's cache of extended public keys, with two members on one+-- chain in RocksDB and a Redis in Podman: one build for a burst of+-- requests, the same refusal from every member, a dead holder of the dibs+-- taken over, exact reads across a change of the pool and a move of the+-- tip, a reorganisation that keeps the cache, the pool filled again after+-- Redis lost its data, and random histories after which every cached entry+-- is what the chain at the tip and the pool give.+module Haskoin.Store.Fleet.CacheSpec (spec) where++import Control.Concurrent (threadDelay)+import Control.Concurrent.Async (mapConcurrently, withAsync)+import Control.Concurrent.STM (readTVarIO)+import Control.Exception (SomeException, try)+import Control.Monad (forM, forM_, join, replicateM_, unless, void, when)+import Data.ByteString (ByteString)+import Data.ByteString qualified as B+import Data.Default (def)+import Data.Functor ((<&>))+import Data.HashMap.Strict qualified as HashMap+import Data.IORef (IORef, atomicModifyIORef', newIORef)+import Data.IntMap.Strict qualified as IntMap+import Data.List (find)+import Data.List.NonEmpty (NonEmpty (..))+import Data.Maybe (catMaybes, fromMaybe)+import Data.Serialize (encode)+import Data.Time.Clock.POSIX (getPOSIXTime)+import Data.Word (Word32, Word64, Word8)+import Database.Redis qualified as Redis+import Haskoin+  ( Block (..),+    BlockHash,+    BlockHeader (..),+    BlockNode (..),+    Tx (..),+    TxHash (..),+    TxIn (..),+    TxOut (..),+    addressToScriptBS,+    btcRegTest,+    buildMerkleRoot,+    deriveXPubKey,+    doubleSHA256,+    isValidPOW,+    makeXPrvKey,+    nullOutPoint,+    scriptToAddressBS,+    txHash,+  )+import Haskoin.Store.BlockStore (toFetch)+import Haskoin.Store.Common (StoreRead (..), sortTxs)+import Haskoin.Store.Data+  ( BlockData (..),+    BlockRef (..),+    DeriveType (..),+    Prev (..),+    TxData (..),+    TxRef,+    Unspent (..),+    XPubBal,+    XPubSpec (..),+    XPubUnspent (..),+    confirmed,+  )+import Haskoin.Store.Database.Query qualified as Q+import Haskoin.Store.Database.Query.Pooled (pooledRead)+import Haskoin.Store.Database.Reader (DatabaseReader (..), Derived (..), derivedAddrs)+import Haskoin.Store.Database.Views (withViewAt)+import Haskoin.Store.Fleet.Member+import Haskoin.Store.Fleet.Redis (FleetKeys (..))+import Haskoin.Store.Fleet.RedisSpec (withRedis)+import Haskoin.Store.Fleet.XPub+  ( Begun (..),+    Cached (..),+    cacheAbort,+    cacheBegin,+    cachePieces,+    cacheSize,+    cachedEntry,+    tipMoves,+    xpubId,+  )+import Haskoin.Store.Histories (openLayered, revert, run, startChain)+import Haskoin.Store.Integration.Podman (podmanUsable)+import Haskoin.Store.Integration.Temp (withSystemTempDirectory)+import Haskoin.Store.Logic (importBlock)+import Haskoin.Store.TestBlocks (script)+import Haskoin.Store.XPub (EntryMoved (..), XPubReads (..))+import Haskoin.Store.XPub.Indexer (Answer (..), Settings (..), defaultSettings, retryAfter)+import Haskoin.Store.XPub.Lock (Lock (..), LockConfig (..), Taken (..), defaultLockConfig, redisLock)+import Haskoin.Store.XPub.Service (Session (..), xpubAsk)+import Test.Hspec+import Test.Hspec.QuickCheck (modifyMaxSuccess)+import Test.QuickCheck++spec :: Spec+spec = do+  usable <- runIO podmanUsable+  describe "xpub cache of a fleet" $ case usable of+    Left why -> it "keeps entries equal to the chain and the pool" (pendingWith why)+    Right () -> aroundAll withRedis $ do+      it "builds a key once for a burst on two members, and both answer it" (`withBed` burstTest)+      it "refuses on two members with the same Retry-After" (`withBed` retryTest)+      it "takes over the dibs of a holder that died" (`withBed` deadHolderTest)+      it "has a request answered again when the pool changes the entry it read" (`withBed` poolChangeTest)+      it "has a request answered again when the tip moves" (`withBed` tipMoveTest)+      it "never answers an entry that a move of the tip is writing" (`withBed` markedTest)+      it "keeps the cache across a reorganisation, as the chain and the pool give it" (`withBed` reorgTest)+      it "fills the pool again after Redis lost its data" (`withBed` refillTest)+      it "fetches no transaction that another member received" (`withBed` seenTest)+      modifyMaxSuccess (const 10) $+        it "keeps every cached entry equal to the chain at the tip and the pool" $ \r ->+          property $ \steps -> ioProperty (withBed r (\bed -> modelTest bed (take 12 steps)) >> return True)++-------------------------------------------------------------------------------+-- The tests+-------------------------------------------------------------------------------++burstTest :: Bed -> IO ()+burstTest bed = do+  let x = xpub 1+  payKey bed x [0, 3] True+  answers <- mapConcurrently (\i -> xpubAsk (if even i then bed.a else bed.b).door x) [1 .. 20 :: Int]+  answers `shouldSatisfy` all (== Indexed)+  threadDelay 300000+  -- one build, which Redis announced to both+  forM_ [bed.a, bed.b] $ \m ->+    (HashMap.lookupDefault 0 (xpubId x) <$> readTVarIO m.built) `shouldReturn` 1+  forM_ [bed.a, bed.b] $ \m -> sameAsChain bed m x++retryTest :: Bed -> IO ()+retryTest bed = do+  let x = xpub 2+  ghost <- redisLock (defaultLockConfig (bed.prefix <> "fleet:") "ghost") bed.conn xpubId+  -- a job that another store expects to finish in a minute, past the wait+  taken ghost x 60000000+  answers <- forM [bed.a, bed.b] (\m -> xpubAsk m.door x)+  case answers of+    [Refused _ ta, Refused _ tb] -> do+      retryAfter ta `shouldBe` retryAfter tb+      retryAfter ta `shouldSatisfy` (> 10)+    other -> expectationFailure ("Expected two refusals: " <> show other)++deadHolderTest :: Bed -> IO ()+deadHolderTest bed = do+  let x = xpub 3+  payKey bed x [1] True+  ghost <- redisLock (defaultLockConfig (bed.prefix <> "fleet:") "ghost") {ttl = 1000} bed.conn xpubId+  -- its holder expects to be done in half a second, and never renews+  taken ghost x 500000+  xpubAsk bed.a.door x `shouldReturn` Indexed+  sameAsChain bed bed.a x++poolChangeTest :: Bed -> IO ()+poolChangeTest bed = do+  let x = xpub 4+  payKey bed x [0] True+  xpubAsk bed.a.door x `shouldReturn` Indexed+  tip <- fleetTipOf bed+  outcome <- answeringCached bed.a tip $ \_ sess _ -> do+    first <- sess.reads.balances x+    first `shouldSatisfy` (/= Nothing)+    payKey bed x [1] False+    try (sess.reads.transactions x def) :: IO (Either EntryMoved (Maybe [TxRef]))+  outcome `shouldBe` Just (Left EntryMoved)+  sameAsChain bed bed.a x++tipMoveTest :: Bed -> IO ()+tipMoveTest bed = do+  let x = xpub 5+  payKey bed x [0] True+  xpubAsk bed.a.door x `shouldReturn` Indexed+  tip <- fleetTipOf bed+  outcome <- answeringCached bed.a tip $ \_ sess _ -> do+    first <- sess.reads.balances x+    first `shouldSatisfy` (/= Nothing)+    payKey bed x [2] True+    try (sess.reads.transactions x def) :: IO (Either SomeException (Maybe [TxRef]))+  fmap (either (const True) (const False)) outcome `shouldBe` Just True+  sameAsChain bed bed.a x+  sameAsChain bed bed.b x++markedTest :: Bed -> IO ()+markedTest bed = do+  let x = xpub 6+  payKey bed x [0] True+  xpubAsk bed.a.door x `shouldReturn` Indexed+  tip <- fleetTipOf bed+  h <- best bed+  height <- maybe 0 (.height) <$> Q.getBlock bed.db h+  dest <- addressScript bed x 0+  coin : _ <- coins bed+  let tx = spend [coin] [(dest, 100000)]+      td =+        TxData+          { block = BlockRef (height + 1) 1,+            tx,+            prevs = IntMap.singleton 0 (Prev coin.script coin.value),+            deleted = False,+            rbf = False,+            timestamp = 0,+            spenders = IntMap.empty+          }+      token = "marked-test"+  cacheBegin bed.conn bed.a.keys token tip >>= \case+    Begun -> return ()+    _ -> expectationFailure "The move of the cache did not begin"+  Just moves <- withViewAt bed.a.views h (\r -> tipMoves bed.conn bed.a.keys r [] [td])+  cachePieces bed.conn bed.a.keys token tip moves `shouldReturn` True+  -- the entry is marked: it is not answered, half written+  cachedEntry bed.conn bed.a.keys tip bed.db.initGap bed.db.maxGap True x >>= \case+    Hit _ -> expectationFailure "A marked entry was answered"+    _ -> return ()+  cacheAbort bed.conn bed.a.keys token+  xpubAsk bed.a.door x `shouldReturn` Indexed+  sameAsChain bed bed.a x++reorgTest :: Bed -> IO ()+reorgTest bed = do+  let x = xpub 7+  payKey bed x [0] True+  xpubAsk bed.a.door x `shouldReturn` Indexed+  payKey bed x [1] True+  payKey bed x [2] True+  sameAsChain bed bed.a x+  -- the two blocks with the payments to 1 and 2 are abandoned for three+  -- empty ones: the payments come back to the pool+  reorganise bed 2+  (entries, _) <- cacheSize bed.conn bed.a.keys+  entries `shouldSatisfy` (> 0)+  forM_ [bed.a, bed.b] $ \m -> sameAsChain bed m x++-- | What one store received from a peer, it does not fetch again; another+-- fetches it until the first has offered it to the pool, and not after:+-- both ask the same set in Redis, which has only what the pool decided on.+seenTest :: Bed -> IO ()+seenTest bed = do+  coin : _ <- coins bed+  let tx = spend [coin] [(script, 100000)]+      other = TxHash (doubleSHA256 ("never sent" :: ByteString))+      fetch m = toFetch (\_ -> return False) (memberSeen m) (\_ -> return False)+  fetch bed.b [txHash tx, other] `shouldReturn` [txHash tx, other]+  offer bed tx+  fetch bed.a [txHash tx, other] `shouldReturn` [other]+  fetch bed.b [txHash tx, other] `shouldReturn` [txHash tx, other]+  settle bed+  inPool bed tx `shouldReturn` True+  fetch bed.a [txHash tx, other] `shouldReturn` [other]+  fetch bed.b [txHash tx, other] `shouldReturn` [other]++refillTest :: Bed -> IO ()+refillTest bed = do+  let x = xpub 8+  payKey bed x [0] True+  dest <- addressScript bed x 1+  coin : _ <- coins bed+  let tx = spend [coin] [(dest, 100000)]+  offer bed tx+  settle bed+  inPool bed tx `shouldReturn` True+  -- Redis loses the fleet's data+  lost <- Redis.runRedis bed.conn (Redis.keys (bed.prefix <> "*"))+  forM_ (either (const []) id lost) $ \k -> void (Redis.runRedis bed.conn (Redis.del (k :| [])))+  settle bed+  readTVarIO bed.a.refills `shouldReturn` 1+  readTVarIO bed.b.refills `shouldReturn` 1+  inPool bed tx `shouldReturn` False+  -- what the web server does then: the peers and the node announce their+  -- mempools again, and the member offers what they send+  offer bed tx+  settle bed+  inPool bed tx `shouldReturn` True+  xpubAsk bed.a.door x `shouldReturn` Indexed+  sameAsChain bed bed.a x++-------------------------------------------------------------------------------+-- The model test+-------------------------------------------------------------------------------++-- | A step of a random history.+data Step+  = -- | pay to an address of a key, in a block or in the pool+    Pay !Int !Word32 !Bool+  | -- | spend an output of a key, in a block or in the pool+    Spend !Int !Bool+  | -- | mine what the pool has+    Mine+  | -- | abandon so many blocks for one more+    Reorg !Int+  | -- | ask for the entry of a key+    Build !Int+  deriving (Eq, Show)++instance Arbitrary Step where+  arbitrary =+    frequency+      [ (4, Pay <$> choose (0, 1) <*> choose (0, 27) <*> arbitrary),+        (2, Spend <$> choose (0, 1) <*> arbitrary),+        (2, pure Mine),+        (1, Reorg <$> choose (1, 4)),+        (3, Build <$> choose (0, 1))+      ]++-- | After every step of a history, every entry that the cache has, read+-- through the scripts of Redis by each member, is what the chain at the+-- fleet tip and the pool give.+modelTest :: Bed -> [Step] -> IO ()+modelTest bed steps =+  forM_ steps $ \s -> do+    step s+    forM_ keys $ \x -> forM_ [bed.a, bed.b] $ \m ->+      cachedSeen m x >>= \case+        Nothing -> return ()+        Just seen -> expected bed x >>= \e -> (s, seen) `shouldBe` (s, e)+  where+    keys = [xpub 101, xpub 102]+    key i = keys !! i+    step = \case+      Pay i n inBlock -> payKey bed (key i) [n] inBlock+      Spend i inBlock -> do+        let x = key i+        outs <-+          atTip bed $ \q -> do+            bals <- q.xPubBals x+            q.xPubUnspents x bals def+        let usable = [u.unspent | u <- outs, not inBlock || confirmed u.unspent.block]+        case usable of+          [] -> return ()+          u : _ -> do+            let tx = spend [u] [(script, 1000)]+            if inBlock then mineBlock bed [tx] else offer bed tx+            settle bed+      Mine -> poolTxs bed >>= mineBlock bed >> settle bed+      Reorg n -> do+        height <- best bed >>= Q.getBlock bed.db <&> maybe 0 (.height)+        when (fromIntegral height > n + 1) (reorganise bed n)+      Build i -> void (xpubAsk bed.a.door (key i))+    cachedSeen m x = do+      tip <- fleetTipOf bed+      join <$> answeringCached m tip (\_ sess _ -> seenIn sess x)++-------------------------------------------------------------------------------+-- The test bed+-------------------------------------------------------------------------------++-- | Two members of a fleet on one database and one Redis.+data Bed = Bed+  { conn :: !Redis.Connection,+    prefix :: !ByteString,+    db :: !DatabaseReader,+    a :: !Member,+    b :: !Member,+    salt :: !(IORef Word32)+  }++-- | A new chain with a few blocks of coins, in a new database, and two+-- members on keys of Redis that no other test uses, with their indexers.+withBed :: (Redis.Connection, IO FleetKeys) -> (Bed -> IO x) -> IO x+withBed (conn, fresh) go = do+  keys <- fresh+  withSystemTempDirectory "haskoin-store-fleet-cache" $ \dir ->+    openLayered btcRegTest dir $ \db -> do+      startChain db+      a <- newMember (config keys.prefix "a") conn db+      b <- newMember (config keys.prefix "b") conn db+      salt <- newIORef 0+      let bed = Bed {conn, prefix = keys.prefix, db, a, b, salt}+      withAsync (memberCacheLoop a) $ \_ ->+        withAsync (memberCacheLoop b) $ \_ -> do+          noted bed+          replicateM_ 3 (mineBlock bed [])+          settle bed+          go bed+  where+    config p name =+      MemberConfig+        { prefix = p,+          name,+          lease = 60000,+          depth = 10,+          expiry = 0,+          cacheMax = 100,+          indexer = defaultSettings {workers = 2},+          stats = Nothing,+          note = \_ _ -> return ()+        }++-- | A key of the tests.+xpub :: Word8 -> XPubSpec+xpub n = XPubSpec {key = deriveXPubKey (makeXPrvKey (B.replicate 32 n)), deriv = DeriveNormal}++-- | The best block of the database.+best :: Bed -> IO BlockHash+best bed = Q.getBestBlock bed.db >>= maybe (fail "No best block") return++-- | The fleet tip, as Redis has it.+fleetTipOf :: Bed -> IO ByteString+fleetTipOf bed = memberTip bed.a >>= maybe (fail "No fleet tip") return++-- | Both members take a view of the best block, as the block store has+-- them do after each block.+noted :: Bed -> IO ()+noted bed = best bed >>= \h -> mapM_ (`noteBest` h) [bed.a, bed.b]++-- | Turns of both members until the fleet tip is the best block and both+-- know it.+settle :: Bed -> IO ()+settle bed = go (10 :: Int)+  where+    go n = do+      memberTick bed.a+      memberTick bed.b+      h <- best bed+      tip <- memberTip bed.a+      ta <- readTVarIO bed.a.tip+      tb <- readTVarIO bed.b.tip+      unless (tip == Just (encode h) && ta == Just h && tb == Just h) $+        if n > 0 then go (n - 1) else expectationFailure "The fleet tip did not reach the best block"++-- | Import a block with these transactions on the best block, with a+-- coinbase of its own, so that a block mined again at a height is another.+mineBlock :: Bed -> [Tx] -> IO ()+mineBlock bed txs = do+  n <- atomicModifyIORef' bed.salt (\s -> (s + 1, s))+  h <- best bed+  parent <- Q.getBlock bed.db h >>= maybe (fail "No data for the best block") return+  let height = parent.height + 1+      coinbase = Tx 1 [TxIn nullOutPoint (encode height <> encode n) maxBound] [TxOut 5000000000 script] [] 0+      included = coinbase : map snd (sortTxs txs)+      merkle = either error id (buildMerkleRoot (map txHash included))+      candidate = BlockHeader 0x20000000 h merkle (parent.header.timestamp + 600) parent.header.bits+      header = fromMaybe (error "No nonce") (find (isValidPOW btcRegTest) (map candidate [0 ..]))+  run bed.db (\w -> importBlock w (Block header included) (BlockNode header height 0 h))+    >>= either (fail . show) (void . return)+  noted bed++-- | Abandon so many blocks for one more, empty, and move the tip there.+reorganise :: Bed -> Int -> IO ()+reorganise bed n = do+  replicateM_ n (revert bed.db >> noted bed)+  replicateM_ (n + 1) (mineBlock bed [])+  settle bed++-- | Read with the queries of the view at the fleet tip and the pool.+atTip :: Bed -> (StoreRead -> IO x) -> IO x+atTip bed f =+  answering bed.a (\r pool -> f (pooledRead r pool)) >>= \case+    Answered x -> return x+    _ -> fail "No view of the fleet tip"++-- | Confirmed outputs to the script of the coinbases that the pool does+-- not spend.+coins :: Bed -> IO [Unspent]+coins bed = do+  let coinAddress = either error id (scriptToAddressBS btcRegTest script)+  filter (confirmed . (.block)) <$> atTip bed (\q -> q.getAddressUnspents coinAddress def)++-- | The output script of an address of the external chain of a key.+addressScript :: Bed -> XPubSpec -> Word32 -> IO ByteString+addressScript bed x i = do+  Derived ext _ <- derivedAddrs bed.db x+  maybe (fail "No address") return (lookup i ext >>= addressToScriptBS)++-- | A transaction that spends outputs and pays amounts to scripts, with+-- the change to the script of the coinbases.+spend :: [Unspent] -> [(ByteString, Word64)] -> Tx+spend us outs =+  Tx 1 [TxIn u.outpoint "" maxBound | u <- us] ([TxOut v s | (s, v) <- outs] <> [TxOut change script]) [] 0+  where+    change = sum (map (.value) us) - sum (map snd outs) - 1000++-- | Pay to addresses of a key from a coin, in a block or in the pool, and+-- move the tip.+payKey :: Bed -> XPubSpec -> [Word32] -> Bool -> IO ()+payKey bed x ns inBlock = do+  dests <- mapM (addressScript bed x) ns+  -- a coin that pays for all of it, not the change of a small spend+  filter ((> 100000 * fromIntegral (length ns) + 1000) . (.value)) <$> coins bed >>= \case+    [] -> return ()+    c : _ -> do+      let tx = spend [c] [(d, 100000) | d <- dests]+      if inBlock then mineBlock bed [tx] else offer bed tx+      settle bed++-- | Hand a transaction to member a, as a peer would.+offer :: Bed -> Tx -> IO ()+offer bed tx = do+  now <- round <$> getPOSIXTime+  memberReceived bed.a now tx++-- | Whether the pool has a transaction.+inPool :: Bed -> Tx -> IO Bool+inPool bed tx = atTip bed $ \q -> do+  found <- q.getTxData (txHash tx)+  return (maybe False (not . confirmed . (.block)) found)++-- | The transactions of the pool.+poolTxs :: Bed -> IO [Tx]+poolTxs bed = atTip bed $ \q -> do+  hs <- map snd <$> q.getMempool+  map (.tx) . catMaybes <$> q.getManyTxData hs++-- | Take the dibs of a key for a store that is not a member.+taken :: Lock XPubSpec -> XPubSpec -> Int -> IO ()+taken lock x eta =+  lock.acquire x eta >>= \case+    Took -> return ()+    _ -> expectationFailure "The ghost did not take the dibs"++-- | What a request reads of a key: balances, transactions, unspent outputs+-- and count.+data Seen = Seen ![XPubBal] ![TxRef] ![XPubUnspent] !Word32+  deriving (Eq, Show)++-- | What the cache has of a key, read in one session.+seenIn :: Session -> XPubSpec -> IO (Maybe Seen)+seenIn sess x =+  sess.reads.balances x >>= \case+    Nothing -> return Nothing+    Just bals -> do+      txs <- sess.reads.transactions x def+      outs <- sess.reads.unspents x bals def+      n <- sess.reads.count x+      return (Seen bals <$> txs <*> outs <*> n)++-- | What the chain at the fleet tip and the pool give for a key.+expected :: Bed -> XPubSpec -> IO Seen+expected bed x = atTip bed $ \q -> do+  bals <- q.xPubBals x+  Seen bals <$> q.xPubTxs x bals def <*> q.xPubUnspents x bals def <*> q.xPubTxCount x bals++-- | The cache answers for a key, through a member, what the chain at the+-- fleet tip and the pool give.+sameAsChain :: Bed -> Member -> XPubSpec -> IO ()+sameAsChain bed m x = do+  tip <- fleetTipOf bed+  seen <- join <$> answeringCached m tip (\_ sess _ -> seenIn sess x)+  e <- expected bed x+  seen `shouldBe` Just e
− test/Haskoin/Store/Fleet/MemberSpec.hs
@@ -1,401 +0,0 @@-module Haskoin.Store.Fleet.MemberSpec (spec) where--import Control.Monad (foldM, foldM_, forM_, void, when)-import Data.Default (def)-import Data.Either (isRight)-import Data.IORef (modifyIORef', newIORef, readIORef)-import Data.Maybe (isJust)-import Data.Serialize (encode)-import Data.Text qualified as T-import Data.Word (Word16)-import Haskoin (btcRegTest, prepareContext, txHash)-import Haskoin.Node (LogLevel (..))-import Haskoin.Store.Common (Limits (..), Start (..), StoreRead)-import Haskoin.Store.Database.Query qualified as Q-import Haskoin.Store.Database.Query.Layered (From (..))-import Haskoin.Store.Database.Query.Pooled (pooledRead)-import Haskoin.Store.Database.Views (closeViews)-import Haskoin.Store.Fleet.Member-import Haskoin.Store.Fleet.Redis (FleetKeys (..), fleetRestarts, poolWithdraw)-import Haskoin.Store.Fleet.RedisSpec (withRedis)-import Haskoin.Store.Fleet.XPub-import Haskoin.Store.Histories-import Haskoin.Store.Integration.Podman (podmanUsable)-import Test.Hspec-import Test.Hspec.QuickCheck (modifyMaxSuccess)-import Test.QuickCheck (forAll, ioProperty, property, sized, vectorOf)-import UnliftIO (withSystemTempDirectory)--spec :: Spec-spec = prepareContext $ \ctx -> do-  usable <- runIO podmanUsable-  describe "member of a fleet" $ case usable of-    Left why -> it "answers as the store does when it is the whole fleet" (pendingWith why)-    Right () -> aroundAll withRedis $ do-      modifyMaxSuccess (const 20) $-        it "answers as the store does when it is the whole fleet" $ \(conn, fresh) ->-          -- A store alone in its fleet, through a random history of-          -- payments, replacements, blocks and reorganisations. The-          -- history is chosen from a database of the layout of version 20-          -- that goes through the same, since that is what 'plan' reads. After each-          -- step it takes its turn as a member, and then every answer that-          -- it gives from the view at the fleet tip with the pool of Redis-          -- on top must be the answer that it gives from its own database.-          property . forAll (vectorOf 60 anyNumber) $ \seeds -> ioProperty $-            withSystemTempDirectory "haskoin-store-member-" $ \dir ->-              openLayered btcRegTest ctx dir $ \db -> withShadow btcRegTest ctx $ \shadow -> do-                keys <- fresh-                let config =-                      MemberConfig-                        { prefix = keys.prefix,-                          name = "a",-                          lease = 60000,-                          depth = 6,-                          expiry = 0,-                          cacheMax = 0,-                          cacheMinAddrs = 0,-                          cacheMinTxs = 0,-                          stats = Nothing,-                          note = \_ _ -> return ()-                        }-                    best = Q.getBestBlock db >>= maybe (fail "no best block") return-                m <- newMember config conn db-                hist0 <- start layeredRules db-                best >>= noteBest m-                memberTick m-                let turn hist seed =-                      plan shadow hist seed >>= \case-                        (hist', Nothing) -> return hist'-                        (hist', Just op) -> do-                          void (apply mixedRules shadow op)-                          out <- apply layeredRules db op-                          let hist'' = record db hist' op out-                          -- the store tells its member what it deletes-                          memberForget m out.removed-                          when (isRight out.result) $ case op of-                            Mine _ _ -> best >>= noteBest m-                            Revert _ _ -> best >>= noteBest m-                            -- a store deletes what is too old, and the pool-                            -- does the same by its own clock-                            Delete _ -> void (poolWithdraw conn keys 32 (map encode out.removed))-                            Pay _ _ -> return ()-                          memberTick m-                          own <- layered db (ask hist'')-                          answering m (\r pool -> ask hist'' (pooledRead r pool)) >>= \case-                            Answered shared -> (show op, shared) `shouldBe` (show op, own)-                            Alone -> expectationFailure "the fleet has no tip"-                            Behind -> expectationFailure "the store has no view of the fleet tip"-                          return hist''-                foldM_ turn hist0 seeds-                closeViews m.views-      modifyMaxSuccess (const 20) $-        it "answers about a wallet from the cache as it does from the database" $ \(conn, fresh) ->-          -- The same store and histories, with the cache of extended-          -- public keys on. After each step, every question about the-          -- wallet of the histories is asked of the view at the fleet tip-          -- with the pool on top, and again with the cache in front: the-          -- answers must be the same, whether the cache has the wallet,-          -- has it with other addresses, or does not have it. The wallet-          -- is cached after some steps and not after others, so that-          -- blocks and payments find it in the cache and out of it. Right-          -- after it is cached, the cache must be what answers, and only-          -- to a caller that has the wallet with as many addresses.-          ---          -- At one step in five the fleet tip is moved by the same store-          -- taking its turn without the cache, as a store does that stops-          -- between moving the tip and bringing the cache along. The-          -- cache is then at another block than the tip, and must not-          -- answer.-          ---          -- The first history is always 'leftAndBack'.-          let histories = sized (\n -> if n == 0 then return leftAndBack else vectorOf 60 anyNumber)-           in property . forAll histories $ \seeds -> ioProperty $-                withSystemTempDirectory "haskoin-store-member-" $ \dir ->-                  openLayered btcRegTest ctx dir $ \db -> withShadow btcRegTest ctx $ \shadow -> do-                    keys <- fresh-                    said <- newIORef []-                    let config =-                          MemberConfig-                            { prefix = keys.prefix,-                              name = "a",-                              lease = 60000,-                              depth = 6,-                              expiry = 0,-                              cacheMax = 10,-                              cacheMinAddrs = 0,-                              cacheMinTxs = 0,-                              stats = Nothing,-                              note = \level line -> modifyIORef' said ((level, line) :)-                            }-                        best = Q.getBestBlock db >>= maybe (fail "no best block") return-                        key = wallet ctx-                    m <- newMember config conn db-                    -- the same store, as it takes its turns without the cache-                    bare <- newMember (config {cacheMax = 0} :: MemberConfig) conn db-                    hits <- newIORef (0 :: Int)-                    hist0 <- start layeredRules db-                    best >>= \b -> noteBest m b >> noteBest bare b-                    memberTick m-                    let tipNow = memberTip m >>= maybe (fail "the fleet has no tip") return-                        inFleet :: (StoreRead -> StoreRead -> IO a) -> IO a-                        inFleet action = do-                          tip <- tipNow-                          answeringCached m tip (\plain cached _ -> action plain cached)-                            >>= maybe (fail "the store has no view of the fleet tip") return-                        turn hist seed =-                          plan shadow hist seed >>= \case-                            (hist', Nothing) -> return hist'-                            (hist', Just op) -> do-                              void (apply mixedRules shadow op)-                              out <- apply layeredRules db op-                              let hist'' = record db hist' op out-                              memberForget m out.removed-                              memberForget bare out.removed-                              when (isRight out.result) $ case op of-                                Mine _ _ -> best >>= \b -> noteBest m b >> noteBest bare b-                                Revert _ _ -> best >>= \b -> noteBest m b >> noteBest bare b-                                Delete _ -> void (poolWithdraw conn keys 32 (map encode out.removed))-                                Pay _ _ -> return ()-                              if seed `mod` 5 == 0-                                then do-                                  -- the tip moves, if the step made a block,-                                  -- and the cache stays where it was: not in-                                  -- every history, but in most-                                  memberTick bare-                                  -- the member with the cache learns the tip-                                  memberTick m-                                else memberTick m-                              let made = map txHash hist''.made-                                  pick xs = xs !! (fromIntegral seed `mod` length xs)-                                  starts =-                                    [Nothing, Just (AtBlock (fromIntegral seed `mod` 8))]-                                      <> [Just (AtTx (pick made)) | not (null made)]-                                  limits =-                                    [ Limits {limit, offset, start = from}-                                    | from <- starts,-                                      (limit, offset) <- [(0, 0), (2, 0), (3, fromIntegral seed `mod` 3)]-                                    ]-                                  asked =-                                    [WalletBalances, WalletCount]-                                      <> map WalletTxs limits-                                      <> map WalletUnspents limits-                              (direct, cached) <- inFleet $ \plain cached ->-                                (,) <$> mapM (answer plain) asked <*> mapM (answer cached) asked-                              (show op, differences asked direct cached) `shouldBe` (show op, [])-                              -- cache the wallet after two steps of three-                              when (seed `mod` 3 /= 0) $ do-                                tip <- tipNow-                                built <- answeringAt m tip (\r pool _ -> cacheBuild conn keys tip r pool 1 key)-                                at <- cacheTip conn keys-                                if at /= Just tip-                                  then do-                                    -- the cache is behind the tip: nothing is-                                    -- written to it, and it does not answer-                                    built `shouldSatisfy` (`elem` [Just Abandoned, Just (Built 0)])-                                    cachedEntry conn keys tip 20 32 1 key `shouldReturn` Miss-                                  else do-                                    built `shouldSatisfy` \case-                                      Just (Built _) -> True-                                      _ -> False-                                    cachedEntry conn keys tip 20 32 1 key >>= \case-                                      Hit e -> do-                                        modifyIORef' hits (+ 1)-                                        let (n0, n1) = e.sizes-                                            isHit = \case-                                              Hit _ -> True-                                              _ -> False-                                        cachedTxs conn keys tip key (n0, n1) Top def >>= (`shouldSatisfy` isHit)-                                        cachedCount conn keys tip key (n0, n1) >>= (`shouldSatisfy` isHit)-                                        -- not for a caller with other addresses-                                        cachedTxs conn keys tip key (n0 + 1, n1) Top def `shouldReturn` Miss-                                        cachedCount conn keys tip key (n0, n1 + 1) `shouldReturn` Miss-                                        let more = e {sizes = (n0 + 1, n1)}-                                        cachedUnspents conn keys tip key more Top def `shouldReturn` Miss-                                      other -> expectationFailure ("the cache did not answer: " <> show other)-                                (direct', cached') <- inFleet $ \plain through ->-                                  (,) <$> mapM (answer plain) asked <*> mapM (answer through) asked-                                (show op, differences asked direct' cached') `shouldBe` (show op, [])-                              return hist''-                    foldM_ turn hist0 seeds-                    -- the least recently used entries go when there are too-                    -- many, and nothing of them is left-                    memberTick m-                    memberTick m-                    tip <- tipNow-                    cacheTip conn keys `shouldReturn` Just tip-                    _ <- answeringAt m tip (\r pool _ -> cacheBuild conn keys tip r pool 1 key)-                    -- its size is one entry, with the addresses of the wallet-                    (n0, n1) <- cacheHave conn keys key-                    cacheSize conn keys `shouldReturn` (1, n0 + n1)-                    cacheEvict conn keys 1 `shouldReturn` 0-                    cacheEvict conn keys 0 `shouldReturn` 1-                    cacheSize conn keys `shouldReturn` (0, 0)-                    cachedEntry conn keys tip 20 32 1 key `shouldReturn` Miss-                    cacheHave conn keys key `shouldReturn` (0, 0)-                    closeViews m.views-                    closeViews bare.views-                    -- what the member logged: that it emptied the cache, and-                    -- why, as information, and nothing else above debugging-                    logged <- readIORef said-                    let emptied = [line | (LevelInfo, line) <- logged]-                    [() | (level, _) <- logged, level /= LevelInfo, level /= LevelDebug] `shouldBe` []-                    emptied `shouldSatisfy` all ("Emptied the xpub cache at block " `T.isPrefixOf`)-                    readIORef hits >>= (`shouldSatisfy` (> 10))-      it "leaves behind a tip that is too far for every store, and fills the pool again" $ \(conn, fresh) ->-        -- Two stores of one fleet, both live and both with a view of the-        -- tip, take no turn while their chain grows by more blocks than a-        -- store goes through to say what they did to the pool: as stores do-        -- that are behind, whose health checks fail until they have caught-        -- up. When they take their turns again, the tip is where it was.-        ---        -- No store will ever say the way from there, and neither is alone,-        -- which is what Redis asked of a store that empties the pool: the-        -- tip stayed there for good, and every request was answered that-        -- the store was behind its fleet. The integration tests met this-        -- when the health check began to run at every block, which made-        -- stores stop taking turns sooner than they used to.-        ---        -- Now the first to take a turn moves the tip to their best block-        -- and empties the pool, and both offer their mempools again: what-        -- they answer afterwards is what the store answers by itself.-        withSystemTempDirectory "haskoin-store-member-" $ \dir ->-          openLayered btcRegTest ctx dir $ \db -> withShadow btcRegTest ctx $ \shadow -> do-            keys <- fresh-            said <- newIORef []-            let config name =-                  MemberConfig-                    { prefix = keys.prefix,-                      name,-                      lease = 60000,-                      depth = 6,-                      expiry = 0,-                      cacheMax = 0,-                      cacheMinAddrs = 0,-                      cacheMinTxs = 0,-                      stats = Nothing,-                      note = \level line -> modifyIORef' said ((level, line) :)-                    }-                best = Q.getBestBlock db >>= maybe (fail "no best block") return-            a <- newMember (config "a") conn db-            b <- newMember (config "b") conn db-            let both f = f a >> f b-                done hist op = do-                  void (apply mixedRules shadow op)-                  out <- apply layeredRules db op-                  both (`memberForget` out.removed)-                  return (record db hist op out)-                pay hist seed =-                  plan shadow hist seed >>= \case-                    (hist', Nothing) -> return hist'-                    (hist', Just op) -> done hist' op-                mine hist = do-                  hist' <- blockOn db [] >>= done hist-                  -- as the block store does after each block-                  best >>= \h -> both (`noteBest` h)-                  return hist'-                same hist = do-                  own <- layered db (ask hist)-                  forM_ [a, b] $ \m ->-                    answering m (\r pool -> ask hist (pooledRead r pool)) >>= \case-                      Answered shared -> shared `shouldBe` own-                      Alone -> expectationFailure "the fleet has no tip"-                      Behind -> expectationFailure "a store has no view of the fleet tip"-            hist0 <- start layeredRules db-            best >>= \h -> both (`noteBest` h)-            both memberTick-            -- payments that both stores have, and that the pool takes-            hist1 <- foldM pay hist0 (onlyPayments [100, 2000, 30000, 4000, 500, 60000])-            both memberTick-            old <- memberTip a-            old `shouldSatisfy` isJust-            same hist1-            Q.getMempool db >>= (`shouldSatisfy` (not . null))-            -- more blocks than a store goes through, and no turn taken-            hist2 <- foldM (\hist _ -> mine hist) hist1 [1 .. farthest + 5]-            restarts <- fleetRestarts conn keys-            memberTip a `shouldReturn` old-            both memberTick-            new <- encode <$> best-            memberTip a `shouldReturn` Just new-            fleetRestarts conn keys `shouldReturn` restarts + 1-            -- the pool was emptied, and has again what the stores have-            both memberTick-            same hist2-            Q.getMempool db >>= (`shouldSatisfy` (not . null))-            closeViews a.views-            closeViews b.views-            -- said once, by the store that did it, as a warning-            logged <- readIORef said-            [line | (LevelWarn, line) <- logged] `shouldSatisfy` \case-              [line] -> "starting the shared mempool again" `T.isInfixOf` line-              _ -> False---- | A history that a run of the tests found: a block is mined, reverted and--- mined again while the cache is not brought along, so that the fleet tip--- comes back to the block that the cache was at. The cache then looked--- current, with entries that the pool had taken the transactions of that--- block out of on the way. It is why the cache is at a move of the tip and--- not at a block.-leftAndBack :: [Word16]-leftAndBack =-  [ 2944,-    33599,-    29992,-    62497,-    57974,-    57422,-    44271,-    60202,-    57257,-    19310,-    53832,-    39324,-    44849,-    28951,-    37975,-    2064,-    15159,-    34699,-    16471,-    3642,-    14550,-    867,-    22752,-    14436,-    39637,-    398,-    25449,-    11005,-    20967,-    47441,-    46443,-    63224,-    4787,-    36990,-    56953,-    1543,-    36370,-    24815,-    19050,-    39880,-    16054,-    22273,-    62670,-    13555,-    30149,-    1048,-    1444,-    45704,-    39517,-    1970,-    43989,-    15518,-    54685,-    58539,-    63687,-    1789,-    3783,-    27016,-    43138,-    28658-  ]
test/Haskoin/Store/Fleet/RedisSpec.hs view
@@ -86,6 +86,28 @@         fleetAdvance conn keys 4 Nothing (Just (block 1)) (block 2) (Just confirms) [] `shouldReturn` MovedTo (block 2)         poolSpends conn keys `shouldReturn` []         poolBody conn keys (block 7) `shouldReturn` Nothing+      it "sees what the pool decided on, and keeps the newest" $ \(conn, fresh) -> do+        keys <- fresh+        fleetReport conn keys Nothing 60000 "a" [block 1] [] `shouldReturn` True+        fleetMove conn keys Nothing Nothing `shouldReturn` MovedTo (block 1)+        let at = Just (block 1)+        -- taken, known and in conflict are seen; late and orphans are not+        poolOffer conn keys 4 at (txOf 7 [out 0 0]) [True] [] `shouldReturn` Taken+        poolOffer conn keys 4 at (txOf 7 [out 0 0]) [True] [] `shouldReturn` Known+        poolOffer conn keys 4 at (txOf 8 [out 0 0]) [True] [] `shouldReturn` Conflict [block 7]+        poolOffer conn keys 4 Nothing (txOf 9 [out 0 1]) [True] [] `shouldReturn` Late+        poolOffer conn keys 4 at (txOf 10 [out 5 0]) [False] [] `shouldReturn` Orphan [out 5 0]+        fleetSeen conn keys (map block [7 .. 10]) `shouldReturn` [True, True, False, False]+        -- withdrawn is seen+        poolOffer conn keys 4 at (txOf 11 [out 0 2]) [True] [] `shouldReturn` Taken+        poolWithdraw conn keys 4 [block 11] `shouldReturn` [block 11]+        fleetSeen conn keys [block 11] `shouldReturn` [True]+        fleetSeen conn keys [] `shouldReturn` []+        -- a bound of 20 is trimmed past 21, to the newest+        forM_ [100 .. 129] $ \i -> poolOffer conn keys 4 at (txOf i [out 1 i]) [True] [] `shouldReturn` Taken+        fleetTrimSeen conn keys 40 `shouldReturn` 33+        fleetTrimSeen conn keys 20 `shouldReturn` 20+        drop 10 <$> fleetSeen conn keys (map block [100 .. 129]) `shouldReturn` replicate 20 True       it "tells what happens to the pool and to the tip, in the order in which it happens" $ \(conn, fresh) -> do         keys <- fresh         fleetReport conn keys Nothing 60000 "a" [block 1] [] `shouldReturn` True
test/Haskoin/Store/Fleet/XPubSpec.hs view
@@ -58,18 +58,24 @@             final m = maximum [n | (m', n, _) <- rows found, m' == m]             lacking m = [r | r@(m', n, _) <- rows found, m' /= m || n /= final m]          in conjoin [window igap gap (lacking m) === Nothing | m <- [0, 1], any (\(m', _, _) -> m' == m) (rows found)]-    it "gives nothing when the entry has an address more" $-      property . forAll key $ \k@(igap, gap, ext, chg) ->-        let found = scan k-            next m = length [() | (m', _, _) <- rows found, m' == m]-            more m used = rows found <> [(m, fromIntegral (next m), balance used m (fromIntegral (next m)))]-            -- one more is another key, unless the scan read no change-            -- addresses at all and the one more is the first of them-            changeless = next 1 == 0-         in conjoin-              [ window igap gap (more 0 ext) === Nothing,-                changeless .||. window igap gap (more 1 chg) === Nothing-              ]+    -- one more address, right after what the scan reads in a chain+    let past k m used =+          let found = scan k+              next = fromIntegral (length [() | (m', _, _) <- rows found, m' == m])+              b = zeroBalance (address m next)+           in (found, rows found <> [(m, next, if used then b {txs = 1, received = 5} else b)])+    it "gives the scan when the entry has an unused address more, its lookahead" $+      property . forAll key $ \k@(igap, gap, _, _) ->+        conjoin+          [ let (found, more) = past k m False in window igap gap more === Just found+          | m <- [0, 1]+          ]+    it "gives nothing when an address past what the scan reads is used" $+      property . forAll key $ \k@(igap, gap, _, _) ->+        conjoin+          [ let (_, more) = past k m True in window igap gap more === Nothing+          | m <- [0, 1]+          ]     it "gives nothing when a chain has a hole" $       property . forAll key $ \k@(igap, gap, _, _) ->         let found = scan k
test/Haskoin/Store/Histories.hs view
@@ -1,777 +1,83 @@--- | Random histories of a store, for tests that compare two ways of keeping--- its database: the layout in use today, where the chain and the mempool--- share their indices, and the layered layout of @docs/layered-database.md@.------ A history is a sequence of operations: transactions for the mempool,--- blocks, deletions and reversals, some of them invalid. Each is chosen from--- a number and from what a database of today's layout holds at that point,--- so that the same history can be given to both sets of rules.------ Payments go to a handful of fixed scripts and to a few addresses of an--- extended public key, 'wallet', so that the queries about extended public--- keys have something to find.+-- | Databases for tests, and blocks to put in them: what the tests of the+-- database, of the block import and of the views share. The database holds+-- the chain alone; the mempool is the pool of the fleet, in Redis. module Haskoin.Store.Histories-  ( -- * Databases and rules-    open20,+  ( -- * Databases     openLayered,-    withShadow,-    Rules (..),-    mixedRules,-    layeredRules,-    run,--    -- * Histories-    History (..),-    Op (..),-    Outcome (..),-    start,-    plan,-    apply,-    record,-    step,-    blockOn,-    coinOf,-    payments,-    anyNumber,-    onlyPayments,-    p2pkh,-    hashLock,-    addressesOf,-    wallet,--    -- * Answers-    mixed,-    layered,-    pooled,-    Answers (..),-    ask,--    -- * Pages-    Question (..),-    Reply (..),-    questions,-    answer,-    differences,-    startOf,-    startsKnown,-    size,--    -- * Column families-    Chain (..),-    chainOf,+    openOlder,     dump,++    -- * Blocks+    run,+    startChain,+    mine,+    revert,   ) where +import Control.Exception (try) import Control.Monad (void)-import Data.Bifunctor (first)-import Data.Bits (shiftR) import Data.ByteString (ByteString)-import Data.ByteString qualified as B-import Data.Default (def)-import Data.Either (isLeft, rights)-import Data.Hashable (hashWithSalt)-import Data.IntMap.Strict (IntMap)-import Data.IntMap.Strict qualified as IntMap-import Data.List (sort, sortOn)-import Data.Maybe (catMaybes, fromMaybe, mapMaybe)-import Data.Serialize (decode, encode, getWord64be, runGet)-import Data.Word (Word16, Word32, Word64, Word8)-import Database.RocksDB (ColumnFamily, iterEntry, iterFirst, iterNext, withIterCF)-import Haskoin-  ( Address,-    Block (..),-    BlockHash,-    BlockHeader (..),-    BlockHeight,-    BlockNode (..),-    Ctx,-    Network,-    OutPoint (..),-    Tx (..),-    TxHash (..),-    TxIn (..),-    TxOut (..),-    addressToScriptBS,-    deriveXPubKey,-    headerHash,-    makeXPrvKey,-    nullOutPoint,-    pubSubKey,-    scriptToAddressBS,-    sha256,-    txHash,-  )+import Database.RocksDB (ColumnFamily, Config (..), iterEntry, iterFirst, iterNext, withIterCF)+import Haskoin (Block (..), BlockHash, Network, Tx, headerHash) import Haskoin.Node (LogLevel (..), Logger (..))-import Haskoin.Store.Common-  ( Limits (..),-    Start (..),-    StoreRead (..),-    deriveAddresses,-    deriveFunction,-    sortTxs,-  )-import Haskoin.Store.Data-  ( Balance (..),-    BlockData (..),-    BlockRef (..),-    DeriveType (..),-    Prev (..),-    Spender (..),-    TxData (..),-    TxRef (..),-    UnixTime,-    Unspent (..),-    XPubBal (..),-    XPubSpec (..),-    XPubUnspent (..),-    confirmed,-    zeroBalance,-  )+import Haskoin.Store.Data (BlockData (..)) import Haskoin.Store.Database.Query qualified as Q-import Haskoin.Store.Database.Query.Layered qualified as L-import Haskoin.Store.Database.Query.Pooled qualified as P import Haskoin.Store.Database.Reader   ( DatabaseReader (..),-    addrOutCF,-    addrTxCF,-    balanceCF,-    spenderCF,-    txCF,-    unspentCF,-    withDatabaseReader,+    layeredFamilyConfig,+    mergeGuardFamily,     withLayeredReader,+    withReader,   )-import Haskoin.Store.Database.Types (BalVal (..), SpenderKey (..), SpenderVal (..))-import Haskoin.Store.Database.Writer (MempoolChanges (..), Writer, runWriterMempool)-import Haskoin.Store.Logic (ImportException)-import Haskoin.Store.Logic qualified as Mixed-import Haskoin.Store.Logic.Layered qualified as Layered-import Haskoin.Store.TestBlocks (nextBlock, script)-import Test.QuickCheck (Gen, choose, elements, frequency, vectorOf)-import UnliftIO (try, withSystemTempDirectory)------------------------------------------------------------------------------------ Databases and rules------------------------------------------------------------------------------------ | Open a database as the store does today, and as the layered layout--- needs it.-open20, openLayered :: Network -> Ctx -> FilePath -> (DatabaseReader -> IO a) -> IO a-open20 net ctx dir = withDatabaseReader net ctx 20 32 dir False Nothing Nothing Nothing Nothing Nothing-openLayered net ctx dir = withLayeredReader net ctx 20 32 dir False Nothing Nothing Nothing Nothing Nothing---- | Run an action with a new database of the layout of version 20 that has--- the start of a history ('start'), in a directory of its own.------ 'plan' chooses the next operation of a history by reading a database of--- that layout: what is unspent, what the mempool has. Given a database of--- the layered layout it finds almost none of the mempool, and its blocks--- confirm next to nothing. So a test of the layered layout keeps one of--- these beside its own database, plans from it, and applies each--- operation to both, as 'mixedRules' and 'layeredRules' take the same--- operations and give the same outcomes.-withShadow :: Network -> Ctx -> (DatabaseReader -> IO a) -> IO a-withShadow net ctx action =-  withSystemTempDirectory "haskoin-store-shadow-" $ \dir ->-    open20 net ctx dir $ \shadow -> start mixedRules shadow >> action shadow---- | The rules that import blocks and transactions, for one layout.-data Rules = Rules-  { initBest :: Writer -> IO (),-    importBlock :: Writer -> Block -> BlockNode -> IO (BlockData, [TxData]),-    revertBlock :: Writer -> UnixTime -> BlockHash -> IO (),-    newMempoolTx :: Writer -> Tx -> UnixTime -> IO Bool,-    deleteUnconfirmedTx :: Writer -> Bool -> TxHash -> IO ()-  }---- | The rules of the layout in use today.-mixedRules :: Rules-mixedRules =-  Rules-    { initBest = Mixed.initBest,-      importBlock = Mixed.importBlock,-      revertBlock = Mixed.revertBlockAt,-      newMempoolTx = Mixed.newMempoolTx,-      deleteUnconfirmedTx = Mixed.deleteUnconfirmedTx-    }---- | The rules of the layered layout.-layeredRules :: Rules-layeredRules =-  Rules-    { initBest = Layered.initBest,-      importBlock = Layered.importBlock,-      revertBlock = Layered.revertBlock,-      newMempoolTx = Layered.newMempoolTx,-      deleteUnconfirmedTx = Layered.deleteUnconfirmedTx-    }---- | Run a batch of writes, and give what it returned with how it changed--- the mempool, or the reason why it wrote nothing.-run :: DatabaseReader -> (Writer -> IO a) -> IO (Either ImportException (a, MempoolChanges))-run db = try . runWriterMempool db.net db.ctx db quiet-  where-    quiet = Logger {level = LevelError, source = "", push = \_ -> return ()}------------------------------------------------------------------------------------ Histories------------------------------------------------------------------------------------ | What happened to a database: its main chain, newest block first, every--- transaction ever made for it, taken or not, coinbases included, the--- outputs made that have no address, which no index of addresses lists, the--- time to give the next mempool transaction, and the scripts that payments--- go to.-data History = History-  { chain :: ![(Block, BlockNode)],-    made :: ![Tx],-    loose :: ![OutPoint],-    clock :: !UnixTime,-    scripts :: ![ByteString]-  }---- | Something that happens to a store.-data Op-  = -- | a transaction arrives for the mempool, at a time-    Pay !Tx !UnixTime-  | -- | a block arrives, on the best block-    Mine !Block !BlockNode-  | -- | a mempool transaction is deleted, with its descendants-    Delete !TxHash-  | -- | the best block is reverted, at a time-    Revert !BlockHash !UnixTime-  deriving (Show)+import Haskoin.Store.Database.Writer (Writer, runWriter)+import Haskoin.Store.Logic (ImportException, importBlock, initBest, revertBlock)+import Haskoin.Store.TestBlocks (nextBlock) --- | What an operation did: what the rules returned or why they refused, and--- the transactions that entered and left the mempool.-data Outcome = Outcome-  { result :: !(Either ImportException String),-    added :: ![TxHash],-    removed :: ![TxHash]-  }-  deriving (Eq, Show)+-- | Open a database as the store does.+openLayered :: Network -> FilePath -> (DatabaseReader -> IO a) -> IO a+openLayered net dir = withLayeredReader net 20 32 dir False Nothing Nothing Nothing Nothing Nothing Nothing --- | Give an operation to a set of rules.-apply :: Rules -> DatabaseReader -> Op -> IO Outcome-apply rules db = \case-  Pay tx time -> outcome <$> run db (\w -> rules.newMempoolTx w tx time)-  -- the data of the transactions is left out: it comes with the spenders-  -- that each layout keeps in a body-  Mine block node -> outcome . fmap (first fst) <$> run db (\w -> rules.importBlock w block node)-  Delete h -> outcome <$> run db (\w -> rules.deleteUnconfirmedTx w False h)-  Revert h time -> outcome <$> run db (\w -> rules.revertBlock w time h)+-- | Open a database as a store of database version 22 did: without merge+-- operators, and without the column family that is there since version 23+-- to keep such a store out. It is how the tests make a database that such+-- a store left, and how they try what such a store would do to a newer one.+openOlder :: Network -> FilePath -> (DatabaseReader -> IO a) -> IO a+openOlder net dir =+  withReader older (\_ -> return ()) net 20 32 dir False Nothing Nothing Nothing Nothing Nothing Nothing   where-    outcome :: (Show a) => Either ImportException (a, MempoolChanges) -> Outcome-    outcome (Left e) = Outcome {result = Left e, added = [], removed = []}-    outcome (Right (x, c)) = Outcome {result = Right (show x), added = sort c.added, removed = sort c.removed}+    older bloom cached =+      [ (name, config {mergeOperator = Nothing})+      | (name, config) <- layeredFamilyConfig bloom cached,+        name /= mergeGuardFamily+      ] --- | Note what an operation did in a history. Transactions are noted whether--- they were taken or not, so that questions are asked about all of them.-record :: DatabaseReader -> History -> Op -> Outcome -> History-record db hist op out = case op of-  Pay tx _ -> noted [tx]-  Mine block node-    | isLeft out.result -> noted block.txs-    | otherwise -> (noted block.txs) {chain = (block, node) : hist.chain}-  Delete _ -> hist-  Revert _ _-    | isLeft out.result -> hist-    | otherwise -> hist {chain = drop 1 hist.chain}+-- | Run a batch of writes, and give what it returned, or the reason why it+-- wrote nothing.+run :: DatabaseReader -> (Writer -> IO a) -> IO (Either ImportException a)+run db = try . runWriter db.net db quiet   where-    known = map txHash hist.made-    noted txs =-      let new = [tx | tx <- txs, txHash tx `notElem` known]-          unlisted =-            [ OutPoint (txHash tx) i-            | tx <- new,-              (i, o) <- zip [0 ..] tx.outputs,-              isLeft (scriptToAddressBS db.net db.ctx o.script)-            ]-       in hist {made = new <> hist.made, loose = unlisted <> hist.loose}+    quiet = Logger {level = LevelError, source = "", push = \_ -> return ()} --- | A database with the genesis block and one block with a coin, under a--- set of rules, and its history so far.-start :: Rules -> DatabaseReader -> IO History-start rules db = do-  void (run db rules.initBest) :: IO ()-  let hist = History {chain = [], made = [], loose = [], clock = 1000, scripts = places db.ctx}-  block <- blockOn db []-  record db hist block <$> apply rules db block+-- | Give a new database its genesis block.+startChain :: DatabaseReader -> IO ()+startChain db = run db initBest >>= either (fail . show) return --- | A block with these transactions on the best block of a database.-blockOn :: DatabaseReader -> [Tx] -> IO Op-blockOn db txs = do+-- | Import a block with these transactions on the best block, and give its+-- hash.+mine :: DatabaseReader -> [Tx] -> IO BlockHash+mine db txs = do   best <- Q.getBestBlock db >>= maybe (fail "no best block") return   parent <- Q.getBlock db best >>= maybe (fail "no data for the best block") return-  return (uncurry Mine (nextBlock db.net parent.height parent.header txs))---- | The output of the coinbase of the best block.-coinOf :: History -> OutPoint-coinOf hist = case hist.chain of-  (Block {txs = coinbase : _}, _) : _ -> OutPoint (txHash coinbase) 0-  _ -> error "no block with a coinbase"---- | The scripts that the payments of a history go to: one that has no--- address on Bitcoin, that of every coinbase, four more of the same kind,--- and those of five addresses of the 'wallet'.-places :: Ctx -> [ByteString]-places ctx = hashLock : script : map p2pkh [1 .. 4] <> mapMaybe (addressToScriptBS ctx) paid-  where-    spec = wallet ctx-    branch m = map snd (deriveAddresses (deriveFunction ctx spec.deriv) (pubSubKey ctx spec.key m) 0)-    -- with addresses that are never paid between them, so that whether the-    -- later ones are within the gap depends on which earlier ones are used-    paid = map (branch 0 !!) [0, 2, 5] <> map (branch 1 !!) [0, 2]---- | An extended public key. Five of its addresses are paid in histories:--- three that it gives out to be paid, and two for its change.-wallet :: Ctx -> XPubSpec-wallet ctx = XPubSpec {key = deriveXPubKey ctx (makeXPrvKey seed), deriv = DeriveNormal}-  where-    seed = "the wallet of the histories of the tests of haskoin-store" :: ByteString--p2pkh :: Word8 -> ByteString-p2pkh n = "\x76\xa9\x14" <> B.replicate 20 n <> "\x88\xac"---- | A script that has no address on Bitcoin.-hashLock :: ByteString-hashLock = "\xaa\x20" <> B.replicate 32 0xbb <> "\x87"---- | The addresses of the scripts of a history that have one on a network.-addressesOf :: Network -> Ctx -> History -> [Address]-addressesOf net ctx hist = rights (map (scriptToAddressBS net ctx) hist.scripts)---- | Eight payments, each from the change of the one before, starting from--- a coin: a chain in the mempool.-payments :: OutPoint -> [Tx]-payments coin = take 8 (go coin 5000000000)-  where-    go from value =-      let left = value - 10000-          tx = Tx 2 [TxIn from "" maxBound] [TxOut left script, TxOut 9000 (p2pkh 3)] [] 0-       in tx : go (OutPoint (txHash tx) 0) left---- | A number for 'plan', any of them as likely as any other. QuickCheck's--- own generator of numbers starts with small ones, which would make the--- first histories of every run nothing but the same payment.-anyNumber :: Gen Word16-anyNumber = choose (minBound, maxBound)---- | Turn numbers into numbers that 'plan' reads as payments.-onlyPayments :: [Word16] -> [Word16]-onlyPayments ps = [p - p `mod` 16 + p `mod` 6 | p <- ps]---- | Choose and carry out one operation under the rules of today's layout,--- which is the layout that 'plan' reads.-step :: DatabaseReader -> History -> Word16 -> IO History-step db hist seed =-  plan db hist seed >>= \case-    (hist', Nothing) -> return hist'-    (hist', Just op) -> record db hist' op <$> apply mixedRules db op---- | Choose the next operation of a history from a number, given a database--- of today's layout that has gone through the history so far. Out of--- sixteen: six are payments between the scripts of the history, from outputs--- that are unspent, in a block or in the mempool, a few of them with ten--- outputs and the others with up to three; one is a transaction that--- must be refused; one conflicts with a mempool transaction; four are--- blocks, with part of the mempool, with a transaction that the mempool has--- not seen, or with one that conflicts with the mempool; two are deletions;--- one is the reversal of the best block; and one sends an earlier--- transaction again. The clock moves on with some of them, so that mempool--- transactions share a time now and then.-plan :: DatabaseReader -> History -> Word16 -> IO (History, Maybe Op)-plan db hist0 seed = (hist,) <$> chosen-  where-    hist = hist0 {clock = hist0.clock + fromIntegral (seed `mod` 2)}-    kind = seed `mod` 16-    s = fromIntegral seed `div` 16 :: Int-    chosen-      | kind <= 5 = pay-      | kind == 6 = refused-      | kind == 7 = rival (\tx -> return (Just (Pay tx hist.clock)))-      | kind <= 9 = block . shuffled =<< some-      | kind == 10 = rival (\tx -> Just <$> blockOn db [tx]) >>= maybe (block []) (return . Just)-      | kind == 11 = unseen-      | kind == 12 = delete-      | kind == 13 = deleteOther-      | kind == 14 = revert-      | otherwise = again-    rotate k xs = let k' = k `mod` max 1 (length xs) in drop k' xs <> take k' xs-    target j = hist.scripts !! ((s `div` 11 + j) `mod` length hist.scripts)-    spending us values =-      let outputs = [TxOut v (target j) | (j, v) <- zip [0 ..] values]-       in Tx 2 [TxIn u.outpoint "" signal | u <- us] outputs [] 0-    -- some signal that they can be replaced-    signal = if even (s `div` 13) then maxBound else 0-    block txs = Just <$> blockOn db txs-    shuffled = sortOn (hashWithSalt s . encode . txHash)-    pool = map snd <$> Q.getMempool db-    -- outputs that nothing spends, in a block or in the mempool-    unspent = do-      listed <- concat <$> mapM (\a -> Q.getAddressUnspents db a def) (addressesOf db.net db.ctx hist)-      (listed <>) . catMaybes <$> mapM (Q.getUnspent db) hist.loose-    -- a payment from some of them, in equal shares: now and then in ten of-    -- them, for a transaction with many outputs to spend-    payment us =-      let total = sum (map (.value) us)-          count = if s `mod` 13 == 5 && total >= 50000 then 10 else 1 + (s `div` 7) `mod` 3-          share = (total - 500) `div` fromIntegral count-       in if null us || total < 500 + 600 * fromIntegral count-            then Nothing-            else Just (spending us (replicate count share))-    pay = do-      us <- take (1 + s `mod` 3) . rotate (s `div` 3) <$> unspent-      return ((`Pay` hist.clock) <$> payment us)-    -- part of the mempool, in an order in which it can be mined-    some = do-      tds <- catMaybes <$> (pool >>= Q.getManyTxData db)-      let ordered = map snd (sortTxs (map (.tx) tds))-      return (take (s `mod` (length ordered + 1)) ordered)-    -- a block with part of the mempool and a payment that the mempool has-    -- not seen, from outputs of blocks-    unseen = do-      us <- take (1 + s `mod` 2) . rotate (s `div` 3) . filter (confirmed . (.block)) <$> unspent-      txs <- some-      block (shuffled (txs <> maybe [] pure (payment us)))-    -- a transaction that spends what a mempool transaction spends, when-    -- that is an output of a block-    rival :: (Tx -> IO (Maybe Op)) -> IO (Maybe Op)-    rival make =-      rotate s <$> pool >>= \case-        [] -> return Nothing-        h : _ ->-          Q.getTxData db h >>= \case-            Just td-              | i : _ <- td.tx.inputs,-                Just p <- IntMap.lookup 0 td.prevs,-                p.value > 2000 ->-                  Q.getTxData db i.outpoint.hash >>= \case-                    Just parent-                      | confirmed parent.block ->-                          let value = p.value - 700 - fromIntegral (s `mod` 50)-                           in make (Tx 2 [TxIn i.outpoint "" maxBound] [TxOut value (target 0)] [] 0)-                    _ -> return Nothing-            _ -> return Nothing-    -- a transaction or a block that the rules must refuse-    refused = do-      us <- rotate (s `div` 8) <$> unspent-      let spent = [i.outpoint | (b, _) <- hist.chain, mined <- drop 1 b.txs, i <- take 1 mined.inputs]-          unknown = Tx 2 [TxIn (OutPoint (TxHash (sha256 (encode seed))) 0) "" maxBound] [TxOut 1000 script] [] 0-          again' p = Tx 2 [TxIn p "" maxBound] [TxOut 1000 script] [] 0-          send tx = return (Just (Pay tx hist.clock))-      case (s `mod` 8, us, rotate s spent) of-        -- an output that a confirmed transaction has spent-        (1, _, p : _) -> send (again' p)-        -- the same output twice-        (2, u : _, _) -> send (Tx 2 (replicate 2 (TxIn u.outpoint "" maxBound)) [TxOut 1000 script] [] 0)-        -- more than it has-        (3, u : _, _) -> send (Tx 2 [TxIn u.outpoint "" maxBound] [TxOut (u.value + 1) script] [] 0)-        -- a coinbase-        (4, _, _) -> send (Tx 1 [TxIn nullOutPoint (encode seed) maxBound] [TxOut 1000 script] [] 0)-        -- a block that spends an output that does not exist-        (5, _, _) -> block [unknown]-        -- a block that spends an output that a confirmed transaction has-        -- spent-        (6, _, p : _) -> block [again' p]-        -- a block that does not build on the best block, but beside it-        (7, _, _) | (_, best) : _ <- hist.chain -> do-          parent <- Q.getBlock db best.header.prev >>= maybe (fail "no data for the parent") return-          return . Just $ uncurry Mine (nextBlock db.net parent.height parent.header [])-        -- an output that does not exist-        _ -> send unknown-    delete =-      fmap Delete . take 1 . rotate s <$> pool >>= \case-        [] -> return Nothing-        d : _ -> return (Just d)-    -- the deletion of a confirmed transaction, or of one that is not there-    deleteOther =-      let mined = [txHash tx | (b, _) <- hist.chain, tx <- b.txs]-       in return . Just . Delete $ case rotate s mined of-            h : _ | even s -> h-            _ -> TxHash (sha256 (encode seed))-    revert = return $ case hist.chain of-      (b, _) : _ : _ -> Just (Revert (headerHash b.header) hist.clock)-      _ -> Nothing-    -- an earlier transaction, which may be confirmed, in the mempool,-    -- deleted, or missing what it spends-    again = return $ case rotate s hist.made of-      tx : _ -> Just (Pay tx hist.clock)-      [] -> Nothing------------------------------------------------------------------------------------ Answers------------------------------------------------------------------------------------ | Run an action with the queries of a database: those of today's layout,--- where one set of indices has the chain and the mempool mixed, or those of--- the layered layout, on a view of the database as it is now.-mixed, layered, pooled :: DatabaseReader -> (StoreRead -> IO a) -> IO a-mixed db action = action (Q.databaseRead db)-layered db action = L.withView db (action . L.layeredRead)--- the queries of a fleet, with the store's own mempool area as the pool-pooled db action = L.withView db $ \v ->-  let r = L.viewReader v in action (P.pooledRead r (P.localPool r))---- | Everything that the API could say about a history, in whole lists.-data Answers = Answers-  { balances :: ![(Address, Balance)],-    addressTxs :: ![(Address, [TxRef])],-    addressUnspents :: ![(Address, [Unspent])],-    txs :: ![(TxHash, Maybe TxData)],-    numbered :: ![(TxHash, [TxData])],-    spenders :: ![(OutPoint, Maybe Spender)],-    unspents :: ![(OutPoint, Maybe Unspent)],-    mempool :: ![(UnixTime, TxHash)]-  }-  deriving (Eq, Show)---- | Ask everything about the addresses of a history and about every--- transaction made, by its hash and by its short number, with each of its--- outputs and one that it does not have. An address that was never used and--- one whose balance is zero are the same to the API, and are the same here.-ask :: History -> StoreRead -> IO Answers-ask hist queries = do-  balances <- mapM (\a -> (a,) . fromMaybe (zeroBalance a) <$> queries.getBalance a) as-  addressTxs <- mapM (\a -> (a,) <$> queries.getAddressTxs a def) as-  addressUnspents <- mapM (\a -> (a,) <$> queries.getAddressUnspents a def) as-  txs <- mapM (\h -> (h,) <$> queries.getTxData h) hashes-  numbered <- mapM (\h -> (h,) <$> queries.getNumTxData (shortNumber h)) hashes-  spenders <- mapM (\p -> (p,) <$> queries.getSpender p) outpoints-  unspents <- mapM (\p -> (p,) <$> queries.getUnspent p) outpoints-  mempool <- queries.getMempool-  return Answers {balances, addressTxs, addressUnspents, txs, numbered, spenders, unspents, mempool}-  where-    as = addressesOf queries.net queries.ctx hist-    hashes = map txHash hist.made-    outpoints = [OutPoint (txHash tx) i | tx <- hist.made, i <- [0 .. fromIntegral (length tx.outputs)]]------------------------------------------------------------------------------------ Pages------------------------------------------------------------------------------------ | A question whose answer is a page of a list, or is read in a way that--- the questions of 'ask' do not use: many at a time, by number, or through--- an extended public key.-data Question-  = -- | a page of the transactions of an address-    AddressTxs !Address !Limits-  | -- | a page of the transactions of some addresses-    AddressesTxs ![Address] !Limits-  | -- | a page of the unspent outputs of an address-    AddressUnspents !Address !Limits-  | -- | a page of the unspent outputs of some addresses-    AddressesUnspents ![Address] !Limits-  | -- | the balances of some addresses-    Balances ![Address]-  | -- | the data of some transactions-    Bodies ![TxHash]-  | -- | the transactions with a short number of the Blockchain.info API-    Numbered !Word64-  | -- | a page of the mempool-    Pool !Limits-  | -- | the balances of the addresses of the 'wallet'-    WalletBalances-  | -- | a page of the transactions of the 'wallet'-    WalletTxs !Limits-  | -- | a page of the unspent outputs of the 'wallet'-    WalletUnspents !Limits-  | -- | the number of transactions of the 'wallet'-    WalletCount-  | -- | a page of the transactions of some addresses, read as those of an-    -- extended public key are-    SetTxs ![Address] !Limits-  | -- | a page of the unspent outputs of some addresses, read as those of-    -- an extended public key are-    SetUnspents ![Address] !Limits-  | -- | the number of distinct transactions of some addresses-    SetCount ![Address]-  deriving (Eq, Show)---- | The answer to a 'Question'.-data Reply-  = Refs ![TxRef]-  | Outputs ![Unspent]-  | Amounts ![Balance]-  | Data ![Maybe TxData]-  | Matches ![TxData]-  | Listed ![(UnixTime, TxHash)]-  | Paths ![XPubBal]-  | PathOutputs ![XPubUnspent]-  | Number !Word32-  deriving (Eq, Show)---- | Ask a question.-answer :: StoreRead -> Question -> IO Reply-answer queries = \case-  AddressTxs a limits -> Refs <$> queries.getAddressTxs a limits-  AddressesTxs as limits -> Refs <$> queries.getAddressesTxs as limits-  AddressUnspents a limits -> Outputs <$> queries.getAddressUnspents a limits-  AddressesUnspents as limits -> Outputs <$> queries.getAddressesUnspents as limits-  Balances as -> Amounts <$> queries.getBalances as-  Bodies hs -> Data <$> queries.getManyTxData hs-  Numbered i -> Matches <$> queries.getNumTxData i-  Pool limits -> Listed <$> queries.getMempoolPage limits-  WalletBalances -> Paths <$> queries.xPubBals spec-  WalletTxs limits -> Refs <$> (queries.xPubBals spec >>= \bals -> queries.xPubTxs spec bals limits)-  WalletUnspents limits -> PathOutputs <$> (queries.xPubBals spec >>= \bals -> queries.xPubUnspents spec bals limits)-  WalletCount -> Number <$> (queries.xPubBals spec >>= queries.xPubTxCount spec)-  SetTxs as limits -> Refs <$> (set as >>= \bals -> queries.xPubTxs spec bals limits)-  SetUnspents as limits -> PathOutputs <$> (set as >>= \bals -> queries.xPubUnspents spec bals limits)-  SetCount as -> Number <$> (set as >>= queries.xPubTxCount spec)-  where-    spec = wallet queries.ctx-    -- the queries about an extended public key take the balances of its-    -- addresses, so any addresses can be given to them-    set as = zipWith (\i b -> XPubBal {path = [0, i], balance = b}) [0 ..] <$> queries.getBalances as---- | Whether the page of a question starts somewhere that both layouts--- know, given a database of version 20: not at a transaction that left--- the mempool unconfirmed. Version 20 keeps the bodies of those, and--- starts a page where such a transaction was; the layered layout keeps--- nothing of them, and its page is empty.-startsKnown :: DatabaseReader -> Question -> IO Bool-startsKnown db question = case startOf question of-  Just (Just (AtTx h)) -> maybe True (not . (.deleted)) <$> Q.getTxRecord db h-  _ -> return True---- | The questions that two databases answer differently, with the answers--- of both.-differences :: [Question] -> [Reply] -> [Reply] -> [(Question, Reply, Reply)]-differences asked was now = [(q, a, b) | (q, a, b) <- zip3 asked was now, a /= b]---- | How much there is in an answer.-size :: Reply -> Int-size = \case-  Refs xs -> length xs-  Outputs xs -> length xs-  Amounts xs -> length xs-  Data xs -> length (catMaybes xs)-  Matches xs -> length xs-  Listed xs -> length xs-  Paths xs -> length xs-  PathOutputs xs -> length xs-  Number n -> fromIntegral n---- | Where the page of a question starts, if it has a page.-startOf :: Question -> Maybe (Maybe Start)-startOf = \case-  AddressTxs _ limits -> Just limits.start-  AddressesTxs _ limits -> Just limits.start-  AddressUnspents _ limits -> Just limits.start-  AddressesUnspents _ limits -> Just limits.start-  WalletTxs limits -> Just limits.start-  WalletUnspents limits -> Just limits.start-  SetTxs _ limits -> Just limits.start-  SetUnspents _ limits -> Just limits.start-  _ -> Nothing---- | Questions about a history: pages with small limits and offsets, from the--- top, from any transaction ever made, whatever became of it, from one that--- was never made, and from heights up to above the best block; about the--- addresses of the history, one at a time and several together, with one--- that was never used and with repetitions; and about the 'wallet'.-questions :: Network -> Ctx -> History -> Int -> Gen [Question]-questions net ctx hist n = vectorOf n question-  where-    question =-      frequency-        [ (4, AddressTxs <$> address <*> limits),-          (4, AddressesTxs <$> several address <*> limits),-          (4, AddressUnspents <$> address <*> limits),-          (4, AddressesUnspents <$> several address <*> limits),-          (1, Balances <$> several address),-          (1, Bodies <$> several hash),-          (1, Numbered . shortNumber <$> hash),-          (1, Pool <$> limits),-          (1, pure WalletBalances),-          (3, WalletTxs <$> limits),-          (3, WalletUnspents <$> limits),-          (1, pure WalletCount),-          (3, SetTxs <$> several address <*> limits),-          (3, SetUnspents <$> several address <*> limits),-          (1, SetCount <$> several address)-        ]-    address = frequency [(12, elements (addressesOf net ctx hist)), (1, elements unused)]-    unused = rights [scriptToAddressBS net ctx (p2pkh 9)]-    hash = frequency [(12, elements (map txHash hist.made)), (1, pure (TxHash (sha256 ("never made" :: ByteString))))]-    several :: Gen a -> Gen [a]-    several one = frequency [(1, pure 0), (12, choose (1, 6))] >>= (`vectorOf` one)-    limits =-      Limits-        <$> frequency [(1, pure 0), (5, choose (1, 6))]-        <*> frequency [(3, pure 0), (2, choose (1, 3))]-        <*> frequency-          [ (3, pure Nothing),-            (6, Just . AtTx <$> hash),-            (2, Just . AtBlock <$> choose (0, best + 2))-          ]-    best :: BlockHeight-    best = case hist.chain of-      (_, node) : _ -> node.height-      [] -> 0---- | The short number of a transaction in the Blockchain.info API: the first--- 53 bits of its hash.-shortNumber :: TxHash -> Word64-shortNumber h = either error (`shiftR` 11) (runGet getWord64be (encode h))------------------------------------------------------------------------------------ The chain's column families------------------------------------------------------------------------------------ | What the chain's column families hold, but for what version 20 left--- there that does not come from the chain: bodies marked as deleted, and--- balances at zero. The time a transaction was first seen is apart from its--- body, because a database that never saw the mempool has another.-data Chain = Chain-  { bodies :: ![(ByteString, (BlockRef, Tx, IntMap Prev, Bool))],-    firstSeen :: ![(ByteString, UnixTime)],-    addressTxs :: ![(ByteString, ByteString)],-    addressOutputs :: ![(ByteString, ByteString)],-    unspents :: ![(ByteString, ByteString)],-    balances :: ![(ByteString, BalVal)],-    -- | the inputs of blocks that spend outputs, each with the output that-    -- it spends, whether the database has it under a key of version 22 or-    -- in the record of the transaction that made the output, as the-    -- versions before did: where they are is not what this compares-    spenders :: ![(ByteString, ByteString)]-  }-  deriving (Eq, Show)+  let (block, node) = nextBlock db.net parent.height parent.header txs+  run db (\w -> importBlock w block node) >>= either (fail . show) (void . return)+  return (headerHash block.header) -chainOf :: DatabaseReader -> IO Chain-chainOf db = do-  txs <- dump db (txCF db.db) >>= mapM (\(k, v) -> (k,) <$> decoded v)-  addressTxs <- dump db (addrTxCF db.db)-  addressOutputs <- dump db (addrOutCF db.db)-  unspents <- dump db (unspentCF db.db)-  balances <- dump db (balanceCF db.db) >>= mapM (\(k, v) -> (k,) <$> decoded v)-  spenders <- dump db (spenderCF db.db)-  let live = [(k, td :: TxData) | (k, td) <- txs, not td.deleted]-  return-    Chain-      { bodies = [(k, (td.block, td.tx, td.prevs, td.rbf)) | (k, td) <- live],-        firstSeen = [(k, td.timestamp) | (k, td) <- live],-        addressTxs,-        addressOutputs,-        unspents,-        balances = filter ((/= def) . snd) balances,-        spenders =-          sort $-            [ (encode (OutPoint h (fromIntegral i)), encode spender)-            | (k, v) <- spenders,-              Right (SpenderKey h _) <- [decode k],-              Right (SpenderVal m) <- [decode v],-              (i, spender) <- IntMap.toList m-            ]-              <> [ (encode (OutPoint (txHash td.tx) (fromIntegral i)), encode spender)-                 | (_, td) <- live,-                   (i, spender) <- IntMap.toList td.spenders-                 ]-      }-  where-    decoded v = either fail return (decode v)+-- | Revert the best block.+revert :: DatabaseReader -> IO ()+revert db = do+  best <- Q.getBestBlock db >>= maybe (fail "no best block") return+  run db (`revertBlock` best) >>= either (fail . show) return  -- | Every key and value of a column family. dump :: DatabaseReader -> ColumnFamily -> IO [(ByteString, ByteString)]
+ test/Haskoin/Store/Integration/TempSpec.hs view
@@ -0,0 +1,34 @@+-- | The temporary directories of the tests: there for the action, and gone+-- after it.+module Haskoin.Store.Integration.TempSpec (spec) where++import Control.Exception (ErrorCall (..), throwIO, try)+import Data.IORef (newIORef, readIORef, writeIORef)+import Data.List (isPrefixOf)+import Haskoin.Store.Integration.Temp+import System.Directory (doesDirectoryExist, doesFileExist)+import System.FilePath (takeFileName, (</>))+import Test.Hspec++spec :: Spec+spec = describe "a temporary directory" $ do+  it "is there for the action, and removed with what is in it afterwards" $ do+    dir <- withSystemTempDirectory "haskoin-store-temp" $ \dir -> do+      takeFileName dir `shouldSatisfy` isPrefixOf "haskoin-store-temp"+      doesDirectoryExist dir `shouldReturn` True+      writeFile (dir </> "file") "kept until the end"+      doesFileExist (dir </> "file") `shouldReturn` True+      return dir+    doesDirectoryExist dir `shouldReturn` False++  it "is removed when the action fails, and made where it is asked" $+    withSystemTempDirectory "haskoin-store-temp" $ \parent -> do+      made <- newIORef ""+      r <- try . withTempDirectory parent "inner" $ \dir -> do+        writeIORef made dir+        writeFile (dir </> "file") "lost"+        throwIO (ErrorCall "failed") :: IO ()+      r `shouldBe` Left (ErrorCall "failed")+      dir <- readIORef made+      dir `shouldSatisfy` isPrefixOf (parent </> "inner")+      doesDirectoryExist dir `shouldReturn` False
test/Haskoin/Store/IntegrationSpec.hs view
@@ -33,6 +33,9 @@ -- limit. Set @HASKOIN_TEST_LOG@ to see the log of the store. module Haskoin.Store.IntegrationSpec (spec) where +import Control.Concurrent (threadDelay)+import Control.Concurrent.Async (async, cancel, forConcurrently, link, withAsync)+import Control.Exception (throwIO, try) import Control.Lens ((&), (.~), (?~), (^.)) import Control.Monad (forM_, forever, replicateM_, unless, void, when) import Data.Aeson (Value (..), eitherDecode, object, toJSON, (.=))@@ -45,13 +48,14 @@ import Data.Default (def) import Data.Either (isRight) import Data.Foldable (toList)+import Data.HashMap.Strict qualified as HashMap import Data.HashSet qualified as HashSet import Data.IORef (IORef, modifyIORef', newIORef, readIORef) import Data.IntMap.Strict qualified as IntMap import Data.List (nub, sort, sortOn) import Data.List.NonEmpty qualified as NonEmpty import Data.Map.Strict qualified as Map-import Data.Maybe (fromMaybe, isJust, mapMaybe)+import Data.Maybe (fromMaybe, isJust, isNothing, mapMaybe) import Data.Ord (Down (..)) import Data.Serialize (decode, encode) import Data.String.Conversions (cs)@@ -60,10 +64,10 @@ import Data.Time.Clock.POSIX (getPOSIXTime) import Data.Word (Word32, Word64) import Database.Redis qualified as Redis+import GHC.Clock (getMonotonicTimeNSec) import Haskoin   ( Address (..),     BloomFlags (..),-    Ctx,     Network (..),     OutPoint (..),     Tx (..),@@ -84,7 +88,6 @@     hexToTxHash,     makeXPrvKey,     marshalJSON,-    prepareContext,     prvSubKey,     pubSubKey,     sha256,@@ -105,50 +108,47 @@     WebConfig (..),     WebLimits (..),     XPubSpec (..),+    cachePrefix,     decodeTxBody,+    defaultXPubConfig,     runWebSocket,     withStore,   )-import Haskoin.Store.Cache-  ( Cache,-    CacheCounters (..),-    Reload (..),-    cacheCounters,-    cacheEvict,-    cacheIsCached,-    cachePrefix,-    cacheRefresh,-    cacheTxData,-    cacheXPubBals,-    xpubId,-  )-import Haskoin.Store.Common (StoreEvent (..))+import Haskoin.Store.Common (Limits (..), StoreEvent (..)) import Haskoin.Store.Data   ( Balance (..),     BlockData (..),+    BlockHealth (..),     BlockRef (..),     Event (..),     EventCommand (..),     Except (..),     HealthCheck (..),-    BlockHealth (..),     RawResult (..),     SerialList (..),     Spender (..),     Transaction (..),     TxData (..),+    TxRef (..),     XPubBal (..),   ) import Haskoin.Store.Database.Query qualified as Q-import Haskoin.Store.Database.Query.Layered qualified as L+import Haskoin.Store.Exception (bracket, tryAny) import Haskoin.Store.Fleet.Member (memberTip)+import Haskoin.Store.Fleet.Redis (fleetKeys)+import Haskoin.Store.Fleet.XPub (cacheHave, xpubId) import Haskoin.Store.Integration.Image (testImage) import Haskoin.Store.Integration.Node import Haskoin.Store.Integration.Podman+import Haskoin.Store.Integration.Temp (withSystemTempDirectory) import Haskoin.Store.Integration.Valkey import Haskoin.Store.WebClient qualified as Client import Haskoin.Store.WebCommon (ApiResource, NoCache (..), NoTx (..))-import NQE (receive, withSubscription)+import Haskoin.Store.XPub (EntryMoved (..), XPubReads (..))+import Haskoin.Store.XPub.Indexer (Settings (..))+import Haskoin.Store.XPub.Service (XPubConfig (..))+import NQE (publish, receive, withSubscription)+import Network.HTTP.Types (ResponseHeaders) import Network.Socket qualified as Socket import Network.WebSockets qualified as WS import Network.Wreq qualified as Wreq@@ -158,8 +158,7 @@ import System.Log.FastLogger (toLogStr) import System.Process (readProcessWithExitCode) import Test.Hspec-import UnliftIO (async, bracket, cancel, link, throwIO, try, tryAny, withAsync, withSystemTempDirectory)-import UnliftIO.Concurrent (threadDelay)+import Text.Read (readMaybe)  spec :: Spec spec = do@@ -168,17 +167,17 @@     Left why ->       it "check the store against Bitcoin Core and Bitcoin Cash Node" $         pendingWith why-    Right () -> prepareContext $ \ctx -> do+    Right () -> do       forM_ [BitcoinCore, BitcoinCashNode] $ \kind ->-        aroundAll (withEnvOf False kind ctx) $+        aroundAll (withEnvOf False kind) $           describe (name kind <> " that does not whitelist the store") $             it "publishes what it verified, and keeps its node when sent what it cannot verify" strangerTest-      aroundAll (withEnvAt True (Just (30 * 24 * 3600)) BitcoinCore ctx) $+      aroundAll (withEnvAt True (Just (30 * 24 * 3600)) BitcoinCore) $         describe "Bitcoin Core with blocks of a month ago" $           it "keeps what a reorganisation of an old block put back in the mempool" oldBlockTest-      aroundAll (withEnv BitcoinCore ctx) $+      aroundAll (withEnv BitcoinCore) $         describe "Bitcoin Core" (chainSpec BitcoinCore)-      aroundAll (withEnv BitcoinCashNode ctx) $+      aroundAll (withEnv BitcoinCashNode) $         describe "Bitcoin Cash Node" (chainSpec BitcoinCashNode)  -- | The name of a node, as people write it.@@ -198,12 +197,6 @@     it "derives native and nested SegWit addresses" segwitTest     it "refuses the wrong kind of address for an xpub" deriveTest   it "refuses unknown parameters and values it cannot read" strictTest-  it "does not use or fill the cache when asked not to" noCacheTest-  it "recovers from damaged entries and keeps newer data" damagedTest-  it "updates cached xpubs from what changed, and reads them whole when unsure" incrementalTest-  it "catches up with changes that the database has before their events" catchUpTest-  it "keeps the spenders of cached transactions up to date" spenderTest-  it "evicts the least recently used xpubs" evictTest   it "answers an error in JSON, whatever was asked for" errorFormatTest   it "answers the Haskell client of haskoin-store-data" clientTest   it "hands what it verified to its peers, and shows only what a node announced back" publishTest@@ -217,10 +210,6 @@   it "tells each client of the transactions it chose, and takes up where it left" eventsTest   it "drops a transaction double-spent in a reorganisation" reorgTest   it "sends again a payment that the nodes forgot and the store lists" resendTest-  it "applies each change once with two stores sharing Redis" onceTest-  it "does not replace what it does not know" unknownTest-  it "leaves the keys of older caches alone" oldKeysTest-  it "answers from its database, and says so, while Redis does not answer" redisDownTest   it "gives the same answers from every store of a fleet, from one block and one mempool" fleetTest  -------------------------------------------------------------------------------@@ -233,7 +222,6 @@   { kind :: !NodeKind,     node :: !Node,     net :: !Network,-    ctx :: !Ctx,     store :: !Store,     -- | base URL of the web server with the cache     cached :: !String,@@ -241,8 +229,6 @@     plain :: !String,     -- | address of the wallet of the node, to mine to     miner :: !Text,-    -- | the cache of the store-    redisCache :: !Cache,     -- | connection to Redis, to look at the keys     redis :: !Redis.Connection,     -- | configuration of a store like this one with its database in a@@ -257,20 +243,20 @@ initGap = 20 gap = 32 -withEnv :: NodeKind -> Ctx -> (Env -> IO ()) -> IO ()+withEnv :: NodeKind -> (Env -> IO ()) -> IO () withEnv = withEnvOf True  -- | The same with a node that whitelists the store, as an operator's own -- node should, or with one that does not. The second announces -- transactions seconds later, so the web servers wait longer for one they -- published, and it does not give its mempool when asked.-withEnvOf :: Bool -> NodeKind -> Ctx -> (Env -> IO ()) -> IO ()+withEnvOf :: Bool -> NodeKind -> (Env -> IO ()) -> IO () withEnvOf whitelisted = withEnvAt whitelisted Nothing  -- | The same, with the first blocks made a number of seconds in the past: -- the node's clock is set back for them with @setmocktime@, and left so.-withEnvAt :: Bool -> Maybe Int -> NodeKind -> Ctx -> (Env -> IO ()) -> IO ()-withEnvAt whitelisted ago kind ctx go = do+withEnvAt :: Bool -> Maybe Int -> NodeKind -> (Env -> IO ()) -> IO ()+withEnvAt whitelisted ago kind go = do   image <- testImage   withContainer (redisContainer image) $ \redis -> do     redisPort <- hostPort redis 6379@@ -299,14 +285,15 @@       void $ generateTo node 150 miner       withRamDir $ \dir -> withTestLogger $ \logger -> do         let net = nodeNetwork kind-            cfg = (storeConfig net ctx logger dir redisURL p2pPort False) {syncMempool = whitelisted}+            cfg = storeConfig net logger dir redisURL p2pPort         withStore cfg $ \store -> do           listening $ \(cachedSock, cachedPort) -> listening $ \(plainSock, plainPort) -> do             let web s = runWebSocket (webConfig logger s)             conn <- Redis.checkedConnect (either error id (Redis.parseConnectInfo redisURL))-            cache <- maybe (fail "The store has no cache") return store.cache             withAsync (web store cachedSock) $ \a1 ->-              withAsync (web store {cache = Nothing} plainSock) $ \a2 -> do+              -- a second server of the same store: the cache cannot be+              -- turned off, so both answer xpubs from it+              withAsync (web store plainSock) $ \a2 -> do                 link a1                 link a2                 let env =@@ -314,15 +301,13 @@                         { kind = kind,                           node = node,                           net = net,-                          ctx = ctx,                           store = store,                           cached = "http://127.0.0.1:" <> show cachedPort,                           plain = "http://127.0.0.1:" <> show plainPort,                           miner = miner,-                          redisCache = cache,                           redis = conn,-                          configIn = \d -> storeConfig net ctx logger d redisURL p2pPort False,-                          memberIn = \d -> storeConfig net ctx logger d redisURL p2pPort True+                          configIn = \d -> storeConfig net logger d redisURL p2pPort,+                          memberIn = \d -> storeConfig net logger d redisURL p2pPort                         }                 waitSynced env                 go env@@ -345,9 +330,9 @@       return (toLogStr (show seconds <> "." <> padded <> " "))  -- | Store with the default gaps, caching every used xpub, with small RocksDB--- buffers and caches, as a member of a fleet or not.-storeConfig :: Network -> Ctx -> Logger -> FilePath -> String -> Int -> Bool -> StoreConfig-storeConfig net ctx logger dir redisURL p2pPort fleet =+-- buffers and caches.+storeConfig :: Network -> Logger -> FilePath -> String -> Int -> StoreConfig+storeConfig net logger dir redisURL p2pPort =   StoreConfig     { maxPeers = 1,       initPeers = ["127.0.0.1:" <> show p2pPort],@@ -355,19 +340,17 @@       db = dir,       net = net,       redis = Just redisURL,-      ctx = ctx,       initGap = initGap,       gap = gap,-      redisMinAddrs = 1,-      redisMinTxs = 1000,       redisMaxXPubs = 10000,-      noMempool = False,-      wipeMempool = False,-      fleet = fleet,-      syncMempool = True,+      -- requests wait three seconds for an xpub, not ten, so that the tests+      -- of the wait run fast+      xpubs = defaultXPubConfig {indexer = defaultXPubConfig.indexer {wait = 3000000}},       blockWindow = 64,       mempoolTimeout = 14,       peerTimeout = 120,+      blockStall = 10,+      blockPing = 5,       maxPeerLife = 48 * 3600,       connect = withConnection,       stats = Nothing,@@ -375,6 +358,7 @@       maxOpenFiles = Just 256,       writeBufferSize = Just (16 * 1024 * 1024),       maxWalSize = Just (64 * 1024 * 1024),+      backgroundJobs = Nothing,       blockCache = Just (16 * 1024 * 1024),       logger = logger     }@@ -390,12 +374,10 @@       minPeers = 0,       limits = def,       version = "test",-      noMempool = False,       stats = Nothing,       tickerRefresh = 90 * 1000 * 1000,       tickerURL = "http://127.0.0.1:9/",       priceHistoryURL = "http://127.0.0.1:9/",-      noXPub = False,       noBlockchainInfo = False,       healthCheckInterval = 5,       accessLog = False,@@ -549,44 +531,44 @@   XPub     { prv = prv,       spec = xspec,-      text = cs (xPubExport env.net env.ctx pub),+      text = cs (xPubExport env.net pub),       scripts =         Map.fromList-          [ (scriptHex env (addrOf env x [m, i]), [m, i])+          [ (scriptHex (addrOf x [m, i]), [m, i])           | m <- [0, 1],             i <- [0 .. 120]           ]     }   where     prv = makeXPrvKey (C8.pack ("haskoin-store integration " <> show env.kind <> " " <> seed))-    pub = deriveXPubKey env.ctx prv+    pub = deriveXPubKey prv     xspec = XPubSpec {key = pub, deriv = deriv}     x = XPub {prv = prv, spec = xspec, text = "", scripts = Map.empty}  -- | Address at a derivation path, computed with haskoin-core, not the store.-addrOf :: Env -> XPub -> [Word32] -> Address-addrOf env x [m, i] = f (pubSubKey env.ctx (pubSubKey env.ctx x.spec.key m) i)+addrOf :: XPub -> [Word32] -> Address+addrOf x [m, i] = f (pubSubKey (pubSubKey x.spec.key m) i)   where     f = case x.spec.deriv of-      DeriveNormal -> xPubAddr env.ctx-      DeriveP2SH -> xPubCompatWitnessAddr env.ctx-      DeriveP2WPKH -> xPubWitnessAddr env.ctx-addrOf _ _ p = error ("Bad path: " <> show p)+      DeriveNormal -> xPubAddr+      DeriveP2SH -> xPubCompatWitnessAddr+      DeriveP2WPKH -> xPubWitnessAddr+addrOf _ p = error ("Bad path: " <> show p)  addrText :: Env -> Address -> Text addrText env a = fromMaybe (error "Cannot encode address") (addrToText env.net a)  -- | Address at a path of an xpub, as text. at :: Env -> XPub -> [Word32] -> Text-at env x = addrText env . addrOf env x+at env x = addrText env . addrOf x -scriptHex :: Env -> Address -> Text-scriptHex env a =-  encodeHex (fromMaybe (error "No script for address") (addressToScriptBS env.ctx a))+scriptHex :: Address -> Text+scriptHex a =+  encodeHex (fromMaybe (error "No script for address") (addressToScriptBS a))  -- | Private key of the address at a path, as WIF. wifOf :: Env -> XPub -> [Word32] -> Text-wifOf env x [m, i] = xPrvWif env.net (prvSubKey env.ctx (prvSubKey env.ctx x.prv m) i)+wifOf env x [m, i] = xPrvWif env.net (prvSubKey (prvSubKey x.prv m) i) wifOf _ _ p = error ("Bad path: " <> show p)  -- | Parameter that selects the derivation of an xpub.@@ -770,14 +752,6 @@     _ <- getJSON env.cached (b <> "/balances" <> q)     waitCached env x   sameAnswers env (xpubPaths x ts heights)-  unless (null e.used) $ do-    -- every body of the xpub is in Redis, and is the body of the database,-    -- spenders included-    let hs = map (\t -> fromMaybe (error "bad txid") (hexToTxHash t)) e.txids-    waitFor "cached bodies" $ do-      fromCache <- cacheTxData env.redisCache hs-      fromDB <- L.withView env.store.db (`L.getManyTxData` hs)-      return (fromCache == fromDB && all isJust fromCache)  -- | Requests about an xpub to compare between the servers. The balances come -- first, as they are what puts an xpub in the cache.@@ -792,10 +766,7 @@     b <> "/transactions/full" <> q <> "&limit=4",     u,     u <> "&limit=1",-    u <> "&limit=2&offset=1",-    b <> q <> "&nocache=true",-    t <> "&nocache=true",-    u <> "&nocache=true"+    u <> "&limit=2&offset=1"   ]     <> concat       [ [ t <> "&height=" <> s <> "&limit=2",@@ -875,7 +846,7 @@         [ object             [ "txid" .= c.txid,               "vout" .= c.vout,-              "scriptPubKey" .= scriptHex env (addrOf env x c.path),+              "scriptPubKey" .= scriptHex (addrOf x c.path),               "amount" .= btc c.value             ]         | c <- cs'@@ -918,11 +889,15 @@   e1.used `shouldBe` [[0, 0], [0, 1], [0, 2], [0, 5], [0, 37]]   e1.balance `shouldBe` 1500000   checkXPub env x ref+  full0 <- redisRun env (Redis.hget (xpubKey env x "meta") "full")   void $ pay env x ref [([0, 69], 700000)]   waitSynced env   e2 <- expect <$> readIORef ref   e2.used `shouldBe` [[0, 0], [0, 1], [0, 2], [0, 5], [0, 37], [0, 69], [0, 70]]   checkXPub env x ref+  -- index 70 came into the gap with its payment, and the cache read it+  -- without reading the xpub whole+  redisRun env (Redis.hget (xpubKey env x "meta") "full") `shouldReturn` full0   void $ mine env 1   checkXPub env x ref @@ -1126,8 +1101,7 @@   forM_     [ "/address/" <> cs a <> "/transactions/full?limit=200",       "/address/transactions/full?limit=200&addresses=" <> cs a,-      "/xpub/" <> x.text <> "/transactions/full?limit=200",-      "/xpub/" <> x.text <> "/transactions/full?limit=200&nocache=true"+      "/xpub/" <> x.text <> "/transactions/full?limit=200"     ]     $ \p -> (arr <$> plainJSON env p) `shouldReturn` wanted   -- a block that is not there is an error, not a response cut short@@ -1261,7 +1235,8 @@   forM_     [ "/block/best?notx=true",       "/blockchain/latestblock?format=json&txidindex=true",-      "/xpub/" <> x.text <> "?derive=normal&nocache=false"+      -- taken from older clients, and ignored+      "/xpub/" <> x.text <> "?derive=normal&nocache=true"     ]     $ \p -> do       (code', _) <- getJSON env.plain p@@ -1362,7 +1337,7 @@           [ object               [ "txid" .= cbTxid,                 "vout" .= (0 :: Int),-                "scriptPubKey" .= scriptHex env (addrOf env k [0, 0]),+                "scriptPubKey" .= scriptHex (addrOf k [0, 0]),                 "amount" .= btc cbValue               ]           ]@@ -1445,7 +1420,7 @@  -- | Whether an xpub is cached. isCached :: Env -> XPub -> IO Bool-isCached env x = cacheIsCached env.redisCache x.spec+isCached env x = (\(a, b) -> a + b > 0) <$> cacheHave env.redis (fleetKeys (cachePrefix env.net) 32) x.spec  -- | Key of the entry of an xpub in Redis. xpubKey :: Env -> XPub -> C8.ByteString -> C8.ByteString@@ -1473,277 +1448,6 @@   void . Redis.runRedis conn $ mapM_ (uncurry Redis.set) oldKeys   Redis.disconnect conn --- | Queries with @nocache=true@ must neither read the cache nor put the xpub--- in it; a query without it then caches the xpub.-noCacheTest :: Env -> IO ()-noCacheTest env = do-  let x = newXPub env "nocache" DeriveNormal-      b = "/xpub/" <> x.text-  ref <- newLedger-  void $ pay env x ref [([0, 0], 100000), ([0, 1], 100000)]-  void $ mine env 1-  forM_ ["/transactions", "/transactions/full", "/unspent", "/balances", ""] $ \p -> do-    (code, _) <- getJSON env.cached (b <> p <> "?nocache=true")-    code `shouldBe` 200-  threadDelay 1000000-  isCached env x `shouldReturn` False-  checkXPub env x ref---- | A cached entry that cannot be decoded is cached again instead of failing--- the request, and a reload does not replace an entry stamped with a higher--- block, as one written by a store that is ahead would be.-damagedTest :: Env -> IO ()-damagedTest env = do-  let x = newXPub env "damaged" DeriveNormal-  ref <- newLedger-  void $ pay env x ref [([0, 0], 100000), ([0, 1], 200000)]-  void $ mine env 1-  checkXPub env x ref-  void . redisRun env $ Redis.set (xpubKey env x "bal") "garbage"-  checkXPub env x ref-  bal <- redisRun env $ Redis.get (xpubKey env x "bal")-  (decode <$> bal :: Maybe (Either String [XPubBal])) `shouldSatisfy` maybe False isRight-  void . redisRun env $ Redis.hset (xpubKey env x "meta") (pure ("height", "999999999"))-  cacheRefresh env.redisCache x.spec HashSet.empty `shouldReturn` Behind-  void . redisRun env $ Redis.hset (xpubKey env x "meta") (pure ("height", "0"))-  cacheRefresh env.redisCache x.spec HashSet.empty `shouldReturn` Reloaded-  checkXPub env x ref-  -- a reload writes only differences, computed from what Redis has, so-  -- members that should not be there go, and the generation goes up-  gen0 <- redisRun env $ Redis.hget (xpubKey env x "meta") "gen"-  void . redisRun env $ Redis.zadd (xpubKey env x "txs") [(1, "not a transaction")]-  void . redisRun env $ Redis.zadd (xpubKey env x "utxo") [(1, "not an output")]-  cacheRefresh env.redisCache x.spec HashSet.empty `shouldReturn` Reloaded-  redisRun env (Redis.zscore (xpubKey env x "txs") "not a transaction") `shouldReturn` Nothing-  redisRun env (Redis.zscore (xpubKey env x "utxo") "not an output") `shouldReturn` Nothing-  gen1 <- redisRun env $ Redis.hget (xpubKey env x "meta") "gen"-  (read . C8.unpack <$> gen1) `shouldBe` ((+ (1 :: Int)) . read . C8.unpack <$> gen0)-  checkXPub env x ref---- | Changes of a cached xpub are applied from the transactions that make--- them, without a full reload: the time of the last full reload in the entry--- stays, and the answers stay those of the database. When Redis has a balance--- that no changed transaction explains, a change was missed, and the next--- reload reads the xpub whole, which repairs it.-incrementalTest :: Env -> IO ()-incrementalTest env = do-  let x = newXPub env "incremental" DeriveNormal-      meta k = redisRun env (Redis.hget (xpubKey env x "meta") k)-  ref <- newLedger-  void $ pay env x ref [([0, 0], 100000), ([0, 1], 200000), ([1, 0], 300000)]-  void $ mine env 1-  checkXPub env x ref-  full0 <- meta "full"-  full0 `shouldSatisfy` isJust-  meta "ver" `shouldReturn` Just "3"-  -- a payment in the mempool, its confirmation, and a spend-  void $ pay env x ref [([0, 2], 400000)]-  waitSynced env-  checkXPub env x ref-  void $ mine env 1-  checkXPub env x ref-  out <- getNewAddress env.node-  l <- readIORef ref-  void $ spend env x ref [coinAt l [0, 1]] [(out, 50000), (at env x [1, 1], 140000)]-  waitSynced env-  checkXPub env x ref-  void $ mine env 1-  checkXPub env x ref-  meta "full" `shouldReturn` full0-  -- the index of outputs by outpoint matches the outputs-  nu <- redisRun env (Redis.zcard (xpubKey env x "utxo"))-  no <- redisRun env (Redis.hlen (xpubKey env x "uop"))-  no `shouldBe` nu-  -- a balance that Redis has wrong, at an address the next change does not-  -- touch, as if a change had been missed-  Just blob <- redisRun env (Redis.get (xpubKey env x "bal"))-  bals <- either fail return (decode blob :: Either String [XPubBal])-  let target = addrOf env x [0, 0]-      wrong :: XPubBal -> XPubBal-      wrong b-        | b.balance.address == target =-            XPubBal {path = b.path, balance = (b.balance :: Balance) {received = b.balance.received + 1}}-        | otherwise = b-  void . redisRun env $ Redis.set (xpubKey env x "bal") (encode (map wrong bals))-  void $ pay env x ref [([0, 3], 500000)]-  waitSynced env-  checkXPub env x ref-  full1 <- meta "full"-  (full1 > full0) `shouldBe` True-  -- the wrong balance is gone: Redis has the balances of the database-  fromDB <- L.withView env.store.db (`L.xPubBals` x.spec)-  Just blob' <- redisRun env (Redis.get (xpubKey env x "bal"))-  decode blob' `shouldBe` Right fromDB---- | Under load, the database can have transactions whose events the cache has--- not applied yet, and the balances that a reload reads include them. The--- reload finds them from the addresses whose balances moved, without reading--- the xpub whole. Here the entry is wound back to before a payment, as if--- its event had not come, and a reload is told only of an older change.-catchUpTest :: Env -> IO ()-catchUpTest env = do-  let x = newXPub env "catch-up" DeriveNormal-      key = xpubKey env x-      meta k = redisRun env (Redis.hget (key "meta") k)-  ref <- newLedger-  old <- pay env x ref [([0, 0], 100000)]-  void $ mine env 1-  checkXPub env x ref-  full0 <- meta "full"-  Just blob0 <- redisRun env (Redis.get (key "bal"))-  -- a payment in the mempool, applied by the cache-  newT <- pay env x ref [([0, 1], 200000)]-  waitSynced env-  checkXPub env x ref-  let new = fromMaybe (error "bad txid") (hexToTxHash newT)-      oldH = fromMaybe (error "bad txid") (hexToTxHash old)-  -- wind the entry back: balances, transaction and outputs of the payment-  uop <- redisRun env (Redis.hgetall (key "uop"))-  let paidBy o = either (const False) (\p -> p.hash == new) (decode o :: Either String OutPoint)-      ours = [(op, u) | (op, u) <- uop, paidBy op]-  outs <- maybe (fail "no outputs of the payment in the entry") return (NonEmpty.nonEmpty ours)-  void . redisRun env $ Redis.set (key "bal") blob0-  void . redisRun env $ Redis.zrem (key "txs") (NonEmpty.singleton (encode new))-  void . redisRun env $ Redis.zrem (key "utxo") (snd <$> outs)-  void . redisRun env $ Redis.hdel (key "uop") (fst <$> outs)-  cacheRefresh env.redisCache x.spec (HashSet.singleton oldH) `shouldReturn` Reloaded-  meta "full" `shouldReturn` full0-  redisRun env (Redis.zscore (key "txs") (encode new)) >>= (`shouldSatisfy` isJust)-  checkXPub env x ref-  fromDB <- L.withView env.store.db (`L.xPubBals` x.spec)-  Just blob1 <- redisRun env (Redis.get (key "bal"))-  decode blob1 `shouldBe` Right fromDB---- | An output of a cached transaction that pays an address outside the xpub--- is spent by a transaction that does not touch the xpub. The xpub is not--- reloaded, but the cached body must show the spender.-spenderTest :: Env -> IO ()-spenderTest env = do-  let x = newXPub env "spender" DeriveNormal-  ref <- newLedger-  w <- getNewAddress env.node-  raw <--    rpc-      env.node-      "createrawtransaction"-      [ toJSON ([] :: [Value]),-        object [Key.fromText (at env x [0, 0]) .= btc 200000, Key.fromText w .= btc 300000]-      ]-  funded <- rpc env.node "fundrawtransaction" [raw]-  signed <- rpc env.node "signrawtransactionwithwallet" [(funded ! "hex")]-  txT <--    rpc env.node "sendrawtransaction" [(signed ! "hex")] >>= \case-      String t -> return t-      v -> fail (show v)-  record env x ref txT-  void $ mine env 1-  checkXPub env x ref-  info <- rpc env.node "getaddressinfo" [String w]-  tx <- getRawTx env.node txT-  let wScript = textField "scriptPubKey" info-  vout <- case [ numV n-               | o <- arr ((tx ! "vout")),-                 Just n <- [field "n" o],-                 (textField "hex" =<< field "scriptPubKey" o) == wScript-               ] of-    v : _ -> return v-    [] -> fail "No output to the wallet address"-  w2 <- getNewAddress env.node-  raw2 <--    rpc-      env.node-      "createrawtransaction"-      [ toJSON [object ["txid" .= txT, "vout" .= vout]],-        object [Key.fromText w2 .= btc 290000]-      ]-  signed2 <- rpc env.node "signrawtransactionwithwallet" [raw2]-  txU <--    rpc env.node "sendrawtransaction" [(signed2 ! "hex")] >>= \case-      String t -> return t-      v -> fail (show v)-  waitSynced env-  let th = fromMaybe (error "bad txid") (hexToTxHash txT)-      spenderOf = \case-        [Just td] -> (.txid) <$> IntMap.lookup (fromIntegral vout) td.spenders-        _ -> Nothing-  waitFor "the spender of a cached body" $-    (== hexToTxHash txU) . spenderOf <$> cacheTxData env.redisCache [th]-  sameAnswers env ["/xpub/" <> x.text <> "/transactions/full", "/transaction/" <> cs txT]-  void $ mine env 1-  sameAnswers env ["/xpub/" <> x.text <> "/transactions/full", "/transaction/" <> cs txT]---- | Evicting down to one xpub keeps the one used last, and removes everything--- of the others: their entries, their addresses in the index, and the--- bodies that no other cached xpub refers to.-evictTest :: Env -> IO ()-evictTest env = do-  let xa = newXPub env "evict a" DeriveNormal-      xb = newXPub env "evict b" DeriveNormal-  ra <- newLedger-  rb <- newLedger-  void $ pay env xa ra [([0, 0], 100000)]-  void $ pay env xb rb [([0, 0], 100000), ([0, 1], 100000)]-  void $ mine env 1-  checkXPub env xb rb-  checkXPub env xa ra-  _ <- getJSON env.cached ("/xpub/" <> xa.text <> "/balances")-  n <- cacheEvict env.redisCache 1-  n `shouldSatisfy` (>= 1)-  isCached env xa `shouldReturn` True-  isCached env xb `shouldReturn` False-  left <- redisRun env $ Redis.keys (xpubKey env xb "*")-  left `shouldBe` []-  addrs <- redisRun env $ Redis.smembers (cachePrefix env.net <> "a:" <> encode (addrOf env xb [0, 0]))-  addrs `shouldBe` []-  eb <- expect <$> readIORef rb-  forM_ eb.txids $ \t -> do-    let h = fromMaybe (error "bad txid") (hexToTxHash t)-    stored <- redisRun env $ Redis.exists (cachePrefix env.net <> "tx:" <> encode h)-    stored `shouldBe` False-  -- the evicted xpub is answered from the database and cached again-  checkXPub env xb rb---- | The keys of older caches, written before the store started, are as they--- were.-oldKeysTest :: Env -> IO ()-oldKeysTest env =-  forM_ oldKeys $ \(k, v) ->-    redisRun env (Redis.get k) `shouldReturn` Just v---- | While Redis does not answer, the store answers from its own database,--- and its health check says that it does with status 203: not a failure,--- since the store works, and not the 200 of a store that has everything.--- When Redis is back, so is the 200. Redis is told to hold every client for--- a while, which is what a server that hangs looks like from outside.-redisDownTest :: Env -> IO ()-redisDownTest env = do-  let x = newXPub env "redisdown" DeriveNormal-  ref <- newLedger-  void $ pay env x ref [([0, 0], 100000), ([1, 0], 50000)]-  void $ mine env 1-  checkXPub env x ref-  let redisOK (_, v) = (v ! "redis") ! "ok"-  (code0, healthy) <- getJSON env.cached "/health"-  code0 `shouldBe` 200-  redisOK (code0, healthy) `shouldBe` Bool True-  -- a store without a cache has no such check-  (_, bare) <- getJSON env.plain "/health"-  bare `shouldSatisfy` \case-    Object o -> not (KeyMap.member "redis" o)-    _ -> False-  held <- redisRun env (Redis.sendRequest ["CLIENT", "PAUSE", "12000"])-  held `shouldBe` Redis.Ok-  -- the first request waits two seconds for Redis and gives up on it-  (code, _) <- getJSON env.cached ("/xpub/" <> x.text <> "/balances")-  code `shouldBe` 200-  waitFor "the health check to notice that Redis is gone" $-    (== 203) . fst <$> getJSON env.cached "/health"-  redisOK <$> getJSON env.cached "/health" `shouldReturn` Bool False-  checkXPub env x ref-  waitFor "the health check to notice that Redis is back" $-    (== 200) . fst <$> getJSON env.cached "/health"-  checkXPub env x ref- -- | An error is answered in JSON, whatever the request asks for, and -- every answer says that it depends on whether the client reads gzip, and -- on nothing else.@@ -1777,7 +1481,7 @@       -- api.haskoin.com has it, and the server of the tests has none       conf = Client.ApiConfig {net = env.net {name = ""}, host = env.plain}       call :: (ApiResource a b) => a -> IO b-      call r = Client.apiCall env.ctx conf r >>= either (fail . show) return+      call r = Client.apiCall conf r >>= either (fail . show) return   heard <- newIORef []   listener <- async . Client.apiEvents conf $ \e -> modifyIORef' heard (e :) >> return True   -- the client is listening once it hears of a block@@ -1796,14 +1500,14 @@   tx.txid `shouldBe` txid   raw <- call (Client.GetTxRaw txid)   txHash raw.get `shouldBe` txid-  balances <- call (Client.GetAddrsBalance [addrOf env x [0, 0]])+  balances <- call (Client.GetAddrsBalance [addrOf x [0, 0]])   map (.confirmed) balances.get `shouldBe` [100000]   keys <- call (Client.GetXPubBalances x.spec.key DeriveNormal (NoCache False))   sum (map (.balance.confirmed) keys.get) `shouldBe` 100000   health <- call Client.GetHealth   health.blocks.blocks `shouldSatisfy` (> 0)   -- an error comes back as the error that it is-  unknown <- Client.apiCall env.ctx conf (Client.GetTx (TxHash (sha256 ("nothing" :: C8.ByteString))))+  unknown <- Client.apiCall conf (Client.GetTx (TxHash (sha256 ("nothing" :: C8.ByteString))))   either Just (const Nothing) unknown `shouldBe` Just ThingNotFound   hash <- maybe (fail "Not a hash") return (hexToBlockHash mined)   waitFor "the client to hear of the block" ((EventBlock hash `elem`) <$> readIORef heard)@@ -2077,7 +1781,9 @@   let amount = case c ! "amount" of         Number n -> Number (n - 0.0001)         other -> other-  rpc env.node "createrawtransaction"+  rpc+    env.node+    "createrawtransaction"     [toJSON [object ["txid" .= (c ! "txid"), "vout" .= (c ! "vout")]], object [Key.fromText to .= amount]]     >>= signed env Nothing @@ -2151,19 +1857,19 @@ -- is, and the output that holds it. payScript :: Env -> H.ScriptOutput -> IO (OutPoint, TxOut) payScript env so = do-  address <- maybe (fail "No address for the script") return (H.outputAddress env.net env.ctx so)+  address <- maybe (fail "No address for the script") return (H.outputAddress env.net so)   txid <- sendMany env.node [(addrText env address, 300000)]   waitSynced env   funding <- getRawTx env.node txid   vout <- case [ numV n                | o <- arr (funding ! "vout"),                  Just n <- [field "n" o],-                 (textField "hex" =<< field "scriptPubKey" o) == Just (scriptHex env address)+                 (textField "hex" =<< field "scriptPubKey" o) == Just (scriptHex address)                ] of     v : _ -> return v     [] -> fail "No output to the script"   spent <- maybe (fail "Not a transaction id") return (hexToTxHash txid)-  return (OutPoint spent (fromIntegral vout), TxOut 300000 (H.marshal env.ctx so))+  return (OutPoint spent (fromIntegral vout), TxOut 300000 (H.encodeSerial so))  -- | A transaction that spends a coin of 300,000 to the wallet of the -- node, with nothing in its input yet.@@ -2184,18 +1890,18 @@ -- | A Schnorr signature of Bitcoin Cash, made the plain way: this is a -- test, and the nonce is no secret worth keeping. haskoin-core verifies -- these and does not make them.-cashSchnorr :: Ctx -> H.SecKey -> H.Hash256 -> C8.ByteString-cashSchnorr ctx k h = case [sig | Just sig <- map attempt [1 ..]] of+cashSchnorr :: H.SecKey -> H.Hash256 -> C8.ByteString+cashSchnorr k h = case [sig | Just sig <- map attempt [1 ..]] of   sig : _ -> sig   [] -> error "No nonce"   where     n = 0xfffffffffffffffffffffffffffffffebaaedce6af48a03bbfd25e8cd0364141 :: Integer     p = 2 ^ (256 :: Int) - 2 ^ (32 :: Int) - 977 :: Integer     word256 i = let b = H.integerToBS i in C8.replicate (32 - C8.length b) '\0' <> b-    public = H.exportPubKey ctx True (H.derivePubKey ctx k)+    public = H.exportPubKey True (H.derivePubKey k)     attempt nonce = do-      r <- H.derivePubKey ctx <$> H.secKey (word256 nonce)-      let (x, y) = C8.splitAt 32 (C8.drop 1 (H.exportPubKey ctx False r))+      r <- H.derivePubKey <$> H.secKey (word256 nonce)+      let (x, y) = C8.splitAt 32 (C8.drop 1 (H.exportPubKey False r))           square = powMod (H.bsToInteger y) ((p - 1) `div` 2) == 1           e = H.bsToInteger (encode (H.sha256 (x <> public <> encode h))) `mod` n       if square then Just (x <> word256 ((nonce + e * H.bsToInteger k.get) `mod` n)) else Nothing@@ -2217,10 +1923,9 @@ -- are not of, is refused. multisigTest :: Env -> IO () multisigTest env = do-  let ctx = env.ctx-      secret n = fromMaybe (error "Not a secret key") (H.secKey (C8.replicate 31 '\0' <> C8.singleton n))+  let secret n = fromMaybe (error "Not a secret key") (H.secKey (C8.replicate 31 '\0' <> C8.singleton n))       secrets = map secret "\x0b\x0c\x0d"-      points = map (H.derivePubKey ctx) secrets+      points = map H.derivePubKey secrets       twoOfThree = H.PayMulSig (map (H.wrapPubKey True) points) 2       right = either (fail . ("Could not make the transaction: " <>)) return       published tx = do@@ -2239,25 +1944,25 @@                   sighash = hashType,                   redeem = Just twoOfThree                 }-        right (H.signTx env.net ctx unsigned [input] (take 2 secrets)) >>= published+        right (H.signTx env.net unsigned [input] (take 2 secrets)) >>= published   case env.kind of     BitcoinCore -> do-      inWitness <- right (H.payToWitnessScriptAddress ctx twoOfThree)+      inWitness <- right (H.payToWitnessScriptAddress twoOfThree)       old (fromMaybe (error "No script") (H.addressToOutput inWitness)) H.sigHashAll       -- Taproot: signed by the first and the third key-      let xOnly = C8.drop 1 . H.exportPubKey ctx True+      let xOnly = C8.drop 1 . H.exportPubKey True           leaf =             H.Script $               [H.opPushData (xOnly (head points)), H.OP_CHECKSIG]                 <> concat [[H.opPushData (xOnly k), H.OP_CHECKSIGADD] | k <- drop 1 points]                 <> [H.OP_2, H.OP_NUMEQUAL]-          internal = H.derivePubKey ctx (secret '\x0e')+          internal = H.derivePubKey (secret '\x0e')           tree = H.TaprootOutput {internalKey = internal, mast = Just (H.MASTLeaf H.tapscriptLeafVersion leaf)}-      (coin, out) <- payScript env (H.taprootScriptOutput ctx tree)+      (coin, out) <- payScript env (H.taprootScriptOutput tree)       unsigned <- spendingTo env coin       let leafOf = Just (H.leafHash H.tapscriptLeafVersion leaf, 0xffffffff)       digest <- right (H.taprootSigHash unsigned [out] 0 H.sigHashDefault Nothing leafOf)-      let sign k = (H.signSchnorr ctx k (H.hashToMsg digest)).get+      let sign k = (H.signSchnorr k (H.hashToMsg digest)).get           spend =             H.ScriptPathSpend               H.ScriptPathData@@ -2265,25 +1970,25 @@                   -- an item for each key, that of the last key first                   stack = [sign (secrets !! 2), "", sign (head secrets)],                   script = leaf,-                  extIsOdd = odd (H.keyParity ctx (H.taprootOutputKey ctx tree)),+                  extIsOdd = odd (H.keyParity (H.taprootOutputKey tree)),                   leafVersion = H.tapscriptLeafVersion,                   internalKey = internal,                   control = [],                   opaqueScript = Nothing                 }-      witness <- right (H.encodeTaprootWitness ctx spend)+      witness <- right (H.encodeTaprootWitness spend)       published (unsigned {witness = [witness]} :: Tx)     BitcoinCashNode -> do-      inScript <- right (H.payToScriptAddress ctx twoOfThree)+      inScript <- right (H.payToScriptAddress twoOfThree)       let locked = fromMaybe (error "No script") (H.addressToOutput inScript)       old locked (H.setForkIdFlag H.sigHashAll)       -- Schnorr signatures of the first and the third key: bits 0 and 2       (coin, out) <- payScript env locked       unsigned <- spendingTo env coin-      code <- right (H.encodeOutput ctx twoOfThree)+      code <- right (H.encodeOutput twoOfThree)       digest <- right (H.txSigHashForkId env.net unsigned code out.value 0 (H.setForkIdFlag H.sigHashAll))-      let sign k = cashSchnorr ctx k digest <> "\x41"-          pushed = [sign (head secrets), sign (secrets !! 2), H.marshal ctx twoOfThree]+      let sign k = cashSchnorr k digest <> "\x41"+          pushed = [sign (head secrets), sign (secrets !! 2), H.encodeSerial twoOfThree]           items = [H.OP_5] <> map H.opPushData pushed       published (unsigned {inputs = [i {script = encode (H.Script items)} | i <- unsigned.inputs]} :: Tx) @@ -2339,7 +2044,7 @@       y = newXPub env "events-other" DeriveNormal       port = read (reverse (takeWhile (/= ':') (reverse env.plain))) :: Int       client path = WS.runClient "127.0.0.1" port path-      json = BL.toStrict . marshalJSON (env.net, env.ctx)+      json = BL.toStrict . marshalJSON env.net       -- every event that a client hears after saying something, in order       hearing path say heard = client path $ \c -> do         mapM_ (WS.sendTextData c) (say :: [C8.ByteString])@@ -2353,14 +2058,14 @@       element = \case         PubKeyAddress h -> runPutS (serialize h)         _ -> error "Not the hash of a key"-      bloom = bloomInsert (bloomCreate 10 0.0000001 0 BloomUpdateNone) (element (addrOf env y [0, 1]))+      bloom = bloomInsert (bloomCreate 10 0.0000001 0 BloomUpdateNone) (element (addrOf y [0, 1]))   [all', key, address, filtered, quiet, keyFirst, addressFirst] <- mapM (const (newIORef [])) [1 .. 7 :: Int]   listeners <-     mapM       async       [ hearing "/events" [] all',         hearing "/events" [json (EventWatchXPub x.spec)] key,-        hearing "/events" [json (EventWatch [addrOf env y [0, 0]])] address,+        hearing "/events" [json (EventWatch [addrOf y [0, 0]])] address,         hearing "/events" [json (EventFilterLoad bloom)] filtered,         -- chosen in the URL, before the first event: blocks alone, a key         -- under a type of address, an address@@ -2495,7 +2200,7 @@         ]       cachedKey = cachePrefix env.net <> "fx:e:" <> xpubId x.spec <> ":meta"       -- the cache has the key, at the fleet tip, and each store answers-      -- about it as it does without the cache, and as the other store does+      -- about it as the other store does       throughCache what = do         waitFor ("the cache to have the key, " <> what) $ do           _ <- getJSON urlA (xpub <> "/balances" <> q)@@ -2506,13 +2211,6 @@           let here = (\n t -> n <> "@" <> t) <$> moves <*> tip           return (has && isJust tip && cached == here)         alike ("the two stores to answer alike from the cache, " <> what) about-        waitFor ("each store to answer from the cache as from its database, " <> what) $ do-          let direct = map (<> "&nocache=true") about-          cachedA <- mapM (getJSON urlA) about-          directA <- mapM (getJSON urlA) direct-          cachedB <- mapM (getJSON urlB) about-          directB <- mapM (getJSON urlB) direct-          return (cachedA == directA && cachedB == directB && cachedA == cachedB)   waitStore env a   waitStore env b   atTip@@ -2635,57 +2333,20 @@     ours <- Q.getBestBlock st.db     return (fmap blockHashToHex ours == Just best) --- | Two stores share a Redis and see the same events. A payment to a cached--- xpub, then the block that confirms it, are each applied once: one store--- claims each change and reloads the xpub, the other finds it done.------ The stores record their own times for mempool transactions, so which of--- them wrote the entry shows in the times of its transactions; only the--- balances are compared with the database here.-onceTest :: Env -> IO ()-onceTest env = withOther env $ \other -> do-  cache2 <- maybe (fail "The second store has no cache") return other.cache-  let x = newXPub env "once" DeriveNormal-      both = do-        a <- cacheCounters env.redisCache-        b <- cacheCounters cache2-        return (a.reloads + b.reloads, a.behind + b.behind)-      synced = waitSynced env >> waitStore env other-      balancesMatch =-        waitFor "the cached balances" $ do-          cached <- cacheXPubBals env.redisCache x.spec-          stored <- L.withView env.store.db (`L.xPubBals` x.spec)-          return (cached == Just stored)-  ref <- newLedger-  void $ pay env x ref [([0, 0], 100000)]-  void $ mine env 1-  synced-  _ <- getJSON env.cached ("/xpub/" <> x.text <> "/balances")-  waitCached env x-  threadDelay 1000000-  (r0, b0) <- both-  void $ pay env x ref [([0, 1], 200000)]-  synced-  balancesMatch-  void $ mine env 1-  synced-  balancesMatch-  threadDelay 2000000-  (r1, b1) <- both-  (r1 - r0, b1 - b0) `shouldBe` (2, 0)+-- | GET a path, with the status, the headers and the body of the answer.+getWithHeaders :: String -> String -> IO (Int, ResponseHeaders, Value)+getWithHeaders base path = do+  r <- Wreq.getWith opts (base <> path)+  return+    ( r ^. Wreq.responseStatus . Wreq.statusCode,+      r ^. Wreq.responseHeaders,+      body (r ^. Wreq.responseBody)+    )+  where+    opts = Wreq.defaults & Wreq.checkResponse ?~ (\_ _ -> return ()) --- | Redis has a transaction of a cached xpub that this store has never seen:--- a store that is behind must not replace the entry, which would drop it.-unknownTest :: Env -> IO ()-unknownTest env = do-  let x = newXPub env "unknown" DeriveNormal-      unknown = encode (TxHash (sha256 ("never seen" :: C8.ByteString)))-  ref <- newLedger-  void $ pay env x ref [([0, 0], 100000)]-  void $ mine env 1-  checkXPub env x ref-  void . redisRun env $ Redis.zadd (xpubKey env x "txs") [(1, unknown)]-  cacheRefresh env.redisCache x.spec HashSet.empty `shouldReturn` Behind-  void . redisRun env $ Redis.zrem (xpubKey env x "txs") (pure unknown)-  cacheRefresh env.redisCache x.spec HashSet.empty `shouldReturn` Reloaded-  checkXPub env x ref+-- | The clock of Redis, in milliseconds.+redisNowMs :: Env -> IO Integer+redisNowMs env = do+  (seconds, micros) <- redisRun env Redis.time+  return (seconds * 1000 + micros `div` 1000)
− test/Haskoin/Store/Logic/LayeredSpec.hs
@@ -1,310 +0,0 @@-module Haskoin.Store.Logic.LayeredSpec (spec) where--import Control.Monad (foldM, forM_, replicateM_, void)-import Data.ByteString (ByteString)-import Data.ByteString qualified as B-import Data.Either (isLeft, isRight)-import Data.List (sort, sortOn)-import Data.Maybe (isJust)-import Data.Serialize (decode)-import Haskoin-  ( Block (..),-    BlockHeight,-    BlockHash,-    Ctx,-    Network,-    OutPoint (..),-    Tx (..),-    TxHash,-    TxIn (..),-    TxOut (..),-    bchRegTest,-    btcRegTest,-    headerHash,-    prepareContext,-    txHash,-  )-import Haskoin.Store.Common (Replaced (..), Start (..), StoreRead (..), replacedKept)-import Haskoin.Store.Data (BlockData (..), TxData (..), Unspent (..))-import Haskoin.Store.Database.Mempool qualified as M-import Haskoin.Store.Database.Migrate (upgradeDatabase)-import Haskoin.Store.Database.Query qualified as Q-import Haskoin.Store.Database.Query.Layered qualified as L-import Haskoin.Store.Database.Reader-  ( DatabaseReader (..),-    blockCF,-    heightCF,-    mempoolAddrCF,-    mempoolCF,-    txCF,-  )-import Haskoin.Store.Histories-import Haskoin.Store.Logic (ImportException (..))-import Haskoin.Store.TestBlocks (coinbaseTx, script)-import Test.Hspec-import Test.Hspec.QuickCheck (modifyMaxSuccess)-import Test.QuickCheck (choose, forAll, ioProperty, property, tabulate, vectorOf)-import Test.QuickCheck.Gen (unGen)-import Test.QuickCheck.Random (mkQCGen)-import UnliftIO (withSystemTempDirectory)--spec :: Spec-spec = prepareContext $ \ctx -> describe "rules of the layered layout" $ do-  forM_ [("Bitcoin", btcRegTest), ("Bitcoin Cash, without replace-by-fee", bchRegTest)] $ \(name, net) ->-    modifyMaxSuccess (const 40) $-      it ("leave the database that today's rules and the migration leave, on " <> name) $-        -- the number says when the third store is upgraded, and which-        -- pages are asked for at the end-        property . forAll ((,) <$> vectorOf 70 anyNumber <*> choose (0, 1000000)) $ \(seeds, number) -> ioProperty $ do-          (outcomes, hist) <- sameHistory ctx net number [\db h -> plan db h seed | seed <- seeds]-          let count p = length (filter p outcomes)-              bucket n-                | n == 0 = "none"-                | n <= 3 = "1 to 3"-                | n <= 9 = "4 to 9"-                | otherwise = "10 or more" :: String-          return-            . tabulate "operations refused" [bucket (count (isLeft . (.result)))]-            . tabulate "operations that removed from the mempool" [bucket (count (not . null . (.removed)))]-            $ tabulate "blocks at the end" [bucket (length hist.chain)] True--  it "give a transaction that is confirmed twice the place of the later block, as today's" $ do-    -- the coinbase of the first block again, in the second; then its output-    -- is spent, the spend confirmed, and both blocks reverted-    let twice = coinbaseTx 1-        spend = Tx 2 [TxIn (OutPoint (txHash twice) 0) "" maxBound] [TxOut 4000 (p2pkh 1)] [] 0-        mine txs db h = (h,) . Just <$> blockOn db txs-        pay tx _ h = return (h, Just (Pay tx h.clock))-        revert _ h = return $ case h.chain of-          (b, _) : _ -> (h, Just (Revert (headerHash b.header) h.clock))-          [] -> (h, Nothing)-    -- the third store is upgraded with the spend in its mempool-    (outcomes, _) <- sameHistory ctx btcRegTest 2 [mine [twice], pay spend, mine [spend], revert, revert]-    map (either show (const "done") . (.result)) outcomes `shouldBe` replicate 5 "done"--  it "refuse a block that spends an output of the mempool, which today's rules import" $-    withSystemTempDirectory "haskoin-store-layered-" $ \dir ->-      openLayered btcRegTest ctx dir $ \db -> do-        hist <- start layeredRules db-        let parent = Tx 2 [TxIn (coinOf hist) "" maxBound] [TxOut 4000 script] [] 0-            child = Tx 2 [TxIn (OutPoint (txHash parent) 0) "" maxBound] [TxOut 3000 (p2pkh 1)] [] 0-        (.result) <$> apply layeredRules db (Pay parent 1000) `shouldReturn` Right "True"-        untouched <- everything db-        -- the parent is not in the block, so its output is not in the chain-        block <- blockOn db [child]-        (.result) <$> apply layeredRules db block `shouldReturn` Left Orphan-        everything db `shouldReturn` untouched--  it "keep what a block replaced, and forget it when the block is reverted" $-    withSystemTempDirectory "haskoin-store-layered-" $ \dir ->-      openLayered btcRegTest ctx dir $ \db -> do-        (gone, height) <- replacedByBlock db-        best <- Q.getBestBlock db >>= maybe (fail "no best block") return-        isRight . (.result) <$> apply layeredRules db (Revert best 2000) `shouldReturn` True-        mapM (Q.getReplaced db) gone `shouldReturn` [Nothing, Nothing]-        Q.getReplacedAt db height `shouldReturn` []--  it "forget what a block replaced 2016 blocks later" $-    withSystemTempDirectory "haskoin-store-layered-" $ \dir ->-      openLayered btcRegTest ctx dir $ \db -> do-        (gone, height) <- replacedByBlock db-        let mineEmpty = blockOn db [] >>= apply layeredRules db-        replicateM_ (fromIntegral replacedKept - 1) mineEmpty-        all isJust <$> mapM (Q.getReplaced db) gone `shouldReturn` True-        void mineEmpty-        mapM (Q.getReplaced db) gone `shouldReturn` [Nothing, Nothing]-        Q.getReplacedAt db height `shouldReturn` []--  it "leave the keys of a mempool transaction alone when a block confirms what it spends" $-    withSystemTempDirectory "haskoin-store-layered-" $ \dir ->-      openLayered btcRegTest ctx dir $ \db -> do-        hist <- start layeredRules db-        let parent = Tx 2 [TxIn (coinOf hist) "" maxBound] [TxOut 4000 script, TxOut 5000 (p2pkh 2)] [] 0-            child = Tx 2 [TxIn (OutPoint (txHash parent) 0) "" maxBound] [TxOut 3000 (p2pkh 1)] [] 0-            keysOf tx = do-              Just td <- M.getMempoolTx db (txHash tx)-              let r = M.mempoolRecords db.net db.ctx 1000 td-              return (fst (M.recordEntries r), M.addressArea [r])-        forM_ [parent, child] $ \tx ->-          (.result) <$> apply layeredRules db (Pay tx 1000) `shouldReturn` Right "True"-        (general, byAddress) <- keysOf child-        block <- blockOn db [parent]-        confirmed <- apply layeredRules db block-        confirmed.removed `shouldBe` [txHash parent]-        -- the child's keys are all that is left of the mempool, unchanged-        dump db (mempoolCF db.db) `shouldReturn` sortOn fst general-        dump db (mempoolAddrCF db.db) `shouldReturn` sortOn fst byAddress-        -- and the spent output of the parent, now in the chain, is still-        -- spent to anyone who asks-        L.withView db (\v -> L.getUnspent v (OutPoint (txHash parent) 0)) `shouldReturn` Nothing-        (fmap (.value) <$> Q.getUnspent db (OutPoint (txHash parent) 0)) `shouldReturn` Just 4000---- | Give the same history to three stores, each with a database of its--- own:------ * one that runs the rules of today's layout;------ * one that runs the rules of the layered layout;------ * one that is upgraded on the way: it runs today's rules for the first---   operations, as many as a number says, then its database is migrated,---   and it runs the rules of the layered layout from there on.------ Each operation is chosen from the first database and the history so far.--- Every operation must have the same outcome in all of them, and leave the--- same answers to every query. At the end all are asked for the same pages,--- chosen from the number. Then the first database is migrated: it must equal--- the second key for key, and the third in everything but the bodies of--- deleted transactions, which each set of rules leaves in its own place.--- Gives the outcomes and the history.-sameHistory ::-  Ctx ->-  Network ->-  Int ->-  [DatabaseReader -> History -> IO (History, Maybe Op)] ->-  IO ([Outcome], History)-sameHistory ctx net number choices =-  withSystemTempDirectory "haskoin-store-mixed-" $ \dirOld ->-    withSystemTempDirectory "haskoin-store-layered-" $ \dirNew ->-      withSystemTempDirectory "haskoin-store-upgraded-" $ \dirUp -> do-        -- the store to upgrade begins with a database of version 20-        open20 net ctx dirUp (\_ -> return ())-        openLayered net ctx dirNew $ \new ->-          openLayered net ctx dirUp $ \up -> do-            (outcomes, hist) <- open20 net ctx dirOld $ \old -> do-              hist0 <- start mixedRules old-              void (start layeredRules new)-              void (start mixedRules up)-              let (earlier, later) = splitAt (number `mod` (length choices + 1)) choices-                  today op _ _ _ = void (apply mixedRules up op)-                  upgraded op was hist answers = do-                    now <- apply layeredRules up op-                    (show op, now) `shouldBe` (show op, was)-                    M.wrongSummaries up `shouldReturn` []-                    layered up (ask hist) `shouldReturn` answers-              half <- foldM (next old new today) ([], hist0) earlier-              void (upgradeDatabase (\_ -> return ()) up)-              done@(_, hist) <- foldM (next old new upgraded) half later-              -- pages, of databases that the new rules built or carried on-              -- with, against today's-              let asked = unGen (questions net ctx hist 60) (mkQCGen number) 30-              was <- mixed old (\queries -> mapM (answer queries) asked)-              -- but for pages that start at a deleted transaction, which-              -- the layered layout does not know-              known <- mapM (startsKnown old) asked-              let both :: [x] -> [x]-                  both xs = [x | (x, True) <- zip xs known]-              forM_ [new, up] $ \db -> do-                now <- layered db (\queries -> mapM (answer queries) asked)-                take 2 (differences (both asked) (both was) (both now)) `shouldBe` []-              -- the queries of a fleet do not know deleted transactions, so-              -- they are not asked for pages that start at one-              let live queries q = case startOf q of-                    Just (Just (AtTx h)) -> isJust <$> queries.getTxData h-                    _ -> return True-              fair <- layered new (\queries -> mapM (live queries) asked)-              let kept :: [x] -> [x]-                  kept xs = [x | (x, True) <- zip xs fair]-              shared <- pooled new (\queries -> mapM (answer queries) (kept asked))-              take 2 (differences (kept asked) (kept was) shared) `shouldBe` []-              return done-            migrated <- openLayered net ctx dirOld $ \old -> upgradeDatabase (\_ -> return ()) old >> everything old-            mine <- everything new-            -- the same, but for the bodies of deleted transactions, which-            -- version 20 left in the chain: the layered rules keep none-            mine `shouldBe` migrated {inChain = [], deleted = []}-            -- the upgraded store has those that it deleted before the-            -- upgrade in the chain, as version 20 left them, even when the-            -- transaction came back later: the queries, compared after-            -- every operation, do not answer with those-            theirs <- everything up-            theirs {deleted = [], inChain = []} `shouldBe` migrated {deleted = [], inChain = []}-            return (reverse outcomes, hist)-  where-    next old new third (outcomes, hist) chosen =-      chosen old hist >>= \case-        (hist', Nothing) -> return (outcomes, hist')-        (hist', Just op) -> do-          was <- apply mixedRules old op-          now <- apply layeredRules new op-          (show op, now) `shouldBe` (show op, was)-          -- each summary is what the entries of its address add up to-          M.wrongSummaries new `shouldReturn` []-          -- and every answer is the same after it-          let hist'' = record old hist' op was-          answers <- mixed old (ask hist'')-          layered new (ask hist'') `shouldReturn` answers-          -- from the queries of a fleet as well, whose pool is here the-          -- store's own mempool-          pooled new (ask hist'') `shouldReturn` answers-          third op was hist'' answers :: IO ()-          return (was : outcomes, hist'')---- | Everything in a database of the layered layout.-data Everything = Everything-  { best :: !(Maybe BlockHash),-    blocks :: ![(ByteString, ByteString)],-    heights :: ![(ByteString, ByteString)],-    chain :: !Chain,-    mempool :: ![(ByteString, ByteString)],-    mempoolByAddress :: ![(ByteString, ByteString)],-    -- | transactions that left the mempool unconfirmed, wherever their-    -- bodies are kept-    deleted :: ![(TxHash, TxData)],-    -- | those of them that are kept in the chain, as version 20 did-    inChain :: ![TxHash]-  }-  deriving (Eq, Show)--everything :: DatabaseReader -> IO Everything-everything db = do-  best <- Q.getBestBlock db-  -- what is kept of replaced transactions is the layered layout's own,-  -- and tested on its own-  blocks <- filter (not . ("\x11" `B.isPrefixOf`) . fst) <$> dump db (blockCF db.db)-  heights <- filter (not . ("\x12" `B.isPrefixOf`) . fst) <$> dump db (heightCF db.db)-  chain <- chainOf db-  area <- dump db (mempoolCF db.db)-  mempoolByAddress <- dump db (mempoolAddrCF db.db)-  bodies <- dump db (txCF db.db) >>= mapM (decoded . snd)-  let (gone, mempool) = span' ((== "\x0e") . B.take 1 . fst) area-      old = [td | td <- bodies, td.deleted]-  new <- mapM (decoded . snd) gone-  return-    Everything-      { best,-        blocks,-        heights,-        chain,-        mempool,-        mempoolByAddress,-        deleted = sortOn fst [(txHash td.tx, td) | td <- old <> new],-        inChain = map (txHash . (.tx)) old-      }-  where-    decoded :: ByteString -> IO TxData-    decoded = either fail return . decode-    span' p xs = (filter p xs, filter (not . p) xs)---- | A payment and its child in the mempool, and a block whose transaction--- spends what the payment spent: both are replaced. Gives their hashes and--- the height of the block, after checking what is kept of them.-replacedByBlock :: DatabaseReader -> IO ([TxHash], BlockHeight)-replacedByBlock db = do-  hist <- start layeredRules db-  let paid = Tx 2 [TxIn (coinOf hist) "" maxBound] [TxOut 4000 (p2pkh 1)] [] 0-      child = Tx 2 [TxIn (OutPoint (txHash paid) 0) "" maxBound] [TxOut 3000 (p2pkh 2)] [] 0-      rival = Tx 2 [TxIn (coinOf hist) "" maxBound] [TxOut 4500 (p2pkh 3)] [] 0-      gone = map txHash [paid, child]-  forM_ [paid, child] $ \tx ->-    (.result) <$> apply layeredRules db (Pay tx 1000) `shouldReturn` Right "True"-  block <- blockOn db [rival]-  replaced <- apply layeredRules db block-  sort replaced.removed `shouldBe` sort gone-  best <- Q.getBestBlock db >>= maybe (fail "no best block") return-  height <- maybe (fail "no block") (return . (.height)) =<< Q.getBlock db best-  -- the payment, and its child, which spent what it paid-  let what = Replaced {by = txHash rival, height = height}-  mapM (Q.getReplaced db) gone `shouldReturn` [Just what, Just what]-  Q.getReplacedAt db height `shouldReturn` sort gone-  return (gone, height)
test/Haskoin/Store/LogicSpec.hs view
@@ -1,492 +1,95 @@ module Haskoin.Store.LogicSpec (spec) where -import Control.Monad (forM_, void)+import Control.Monad (forM) import Data.ByteString qualified as B-import Data.Functor ((<&>))-import Data.List (sort, sortOn)-import Data.Maybe (fromMaybe)-import Data.Ord (Down (..))-import Data.Text qualified as T-import Data.Word (Word32, Word64)-import Database.RocksDB.Query (insert, insertCF, matchingAsListCF, retrieve)+import Data.Serialize (encode)+import Database.RocksDB (columnFamilies) import Haskoin-  ( Address,-    Block (..),-    BlockHash,-    BlockHeader (..),-    BlockNode (..),-    Ctx,-    Network,+  ( Block (..),+    Network (..),     OutPoint (..),     Tx (..),     TxHash (..),     TxIn (..),     TxOut (..),-    bchRegTest,     btcRegTest,-    genesisNode,-    headerHash,-    prepareContext,-    scriptToAddressBS,     sha256,     txHash,   )-import Haskoin.Node (LogLevel (..), Logger (..))-import Haskoin.Store.Common (Limits (..), StoreRead (..))-import Haskoin.Store.Data-  ( Balance (..),-    BlockData (..),-    BlockRef (..),-    Spender (..),-    TxData (..),-    Unspent (..),-    zeroBalance,-  )+import Haskoin.Store.Common (sortTxs)+import Haskoin.Store.Data (BlockData (..)) import Haskoin.Store.Database.Query qualified as Q-import Haskoin.Store.Database.Reader (DatabaseReader (..), memoryReport, mempoolCF, withDatabaseReader)-import Haskoin.Store.Database.Types (MemKey (..), MemTimeKey (..), MemTxKey (..), VersionKey (..))-import Haskoin.Store.Database.Writer-  ( MempoolChanges (..),-    Writer,-    addToMempool,-    deleteFromMempool,-    runWriter,-    runWriterMempool,-  )-import Haskoin.Store.Logic-  ( ImportException (..),-    deleteUnconfirmedTx,-    importBlock,-    initBest,-    newMempoolTx,-    revertBlock,-  )-import Haskoin.Store.TestBlocks (coinbaseTx, nextBlock, script)-import System.Log.FastLogger (fromLogStr)+import Haskoin.Store.Database.Reader (DatabaseReader (..))+import Haskoin.Store.Histories+import Haskoin.Store.Integration.Temp (withSystemTempDirectory)+import Haskoin.Store.Logic (PreparedBlock (..), PreparedTx (..), prepareBlock)+import Haskoin.Store.TestBlocks (coinbaseTx, script) import Test.Hspec-import UnliftIO (catch, evaluate, try, withSystemTempDirectory)+import Test.Hspec.QuickCheck (modifyMaxSuccess)+import Test.QuickCheck (Gen, choose, forAll, ioProperty, property, vectorOf)  spec :: Spec-spec = prepareContext $ \ctx -> do-  describe "initBest" $-    it "imports the genesis block once" $-      withDB btcRegTest ctx $ \db -> do-        importing db initBest `shouldReturn` Right ()-        importing db initBest `shouldReturn` Right ()-        Q.getBestBlock db `shouldReturn` Just genesis--  describe "memoryReport" $-    it "lists the memory of every column family" $-      withCoin btcRegTest ctx $ \db _ -> do-        report <- memoryReport db.db-        forM_ ["default", "addr-tx", "unspent", "balance", "total", "block cache"] $ \name ->-          report `shouldSatisfy` T.isInfixOf name-        report `shouldNotSatisfy` T.isInfixOf "?"--  describe "data version 18" $-    it "moves the mempool into its column family when opened" $-      withSystemTempDirectory "haskoin-store-migrate-" $ \dir -> do-        let open :: (DatabaseReader -> IO ()) -> IO ()-            open = withDatabaseReader btcRegTest ctx 20 32 dir False Nothing Nothing Nothing Nothing Nothing-            h1 = TxHash (sha256 ("one" :: B.ByteString))-            h2 = TxHash (sha256 ("two" :: B.ByteString))-            old = [(200, h2), (100, h1)] :: [(Word64, TxHash)]-        open $ \r -> do-          insert r.db VersionKey (18 :: Word32)-          insert r.db MemKey old-        open $ \r -> do-          Q.getMempool r `shouldReturn` old-          Q.getMempoolPage r (Limits 1 1 Nothing) `shouldReturn` drop 1 old-          (retrieve r.db MemKey :: IO (Maybe [(Word64, TxHash)])) `shouldReturn` Nothing-          retrieve r.db VersionKey `shouldReturn` Just (20 :: Word32)--  describe "data version 19" $-    it "indexes the mempool by time when opened" $-      withSystemTempDirectory "haskoin-store-migrate-" $ \dir -> do-        let open :: (DatabaseReader -> IO ()) -> IO ()-            open = withDatabaseReader btcRegTest ctx 20 32 dir False Nothing Nothing Nothing Nothing Nothing-            old = sortOn Down (zip [300, 100, 200] (hashes 3))-        open $ \r -> do-          insert r.db VersionKey (19 :: Word32)-          forM_ old $ \(t, h) -> insertCF r.db (mempoolCF r.db) (MemTxKey h) t-          (matchingAsListCF r.db (mempoolCF r.db) MemTimeKeyS :: IO [(MemTimeKey, ())])-            `shouldReturn` []-        open $ \r -> do-          Q.getMempool r `shouldReturn` old-          retrieve r.db VersionKey `shouldReturn` Just (20 :: Word32)--  describe "mempool index" $ do-    let entries = zip [5, 3, 5, 1, 4, 3, 2] (hashes 7)-        newest = sortOn Down entries-        fill db = importing db (\w -> forM_ entries (\(t, h) -> addToMempool w h t))-    it "pages the mempool newest first" $-      withDB btcRegTest ctx $ \db -> do-        _ <- fill db-        Q.getMempool db `shouldReturn` newest-        forM_ [(0, 0), (3, 0), (3, 2), (2, 5), (4, 7), (0, 3)] $ \(l, o) ->-          Q.getMempoolPage db (Limits l o Nothing)-            `shouldReturn` take' l (drop (fromIntegral o) newest)-    it "reads the oldest entries before a time" $-      withDB btcRegTest ctx $ \db -> do-        _ <- fill db-        let oldest = reverse newest-        Q.getMempoolBefore db 4 10 `shouldReturn` takeWhile ((< 4) . fst) oldest-        Q.getMempoolBefore db 4 2 `shouldReturn` take 2 oldest-        Q.getMempoolBefore db 1 10 `shouldReturn` []-    it "reports and indexes what a batch changes" $-      withDB btcRegTest ctx $ \db -> do-        let hs = hashes 3-            (h1, h2, h3) = (hs !! 0, hs !! 1, hs !! 2)-            batch :: (Writer -> IO ()) -> IO MempoolChanges-            batch = fmap snd . runWriterMempool db.net db.ctx db quietLogger-        c <- batch (\w -> addToMempool w h1 10 >> addToMempool w h2 20)-        (sort c.added, c.removed) `shouldBe` (sort [h1, h2], [])-        -- a new time moves the entry in the index, and is no change-        batch (\w -> addToMempool w h1 30) `shouldReturn` MempoolChanges [] []-        Q.getMempool db `shouldReturn` [(30, h1), (20, h2)]-        -- removing what is not there, or adding and removing in one batch,-        -- is no change either-        batch (\w -> deleteFromMempool w h3) `shouldReturn` MempoolChanges [] []-        batch (\w -> addToMempool w h3 40 >> deleteFromMempool w h3) `shouldReturn` MempoolChanges [] []-        batch (\w -> deleteFromMempool w h2) `shouldReturn` MempoolChanges [] [h2]-        Q.getMempool db `shouldReturn` [(30, h1)]-        (matchingAsListCF db.db (mempoolCF db.db) MemTimeKeyS :: IO [(MemTimeKey, ())])-          `shouldReturn` [(MemTimeKey 30 h1, ())]--  describe "newMempoolTx" $ do-    it "imports a transaction and spends its input" $-      withCoin btcRegTest ctx $ \db coin -> do-        let tx = spend [coin] [1000, 2000]-        importing db (\w -> newMempoolTx w tx 1700000000) `shouldReturn` Right True-        Q.getMempool db `shouldReturn` [(1700000000, txHash tx)]-        Q.getSpender db coin `shouldReturn` Just (Spender (txHash tx) 0)-        fmap (.value) <$> Q.getUnspent db (out tx 1) `shouldReturn` Just 2000--    it "ignores a transaction it already has" $-      withCoin btcRegTest ctx $ \db coin -> do-        let tx = spend [coin] [1000]-        mapM (addTx db) [tx, tx] `shouldReturn` [Right True, Right False]-        _ <- mine db [tx]-        addTx db tx `shouldReturn` Right False-        fmap (.block) <$> txData db tx `shouldReturn` Just (BlockRef 2 1)--    it "rejects invalid transactions" $-      withCoin btcRegTest ctx $ \db coin -> do-        addTx db (spend [out (coinbaseTx 9) 0] [1]) `shouldReturn` Left Orphan-        addTx db (spend [coin, coin] [1]) `shouldReturn` Left DuplicatePrevOutput-        addTx db (spend [coin] [5000000001]) `shouldReturn` Left InsufficientFunds-        addTx db (coinbaseTx 9) `shouldReturn` Left UnexpectedCoinbase-        mempool db `shouldReturn` []--    it "flags BIP125 signalling transactions and their descendants" $-      withCoin btcRegTest ctx $ \db coin -> do-        let parent = spendRBF [coin] [1000]-            child = spend [out parent 0] [900]-        importing db (\w -> mapM (\t -> newMempoolTx w t 0) [parent, child])-          `shouldReturn` Right [True, True]-        mapM (rbfFlag db) [parent, child] `shouldReturn` [Just True, Just True]--    it "does not flag other transactions" $ do-      withCoin btcRegTest ctx $ \db coin -> do-        let parent = spend [coin] [1000]-            child = spend [out parent 0] [900]-        mapM (addTx db) [parent, child] `shouldReturn` [Right True, Right True]-        mapM (rbfFlag db) [parent, child] `shouldReturn` [Just False, Just False]-      withCoin bchRegTest ctx $ \db coin -> do-        let tx = spendRBF [coin] [1000]-        addTx db tx `shouldReturn` Right True-        rbfFlag db tx `shouldReturn` Just False--    it "replaces conflicting transactions and their descendants" $-      withCoin btcRegTest ctx $ \db coin -> do-        let original = spend [coin] [1000]-            child = spend [out original 0] [900]-            replacement = spend [coin] [2000]-        mapM (addTx db) [original, child, replacement]-          `shouldReturn` [Right True, Right True, Right True]-        mempool db `shouldReturn` [txHash replacement]-        Q.getSpender db coin `shouldReturn` Just (Spender (txHash replacement) 0)-        fmap (.deleted) <$> Q.getTxData db (txHash original) `shouldReturn` Just True-        -- the database keeps the replaced ones, and the queries that the API-        -- answers with do not return them, by hash or among others-        let queries = Q.databaseRead db-            wanted = map txHash [original, child, replacement]-        queries.getTxData (txHash original) `shouldReturn` Nothing-        map (fmap (txHash . (.tx))) <$> queries.getManyTxData wanted-          `shouldReturn` [Nothing, Nothing, Just (txHash replacement)]--    it "replaces a transaction and its descendant when both conflict" $-      withCoin btcRegTest ctx $ \db coin1 -> do-        _ <- mine db []-        let coin2 = out (coinbaseTx 2) 0-            parent = spend [coin1] [1000]-            child = spend [out parent 0, coin2] [900]-            replacement = spend [coin1, coin2] [2000]-        mapM (addTx db) [parent, child, replacement]-          `shouldReturn` [Right True, Right True, Right True]-        mempool db `shouldReturn` [txHash replacement]--    it "leaves the mempool unchanged when it rejects a replacement" $-      withCoin btcRegTest ctx $ \db coin1 -> do-        _ <- mine db []-        let coin2 = out (coinbaseTx 2) 0-            original = spend [coin1] [1000]-        _ <- mine db [spend [coin2] [1000]]-        addTx db original `shouldReturn` Right True-        -- The block store imports mempool transactions in batches and skips-        -- the ones that are rejected.-        let tryAdd w tx = newMempoolTx w tx 0 `catch` skip-            skip :: ImportException -> IO Bool-            skip _ = return False-            unfunded = spend [coin1] [5000000001]-            confirmedConflict = spend [coin1, coin2] [2000]-            spendsReplaced = spend [coin1, out original 0] [900]-        importing db (\w -> mapM (tryAdd w) [unfunded, confirmedConflict, spendsReplaced])-          `shouldReturn` Right [False, False, False]-        mempool db `shouldReturn` [txHash original]--    it "rejects double spends on networks without RBF" $-      withCoin bchRegTest ctx $ \db coin -> do-        let original = spendRBF [coin] [1000]-        mapM (addTx db) [original, spend [coin] [2000]]-          `shouldReturn` [Right True, Left DoubleSpend]-        mempool db `shouldReturn` [txHash original]--    it "rejects double spends of confirmed transactions" $-      withCoin btcRegTest ctx $ \db coin -> do-        _ <- mine db [spend [coin] [1000]]-        addTx db (spend [coin] [2000]) `shouldReturn` Left TxConfirmed--    it "commits nothing from a batch that fails" $-      withCoin btcRegTest ctx $ \db coin -> do-        let good = spend [coin] [1000]-            orphan = spend [out (coinbaseTx 9) 0] [1]-        importing db (\w -> newMempoolTx w good 0 >> newMempoolTx w orphan 0)-          `shouldReturn` Left Orphan-        mempool db `shouldReturn` []-        Q.getSpender db coin `shouldReturn` Nothing--  describe "importBlock" $ do-    it "confirms mempool transactions" $-      withCoin btcRegTest ctx $ \db coin -> do-        let tx = spendRBF [coin] [1000]-        addTx db tx `shouldReturn` Right True-        _ <- mine db [tx]-        mempool db `shouldReturn` []-        fmap (\t -> (t.block, t.rbf)) <$> txData db tx `shouldReturn` Just (BlockRef 2 1, False)-        fmap (.block) <$> Q.getUnspent db (out tx 0) `shouldReturn` Just (BlockRef 2 1)--    it "confirms a transaction whose outputs are spent in the mempool" $-      withCoin btcRegTest ctx $ \db coin -> do-        let parent = spend [coin] [1000]-            child = spend [out parent 0] [900]-        mapM (addTx db) [parent, child] `shouldReturn` [Right True, Right True]-        _ <- mine db [parent]-        mempool db `shouldReturn` [txHash child]-        fmap (.block) <$> txData db parent `shouldReturn` Just (BlockRef 2 1)-        Q.getSpender db (out parent 0) `shouldReturn` Just (Spender (txHash child) 0)--    it "stops flagging a descendant once its signalling ancestor confirms" $-      withCoin btcRegTest ctx $ \db coin -> do-        let parent = spendRBF [coin] [1000]-            child = spend [out parent 0] [900]-        mapM (addTx db) [parent, child] `shouldReturn` [Right True, Right True]-        _ <- mine db [parent]-        rbfFlag db child `shouldReturn` Just False--    it "imports transactions in any order and records the block" $-      withCoin btcRegTest ctx $ \db coin -> do-        let parent = spend [coin] [4000000000]-            child = spend [out parent 0] [3999990000]-        block <- mine db [child, parent]-        Just bd <- Q.getBlock db (headerHash block.header)-        (bd.height, bd.main, bd.txs) `shouldBe` (2, True, map txHash block.txs)-        (bd.subsidy, bd.fee, bd.outputs) `shouldBe` (5000000000, 1000010000, 7999990000)-        fmap (.block) <$> txData db child `shouldReturn` Just (BlockRef 2 1)--    it "evicts conflicting mempool transactions and their descendants" $-      forM_ [btcRegTest, bchRegTest] $ \net ->-        withCoin net ctx $ \db coin -> do-          let original = spend [coin] [1000]-              child = spend [out original 0] [900]-          mapM (addTx db) [original, child] `shouldReturn` [Right True, Right True]-          _ <- mine db [spend [coin] [2000]]-          mempool db `shouldReturn` []-          mapM (txData db) [original, child] `shouldReturn` [Nothing, Nothing]--    it "rejects blocks that spend unknown outputs, changing nothing" $-      withCoin btcRegTest ctx $ \db _ -> do-        best <- Q.getBestBlock db-        tryMine db [spend [out (coinbaseTx 9) 0] [1]] `shouldReturn` Left Orphan-        Q.getBestBlock db `shouldReturn` best--    it "rejects blocks that do not build on the best block" $-      withDB btcRegTest ctx $ \db -> do-        let (block1, node1) = nextBlock btcRegTest 0 (genesisNode btcRegTest).header []-            (block2, node2) = nextBlock btcRegTest 1 node1.header []-        importing db (\w -> void (importBlock w block1 node1)) `shouldReturn` Left BestBlockNotFound-        importing db initBest `shouldReturn` Right ()-        importing db (\w -> void (importBlock w block2 node2)) `shouldReturn` Left PrevBlockNotBest--  describe "revertBlock" $ do-    it "returns the block's transactions to the mempool" $-      withCoin btcRegTest ctx $ \db coin -> do-        let tx = spendRBF [coin] [1000]-        block <- mine db [tx]-        let h = headerHash block.header-        importing db (`revertBlock` h) `shouldReturn` Right ()-        Q.getBestBlock db `shouldReturn` Just block.header.prev-        fmap (.main) <$> Q.getBlock db h `shouldReturn` Just False-        Q.getMempool db `shouldReturn` [(1296689802, txHash tx)]-        rbfFlag db tx `shouldReturn` Just True-        txData db (coinbaseTx 2) `shouldReturn` Nothing--    it "flags mempool descendants of the signalling transactions it returns" $-      withCoin btcRegTest ctx $ \db coin -> do-        let parent = spendRBF [coin] [1000]-            child = spend [out parent 0] [900]-        block <- mine db [parent]-        addTx db child `shouldReturn` Right True-        rbfFlag db child `shouldReturn` Just False-        importing db (`revertBlock` (headerHash block.header)) `shouldReturn` Right ()-        rbfFlag db child `shouldReturn` Just True--    it "only reverts the best block" $-      withDB btcRegTest ctx $ \db -> do-        importing db (`revertBlock` genesis) `shouldReturn` Left BestBlockNotFound-        importing db initBest `shouldReturn` Right ()-        _ <- mine db []-        importing db (`revertBlock` genesis) `shouldReturn` Left BlockNotBest--  describe "deleteUnconfirmedTx" $ do-    it "deletes a transaction and its descendants" $-      withCoin btcRegTest ctx $ \db coin -> do-        let parent = spend [coin] [1000]-            child = spend [out parent 0] [900]-        mapM (addTx db) [parent, child] `shouldReturn` [Right True, Right True]-        importing db (\w -> deleteUnconfirmedTx w False (txHash parent)) `shouldReturn` Right ()-        mempool db `shouldReturn` []-        fmap (.value) <$> Q.getUnspent db coin `shouldReturn` Just 5000000000--    it "ignores unknown transactions" $-      withCoin btcRegTest ctx $ \db _ ->-        importing db (\w -> deleteUnconfirmedTx w False (txHash (coinbaseTx 9)))-          `shouldReturn` Right ()--    it "refuses to delete confirmed transactions" $-      withCoin btcRegTest ctx $ \db _ ->-        importing db (\w -> deleteUnconfirmedTx w False (txHash (coinbaseTx 1)))-          `shouldReturn` Left TxConfirmed--  describe "address balances" $-    it "follow transactions through the mempool, a block and its reversal" $-      withCoin btcRegTest ctx $ \db coin -> do-        a <- either fail return (scriptToAddressBS btcRegTest ctx script)-        let tx = spend [coin] [1000, 2000]-            -- confirmed, unconfirmed, outputs, transactions, received-            balance = do-              b <- defaultBalance db a-              return (b.confirmed, b.unconfirmed, b.utxo, b.txs, b.received)-        balance `shouldReturn` (5000000000, 0, 1, 1, 5000000000)-        addTx db tx `shouldReturn` Right True-        balance `shouldReturn` (0, 3000, 2, 2, 5000003000)-        block <- mine db [tx]-        balance `shouldReturn` (5000003000, 0, 3, 3, 10000003000)-        importing db (`revertBlock` (headerHash block.header)) `shouldReturn` Right ()-        balance `shouldReturn` (0, 3000, 2, 2, 5000003000)--  describe "a coin whose script has no address on the network" $-    it "is kept without an address" $-      withCoin btcRegTest ctx $ \db coin -> do-        -- OP_HASH256, a push of 32 bytes, OP_EQUAL. Bitcoin Cash has an-        -- address for this script. Bitcoin has none.-        let hashLock = "\xaa\x20" <> B.replicate 32 0xbb <> "\x87"-            tx = Tx 2 [TxIn coin "" maxBound] [TxOut 1000 hashLock] [] 0-        addTx db tx `shouldReturn` Right True-        coinAddress <- fmap (.address) <$> Q.getUnspent db (out tx 0)-        coinAddress `shouldBe` Just Nothing---- | Run with an empty database in a temporary directory.-withDB :: Network -> Ctx -> (DatabaseReader -> IO a) -> IO a-withDB net ctx go =-  withSystemTempDirectory "haskoin-store-logic-" $ \dir ->-    withDatabaseReader net ctx 20 32 dir False Nothing Nothing Nothing Nothing Nothing go---- | Run with the genesis block and one mined block, passing the output of--- that block's coinbase, worth 5000000000.-withCoin :: Network -> Ctx -> (DatabaseReader -> OutPoint -> IO a) -> IO a-withCoin net ctx go =-  withDB net ctx $ \db -> do-    importing db initBest `shouldReturn` Right ()-    _ <- mine db []-    go db (out (coinbaseTx 1) 0)--importing :: DatabaseReader -> (Writer -> IO a) -> IO (Either ImportException a)-importing db = try . runWriter db.net db.ctx db quietLogger---- | Build every message, so that log lines are tested too, but print none.-quietLogger :: Logger-quietLogger =-  Logger-    { level = LevelDebug,-      source = "",-      push = void . evaluate . B.length . fromLogStr-    }---- | Distinct transaction hashes.-hashes :: Int -> [TxHash]-hashes n = [TxHash (sha256 (B.singleton (fromIntegral i))) | i <- [1 .. n]]---- | 'take' where a limit of 0 means no limit, as in 'Limits'.-take' :: Word32 -> [a] -> [a]-take' 0 = id-take' l = take (fromIntegral l)---- | Balance of an address, or a zero balance if never used.-defaultBalance :: DatabaseReader -> Address -> IO Balance-defaultBalance db a = fromMaybe (zeroBalance a) <$> Q.getBalance db a--addTx :: DatabaseReader -> Tx -> IO (Either ImportException Bool)-addTx db tx = importing db (\w -> newMempoolTx w tx 0)--txData :: DatabaseReader -> Tx -> IO (Maybe TxData)-txData db tx =-  Q.getTxData db (txHash tx) <&> \case-    Just td | not td.deleted -> Just td-    _ -> Nothing--rbfFlag :: DatabaseReader -> Tx -> IO (Maybe Bool)-rbfFlag db tx = fmap (.rbf) <$> txData db tx--mempool :: DatabaseReader -> IO [TxHash]-mempool db = sort . map snd <$> Q.getMempool db---- | Mine a block with these transactions on top of the best block.-tryMine :: DatabaseReader -> [Tx] -> IO (Either ImportException Block)-tryMine db txs = do-  Just h <- Q.getBestBlock db-  Just parent <- Q.getBlock db h-  let (block, node) = nextBlock db.net parent.height parent.header txs-  fmap (const block) <$> importing db (\w -> importBlock w block node)--mine :: DatabaseReader -> [Tx] -> IO Block-mine db txs = tryMine db txs >>= either (fail . ("Block rejected: " <>) . show) return---- | Spend outputs into new ones of the given values.-spend :: [OutPoint] -> [Word64] -> Tx-spend = spendWith maxBound---- | Like 'spend', but signal BIP125 replaceability.-spendRBF :: [OutPoint] -> [Word64] -> Tx-spendRBF = spendWith 0--spendWith :: Word32 -> [OutPoint] -> [Word64] -> Tx-spendWith s ops vs =-  Tx 2 [TxIn op "" s | op <- ops] [TxOut v script | v <- vs] [] 0+spec = describe "the rules of the chain" $ do+  it "take out with a revert exactly what the import of a block put in" $+    withSystemTempDirectory "haskoin-store-logic-" $ \dir ->+      openLayered btcRegTest dir $ \db -> do+        startChain db+        first <- mine db []+        coinbase <-+          Q.getBlock db first >>= \case+            Just bd | h : _ <- bd.txs -> return h+            _ -> fail "no coinbase"+        kept <- chainFamilies db+        let spend =+              Tx+                1+                [TxIn (OutPoint coinbase 0) "" maxBound]+                [TxOut 1000 script, TxOut 2000 ("\x00\x14" <> B.replicate 20 0x22)]+                []+                0+        second <- mine db [spend]+        Q.getTxData db (txHash spend) >>= (`shouldSatisfy` maybe False (const True))+        Q.getUnspent db (OutPoint coinbase 0) >>= (`shouldBe` Nothing) . fmap (const ())+        revert db+        Q.getBestBlock db `shouldReturn` Just first+        fmap (.main) <$> Q.getBlock db second `shouldReturn` Just False+        Q.getTxData db (txHash spend) >>= (`shouldBe` Nothing) . fmap (const ())+        chainFamilies db `shouldReturn` kept -out :: Tx -> Word32 -> OutPoint-out tx = OutPoint (txHash tx)+  describe "prepareBlock" $+    modifyMaxSuccess (const 50) $+      it "gives the hash of each transaction, the order of sortTxs, and the size and weight of the block" $+        property . forAll payments $ \txs -> ioProperty $ do+          let block = Block btcRegTest.genesisHeader (coinbaseTx 1 : txs)+              stripped = (block :: Block) {txs = map (\tx -> (tx :: Tx) {witness = []}) block.txs}+          pb <- prepareBlock btcRegTest block+          map (.txid) pb.txs `shouldBe` map txHash block.txs+          map (.tx) pb.txs `shouldBe` block.txs+          [(i, p.tx) | (i, p) <- pb.ordered] `shouldBe` sortTxs block.txs+          pb.size `shouldBe` B.length (encode block)+          pb.weight `shouldBe` 3 * B.length (encode stripped) + B.length (encode block)+  where+    -- the column families of the chain whose keys a block adds and a revert+    -- takes out: addresses, outputs, transactions, unspent outputs,+    -- spenders and balances+    chainFamilies db = forM [0, 1, 2, 3, 4, 7] $ \i -> dump db (columnFamilies db.db !! i) --- | Bitcoin regtest genesis block.-genesis :: BlockHash-genesis = "0f9188f13cb7b2c71f2a335e3a4fc328bf5beb436012afca590b1a11466e2206"+-- | Transactions that spend one another and some coins, some with a+-- witness, in a random order.+payments :: Gen [Tx]+payments = do+  n <- choose (0, 20)+  seeds <- vectorOf n (choose (0, 3 :: Int))+  let coin i = OutPoint (TxHash (sha256 (encode i))) 0+      -- one of kind 0 spends the one before it, and one of kind 2 has a+      -- witness+      tx prev (i, kind) =+        Tx+          1+          [TxIn (maybe (coin i) (\p -> if kind == 0 then OutPoint (txHash p) 0 else coin i) prev) "" maxBound]+          [TxOut (fromIntegral (1000 + i)) script]+          (if kind == 2 then [["\x01\x02"]] else [])+          0+      chained _ [] = []+      chained prev (x : xs) = let t = tx prev x in t : chained (Just t) xs+  return (reverse (chained Nothing (zip [0 :: Int ..] seeds)))
+ test/Haskoin/Store/Web/AdmissionSpec.hs view
@@ -0,0 +1,298 @@+{-# LANGUAGE OverloadedRecordDot #-}++-- | The limit on the requests answered at once: how many run, who waits, in+-- what order, and who is refused, and why. The clock is one that the tests+-- move by hand, so that CoDel and the limit of a wait are tested without+-- waiting, and without depending on how fast the machine is; the tests wait+-- only for the threads of the requests to get where they are going.+module Haskoin.Store.Web.AdmissionSpec (spec) where++import Control.Concurrent (threadDelay)+import Control.Concurrent.Async (async, cancel, forConcurrently, wait)+import Control.Concurrent.MVar (newEmptyMVar, putMVar, takeMVar)+import Control.Exception (ErrorCall (..), throwIO, try)+import Control.Monad (forM, replicateM)+import Data.IORef (atomicModifyIORef', modifyIORef', newIORef, readIORef)+import Haskoin.Store.Web.Admission+import Test.Hspec++spec :: Spec+spec = describe "the limit on the requests answered at once" $ do+  it "runs no more than its slots at once, and everything in the end" $ do+    (clock, _) <- handClock+    a <- newAdmissionWith clock (fixed 3 100)+    now <- newIORef (0 :: Int)+    most <- newIORef (0 :: Int)+    done <- forConcurrently [1 :: Int .. 40] $ \_ ->+      admitted a (const (return False)) $ do+        n <- atomicModifyIORef' now (\n -> (n + 1, n + 1))+        modifyIORef' most (max n)+        threadDelay 2000+        atomicModifyIORef' now (\m -> (m - 1, ()))+        return True+    done `shouldSatisfy` and+    readIORef most `shouldReturn` 3+    l <- load a+    (l.busy, l.waiting, l.refused) `shouldBe` (0, 0, Refusals 0 0 0)++  it "answers those that wait in the order that they came" $ do+    (clock, _) <- handClock+    a <- newAdmissionWith clock (fixed 1 10)+    gate <- newEmptyMVar+    order <- newIORef []+    first <- async (admitted a (const (return ())) (takeMVar gate))+    settle a 1 0+    rest <- forM [1 :: Int .. 5] $ \i -> do+      t <- async (admitted a (const (return ())) (modifyIORef' order (<> [i])))+      settle a 1 i+      return t+    putMVar gate ()+    mapM_ wait (first : rest)+    readIORef order `shouldReturn` [1 .. 5]++  it "refuses a request that finds the line full, without making it wait" $ do+    (clock, _) <- handClock+    a <- newAdmissionWith clock (fixed 1 2)+    gate <- newEmptyMVar+    held <- async (admitted a (return . Left) (Right "answered" <$ takeMVar gate))+    settle a 1 0+    queued <- replicateM 2 (async (admitted a (return . Left) (return (Right "answered"))))+    settle a 1 2+    admitted a (return . Left) (return (Right ("answered" :: String))) `shouldReturn` Left LineFull+    (.refused) <$> load a `shouldReturn` Refusals {full = 1, waited = 0, shed = 0}+    putMVar gate ()+    mapM wait (held : queued) `shouldReturn` replicate 3 (Right ("answered" :: String))++  it "refuses a request that has waited too long, and lets the next ones in" $ do+    (clock, move) <- handClock+    a <- newAdmissionWith clock (fixed 1 5) {patience = 1000000}+    gate <- newEmptyMVar+    held <- async (admitted a (return . Left) (Right () <$ takeMVar gate))+    settle a 1 0+    late <- async (admitted a (return . Left) (return (Right ())))+    settle a 1 1+    -- not long enough yet+    move 999 >> sweep a+    settle a 1 1+    move 1 >> sweep a+    wait late `shouldReturn` Left WaitedTooLong+    l <- load a+    (l.busy, l.waiting, l.refused) `shouldBe` (1, 0, Refusals 0 1 0)+    putMVar gate ()+    wait held `shouldReturn` Right ()+    admitted a (return . Left) (return (Right ())) `shouldReturn` Right ()++  it "gives the slot back when the request fails" $ do+    (clock, _) <- handClock+    a <- newAdmissionWith clock (fixed 1 0)+    r <- try (admitted a (const (return ())) (throwIO (ErrorCall "broken")))+    r `shouldBe` Left (ErrorCall "broken")+    admitted a (const (return False)) (return True) `shouldReturn` True++  it "takes a request that is stopped while it waits out of the line" $ do+    (clock, _) <- handClock+    a <- newAdmissionWith clock (fixed 1 5)+    gate <- newEmptyMVar+    held <- async (admitted a (const (return ())) (takeMVar gate))+    settle a 1 0+    stopped <- async (admitted a (const (return ())) (return ()))+    settle a 1 1+    behind <- async (admitted a (const (return False)) (return True))+    settle a 1 2+    cancel stopped+    settle a 1 1+    putMVar gate ()+    wait held+    -- the one behind it is not stuck behind a request that is gone+    wait behind `shouldReturn` True++  it "is no limit at all without a ceiling" $ do+    a <- newAdmission (fixed 0 0)+    rs <- forConcurrently [1 :: Int .. 20] $ \_ ->+      admitted a (const (return False)) (True <$ threadDelay 1000)+    rs `shouldSatisfy` and++  it "gives a slot that the limit gains to the first in line at once" $ do+    (clock, _) <- handClock+    a <- newAdmissionWith clock (fixed 3 10) {floor = 1}+    (.limit) <$> load a `shouldReturn` 1+    gate <- newEmptyMVar+    gates <- replicateM 3 newEmptyMVar+    held <- async (admitted a (const (return ())) (takeMVar gate))+    settle a 1 0+    waiting <- forM (zip [1 :: Int ..] gates) $ \(i, g) -> do+      t <- async (admitted a (const (return ())) (takeMVar g))+      settle a 1 i+      return t+    setLimit a 3+    settle a 3 1+    -- above the ceiling is the ceiling+    setLimit a 10+    (.limit) <$> load a `shouldReturn` 3+    mapM_ (`putMVar` ()) (gate : gates)+    mapM_ wait (held : waiting)++  it "lets the requests over a limit that was cut end before handing on a slot" $ do+    (clock, _) <- handClock+    a <- newAdmissionWith clock (fixed 3 10) {floor = 1}+    setLimit a 3+    gates <- replicateM 3 newEmptyMVar+    held <- forM (zip [1 ..] gates) $ \(i, g) -> do+      t <- async (admitted a (const (return ())) (takeMVar g))+      settle a i 0+      return t+    -- below the floor is the floor+    setLimit a 0+    (.limit) <$> load a `shouldReturn` 1+    behind <- async (admitted a (const (return False)) (return True))+    settle a 3 1+    let end (g, t) = putMVar g () >> wait t+    -- the slots of the first two come free, and the third one's goes on+    mapM_ (\(k, x) -> end x >> settle a (3 - k) 1) (zip [1 ..] (take 2 (zip gates held)))+    mapM_ end (drop 2 (zip gates held))+    wait behind `shouldReturn` True++  describe "CoDel" $ do+    it "refuses at once, and sheds those that waited, while the line stands" $ do+      (clock, move) <- handClock+      a <- newAdmissionWith clock (fixed 1 10)+      let ask g = async (admitted a (return . Left) (Right () <$ takeMVar g))+      gate1 <- newEmptyMVar+      gate2 <- newEmptyMVar+      gate3 <- newEmptyMVar+      -- the first interval: one request goes in at once, three wait+      first <- ask gate1+      settle a 1 0+      second <- ask gate2+      settle a 1 1+      thirds <- replicateM 2 (async (admitted a (return . Left) (return (Right ()))))+      settle a 1 3+      -- the first ends after the interval, which let a request in at once:+      -- the line worked, and the second has its slot after 110 ms+      move 110+      (putMVar gate1 () >> wait first) `shouldReturn` Right ()+      settle a 1 2+      (.overloaded) <$> load a `shouldReturn` False+      -- in the second interval the only one let in had waited 110 ms: the+      -- line stood, the two that waited 220 ms are refused, and the slot+      -- comes free+      move 110+      (putMVar gate2 () >> wait second) `shouldReturn` Right ()+      mapM wait thirds `shouldReturn` replicate 2 (Left Overloaded)+      l <- load a+      (l.busy, l.waiting, l.overloaded) `shouldBe` (0, 0, True)+      -- a free slot is taken; with none free, a request is refused at once+      third <- ask gate3+      settle a 1 0+      admitted a (return . Left) (return (Right ())) `shouldReturn` Left Overloaded+      -- the third went in at once: in the next interval the line works+      move 110+      (putMVar gate3 () >> wait third) `shouldReturn` Right ()+      (.overloaded) <$> load a `shouldReturn` False+      (.refused) <$> load a `shouldReturn` Refusals {full = 0, waited = 0, shed = 3}++    it "takes a line whose slots nobody leaves for one that stands" $ do+      (clock, move) <- handClock+      a <- newAdmissionWith clock (fixed 1 10)+      gate <- newEmptyMVar+      held <- async (admitted a (return . Left) (Right () <$ takeMVar gate))+      settle a 1 0+      waiting <- async (admitted a (return . Left) (return (Right ())))+      settle a 1 1+      -- the first interval let a request in at once+      move 150+      behind <- async (admitted a (return . Left) (return (Right ())))+      settle a 1 2+      (.overloaded) <$> load a `shouldReturn` False+      -- in the second nobody was let in, and the first in line had waited+      -- 260 ms by its end+      move 110+      admitted a (return . Left) (return (Right ())) `shouldReturn` Left Overloaded+      (.overloaded) <$> load a `shouldReturn` True+      putMVar gate ()+      wait held `shouldReturn` Right ()+      mapM wait [waiting, behind] `shouldReturn` replicate 2 (Left Overloaded)++    it "lets the line stand while the limit is free to rise" $ do+      (clock, move) <- handClock+      a <- newAdmissionWith clock (fixed 1 10)+      setClimbing a True+      gate <- newEmptyMVar+      held <- async (admitted a (return . Left) (Right () <$ takeMVar gate))+      settle a 1 0+      waiting <- async (admitted a (return . Left) (return (Right ())))+      settle a 1 1+      -- two intervals in which nobody was let in: the line stood, but the+      -- limit is about to grow, so the request waits for it+      move 150+      sweep a+      move 110+      sweep a+      (.overloaded) <$> load a `shouldReturn` False+      putMVar gate ()+      mapM wait [held, waiting] `shouldReturn` replicate 2 (Right ())+      (.refused) <$> load a `shouldReturn` Refusals 0 0 0++    it "absorbs a burst that the slots clear within an interval" $ do+      (clock, move) <- handClock+      a <- newAdmissionWith clock (fixed 1 100)+      gates <- replicateM 20 newEmptyMVar+      ts <- forM (zip [0 :: Int ..] gates) $ \(i, g) -> do+        t <- async (admitted a (return . Left) (Right () <$ takeMVar g))+        settle a 1 i+        return t+      -- one ends every 4 ms: the last one waits 76 ms, and the line is+      -- empty again before the end of the interval+      mapM_ (\(g, t) -> move 4 >> putMVar g () >> wait t) (zip gates ts)+      (.refused) <$> load a `shouldReturn` Refusals 0 0 0++  it "lets a request step aside from its slot, and takes it back without the line" $ do+    (clock, _) <- handClock+    a <- newAdmissionWith clock (fixed 1 10)+    outside <- newEmptyMVar+    back <- newEmptyMVar+    gate <- newEmptyMVar+    stepping <- async $ admittedAside a (const (return False)) $ \out -> do+      aside out (putMVar outside () >> takeMVar back)+      takeMVar gate+      return True+    takeMVar outside+    -- its slot is free while it is aside+    settle a 0 0+    other <- async (admitted a (const (return False)) (True <$ takeMVar gate))+    settle a 1 0+    -- back in, past the limit+    putMVar back ()+    settle a 2 0+    putMVar gate ()+    putMVar gate ()+    mapM wait [stepping, other] `shouldReturn` [True, True]+    l <- load a+    (l.busy, l.waiting, l.refused) `shouldBe` (0, 0, Refusals 0 0 0)++-- | A fixed limit of so many slots, with room for so many in line, who may+-- wait ten seconds.+fixed :: Int -> Int -> Settings+fixed n room =+  Settings+    { ceiling = n,+      floor = n,+      target = 25000,+      room = room,+      patience = 10000000+    }++-- | A clock that moves only when told to, by so many milliseconds.+handClock :: IO (IO Int, Int -> IO ())+handClock = do+  v <- newIORef 0+  return (readIORef v, \ms -> atomicModifyIORef' v (\t -> (t + ms * 1000000, ())))++-- | Wait until so many requests run and so many wait.+settle :: Admission -> Int -> Int -> IO ()+settle a busy waiting = go (20000 :: Int)+  where+    go 0 = expectationFailure "the requests did not settle"+    go n = do+      l <- load a+      if l.busy == busy && l.waiting == waiting then return () else threadDelay 500 >> go (n - 1)
+ test/Haskoin/Store/Web/CongestionSpec.hs view
@@ -0,0 +1,189 @@+{-# LANGUAGE OverloadedRecordDot #-}++-- | The controller of the limit on long requests, alone, and the probes of+-- the scheduler's delay that it follows.+module Haskoin.Store.Web.CongestionSpec (spec) where++import Control.Concurrent (getNumCapabilities, threadDelay)+import Control.Concurrent.Async (withAsyncOn)+import Control.Exception (evaluate)+import Control.Monad (forever, unless, when)+import Data.IORef (newIORef, readIORef, writeIORef)+import Data.IntMap.Strict qualified as IntMap+import GHC.Stats (getRTSStatsEnabled)+import Haskoin.Store.Web.Congestion+import System.Mem (performMajorGC)+import Test.Hspec+import Test.Hspec.QuickCheck (prop)+import Test.QuickCheck++spec :: Spec+spec = do+  describe "the controller of the limit" $ do+    prop "keeps the limit between the floor and the ceiling" $+      forAll settings $ \c -> forAll (start c) $ \w ->+        forAll (listOf (observation c)) $ \os ->+          all (\x -> c.floor <= x.limit && x.limit <= c.ceiling) (steps c w os)++    prop "comes down to the floor while the delay stays over the target" $+      forAll settings $ \c -> forAll (start c) $ \w ->+        forAll (vectorOf 64 (congested c)) $ \os ->+          let ms = map (.limit) (steps c w os)+           in last ms == c.floor && and (zipWith (>=) ms (drop 1 ms))++    prop "cuts at most once a round trip" $+      forAll settings $ \c -> forAll (start c) $ \w -> forAll (chooseInt (0, 5000000)) $ \rtt ->+        forAll (listOf (congested c)) $ \os ->+          let os' = [o {roundTrip = rtt} | o <- os]+              ws = drop 1 (steps c w os')+              -- microseconds since the last cut, before each step+              since = scanl (\t (o, x) -> if x.sinceCut == 0 then 0 else t + o.elapsed) w.sinceCut (zip os' ws)+           in and [t + o.elapsed >= rtt | (t, o, x) <- zip3 since os' ws, x.sinceCut == 0]++    prop "never grows while it is not saturated, or while the cores are busy" $+      forAll settings $ \c -> forAll (start c) $ \w ->+        forAll (listOf (observation c)) $ \os -> forAll (chooseInt (90, 200)) $ \busy ->+          let idle = map (.limit) (steps c w [o {saturated = False} | o <- os])+              full = map (.limit) (steps c w [o {used = busy} | o <- os])+           in and (zipWith (>=) idle (drop 1 idle)) && and (zipWith (>=) full (drop 1 full))++    prop "never cuts while the cores have room" $+      forAll settings $ \c -> forAll (start c) $ \w ->+        forAll (listOf (congested c)) $ \os -> forAll (chooseInt (0, 89)) $ \idle ->+          let ms = map (.limit) (steps c w [o {used = idle} | o <- os])+           in and (zipWith (<=) ms (drop 1 ms))++    prop "is free to rise only while the cores have room, under the ceiling" $+      forAll settings $ \c -> forAll (start c) $ \w -> forAll (observation c) $ \o ->+        rising c o w == (o.used < 90 && w.limit < c.ceiling)++    prop "grows back to the level of the last cut, then past it to the ceiling" $+      forAll settings $ \c -> forAll (chooseInt (c.floor, c.ceiling)) $ \n ->+        let w = (window c) {limit = n, level = fromIntegral n, peak = fromIntegral n}+            cut = control c (Observation (c.target + 1) 95 True 0 500000) w+            climb = steps c cut (replicate 400 (calm c))+            -- seconds on the curve at each step+            times = [0.5 * fromIntegral i :: Double | i <- [0 :: Int ..]]+            early = [x.limit | (x, t) <- zip climb times, t < cut.knee]+            past = [x.limit | (x, t) <- zip climb times, t >= cut.knee + 0.5]+         in all (<= n) early+              && all (>= n) (take 1 past)+              && last (map (.limit) climb) == c.ceiling+              && and (zipWith (<=) (map (.limit) climb) (drop 1 (map (.limit) climb)))++    prop "never moves a fixed limit" $+      forAll settings $ \c0 -> forAll (listOf (observation c0)) $ \os ->+        let c = c0 {ceiling = c0.floor}+         in all ((== c.floor) . (.limit)) (steps c (window c) os)++  describe "the percentile" $ do+    it "is the value at its rank" $ do+      percentile 90 [1 .. 100] `shouldBe` 90+      percentile 90 [100, 99 .. 1] `shouldBe` 90+      percentile 90 [7] `shouldBe` 7+      percentile 50 [1, 2, 3, 4] `shouldBe` 2+    it "is zero for no values" $+      percentile 90 [] `shouldBe` 0+    it "counts each value for its weight, when weighted" $ do+      weighted 90 [(x, 1) | x <- [1 .. 100]] `shouldBe` 90+      -- thirty wakes on time on an idle capability, and one of a busy+      -- capability, 500 µs late, which covers more time than all of them+      weighted 90 ((500, 510) : replicate 30 (1, 10)) `shouldBe` 500+      percentile 90 (500 : replicate 30 1) `shouldBe` 1+      weighted 90 [] `shouldBe` 0++  describe "the probes of the scheduler's delay" $ do+    -- the target is 25 ms; the thresholds leave a factor of two either way+    it "stay well under the target on idle capabilities" $+      withProbes $ \ps -> do+        _ <- lateness ps+        threadDelay 1000000+        d <- lateness ps+        d `shouldSatisfy` (< 12500)++    it "rise well over the target with four busy threads on each capability" $ do+      caps <- getNumCapabilities+      fourOn [0 .. caps - 1] $ withProbes $ \ps -> do+        _ <- lateness ps+        threadDelay 1000000+        d <- lateness ps+        d `shouldSatisfy` (> 50000)++    it "rise well over the target with four busy threads on one capability" $+      fourOn [0] $+        withProbes $ \ps -> do+          _ <- lateness ps+          threadDelay 1000000+          d <- lateness ps+          d `shouldSatisfy` (> 50000)++    it "take the pauses of the collector out of the scheduler's delay" $ do+      counted <- getRTSStatsEnabled+      unless counted $ pendingWith "GHC keeps no statistics without +RTS -T"+      -- a map of a million entries, copied by every major collection,+      -- which nothing else runs between+      held <- evaluate (IntMap.fromList [(i, i) | i <- [1 .. 1_000_000 :: Int]])+      withProbes $ \ps -> do+        _ <- delays ps+        before <- delays ps+        let collect n = when (n > (0 :: Int)) (performMajorGC >> collect (n - 1))+        collect 20+        d <- delays ps+        IntMap.size held `shouldBe` 1_000_000+        d.collected `shouldSatisfy` (> before.collected)+        d.raw `shouldSatisfy` (> 5000)+        d.scheduling `shouldSatisfy` (< d.raw `div` 2)+  where+    steps c = scanl (flip (control c))++-- | Run an action with four threads that compute on each of these+-- capabilities.+fourOn :: [Int] -> IO a -> IO a+fourOn caps act = foldr (\i k -> busy i (busy i (busy i (busy i k)))) act caps+  where+    busy i k = withAsyncOn i spin (const k)++-- | A thread that computes, and allocates, so that GHC can take its core+-- from it at the end of its timeslice.+spin :: IO ()+spin = do+  r <- newIORef (0 :: Int)+  forever $ readIORef r >>= \x -> writeIORef r $! x + 1++-- | Settings of the controller.+settings :: Gen Control+settings = do+  lo <- chooseInt (1, 64)+  hi <- chooseInt (lo, 1024)+  target <- chooseInt (1, 100000)+  return Control {floor = lo, ceiling = hi, target = target}++-- | A window that some steps led to from the start.+start :: Control -> Gen Window+start c = foldl (flip (control c)) (window c) <$> listOf (observation c)++-- | Anything that a step may see.+observation :: Control -> Gen Observation+observation c =+  Observation+    <$> chooseInt (0, 2 * c.target)+    <*> chooseInt (0, 150)+    <*> arbitrary+    <*> chooseInt (0, 2000000)+    <*> chooseInt (0, 1000000)++-- | A step with a delay over the target and the cores busy, a round trip+-- after the last cut.+congested :: Control -> Gen Observation+congested c =+  Observation+    <$> chooseInt (c.target + 1, 4 * c.target)+    <*> chooseInt (90, 150)+    <*> arbitrary+    <*> pure 0+    <*> chooseInt (1, 1000000)++-- | A step of half a second with a delay under the target, every slot+-- taken, and cores with room.+calm :: Control -> Observation+calm c = Observation (c.target - 1) 50 True 0 500000
test/Haskoin/Store/Web/EventsSpec.hs view
@@ -20,28 +20,28 @@ address n = PubKeyAddress (addressHash (B.replicate 33 n))  -- | A transaction that pays to addresses, told apart by a number.-paying :: Ctx -> Word32 -> [Address] -> Seen-paying ctx n to =+paying :: Word32 -> [Address] -> Seen+paying n to =   Seen {tx, addresses = HashSet.fromList to, size = 1000}   where     tx = Tx {version = 1, inputs = [], outputs = map out to, witness = [], locktime = n}-    out a = TxOut {value = 1000, script = fromMaybe (error "No script") (addressToScriptBS ctx a)}+    out a = TxOut {value = 1000, script = fromMaybe (error "No script") (addressToScriptBS a)}  -- | A key, and its address at a place on a chain.-key :: Ctx -> DeriveType -> XPubSpec-key ctx deriv = XPubSpec {key = deriveXPubKey ctx (makeXPrvKey (B.replicate 32 7)), deriv}+key :: DeriveType -> XPubSpec+key deriv = XPubSpec {key = deriveXPubKey (makeXPrvKey (B.replicate 32 7)), deriv} -keyAddress :: Ctx -> XPubSpec -> Word32 -> Word32 -> Address-keyAddress ctx x chain = deriveFunction ctx x.deriv (pubSubKey ctx x.key chain)+keyAddress :: XPubSpec -> Word32 -> Word32 -> Address+keyAddress x chain = deriveFunction x.deriv (pubSubKey x.key chain)  -- | A balance of a key's address that has been paid to.-used :: Ctx -> XPubSpec -> Word32 -> Word32 -> XPubBal-used ctx x chain i =+used :: XPubSpec -> Word32 -> Word32 -> XPubBal+used x chain i =   XPubBal     { path = [chain, i],       balance =         Balance-          { address = keyAddress ctx x chain i,+          { address = keyAddress x chain i,             confirmed = 1,             unconfirmed = 0,             utxo = 1,@@ -51,13 +51,13 @@     }  spec :: Spec-spec = prepareContext $ \ctx -> do+spec = do   describe "log of events" $ do     it "gives a reader what came after its place, and waits at the end" $ do       events <- newEventLog-      let one = paying ctx 1 [address 1]+      let one = paying 1 [address 1]           h1 = txHash one.tx-          block = BlockHash (txHash (paying ctx 2 []).tx).get+          block = BlockHash (txHash (paying 2 []).tx).get       start <- atomically (eventsEnd events)       atomically $ do         appendEvent events (TxEntered h1 (Just one))@@ -72,7 +72,7 @@       fmap (fmap length) waiting `shouldBe` Nothing     it "takes up after the first event about a hash, and not after one it does not have" $ do       events <- newEventLog-      let seen n = paying ctx n [address 1]+      let seen n = paying n [address 1]           hashes = map (txHash . (.tx) . seen) [1 .. 3]           h1 = hashes !! 0           h2 = hashes !! 1@@ -87,7 +87,7 @@       atomically (eventsAfter events Nothing (Just (txHash (seen 9).tx))) `shouldReturn` Nothing     it "forgets the oldest events, and tells a reader that fell off the start" $ do       events <- newEventLog-      let seen n = paying ctx n [address 1]+      let seen n = paying n [address 1]           enter n = appendEvent events (TxEntered (txHash (seen n).tx) (Just (seen n)))           extra = 5       atomically (mapM_ enter [1 .. fromIntegral logLength + extra])@@ -104,7 +104,7 @@       events <- newEventLog       let large :: Word32 -> Seen           large n =-            let s = paying ctx n [address 1]+            let s = paying n [address 1]              in Seen {tx = s.tx, addresses = s.addresses, size = logWeight `div` 4}       atomically (mapM_ (\n -> appendEvent events (TxEntered (txHash (large n).tx) (Just (large n)))) [1 .. 10])       end <- atomically (eventsEnd events)@@ -115,77 +115,77 @@   describe "choice of a client" $ do     let gap = 3 :: Word32         unused _ = return [] :: IO [XPubBal]-        told c var seen = atomically (stateTVar var (relevant c gap seen))+        told var seen = atomically (stateTVar var (relevant gap seen))     it "is everything until the client chooses, and again when it says so" $ do       var <- newTVarIO everything-      told ctx var (Just (paying ctx 1 [address 1])) `shouldReturn` True-      command ctx gap unused var (EventWatch [address 2]) `shouldReturn` Right ()-      told ctx var (Just (paying ctx 1 [address 1])) `shouldReturn` False-      told ctx var (Just (paying ctx 2 [address 1, address 2])) `shouldReturn` True+      told var (Just (paying 1 [address 1])) `shouldReturn` True+      command gap unused var (EventWatch [address 2]) `shouldReturn` Right ()+      told var (Just (paying 1 [address 1])) `shouldReturn` False+      told var (Just (paying 2 [address 1, address 2])) `shouldReturn` True       -- what the server does not know is told-      told ctx var Nothing `shouldReturn` True-      command ctx gap unused var (EventUnwatch [address 2]) `shouldReturn` Right ()-      told ctx var (Just (paying ctx 2 [address 1, address 2])) `shouldReturn` False-      command ctx gap unused var EventAll `shouldReturn` Right ()+      told var Nothing `shouldReturn` True+      command gap unused var (EventUnwatch [address 2]) `shouldReturn` Right ()+      told var (Just (paying 2 [address 1, address 2])) `shouldReturn` False+      command gap unused var EventAll `shouldReturn` Right ()       readTVarIO var `shouldReturn` everything       -- blocks alone, which are events without a transaction-      command ctx gap unused var EventNone `shouldReturn` Right ()-      told ctx var (Just (paying ctx 1 [address 1])) `shouldReturn` False-      told ctx var Nothing `shouldReturn` True+      command gap unused var EventNone `shouldReturn` Right ()+      told var (Just (paying 1 [address 1])) `shouldReturn` False+      told var Nothing `shouldReturn` True     it "watches a key up to its gap, which moves on with its payments" $ do       var <- newTVarIO everything-      let x = key ctx DeriveNormal-          pay chain i = Just (paying ctx (chain * 1000 + i) [keyAddress ctx x chain i])+      let x = key DeriveNormal+          pay chain i = Just (paying (chain * 1000 + i) [keyAddress x chain i])       -- used up to the second address of the external chain-      command ctx gap (\_ -> return [used ctx x 0 1]) var (EventWatchXPub x) `shouldReturn` Right ()+      command gap (\_ -> return [used x 0 1]) var (EventWatchXPub x) `shouldReturn` Right ()       w <- readTVarIO var       HashMap.lookup x w.keys `shouldBe` Just (5, 3)       HashMap.size w.derived `shouldBe` 8-      told ctx var (pay 0 5) `shouldReturn` False-      told ctx var (pay 1 3) `shouldReturn` False-      told ctx var (pay 0 4) `shouldReturn` True+      told var (pay 0 5) `shouldReturn` False+      told var (pay 1 3) `shouldReturn` False+      told var (pay 0 4) `shouldReturn` True       -- the payment to the fifth address moved the gap to the eighth-      told ctx var (pay 0 7) `shouldReturn` True-      told ctx var (pay 0 11) `shouldReturn` False-      told ctx var (pay 1 2) `shouldReturn` True-      told ctx var (pay 1 5) `shouldReturn` True+      told var (pay 0 7) `shouldReturn` True+      told var (pay 0 11) `shouldReturn` False+      told var (pay 1 2) `shouldReturn` True+      told var (pay 1 5) `shouldReturn` True       -- the same key with another type of address is another key-      let y = key ctx DeriveP2WPKH-          other = Just (paying ctx 77 [keyAddress ctx y 0 0])-      told ctx var other `shouldReturn` False-      command ctx gap unused var (EventWatchXPub y) `shouldReturn` Right ()-      told ctx var other `shouldReturn` True-      command ctx gap unused var (EventUnwatchXPub x) `shouldReturn` Right ()-      told ctx var (pay 0 0) `shouldReturn` False-      told ctx var other `shouldReturn` True+      let y = key DeriveP2WPKH+          other = Just (paying 77 [keyAddress y 0 0])+      told var other `shouldReturn` False+      command gap unused var (EventWatchXPub y) `shouldReturn` Right ()+      told var other `shouldReturn` True+      command gap unused var (EventUnwatchXPub x) `shouldReturn` Right ()+      told var (pay 0 0) `shouldReturn` False+      told var other `shouldReturn` True     it "matches a Bloom filter as a node does, next to the addresses" $ do       var <- newTVarIO everything       let element = \case             PubKeyAddress h -> runPutS (serialize h)             _ -> error "Not the hash of a key"           filter' = bloomInsert (bloomCreate 10 0.0000001 0 BloomUpdateAll) (element (address 1))-          one = paying ctx 1 [address 1]-          two = paying ctx 2 [address 2]-          three = paying ctx 3 [address 3]-      command ctx gap unused var (EventFilterAdd "x") `shouldReturn` Left "No filter to add to"-      command ctx gap unused var (EventFilterLoad filter') `shouldReturn` Right ()-      command ctx gap unused var (EventWatch [address 3]) `shouldReturn` Right ()-      told ctx var (Just one) `shouldReturn` True-      told ctx var (Just two) `shouldReturn` False-      told ctx var (Just three) `shouldReturn` True+          one = paying 1 [address 1]+          two = paying 2 [address 2]+          three = paying 3 [address 3]+      command gap unused var (EventFilterAdd "x") `shouldReturn` Left "No filter to add to"+      command gap unused var (EventFilterLoad filter') `shouldReturn` Right ()+      command gap unused var (EventWatch [address 3]) `shouldReturn` Right ()+      told var (Just one) `shouldReturn` True+      told var (Just two) `shouldReturn` False+      told var (Just three) `shouldReturn` True       -- the server does not add to the filter by itself       (.bloom) <$> readTVarIO var `shouldReturn` Just filter'       -- by the hash of the transaction, and by what the client adds-      command ctx gap unused var (EventFilterAdd (runPutS (serialize (txHash two.tx)))) `shouldReturn` Right ()-      told ctx var (Just two) `shouldReturn` True-      command ctx gap unused var EventFilterClear `shouldReturn` Right ()-      told ctx var (Just one) `shouldReturn` False-      told ctx var (Just three) `shouldReturn` True+      command gap unused var (EventFilterAdd (runPutS (serialize (txHash two.tx)))) `shouldReturn` Right ()+      told var (Just two) `shouldReturn` True+      command gap unused var EventFilterClear `shouldReturn` Right ()+      told var (Just one) `shouldReturn` False+      told var (Just three) `shouldReturn` True     it "refuses more addresses or keys than a client may name" $ do       var <- newTVarIO everything       let numbered n = PubKeyAddress (addressHash (runPutS (serialize n)))           many = map numbered [1 .. fromIntegral maxWatchAddresses :: Word32]-      command ctx gap unused var (EventWatch many) `shouldReturn` Right ()-      command ctx gap unused var (EventWatch [address 1]) `shouldReturn` Left "Too many addresses"+      command gap unused var (EventWatch many) `shouldReturn` Right ()+      command gap unused var (EventWatch [address 1]) `shouldReturn` Left "Too many addresses"       -- a refused command changes nothing       HashSet.size . (.plain) <$> readTVarIO var `shouldReturn` maxWatchAddresses
test/Haskoin/Store/Web/PiecesSpec.hs view
@@ -3,13 +3,13 @@ module Haskoin.Store.Web.PiecesSpec (spec) where  import Control.Concurrent (threadDelay)-import Control.Monad (forM_) import Control.Exception (ErrorCall (..), throwIO, try)+import Control.Monad (forM_) import Data.Aeson (encode, toEncoding) import Data.ByteString.Builder (Builder, toLazyByteString) import Data.ByteString.Lazy qualified as L-import Data.List (sort) import Data.IORef (IORef, atomicModifyIORef', modifyIORef', newIORef, readIORef)+import Data.List (sort) import Haskoin.Store.Web.Pieces import Network.Wai.Internal (Response (..)) import Test.Hspec hiding (around)@@ -23,14 +23,14 @@   it "has the bytes of a list encoded at once, for pieces of its items" $ do     let lists = [[], [[]], [[1]], [[1, 2], [3]], [[], [1], [], [2, 3], []], [[1 .. 100], [101 .. 150]]]         listed :: [[Int]] -> IO (L.ByteString, Bool)-        listed l = listPieces toEncoding (map return l) >>= answer+        listed l = listPieces toEncoding [(length x, return x) | x <- l] >>= answer     mapM_ (\l -> listed l `shouldReturn` (encode (concat l), True)) lists   it "has the same bytes when its pieces take turns to finish" $ do     -- later pieces finish first, so that what is settled in order, the     -- commas, is settled after all of them are encoded     let slow n items = threadDelay (1000 * (5 - n `mod` 5)) >> return items         l = [[], [1, 2], [], [3], [4 .. 9], [], [10 :: Int]]-    pieces <- listPieces toEncoding (zipWith slow [0 ..] l)+    pieces <- listPieces toEncoding (zipWith (\n x -> (length x, slow n x)) [0 ..] l)     answer pieces `shouldReturn` (encode (concat l), True)   it "cuts keys into pieces of the size, and the last of what is left" $ do     read' <- newIORef []@@ -39,6 +39,7 @@     answer pieces `shouldReturn` (encode [1 .. 8 :: Int], True)     -- pieces are read at the same time, in no fixed order     sort <$> readIORef read' `shouldReturn` [[1, 2, 3], [4, 5, 6], [7, 8]]+    map fst (pieced 3 fetch [1 .. 8 :: Int]) `shouldBe` [3, 3, 2]   it "puts bytes around pieces without a piece more" $ do     let wrapped ps = around "<" ">" (map plain ps)     let cases = [[], ["a"], ["a", "b"], ["a", "b", "c"]]
test/Haskoin/Store/Web/PublishSpec.hs view
@@ -9,8 +9,8 @@ import Data.ByteString qualified as B import Data.HashMap.Strict qualified as HashMap import Data.IORef (modifyIORef', newIORef, readIORef)-import Data.Time.Clock.POSIX (posixSecondsToUTCTime) import Data.Serialize (encode)+import Data.Time.Clock.POSIX (posixSecondsToUTCTime) import Data.Word (Word32, Word64, Word8) import Haskoin hiding (Coin (..)) import Haskoin.Store.Web.Publish@@ -199,8 +199,11 @@         `shouldBe` Left (TokensUnbacked (concatMap (const "09") [1 .. 32 :: Int]))     it "makes tokens of a new category at will, from an input that spends an output 0" $ do       let genesisId = TxHash (doubleSHA256 ("genesis" :: ByteString))-          genesis = (tx [] [fungible newCategory 5000, nft newCategory 2 ""])-            {inputs = [TxIn (OutPoint genesisId 0) "" maxBound]} :: Tx+          genesis =+            (tx [] [fungible newCategory 5000, nft newCategory 2 ""])+              { inputs = [TxIn (OutPoint genesisId 0) "" maxBound]+              } ::+              Tx       tokenRules genesis [coin plain] `shouldBe` Right ()       -- the same category from an input that spends an output 1 is not new       let notNew = genesis {inputs = [TxIn (OutPoint genesisId 1) "" maxBound]} :: Tx
test/Haskoin/Store/Web/RouterSpec.hs view
@@ -33,7 +33,7 @@   it "answers inside the scope of a request, and makes one for each route that it tries" $ do     events <- newIORef []     let note e = modifyIORef' events (<> [e])-        scoped answer = note "open" >> answer routes <* note "close"+        scoped _ answer = note "open" >> answer routes <* note "close"         req = defaultRequest {requestMethod = methodGet, pathInfo = ["block", "best"]}     _ <- routerScoped routes scoped (\_ _ -> fail "no route") req $ \_ ->       note "respond" >> return ResponseReceived
+ test/Haskoin/Store/XPub/IndexerSpec.hs view
@@ -0,0 +1,408 @@+-- | The indexer of xpubs: its decisions, as a pure function, and its queue,+-- its waits and its lock, with jobs run one by one on a clock that the tests+-- move by hand, and once with its own threads on the clock of the system.+module Haskoin.Store.XPub.IndexerSpec (spec) where++import Control.Concurrent (threadDelay)+import Control.Concurrent.Async (async, forConcurrently, wait, withAsync)+import Control.Concurrent.MVar (newEmptyMVar, putMVar, readMVar)+import Control.Monad (forM_, replicateM, void, when)+import Data.IORef (IORef, atomicModifyIORef', modifyIORef', newIORef, readIORef, writeIORef)+import Data.List (sort)+import GHC.Clock (getMonotonicTimeNSec)+import Haskoin.Store.XPub (Budget (..), noBudget)+import Haskoin.Store.XPub.Indexer+import Haskoin.Store.XPub.Lock (Lock (..), Taken (..), noLock)+import Test.Hspec++spec :: Spec+spec = do+  describe "the decision for a key that the cache does not have" $ do+    let s holder est room = Situation {expected = est, patience = 5, reach = 30, hasRoom = room, holder}+    it "takes nothing when another store holds the key, and retries from its lock" $ do+      decide (s (AnotherStore 7) 1 True) `shouldBe` RefuseNothing OtherStore 7+      decide (s (AnotherStore (-3)) 1 True) `shouldBe` RefuseNothing OtherStore 0+    it "joins the job of this store within the wait, whatever the room" $ do+      decide (s ThisStore 5 False) `shouldBe` Wait+      decide (s ThisStore 1 True) `shouldBe` Wait+    it "keeps the job of this store beyond the wait, and retries from the estimate" $ do+      decide (s ThisStore 6 True) `shouldBe` RefuseKeep 6+      decide (s ThisStore 60 False) `shouldBe` RefuseKeep 60+    it "takes nothing with a full queue, and retries at the horizon" $ do+      decide (s Nobody 1 False) `shouldBe` RefuseNothing QueueFull 30+      decide (s Nobody 60 False) `shouldBe` RefuseNothing QueueFull 30+    it "takes nothing beyond the horizon, and retries at the horizon" $+      decide (s Nobody 31 True) `shouldBe` RefuseNothing BeyondHorizon 30+    it "queues, locks and waits within the wait" $ do+      decide (s Nobody 5 True) `shouldBe` Wait+      decide (s Nobody 0 True) `shouldBe` Wait+    it "queues and locks, and retries from the estimate, within the horizon" $ do+      decide (s Nobody 6 True) `shouldBe` RefuseKeep 6+      decide (s Nobody 30 True) `shouldBe` RefuseKeep 30+    it "estimates a place in the queue from the median and the threads" $ do+      estimate 100 4 0 `shouldBe` 100+      estimate 100 4 4 `shouldBe` 200+      estimate 100 4 6 `shouldBe` 250+      estimate 100 1 3 `shouldBe` 400+    it "says Retry-After in whole seconds, at least one" $+      map retryAfter [0, 1, 999999, 1000000, 1000001, 4200000] `shouldBe` [1, 1, 1, 1, 2, 5]++  describe "the indexer" $ do+    it "runs one job for a key however many requests ask, and wakes them all when it is done" $ do+      t <- testbed settings+      answers <- withAsync (forConcurrently [1 .. 20] (\n -> request t.indexer 'x' [n])) $ \a -> do+        waiting t 20+        runNext t.indexer `shouldReturn` True+        runNext t.indexer `shouldReturn` False+        wait a+      -- each told that the cache has the key now, to read it there+      answers `shouldBe` replicate 20 Indexed+      -- the job with what each request brought+      map (fmap sort) <$> readIORef t.ran `shouldReturn` [('x', [1 .. 20])]+      readIORef t.locks `shouldReturn` ["take x", "release x"]+      s <- indexerStats t.indexer+      (s.counts.done, s.counts.answered, s.waiting) `shouldBe` (1, 20, 0)++    it "runs the jobs that requests wait for before upkeep, in the order they came" $ do+      t <- testbed settings+      upkeep t.indexer 'a' [1]+      upkeep t.indexer 'b' [2]+      upkeep t.indexer 'c' [3]+      r1 <- async (request t.indexer 'd' [4])+      waiting t 1+      -- a request for a key queued for upkeep moves it to the first part+      r2 <- async (request t.indexer 'c' [5])+      waiting t 2+      replicateM 5 (runNext t.indexer) `shouldReturn` [True, True, True, True, False]+      map fst <$> readIORef t.ran `shouldReturn` "dcab"+      lookup 'c' <$> readIORef t.ran `shouldReturn` Just [3, 5]+      mapM wait [r1, r2] `shouldReturn` [Indexed, Indexed]++    it "goes on with a job after the requests that wait for it are refused" $ do+      t <- testbed settings+      gate <- newEmptyMVar+      writeIORef t.during (readMVar gate)+      r <- async (request t.indexer 'x' [])+      waiting t 1+      job <- async (runNext t.indexer)+      waitFor (not . null <$> readIORef t.started)+      t.advance (settings.wait + 1)+      sweep t.indexer+      wait r >>= (`shouldSatisfy` refusedFor WaitedOut)+      putMVar gate ()+      wait job `shouldReturn` True+      readIORef t.ran `shouldReturn` [('x', [])]+      readIORef t.locks `shouldReturn` ["take x", "release x"]++    it "refuses a waiting request at a sweep once its time is up, by its own clock" $ do+      t <- testbed settings+      r <- async (request t.indexer 'x' [])+      waiting t 1+      -- nothing is refused before its time, however often the sweeper runs+      t.advance (settings.wait - 1)+      forM_ [1 .. 3 :: Int] $ \_ -> sweep t.indexer+      threadDelay 20000+      (.waiting) <$> indexerStats t.indexer `shouldReturn` 1+      t.advance 1+      sweep t.indexer+      a <- wait r+      a `shouldSatisfy` refusedFor WaitedOut+      -- the job has not run, so it is still where the request found it+      a `shouldBe` Refused WaitedOut settings.guess+      (.counts.refused.waitedOut) <$> indexerStats t.indexer `shouldReturn` 1++    it "refuses a waiting request in time with its own threads, which set no timer" $ do+      gate <- newEmptyMVar+      let runner = (plainRunner (\_ _ () -> readMVar gate) :: Runner Char ()) {ready = return True}+      withIndexer settings {wait = 300000, workers = 1} noLock runner $ \i -> do+        asked <- now+        a <- request i 'x' ()+        told <- now+        putMVar gate ()+        a `shouldSatisfy` refusedFor WaitedOut+        -- the sweeper runs ten times a second+        (told - asked) `shouldSatisfy` (\d -> d >= 300000 && d < 1500000)++    it "estimates from the median time of recent jobs and the place in the queue" $ do+      t <- testbed settings {workers = 2}+      -- each job takes as long as its payload says, on the hand clock+      writeIORef t.length' True+      forM_ (zip "abc" [100, 300, 200]) $ \(k, d) -> do+        void (prefetch t.indexer k [d])+        void (runNext t.indexer)+      (.median) <$> indexerStats t.indexer `shouldReturn` 200+      -- a request that does not wait is told the estimate: the median for+      -- the first in the queue, then a half more for each, two threads+      prefetch t.indexer 'd' [200] `shouldReturn` Refused TooLong 200+      prefetch t.indexer 'e' [200] `shouldReturn` Refused TooLong 300+      prefetch t.indexer 'f' [200] `shouldReturn` Refused TooLong 400+      -- and a request that joins one of them, its place+      prefetch t.indexer 'e' [] `shouldReturn` Refused TooLong 300+      -- the jobs of upkeep are not timed+      upkeep t.indexer 'g' [1000000]+      void (runNext t.indexer >> runNext t.indexer >> runNext t.indexer >> runNext t.indexer)+      (.median) <$> indexerStats t.indexer `shouldReturn` 200++    it "takes nothing for a key that another store holds, and gives the payload of upkeep back" $ do+      t <- testbed settings+      writeIORef t.other (Just 4000000)+      request t.indexer 'x' [1] `shouldReturn` Refused OtherStore 4000000+      upkeep t.indexer 'y' [2]+      -- the payloads of both come back, that of upkeep to be queued again+      readIORef t.blocked `shouldReturn` [('x', [1], 4000000), ('y', [2], 4000000)]+      runNext t.indexer `shouldReturn` False+      s <- indexerStats t.indexer+      (s.forRequests, s.forUpkeep, s.counts.refused.otherStore) `shouldBe` (0, 0, 1)++    it "joins upkeep to a queued job, and runs a key again when upkeep comes while it runs" $ do+      t <- testbed settings+      upkeep t.indexer 'x' [1]+      upkeep t.indexer 'x' [2]+      gate <- newEmptyMVar+      writeIORef t.during (readMVar gate)+      job <- async (runNext t.indexer)+      waitFor (not . null <$> readIORef t.started)+      upkeep t.indexer 'x' [3]+      upkeep t.indexer 'x' [4]+      putMVar gate ()+      wait job `shouldReturn` True+      writeIORef t.during (return ())+      runNext t.indexer `shouldReturn` True+      runNext t.indexer `shouldReturn` False+      readIORef t.ran `shouldReturn` [('x', [1, 2]), ('x', [3, 4])]+      -- the lock is kept from one run to the next+      readIORef t.locks `shouldReturn` ["take x", "release x"]++    it "answers a request while it lets go of a lock, and takes it anew for upkeep that came meanwhile" $ do+      t <- testbed settings+      gate <- newEmptyMVar+      writeIORef t.onRelease (readMVar gate)+      void (prefetch t.indexer 'x' [1])+      job <- async (runNext t.indexer)+      waitFor ((== ["take x", "release x"]) <$> readIORef t.locks)+      request t.indexer 'x' [2] `shouldReturn` Indexed+      upkeep t.indexer 'x' [3]+      writeIORef t.onRelease (return ())+      putMVar gate ()+      wait job `shouldReturn` True+      runNext t.indexer `shouldReturn` True+      readIORef t.ran `shouldReturn` [('x', [1]), ('x', [3])]+      readIORef t.locks `shouldReturn` ["take x", "release x", "take x", "release x"]++    it "takes nothing with a full queue, or beyond the horizon" $ do+      t <- testbed settings {room = 1}+      prefetch t.indexer 'a' [] `shouldReturn` Refused TooLong settings.guess+      request t.indexer 'b' [] `shouldReturn` Refused QueueFull settings.horizon+      u <- testbed settings {guess = settings.horizon + 1}+      request u.indexer 'b' [] `shouldReturn` Refused BeyondHorizon settings.horizon+      readIORef u.locks `shouldReturn` []++    it "counts prefetches apart from the refusals of requests" $ do+      t <- testbed settings {room = 1}+      -- a prefetch that queues a job, and one that finds it+      prefetch t.indexer 'a' [] `shouldReturn` Refused TooLong settings.guess+      prefetch t.indexer 'a' [] `shouldReturn` Refused TooLong settings.guess+      -- one that takes nothing, the queue being full+      prefetch t.indexer 'b' [] `shouldReturn` Refused QueueFull settings.horizon+      c <- (.counts) <$> indexerStats t.indexer+      (c.prefetched, c.declined, c.refused) `shouldBe` (2, 1, Refusals 0 0 0 0 0 0)+      -- a request is refused, and counted so+      request t.indexer 'b' [] `shouldReturn` Refused QueueFull settings.horizon+      d <- (.counts) <$> indexerStats t.indexer+      (d.prefetched, d.declined, d.refused.queueFull) `shouldBe` (2, 1, 1)++    it "takes no job while the store is not ready, and drops those it has" $ do+      t <- testbed settings+      upkeep t.indexer 'a' [1]+      writeIORef t.ready' False+      request t.indexer 'b' [2] `shouldReturn` Refused NotInSync settings.horizon+      upkeep t.indexer 'c' [3]+      runNext t.indexer `shouldReturn` True+      readIORef t.ran `shouldReturn` []+      sort <$> readIORef t.abandoned `shouldReturn` [('a', [1]), ('c', [3])]+      readIORef t.locks `shouldReturn` ["take a", "release a"]++    it "answers a request whose job failed, and goes on" $ do+      t <- testbed settings+      writeIORef t.during (fail "the database is gone")+      r <- async (request t.indexer 'x' [1])+      waiting t 1+      runNext t.indexer `shouldReturn` True+      wait r `shouldReturn` Failed+      readIORef t.failures `shouldReturn` [('x', [1])]+      readIORef t.locks `shouldReturn` ["take x", "release x"]++    it "tells a request that joins a running job what its size leaves of it, never more as it runs" $ do+      t <- testbed settings+      -- a job of 1,000 transactions that takes 100 ms: 100 microseconds each+      writeIORef t.length' True+      writeIORef t.size (Just 1000)+      void (prefetch t.indexer 'a' [100000])+      void (runNext t.indexer)+      -- a job of 50,000, which should take five seconds+      writeIORef t.length' False+      writeIORef t.size (Just 50000)+      gate <- newEmptyMVar+      writeIORef t.during (readMVar gate)+      void (prefetch t.indexer 'b' [])+      job <- async (runNext t.indexer)+      waitFor (elem 'b' <$> readIORef t.started)+      -- two seconds in, three are left+      t.advance 2000000+      prefetch t.indexer 'b' [] `shouldReturn` Refused TooLong 3000000+      -- six seconds in, it has run longer than its size says: the median+      t.advance 4000000+      prefetch t.indexer 'b' [] `shouldReturn` Refused TooLong 100000+      putMVar gate ()+      wait job `shouldReturn` True++    it "puts the estimate of a job in its lock when it starts, and when it tells its size" $ do+      t <- testbed settings {workers = 1}+      writeIORef t.size (Just 5000)+      upkeep t.indexer 'a' [1]+      void (runNext t.indexer)+      -- the median at the start, then 5,000 transactions at the first+      -- guess of the rate, 100 microseconds each+      readIORef t.renewed `shouldReturn` [[('a', 100000)], [('a', 500000)]]++    it "joins and waits for the job of this store when the door decided so, whatever its estimate" $ do+      t <- testbed settings {workers = 1}+      -- a job that tells it has 50,000 transactions: five seconds+      writeIORef t.size (Just 50000)+      upkeep t.indexer 'a' [5000000]+      gate <- newEmptyMVar+      writeIORef t.during (readMVar gate)+      job <- async (runNext t.indexer)+      waitFor ((== "a") <$> readIORef t.started)+      -- this store's own estimate is beyond the wait of a second+      request t.indexer 'a' [] `shouldReturn` Refused TooLong 5000000+      -- but the door read the estimate in the lock, and decided to wait+      waiter <- async (requestWithin t.indexer 1000000 'a' [])+      waitFor ((== 1) . (.waiting) <$> indexerStats t.indexer)+      putMVar gate ()+      void (wait job)+      wait waiter `shouldReturn` Indexed++    it "renews the locks of the jobs that it holds, with their estimates" $ do+      t <- testbed settings {workers = 1}+      upkeep t.indexer 'a' [1]+      void (prefetch t.indexer 'b' [2])+      renewHeld t.indexer+      readIORef t.renewed `shouldReturn` [[('a', 200000), ('b', 100000)]]+      sort <$> readIORef t.kept `shouldReturn` [('a', [1]), ('b', [2])]+  where+    settings = defaultSettings {workers = 1, wait = 1000000, horizon = 30000000, room = 10, guess = 100000}+    refusedFor why = \case+      Refused w _ -> w == why+      _ -> False+    now = (`div` 1000) . fromIntegral <$> getMonotonicTimeNSec :: IO Int++-- | An indexer of characters with lists of numbers as payloads, on a hand+-- clock, with no threads of its own, and what it did.+data Testbed = Testbed+  { indexer :: Indexer Char [Int],+    advance :: Int -> IO (),+    -- | the jobs run, in order, with their payloads+    ran :: IORef [(Char, [Int])],+    -- | jobs started+    started :: IORef [Char],+    -- | what a job does before it ends+    during :: IORef (IO ()),+    -- | whether a job moves the clock on by the sum of its payload+    length' :: IORef Bool,+    -- | the size that a job tells, if any+    size :: IORef (Maybe Int),+    -- | what the lock did+    locks :: IORef [String],+    -- | what letting go of a lock does after it is noted+    onRelease :: IORef (IO ()),+    -- | when another store holds every lock, until when+    other :: IORef (Maybe Int),+    ready' :: IORef Bool,+    abandoned :: IORef [(Char, [Int])],+    blocked :: IORef [(Char, [Int], Int)],+    failures :: IORef [(Char, [Int])],+    renewed :: IORef [[(Char, Int)]],+    kept :: IORef [(Char, [Int])]+  }++testbed :: Settings -> IO Testbed+testbed s = do+  clock <- newIORef 0+  ran <- newIORef []+  started <- newIORef []+  during <- newIORef (return ())+  length' <- newIORef False+  size <- newIORef Nothing+  locks <- newIORef []+  onRelease <- newIORef (return ())+  other <- newIORef Nothing+  ready' <- newIORef True+  abandoned <- newIORef []+  blocked <- newIORef []+  failures <- newIORef []+  renewed <- newIORef []+  kept <- newIORef []+  let note r x = atomicModifyIORef' r (\xs -> (xs <> [x], ()))+      lock =+        Lock+          { acquire = \k _ ->+              readIORef other >>= \case+                Just t -> return (Held t)+                Nothing -> note locks ("take " <> [k]) >> return Took,+            renew = \ks -> note renewed (sort ks) >> return 0,+            release = \k -> note locks ("release " <> [k]) >> (readIORef onRelease >>= id),+            holder = \_ -> return Nothing,+            every = 10000000+          }+      runner =+        Runner+          { index = \b k p -> do+              readIORef size >>= mapM_ b.sized+              note started k+              readIORef during >>= id+              moves <- readIORef length'+              when moves $ modifyIORef' clock (+ sum p)+              note ran (k, p),+            budget = \_ -> return noBudget,+            ready = readIORef ready',+            abandon = \k p -> note abandoned (k, p),+            blocked = \k p t -> note blocked (k, p, t),+            failed = \k p _ -> note failures (k, p),+            renew = \jobs -> forM_ jobs (note kept)+          }+  i <- newIndexerWith (readIORef clock) s lock runner+  return+    Testbed+      { indexer = i,+        advance = \d -> modifyIORef' clock (+ d),+        ran,+        started,+        during,+        length',+        size,+        locks,+        onRelease,+        other,+        ready',+        abandoned,+        blocked,+        failures,+        renewed,+        kept+      }++-- | Wait until so many requests wait for a job.+waiting :: Testbed -> Int -> IO ()+waiting t n = waitFor ((== n) . (.waiting) <$> indexerStats t.indexer)++-- | Wait for something, for up to five seconds.+waitFor :: IO Bool -> IO ()+waitFor check = go (5000 :: Int)+  where+    go k =+      check >>= \case+        True -> return ()+        False+          | k <= 0 -> expectationFailure "timed out"+          | otherwise -> threadDelay 1000 >> go (k - 1)
test/Haskoin/StoreSpec.hs view
@@ -3,6 +3,9 @@ module Haskoin.StoreSpec (spec) where  import Conduit+import Control.Concurrent (threadDelay)+import Control.Concurrent.Async (withAsync)+import Control.Concurrent.STM (TVar, atomically, modifyTVar', newTVarIO, readTVarIO) import Control.Monad import Data.ByteString (ByteString) import Data.ByteString qualified as B@@ -18,13 +21,14 @@ import Haskoin.Node import Haskoin.Store import Haskoin.Store.Database.Query qualified as Q+import Haskoin.Store.Integration.Temp (withSystemTempDirectory) import Haskoin.Store.TestBlocks (nextBlock) import NQE import Network.Socket import System.Mem (performMajorGC) import System.Random+import System.Timeout (timeout) import Test.Hspec-import UnliftIO  data TestStore = TestStore   { testStoreDB :: !DatabaseReader,@@ -36,10 +40,10 @@   }  spec :: Spec-spec = prepareContext $ \ctx -> do+spec = do   describe "Download" $ do     it "gets 8 blocks" $-      withTestStore bchRegTest ctx "eight-blocks" $ \TestStore {..} -> do+      withTestStore bchRegTest "eight-blocks" $ \TestStore {..} -> do         bs <- replicateM 8 . receiveMatch testStoreEvents $ \case           StoreBestBlock b -> Just b           _ -> Nothing@@ -50,7 +54,7 @@             bestHeight = bestNode.height         bestHeight `shouldBe` 8     it "get a block and its transactions" $-      withTestStore bchRegTest ctx "get-block-txs" $ \TestStore {..} ->+      withTestStore bchRegTest "get-block-txs" $ \TestStore {..} ->         do           let h1 = "5369ef2386c72acdf513ffd80aeba2a1774e2f004d120761e54a8bf614173f3e"               get_the_block h =@@ -69,13 +73,25 @@           listToMaybe bd.txs `shouldBe` Just h1           fmap (txHash . (.tx)) t1 `shouldBe` Just h1     it "downloads 80 blocks with a request for 64 and one for the remaining 16" $-      withTestStore bchRegTest ctx "block-window" $ \TestStore {..} -> do+      withTestStore bchRegTest "block-window" $ \TestStore {..} -> do         bs <- replicateM 80 . receiveMatch testStoreEvents $ \case           StoreBestBlock b -> Just b           _ -> Nothing         bestNodeM <- chainGetBlock testStoreChain (last bs)         fmap (.height) bestNodeM `shouldBe` Just 80         readTVarIO testStoreBlockRequests `shouldReturn` [64, 16]+    it "asks another peer for the blocks that the first one does not deliver" $+      -- The first peer to connect sends headers and never a block; the+      -- second connects a second later and delivers. With a stall of one+      -- second the store has its blocks in a few; waiting out the timeout+      -- of the first peer, as it used to, would take a minute.+      withStalledStore bchRegTest "stalled-peer" $ \TestStore {..} -> do+        got <- timeout 20000000 . replicateM 8 . receiveMatch testStoreEvents $ \case+          StoreBestBlock b -> Just b+          _ -> Nothing+        fmap length got `shouldBe` Just 8+        -- both were asked: the silent one first, then the one that delivers+        length <$> readTVarIO testStoreBlockRequests `shouldReturn` 2    describe "Memory" $     it "keeps the heap flat while syncing" $ do@@ -89,7 +105,7 @@       -- length is a multiple of 2000, so the last headers message the mock       -- peer sends is empty, which haskoin-node before 2.1.2 did not handle.       let blocks = chainOf 6000-      withTestStoreBlocks blocks bchRegTest ctx "heap" $ \TestStore {..} -> do+      withTestStoreBlocks blocks bchRegTest "heap" $ \TestStore {..} -> do         let wait_for height =               receive testStoreEvents >>= \case                 StoreBestBlock b ->@@ -110,7 +126,6 @@  withTestStore ::   Network ->-  Ctx ->   String ->   (TestStore -> IO a) ->   IO a@@ -120,11 +135,38 @@ withTestStoreBlocks ::   [Block] ->   Network ->-  Ctx ->   String ->   (TestStore -> IO a) ->   IO a-withTestStoreBlocks blocks net ctx t f =+withTestStoreBlocks blocks = withTestStoreConfig (dummyPeerConnect 0 mockPeerReact blocks) id++-- | A store with two mock peers: one that connects at once and never sends+-- a block, and one that connects a second later and behaves. The store+-- passes over a peer that delivers nothing for a second.+withStalledStore ::+  Network ->+  String ->+  (TestStore -> IO a) ->+  IO a+withStalledStore = withTestStoreConfig peers $ \cfg ->+  cfg {initPeers = ["127.0.0.1:18444", "127.0.0.1:18445"], blockStall = 1}+  where+    peers requests net ad sa = case sa of+      SockAddrInet 18444 _ -> dummyPeerConnect 0 silent allBlocks requests net ad sa+      _ -> dummyPeerConnect 1000000 mockPeerReact allBlocks requests net ad sa+    silent _ (MGetData _) = []+    silent blocks msg = mockPeerReact blocks msg++-- | A store whose peers are made by a function, with its configuration+-- changed by another.+withTestStoreConfig ::+  (TVar [Int] -> Network -> NetworkAddress -> SockAddr -> WithConnection) ->+  (StoreConfig -> StoreConfig) ->+  Network ->+  String ->+  (TestStore -> IO a) ->+  IO a+withTestStoreConfig peers change net t f =   withSystemTempDirectory ("haskoin-store-test-" <> t <> "-") $ \w -> do     requests <- newTVarIO []     let ad =@@ -132,38 +174,36 @@             nodeNetwork             (sockToHostAddress (SockAddrInet 0 0))         cfg =-          StoreConfig-            { maxPeers = 20,-              -- the mock peer, by name: since haskoin-node 2.5 the seed of-              -- a regtest network is not an address that discovery takes-              initPeers = ["127.0.0.1:18444"],-              discover = False,-              db = w,-              net = net,-              ctx = ctx,-              redis = Nothing,-              gap = gap,-              initGap = 20,-              redisMinAddrs = 100,-              redisMinTxs = 1000,-              redisMaxXPubs = 10000,-              noMempool = False,-              wipeMempool = False,-              fleet = False,-              syncMempool = False,-              blockWindow = 64,-              mempoolTimeout = 14,-              peerTimeout = 60,-              maxPeerLife = 48 * 3600,-              connect = dummyPeerConnect blocks requests net ad,-              stats = Nothing,-              bloom = True,-              maxOpenFiles = Nothing,-              writeBufferSize = Nothing,-              maxWalSize = Nothing,-              blockCache = Nothing,-              logger = Logger {level = LevelError, source = "", push = \_ -> return ()}-            }+          change+            StoreConfig+              { maxPeers = 20,+                -- the mock peer, by name: since haskoin-node 2.5 the seed of+                -- a regtest network is not an address that discovery takes+                initPeers = ["127.0.0.1:18444"],+                discover = False,+                db = w,+                net = net,+                redis = Nothing,+                gap = gap,+                initGap = 20,+                redisMaxXPubs = 10000,+                xpubs = defaultXPubConfig,+                blockWindow = 64,+                mempoolTimeout = 14,+                peerTimeout = 60,+                blockStall = 10,+                blockPing = 5,+                maxPeerLife = 48 * 3600,+                connect = peers requests net ad,+                stats = Nothing,+                bloom = True,+                maxOpenFiles = Nothing,+                writeBufferSize = Nothing,+                maxWalSize = Nothing,+                backgroundJobs = Nothing,+                blockCache = Nothing,+                logger = Logger {level = LevelError, source = "", push = \_ -> return ()}+              }     withStore cfg $ \store ->       withSubscription store.pub $ \sub ->         f@@ -252,30 +292,41 @@   \AAAAAAAAAAAAAAAAAAD/////DF8BAQgvRUIzMi4wL/////8BAPIFKgEAAAAjIQME7KZAozHsyrOO\   \wT6Wn6LtY47B39Th44JKsZ8BGJCvc6wAAAAA" -dummyPeerConnect :: [Block] -> TVar [Int] -> Network -> NetworkAddress -> SockAddr -> WithConnection-dummyPeerConnect blocks requests net ad sa f = do+-- | A mock peer that waits so many microseconds before it says anything,+-- and then answers each message with a function of the blocks that it has.+dummyPeerConnect ::+  Int ->+  ([Block] -> Message -> [Message]) ->+  [Block] ->+  TVar [Int] ->+  Network ->+  NetworkAddress ->+  SockAddr ->+  WithConnection+dummyPeerConnect wait react blocks requests net ad sa f = do   r <- newInbox   s <- newInbox   let s' = inboxToMailbox s   withAsync (go r s') $ \_ -> do-    let o = awaitForever (`send` r)-        i = forever (receive s >>= yield)+    let o = conduitMailbox r+        i = repeatMC (receive s)     f (Conduits i o) :: IO ()   where     go :: Inbox ByteString -> Mailbox ByteString -> IO ()     go r s = do+      threadDelay wait       nonce <- randomIO-      now <- round <$> liftIO getPOSIXTime+      now <- round <$> getPOSIXTime       let rmt = NetworkAddress 0 (sockToHostAddress sa)-          ver = buildVersion net nonce 0 ad rmt now+          ver = buildVersion net nonce (fromIntegral (length blocks)) ad rmt now       runPut (putMessage net (MVersion ver)) `send` s       runConduit $-        forever (receive r >>= yield)+        repeatMC (receive r)           .| inc           .| iterMC (recordBlockRequest requests)-          .| concatMapC (mockPeerReact blocks)+          .| concatMapC (react blocks)           .| outc-          .| awaitForever (`send` s)+          .| conduitMailbox s     outc = mapMC $ \msg' -> return $ runPut (putMessage net msg')     inc = forever $ do       x <- takeCE 24 .| foldC