{-# LANGUAGE FlexibleInstances #-}
{-# LANGUAGE ImportQualifiedPost #-}
{-# LANGUAGE MultiParamTypeClasses #-}
-- |
-- Tests of "Database.RocksDB.Query" against a real RocksDB, in temporary
-- directories.
--
-- The first group pins down what each function returns, on a handful of
-- records. The second is about how a prefix is read: in chunks, from one
-- call into RocksDB for many entries. Chunks must not show in the results,
-- so those tests use lists longer than any chunk, lists that end exactly
-- where a chunk does, and values larger than a chunk may be. The suite runs
-- with a stack of 1 MiB at most, so that a query whose stack grows with its
-- result fails on the long lists.
module Main (main) where
import Conduit
import Control.Applicative
import Control.Exception (ErrorCall, evaluate)
import Control.Monad
import Data.ByteString (ByteString)
import Data.ByteString qualified as B
import Data.Default
import Data.Serialize as S
import Data.Word
import Database.RocksDB as R
import Database.RocksDB.Query
import System.IO.Temp
import Test.Hspec
newtype KeyOne = KeyOne Word32 deriving (Show, Eq)
data KeyTwo
= KeyTwo
Word32
Word32
| KeyTwoBase Word32
deriving (Show, Eq)
-- | Keys of the tests of chunks: a group, and a number within it. They sort
-- after every 'KeyTwo'.
data KeyRun
= KeyRun Word32 Word32
| KeyRunBase Word32
deriving (Show, Eq)
instance KeyValue KeyTwo String
instance KeyValue KeyOne String
instance KeyValue KeyRun ByteString
instance Serialize KeyOne where
put (KeyOne x) = do
putWord8 0x01
S.put x
get = do
getWord8 >>= guard . (== 0x01)
KeyOne <$> S.get
instance Serialize KeyTwo where
put k = do
putWord8 0x02
case k of
KeyTwoBase x -> S.put x
KeyTwo x y -> S.put x >> S.put y
get =
(guard . (== 0x02) =<< getWord8)
>> KeyTwo <$> S.get <*> S.get <|> KeyTwoBase <$> S.get
instance Serialize KeyRun where
put k = do
putWord8 0x03
case k of
KeyRunBase g -> S.put g
KeyRun g n -> S.put g >> S.put n
get = do
getWord8 >>= guard . (== 0x03)
KeyRun <$> S.get <*> S.get
main :: IO ()
main = hspec $ do
around withRecords recordSpec
around withEmpty chunkSpec
-- | A new database in a temporary directory, with a second column family.
withEmpty :: (DB -> IO a) -> IO a
withEmpty f =
withSystemTempDirectory "rocksdb-query-test-" $ \d ->
withDBCF d cfg [("other", cfg)] f
where
cfg = def {createIfMissing = True, createMissingColumnFamilies = True}
-- | A new database with the records of 'insertTestRecords'.
withRecords :: (DB -> IO a) -> IO a
withRecords f = withEmpty $ \db -> insertTestRecords db >> f db
-- | The second column family of the test database.
other :: DB -> ColumnFamily
other db = case columnFamilies db of
cf : _ -> cf
[] -> error "the test database has no column family"
insertTestRecords :: DB -> IO ()
insertTestRecords db = do
insert db (KeyOne 1) "Hello World!"
insert db (KeyOne 2) "Hello World Again!"
insert db (KeyTwo 1 1) "Hello First World!"
insert db (KeyTwo 1 2) "Hello First World Again!"
insert db (KeyTwo 2 1) "Hello Second World!"
insert db (KeyTwo 2 2) "Hello Second World Again!"
recordSpec :: SpecWith DB
recordSpec =
describe "database" $ do
it "reads a record" $ \db -> do
r <- retrieve db (KeyTwo 1 2)
r `shouldBe` Just "Hello First World Again!"
it "reads two records at the end" $ \db -> do
let ls =
[ (KeyTwo 2 1, "Hello Second World!"),
(KeyTwo 2 2, "Hello Second World Again!")
]
rs <- matchingAsList db (KeyTwoBase 2)
rs `shouldBe` ls
it "reads two records in the middle" $ \db -> do
let ls =
[ (KeyTwo 1 1, "Hello First World!"),
(KeyTwo 1 2, "Hello First World Again!")
]
rs <- matchingAsList db (KeyTwoBase 1)
rs `shouldBe` ls
it "query and skip" $ \db -> do
let ex = (KeyTwo 2 2, "Hello Second World Again!")
rs <-
matchingSkipAsList
db
(KeyTwoBase 2)
(KeyTwo 2 2)
rs `shouldBe` [ex]
it "reads nothing for a prefix that no key has" $ \db -> do
rs <- matchingAsList db (KeyTwoBase 3)
rs `shouldBe` ([] :: [(KeyTwo, String)])
r <- firstMatching db (KeyTwoBase 3)
r `shouldBe` nothing
r' <- firstMatching db (KeyTwoBase 0)
r' `shouldBe` nothing
it "reads the first record of a prefix" $ \db -> do
r <- firstMatching db (KeyTwoBase 1)
r `shouldBe` Just (KeyTwo 1 1, "Hello First World!")
r' <- firstMatching db (KeyTwoBase 2)
r' `shouldBe` Just (KeyTwo 2 1, "Hello Second World!")
it "reads the first record from a key on" $ \db -> do
r <- firstMatchingSkip db (KeyTwoBase 1) (KeyTwo 1 2)
r `shouldBe` Just (KeyTwo 1 2, "Hello First World Again!")
-- past the last key of the prefix, and of the database
r' <- firstMatchingSkip db (KeyTwoBase 1) (KeyTwo 1 3)
r' `shouldBe` nothing
r'' <- firstMatchingSkip db (KeyTwoBase 2) (KeyTwo 2 3)
r'' `shouldBe` nothing
it "streams the records of a prefix" $ \db -> do
rs <- withIter db $ \it -> runConduit $ matching it (KeyTwoBase 1) .| sinkList
rs
`shouldBe` [ (KeyTwo 1 1, "Hello First World!"),
(KeyTwo 1 2, "Hello First World Again!")
]
rs' <- withIter db $ \it ->
runConduit $ matchingSkip it (KeyTwoBase 2) (KeyTwo 2 2) .| sinkList
rs' `shouldBe` [(KeyTwo 2 2, "Hello Second World Again!")]
-- | No record under a 'KeyTwo', for where nothing else says the type of the
-- value that was not found.
nothing :: Maybe (KeyTwo, String)
nothing = Nothing
-- | The value of an entry of a run: short, and different for every key.
runValue :: Word32 -> Word32 -> ByteString
runValue g n = encode (g * 1000003 + n)
-- | The entries of a run of so many keys in a group, in order.
run :: Word32 -> Word32 -> [(KeyRun, ByteString)]
run g count = [(KeyRun g n, runValue g n) | n <- [0 .. count - 1], count > 0]
-- | Write a run to a column family, or to the default one.
insertRun :: DB -> Maybe ColumnFamily -> Word32 -> Word32 -> IO ()
insertRun db mcf g count = mapM_ (writeBatch db) (pieces (map op (run g count)))
where
op (k, v) = maybe (insertOp k v) (\cf -> insertOpCF cf k v) mcf
pieces [] = []
pieces ops = let (a, b) = splitAt 1000 ops in a : pieces b
-- | Every way of reading a whole prefix, which all have to agree.
everyWay :: DB -> KeyRun -> IO [[(KeyRun, ByteString)]]
everyWay db base =
sequence
[ matchingAsList db base,
withIter db $ \it -> runConduit $ matching it base .| sinkList
]
chunkSpec :: SpecWith DB
chunkSpec =
describe "prefixes read in chunks" $ do
it "reads a list of a hundred thousand records" $ \db -> do
insertRun db Nothing 1 100000
insertRun db Nothing 2 10
rs <- matchingAsList db (KeyRunBase 1)
length rs `shouldBe` 100000
rs `shouldBe` run 1 100000
it "streams a hundred thousand records" $ \db -> do
insertRun db Nothing 1 100000
insertRun db Nothing 2 10
withIter db $ \it -> do
rs <- runConduit $ matching it (KeyRunBase 1) .| sinkList
rs `shouldBe` run 1 100000
n <- runConduit $ stream it (KeyRunBase 1) .| lengthC
n `shouldBe` (100000 :: Int)
it "reads long lists from a key on, in a column family" $ \db -> do
insertRun db (Just (other db)) 1 30000
insertRun db (Just (other db)) 2 10
rs <- matchingSkipAsListCF db (other db) (KeyRunBase 1) (KeyRun 1 12345)
rs `shouldBe` drop 12345 (run 1 30000)
rs' <- matchingAsListCF db (other db) (KeyRunBase 1)
rs' `shouldBe` run 1 30000
rs'' <- withIterCF db (other db) $ \it ->
runConduit $ matchingSkip it (KeyRunBase 1) (KeyRun 1 29000) .| sinkList
rs'' `shouldBe` drop 29000 (run 1 30000)
list db (KeyRunBase 1) `shouldReturn` []
it "ends a list where its prefix ends, wherever the chunks end" $ \db -> do
-- Lists read 256 entries at a time, and conduits 16, then 64, then
-- 256: each group has a number of keys at, just below or just above
-- where one of those chunks ends, and the next group follows at once.
let counts = [0, 1, 15, 16, 17, 79, 80, 81, 255, 256, 257, 335, 336, 337, 511, 512, 513, 592, 1024]
groups = zip [10 ..] counts
forM_ groups $ \(g, count) -> insertRun db Nothing g count
forM_ groups $ \(g, count) -> do
results <- everyWay db (KeyRunBase g)
forM_ results (`shouldBe` run g count)
first <- firstMatching db (KeyRunBase g)
first `shouldBe` (if count == 0 then Nothing else Just (KeyRun g 0, runValue g 0))
it "ends a list where the data ends, wherever the chunks end" $ \db ->
forM_ [16, 256, 336, 512] $ \count -> do
-- the last group of the database, longer with every round
insertRun db Nothing 5 count
results <- everyWay db (KeyRunBase 5)
forM_ results (`shouldBe` run 5 count)
it "reads values larger than a chunk may be" $ \db -> do
-- a megabyte is the limit of a chunk: values above it, below it, and
-- small ones between them
let sizes = [10, 1500000, 20, 700000, 700000, 700000, 30, 2000000, 2000000, 40]
entries = [(KeyRun 7 n, B.replicate size (fromIntegral n)) | (n, size) <- zip [0 ..] sizes]
forM_ entries $ uncurry (insert db)
insertRun db Nothing 8 10
results <- everyWay db (KeyRunBase 7)
forM_ results (`shouldBe` entries)
it "reads only a little ahead of a consumer that stops" $ \db -> do
insertRun db Nothing 1 1000
withIter db $ \it -> do
rs <- runConduit $ matching it (KeyRunBase 1) .| takeC 3 .| sinkList
rs `shouldBe` take 3 (run 1 1000)
-- the first chunk is of 16 entries, and no other was read
iterKey it `shouldReturn` Just (encode (KeyRun 1 16))
rs' <- runConduit $ matching it (KeyRunBase 1) .| takeC 17 .| sinkList
rs' `shouldBe` take 17 (run 1 1000)
-- the second is of 64
iterKey it `shouldReturn` Just (encode (KeyRun 1 80))
it "decodes an entry only when a consumer gets to it" $ \db -> do
insertRun db Nothing 1 3
-- a value that is not a ByteString: its length says more bytes than
-- there are
R.put db (encode (KeyRun 1 3)) (B.pack [0, 0, 0, 0, 0, 0, 0, 9, 1, 2, 3])
withIter db $ \it -> do
rs <- runConduit $ matching it (KeyRunBase 1) .| takeC 3 .| sinkList
rs `shouldBe` run 1 3
runConduit (stream it (KeyRunBase 1) .| sinkList) `shouldThrow` anyErrorCall
list db (KeyRunBase 1) `shouldThrow` anyErrorCall
firstMatching db (KeyRunBase 1) `shouldReturn` Just (KeyRun 1 0, runValue 1 0)
firstFrom db (KeyRunBase 1) (KeyRun 1 3) `shouldThrow` anyErrorCall
it "returns a list that does not need the iterator any more" $ \db -> do
insertRun db Nothing 1 1000
rs <- matchingAsList db (KeyRunBase 1)
-- the database changes before the list is looked at
forM_ (run 1 1000) $ \(k, _) -> R.delete db (encode k)
n <- evaluate (length rs)
n `shouldBe` 1000
rs `shouldBe` run 1 1000
where
anyErrorCall :: Selector ErrorCall
anyErrorCall = const True
-- the queries at the types of a run, for where nothing else says them
list :: DB -> KeyRun -> IO [(KeyRun, ByteString)]
list = matchingAsList
stream :: Iterator -> KeyRun -> ConduitT () (KeyRun, ByteString) IO ()
stream = matching
firstFrom :: DB -> KeyRun -> KeyRun -> IO (Maybe (KeyRun, ByteString))
firstFrom = firstMatchingSkip