packages feed

rocksdb-query-0.7.0: test/Spec.hs

{-# LANGUAGE FlexibleInstances #-}
{-# LANGUAGE ImportQualifiedPost #-}
{-# LANGUAGE MultiParamTypeClasses #-}

-- |
-- Tests of "Database.RocksDB.Query" against a real RocksDB, in temporary
-- directories.
--
-- The first group pins down what each function returns, on a handful of
-- records. The second is about how a prefix is read: in chunks, from one
-- call into RocksDB for many entries. Chunks must not show in the results,
-- so those tests use lists longer than any chunk, lists that end exactly
-- where a chunk does, and values larger than a chunk may be. The suite runs
-- with a stack of 1 MiB at most, so that a query whose stack grows with its
-- result fails on the long lists.
module Main (main) where

import Conduit
import Control.Applicative
import Control.Exception (ErrorCall, evaluate)
import Control.Monad
import Data.ByteString (ByteString)
import Data.ByteString qualified as B
import Data.Default
import Data.Serialize as S
import Data.Word
import Database.RocksDB as R
import Database.RocksDB.Query
import System.IO.Temp
import Test.Hspec

newtype KeyOne = KeyOne Word32 deriving (Show, Eq)

data KeyTwo
  = KeyTwo
      Word32
      Word32
  | KeyTwoBase Word32
  deriving (Show, Eq)

-- | Keys of the tests of chunks: a group, and a number within it. They sort
-- after every 'KeyTwo'.
data KeyRun
  = KeyRun Word32 Word32
  | KeyRunBase Word32
  deriving (Show, Eq)

instance KeyValue KeyTwo String

instance KeyValue KeyOne String

instance KeyValue KeyRun ByteString

instance Serialize KeyOne where
  put (KeyOne x) = do
    putWord8 0x01
    S.put x
  get = do
    getWord8 >>= guard . (== 0x01)
    KeyOne <$> S.get

instance Serialize KeyTwo where
  put k = do
    putWord8 0x02
    case k of
      KeyTwoBase x -> S.put x
      KeyTwo x y -> S.put x >> S.put y
  get =
    (guard . (== 0x02) =<< getWord8)
      >> KeyTwo <$> S.get <*> S.get <|> KeyTwoBase <$> S.get

instance Serialize KeyRun where
  put k = do
    putWord8 0x03
    case k of
      KeyRunBase g -> S.put g
      KeyRun g n -> S.put g >> S.put n
  get = do
    getWord8 >>= guard . (== 0x03)
    KeyRun <$> S.get <*> S.get

main :: IO ()
main = hspec $ do
  around withRecords recordSpec
  around withEmpty chunkSpec

-- | A new database in a temporary directory, with a second column family.
withEmpty :: (DB -> IO a) -> IO a
withEmpty f =
  withSystemTempDirectory "rocksdb-query-test-" $ \d ->
    withDBCF d cfg [("other", cfg)] f
  where
    cfg = def {createIfMissing = True, createMissingColumnFamilies = True}

-- | A new database with the records of 'insertTestRecords'.
withRecords :: (DB -> IO a) -> IO a
withRecords f = withEmpty $ \db -> insertTestRecords db >> f db

-- | The second column family of the test database.
other :: DB -> ColumnFamily
other db = case columnFamilies db of
  cf : _ -> cf
  [] -> error "the test database has no column family"

insertTestRecords :: DB -> IO ()
insertTestRecords db = do
  insert db (KeyOne 1) "Hello World!"
  insert db (KeyOne 2) "Hello World Again!"
  insert db (KeyTwo 1 1) "Hello First World!"
  insert db (KeyTwo 1 2) "Hello First World Again!"
  insert db (KeyTwo 2 1) "Hello Second World!"
  insert db (KeyTwo 2 2) "Hello Second World Again!"

recordSpec :: SpecWith DB
recordSpec =
  describe "database" $ do
    it "reads a record" $ \db -> do
      r <- retrieve db (KeyTwo 1 2)
      r `shouldBe` Just "Hello First World Again!"
    it "reads two records at the end" $ \db -> do
      let ls =
            [ (KeyTwo 2 1, "Hello Second World!"),
              (KeyTwo 2 2, "Hello Second World Again!")
            ]
      rs <- matchingAsList db (KeyTwoBase 2)
      rs `shouldBe` ls
    it "reads two records in the middle" $ \db -> do
      let ls =
            [ (KeyTwo 1 1, "Hello First World!"),
              (KeyTwo 1 2, "Hello First World Again!")
            ]
      rs <- matchingAsList db (KeyTwoBase 1)
      rs `shouldBe` ls
    it "query and skip" $ \db -> do
      let ex = (KeyTwo 2 2, "Hello Second World Again!")
      rs <-
        matchingSkipAsList
          db
          (KeyTwoBase 2)
          (KeyTwo 2 2)
      rs `shouldBe` [ex]
    it "reads nothing for a prefix that no key has" $ \db -> do
      rs <- matchingAsList db (KeyTwoBase 3)
      rs `shouldBe` ([] :: [(KeyTwo, String)])
      r <- firstMatching db (KeyTwoBase 3)
      r `shouldBe` nothing
      r' <- firstMatching db (KeyTwoBase 0)
      r' `shouldBe` nothing
    it "reads the first record of a prefix" $ \db -> do
      r <- firstMatching db (KeyTwoBase 1)
      r `shouldBe` Just (KeyTwo 1 1, "Hello First World!")
      r' <- firstMatching db (KeyTwoBase 2)
      r' `shouldBe` Just (KeyTwo 2 1, "Hello Second World!")
    it "reads the first record from a key on" $ \db -> do
      r <- firstMatchingSkip db (KeyTwoBase 1) (KeyTwo 1 2)
      r `shouldBe` Just (KeyTwo 1 2, "Hello First World Again!")
      -- past the last key of the prefix, and of the database
      r' <- firstMatchingSkip db (KeyTwoBase 1) (KeyTwo 1 3)
      r' `shouldBe` nothing
      r'' <- firstMatchingSkip db (KeyTwoBase 2) (KeyTwo 2 3)
      r'' `shouldBe` nothing
    it "streams the records of a prefix" $ \db -> do
      rs <- withIter db $ \it -> runConduit $ matching it (KeyTwoBase 1) .| sinkList
      rs
        `shouldBe` [ (KeyTwo 1 1, "Hello First World!"),
                     (KeyTwo 1 2, "Hello First World Again!")
                   ]
      rs' <- withIter db $ \it ->
        runConduit $ matchingSkip it (KeyTwoBase 2) (KeyTwo 2 2) .| sinkList
      rs' `shouldBe` [(KeyTwo 2 2, "Hello Second World Again!")]

-- | No record under a 'KeyTwo', for where nothing else says the type of the
-- value that was not found.
nothing :: Maybe (KeyTwo, String)
nothing = Nothing

-- | The value of an entry of a run: short, and different for every key.
runValue :: Word32 -> Word32 -> ByteString
runValue g n = encode (g * 1000003 + n)

-- | The entries of a run of so many keys in a group, in order.
run :: Word32 -> Word32 -> [(KeyRun, ByteString)]
run g count = [(KeyRun g n, runValue g n) | n <- [0 .. count - 1], count > 0]

-- | Write a run to a column family, or to the default one.
insertRun :: DB -> Maybe ColumnFamily -> Word32 -> Word32 -> IO ()
insertRun db mcf g count = mapM_ (writeBatch db) (pieces (map op (run g count)))
  where
    op (k, v) = maybe (insertOp k v) (\cf -> insertOpCF cf k v) mcf
    pieces [] = []
    pieces ops = let (a, b) = splitAt 1000 ops in a : pieces b

-- | Every way of reading a whole prefix, which all have to agree.
everyWay :: DB -> KeyRun -> IO [[(KeyRun, ByteString)]]
everyWay db base =
  sequence
    [ matchingAsList db base,
      withIter db $ \it -> runConduit $ matching it base .| sinkList
    ]

chunkSpec :: SpecWith DB
chunkSpec =
  describe "prefixes read in chunks" $ do
    it "reads a list of a hundred thousand records" $ \db -> do
      insertRun db Nothing 1 100000
      insertRun db Nothing 2 10
      rs <- matchingAsList db (KeyRunBase 1)
      length rs `shouldBe` 100000
      rs `shouldBe` run 1 100000
    it "streams a hundred thousand records" $ \db -> do
      insertRun db Nothing 1 100000
      insertRun db Nothing 2 10
      withIter db $ \it -> do
        rs <- runConduit $ matching it (KeyRunBase 1) .| sinkList
        rs `shouldBe` run 1 100000
        n <- runConduit $ stream it (KeyRunBase 1) .| lengthC
        n `shouldBe` (100000 :: Int)
    it "reads long lists from a key on, in a column family" $ \db -> do
      insertRun db (Just (other db)) 1 30000
      insertRun db (Just (other db)) 2 10
      rs <- matchingSkipAsListCF db (other db) (KeyRunBase 1) (KeyRun 1 12345)
      rs `shouldBe` drop 12345 (run 1 30000)
      rs' <- matchingAsListCF db (other db) (KeyRunBase 1)
      rs' `shouldBe` run 1 30000
      rs'' <- withIterCF db (other db) $ \it ->
        runConduit $ matchingSkip it (KeyRunBase 1) (KeyRun 1 29000) .| sinkList
      rs'' `shouldBe` drop 29000 (run 1 30000)
      list db (KeyRunBase 1) `shouldReturn` []
    it "ends a list where its prefix ends, wherever the chunks end" $ \db -> do
      -- Lists read 256 entries at a time, and conduits 16, then 64, then
      -- 256: each group has a number of keys at, just below or just above
      -- where one of those chunks ends, and the next group follows at once.
      let counts = [0, 1, 15, 16, 17, 79, 80, 81, 255, 256, 257, 335, 336, 337, 511, 512, 513, 592, 1024]
          groups = zip [10 ..] counts
      forM_ groups $ \(g, count) -> insertRun db Nothing g count
      forM_ groups $ \(g, count) -> do
        results <- everyWay db (KeyRunBase g)
        forM_ results (`shouldBe` run g count)
        first <- firstMatching db (KeyRunBase g)
        first `shouldBe` (if count == 0 then Nothing else Just (KeyRun g 0, runValue g 0))
    it "ends a list where the data ends, wherever the chunks end" $ \db ->
      forM_ [16, 256, 336, 512] $ \count -> do
        -- the last group of the database, longer with every round
        insertRun db Nothing 5 count
        results <- everyWay db (KeyRunBase 5)
        forM_ results (`shouldBe` run 5 count)
    it "reads values larger than a chunk may be" $ \db -> do
      -- a megabyte is the limit of a chunk: values above it, below it, and
      -- small ones between them
      let sizes = [10, 1500000, 20, 700000, 700000, 700000, 30, 2000000, 2000000, 40]
          entries = [(KeyRun 7 n, B.replicate size (fromIntegral n)) | (n, size) <- zip [0 ..] sizes]
      forM_ entries $ uncurry (insert db)
      insertRun db Nothing 8 10
      results <- everyWay db (KeyRunBase 7)
      forM_ results (`shouldBe` entries)
    it "reads only a little ahead of a consumer that stops" $ \db -> do
      insertRun db Nothing 1 1000
      withIter db $ \it -> do
        rs <- runConduit $ matching it (KeyRunBase 1) .| takeC 3 .| sinkList
        rs `shouldBe` take 3 (run 1 1000)
        -- the first chunk is of 16 entries, and no other was read
        iterKey it `shouldReturn` Just (encode (KeyRun 1 16))
        rs' <- runConduit $ matching it (KeyRunBase 1) .| takeC 17 .| sinkList
        rs' `shouldBe` take 17 (run 1 1000)
        -- the second is of 64
        iterKey it `shouldReturn` Just (encode (KeyRun 1 80))
    it "decodes an entry only when a consumer gets to it" $ \db -> do
      insertRun db Nothing 1 3
      -- a value that is not a ByteString: its length says more bytes than
      -- there are
      R.put db (encode (KeyRun 1 3)) (B.pack [0, 0, 0, 0, 0, 0, 0, 9, 1, 2, 3])
      withIter db $ \it -> do
        rs <- runConduit $ matching it (KeyRunBase 1) .| takeC 3 .| sinkList
        rs `shouldBe` run 1 3
        runConduit (stream it (KeyRunBase 1) .| sinkList) `shouldThrow` anyErrorCall
      list db (KeyRunBase 1) `shouldThrow` anyErrorCall
      firstMatching db (KeyRunBase 1) `shouldReturn` Just (KeyRun 1 0, runValue 1 0)
      firstFrom db (KeyRunBase 1) (KeyRun 1 3) `shouldThrow` anyErrorCall
    it "returns a list that does not need the iterator any more" $ \db -> do
      insertRun db Nothing 1 1000
      rs <- matchingAsList db (KeyRunBase 1)
      -- the database changes before the list is looked at
      forM_ (run 1 1000) $ \(k, _) -> R.delete db (encode k)
      n <- evaluate (length rs)
      n `shouldBe` 1000
      rs `shouldBe` run 1 1000
  where
    anyErrorCall :: Selector ErrorCall
    anyErrorCall = const True
    -- the queries at the types of a run, for where nothing else says them
    list :: DB -> KeyRun -> IO [(KeyRun, ByteString)]
    list = matchingAsList
    stream :: Iterator -> KeyRun -> ConduitT () (KeyRun, ByteString) IO ()
    stream = matching
    firstFrom :: DB -> KeyRun -> KeyRun -> IO (Maybe (KeyRun, ByteString))
    firstFrom = firstMatchingSkip