streamly-lmdb 0.4.0 → 0.5.0

raw patch · 5 files changed

+100/−28 lines, 5 filesPVP ok

version bump matches the API change (PVP)

API changes (from Hackage documentation)

Files

• ChangeLog.md +4/−0
• src/Streamly/External/LMDB.hs +73/−22
• src/Streamly/External/LMDB/Internal/Foreign.hsc +6/−0
• streamly-lmdb.cabal +1/−1
• test/Streamly/External/LMDB/Tests.hs +16/−5

@@ -1,3 +1,7 @@+## 0.5.0++* Added the ability to close databases and environments.+ ## 0.4.0  * Added support for precreating read-only transactions.

@@ -14,6 +14,7 @@     Environment,     openEnvironment,     isReadOnlyEnvironment,+    closeEnvironment,      -- ** Mode     Mode,@@ -37,15 +38,19 @@     Database,     getDatabase,     clearDatabase,+    closeDatabase,      -- * Reading     readLMDB,     unsafeReadLMDB, -    -- ** Read-only transactions+    -- ** Read-only transactions and cursors     ReadOnlyTxn,     beginReadOnlyTxn,     abortReadOnlyTxn,+    Cursor,+    openCursor,+    closeCursor,      -- ** Read options     ReadOptions (..),@@ -82,11 +87,14 @@ import Streamly.External.LMDB.Internal.Foreign   ( LMDB_Error (..),     MDB_ErrCode (..),+    MDB_cursor,     MDB_env,     MDB_txn,     MDB_val (MDB_val, mv_data, mv_size),     c_mdb_cursor_close,     c_mdb_cursor_get,+    c_mdb_dbi_close,+    c_mdb_env_close,     c_mdb_get,     c_mdb_txn_abort,     combineOptions,@@ -205,6 +213,22 @@    return env +-- | Closes the given environment.+--+-- If you have merely a few dozen environments at most, there should be no need for this. (It is a+-- common practice with LMDB to create one’s environments once and reuse them for the remainder of+-- the program’s execution.) If you find yourself needing this, it is your responsibility to heed+-- the [documented+-- caveats](https://github.com/LMDB/lmdb/blob/8d0cbbc936091eb85972501a9b31a8f86d4c51a7/libraries/liblmdb/lmdb.h#L787).+--+-- In particular, you will probably, before calling this function, want to (a) use 'closeDatabase',+-- and (b) pass in precreated transactions and cursors to 'readLMDB' and 'unsafeReadLMDB' to make+-- sure there are no transactions or cursors still left to be cleaned up by the garbage collector.+-- (As an alternative to (b), one could try manually triggering the garbage collector.)+closeEnvironment :: (Mode mode) => Environment mode -> IO ()+closeEnvironment (Environment penv) =+  c_mdb_env_close penv+ -- | Gets a database with the given name. When creating a database (i.e., getting it for the first -- time), one must do so in 'ReadWrite' mode. --@@ -233,14 +257,26 @@     )     >>= wait +-- | Closes the given database.+--+-- If you have merely a few dozen databases at most, there should be no need for this. (It is a+-- common practice with LMDB to create one’s databases once and reuse them for the remainder of the+-- program’s execution.) If you find yourself needing this, it is your responsibility to heed the+-- [documented+-- caveats](https://github.com/LMDB/lmdb/blob/8d0cbbc936091eb85972501a9b31a8f86d4c51a7/libraries/liblmdb/lmdb.h#L1200).+closeDatabase :: (Mode mode) => Database mode -> IO ()+closeDatabase (Database penv dbi) =+  c_mdb_dbi_close penv dbi+ -- | Creates an unfold with which we can stream key-value pairs from the given database. ----- If an existing read-only transaction is not provided, a read-only transaction is automatically--- created and kept open for the duration of the unfold; we suggest doing this as a first option.--- However, if you find this to be a bottleneck (e.g., if you find upon profiling that a significant--- time is being spent at @mdb_txn_begin@, or if you find yourself having to increase 'maxReaders'--- in the environment’s limits because the transactions are not being garbage collected fast--- enough), consider precreating a transaction using 'beginReadOnlyTxn'.+-- If an existing read-only transaction and cursor are not provided, a read-only transaction and+-- cursor are automatically created and kept open for the duration of the unfold; we suggest doing+-- this as a first option. However, if you find this to be a bottleneck (e.g., if you find upon+-- profiling that a significant time is being spent at @mdb_txn_begin@, or if you find yourself+-- having to increase 'maxReaders' in the environment’s limits because the transactions and cursors+-- are not being garbage collected fast enough), consider precreating a transaction and cursor using+-- 'beginReadOnlyTxn' and 'openCursor'. -- -- In any case, bear in mind at all times LMDB’s [caveats regarding long-lived -- transactions](https://github.com/LMDB/lmdb/blob/8d0cbbc936091eb85972501a9b31a8f86d4c51a7/libraries/liblmdb/lmdb.h#L107).@@ -251,10 +287,10 @@ readLMDB ::   (MonadIO m, Mode mode) =>   Database mode ->-  Maybe ReadOnlyTxn ->+  Maybe (ReadOnlyTxn, Cursor) ->   ReadOptions ->   Unfold m Void (ByteString, ByteString)-readLMDB db mtxn ropts = unsafeReadLMDB db mtxn ropts packCStringLen packCStringLen+readLMDB db mtxncurs ropts = unsafeReadLMDB db mtxncurs ropts packCStringLen packCStringLen  -- | Similar to 'readLMDB', except that the keys and values are not automatically converted into -- Haskell @ByteString@s.@@ -267,12 +303,12 @@ unsafeReadLMDB ::   (MonadIO m, Mode mode) =>   Database mode ->-  Maybe ReadOnlyTxn ->+  Maybe (ReadOnlyTxn, Cursor) ->   ReadOptions ->   (CStringLen -> IO k) ->   (CStringLen -> IO v) ->   Unfold m Void (k, v)-unsafeReadLMDB (Database penv dbi) mtxn ropts kmap vmap =+unsafeReadLMDB (Database penv dbi) mtxncurs ropts kmap vmap =   let (firstOp, subsequentOp) = case (readDirection ropts, readStart ropts) of         (Forward, Nothing) -> (mdb_first, mdb_next)         (Forward, Just _) -> (mdb_set_range, mdb_next)@@ -322,10 +358,13 @@           ( \op -> do               (pcurs, pk, pv, ref) <- liftIO $                 mask_ $ do-                  ptxn <- case mtxn of-                    Nothing -> liftIO $ mdb_txn_begin penv nullPtr mdb_rdonly-                    Just (ReadOnlyTxn ptxn') -> return ptxn'-                  pcurs <- liftIO $ mdb_cursor_open ptxn dbi+                  (ptxn, pcurs) <- case mtxncurs of+                    Nothing -> liftIO $ do+                      ptxn <- mdb_txn_begin penv nullPtr mdb_rdonly+                      pcurs <- mdb_cursor_open ptxn dbi+                      return (ptxn, pcurs)+                    Just (ReadOnlyTxn ptxn, Cursor pcurs) ->+                      return (ptxn, pcurs)                   pk <- liftIO malloc                   pv <- liftIO malloc @@ -336,14 +375,9 @@                    ref <- liftIO . newIOFinalizer $ do                     free pv >> free pk-                    -- Note: If no transaction is provided to this function, it could happen that-                    -- mdb_cursor_close() gets called after mdb_txn_abort() -- the former being-                    -- called during garbage collection. This is allowed by LMDB for read-only-                    -- transactions.-                    c_mdb_cursor_close pcurs-                    when (isNothing mtxn) $+                    when (isNothing mtxncurs) $                       -- There is no need to commit this read-only transaction.-                      c_mdb_txn_abort ptxn+                      c_mdb_cursor_close pcurs >> c_mdb_txn_abort ptxn                   return (pcurs, pk, pv, ref)               return (op, pcurs, pk, pv, ref)           )@@ -360,6 +394,23 @@ -- | Disposes of a read-only transaction created with 'beginReadOnlyTxn'. abortReadOnlyTxn :: ReadOnlyTxn -> IO () abortReadOnlyTxn (ReadOnlyTxn ptxn) = c_mdb_txn_abort ptxn++newtype Cursor = Cursor (Ptr MDB_cursor)++-- | Opens a cursor for use with 'readLMDB' or 'unsafeReadLMDB'. It is your responsibility to (a)+-- make sure the cursor only gets used by a single 'readLMDB' or 'unsafeReadLMDB' @Unfold@ at the+-- same time (to be safe, one can open a new cursor for every 'readLMDB' or 'unsafeReadLMDB' call),+-- (b) make sure the provided database is within the environment on which the provided transaction+-- was begun, and (c) dispose of the cursor with 'closeCursor' (logically before 'abortReadOnlyTxn',+-- although the order doesn’t really matter for read-only transactions).+openCursor :: ReadOnlyTxn -> Database mode -> IO Cursor+openCursor (ReadOnlyTxn ptxn) (Database _ dbi) =+  Cursor <$> mdb_cursor_open ptxn dbi++-- | Disposes of a cursor created with 'openCursor'.+closeCursor :: Cursor -> IO ()+closeCursor (Cursor pcurs) =+  c_mdb_cursor_close pcurs  data ReadOptions = ReadOptions   { readDirection :: !ReadDirection,

@@ -79,6 +79,12 @@ foreign import ccall unsafe "lmdb.h mdb_cursor_close"     c_mdb_cursor_close :: Ptr MDB_cursor -> IO () +foreign import ccall unsafe "lmdb.h mdb_dbi_close"+    c_mdb_dbi_close :: Ptr MDB_env -> MDB_dbi_t -> IO ()++foreign import ccall unsafe "lmdb.h mdb_env_close"+    c_mdb_env_close :: Ptr MDB_env -> IO ()+ foreign import ccall unsafe "lmdb.h mdb_get"     c_mdb_get :: Ptr MDB_txn -> MDB_dbi_t -> Ptr MDB_val -> Ptr MDB_val -> IO CInt 

@@ -5,7 +5,7 @@ -- see: https://github.com/sol/hpack  name:           streamly-lmdb-version:        0.4.0+version:        0.5.0 synopsis:       Stream data to or from LMDB databases using the streamly library. description:    Please see the README on GitHub at <https://github.com/shlok/streamly-lmdb#readme> category:       Database, Streaming, Streamly

@@ -12,7 +12,8 @@ import Data.Word (Word8) import Foreign (castPtr, nullPtr, with) import Streamly.External.LMDB-  ( Environment,+  ( Cursor,+    Environment,     Mode,     OverwriteOptions (..),     ReadDirection (..),@@ -23,8 +24,10 @@     abortReadOnlyTxn,     beginReadOnlyTxn,     clearDatabase,+    closeCursor,     defaultReadOptions,     defaultWriteOptions,+    openCursor,     readLMDB,     unsafeReadLMDB,     writeLMDB,@@ -55,8 +58,16 @@     testBetween   ] -withReadOnlyTxn :: (Mode mode) => Environment mode -> (ReadOnlyTxn -> IO r) -> IO r-withReadOnlyTxn env = bracket (beginReadOnlyTxn env) abortReadOnlyTxn+withReadOnlyTxnAndCurs ::+  (Mode mode) =>+  Environment mode ->+  Database mode ->+  ((ReadOnlyTxn, Cursor) -> IO r) ->+  IO r+withReadOnlyTxnAndCurs env db =+  bracket+    (beginReadOnlyTxn env >>= \txn -> openCursor txn db >>= \curs -> return (txn, curs))+    (\(txn, curs) -> closeCursor curs >> abortReadOnlyTxn txn)  -- | Clear the database, write key-value pairs to it in a normal manner, read -- them back using our library, and make sure the result is what we wrote.@@ -72,7 +83,7 @@   (readOpts, expectedResults) <- pick $ readOptionsAndResults keyValuePairsInDb   let unf txn = toList $ unfold (readLMDB db txn readOpts) undefined   results <- run $ unf Nothing-  resultsTxn <- run $ withReadOnlyTxn env (unf . Just)+  resultsTxn <- run $ withReadOnlyTxnAndCurs env db (unf . Just)    return $ results == expectedResults && resultsTxn == expectedResults @@ -92,7 +103,7 @@         toList $           unfold (unsafeReadLMDB db txn readOpts (return . snd) (return . snd)) undefined   lengths <- run $ unf Nothing-  lengthsTxn <- run $ withReadOnlyTxn env (unf . Just)+  lengthsTxn <- run $ withReadOnlyTxnAndCurs env db (unf . Just)    return $ lengths == expectedLengths && lengthsTxn == expectedLengths