diff --git a/ChangeLog.md b/ChangeLog.md --- a/ChangeLog.md +++ b/ChangeLog.md @@ -1,3 +1,8 @@ -# Changelog for streamly-lmdb +## 0.0.1.1 -## Unreleased changes +* Fixed `install-includes` and `include-dirs` in the Cabal file. +* Added safety check for bound threads in `writeLMDB`. + +## 0.0.1 + +* Initial release. diff --git a/README.md b/README.md --- a/README.md +++ b/README.md @@ -1,5 +1,8 @@ # streamly-lmdb +[![Hackage](https://img.shields.io/hackage/v/streamly-lmdb.svg?style=flat)](https://hackage.haskell.org/package/streamly-lmdb) +[![Build Status](https://travis-ci.org/shlok/streamly-lmdb.svg?branch=master)](https://travis-ci.org/shlok/streamly-lmdb) + Stream data to or from LMDB databases using the Haskell [streamly](https://hackage.haskell.org/package/streamly) library. ## Requirements @@ -52,9 +55,9 @@ ## Benchmarks -See `bench/README.md`. Summary (with rough figures from our machine[1](#fn1)): +See `bench/README.md`. Summary (with rough figures from our machine†): * **Reading.** For reading a fully cached LMDB database, this library (when `unsafeReadLMDB` is used instead of `readLMDB`) has a 10 ns/pair overhead compared to plain Haskell `IO` code, which has another 10 ns/pair overhead compared to C. (The first two being similar fulfills the promise of [streamly](https://hackage.haskell.org/package/streamly) and stream fusion.) We deduce that if your total workload per pair takes longer than 20 ns, your bottleneck will not be your usage of this library as opposed to C. * **Writing**. Writing with plain Haskell `IO` code and with this library is, respectively, 10% and 20% slower than writing with C. We have not dug further into these differences because this write performance is currently good enough for our purposes. -1 [Linode](https://linode.com); Debian 10, Dedicated 32GB: 16 CPU, 640GB Storage, 32GB RAM. [↩](#a1) +† [Linode](https://linode.com); Debian 10, Dedicated 32GB: 16 CPU, 640GB Storage, 32GB RAM. diff --git a/src/Streamly/External/LMDB.hs b/src/Streamly/External/LMDB.hs --- a/src/Streamly/External/LMDB.hs +++ b/src/Streamly/External/LMDB.hs @@ -34,7 +34,7 @@ defaultWriteOptions, writeLMDB) where -import Control.Concurrent (isCurrentThreadBound) +import Control.Concurrent (isCurrentThreadBound, myThreadId) import Control.Concurrent.Async (asyncBound, wait) import Control.Exception (Exception, catch, tryJust, mask_, throw) import Control.Monad (guard, when) @@ -226,7 +226,17 @@ writeLMDB (Database penv dbi) options = let txnSize = max 1 (writeTransactionSize options) flags = combineOptions $ [mdb_nooverwrite | noOverwrite options] ++ [mdb_append | writeAppend options] - in Fold (\(currChunkSz, mtxn) (k, v) -> do + in Fold (\(threadId, iter, currChunkSz, mtxn) (k, v) -> do + -- In the first few iterations, ascertain that we are still on the same (bound) thread. + iter' <- liftIO $ + if iter < 3 then do + threadId' <- myThreadId + when (threadId' /= threadId) $ + (throw $ ExceptionString "Error: writeLMDB veered off the original bound thread") + return $ iter + 1 + else + return iter + currChunkSz' <- liftIO $ if currChunkSz >= txnSize then do let (_, ref) = fromJust mtxn @@ -248,15 +258,19 @@ catch (mdb_put_ ptxn dbi kp (fromIntegral kl) vp (fromIntegral vl) flags) (\(e :: LMDB_Error) -> runIORefFinalizer ref >> throw e) - return (currChunkSz' + 1, Just (ptxn, ref))) + return (threadId, iter', currChunkSz' + 1, Just (ptxn, ref))) (do isBound <- liftIO isCurrentThreadBound + threadId <- liftIO $ myThreadId if isBound then - return (0, Nothing) + return (threadId, 0 :: Int, 0, Nothing) else throw $ ExceptionString "Error: writeLMDB should be executed on a bound thread") -- This final part is incompatible with scans. - (\(_, mtxn) -> liftIO $ + (\(threadId, _, _, mtxn) -> liftIO $ do + threadId' <- myThreadId + when (threadId' /= threadId) $ + (throw $ ExceptionString "Error: writeLMDB veered off the original bound thread at the end") case mtxn of Nothing -> return () Just (_, ref) -> runIORefFinalizer ref) diff --git a/src/Streamly/External/LMDB/Internal/streamly_lmdb_foreign.h b/src/Streamly/External/LMDB/Internal/streamly_lmdb_foreign.h new file mode 100644 --- /dev/null +++ b/src/Streamly/External/LMDB/Internal/streamly_lmdb_foreign.h @@ -0,0 +1,3 @@ +#include + +int mdb_put_(MDB_txn *txn, MDB_dbi dbi, char *key, size_t key_size, char *val, size_t val_size, unsigned int flags); diff --git a/streamly-lmdb.cabal b/streamly-lmdb.cabal --- a/streamly-lmdb.cabal +++ b/streamly-lmdb.cabal @@ -4,10 +4,10 @@ -- -- see: https://github.com/sol/hpack -- --- hash: 842ffd0abdf7b09639ffae5af90237140b800743e7e8c53744189c32edb85f39 +-- hash: d4449c158999353b1715fa4877104b424051746bec5dccfc6adaf535b84e0fba name: streamly-lmdb -version: 0.0.1 +version: 0.0.1.1 synopsis: Stream data to or from LMDB databases using the streamly library. description: Please see the README on GitHub at category: Database, Streaming, Streamly @@ -37,6 +37,10 @@ hs-source-dirs: src ghc-options: -Wall + include-dirs: + src/Streamly/External/LMDB/Internal + install-includes: + src/Streamly/External/LMDB/Internal/streamly_lmdb_foreign.h c-sources: src/Streamly/External/LMDB/Internal/streamly_lmdb_foreign.c extra-libraries: @@ -66,7 +70,7 @@ , bytestring >=0.10.10.0 && <0.11 , directory >=1.3.6.0 && <1.4 , streamly >=0.7.2 && <0.8 - , streamly-lmdb ==0.0.1 + , streamly-lmdb ==0.0.1.1 , tasty >=1.2.3 && <1.3 , tasty-quickcheck >=0.10.1.1 && <0.11 , temporary >=1.3 && <1.4