rocksdb-haskell-jprupp 2.4.0 → 2.5.0

raw patch · 6 files changed

+120/−16 lines, 6 filesPVP ok

version bump matches the API change (PVP)

API changes (from Hackage documentation)

Files

• CHANGELOG.md +14/−0
• rocksdb-haskell-jprupp.cabal +1/−1
• src/Database/RocksDB/Base.hs +49/−15
• src/Database/RocksDB/C.hs +12/−0
• src/Database/RocksDB/Internal.hs +15/−0
• test/Spec.hs +29/−0

@@ -1,3 +1,17 @@+## 2.5.0++ * Add the `EstimateTableReadersMem`, `CurSizeAllMemTables` and+   `BlockCacheUsage` properties, and `PropertyName` for any other property by+   its name.+ * Add `getPropertyCF` to read a property of a column family.+ * Add `maxTotalWalSize` (`max_total_wal_size`) to `Config`, to bound the+   write-ahead log.+ * Add `createMissingColumnFamilies` (`create_missing_column_families`) to+   `Config`, to add column families to an existing database.+ * Destroy the handle of the default column family when `withDBCF` opens an+   existing database. It was leaked, and RocksDB built with assertions+   aborted when the database was closed.+ ## 2.4.0   * Add `writeBufferSize` (`write_buffer_size`) and `dbWriteBufferSize`

@@ -5,7 +5,7 @@ -- see: https://github.com/sol/hpack  name:           rocksdb-haskell-jprupp-version:        2.4.0+version:        2.5.0 synopsis:       Haskell bindings for RocksDB description:    See README at <https://github.com/jprupp/rocksdb-haskell#readme> category:       Database, FFI

@@ -39,7 +39,7 @@     , releaseSnapshot      -- * Administrative Functions-    , Property (..), getProperty+    , Property (..), getProperty, getPropertyCF     , destroy     , repair     , approximateSize@@ -67,7 +67,19 @@ import           System.Directory  -- | Properties exposed by RocksDB-data Property = NumFilesAtLevel Int | Stats | SSTables+data Property+    = NumFilesAtLevel Int+    | Stats+    | SSTables+    -- | @rocksdb.estimate-table-readers-mem@: memory for index and filter+    -- blocks of open table files, outside the block cache+    | EstimateTableReadersMem+    -- | @rocksdb.cur-size-all-mem-tables@: memory of all memtables+    | CurSizeAllMemTables+    -- | @rocksdb.block-cache-usage@: memory used by the block cache+    | BlockCacheUsage+    -- | any other property, by its full name+    | PropertyName String     deriving (Eq, Show)  data BatchOp = Put !ByteString !ByteString@@ -165,9 +177,15 @@                         cf_names_array                         cf_opts_array                         cf_ptrs_array-                    cfs <- peekArray (length cf_cfgs + 1) cf_ptrs_array+                    (deflt, cfs) <- peekArray (length cf_cfgs + 1) cf_ptrs_array >>= \case+                        d : cs -> return (d, cs)+                        [] -> error "rocksdb_open_column_families returned no handles"+                    -- Every handle must be destroyed before the database is+                    -- closed. The default column family is not kept, so+                    -- release its handle now; the database keeps its own.+                    c_rocksdb_column_family_handle_destroy deflt                     return DB { rocksDB = db_ptr-                              , columnFamilies = tail cfs+                              , columnFamilies = cfs                               , readOpts = read_opts                               , writeOpts = write_opts                               }@@ -197,17 +215,33 @@ -- | Get a DB property. getProperty :: DB -> Property -> IO (Maybe ByteString) getProperty DB{rocksDB = db_ptr} p =-    withCString (prop p) $-    c_rocksdb_property_value db_ptr >=> \case-    val_ptr | val_ptr == nullPtr -> return Nothing-            | otherwise -> do-                  res <- Just <$> BS.packCString val_ptr-                  freeCString val_ptr-                  return res-  where-    prop (NumFilesAtLevel i) = "rocksdb.num-files-at-level" ++ show i-    prop Stats               = "rocksdb.stats"-    prop SSTables            = "rocksdb.sstables"+    withCString (propertyName p) $ \name ->+    c_rocksdb_property_value db_ptr name >>= propertyResult++-- | Get a property of a column family. Many properties, such as the memory+-- of memtables and table readers, are kept per column family.+getPropertyCF :: DB -> ColumnFamily -> Property -> IO (Maybe ByteString)+getPropertyCF DB{rocksDB = db_ptr} cf p =+    withCString (propertyName p) $ \name ->+    c_rocksdb_property_value_cf db_ptr cf name >>= propertyResult++propertyResult :: CString -> IO (Maybe ByteString)+propertyResult val_ptr+    | val_ptr == nullPtr = return Nothing+    | otherwise = do+          res <- BS.packCString val_ptr+          freeCString val_ptr+          return (Just res)++propertyName :: Property -> String+propertyName = \case+    NumFilesAtLevel i       -> "rocksdb.num-files-at-level" ++ show i+    Stats                   -> "rocksdb.stats"+    SSTables                -> "rocksdb.sstables"+    EstimateTableReadersMem -> "rocksdb.estimate-table-readers-mem"+    CurSizeAllMemTables     -> "rocksdb.cur-size-all-mem-tables"+    BlockCacheUsage         -> "rocksdb.block-cache-usage"+    PropertyName n          -> n  -- | Destroy the given RocksDB database. destroy :: FilePath -> Options -> IO ()

@@ -44,6 +44,7 @@     , c_rocksdb_create_snapshot     , c_rocksdb_release_snapshot     , c_rocksdb_property_value+    , c_rocksdb_property_value_cf     , c_rocksdb_approximate_sizes     , c_rocksdb_destroy_db     , c_rocksdb_repair_db@@ -74,6 +75,8 @@     , c_rocksdb_options_set_max_open_files     , c_rocksdb_options_set_write_buffer_size     , c_rocksdb_options_set_db_write_buffer_size+    , c_rocksdb_options_set_max_total_wal_size+    , c_rocksdb_options_set_create_missing_column_families     , c_rocksdb_options_set_prefix_extractor     , c_rocksdb_options_set_block_based_table_factory     , c_rocksdb_block_based_options_create@@ -249,6 +252,9 @@ foreign import ccall safe "rocksdb/c.h rocksdb_property_value"   c_rocksdb_property_value :: RocksDB -> CString -> IO CString +foreign import ccall safe "rocksdb/c.h rocksdb_property_value_cf"+  c_rocksdb_property_value_cf :: RocksDB -> ColumnFamily -> CString -> IO CString+ foreign import ccall safe "rocksdb/c.h rocksdb_approximate_sizes"   c_rocksdb_approximate_sizes :: RocksDB                               -> CInt@@ -378,6 +384,12 @@  foreign import ccall safe "rocksdb/c.h rocksdb_options_set_db_write_buffer_size"   c_rocksdb_options_set_db_write_buffer_size :: Options -> CSize -> IO ()++foreign import ccall safe "rocksdb/c.h rocksdb_options_set_max_total_wal_size"+  c_rocksdb_options_set_max_total_wal_size :: Options -> Word64 -> IO ()++foreign import ccall safe "rocksdb/c.h rocksdb_options_set_create_missing_column_families"+  c_rocksdb_options_set_create_missing_column_families :: Options -> CBool -> IO ()  foreign import ccall safe "rocksdb/c.h rocksdb_options_set_prefix_extractor"   c_rocksdb_options_set_prefix_extractor :: Options

@@ -48,6 +48,9 @@ data Config = Config { createIfMissing :: !Bool                      , errorIfExists   :: !Bool                      , paranoidChecks  :: !Bool+                     , createMissingColumnFamilies :: !Bool+                       -- ^ create the column families passed to 'withDBCF'+                       -- that an existing database does not have yet                      , maxFiles        :: !(Maybe Int)                      , prefixLength    :: !(Maybe Int)                      , bloomFilter     :: !Bool@@ -58,17 +61,25 @@                        -- ^ bytes for all memtables across all column                        -- families (RocksDB default unlimited); database-wide,                        -- ignored in column family configs+                     , maxTotalWalSize :: !(Maybe Int)+                       -- ^ bytes of write-ahead log after which the column+                       -- families holding the oldest log are flushed, so it+                       -- can be deleted (RocksDB default: four times the+                       -- memtables); database-wide, ignored in column family+                       -- configs                      } deriving (Eq, Show)  instance Default Config where     def = Config { createIfMissing  = False                  , errorIfExists    = False                  , paranoidChecks   = False+                 , createMissingColumnFamilies = False                  , maxFiles         = Nothing                  , prefixLength     = Nothing                  , bloomFilter      = False                  , writeBufferSize  = Nothing                  , dbWriteBufferSize = Nothing+                 , maxTotalWalSize  = Nothing                  }  withOptions :: Config -> (Options -> IO a) -> IO a@@ -87,12 +98,16 @@         c_rocksdb_options_set_write_buffer_size opts . intToCSize     forM_ dbWriteBufferSize $         c_rocksdb_options_set_db_write_buffer_size opts . intToCSize+    forM_ maxTotalWalSize $+        c_rocksdb_options_set_max_total_wal_size opts . fromIntegral     c_rocksdb_options_set_create_if_missing         opts (boolToCBool createIfMissing)     c_rocksdb_options_set_error_if_exists         opts (boolToCBool errorIfExists)     c_rocksdb_options_set_paranoid_checks         opts (boolToCBool paranoidChecks)+    c_rocksdb_options_set_create_missing_column_families+        opts (boolToCBool createMissingColumnFamilies)     f opts   where     with_opts =

@@ -250,3 +250,32 @@             getCF db two (key 20000) `shouldReturn` Just value             files <- getProperty db SSTables             files `shouldSatisfy` maybe False (not . C.null)+    describe "Properties" $ do+      it "reports memory use, per database and per column family" $ \db -> do+        let one = head (columnFamilies db)+            isNumber = maybe False (\v -> not (C.null v) && C.all (`elem` ['0' .. '9']) v)+        forM_ [1 .. 1000 :: Int] $ \i -> putCF db one (C.pack (show i)) "value"+        mem <- getPropertyCF db one CurSizeAllMemTables+        mem `shouldSatisfy` isNumber+        mem `shouldSatisfy` maybe False ((> 0) . read . C.unpack)+        getProperty db EstimateTableReadersMem >>= (`shouldSatisfy` isNumber)+        getProperty db BlockCacheUsage >>= (`shouldSatisfy` isNumber)+        getProperty db (PropertyName "rocksdb.num-immutable-mem-table")+          >>= (`shouldSatisfy` isNumber)+        getProperty db (PropertyName "rocksdb.no-such-property")+          `shouldReturn` Nothing+      it "opens with a bound on the write-ahead log" $ \_ ->+        withSystemTempDirectory "rocksdb-tests-wal" $ \path ->+          withDBCF path conf {maxTotalWalSize = Just (1024 * 1024)} [("one", conf)] $ \db -> do+            put db "key" "value"+            get db "key" `shouldReturn` Just "value"+    describe "Column families" $ do+      it "adds a column family to an existing database" $ \_ ->+        withSystemTempDirectory "rocksdb-tests-newcf" $ \path -> do+          let open = conf {errorIfExists = False}+          withDBCF path open [("one", open)] $ \db -> put db "key" "value"+          withDBCF path open {createMissingColumnFamilies = True} [("one", open), ("two", open)] $ \db -> do+            get db "key" `shouldReturn` Just "value"+            let two = columnFamilies db !! 1+            putCF db two "new" "cf"+            getCF db two "new" `shouldReturn` Just "cf"