diff --git a/CHANGELOG.md b/CHANGELOG.md --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,17 @@ +## 2.5.0 + + * Add the `EstimateTableReadersMem`, `CurSizeAllMemTables` and + `BlockCacheUsage` properties, and `PropertyName` for any other property by + its name. + * Add `getPropertyCF` to read a property of a column family. + * Add `maxTotalWalSize` (`max_total_wal_size`) to `Config`, to bound the + write-ahead log. + * Add `createMissingColumnFamilies` (`create_missing_column_families`) to + `Config`, to add column families to an existing database. + * Destroy the handle of the default column family when `withDBCF` opens an + existing database. It was leaked, and RocksDB built with assertions + aborted when the database was closed. + ## 2.4.0 * Add `writeBufferSize` (`write_buffer_size`) and `dbWriteBufferSize` diff --git a/rocksdb-haskell-jprupp.cabal b/rocksdb-haskell-jprupp.cabal --- a/rocksdb-haskell-jprupp.cabal +++ b/rocksdb-haskell-jprupp.cabal @@ -5,7 +5,7 @@ -- see: https://github.com/sol/hpack name: rocksdb-haskell-jprupp -version: 2.4.0 +version: 2.5.0 synopsis: Haskell bindings for RocksDB description: See README at category: Database, FFI diff --git a/src/Database/RocksDB/Base.hs b/src/Database/RocksDB/Base.hs --- a/src/Database/RocksDB/Base.hs +++ b/src/Database/RocksDB/Base.hs @@ -39,7 +39,7 @@ , releaseSnapshot -- * Administrative Functions - , Property (..), getProperty + , Property (..), getProperty, getPropertyCF , destroy , repair , approximateSize @@ -67,7 +67,19 @@ import System.Directory -- | Properties exposed by RocksDB -data Property = NumFilesAtLevel Int | Stats | SSTables +data Property + = NumFilesAtLevel Int + | Stats + | SSTables + -- | @rocksdb.estimate-table-readers-mem@: memory for index and filter + -- blocks of open table files, outside the block cache + | EstimateTableReadersMem + -- | @rocksdb.cur-size-all-mem-tables@: memory of all memtables + | CurSizeAllMemTables + -- | @rocksdb.block-cache-usage@: memory used by the block cache + | BlockCacheUsage + -- | any other property, by its full name + | PropertyName String deriving (Eq, Show) data BatchOp = Put !ByteString !ByteString @@ -165,9 +177,15 @@ cf_names_array cf_opts_array cf_ptrs_array - cfs <- peekArray (length cf_cfgs + 1) cf_ptrs_array + (deflt, cfs) <- peekArray (length cf_cfgs + 1) cf_ptrs_array >>= \case + d : cs -> return (d, cs) + [] -> error "rocksdb_open_column_families returned no handles" + -- Every handle must be destroyed before the database is + -- closed. The default column family is not kept, so + -- release its handle now; the database keeps its own. + c_rocksdb_column_family_handle_destroy deflt return DB { rocksDB = db_ptr - , columnFamilies = tail cfs + , columnFamilies = cfs , readOpts = read_opts , writeOpts = write_opts } @@ -197,17 +215,33 @@ -- | Get a DB property. getProperty :: DB -> Property -> IO (Maybe ByteString) getProperty DB{rocksDB = db_ptr} p = - withCString (prop p) $ - c_rocksdb_property_value db_ptr >=> \case - val_ptr | val_ptr == nullPtr -> return Nothing - | otherwise -> do - res <- Just <$> BS.packCString val_ptr - freeCString val_ptr - return res - where - prop (NumFilesAtLevel i) = "rocksdb.num-files-at-level" ++ show i - prop Stats = "rocksdb.stats" - prop SSTables = "rocksdb.sstables" + withCString (propertyName p) $ \name -> + c_rocksdb_property_value db_ptr name >>= propertyResult + +-- | Get a property of a column family. Many properties, such as the memory +-- of memtables and table readers, are kept per column family. +getPropertyCF :: DB -> ColumnFamily -> Property -> IO (Maybe ByteString) +getPropertyCF DB{rocksDB = db_ptr} cf p = + withCString (propertyName p) $ \name -> + c_rocksdb_property_value_cf db_ptr cf name >>= propertyResult + +propertyResult :: CString -> IO (Maybe ByteString) +propertyResult val_ptr + | val_ptr == nullPtr = return Nothing + | otherwise = do + res <- BS.packCString val_ptr + freeCString val_ptr + return (Just res) + +propertyName :: Property -> String +propertyName = \case + NumFilesAtLevel i -> "rocksdb.num-files-at-level" ++ show i + Stats -> "rocksdb.stats" + SSTables -> "rocksdb.sstables" + EstimateTableReadersMem -> "rocksdb.estimate-table-readers-mem" + CurSizeAllMemTables -> "rocksdb.cur-size-all-mem-tables" + BlockCacheUsage -> "rocksdb.block-cache-usage" + PropertyName n -> n -- | Destroy the given RocksDB database. destroy :: FilePath -> Options -> IO () diff --git a/src/Database/RocksDB/C.hs b/src/Database/RocksDB/C.hs --- a/src/Database/RocksDB/C.hs +++ b/src/Database/RocksDB/C.hs @@ -44,6 +44,7 @@ , c_rocksdb_create_snapshot , c_rocksdb_release_snapshot , c_rocksdb_property_value + , c_rocksdb_property_value_cf , c_rocksdb_approximate_sizes , c_rocksdb_destroy_db , c_rocksdb_repair_db @@ -74,6 +75,8 @@ , c_rocksdb_options_set_max_open_files , c_rocksdb_options_set_write_buffer_size , c_rocksdb_options_set_db_write_buffer_size + , c_rocksdb_options_set_max_total_wal_size + , c_rocksdb_options_set_create_missing_column_families , c_rocksdb_options_set_prefix_extractor , c_rocksdb_options_set_block_based_table_factory , c_rocksdb_block_based_options_create @@ -249,6 +252,9 @@ foreign import ccall safe "rocksdb/c.h rocksdb_property_value" c_rocksdb_property_value :: RocksDB -> CString -> IO CString +foreign import ccall safe "rocksdb/c.h rocksdb_property_value_cf" + c_rocksdb_property_value_cf :: RocksDB -> ColumnFamily -> CString -> IO CString + foreign import ccall safe "rocksdb/c.h rocksdb_approximate_sizes" c_rocksdb_approximate_sizes :: RocksDB -> CInt @@ -378,6 +384,12 @@ foreign import ccall safe "rocksdb/c.h rocksdb_options_set_db_write_buffer_size" c_rocksdb_options_set_db_write_buffer_size :: Options -> CSize -> IO () + +foreign import ccall safe "rocksdb/c.h rocksdb_options_set_max_total_wal_size" + c_rocksdb_options_set_max_total_wal_size :: Options -> Word64 -> IO () + +foreign import ccall safe "rocksdb/c.h rocksdb_options_set_create_missing_column_families" + c_rocksdb_options_set_create_missing_column_families :: Options -> CBool -> IO () foreign import ccall safe "rocksdb/c.h rocksdb_options_set_prefix_extractor" c_rocksdb_options_set_prefix_extractor :: Options diff --git a/src/Database/RocksDB/Internal.hs b/src/Database/RocksDB/Internal.hs --- a/src/Database/RocksDB/Internal.hs +++ b/src/Database/RocksDB/Internal.hs @@ -48,6 +48,9 @@ data Config = Config { createIfMissing :: !Bool , errorIfExists :: !Bool , paranoidChecks :: !Bool + , createMissingColumnFamilies :: !Bool + -- ^ create the column families passed to 'withDBCF' + -- that an existing database does not have yet , maxFiles :: !(Maybe Int) , prefixLength :: !(Maybe Int) , bloomFilter :: !Bool @@ -58,17 +61,25 @@ -- ^ bytes for all memtables across all column -- families (RocksDB default unlimited); database-wide, -- ignored in column family configs + , maxTotalWalSize :: !(Maybe Int) + -- ^ bytes of write-ahead log after which the column + -- families holding the oldest log are flushed, so it + -- can be deleted (RocksDB default: four times the + -- memtables); database-wide, ignored in column family + -- configs } deriving (Eq, Show) instance Default Config where def = Config { createIfMissing = False , errorIfExists = False , paranoidChecks = False + , createMissingColumnFamilies = False , maxFiles = Nothing , prefixLength = Nothing , bloomFilter = False , writeBufferSize = Nothing , dbWriteBufferSize = Nothing + , maxTotalWalSize = Nothing } withOptions :: Config -> (Options -> IO a) -> IO a @@ -87,12 +98,16 @@ c_rocksdb_options_set_write_buffer_size opts . intToCSize forM_ dbWriteBufferSize $ c_rocksdb_options_set_db_write_buffer_size opts . intToCSize + forM_ maxTotalWalSize $ + c_rocksdb_options_set_max_total_wal_size opts . fromIntegral c_rocksdb_options_set_create_if_missing opts (boolToCBool createIfMissing) c_rocksdb_options_set_error_if_exists opts (boolToCBool errorIfExists) c_rocksdb_options_set_paranoid_checks opts (boolToCBool paranoidChecks) + c_rocksdb_options_set_create_missing_column_families + opts (boolToCBool createMissingColumnFamilies) f opts where with_opts = diff --git a/test/Spec.hs b/test/Spec.hs --- a/test/Spec.hs +++ b/test/Spec.hs @@ -250,3 +250,32 @@ getCF db two (key 20000) `shouldReturn` Just value files <- getProperty db SSTables files `shouldSatisfy` maybe False (not . C.null) + describe "Properties" $ do + it "reports memory use, per database and per column family" $ \db -> do + let one = head (columnFamilies db) + isNumber = maybe False (\v -> not (C.null v) && C.all (`elem` ['0' .. '9']) v) + forM_ [1 .. 1000 :: Int] $ \i -> putCF db one (C.pack (show i)) "value" + mem <- getPropertyCF db one CurSizeAllMemTables + mem `shouldSatisfy` isNumber + mem `shouldSatisfy` maybe False ((> 0) . read . C.unpack) + getProperty db EstimateTableReadersMem >>= (`shouldSatisfy` isNumber) + getProperty db BlockCacheUsage >>= (`shouldSatisfy` isNumber) + getProperty db (PropertyName "rocksdb.num-immutable-mem-table") + >>= (`shouldSatisfy` isNumber) + getProperty db (PropertyName "rocksdb.no-such-property") + `shouldReturn` Nothing + it "opens with a bound on the write-ahead log" $ \_ -> + withSystemTempDirectory "rocksdb-tests-wal" $ \path -> + withDBCF path conf {maxTotalWalSize = Just (1024 * 1024)} [("one", conf)] $ \db -> do + put db "key" "value" + get db "key" `shouldReturn` Just "value" + describe "Column families" $ do + it "adds a column family to an existing database" $ \_ -> + withSystemTempDirectory "rocksdb-tests-newcf" $ \path -> do + let open = conf {errorIfExists = False} + withDBCF path open [("one", open)] $ \db -> put db "key" "value" + withDBCF path open {createMissingColumnFamilies = True} [("one", open), ("two", open)] $ \db -> do + get db "key" `shouldReturn` Just "value" + let two = columnFamilies db !! 1 + putCF db two "new" "cf" + getCF db two "new" `shouldReturn` Just "cf"