diff --git a/ChangeLog b/ChangeLog --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,6 @@ +Version 0.0.4.0 2017-11-27 by luispedro + * Load hash table into memory + Version 0.0.3.2 2017-11-09 by luispedro * Fix Haskell distribution diff --git a/diskhash.cabal b/diskhash.cabal --- a/diskhash.cabal +++ b/diskhash.cabal @@ -1,5 +1,5 @@ name: diskhash -version: 0.0.3.2 +version: 0.0.4.0 synopsis: Disk-based hash table description: Disk-based hash table category: Data diff --git a/haskell/Data/DiskHash.hs b/haskell/Data/DiskHash.hs --- a/haskell/Data/DiskHash.hs +++ b/haskell/Data/DiskHash.hs @@ -19,6 +19,7 @@ ( DiskHashRO , DiskHashRW , htOpenRO + , htLoadRO , htOpenRW , withDiskHashRW , htLookupRO @@ -33,6 +34,7 @@ import qualified Data.ByteString as B import qualified Data.ByteString.Char8 as B8 import Control.Exception (throwIO) +import Control.Monad (when) import System.IO.Unsafe (unsafeDupablePerformIO) import Foreign.Ptr (Ptr, FunPtr, castPtr, nullPtr) import Foreign.ForeignPtr (ForeignPtr, newForeignPtr, withForeignPtr, finalizeForeignPtr) @@ -54,6 +56,7 @@ foreign import ccall "dht_reserve" c_dht_reserve :: Ptr () -> CInt -> Ptr CString -> IO () foreign import ccall "dht_insert" c_dht_insert :: Ptr () -> CString -> Ptr () -> Ptr CString -> IO CInt foreign import ccall "dht_size" c_dht_size :: Ptr () -> IO CSize +foreign import ccall "dht_load_to_memory" c_dht_load_to_memory :: Ptr () -> Ptr CString -> IO CInt foreign import ccall "&dht_free" c_dht_free_p :: FunPtr (Ptr () -> IO ()) -- | Internal function to handle error message interface @@ -76,7 +79,7 @@ -> Int -- ^ maximum key size -> IO (DiskHashRW a) -htOpenRW fpath maxk = DiskHashRW <$> open' (undefined :: a) fpath maxk 66 +htOpenRW fpath maxk = DiskHashRW <$> open' (undefined :: a) fpath maxk 66 False -- | open a hash table in read-only mode -- @@ -88,10 +91,24 @@ -> Int -- ^ maximum key size -> IO (DiskHashRO a) -htOpenRO fpath maxk = DiskHashRO <$> open' (undefined :: a) fpath maxk 0 +htOpenRO fpath maxk = DiskHashRO <$> open' (undefined :: a) fpath maxk 0 False -open' :: forall a. (Storable a) => a -> FilePath -> Int -> CInt -> IO HashTable_t -open' unused fpath maxk flags = B.useAsCString (B8.pack fpath) $ \fpath' -> +-- | open a hash table in read-only mode and load it into memory +-- +-- The 'maxk' argument can be 0, in which case the value of the maximum key +-- will be taken from the disk file. If not zero, then it is checked against +-- the value on disk and an exception is raised if there is a mismatch. +-- +-- @since 0.0.4.0 +htLoadRO :: forall a. (Storable a) => FilePath + -- ^ file path + -> Int + -- ^ maximum key size + -> IO (DiskHashRO a) +htLoadRO fpath maxk = DiskHashRO <$> open' (undefined :: a) fpath maxk 0 True + +open' :: forall a. (Storable a) => a -> FilePath -> Int -> CInt -> Bool -> IO HashTable_t +open' unused fpath maxk flags load = B.useAsCString (B8.pack fpath) $ \fpath' -> alloca $ \err -> do poke err nullPtr ht <- c_dht_open2 fpath' (fromIntegral maxk) (fromIntegral $ sizeOf unused) flags err @@ -99,7 +116,12 @@ then do errmsg <- getError err throwIO $ userError ("Could not open hash table: " ++ show errmsg) - else + else do + when load $ do + e <- c_dht_load_to_memory ht err + when (e == 2) $ do + errmsg <- getError err + throwIO $ userError ("Could not load hash table into memory: " ++ show errmsg) newForeignPtr c_dht_free_p ht -- | Open a hash table in read-write mode and pass it to an action diff --git a/haskell/Data/DiskHash/Tests.hs b/haskell/Data/DiskHash/Tests.hs --- a/haskell/Data/DiskHash/Tests.hs +++ b/haskell/Data/DiskHash/Tests.hs @@ -59,6 +59,17 @@ assertEqual "Lookup" (Just (9 :: Int64)) (htLookupRO "key" ht) removeFileIfExists outname +case_open_close_load = do + withDiskHashRW outname 15 $ \ht -> do + s <- htSizeRW ht + assertEqual "new table has size 0" s 0 + inserted <- htInsert "key" (9 :: Int64) ht + assertBool "inserted should have return True" inserted + ht <- htLoadRO outname 15 + assertEqual "read-only table after reopen (load)" (htSizeRO ht) 1 + assertEqual "Lookup" (Just (9 :: Int64)) (htLookupRO "key" ht) + removeFileIfExists outname + -- prop_insert_find :: [(String, Int64)] -> IO Bool prop_insert_find args = ioProperty $ do let args' = normArgs args diff --git a/src/diskhash.c b/src/diskhash.c --- a/src/diskhash.c +++ b/src/diskhash.c @@ -19,6 +19,7 @@ enum { HT_FLAG_CAN_WRITE = 1, HT_FLAG_HASH_2 = 2, + HT_FLAG_IS_LOADED = 4, }; typedef struct HashTableHeader { @@ -240,8 +241,39 @@ return rp; } -void dht_free(HashTable* ht) { +int dht_load_to_memory(HashTable* ht, char** err) { + if (ht->flags_ & HT_FLAG_CAN_WRITE) { + if (err) *err = "Cannot call dht_load_to_memory on a read/write Diskhash"; + return 1; + } + if (ht->flags_ & HT_FLAG_IS_LOADED) { + if (err) *err = "dht_load_to_memory had already been called."; + return 1; + } munmap(ht->data_, ht->datasize_); + ht->data_ = malloc(ht->datasize_); + if (ht->data_) { + size_t n = read(ht->fd_, ht->data_, ht->datasize_); + if (n == ht->datasize_) return 0; + else if (err) *err = "dht_load_to_memory: could not read data from file"; + } else { + if (err) *err = "dht_load_to_memory: could not allocate memory."; + } + free(ht->data_); + fsync(ht->fd_); + close(ht->fd_); + free((char*)ht->fname_); + free(ht); + return 2; + +} + +void dht_free(HashTable* ht) { + if (ht->flags_ & HT_FLAG_IS_LOADED) { + free(ht->data_); + } else { + munmap(ht->data_, ht->datasize_); + } fsync(ht->fd_); close(ht->fd_); free((char*)ht->fname_); diff --git a/src/diskhash.h b/src/diskhash.h --- a/src/diskhash.h +++ b/src/diskhash.h @@ -73,6 +73,18 @@ */ HashTable* dht_open(const char* fpath, HashTableOpts opts, int flags, char**); +/** Load table into memory + * + * Return: + * 0 : success + * + * 1 : impossible operation: nothing has been done. Attempting to load a + * previously loaded table or a read/write table is impossible. + * + * 2 : error: the HashTable has been freed and must not be used. + */ +int dht_load_to_memory(HashTable*, char**); + /** Lookup a value by key * * If the hash table was opened in read-write mode, then the memory returned