hmatrix-mmap 0.0.2 → 0.0.3

raw patch · 2 files changed

+103/−2 lines, 2 filesPVP ok

version bump matches the API change (PVP)

API changes (from Hackage documentation)

Files

• Data/Packed/Vector/MMap.hs +101/−0
• hmatrix-mmap.cabal +2/−2

@@ -28,6 +28,7 @@ module Data.Packed.Vector.MMap (   -- * Memory-mapping 'Vector' from disk   unsafeMMapVector,+  unsafeLazyMMapVectors,    -- * Writing 'Vector' to disk @@ -37,8 +38,11 @@   writeVector ) where +import Control.Monad (when)+ import System.IO import System.IO.MMap+import System.IO.Unsafe  import Foreign.ForeignPtr import Foreign.Ptr@@ -65,6 +69,103 @@           Nothing -> Nothing           Just (start, length) -> Just (start, length * sizeOf (undefined :: a))      return $ I.unsafeFromForeignPtr foreignPtr offset (size `div` sizeOf (undefined :: a))++-- | Map a file into memory as a lazy-list of equal-sized 'Vector',+-- even if they can't all fit in the address space at the same time.+--+-- > (numVectors,vectors) <- unsafeLazyMMapVectors filename Nothing vectorSize+--+-- Commonly, a data file will contain multiple vectors of equal length+-- (matrix). This function is convenient for those uses, but it plays+-- a more important role: supporting data-sets that cannot fit in the+-- address space of the current machine.+--+-- On 32-bit machines the address space is only 4GB, and it is+-- actually pretty easy to find data-sets that are too large to be+-- represented, even in virtual memory.+--+-- This function loads the data in chunks, and as long as you drop+-- your reference to the vectors as you consume the data, the old+-- chunks will be unmapped before mapping the next chunk.+--+-- The number of vectors in the list is returned because it's often+-- needed, yet calculating it using 'length' would demand the whole+-- list.+unsafeLazyMMapVectors :: forall a. Storable a => FilePath -- ^ Path of the file to map+                      -> Maybe (Int64, Int64)+                      -- ^ 'Nothing' to map entire file into memory,+                      -- otherwise @'Just' (fileOffset, totalElementCount)@+                      -> Int -- ^ The number of elements in each 'Vector'+                      -> IO (Int64,[I.Vector a]) -- ^ Return @(numberOfVectors,vectors)@+unsafeLazyMMapVectors path range vsize = do+  when (vecSize > maxChunkSize) vecTooBigError+  filesize <- withFile path ReadMode hFileSize+  let filesize' :: Int64+      filesize' = fI filesize+  imgs <- unsafeLazyMMapVectors' filesize' path range vsize+  return (nimages range filesize', imgs)+      where+        nimages :: Maybe (Int64, Int64) -> Int64 -> Int64+        nimages Nothing fsz = fsz `div` imageSize+        nimages (Just (_,sz)) _ = sz `div` imageSize+        imageSize = fI vsize * eltSize+        eltSize = fI (sizeOf (undefined :: a))+        vecSize = fI vsize * eltSize+        vecTooBigError = fail "The requested vector size can't be mapped into memory"++unsafeLazyMMapVectors' :: forall a. Storable a => Int64+                       -> FilePath+                       -> Maybe (Int64, Int64)+                       -> Int+                       -> IO [I.Vector a]+unsafeLazyMMapVectors' fileSize+                       fileName+                       fileRange+                       numEltsPerVec+                           | mapSize < maxChunkSize = mmapAll+                           | otherwise = mmapChunks 0+    where+      mapSize, eltSize, vecSize, chunkSize, baseOffset :: Int64+      eltSize = fI $ sizeOf (undefined :: a)+      (baseOffset,mapSize) = case fileRange of+                               Just (off,nelts) -> (off,nelts*eltSize)+                               _ -> (0,fileSize)+      vecSize = fI numEltsPerVec * eltSize+      chunkSize = (maxChunkSize `div` vecSize) * vecSize++      fileRange' = do+        (offset, nelts) <- fileRange+        return (offset, fI nelts)++      splitVecs :: I.Vector a -> [I.Vector a]+      splitVecs bigVec = let nvecs = I.dim bigVec `div` numEltsPerVec+                         in I.takesV (replicate nvecs numEltsPerVec) bigVec++      mmapAll :: IO [I.Vector a]+      mmapAll = do+        allVecs <- unsafeMMapVector fileName fileRange'+        return $ splitVecs allVecs++      mmapChunks :: Int64 -> IO [I.Vector a]+      mmapChunks offs | remaining <= 0 = return []+                      | otherwise = do+        chunk <- unsafeMMapVector fileName mmapRange+        rest <- unsafeInterleaveIO $ mmapChunks (offs+chunkSize')+        return $ splitVecs chunk ++ rest+          where+            mmapRange = Just (baseOffset+offs,fI (chunkSize' `div` eltSize))+            remaining = mapSize-offs+            chunkSize' = min chunkSize remaining+++-- Maximum size for chunks+maxChunkSize :: Int64+maxChunkSize = fI (maxBound `div` 4 :: Int)++-- Handy alias for 'fromIntegral'+fI :: (Integral a, Num b) => a -> b+fI = fromIntegral+  --------------------------- -- Writing 'Vector' to disk

@@ -1,12 +1,12 @@ Name:           hmatrix-mmap-Version:        0.0.2+Version:        0.0.3 License:        BSD3 License-File:   LICENSE Author:         Daniel Peebles <pumpkingod@gmail.com>, Alan Falloon <alan.falloon@gmail.com> Maintainer:     Alan Falloon <alan.falloon@gmail.com> Copyright:      (c) Daniel Peebles 2010, (c) Alan Falloon 2011 Homepage:       http://github.com/alanfalloon/hmatrix-mmap-Category:       Data, Data Structures+Category:       System, Math Synopsis:       Memory map Vector from disk into memory efficiently Description:         Memory map Data.Packed.Vector.Vector from disk into memory efficiently.