mmap (empty) → 0.1
raw patch · 7 files changed
+591/−0 lines, 7 filesdep +basedep +bytestringsetup-changed
Dependencies added: base, bytestring
Files
- LICENSE +27/−0
- Setup.hs +2/−0
- System/IO/MMap.hs +252/−0
- cbits/posix.c +103/−0
- cbits/win32.c +132/−0
- mmap.cabal +31/−0
- tests/SimpleMMap.hs +44/−0
+ LICENSE view
@@ -0,0 +1,27 @@+Copyright (c) Gracjan Polak++All rights reserved.++Redistribution and use in source and binary forms, with or without+modification, are permitted provided that the following conditions+are met:+1. Redistributions of source code must retain the above copyright+ notice, this list of conditions and the following disclaimer.+2. Redistributions in binary form must reproduce the above copyright+ notice, this list of conditions and the following disclaimer in the+ documentation and/or other materials provided with the distribution.+3. Neither the name of the author nor the names of his contributors+ may be used to endorse or promote products derived from this software+ without specific prior written permission.++THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE+ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS+OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)+HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT+LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY+OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF+SUCH DAMAGE.
+ Setup.hs view
@@ -0,0 +1,2 @@+import Distribution.Simple +main = defaultMain
+ System/IO/MMap.hs view
@@ -0,0 +1,252 @@+{-# OPTIONS_GHC -fglasgow-exts #-} + +module System.IO.MMap +( + -- $mmap_intro + + -- * Memory mapped files strict interface + mmapFilePtr, + mmapFileForeignPtr, + mmapFileByteString, + + -- * Memory mapped files lazy interface + mmapFilePtrLazy, + mmapFileForeignPtrLazy, + mmapFileByteStringLazy, + + -- * Mapping mode + Mode(..) +) +where + +import System.IO () +import Foreign.Ptr (Ptr,FunPtr,nullPtr,plusPtr) +import Foreign.C.Types (CInt,CLLong) +import Foreign.C.String (CString,withCString) +import Foreign.ForeignPtr (ForeignPtr,withForeignPtr,finalizeForeignPtr,newForeignPtr) +import Foreign.Concurrent (newForeignPtr) +import System.IO.Unsafe (unsafePerformIO) +import qualified Data.ByteString.Unsafe as BS (unsafePackCStringFinalizer) +import Data.Int (Int64) +import Control.Monad (when) +import Control.Exception (bracket) +import qualified Data.ByteString as BS (ByteString) +import qualified Data.ByteString.Lazy as BSL (ByteString,fromChunks) + +-- $mmap_intro +-- +-- This module is an interface to mmap(2) system call under POSIX (Unix, Linux, +-- Mac OS X) and CreateFileMapping,MapViewOfFile under Windows. +-- +-- We can consider mmap as lazy IO pushed into the virtual memory +-- subsystem. +-- +-- It is only safe to mmap a file if you know you are the sole user. +-- +-- For more details about mmap, and its consequences, see: +-- +-- * <http://opengroup.org/onlinepubs/009695399/functions/mmap.html> +-- +-- * <http://www.gnu.org/software/libc/manual/html_node/Memory_002dmapped-I_002fO.html> +-- +-- * <http://msdn2.microsoft.com/en-us/library/aa366781(VS.85).aspx> +-- + +-- | Mode of mapping. Three cases are supported. +data Mode = ReadOnly -- ^ file is mapped read-only + | ReadWrite -- ^ file is mapped read-write + | WriteCopy -- ^ file is mapped read-write, but changes aren't propagated to disk + deriving (Eq,Ord,Enum) + +-- | The 'mmapFilePtr' function maps a file or device into memory, +-- returning a tripple containing pointer that accesses the mapped file, +-- the finalizer to run to unmap region and size of mmaped memory. +-- +-- If the mmap fails for some reason, an error is thrown. +-- +-- Memory mapped files will behave as if they were read lazily -- +-- pages from the file will be loaded into memory on demand. +-- +-- The storage manager is used to free the mapped memory. When +-- the garbage collector notices there are no further references to the +-- mapped memory, a call to munmap is made. It is not necessary to do +-- this yourself. In tight memory situations, it may be profitable to +-- use 'System.Mem.performGC' or 'finalizeForeignPtr' to force an unmap. +-- +-- File must be created with correct attributes prior to mapping it +-- into memory. +-- +-- If mode is 'ReadWrite' or 'WriteCopy', the returned memory region may +-- be written to with 'Foreign.Storable.poke' and friends. +-- +-- Range specified may be 'Nothing', then whole file is mapped. Otherwise +-- range should be 'Just (offset,size)' where offsets is the beginning byte +-- of file region to map and size tells its length. There are no alignment +-- requirements. +-- +-- If range to map extends beyond end of file, it will be resized accordingly. +-- +mmapFilePtr :: FilePath -- ^ name of file to mmap + -> Mode -- ^ access mode + -> Maybe (Int64,Int) -- ^ range to map, maps whole file if Nothing + -> IO (Ptr a,IO (),Int) -- ^ pointer, finalizer and size +mmapFilePtr filepath mode offsetsize = do + bracket (mmapFileOpen filepath mode) + (finalizeForeignPtr) mmap + where + mmap handle = do + (offset,size) <- case offsetsize of + Just (offset,size) -> return (offset,size) + Nothing -> do + longsize <- withForeignPtr handle c_system_io_file_size + when (longsize > fromIntegral (maxBound :: Int)) $ + error ("file is longer (" ++ show longsize ++ ") than maxBound::Int") + return (0,fromIntegral longsize) + withForeignPtr handle $ \handle -> do + let align = offset `mod` fromIntegral c_system_io_granularity + offsetraw = offset - align + sizeraw = size + fromIntegral align + ptr <- c_system_io_mmap_mmap handle (fromIntegral $ fromEnum mode) (fromIntegral offsetraw) (fromIntegral sizeraw) + when (ptr == nullPtr) $ + error "c_system_io_mmap_mmap returned NULL" + let finalizer = c_system_io_mmap_munmap ptr (fromIntegral sizeraw) + return (ptr `plusPtr` fromIntegral align,finalizer,fromIntegral size) + +-- | Maps region of file and returns it as 'ForeignPtr'. See 'mmapFilePtr' for details. +mmapFileForeignPtr :: FilePath -- ^ name of file to map + -> Mode -- ^ access mode + -> Maybe (Int64,Int) -- ^ range to map, maps whole file if Nothing + -> IO (ForeignPtr a,Int) -- ^ foreign pointer to beginning of region and size +mmapFileForeignPtr filepath mode offsetsize = do + (ptr,finalizer,size) <- mmapFilePtr filepath mode offsetsize + foreignptr <- Foreign.Concurrent.newForeignPtr ptr finalizer + return (foreignptr,size) + + +-- | Maps region of file and returns it as 'Data.ByteString.ByteString'. +-- File is mapped in in 'ReadOnly' mode. See 'mmapFilePtr' for details +-- +-- Note: this operation may break referential transparency! If +-- any other process on the system changes the file when it is mapped +-- into Haskell, the contents of your 'Data.ByteString.ByteString' may change. +-- +mmapFileByteString :: FilePath -- ^ name of file to map + -> Maybe (Int64,Int) -- ^ range to map, maps whole file if Nothing + -> IO BS.ByteString -- ^ bytestring with file content +mmapFileByteString filepath offsetsize = do + (ptr,finalizer,size) <- mmapFilePtr filepath ReadOnly offsetsize + bytestring <- BS.unsafePackCStringFinalizer ptr size finalizer + return bytestring + +-- | The 'mmapFilePtrLazy' function maps a file or device into memory, +-- returning a list of tripples containing pointer that accesses the mapped file, +-- the finalizer to run to unmap that region and size of mapped memory. +-- +-- If the mmap fails for some reason, an error is thrown. +-- +-- Memory mapped files will behave as if they were read lazily -- +-- pages from the file will be loaded into memory on demand. +-- +-- The storage manager is used to free the mapped memory. When +-- the garbage collector notices there are no further references to the +-- mapped memory, a call to munmap is made. It is not necessary to do +-- this yourself. In tight memory situations, it may be profitable to +-- use 'System.Mem.performGC' or 'finalizeForeignPtr' to force an unmap. +-- +-- File must be created with correct attributes prior to mapping it +-- into memory. +-- +-- If mode is 'ReadWrite' or 'WriteCopy', the returned memory region may +-- be written to with 'Foreign.Storable.poke' and friends. +-- +-- Range specified may be 'Nothing', then whole file is mapped. Otherwise +-- range should be 'Just (offset,size)' where offsets is the beginning byte +-- of file region to map and size tells its length. There are no alignment +-- requirements. +-- +-- If range to map extends beyond end of file, it will be resized accordingly. +-- +mmapFilePtrLazy :: FilePath -- ^ name of file to mmap + -> Mode -- ^ access mode + -> Maybe (Int64,Int64) -- ^ range to map, maps whole file if Nothing + -> IO [(Ptr a,IO (),Int)] -- ^ list of pointer, finalizer and size +mmapFilePtrLazy filepath mode offsetsize = do + handle <- mmapFileOpen filepath mode + mmap handle + where + mmap handle = do + (offset,size) <- case offsetsize of + Just (offset,size) -> return (offset,size) + Nothing -> do + longsize <- withForeignPtr handle c_system_io_file_size + return (0,fromIntegral longsize) + return $ map (mapChunk handle) (chunks offset size) + mapChunk handle (offset,size) = unsafePerformIO $ + withForeignPtr handle $ \handle -> do + let align = offset `mod` fromIntegral c_system_io_granularity + offsetraw = offset - align + sizeraw = size + fromIntegral align + ptr <- c_system_io_mmap_mmap handle (fromIntegral $ fromEnum mode) (fromIntegral offsetraw) (fromIntegral sizeraw) + when (ptr == nullPtr) $ + error "c_system_io_mmap_mmap returned NULL" + let finalizer = c_system_io_mmap_munmap ptr (fromIntegral sizeraw) + return (ptr `plusPtr` fromIntegral align,finalizer,fromIntegral size) + +chunks :: Int64 -> Int64 -> [(Int64,Int)] +chunks offset size | size <= fromIntegral chunkSize = [(offset,fromIntegral size)] + | otherwise = let offset2 = offset + fromIntegral chunkSize `div` fromIntegral chunkSize * fromIntegral chunkSize + size2 = fromIntegral (offset2 - offset) + in (offset,size2) : chunks (offset2) (size-fromIntegral size2) + +-- | Maps region of file and returns it as list of 'ForeignPtr's. See 'mmapFilePtr' for details. +-- Each chunk is mapped in on demand only. +mmapFileForeignPtrLazy :: FilePath -- ^ name of file to map + -> Mode -- ^ access mode + -> Maybe (Int64,Int64) -- ^ range to map, maps whole file if Nothing + -> IO [(ForeignPtr a,Int)] -- ^ foreign pointer to beginning of region and size +mmapFileForeignPtrLazy filepath mode offsetsize = do + list <- mmapFilePtrLazy filepath mode offsetsize + return (map turn list) + where + turn (ptr,finalizer,size) = unsafePerformIO $ do + foreignptr <- Foreign.Concurrent.newForeignPtr ptr finalizer + return (foreignptr,size) + +-- | Maps region of file and returns it as 'Data.ByteString.Lazy.ByteString'. +-- File is mapped in in 'ReadOnly' mode. See 'mmapFilePtrLazy' for details. +-- Chunks are mapped in on demand. +-- +-- Note: this operation may break referential transparency! If +-- any other process on the system changes the file when it is mapped +-- into Haskell, the contents of your 'Data.ByteString.Lazy.ByteString' may change. +-- +mmapFileByteStringLazy :: FilePath -- ^ name of file to map + -> Maybe (Int64,Int64) -- ^ range to map, maps whole file if Nothing + -> IO BSL.ByteString -- ^ bytestring with file content +mmapFileByteStringLazy filepath offsetsize = do + list <- mmapFilePtrLazy filepath ReadOnly offsetsize + return (BSL.fromChunks (map turn list)) + where + turn (ptr,finalizer,size) = unsafePerformIO $ do + bytestring <- BS.unsafePackCStringFinalizer ptr size finalizer + return bytestring + +chunkSize :: Int +chunkSize = fromIntegral $ (128*1024 `div` c_system_io_granularity) * c_system_io_granularity + +mmapFileOpen :: FilePath -> Mode -> IO (ForeignPtr ()) +mmapFileOpen filepath mode = do + ptr <- withCString filepath $ \filepath -> c_system_io_mmap_file_open filepath (fromIntegral $ fromEnum mode) + when (ptr == nullPtr) $ + error "c_system_io_mmap_file_open returned NULL" + handle <- Foreign.ForeignPtr.newForeignPtr c_system_io_mmap_file_close ptr + return handle + +foreign import ccall unsafe "system_io_mmap_file_open" c_system_io_mmap_file_open :: CString -> CInt -> IO (Ptr ()) +foreign import ccall unsafe "&system_io_mmap_file_close" c_system_io_mmap_file_close :: FunPtr(Ptr () -> IO ()) +foreign import ccall unsafe "system_io_mmap_mmap" c_system_io_mmap_mmap :: Ptr () -> CInt -> CLLong -> CInt -> IO (Ptr ()) +foreign import ccall unsafe "system_io_mmap_munmap" c_system_io_mmap_munmap :: Ptr () -> CInt -> IO () +foreign import ccall unsafe "system_io_mmap_file_size" c_system_io_file_size :: Ptr () -> IO (CLLong) +foreign import ccall unsafe "system_io_mmap_granularity" c_system_io_granularity :: CInt + +
+ cbits/posix.c view
@@ -0,0 +1,103 @@+ + +#define _LARGEFILE64_SOURCE1 +#define _FILE_OFFSET_BITS 64 + +#include <sys/types.h> +#include <sys/stat.h> +#include <sys/mman.h> +#include <fcntl.h> +#include <unistd.h> +#include <sys/errno.h> + +//foreign import ccall unsafe "system_io_mmap_file_open" c_system_io_mmap_file_open :: CString -> CInt -> IO (Ptr ()) +void *system_io_mmap_file_open(const char *filepath, int mode) +{ + void *handle = NULL; + int access; + if( !filepath ) + return NULL; + switch(mode) { + case 0: + access = O_RDONLY; + break; + case 1: + access = O_RDWR; + break; + case 2: + access = O_RDONLY; + break; + default: + return NULL; + } + handle = (void *)open(filepath,access|O_SHLOCK,0666); + if( handle==(void*)(-1) ) { + //fprintf(stderr,"open errno %d\n",errno); + return NULL; + } + return handle; +} + +//foreign import ccall unsafe "system_io_mmap_file_close" c_system_io_mmap_file_close :: FunPtr(Ptr () -> IO ()) +void system_io_mmap_file_close(void *handle) +{ + close((int)handle); +} + +//foreign import ccall unsafe "system_io_mmap_mmap" c_system_io_mmap_mmap :: Ptr () -> CInt -> CLLong -> CInt -> IO (Ptr ()) +void *system_io_mmap_mmap(void *handle, int mode, long long offset, int size) +{ + void *ptr = NULL; + int prot; + int flags; + switch(mode) { + case 0: + prot = PROT_READ; + flags = MAP_PRIVATE; + break; + case 1: + prot = PROT_READ|PROT_WRITE; + flags = MAP_SHARED; + break; + case 2: + prot = PROT_READ|PROT_WRITE; + flags = MAP_PRIVATE; + break; + default: + return NULL; + } + + struct stat st; + fstat((int)handle,&st); + if( st.st_size<offset+size) { + ftruncate((int)handle,offset+size); + } + + ptr = mmap(NULL,size,prot,flags,(int)handle,offset); + + if( ptr == (void*)(-1)) { + //fprintf(stderr,"mmap errno %d\n",errno); + return NULL; + } + return ptr; +} + +//foreign import ccall unsafe "system_io_mmap_munmap" c_system_io_mmap_munmap :: Ptr () -> CInt -> IO () +void system_io_mmap_munmap(void *ptr,int size) +{ + munmap(ptr,size); +} + +//foreign import ccall unsafe "system_io_mmap_file_size" c_system_io_file_size :: Ptr () -> IO (CLLong) +long long system_io_mmap_file_size(void *handle) +{ + struct stat st; + fstat((int)handle,&st); + return st.st_size; +} + +//foreign import ccall unsafe "system_io_mmap_granularity" c_system_io_granularity :: CInt +int system_io_mmap_granularity() +{ + return getpagesize(); +}
+ cbits/win32.c view
@@ -0,0 +1,132 @@+ + + +#include <windows.h> +#include <stdio.h> + +//foreign import ccall unsafe "system_io_mmap_file_open" c_system_io_mmap_file_open :: CString -> CInt -> IO (Ptr ()) +void *system_io_mmap_file_open(const char *filepath, int mode) +{ + /* + HANDLE WINAPI CreateFileA( + __in LPCTSTR lpFileName, + __in DWORD dwDesiredAccess, + __in DWORD dwShareMode, + __in_opt LPSECURITY_ATTRIBUTES lpSecurityAttributes, + __in DWORD dwCreationDisposition, + __in DWORD dwFlagsAndAttributes, + __in_opt HANDLE hTemplateFile + ); + */ + void *handle = NULL; + DWORD dwDesiredAccess; + if( !filepath ) + return NULL; + switch(mode) { + case 0: + dwDesiredAccess = GENERIC_READ; + break; + case 1: + dwDesiredAccess = GENERIC_WRITE|GENERIC_READ; + break; + case 2: + dwDesiredAccess = GENERIC_READ; + break; + default: + return NULL; + } + handle = CreateFileA(filepath, + dwDesiredAccess, + FILE_SHARE_READ|FILE_SHARE_WRITE|FILE_SHARE_DELETE, + NULL, + OPEN_EXISTING, + FILE_ATTRIBUTE_NORMAL, + NULL); + if( handle==INVALID_HANDLE_VALUE ) + return NULL; + return handle; +} + +//foreign import ccall unsafe "system_io_mmap_file_close" c_system_io_mmap_file_close :: FunPtr(Ptr () -> IO ()) +void system_io_mmap_file_close(void *handle) +{ + CloseHandle(handle); +} + +//foreign import ccall unsafe "system_io_mmap_mmap" c_system_io_mmap_mmap :: Ptr () -> CInt -> CLLong -> CInt -> IO (Ptr ()) +void *system_io_mmap_mmap(void *handle, int mode, long long offset, int size) +{ + /* + HANDLE WINAPI CreateFileMapping( + __in HANDLE hFile, + __in_opt LPSECURITY_ATTRIBUTES lpAttributes, + __in DWORD flProtect, + __in DWORD dwMaximumSizeHigh, + __in DWORD dwMaximumSizeLow, + __in_opt LPCTSTR lpName + ); + LPVOID WINAPI MapViewOfFile( + __in HANDLE hFileMappingObject, + __in DWORD dwDesiredAccess, + __in DWORD dwFileOffsetHigh, + __in DWORD dwFileOffsetLow, + __in SIZE_T dwNumberOfBytesToMap + ); + */ + HANDLE mapping; + void *ptr = NULL; + DWORD flProtect; + DWORD dwDesiredAccess; + switch(mode) { + case 0: + flProtect = PAGE_READONLY; + dwDesiredAccess = FILE_MAP_READ; + break; + case 1: + flProtect = PAGE_READWRITE; + dwDesiredAccess = FILE_MAP_WRITE; + break; + case 2: + flProtect = PAGE_WRITECOPY; + dwDesiredAccess = FILE_MAP_COPY; + break; + default: + return NULL; + } + mapping = CreateFileMapping(handle, NULL, flProtect, (DWORD) ((offset + size)>>32), (DWORD)(offset + size), NULL); + if( !mapping ) { + DWORD dw = GetLastError(); + fprintf(stderr,"CreateFileMapping %d\n",(int)dw); + } + ptr = MapViewOfFile(mapping,dwDesiredAccess, (DWORD)(offset>>32), (DWORD)(offset), size ); + if( !ptr ) { + DWORD dw = GetLastError(); + fprintf(stderr,"MapViewOfFile %d\n",(int)dw); + } + CloseHandle(mapping); + return ptr; +} + +//foreign import ccall unsafe "system_io_mmap_munmap" c_system_io_mmap_munmap :: Ptr () -> CInt -> IO () +void system_io_mmap_munmap(void *ptr,int size) +{ + UnmapViewOfFile(ptr); +} + +//foreign import ccall unsafe "system_io_mmap_file_size" c_system_io_file_size :: Ptr () -> IO (CLLong) +long long system_io_mmap_file_size(void *handle) +{ + DWORD lobits, hibits; + lobits = GetFileSize(handle,&hibits); + return (long long)lobits + ((long long)hibits << 32); +} + +//foreign import ccall unsafe "system_io_mmap_granularity" c_system_io_granularity :: CInt +int system_io_mmap_granularity() +{ + SYSTEM_INFO sysinfo; + GetSystemInfo(&sysinfo); + return sysinfo.dwAllocationGranularity; +} + +
+ mmap.cabal view
@@ -0,0 +1,31 @@+Name: mmap+Version: 0.1+Stability: alpha+License: BSD3+License-File: LICENSE+Copyright: 2008, Gracjan Polak+Author: Gracjan Polak <gracjanpolak@gmail.com>+Maintainer: Gracjan Polak <gracjanpolak@gmail.com>+Synopsis: Memory mapped files for POSIX and Windows+Description:+ This library provides a wrapper to mmap(2) or MapViewOfFile, + allowing files or devices to be lazily loaded into memory as + strict or lazy ByteStrings, ForeignPtrs or plain Ptrs, using + the virtual memory subsystem to do on-demand loading. + Modifications are also supported.+Cabal-version: >= 1.2+Category: System+Build-type: Simple++Library+ Build-depends: base, bytestring+ Extensions: ForeignFunctionInterface+ Exposed-modules: System.IO.MMap+ Hs-source-dirs: .+ Ghc-options: -O2+ CC-options: -O2 -Wall+ if os(mingw32)+ C-sources: cbits/win32.c+ else+ C-sources: cbits/posix.c+
+ tests/SimpleMMap.hs view
@@ -0,0 +1,44 @@+ + +module Main where + +import System.IO.MMap +import Data.ByteString.Char8 as BSC +import qualified Data.ByteString.Lazy as BSL +import Data.Word +import Foreign.ForeignPtr +import Foreign.Storable +import Foreign.Ptr +import System.Mem +import Control.Concurrent +import Control.Exception as E + + +content = BSC.pack "Memory mapping of files for POSIX and Windows" + +main = do + BSC.writeFile "test.bin" content + bs <- mmapFileByteString "test.bin" Nothing + BSC.putStrLn bs + print (bs == content) + bs2 <- mmapFileByteString "test.bin" (Just (5,5)) + print (bs2 == BSC.take 5 (BSC.drop 5 content)) + + -- create 5 gigabyte file + let l = 1024*1024*1024*5 + (f,s) <- mmapFileForeignPtr "test.bin" ReadWrite (Just (l,5)) + withForeignPtr f $ \f -> poke (castPtr f) (64::Word8) + + E.catch (do + bs3 <- mmapFileByteString "test.bin" Nothing + print (fromIntegral l==BSC.length bs3 + 5 )) + (\e -> print True -- exception here is also ok + ) + bs4 <- mmapFileByteStringLazy "test.bin" Nothing + print (BSL.fromChunks [content] == BSL.take (fromIntegral $ BSC.length content) bs4) + bs5 <- mmapFileByteStringLazy "test.bin" (Just (5,5)) + print (BSC.take 5 (BSC.drop 5 content) == BSC.concat (BSL.toChunks bs5)) + + System.Mem.performGC + threadDelay 10000 +