mnist-idx 0.1.2.4 → 0.1.2.5

raw patch · 3 files changed

+136/−136 lines, 3 filesdep ~basePVP: major bump suggested

API removals or changes: PVP suggests a major version bump

Dependency ranges changed: base

API changes (from Hackage documentation)

Files

• mnist-idx.cabal +1/−1
• src/Data/IDX.hs +1/−135
• src/Data/IDX/Internal.hs +134/−0

@@ -1,6 +1,6 @@ name:                mnist-idx -version:             0.1.2.4+version:             0.1.2.5  -- A short (one-line) description of the package. synopsis:            Read and write IDX data that is used in e.g. the MNIST database.

@@ -10,7 +10,7 @@ -- -- A package for reading and writing data in the IDX format. -- This data format is used for machine-learning data sets like the--- MINST database of handwritten digits (<http://yann.lecun.com/exdb/mnist/>)+-- MNIST database of handwritten digits (<http://yann.lecun.com/exdb/mnist/>) -------------------------------------------------------------------------------- module Data.IDX (                 -- * Data types@@ -70,41 +70,6 @@ import           Data.Vector.Unboxed ((!)) import           Data.Word -import Debug.Trace--instance Binary IDXContentType where-    get = do-      w <- getWord8-      case w of-        0x08 -> return IDXUnsignedByte-        0x09 -> return IDXSignedByte-        0x0B -> return IDXShort-        0x0C -> return IDXInt-        0x0D -> return IDXFloat-        0x0E -> return IDXDouble-        _ -> fail $ "Unrecognized IDX content type: " ++ (show w)-        -    put IDXUnsignedByte = putWord8 0x08-    put IDXSignedByte   = putWord8 0x09-    put IDXShort        = putWord8 0x0B-    put IDXInt          = putWord8 0x0C-    put IDXFloat        = putWord8 0x0D-    put IDXDouble       = putWord8 0x0E--instance Binary IDXLabels where-  get = do-    getInt32-    nItems <- fromIntegral <$> getInt32-    let readEntries n = V.replicateM n $ fromIntegral <$> getWord8 >>= (return $!)-    v <- readContent readEntries 500 nItems-    return $ IDXLabels v--  put (IDXLabels v) = do-    put (0 :: Int32)-    let len = V.length v-    put (fromIntegral len :: Int32)-    V.forM_ v (\x -> put $! (fromIntegral x :: Word8))- -- | Partition a dataset and label each subpartition, return int values labeledIntData:: IDXLabels -> IDXData -> Maybe [(Int, V.Vector Int)] labeledIntData (IDXLabels v) dat =@@ -133,105 +98,6 @@     content = idxDoubleContent dat     entrySize = (V.product $ idxDimensions dat) `div` dim0 -instance Binary IDXData where-    get = do-      -- Get header information (4 bytes total)-      getWord8-      getWord8 -      idxType <- get :: Get IDXContentType-      nDimensions <- fromIntegral <$> getWord8--      -- Each dimension size is encoded as a 32 bit integer-      dimensionSizes <- replicateM nDimensions (fromIntegral <$> getInt32)      -      let nEntries = fromIntegral $ product dimensionSizes-          dimV = V.fromList dimensionSizes--      -- Retrieve the data, depending on the type specified in the file-      -- Cast all integral types to Int and all decimal numbers tod double-      case idxType of-        t@IDXUnsignedByte -> buildIntResult nEntries t dimV getWord8-        t@IDXSignedByte   -> buildIntResult nEntries t dimV getInt8-        t@IDXShort        -> buildIntResult nEntries t dimV getInt16-        t@IDXInt          -> buildIntResult nEntries t dimV getInt32--        t@IDXFloat        -> buildDoubleResult nEntries t dimV getFloat-        t@IDXDouble       -> buildDoubleResult nEntries t dimV getDouble-        -    put d = do-      -- First four bytes are meta information-      putWord8 0-      putWord8 0-      -- Third byte is content type-      put $ idxType d-      -      -- Fourth byte is number of dimensions-      let dimensions = idxDimensions d-      put $ (fromIntegral $ V.length dimensions :: Word8)--      -- Put size of each dimension as an Int32-      V.forM_ dimensions $ (\x -> put $! (fromIntegral x :: Int32))--      -- Put the individual values-      case d of-        IDXDoubles t _ content -> V.forM_ content $ putReal     t-        IDXInts    t _ content -> V.forM_ content $ putIntegral t---- | Helper function for parsing integer data from the--- IDX content. Returns a full IDX result.-buildIntResult :: Integral a-                  => Int                -- ^ Expected number of entries-                      -> IDXContentType -- ^ Description of content-                      -> V.Vector Int   -- ^ Dimension sizes-                      -> Get a          -- ^ Monadic action to get content element-                      -> Get IDXData-buildIntResult nEntries typ dimV getContent = do-  content <- readContent readEntries 500 nEntries-  return $ IDXInts typ dimV content-  where-    readEntries n = V.replicateM n $ fromIntegral <$> getContent >>= (return $!)---- | Helper function for parsing real number data from--- the IDX content.-buildDoubleResult :: Real a-                  => Int                -- ^ Expected number of entries-                      -> IDXContentType -- ^ Description of content-                      -> V.Vector Int   -- ^ Dimension sizes-                      -> Get a          -- ^ Monadic action to get content element-                      -> Get IDXData-buildDoubleResult nEntries typ dimV getContent = do-  content <- readContent readEntries 500 nEntries-  return $ IDXDoubles typ dimV content-  where-    readEntries n = V.replicateM n $ realToFrac <$> getContent >>= (return $!)---- | Put values that are saved as Int-putIntegral :: IDXContentType -> Int -> Put-putIntegral IDXUnsignedByte n = put $! (fromIntegral n :: Word8)-putIntegral IDXSignedByte   n = put $! (fromIntegral n :: Int8 )-putIntegral IDXShort        n = put $! (fromIntegral n :: Int16)-putIntegral IDXInt          n = put $! (fromIntegral n :: Int32)-putIntegral t               _ = error $ "IDX.putIntegral " ++ show t ---- | Put real values that are saved as Double-putReal :: IDXContentType -> Double -> Put-putReal IDXDouble n = put n-putReal IDXFloat  n = put $! (realToFrac n :: Float )---- Haskell's Data.Binary uses big-endian format-getInt8 :: Get Int8-getInt8 = get--getInt16 :: Get Int16-getInt16 = get--getInt32 :: Get Int32-getInt32 = get--getFloat :: Get Float-getFloat = get--getDouble :: Get Double-getDouble = get  -- | Read labels from a file, return 'Nothing' if something doesn't work decodeIDXLabelsFile :: FilePath -> IO (Maybe IDXLabels)

@@ -19,6 +19,7 @@  import           Control.Monad (replicateM) import           Data.Binary+import           Data.Int  import qualified Data.Vector.Unboxed as V import           Data.Vector.Unboxed ((!))@@ -33,7 +34,25 @@    IDXDouble       :: IDXContentType    deriving Show +instance Binary IDXContentType where+    get = do+      w <- getWord8+      case w of+        0x08 -> return IDXUnsignedByte+        0x09 -> return IDXSignedByte+        0x0B -> return IDXShort+        0x0C -> return IDXInt+        0x0D -> return IDXFloat+        0x0E -> return IDXDouble+        _ -> fail $ "Unrecognized IDX content type: " ++ (show w) +    put IDXUnsignedByte = putWord8 0x08+    put IDXSignedByte   = putWord8 0x09+    put IDXShort        = putWord8 0x0B+    put IDXInt          = putWord8 0x0C+    put IDXFloat        = putWord8 0x0D+    put IDXDouble       = putWord8 0x0E+ -- | Datatype for storing IDXData. Internally data is always stored either -- as 'Int' or 'Double' unboxed vectors. However when binary serialization -- is used, the data is serialized according to the 'IDXContentType'.@@ -41,9 +60,68 @@              | IDXDoubles IDXContentType (V.Vector Int) (V.Vector Double)              deriving Show ++instance Binary IDXData where+    get = do+      -- Get header information (4 bytes total)+      getWord8+      getWord8 +      idxType <- get :: Get IDXContentType+      nDimensions <- fromIntegral <$> getWord8++      -- Each dimension size is encoded as a 32 bit integer+      dimensionSizes <- replicateM nDimensions (fromIntegral <$> getInt32)      +      let nEntries = fromIntegral $ product dimensionSizes+          dimV = V.fromList dimensionSizes++      -- Retrieve the data, depending on the type specified in the file+      -- Cast all integral types to Int and all decimal numbers tod double+      case idxType of+        t@IDXUnsignedByte -> buildIntResult nEntries t dimV getWord8+        t@IDXSignedByte   -> buildIntResult nEntries t dimV getInt8+        t@IDXShort        -> buildIntResult nEntries t dimV getInt16+        t@IDXInt          -> buildIntResult nEntries t dimV getInt32++        t@IDXFloat        -> buildDoubleResult nEntries t dimV getFloat+        t@IDXDouble       -> buildDoubleResult nEntries t dimV getDouble+        +    put d = do+      -- First four bytes are meta information+      putWord8 0+      putWord8 0+      -- Third byte is content type+      put $ idxType d+      +      -- Fourth byte is number of dimensions+      let dimensions = idxDimensions d+      put $ (fromIntegral $ V.length dimensions :: Word8)++      -- Put size of each dimension as an Int32+      V.forM_ dimensions $ (\x -> put $! (fromIntegral x :: Int32))++      -- Put the individual values+      case d of+        IDXDoubles t _ content -> V.forM_ content $ putReal     t+        IDXInts    t _ content -> V.forM_ content $ putIntegral t++-- | A data type that holds 'Int' labels for a set of 'IDXData' newtype IDXLabels = IDXLabels (V.Vector Int) +instance Binary IDXLabels where+  get = do+    getInt32+    nItems <- fromIntegral <$> getInt32+    let readEntries n = V.replicateM n $ fromIntegral <$> getWord8 >>= (return $!)+    v <- readContent readEntries 500 nItems+    return $ IDXLabels v +  put (IDXLabels v) = do+    put (0 :: Int32)+    let len = V.length v+    put (fromIntegral len :: Int32)+    V.forM_ v (\x -> put $! (fromIntegral x :: Word8))++ -- | Return the what type the data is stored in idxType :: IDXData -> IDXContentType idxType (IDXInts    t _ _) = t@@ -101,3 +179,59 @@     rest <- readEntries n     return $! rest +-- Haskell's Data.Binary uses big-endian format+getInt8 :: Get Int8+getInt8 = get++getInt16 :: Get Int16+getInt16 = get++getInt32 :: Get Int32+getInt32 = get++getFloat :: Get Float+getFloat = get++getDouble :: Get Double+getDouble = get++-- | Helper function for parsing integer data from the+-- IDX content. Returns a full IDX result.+buildIntResult :: Integral a+                  => Int                -- ^ Expected number of entries+                      -> IDXContentType -- ^ Description of content+                      -> V.Vector Int   -- ^ Dimension sizes+                      -> Get a          -- ^ Monadic action to get content element+                      -> Get IDXData+buildIntResult nEntries typ dimV getContent = do+  content <- readContent readEntries 500 nEntries+  return $ IDXInts typ dimV content+  where+    readEntries n = V.replicateM n $ fromIntegral <$> getContent >>= (return $!)++-- | Helper function for parsing real number data from+-- the IDX content.+buildDoubleResult :: Real a+                  => Int                -- ^ Expected number of entries+                      -> IDXContentType -- ^ Description of content+                      -> V.Vector Int   -- ^ Dimension sizes+                      -> Get a          -- ^ Monadic action to get content element+                      -> Get IDXData+buildDoubleResult nEntries typ dimV getContent = do+  content <- readContent readEntries 500 nEntries+  return $ IDXDoubles typ dimV content+  where+    readEntries n = V.replicateM n $ realToFrac <$> getContent >>= (return $!)++-- | Put values that are saved as Int+putIntegral :: IDXContentType -> Int -> Put+putIntegral IDXUnsignedByte n = put $! (fromIntegral n :: Word8)+putIntegral IDXSignedByte   n = put $! (fromIntegral n :: Int8 )+putIntegral IDXShort        n = put $! (fromIntegral n :: Int16)+putIntegral IDXInt          n = put $! (fromIntegral n :: Int32)+putIntegral t               _ = error $ "IDX.putIntegral " ++ show t ++-- | Put real values that are saved as Double+putReal :: IDXContentType -> Double -> Put+putReal IDXDouble n = put n+putReal IDXFloat  n = put $! (realToFrac n :: Float )