datasets 0.3.0 → 0.4.0

raw patch · 28 files changed

+865/−198 lines, 28 filesdep +JuicyPixelsdep +QuickCheckdep +criteriondep −data-default-classdep ~aesondep ~bytestringdep ~cassavaPVP ok

version bump matches the API change (PVP)

Dependencies added: JuicyPixels, QuickCheck, criterion, datasets, deepseq, exceptions, hspec, mtl, mwc-random, parallel, safe-exceptions, streaming, streaming-attoparsec, streaming-bytestring, streaming-cassava, streaming-commons, tar, transformers, zlib

Dependencies removed: data-default-class

Dependency ranges changed: aeson, bytestring, cassava, directory, file-embed, filepath, hashable, microlens, req, stringsearch, text, time, vector

API changes (from Hackage documentation)

Files

• bench/Main.hs +7/−0
• bench/Numeric/Dataloader/Benchmark.hs +112/−0
• changelog.md +4/−0
• datasets.cabal +151/−100
• src/Numeric/Dataloader.hs +115/−0
• src/Numeric/Datasets.hs +139/−71
• src/Numeric/Datasets/Abalone.hs +1/−1
• src/Numeric/Datasets/Adult.hs +2/−2
• src/Numeric/Datasets/BostonHousing.hs +3/−3
• src/Numeric/Datasets/BreastCancerWisconsin.hs +3/−3
• src/Numeric/Datasets/CIFAR10.hs +175/−0
• src/Numeric/Datasets/CO2.hs +1/−1
• src/Numeric/Datasets/Car.hs +1/−1
• src/Numeric/Datasets/Coal.hs +1/−1
• src/Numeric/Datasets/Gapminder.hs +1/−1
• src/Numeric/Datasets/Internal/Streaming.hs +99/−0
• src/Numeric/Datasets/Mushroom.hs +2/−2
• src/Numeric/Datasets/OldFaithful.hs +1/−1
• src/Numeric/Datasets/Quakes.hs +1/−1
• src/Numeric/Datasets/States.hs +1/−1
• src/Numeric/Datasets/Sunspots.hs +1/−1
• src/Numeric/Datasets/Titanic.hs +4/−4
• src/Numeric/Datasets/UN.hs +1/−1
• src/Numeric/Datasets/Vocabulary.hs +1/−1
• src/Numeric/Datasets/Wine.hs +1/−1
• src/Numeric/Datasets/WineQuality.hs +1/−1
• src/Streaming/Instances.hs +21/−0
• test/Spec.hs +15/−0

@@ -0,0 +1,7 @@+module Main where++import qualified Numeric.Dataloader.Benchmark++main :: IO ()+main = Numeric.Dataloader.Benchmark.main+

@@ -0,0 +1,112 @@+{-# OPTIONS_GHC -Wno-orphans  #-}+{-# LANGUAGE DataKinds #-}+{-# LANGUAGE LambdaCase #-}+module Numeric.Dataloader.Benchmark where++import Control.DeepSeq+import Control.Concurrent+import Numeric.Dataloader+import Numeric.Datasets+import Numeric.Datasets.Abalone+import Streaming (Of)+import qualified Streaming.Prelude as S+import qualified System.Random.MWC as MWC++-- ImageLoading bench+import Numeric.Datasets.Internal.Streaming+import Numeric.Datasets.CIFAR10+import System.FilePath+import System.Directory+import Codec.Picture+import Control.Exception.Safe+import Text.Read+import System.IO.Unsafe+import qualified Data.List.NonEmpty   as NonEmpty++import Criterion.Main++instance NFData Abalone+instance NFData Sex+instance (NFData a, NFData b) => NFData (Of a b)+++mkDataloaderWithIx :: Dataset a -> IO (Dataloader a a)+mkDataloaderWithIx ds = MWC.withSystemRandom $ \g -> do+  ixl <- uniformIxline ds g+  pure $ Dataloader 1 (Just ixl) ds id+++main :: IO ()+main = do+  dl <- mkDataloaderWithIx abalone+  cifar10l <- cifar10ImageLoader+  defaultMain+    [ bgroup "Numeric.Dataloader"+      [ bench "making an ixline" $ nfIO $ MWC.withSystemRandom (uniformIxline abalone)+      , bgroup "testStream"+        [ bench "with ixline" . nfIO $ foldStream (S.take 100 $ slow . stream $ dl)+        , bench "no ixline"   . nfIO $ foldStream (S.take 100 $ slow . stream $ Dataloader 1 Nothing abalone id)+        ]+      , bench "cifar10 image folder" $ nfIO $ foldStream $ S.take 1000 $ stream cifar10l+      , bench "cifar10 batch folder" $ nfIO $ foldStream $ S.take 1  $ batchStream (cifar10l { batchSize = 1000 })+      ]+    ]++slow :: S.Stream (Of a) IO r -> S.Stream (Of a) IO r+slow = S.mapM (\a -> threadDelay 2 >> pure a)++foldStream :: Show a => S.Stream (Of a) IO () -> IO (Of [a] ())+foldStream = S.fold (\memo a -> a:memo) [] id++-------------------------------------------------------------------------------+-- Image Folder loading++-- may be required if you don't already have CIFAR10 loaded+provision :: IO ()+provision = do+  xdgCache <- getXdgDirectory XdgCache "datasets-hs"+  let imfolder = xdgCache </> "cifar-10-imagefolder"+  createDirectoryIfMissing True imfolder++  -- build the image folder+  S.mapM_ (go imfolder) $ streamDataset cifar10+ where+  go :: FilePath -> CIFARImage -> IO ()+  go cachefolder (CIFARImage (im, lbl)) = do+    let labelfolder = cachefolder </> show lbl+    createDirectoryIfMissing True labelfolder+    ix <- length <$> listDirectory labelfolder+    writePng (labelfolder </> (show lbl ++ "_" ++ show ix ++ ".png")) im+++-- | dataloading the image folder dataset+cifar10ImageLoader :: IO (Dataloader (String, FilePath) CIFARImage)+cifar10ImageLoader = do+  xdgCache <- getXdgDirectory XdgCache "datasets-hs"+  let imfolder = xdgCache </> "cifar-10-imagefolder"+  pure $ Dataloader 1 Nothing (imgFolderDataset imfolder) load++ where+  labelFolders :: NonEmpty.NonEmpty String+  labelFolders = show <$> NonEmpty.fromList [minBound..maxBound::Label]++  imgFolderDataset :: FilePath -> Dataset (String, FilePath)+  imgFolderDataset fp =+    Dataset+      (ImgFolder fp labelFolders)+      Nothing+      Nothing+      (ImageFolder labelFolders)++  load :: (String, FilePath) -> CIFARImage+  load (str, fp) = CIFARImage (img, lbl)+    where+      lbl :: Label+      lbl = either error id $ readEither str++      img :: Image PixelRGB8+      img = case unsafePerformIO (readPng fp) of+        Left err -> error err+        Right (ImageRGB8 i) -> i+        Right _ -> error "decoded image was not rgb8"+

@@ -1,3 +1,7 @@+0.4+	* Get rid of dependency on 'data-default' (introduced by previous versions of 'req')+	+	* Bump 'req' dependency to 2.0.0  0.3 	* 'datasets' hosted within the DataHaskell/dh-core project 

@@ -1,105 +1,156 @@-Name:                datasets-Version:             0.3.0-Synopsis:            Classical data sets for statistics and machine learning-Description:         Classical machine learning and statistics datasets from-                     the UCI Machine Learning Repository and other sources.-                     .-                     The @datasets@ package defines two different kinds of datasets:-                     .-                     * small data sets which are directly (or indirectly with `file-embed`)-                       embedded in the package as pure values and do not require network or IO to download-                       the data set. This includes Iris, Anscombe and OldFaithful.-                     .-                     * other data sets which need to be fetched over the network with-                     `Numeric.Datasets.getDataset` and are cached in a local temporary directory.-                     .-                     The @datafiles/@ directory of this package includes copies of a few famous datasets, such as Titanic, Nightingale and Michelson.-                     .-                     Example :-                     .-                     > import Numeric.Datasets (getDataset)-                     > import Numeric.Datasets.Iris (iris)-                     > import Numeric.Datasets.Abalone (abalone)-                     >-                     > main = do-                     >   -- The Iris data set is embedded-                     >   print (length iris)-                     >   print (head iris)-                     >   -- The Abalone dataset is fetched-                     >   abas <- getDataset abalone-                     >   print (length abas)-                     >   print (head abas)--License:             MIT-License-file:        LICENSE-Author:              Tom Nielsen <tanielsen@gmail.com>-Maintainer:          Marco Zocca <ocramz fripost org>-build-type:          Simple-Cabal-Version: 	     >= 1.10-homepage:            https://github.com/DataHaskell/dh-core-bug-reports:         https://github.com/DataHaskell/dh-core/issues-category:            Statistics, Machine Learning, Data Mining, Data-Tested-With:         GHC == 7.10.2, GHC == 7.10.3, GHC == 8.0.1, GHC == 8.4.3+cabal-version: >=1.10+name: datasets+version: 0.4.0+license: MIT+license-file: LICENSE+maintainer: Marco Zocca <ocramz fripost org>+author: Tom Nielsen <tanielsen@gmail.com>+tested-with: ghc ==7.10.2 ghc ==7.10.3 ghc ==8.0.1 ghc ==8.4.3+homepage: https://github.com/DataHaskell/dh-core+bug-reports: https://github.com/DataHaskell/dh-core/issues+synopsis: Classical data sets for statistics and machine learning+description:+    Classical machine learning and statistics datasets from+    the UCI Machine Learning Repository and other sources.+    .+    The @datasets@ package defines two different kinds of datasets:+    .+    * small data sets which are directly (or indirectly with `file-embed`)+    embedded in the package as pure values and do not require network or IO to download+    the data set. This includes Iris, Anscombe and OldFaithful.+    .+    * other data sets which need to be fetched over the network with+    `Numeric.Datasets.getDataset` and are cached in a local temporary directory.+    .+    The @datafiles/@ directory of this package includes copies of a few famous datasets, such as Titanic, Nightingale and Michelson.+    .+    Example :+    .+    > import Numeric.Datasets (getDataset)+    > import Numeric.Datasets.Iris (iris)+    > import Numeric.Datasets.Abalone (abalone)+    >+    > main = do+    >   -- The Iris data set is embedded+    >   print (length iris)+    >   print (head iris)+    >   -- The Abalone dataset is fetched+    >   abas <- getDataset abalone+    >   print (length abas)+    >   print (head abas)+category: Statistics, Machine Learning, Data Mining, Data+build-type: Simple extra-source-files:-                   changelog.md-                   datafiles/iris.data-                   datafiles/michelson.json-                   datafiles/nightingale.json-                   datafiles/titanic2_full.tsv-                   datafiles/netflix/training/mv_0000001.txt-                   datafiles/netflix/test/qualifying.txt-                   datafiles/netflix/movies/movie_titles.txt+    changelog.md+    datafiles/iris.data+    datafiles/michelson.json+    datafiles/nightingale.json+    datafiles/titanic2_full.tsv+    datafiles/netflix/training/mv_0000001.txt+    datafiles/netflix/test/qualifying.txt+    datafiles/netflix/movies/movie_titles.txt  source-repository head-  type:     git-  location: https://github.com/DataHaskell/dh-core/datasets+    type: git+    location: https://github.com/DataHaskell/dh-core/datasets -Library-   ghc-options:       -Wall -fno-warn-unused-imports-   hs-source-dirs:    src-   other-extensions: TemplateHaskell-   default-language:  Haskell2010+library+    exposed-modules:+        Numeric.Dataloader+        Numeric.Datasets+        Numeric.Datasets.Anscombe+        Numeric.Datasets.BostonHousing+        Numeric.Datasets.CIFAR10+        Numeric.Datasets.OldFaithful+        Numeric.Datasets.Abalone+        Numeric.Datasets.Adult+        Numeric.Datasets.BreastCancerWisconsin+        Numeric.Datasets.Car+        Numeric.Datasets.Coal+        Numeric.Datasets.CO2+        Numeric.Datasets.Gapminder+        Numeric.Datasets.Iris+        Numeric.Datasets.Internal.Streaming+        Numeric.Datasets.Michelson+        Numeric.Datasets.Mushroom+        Numeric.Datasets.Nightingale+        Numeric.Datasets.Quakes+        Numeric.Datasets.States+        Numeric.Datasets.Sunspots+        Numeric.Datasets.Titanic+        Numeric.Datasets.UN+        Numeric.Datasets.Vocabulary+        Numeric.Datasets.Wine+        Numeric.Datasets.WineQuality+        Numeric.Datasets.Netflix+    hs-source-dirs: src+    other-modules:+        Streaming.Instances+    default-language: Haskell2010+    other-extensions: TemplateHaskell+    ghc-options: -Wall -fno-warn-unused-imports+    build-depends:+        base >=4.6 && <5,+        aeson >=1.4.2.0,+        attoparsec >=0.13,+        bytestring >=0.10.8.2,+        cassava >=0.5.1.0,+        deepseq >=1.4.4.0,+        directory >=1.3.3.0,+        exceptions >=0.10.0,+        file-embed >=0.0.11,+        filepath >=1.4.2.1,+        hashable >=1.2.7.0,+        JuicyPixels >=3.3.3,+        microlens >=0.4.10,+        mtl >=2.2.2,+        mwc-random >=0.14.0.0,+        parallel >=3.2.2.0,+        req >=2.0.0,+        safe-exceptions >=0.1.7.0,+        streaming >=0.2.2.0,+        streaming-attoparsec >=1.0.0,+        streaming-bytestring >=0.1.6,+        streaming-cassava >=0.1.0.1,+        streaming-commons >=0.2.1.0,+        stringsearch >=0.3.6.6,+        tar >=0.5.1.0,+        text >=1.2.3.1,+        time >=1.8.0.2,+        transformers >=0.5.5.0,+        vector >=0.12.0.2,+        safe-exceptions >=0.1.7.0,+        zlib >=0.6.2 -   Exposed-modules:-                   Numeric.Datasets-                 , Numeric.Datasets.Anscombe-                 , Numeric.Datasets.BostonHousing-                 , Numeric.Datasets.OldFaithful-                 , Numeric.Datasets.Abalone-                 , Numeric.Datasets.Adult-                 , Numeric.Datasets.BreastCancerWisconsin-                 , Numeric.Datasets.Car-                 , Numeric.Datasets.Coal-                 , Numeric.Datasets.CO2-                 , Numeric.Datasets.Gapminder-                 , Numeric.Datasets.Iris-                 , Numeric.Datasets.Michelson-                 , Numeric.Datasets.Mushroom                 -                 , Numeric.Datasets.Nightingale-                 , Numeric.Datasets.Quakes-                 , Numeric.Datasets.States-                 , Numeric.Datasets.Sunspots-                 , Numeric.Datasets.Titanic                 -                 , Numeric.Datasets.UN-                 , Numeric.Datasets.Vocabulary-                 , Numeric.Datasets.Wine-                 , Numeric.Datasets.WineQuality-                 , Numeric.Datasets.Netflix-   Build-depends:-                 base          >= 4.6 && < 5-               , aeson-               , attoparsec    >= 0.13-               , bytestring-               , cassava-               , data-default-class-               , directory-               , file-embed-               , filepath-               , hashable-               , microlens-               , req            >= 1.0.0-               , stringsearch-               , text-               , time-               , vector-                 -- , wreq+test-suite spec+    type: exitcode-stdio-1.0+    main-is: Spec.hs+    hs-source-dirs: test+    default-language: Haskell2010+    default-extensions: OverloadedStrings+    ghc-options: -threaded -rtsopts -with-rtsopts=-N+    build-depends:+        base >=4.6 && <5,+        QuickCheck >=2.12.6.1,+        hspec >=2.6.0++benchmark bench+    type: exitcode-stdio-1.0+    main-is: Main.hs+    hs-source-dirs: bench+    other-modules:+        Numeric.Dataloader.Benchmark+    default-language: Haskell2010+    ghc-options: -Wall -O2 -rtsopts -threaded+    build-depends:+        base >=4.6 && <5,+        criterion >=1.5.3.0,+        datasets -any,+        deepseq >=1.4.4.0,+        directory >=1.3.3.0,+        filepath >=1.4.2.1,+        JuicyPixels >=3.3.3,+        mwc-random >=0.14.0.0,+        req >=2.0.0,+        safe-exceptions >=0.1.7.0,+        streaming >=0.2.2.0

@@ -0,0 +1,115 @@+-------------------------------------------------------------------------------+-- |+-- Module    :  Numeric.Dataloader+-- Stability :  experimental+-- Portability: non-portable+--+-- A Dataloader is an extension of a Dataset and is primarily intended for+-- compute-intensive, batch loading interfaces. When used with ImageFolder+-- representations of Datasets, it shuffles the order of files to be loaded+-- and leverages the async library when possible.+--+-- Concurrent loading primarily takes place in 'batchStream'. 'stream' exists+-- primarily to provide a unified API with training that is not batch-oriented.+-------------------------------------------------------------------------------+{-# LANGUAGE ScopedTypeVariables #-}+module Numeric.Dataloader+  ( Dataloader(..)+  , uniformIxline++  , stream+  , batchStream+  ) where++import Control.Monad ((>=>))+import Control.Monad.IO.Class (MonadIO, liftIO)+import Data.Vector (Vector)+import Streaming (Stream, Of(..))+import System.Random.MWC (GenIO)+import qualified Data.Vector          as V+import qualified Streaming            as S+import qualified Streaming.Prelude    as S+import qualified System.Random.MWC.Distributions as MWC+import Control.Exception.Safe (MonadThrow)+import Streaming.Instances ()++import Control.Parallel.Strategies++import Numeric.Datasets++-- * Configuring data loaders+++-- | Options for a data loading functions.+data Dataloader a b = Dataloader+  { batchSize :: Int                 -- ^ Batch size used with 'batchStream'.+  , shuffle :: Maybe (Vector Int)    -- ^ Optional shuffle order (forces the dataset to be loaded in memory if it wasn't already).+  , dataset :: Dataset a             -- ^ Dataset associated with the dataloader.+  , transform :: a -> b              -- ^ Transformation associated with the dataloader which will be run in parallel. If using an+                                     --   ImageFolder, this is where you would transform image filepaths to an image (or other+                                     --   compute-optimized form). Additionally, this is where you should perform any+                                     --   static normalization.+  }+++-- | Generate a uniformly random index line from a dataset and a generator.+uniformIxline+  :: Dataset a+  -> GenIO+  -> IO (Vector Int)+uniformIxline ds gen = do+  len <- V.length <$> getDatavec ds+  MWC.uniformPermutation len gen++-- * Data loading functions+++-- | Stream a dataset in-memory, applying a transformation function.+stream+  :: (MonadThrow io, MonadIO io)+  => Dataloader a b+  -> Stream (Of b) io ()+stream dl = S.maps (\(a:>b) -> (transform dl a `using` rpar) :> b) (sourceStream dl)+++-- | Stream batches of a dataset, concurrently processing each element+--+-- NOTE: Run with @-threaded -rtsopts@ to concurrently load data in-memory.+batchStream+  :: (MonadThrow io, MonadIO io, NFData b)+  => Dataloader a b+  -> Stream (Of [b]) io ()+batchStream dl+  = S.mapsM (S.toList >=> liftIO . firstOfM go)+  $ S.chunksOf (batchSize dl)+  $ sourceStream dl+  where+    go as = fmap (transform dl) as `usingIO` parList rdeepseq+++-- * helper functions (not for export)+++-- | Stream a dataset in-memory+sourceStream+  :: (MonadThrow io, MonadIO io)+  => Dataloader a b+  -> Stream (Of a) io ()+sourceStream loader+  = permute loader <$> getDatavec (dataset loader)+  >>= S.each+  where+    -- Use a dataloader's shuffle order to return a permuted vector (or return the+    -- identity vector).+    permute :: Dataloader a b -> Vector a -> Vector a+    permute loader va = maybe va (V.backpermute va) (shuffle loader)+++-- | Monadic, concrete version of Control.Arrow.first for @Of@+firstOfM :: Monad m => (a -> m b) -> Of a c -> m (Of b c)+firstOfM fm (a:>c) = do+  b <- fm a+  pure (b:>c)+++

@@ -2,12 +2,12 @@  The @datasets@ package defines three different kinds of datasets: -* Tiny datasets (up to a few tens of rows) are embedded as part of the library source code, as lists of values. +* Tiny datasets (up to a few tens of rows) are embedded as part of the library source code, as lists of values.  * Small data sets are embedded indirectly (via @file-embed@)   in the package as pure values and do not require IO   to be downloaded (i.e. the data is loaded and parsed at compile time).-  + * Larger data sets which need to be fetched over the network   and are cached in a local temporary directory for subsequent use. @@ -21,107 +21,173 @@ -}  {-# LANGUAGE OverloadedStrings, GADTs, DataKinds #-}+{-# LANGUAGE LambdaCase #-}+{-# LANGUAGE ExistentialQuantification #-}+{-# LANGUAGE ScopedTypeVariables #-} -- {-# OPTIONS_GHC -fno-warn-unused-imports #-} -module Numeric.Datasets (getDataset, Dataset(..), Source(..),-                         -- * Parsing datasets -                         readDataset, ReadAs(..), csvRecord,+module Numeric.Datasets (getDataset, Dataset(..), Source(..), getDatavec, defaultTempDir, getFileFromSource,+                         -- * Parsing datasets+                         readDataset, safeReadDataset, ReadAs(..), csvRecord,                         -- * Defining datasets                         csvDataset, csvHdrDataset, csvHdrDatasetSep, csvDatasetSkipHdr,                         jsonDataset,                         -- ** Dataset options-                        withPreprocess, withTempDir,                        +                        withPreprocess, withTempDir,                         -- ** Preprocessing functions                         --                         -- | These functions are to be used as first argument of 'withPreprocess' in order to improve the quality of the parser inputs.                         dropLines, fixedWidthToCSV, removeEscQuotes, fixAmericanDecimals,                         -- ** Helper functions-                         parseReadField, parseDashToCamelField, -                         yearToUTCTime, +                         parseReadField, parseDashToCamelField,+                         yearToUTCTime,                         -- * Dataset source URLs                         umassMLDB, uciMLDB) where  import Data.Csv-import System.FilePath+import Data.Monoid+import Data.Foldable+import Data.List (isSuffixOf)+import Data.List.NonEmpty (NonEmpty((:|)))+import qualified Data.List.NonEmpty as NE+import System.FilePath (takeExtensions, (</>)) import System.Directory import Data.Hashable import Data.Monoid import qualified Data.ByteString.Lazy as BL-import qualified Data.Vector as V import Data.Aeson as JSON import Control.Applicative import Data.Time--- import qualified Network.Wreq as Wreq-import Data.Default.Class (Default(..))-import Network.HTTP.Req (req, runReq, Url, (/:), http, https, Scheme(..), LbsResponse, lbsResponse, responseBody, GET(..), NoReqBody(..), HttpMethod(..))+import Network.HTTP.Req (req, runReq, Url, (/:), http, https, Scheme(..), LbsResponse, lbsResponse, responseBody, GET(..), NoReqBody(..), HttpMethod(..), defaultHttpConfig) -- import Lens.Micro ((^.)) +import Control.Exception.Safe import Data.Char (ord, toUpper) import Text.Read (readMaybe) import Data.Maybe (fromMaybe) import Data.ByteString.Char8 (unpack)+import qualified Data.ByteString.Char8 as B8 import qualified Data.ByteString.Lazy.Char8 as BL8 import Data.ByteString.Lazy.Search (replace)+import Control.Monad.IO.Class (MonadIO, liftIO)+import Data.Vector.Generic (Vector)+import qualified Data.Vector         as VB+import qualified Data.Vector.Generic as V+import qualified Data.Attoparsec.ByteString as Atto'+import qualified Data.Attoparsec.ByteString.Lazy as Atto  -- * Using datasets --- | Load a dataset, using the system temporary directory as a cache-getDataset :: Dataset h a -> IO [a]-getDataset ds = do-  dir <- tempDirForDataset ds-  bs <- fmap (fromMaybe id $ preProcess ds) $ getFileFromSource dir $ source ds-  return $ readDataset (readAs ds) bs+-- | Load a dataset into memory+getDataset :: (MonadThrow io, MonadIO io) => Dataset a -> io [a]+getDataset ds = VB.toList <$> getDatavec ds +-- | Load a dataset into memory as a vector+getDatavec :: (MonadThrow io, MonadIO io, Vector v a) => Dataset a -> io (v a)+getDatavec ds = liftIO $ do+  folder <- tempDirForDataset ds+  files <- getFileFromSource folder (source ds)+  safeReadDataset (readAs ds) (fromMaybe id (preProcess ds) <$> files)+ -- | Get a ByteString from the specified Source-getFileFromSource ::-     FilePath  -- ^ Cache directory-  -> Source h  -  -> IO BL.ByteString+getFileFromSource+  :: FilePath  -- ^ Cache directory+  -> Source+  -> IO (NonEmpty BL.ByteString) getFileFromSource cacheDir (URL url) = do   createDirectoryIfMissing True cacheDir   let fnm = cacheDir </> "ds" <> show (hash $ show url)   ex <- doesFileExist fnm   if ex-     then BL.readFile fnm+     then (:|[]) <$> BL.readFile fnm      else do-       rsp <- runReq def $ req GET url NoReqBody lbsResponse mempty +       rsp <- runReq defaultHttpConfig $ req GET url NoReqBody lbsResponse mempty        let bs = responseBody rsp        BL.writeFile fnm bs-       return bs-getFileFromSource _ (File fnm) = -  BL.readFile fnm  +       return (bs:|[])+getFileFromSource _ (File fnm) = (:|[]) <$> BL.readFile fnm+getFileFromSource _ (ImgFolder root labels) =+  NE.fromList <$> foldrM allImFolderData [] labels+  where+    allImFolderData :: String -> [BL.ByteString] -> IO [BL.ByteString]+    allImFolderData label agg = (agg ++) <$> toImFolderData label +    toImFolderData :: String -> IO [BL.ByteString]+    toImFolderData l = map (asBytes l) . filter hasValidExt <$> listDirectory (root </> l)++    asBytes :: String -> FilePath -> BL8.ByteString+    asBytes label fp = BL8.pack $ label ++ "<<.>>" ++ (root </> label </> fp)++    hasValidExt :: FilePath -> Bool+    hasValidExt fp = any (`isExtensionOf` fp) ["png", "jpeg", "bitmap", "tiff"] -- "gif", "tga"]++    -- For backwards compatability (only in filepath >= 1.4.2)+    isExtensionOf :: String -> FilePath -> Bool+    isExtensionOf ext@('.':_) = isSuffixOf ext . takeExtensions+    isExtensionOf ext         = isSuffixOf ('.':ext) . takeExtensions++ -- | Parse a ByteString into a list of Haskell values-readDataset ::-     ReadAs a      -- ^ How to parse the raw data string -  -> BL.ByteString -- ^ The data string+readDataset+  :: ReadAs a               -- ^ How to parse the raw data string+  -> BL.ByteString -- ^ The data strings   -> [a]-readDataset JSON bs =-  case JSON.decode bs of-    Just theData ->  theData-    Nothing -> error "failed to parse json"-readDataset (CSVRecord hhdr opts) bs =-  case decodeWith opts hhdr bs of-    Right theData -> V.toList theData-    Left err -> error err-readDataset (CSVNamedRecord opts) bs =-  case decodeByNameWith opts bs of-    Right (_,theData) -> V.toList theData-    Left err -> error err+readDataset ra bs =+  case safeReadDataset ra (bs:|[]) of+    Left e    -> error (show e)+    Right dat -> VB.toList dat -tempDirForDataset :: Dataset h a -> IO FilePath-tempDirForDataset ds =-  case temporaryDirectory ds of-    Nothing -> getTemporaryDirectory-    Just tdir -> return tdir +-- | Read a ByteString into a Haskell value+safeReadDataset :: (Vector v a, MonadThrow m) => ReadAs a -> NonEmpty BL.ByteString -> m (v a)+safeReadDataset ra bss = either throwString pure $+  case (ra, bss) of+    (JSON, bs:|[]) ->  V.fromList <$> JSON.eitherDecode' bs+    (CSVRecord hhdr opts, bs:|[]) -> V.convert <$> decodeWith opts hhdr bs+    (CSVNamedRecord opts, bs:|[]) -> V.convert . snd <$> decodeByNameWith opts bs+    (Parsable psr, bs:|[]) -> V.fromList <$> Atto.eitherResult (Atto.parse (Atto.many' psr) bs)++    (ImageFolder labels, _) -> do+      ds <- mapM (getImFiles labels) bss+      pure $ V.fromList (toList ds)++    _ -> Left $+      if length bss > 1+      then "Cannot parse more than one file for this data format"+      else "impossible: logic has changed, please file this issue on dh-core"++  where+    getImFiles :: NonEmpty String -> BL.ByteString -> Either String (String, FilePath)+    getImFiles labels bs' = Atto.eitherResult (Atto.parse (parseTaggedFile labels) bs')++    parseTaggedFile :: NonEmpty String -> Atto.Parser (String, FilePath)+    parseTaggedFile (l0:|ls) = do+      lbl <- Atto.choice $ Atto.string . B8.pack <$> (l0:ls)+      _ <- Atto.string "<<.>>"+      fp <- Atto.takeByteString+      pure (B8.unpack lbl, B8.unpack fp)+++-- | Get a temporary directory for a dataset.+tempDirForDataset :: Dataset a -> IO FilePath+tempDirForDataset = defaultTempDir . temporaryDirectory++-- | Reify an optional temporary directory+defaultTempDir :: Maybe FilePath -> IO FilePath+defaultTempDir = \case+  Nothing -> getTemporaryDirectory+  Just tdir -> return tdir++ -- | A 'Dataset' source can be either a URL (for remotely-hosted datasets) or the filepath of a local file.-data Source h = URL (Url h)-              | File FilePath+data Source+  = forall h . URL (Url h)+  | File FilePath+  | ImgFolder FilePath (NonEmpty String)  -- | A 'Dataset' contains metadata for loading, caching, preprocessing and parsing data.-data Dataset h a = Dataset-  { source :: Source h  -- ^ Dataset source+data Dataset a = Dataset+  { source :: Source    -- ^ Dataset source   , temporaryDirectory :: Maybe FilePath  -- ^ Temporary directory (optional)   , preProcess :: Maybe (BL.ByteString -> BL.ByteString)  -- ^ Dataset preprocessing function (optional)   , readAs :: ReadAs a@@ -132,6 +198,10 @@   JSON :: FromJSON a => ReadAs a   CSVRecord :: FromRecord a => HasHeader -> DecodeOptions -> ReadAs a   CSVNamedRecord :: FromNamedRecord a => DecodeOptions -> ReadAs a+  Parsable :: Atto.Parser a -> ReadAs a+  ImageFolder+    :: NonEmpty String           -- labels used as folders+    -> ReadAs (String, FilePath) -- FilePaths representing images on disk, Strings are labels  -- | A CSV record with default decoding options (i.e. columns are separated by commas) csvRecord :: FromRecord a => ReadAs a@@ -140,37 +210,37 @@ -- * Defining datasets  -- | Define a dataset from a source for a CSV file-csvDataset :: FromRecord a =>  Source h -> Dataset h a-csvDataset src = Dataset src Nothing Nothing csvRecord +csvDataset :: FromRecord a =>  Source   -> Dataset a+csvDataset src = Dataset src Nothing Nothing csvRecord  -- | Define a dataset from a source for a CSV file, skipping the header line-csvDatasetSkipHdr :: FromRecord a => Source h -> Dataset h a+csvDatasetSkipHdr :: FromRecord a => Source -> Dataset a csvDatasetSkipHdr src = Dataset src Nothing Nothing $ CSVRecord HasHeader defaultDecodeOptions   -- |Define a dataset from a source for a CSV file with a known header-csvHdrDataset :: FromNamedRecord a => Source h -> Dataset h a+csvHdrDataset :: FromNamedRecord a => Source -> Dataset a csvHdrDataset src = Dataset src Nothing Nothing $ CSVNamedRecord defaultDecodeOptions  -- |Define a dataset from a source for a CSV file with a known header and separator-csvHdrDatasetSep :: FromNamedRecord a => Char -> Source h -> Dataset h a+csvHdrDatasetSep :: FromNamedRecord a => Char -> Source -> Dataset a csvHdrDatasetSep sepc src    = Dataset src Nothing Nothing        $ CSVNamedRecord defaultDecodeOptions { decDelimiter = fromIntegral (ord sepc)} --- | Define a dataset from a source for a JSON file -jsonDataset :: FromJSON a => Source h -> Dataset h a+-- | Define a dataset from a source for a JSON file+jsonDataset :: FromJSON a => Source -> Dataset a jsonDataset src = Dataset src Nothing Nothing JSON   -- * Modifying datasets  -- | Include a preprocessing stage to a Dataset: each field in the raw data will be preprocessed with the given function.-withPreprocess :: (BL8.ByteString -> BL8.ByteString) -> Dataset h a -> Dataset h a+withPreprocess :: (BL8.ByteString -> BL8.ByteString) -> Dataset a -> Dataset a withPreprocess preF ds = ds { preProcess = Just preF}  -- | Include a temporary directory for caching the dataset after this has been downloaded one first time.-withTempDir :: FilePath -> Dataset h a -> Dataset h a+withTempDir :: FilePath -> Dataset a -> Dataset a withTempDir dir ds = ds { temporaryDirectory = Just dir }  @@ -220,21 +290,19 @@  -- | Filter out escaped double quotes from a field removeEscQuotes :: BL8.ByteString -> BL8.ByteString-removeEscQuotes = BL8.filter (/= '\"')    +removeEscQuotes = BL8.filter (/= '\"')  -- * Helper functions for data analysis  -- | Convert a fractional year to UTCTime with second-level precision (due to not taking into account leap seconds) yearToUTCTime :: Double -> UTCTime-yearToUTCTime yearDbl =-  let (yearn,yearFrac)  = properFraction yearDbl-      dayYearBegin = fromGregorian yearn 1 1-      (dayn, dayFrac) = properFraction $ yearFrac * (if isLeapYear yearn then 366 else 365)-      day = addDays dayn dayYearBegin-      dt = secondsToDiffTime $ round $ dayFrac * 86400-  in UTCTime day dt--+yearToUTCTime yearDbl = UTCTime day dt+  where+    (yearn,yearFrac)  = properFraction yearDbl+    dayYearBegin = fromGregorian yearn 1 1+    (dayn, dayFrac) = properFraction $ yearFrac * (if isLeapYear yearn then 366 else 365)+    day = addDays dayn dayYearBegin+    dt = secondsToDiffTime $ round $ dayFrac * 86400  -- * URLs 

@@ -36,6 +36,6 @@  instance FromRecord Abalone -abalone :: Dataset 'Http Abalone+abalone :: Dataset Abalone abalone = csvDataset $ URL $ umassMLDB /: "abalone" /: "abalone.data" 

@@ -95,10 +95,10 @@                         <*> v.!4 <*> v.!5 <*> (v.!6 <|> return Nothing) <*> v.!7 <*> v.!8                         <*> v.!9 <*> v.!10 <*> v.!11 <*> v.!12 <*> v.!13 <*> v.!14 -adult :: Dataset 'Http Adult+adult :: Dataset Adult adult = csvDataset $ URL $ umassMLDB /: "adult" /: "adult.data" -adultTestSet :: Dataset 'Http Adult+adultTestSet :: Dataset Adult adultTestSet = withPreprocess (dropLines 1) $ csvDataset $ URL $ umassMLDB /: "adult" /: "adult.test"  -- "http://mlr.cs.umass.edu/ml/machine-learning-databases/adult/adult.test"

@@ -58,7 +58,7 @@              intToBool 0 = False              intToBool 1 = True              intToBool _ = error "intToBool"-             -bostonHousing :: Dataset 'Http BostonHousing-bostonHousing = withPreprocess fixedWidthToCSV $ ++bostonHousing :: Dataset BostonHousing+bostonHousing = withPreprocess fixedWidthToCSV $             csvDataset $ URL $ umassMLDB /: "housing" /: "housing.data"

@@ -43,7 +43,7 @@ instance FromRecord BreastCancerEntry where   parseRecord v = BreastCancerEntry <$> v .! 0 <*> v .! 1 <*> v .! 2 <*> v .! 3 <*> v .! 4 <*> v .! 5 <*> (v .! 6 <|> return Nothing) <*> v .! 7  <*> v .! 8  <*> v .! 9  <*> (intToDiagnosis <$> v .! 10) -breastCancerDatabase :: Dataset 'Http BreastCancerEntry+breastCancerDatabase :: Dataset BreastCancerEntry breastCancerDatabase = csvDataset    $ URL $ umassMLDB /: "breast-cancer-wisconsin" /: "breast-cancer-wisconsin.data" @@ -90,10 +90,10 @@ instance FromRecord CellFeatures where   parseRecord v = CellFeatures <$> v .! 2 <*> v .! 3 <*> v .! 4 <*> v .! 5 <*> v .! 6  <*> v .! 7  <*> v .! 8  <*> v .! 9  <*> v .! 10 -diagnosticBreastCancer :: Dataset 'Http DiagnosticBreastCancer+diagnosticBreastCancer :: Dataset DiagnosticBreastCancer diagnosticBreastCancer = csvDataset    $ URL $ umassMLDB /: "breast-cancer-wisconsin" /: "wdbc.data" -prognosticBreastCancer :: Dataset 'Http PrognosticBreastCancer+prognosticBreastCancer :: Dataset PrognosticBreastCancer prognosticBreastCancer = csvDataset    $ URL $ umassMLDB /: "breast-cancer-wisconsin" /: "wpbc.data"

@@ -0,0 +1,175 @@+-------------------------------------------------------------------------------+-- |+-- Module    :  Numeric.Datasets.CIFAR10+-- License   :  BSD-3-Clause+-- Stability :  experimental+-- Portability: non-portable+--+-- The binary version contains the files data_batch_1.bin, data_batch_2.bin,+-- ..., data_batch_5.bin, as well as test_batch.bin. Each of these files is+-- formatted as follows:+--+--     <1 x label><3072 x pixel>+--     ...+--     <1 x label><3072 x pixel>+--+-- In other words, the first byte is the label of the first image, which is a+-- number in the range 0-9. The next 3072 bytes are the values of the pixels of+-- the image. The first 1024 bytes are the red channel values, the next 1024+-- the green, and the final 1024 the blue. The values are stored in row-major+-- order, so the first 32 bytes are the red channel values of the first row of+-- the image.+-------------------------------------------------------------------------------+{-# LANGUAGE CPP #-}+{-# LANGUAGE DataKinds #-}+{-# LANGUAGE DeriveAnyClass #-}+{-# LANGUAGE DeriveGeneric #-}+#if MIN_VERSION_GLASGOW_HASKELL(8,2,0,0)+{-# LANGUAGE DerivingStrategies #-}+#endif+{-# LANGUAGE GeneralizedNewtypeDeriving #-}+{-# LANGUAGE OverloadedStrings #-}+{-# LANGUAGE ScopedTypeVariables #-}+{-# LANGUAGE TupleSections #-}+#if MIN_VERSION_JuicyPixels(3,3,0)+#else+{-# LANGUAGE UndecidableInstances #-}+#endif+module Numeric.Datasets.CIFAR10+  ( Label(..)+  , CIFARImage(..), height, width, image, label+  , cifarURL+  , cifar10+  , parseCifar+  ) where++import Codec.Picture (Image, PixelRGB8(PixelRGB8), Pixel8, writePixel)+import Codec.Picture.Types (newMutableImage, freezeImage)+import Control.DeepSeq+import Control.Exception (throw)+import Control.Monad.ST (runST)+import Data.List (zipWith4)+import GHC.Generics (Generic)+import Network.HTTP.Req (Url, (/:), https, Scheme(..))+import qualified Codec.Archive.Tar as Tar+import qualified Codec.Compression.GZip as GZip+import qualified Data.Attoparsec.ByteString.Lazy as Atto+import qualified Data.ByteString as BS+import qualified Data.ByteString.Lazy as BL++import Numeric.Datasets++#if MIN_VERSION_JuicyPixels(3,3,0)+#else+import Foreign.Storable (Storable)+import qualified Codec.Picture as Compat+#endif+-- ========================================================================= --++-- | labels of CIFAR-10 dataset. Enum corresponds to binary-based uint8 label.+data Label+  = Airplane+  | Automobile+  | Bird+  | Cat+  | Deer+  | Dog+  | Frog+  | Horse+  | Ship+  | Truck+  deriving (Show, Eq, Generic, Bounded, Enum, Read, NFData)++#if MIN_VERSION_JuicyPixels(3,3,0)+#else+instance (Eq (Compat.PixelBaseComponent a), Storable (Compat.PixelBaseComponent a))+    => Eq (Image a) where+  a == b = Compat.imageWidth  a == Compat.imageWidth  b &&+           Compat.imageHeight a == Compat.imageHeight b &&+           Compat.imageData   a == Compat.imageData   b+#endif++-- | Data representation of a CIFAR image is a 32x32 RGB image+newtype CIFARImage = CIFARImage { getXY :: (Image PixelRGB8, Label) }+#if MIN_VERSION_GLASGOW_HASKELL(8,2,0,0)+  deriving newtype (Eq, NFData)+#else+  deriving (Eq, Generic, NFData)+#endif++instance Show CIFARImage where+  show im = "CIFARImage{Height: 32, Width: 32, Pixel: RGB8, Label: " ++ show (label im) ++ "}"++-- | height of 'CIFARImage'+height :: Int+height = 32++-- | width of 'CIFARImage'+width :: Int+width = 32++-- | extract the JuicyPixel representation from a CIFAR datapoint+image :: CIFARImage -> Image PixelRGB8+image = fst . getXY++-- | extract the label from a CIFAR datapoint+label :: CIFARImage -> Label+label = snd . getXY++-- | Source URL for cifar-10 and cifar-100+cifarURL :: Url 'Https+cifarURL = https "www.cs.toronto.edu" /: "~kriz"++-------------------------------------------------------------------------------+tempdir :: Maybe FilePath+tempdir = Nothing++-- | Define a dataset from a source for a CSV file+cifar10 :: Dataset CIFARImage+cifar10 = Dataset+  (URL $ cifarURL /: "cifar-10-binary.tar.gz")+  tempdir+  (Just unzipCifar)+  (Parsable parseCifar)++-- cifar10Sha256 = "c4a38c50a1bc5f3a1c5537f2155ab9d68f9f25eb1ed8d9ddda3db29a59bca1dd"++-- | parser for a cifar binary+parseCifar :: Atto.Parser CIFARImage+parseCifar = do+  lbl :: Label <- toEnum . fromIntegral <$> Atto.anyWord8+  rs :: [Pixel8] <- BS.unpack <$> Atto.take 1024+  gs :: [Pixel8] <- BS.unpack <$> Atto.take 1024+  bs :: [Pixel8] <- BS.unpack <$> Atto.take 1024+  let ipixels = zipWith4 (\ix r g b -> (ix, PixelRGB8 r g b)) ixs rs gs bs+  pure $ CIFARImage (newImage ipixels, lbl)+  where+    newImage :: [((Int, Int), PixelRGB8)] -> Image PixelRGB8+    newImage ipixels = runST $ do+      mim <- newMutableImage height width+      mapM_ (\((x, y), rgb) -> writePixel mim x y rgb) ipixels+      freezeImage mim++    ixs :: [(Int, Int)]+    ixs = concat $ zipWith (\(row::Int) cols -> (row,) <$> cols) [0..] (replicate height [0..width - 1])++-- | how to unpack the tarball+--+-- FIXME: this should be in MonadThrow+unzipCifar :: BL.ByteString -> BL.ByteString+unzipCifar zipbs = do+  either (throw . fst) (BL.concat) $ Tar.foldlEntries go [] entries+  where+    entries :: Tar.Entries Tar.FormatError+    entries = Tar.read $ GZip.decompress zipbs++    go :: [BL.ByteString] -> Tar.Entry -> [BL.ByteString]+    go agg entry =+      case Tar.entryContent entry of+        Tar.NormalFile ps fs ->+          -- Each file is exactly 30730000 bytes long. All other files are metadata. See https://www.cs.toronto.edu/~kriz/cifar.html+          if fs == 30730000+          then ps:agg+          else agg+        _ -> agg+

@@ -26,7 +26,7 @@  instance FromNamedRecord CO2 -maunaLoaCO2 :: Dataset 'Https CO2+maunaLoaCO2 :: Dataset CO2 maunaLoaCO2 = csvHdrDataset    $ URL $ https "raw.githubusercontent.com" /: "vincentarelbundock" /: "Rdatasets" /: "master" /: "csv" /: "datasets" /: "CO2.csv" 

@@ -66,6 +66,6 @@  instance FromRecord Car -car :: Dataset 'Http Car+car :: Dataset Car car = csvDataset           $ URL $ umassMLDB /: "car" /: "car.data"

@@ -28,6 +28,6 @@ instance FromRecord Coal where   parseRecord v = Coal <$> v .! 1 -coal :: Dataset 'Http Coal+coal :: Dataset Coal coal = let src = URL $ http "vincentarelbundock.github.io" /: "Rdatasets" /: "csv" /: "boot" /: "coal.csv"        in Dataset src Nothing Nothing $ CSVRecord HasHeader defaultDecodeOptions

@@ -41,6 +41,6 @@           where roundIt :: Double -> Integer                 roundIt = round -gapminder :: Dataset 'Https Gapminder+gapminder :: Dataset Gapminder gapminder = csvHdrDataset    $ URL $ https "raw.githubusercontent.com" /: "plotly" /: "datasets" /: "master" /: "gapminderDataFiveYear.csv"

@@ -0,0 +1,99 @@+{-# LANGUAGE FlexibleContexts #-}+{-# LANGUAGE GADTs #-}+{-# LANGUAGE LambdaCase #-}+{-# LANGUAGE OverloadedStrings #-}+{-# LANGUAGE ScopedTypeVariables #-}+{-# OPTIONS_GHC -Wno-unused-top-binds #-}+module Numeric.Datasets.Internal.Streaming+    ( streamDataset+    , streamByteString+    ) where++import Control.Exception.Safe (MonadThrow, Exception, throwString, throwM)+import Control.Monad.Error.Class (MonadError)+import Control.Monad.IO.Class (MonadIO, liftIO)+import Data.Attoparsec.ByteString.Lazy (Parser)+import Data.Maybe (fromMaybe)+import Data.Text.Encoding (decodeUtf8)+import Data.List.NonEmpty (NonEmpty, toList)+import Streaming (Stream, Of((:>)), lift)+import qualified Data.ByteString.Streaming as BS (fromLazy, ByteString, null)+import qualified Data.ByteString      as B' (pack)+import qualified Data.ByteString.Lazy as B (ByteString, concat)+import qualified Data.List as L (intercalate)+import qualified Data.Attoparsec.ByteString.Streaming as Atto (parse)+import qualified Data.Attoparsec.ByteString.Lazy as Atto (anyWord8)+import qualified Streaming         as S (hoist, unfold)+import qualified Streaming.Cassava as S (decodeWith, decodeByNameWith, CsvParseException)+import qualified Streaming.Prelude as S (print, maps)++import Numeric.Datasets+import Streaming.Instances ()++-- | Stream a dataset+streamDataset+  :: forall io a . (MonadThrow io, MonadIO io)+  => Dataset a+  -> Stream (Of a) io ()+streamDataset ds = do+  folder <- liftIO $ defaultTempDir (temporaryDirectory ds)+  files  <- liftIO $ getFileFromSource folder (source ds)+  streamByteString (readAs ds) (fromMaybe id (preProcess ds) <$> files)+++-- | Stream a ByteString into a Haskell value+streamByteString+  :: forall m a+  .  (MonadThrow m)+  => ReadAs a+  -> NonEmpty B.ByteString+  -> Stream (Of a) m ()+streamByteString ra bs = _streamDataset ra (BS.fromLazy $ B.concat $ toList bs)+++-- private function which uses the streaming interface of bytestring+_streamDataset+  :: forall mt a e+  .  (MonadThrow mt, Exception e)+  => (MonadError S.CsvParseException (Either e))+  => ReadAs a+  -> BS.ByteString (Either e) ()+  -> Stream (Of a) mt ()+_streamDataset ra bs =+  case ra of+    JSON -> lift $ throwString "Not implemented: JSON streaming"+    CSVRecord hhdr opts -> S.hoist either2Throw $ S.decodeWith opts hhdr bs+    CSVNamedRecord opts -> S.hoist either2Throw $ S.decodeByNameWith opts bs+    Parsable psr -> parseStream psr (S.hoist either2Throw bs)+    ImageFolder _ -> lift $ throwString "Not implemented: Image Folder streaming, use Dataloader"+  where+    either2Throw :: MonadThrow m => (forall x e . Exception e => Either e x -> m x)+    either2Throw = \case+      Left e -> throwM e+      Right r -> pure r+++-- private function to generate a stream from a parser+parseStream+  :: forall m a . MonadThrow m => Parser a -> BS.ByteString m () -> Stream (Of a) m ()+parseStream psr = S.unfold go+  where+    go :: BS.ByteString m () -> m (Either () (Of a (BS.ByteString m ())))+    go bs = do+      (eea, rest) <- Atto.parse psr bs+      BS.null rest >>= \(empty :> _) ->+        if empty+        then pure $ Left ()+        else case eea of+          Left (es, lst) -> throwString (lst ++ "\n" ++ L.intercalate "\n" es)+          Right a -> pure $ Right (a :> rest)+++-- make this a real test+test :: IO ()+test = do+  S.print $ S.maps render $ parseStream Atto.anyWord8 (BS.fromLazy "1")+  S.print $ S.maps render $ parseStream Atto.anyWord8 (BS.fromLazy "1 2 3 4")+  where+    render (a:>b) = (decodeUtf8 (B'.pack [a]) :> b)+

@@ -354,9 +354,9 @@   'u' -> Urban   'w' -> Waste   'd' -> Woods-  x -> error $ unwords ["Unexpected feature value :", show x]    +  x -> error $ unwords ["Unexpected feature value :", show x]  -mushroom :: Dataset 'Https MushroomEntry+mushroom :: Dataset MushroomEntry mushroom = csvDataset    $ URL $ uciMLDB /: "mushroom" /: "agaricus-lepiota.data"

@@ -30,7 +30,7 @@ instance FromRecord OldFaithful where   parseRecord v = OldFaithful <$> v .! 2 <*> v.! 1 -oldFaithful :: Dataset 'Https OldFaithful+oldFaithful :: Dataset OldFaithful oldFaithful   = let src = URL $ https "raw.githubusercontent.com" /: "vincentarelbundock" /: "Rdatasets" /: "master" /: "csv" /: "datasets" /: "faithful.csv"     in Dataset src Nothing Nothing $ CSVRecord HasHeader defaultDecodeOptions

@@ -27,6 +27,6 @@  instance FromNamedRecord Quake -quakes :: Dataset 'Http Quake+quakes :: Dataset Quake quakes = csvHdrDataset    $ URL $ http "vincentarelbundock.github.io" /: "Rdatasets" /: "csv" /: "datasets" /: "quakes.csv"

@@ -40,7 +40,7 @@                          m .: "dollars"  <*>                          m .: "pay" -states :: Dataset 'Http StateEdu+states :: Dataset StateEdu states = csvHdrDataset    $ URL $ http "vincentarelbundock.github.io" /: "Rdatasets" /: "csv" /: "car" /: "States.csv" 

@@ -30,7 +30,7 @@                          m .: "time" <*>                          m .: "value" -sunspots :: Dataset 'Http Sunspot+sunspots :: Dataset Sunspot sunspots = csvHdrDataset    $ URL $ http "vincentarelbundock.github.io" /: "Rdatasets" /: "csv" /: "datasets" /: "sunspot.month.csv" 

@@ -68,20 +68,20 @@ parseSex = \case   "female" -> Female   "male" -> Male-  x -> error $ unwords ["Unexpected feature value :", show x]      +  x -> error $ unwords ["Unexpected feature value :", show x]  parseBool :: String -> Bool parseBool = \case   "1" -> True   "0" -> False-  x -> error $ unwords ["Unexpected feature value :", show x]   +  x -> error $ unwords ["Unexpected feature value :", show x]  -- | The Titanic dataset, to be downloaded from <https://raw.githubusercontent.com/JackStat/6003Data/master/Titanic.txt>-titanicRemote :: Dataset 'Https TitanicEntry+titanicRemote :: Dataset TitanicEntry titanicRemote = withPreprocess removeEscQuotes $ csvHdrDatasetSep '\t' $ URL $ https "raw.githubusercontent.com" /: "JackStat" /: "6003Data" /: "master" /: "Titanic.txt"  -- | The Titanic dataset, parsed from a local copy-titanicLocal :: Dataset h TitanicEntry+titanicLocal :: Dataset TitanicEntry titanicLocal = withPreprocess removeEscQuotes $ csvHdrDatasetSep '\t' $ File "datafiles/titanic2_full.tsv"  

@@ -29,7 +29,7 @@                          (m .: "infant.mortality" <|> return Nothing) <*>                          (m .: "gdp" <|> return Nothing) -gdpMortalityUN :: Dataset 'Http GdpMortality+gdpMortalityUN :: Dataset GdpMortality gdpMortalityUN = csvHdrDataset    $ URL $ http "vincentarelbundock.github.io" /: "Rdatasets" /: "csv" /: "car" /: "UN.csv" 

@@ -31,6 +31,6 @@  instance FromNamedRecord Vocab -vocab :: Dataset 'Http Vocab+vocab :: Dataset Vocab vocab = csvHdrDataset    $ URL $ http "vincentarelbundock.github.io" /: "Rdatasets" /: "csv" /: "car" /: "Vocab.csv"

@@ -35,6 +35,6 @@  instance FromRecord Wine -wine :: Dataset 'Http Wine+wine :: Dataset Wine wine = withPreprocess fixAmericanDecimals $           csvDataset $ URL $ umassMLDB /: "wine" /: "wine.data"

@@ -48,7 +48,7 @@                          m .: "alcohol" <*>                          m .: "quality" -redWineQuality, whiteWineQuality :: Dataset 'Http WineQuality+redWineQuality, whiteWineQuality :: Dataset WineQuality redWineQuality = csvHdrDatasetSep ';'    $ URL $ umassMLDB /: "wine-quality" /: "winequality-red.csv" 

@@ -0,0 +1,21 @@+{-# LANGUAGE TypeSynonymInstances #-}+{-# LANGUAGE FlexibleInstances #-}+{-# OPTIONS_GHC -Wno-orphans #-}+module Streaming.Instances () where++import Streaming (Stream, lift)+import Data.Attoparsec.ByteString.Streaming (Errors)+import Control.Monad.Catch (MonadThrow(throwM), Exception)++instance (Functor f, MonadThrow m) => MonadThrow (Stream f m) where+  throwM e = lift (throwM e)++instance Exception Errors where++++++++

@@ -0,0 +1,15 @@+module Main where++import Test.Hspec+import Test.Hspec.QuickCheck+import Test.QuickCheck+++main :: IO ()+main = hspec spec++spec :: Spec+spec = do+  describe "Spec" $ +    it "works" $ do+      41 + 1 `shouldBe` 42