diff --git a/bench/Main.hs b/bench/Main.hs
new file mode 100644
--- /dev/null
+++ b/bench/Main.hs
@@ -0,0 +1,7 @@
+module Main where
+
+import qualified Numeric.Dataloader.Benchmark
+
+main :: IO ()
+main = Numeric.Dataloader.Benchmark.main
+
diff --git a/bench/Numeric/Dataloader/Benchmark.hs b/bench/Numeric/Dataloader/Benchmark.hs
new file mode 100644
--- /dev/null
+++ b/bench/Numeric/Dataloader/Benchmark.hs
@@ -0,0 +1,112 @@
+{-# OPTIONS_GHC -Wno-orphans  #-}
+{-# LANGUAGE DataKinds #-}
+{-# LANGUAGE LambdaCase #-}
+module Numeric.Dataloader.Benchmark where
+
+import Control.DeepSeq
+import Control.Concurrent
+import Numeric.Dataloader
+import Numeric.Datasets
+import Numeric.Datasets.Abalone
+import Streaming (Of)
+import qualified Streaming.Prelude as S
+import qualified System.Random.MWC as MWC
+
+-- ImageLoading bench
+import Numeric.Datasets.Internal.Streaming
+import Numeric.Datasets.CIFAR10
+import System.FilePath
+import System.Directory
+import Codec.Picture
+import Control.Exception.Safe
+import Text.Read
+import System.IO.Unsafe
+import qualified Data.List.NonEmpty   as NonEmpty
+
+import Criterion.Main
+
+instance NFData Abalone
+instance NFData Sex
+instance (NFData a, NFData b) => NFData (Of a b)
+
+
+mkDataloaderWithIx :: Dataset a -> IO (Dataloader a a)
+mkDataloaderWithIx ds = MWC.withSystemRandom $ \g -> do
+  ixl <- uniformIxline ds g
+  pure $ Dataloader 1 (Just ixl) ds id
+
+
+main :: IO ()
+main = do
+  dl <- mkDataloaderWithIx abalone
+  cifar10l <- cifar10ImageLoader
+  defaultMain
+    [ bgroup "Numeric.Dataloader"
+      [ bench "making an ixline" $ nfIO $ MWC.withSystemRandom (uniformIxline abalone)
+      , bgroup "testStream"
+        [ bench "with ixline" . nfIO $ foldStream (S.take 100 $ slow . stream $ dl)
+        , bench "no ixline"   . nfIO $ foldStream (S.take 100 $ slow . stream $ Dataloader 1 Nothing abalone id)
+        ]
+      , bench "cifar10 image folder" $ nfIO $ foldStream $ S.take 1000 $ stream cifar10l
+      , bench "cifar10 batch folder" $ nfIO $ foldStream $ S.take 1  $ batchStream (cifar10l { batchSize = 1000 })
+      ]
+    ]
+
+slow :: S.Stream (Of a) IO r -> S.Stream (Of a) IO r
+slow = S.mapM (\a -> threadDelay 2 >> pure a)
+
+foldStream :: Show a => S.Stream (Of a) IO () -> IO (Of [a] ())
+foldStream = S.fold (\memo a -> a:memo) [] id
+
+-------------------------------------------------------------------------------
+-- Image Folder loading
+
+-- may be required if you don't already have CIFAR10 loaded
+provision :: IO ()
+provision = do
+  xdgCache <- getXdgDirectory XdgCache "datasets-hs"
+  let imfolder = xdgCache </> "cifar-10-imagefolder"
+  createDirectoryIfMissing True imfolder
+
+  -- build the image folder
+  S.mapM_ (go imfolder) $ streamDataset cifar10
+ where
+  go :: FilePath -> CIFARImage -> IO ()
+  go cachefolder (CIFARImage (im, lbl)) = do
+    let labelfolder = cachefolder </> show lbl
+    createDirectoryIfMissing True labelfolder
+    ix <- length <$> listDirectory labelfolder
+    writePng (labelfolder </> (show lbl ++ "_" ++ show ix ++ ".png")) im
+
+
+-- | dataloading the image folder dataset
+cifar10ImageLoader :: IO (Dataloader (String, FilePath) CIFARImage)
+cifar10ImageLoader = do
+  xdgCache <- getXdgDirectory XdgCache "datasets-hs"
+  let imfolder = xdgCache </> "cifar-10-imagefolder"
+  pure $ Dataloader 1 Nothing (imgFolderDataset imfolder) load
+
+ where
+  labelFolders :: NonEmpty.NonEmpty String
+  labelFolders = show <$> NonEmpty.fromList [minBound..maxBound::Label]
+
+  imgFolderDataset :: FilePath -> Dataset (String, FilePath)
+  imgFolderDataset fp =
+    Dataset
+      (ImgFolder fp labelFolders)
+      Nothing
+      Nothing
+      (ImageFolder labelFolders)
+
+  load :: (String, FilePath) -> CIFARImage
+  load (str, fp) = CIFARImage (img, lbl)
+    where
+      lbl :: Label
+      lbl = either error id $ readEither str
+
+      img :: Image PixelRGB8
+      img = case unsafePerformIO (readPng fp) of
+        Left err -> error err
+        Right (ImageRGB8 i) -> i
+        Right _ -> error "decoded image was not rgb8"
+
diff --git a/changelog.md b/changelog.md
--- a/changelog.md
+++ b/changelog.md
@@ -1,3 +1,7 @@
+0.4
+	* Get rid of dependency on 'data-default' (introduced by previous versions of 'req')
+	
+	* Bump 'req' dependency to 2.0.0 
 0.3
 	* 'datasets' hosted within the DataHaskell/dh-core project
 
diff --git a/datasets.cabal b/datasets.cabal
--- a/datasets.cabal
+++ b/datasets.cabal
@@ -1,105 +1,156 @@
-Name:                datasets
-Version:             0.3.0
-Synopsis:            Classical data sets for statistics and machine learning
-Description:         Classical machine learning and statistics datasets from
-                     the UCI Machine Learning Repository and other sources.
-                     .
-                     The @datasets@ package defines two different kinds of datasets:
-                     .
-                     * small data sets which are directly (or indirectly with `file-embed`)
-                       embedded in the package as pure values and do not require network or IO to download
-                       the data set. This includes Iris, Anscombe and OldFaithful.
-                     .
-                     * other data sets which need to be fetched over the network with
-                     `Numeric.Datasets.getDataset` and are cached in a local temporary directory.
-                     .
-                     The @datafiles/@ directory of this package includes copies of a few famous datasets, such as Titanic, Nightingale and Michelson.
-                     .
-                     Example :
-                     .
-                     > import Numeric.Datasets (getDataset)
-                     > import Numeric.Datasets.Iris (iris)
-                     > import Numeric.Datasets.Abalone (abalone)
-                     >
-                     > main = do
-                     >   -- The Iris data set is embedded
-                     >   print (length iris)
-                     >   print (head iris)
-                     >   -- The Abalone dataset is fetched
-                     >   abas <- getDataset abalone
-                     >   print (length abas)
-                     >   print (head abas)
-
-License:             MIT
-License-file:        LICENSE
-Author:              Tom Nielsen <tanielsen@gmail.com>
-Maintainer:          Marco Zocca <ocramz fripost org>
-build-type:          Simple
-Cabal-Version: 	     >= 1.10
-homepage:            https://github.com/DataHaskell/dh-core
-bug-reports:         https://github.com/DataHaskell/dh-core/issues
-category:            Statistics, Machine Learning, Data Mining, Data
-Tested-With:         GHC == 7.10.2, GHC == 7.10.3, GHC == 8.0.1, GHC == 8.4.3
+cabal-version: >=1.10
+name: datasets
+version: 0.4.0
+license: MIT
+license-file: LICENSE
+maintainer: Marco Zocca <ocramz fripost org>
+author: Tom Nielsen <tanielsen@gmail.com>
+tested-with: ghc ==7.10.2 ghc ==7.10.3 ghc ==8.0.1 ghc ==8.4.3
+homepage: https://github.com/DataHaskell/dh-core
+bug-reports: https://github.com/DataHaskell/dh-core/issues
+synopsis: Classical data sets for statistics and machine learning
+description:
+    Classical machine learning and statistics datasets from
+    the UCI Machine Learning Repository and other sources.
+    .
+    The @datasets@ package defines two different kinds of datasets:
+    .
+    * small data sets which are directly (or indirectly with `file-embed`)
+    embedded in the package as pure values and do not require network or IO to download
+    the data set. This includes Iris, Anscombe and OldFaithful.
+    .
+    * other data sets which need to be fetched over the network with
+    `Numeric.Datasets.getDataset` and are cached in a local temporary directory.
+    .
+    The @datafiles/@ directory of this package includes copies of a few famous datasets, such as Titanic, Nightingale and Michelson.
+    .
+    Example :
+    .
+    > import Numeric.Datasets (getDataset)
+    > import Numeric.Datasets.Iris (iris)
+    > import Numeric.Datasets.Abalone (abalone)
+    >
+    > main = do
+    >   -- The Iris data set is embedded
+    >   print (length iris)
+    >   print (head iris)
+    >   -- The Abalone dataset is fetched
+    >   abas <- getDataset abalone
+    >   print (length abas)
+    >   print (head abas)
+category: Statistics, Machine Learning, Data Mining, Data
+build-type: Simple
 extra-source-files:
-                   changelog.md
-                   datafiles/iris.data
-                   datafiles/michelson.json
-                   datafiles/nightingale.json
-                   datafiles/titanic2_full.tsv
-                   datafiles/netflix/training/mv_0000001.txt
-                   datafiles/netflix/test/qualifying.txt
-                   datafiles/netflix/movies/movie_titles.txt
+    changelog.md
+    datafiles/iris.data
+    datafiles/michelson.json
+    datafiles/nightingale.json
+    datafiles/titanic2_full.tsv
+    datafiles/netflix/training/mv_0000001.txt
+    datafiles/netflix/test/qualifying.txt
+    datafiles/netflix/movies/movie_titles.txt
 
 source-repository head
-  type:     git
-  location: https://github.com/DataHaskell/dh-core/datasets
+    type: git
+    location: https://github.com/DataHaskell/dh-core/datasets
 
-Library
-   ghc-options:       -Wall -fno-warn-unused-imports
-   hs-source-dirs:    src
-   other-extensions: TemplateHaskell
-   default-language:  Haskell2010
+library
+    exposed-modules:
+        Numeric.Dataloader
+        Numeric.Datasets
+        Numeric.Datasets.Anscombe
+        Numeric.Datasets.BostonHousing
+        Numeric.Datasets.CIFAR10
+        Numeric.Datasets.OldFaithful
+        Numeric.Datasets.Abalone
+        Numeric.Datasets.Adult
+        Numeric.Datasets.BreastCancerWisconsin
+        Numeric.Datasets.Car
+        Numeric.Datasets.Coal
+        Numeric.Datasets.CO2
+        Numeric.Datasets.Gapminder
+        Numeric.Datasets.Iris
+        Numeric.Datasets.Internal.Streaming
+        Numeric.Datasets.Michelson
+        Numeric.Datasets.Mushroom
+        Numeric.Datasets.Nightingale
+        Numeric.Datasets.Quakes
+        Numeric.Datasets.States
+        Numeric.Datasets.Sunspots
+        Numeric.Datasets.Titanic
+        Numeric.Datasets.UN
+        Numeric.Datasets.Vocabulary
+        Numeric.Datasets.Wine
+        Numeric.Datasets.WineQuality
+        Numeric.Datasets.Netflix
+    hs-source-dirs: src
+    other-modules:
+        Streaming.Instances
+    default-language: Haskell2010
+    other-extensions: TemplateHaskell
+    ghc-options: -Wall -fno-warn-unused-imports
+    build-depends:
+        base >=4.6 && <5,
+        aeson >=1.4.2.0,
+        attoparsec >=0.13,
+        bytestring >=0.10.8.2,
+        cassava >=0.5.1.0,
+        deepseq >=1.4.4.0,
+        directory >=1.3.3.0,
+        exceptions >=0.10.0,
+        file-embed >=0.0.11,
+        filepath >=1.4.2.1,
+        hashable >=1.2.7.0,
+        JuicyPixels >=3.3.3,
+        microlens >=0.4.10,
+        mtl >=2.2.2,
+        mwc-random >=0.14.0.0,
+        parallel >=3.2.2.0,
+        req >=2.0.0,
+        safe-exceptions >=0.1.7.0,
+        streaming >=0.2.2.0,
+        streaming-attoparsec >=1.0.0,
+        streaming-bytestring >=0.1.6,
+        streaming-cassava >=0.1.0.1,
+        streaming-commons >=0.2.1.0,
+        stringsearch >=0.3.6.6,
+        tar >=0.5.1.0,
+        text >=1.2.3.1,
+        time >=1.8.0.2,
+        transformers >=0.5.5.0,
+        vector >=0.12.0.2,
+        safe-exceptions >=0.1.7.0,
+        zlib >=0.6.2
 
-   Exposed-modules:
-                   Numeric.Datasets
-                 , Numeric.Datasets.Anscombe
-                 , Numeric.Datasets.BostonHousing
-                 , Numeric.Datasets.OldFaithful
-                 , Numeric.Datasets.Abalone
-                 , Numeric.Datasets.Adult
-                 , Numeric.Datasets.BreastCancerWisconsin
-                 , Numeric.Datasets.Car
-                 , Numeric.Datasets.Coal
-                 , Numeric.Datasets.CO2
-                 , Numeric.Datasets.Gapminder
-                 , Numeric.Datasets.Iris
-                 , Numeric.Datasets.Michelson
-                 , Numeric.Datasets.Mushroom                 
-                 , Numeric.Datasets.Nightingale
-                 , Numeric.Datasets.Quakes
-                 , Numeric.Datasets.States
-                 , Numeric.Datasets.Sunspots
-                 , Numeric.Datasets.Titanic                 
-                 , Numeric.Datasets.UN
-                 , Numeric.Datasets.Vocabulary
-                 , Numeric.Datasets.Wine
-                 , Numeric.Datasets.WineQuality
-                 , Numeric.Datasets.Netflix
-   Build-depends:
-                 base          >= 4.6 && < 5
-               , aeson
-               , attoparsec    >= 0.13
-               , bytestring
-               , cassava
-               , data-default-class
-               , directory
-               , file-embed
-               , filepath
-               , hashable
-               , microlens
-               , req            >= 1.0.0
-               , stringsearch
-               , text
-               , time
-               , vector
-                 -- , wreq
+test-suite spec
+    type: exitcode-stdio-1.0
+    main-is: Spec.hs
+    hs-source-dirs: test
+    default-language: Haskell2010
+    default-extensions: OverloadedStrings
+    ghc-options: -threaded -rtsopts -with-rtsopts=-N
+    build-depends:
+        base >=4.6 && <5,
+        QuickCheck >=2.12.6.1,
+        hspec >=2.6.0
+
+benchmark bench
+    type: exitcode-stdio-1.0
+    main-is: Main.hs
+    hs-source-dirs: bench
+    other-modules:
+        Numeric.Dataloader.Benchmark
+    default-language: Haskell2010
+    ghc-options: -Wall -O2 -rtsopts -threaded
+    build-depends:
+        base >=4.6 && <5,
+        criterion >=1.5.3.0,
+        datasets -any,
+        deepseq >=1.4.4.0,
+        directory >=1.3.3.0,
+        filepath >=1.4.2.1,
+        JuicyPixels >=3.3.3,
+        mwc-random >=0.14.0.0,
+        req >=2.0.0,
+        safe-exceptions >=0.1.7.0,
+        streaming >=0.2.2.0
diff --git a/src/Numeric/Dataloader.hs b/src/Numeric/Dataloader.hs
new file mode 100644
--- /dev/null
+++ b/src/Numeric/Dataloader.hs
@@ -0,0 +1,115 @@
+-------------------------------------------------------------------------------
+-- |
+-- Module    :  Numeric.Dataloader
+-- Stability :  experimental
+-- Portability: non-portable
+--
+-- A Dataloader is an extension of a Dataset and is primarily intended for
+-- compute-intensive, batch loading interfaces. When used with ImageFolder
+-- representations of Datasets, it shuffles the order of files to be loaded
+-- and leverages the async library when possible.
+--
+-- Concurrent loading primarily takes place in 'batchStream'. 'stream' exists
+-- primarily to provide a unified API with training that is not batch-oriented.
+-------------------------------------------------------------------------------
+{-# LANGUAGE ScopedTypeVariables #-}
+module Numeric.Dataloader
+  ( Dataloader(..)
+  , uniformIxline
+
+  , stream
+  , batchStream
+  ) where
+
+import Control.Monad ((>=>))
+import Control.Monad.IO.Class (MonadIO, liftIO)
+import Data.Vector (Vector)
+import Streaming (Stream, Of(..))
+import System.Random.MWC (GenIO)
+import qualified Data.Vector          as V
+import qualified Streaming            as S
+import qualified Streaming.Prelude    as S
+import qualified System.Random.MWC.Distributions as MWC
+import Control.Exception.Safe (MonadThrow)
+import Streaming.Instances ()
+
+import Control.Parallel.Strategies
+
+import Numeric.Datasets
+
+-- * Configuring data loaders
+
+
+-- | Options for a data loading functions.
+data Dataloader a b = Dataloader
+  { batchSize :: Int                 -- ^ Batch size used with 'batchStream'.
+  , shuffle :: Maybe (Vector Int)    -- ^ Optional shuffle order (forces the dataset to be loaded in memory if it wasn't already).
+  , dataset :: Dataset a             -- ^ Dataset associated with the dataloader.
+  , transform :: a -> b              -- ^ Transformation associated with the dataloader which will be run in parallel. If using an
+                                     --   ImageFolder, this is where you would transform image filepaths to an image (or other
+                                     --   compute-optimized form). Additionally, this is where you should perform any
+                                     --   static normalization.
+  }
+
+
+-- | Generate a uniformly random index line from a dataset and a generator.
+uniformIxline
+  :: Dataset a
+  -> GenIO
+  -> IO (Vector Int)
+uniformIxline ds gen = do
+  len <- V.length <$> getDatavec ds
+  MWC.uniformPermutation len gen
+
+-- * Data loading functions
+
+
+-- | Stream a dataset in-memory, applying a transformation function.
+stream
+  :: (MonadThrow io, MonadIO io)
+  => Dataloader a b
+  -> Stream (Of b) io ()
+stream dl = S.maps (\(a:>b) -> (transform dl a `using` rpar) :> b) (sourceStream dl)
+
+
+-- | Stream batches of a dataset, concurrently processing each element
+--
+-- NOTE: Run with @-threaded -rtsopts@ to concurrently load data in-memory.
+batchStream
+  :: (MonadThrow io, MonadIO io, NFData b)
+  => Dataloader a b
+  -> Stream (Of [b]) io ()
+batchStream dl
+  = S.mapsM (S.toList >=> liftIO . firstOfM go)
+  $ S.chunksOf (batchSize dl)
+  $ sourceStream dl
+  where
+    go as = fmap (transform dl) as `usingIO` parList rdeepseq
+
+
+-- * helper functions (not for export)
+
+
+-- | Stream a dataset in-memory
+sourceStream
+  :: (MonadThrow io, MonadIO io)
+  => Dataloader a b
+  -> Stream (Of a) io ()
+sourceStream loader
+  = permute loader <$> getDatavec (dataset loader)
+  >>= S.each
+  where
+    -- Use a dataloader's shuffle order to return a permuted vector (or return the
+    -- identity vector).
+    permute :: Dataloader a b -> Vector a -> Vector a
+    permute loader va = maybe va (V.backpermute va) (shuffle loader)
+
+
+-- | Monadic, concrete version of Control.Arrow.first for @Of@
+firstOfM :: Monad m => (a -> m b) -> Of a c -> m (Of b c)
+firstOfM fm (a:>c) = do
+  b <- fm a
+  pure (b:>c)
+
+
+
diff --git a/src/Numeric/Datasets.hs b/src/Numeric/Datasets.hs
--- a/src/Numeric/Datasets.hs
+++ b/src/Numeric/Datasets.hs
@@ -2,12 +2,12 @@
 
 The @datasets@ package defines three different kinds of datasets:
 
-* Tiny datasets (up to a few tens of rows) are embedded as part of the library source code, as lists of values. 
+* Tiny datasets (up to a few tens of rows) are embedded as part of the library source code, as lists of values.
 
 * Small data sets are embedded indirectly (via @file-embed@)
   in the package as pure values and do not require IO
   to be downloaded (i.e. the data is loaded and parsed at compile time).
-  
+
 * Larger data sets which need to be fetched over the network
   and are cached in a local temporary directory for subsequent use.
 
@@ -21,107 +21,173 @@
 -}
 
 {-# LANGUAGE OverloadedStrings, GADTs, DataKinds #-}
+{-# LANGUAGE LambdaCase #-}
+{-# LANGUAGE ExistentialQuantification #-}
+{-# LANGUAGE ScopedTypeVariables #-}
 -- {-# OPTIONS_GHC -fno-warn-unused-imports #-}
 
-module Numeric.Datasets (getDataset, Dataset(..), Source(..),
-                         -- * Parsing datasets 
-                         readDataset, ReadAs(..), csvRecord,
+module Numeric.Datasets (getDataset, Dataset(..), Source(..), getDatavec, defaultTempDir, getFileFromSource,
+                         -- * Parsing datasets
+                         readDataset, safeReadDataset, ReadAs(..), csvRecord,
                         -- * Defining datasets
                         csvDataset, csvHdrDataset, csvHdrDatasetSep, csvDatasetSkipHdr,
                         jsonDataset,
                         -- ** Dataset options
-                        withPreprocess, withTempDir,                        
+                        withPreprocess, withTempDir,
                         -- ** Preprocessing functions
                         --
                         -- | These functions are to be used as first argument of 'withPreprocess' in order to improve the quality of the parser inputs.
                         dropLines, fixedWidthToCSV, removeEscQuotes, fixAmericanDecimals,
                         -- ** Helper functions
-                         parseReadField, parseDashToCamelField, 
-                         yearToUTCTime, 
+                         parseReadField, parseDashToCamelField,
+                         yearToUTCTime,
                         -- * Dataset source URLs
                         umassMLDB, uciMLDB) where
 
 import Data.Csv
-import System.FilePath
+import Data.Monoid
+import Data.Foldable
+import Data.List (isSuffixOf)
+import Data.List.NonEmpty (NonEmpty((:|)))
+import qualified Data.List.NonEmpty as NE
+import System.FilePath (takeExtensions, (</>))
 import System.Directory
 import Data.Hashable
 import Data.Monoid
 import qualified Data.ByteString.Lazy as BL
-import qualified Data.Vector as V
 import Data.Aeson as JSON
 import Control.Applicative
 import Data.Time
--- import qualified Network.Wreq as Wreq
-import Data.Default.Class (Default(..))
-import Network.HTTP.Req (req, runReq, Url, (/:), http, https, Scheme(..), LbsResponse, lbsResponse, responseBody, GET(..), NoReqBody(..), HttpMethod(..))
+import Network.HTTP.Req (req, runReq, Url, (/:), http, https, Scheme(..), LbsResponse, lbsResponse, responseBody, GET(..), NoReqBody(..), HttpMethod(..), defaultHttpConfig)
 -- import Lens.Micro ((^.))
 
+import Control.Exception.Safe
 import Data.Char (ord, toUpper)
 import Text.Read (readMaybe)
 import Data.Maybe (fromMaybe)
 import Data.ByteString.Char8 (unpack)
+import qualified Data.ByteString.Char8 as B8
 import qualified Data.ByteString.Lazy.Char8 as BL8
 import Data.ByteString.Lazy.Search (replace)
+import Control.Monad.IO.Class (MonadIO, liftIO)
+import Data.Vector.Generic (Vector)
+import qualified Data.Vector         as VB
+import qualified Data.Vector.Generic as V
+import qualified Data.Attoparsec.ByteString as Atto'
+import qualified Data.Attoparsec.ByteString.Lazy as Atto
 
 -- * Using datasets
 
--- | Load a dataset, using the system temporary directory as a cache
-getDataset :: Dataset h a -> IO [a]
-getDataset ds = do
-  dir <- tempDirForDataset ds
-  bs <- fmap (fromMaybe id $ preProcess ds) $ getFileFromSource dir $ source ds
-  return $ readDataset (readAs ds) bs
+-- | Load a dataset into memory
+getDataset :: (MonadThrow io, MonadIO io) => Dataset a -> io [a]
+getDataset ds = VB.toList <$> getDatavec ds
 
+-- | Load a dataset into memory as a vector
+getDatavec :: (MonadThrow io, MonadIO io, Vector v a) => Dataset a -> io (v a)
+getDatavec ds = liftIO $ do
+  folder <- tempDirForDataset ds
+  files <- getFileFromSource folder (source ds)
+  safeReadDataset (readAs ds) (fromMaybe id (preProcess ds) <$> files)
+
 -- | Get a ByteString from the specified Source
-getFileFromSource ::
-     FilePath  -- ^ Cache directory
-  -> Source h  
-  -> IO BL.ByteString
+getFileFromSource
+  :: FilePath  -- ^ Cache directory
+  -> Source
+  -> IO (NonEmpty BL.ByteString)
 getFileFromSource cacheDir (URL url) = do
   createDirectoryIfMissing True cacheDir
   let fnm = cacheDir </> "ds" <> show (hash $ show url)
   ex <- doesFileExist fnm
   if ex
-     then BL.readFile fnm
+     then (:|[]) <$> BL.readFile fnm
      else do
-       rsp <- runReq def $ req GET url NoReqBody lbsResponse mempty 
+       rsp <- runReq defaultHttpConfig $ req GET url NoReqBody lbsResponse mempty
        let bs = responseBody rsp
        BL.writeFile fnm bs
-       return bs
-getFileFromSource _ (File fnm) = 
-  BL.readFile fnm  
+       return (bs:|[])
+getFileFromSource _ (File fnm) = (:|[]) <$> BL.readFile fnm
+getFileFromSource _ (ImgFolder root labels) =
+  NE.fromList <$> foldrM allImFolderData [] labels
+  where
+    allImFolderData :: String -> [BL.ByteString] -> IO [BL.ByteString]
+    allImFolderData label agg = (agg ++) <$> toImFolderData label
 
+    toImFolderData :: String -> IO [BL.ByteString]
+    toImFolderData l = map (asBytes l) . filter hasValidExt <$> listDirectory (root </> l)
+
+    asBytes :: String -> FilePath -> BL8.ByteString
+    asBytes label fp = BL8.pack $ label ++ "<<.>>" ++ (root </> label </> fp)
+
+    hasValidExt :: FilePath -> Bool
+    hasValidExt fp = any (`isExtensionOf` fp) ["png", "jpeg", "bitmap", "tiff"] -- "gif", "tga"]
+
+    -- For backwards compatability (only in filepath >= 1.4.2)
+    isExtensionOf :: String -> FilePath -> Bool
+    isExtensionOf ext@('.':_) = isSuffixOf ext . takeExtensions
+    isExtensionOf ext         = isSuffixOf ('.':ext) . takeExtensions
+
+
 -- | Parse a ByteString into a list of Haskell values
-readDataset ::
-     ReadAs a      -- ^ How to parse the raw data string 
-  -> BL.ByteString -- ^ The data string
+readDataset
+  :: ReadAs a               -- ^ How to parse the raw data string
+  -> BL.ByteString -- ^ The data strings
   -> [a]
-readDataset JSON bs =
-  case JSON.decode bs of
-    Just theData ->  theData
-    Nothing -> error "failed to parse json"
-readDataset (CSVRecord hhdr opts) bs =
-  case decodeWith opts hhdr bs of
-    Right theData -> V.toList theData
-    Left err -> error err
-readDataset (CSVNamedRecord opts) bs =
-  case decodeByNameWith opts bs of
-    Right (_,theData) -> V.toList theData
-    Left err -> error err
+readDataset ra bs =
+  case safeReadDataset ra (bs:|[]) of
+    Left e    -> error (show e)
+    Right dat -> VB.toList dat
 
-tempDirForDataset :: Dataset h a -> IO FilePath
-tempDirForDataset ds =
-  case temporaryDirectory ds of
-    Nothing -> getTemporaryDirectory
-    Just tdir -> return tdir
 
+-- | Read a ByteString into a Haskell value
+safeReadDataset :: (Vector v a, MonadThrow m) => ReadAs a -> NonEmpty BL.ByteString -> m (v a)
+safeReadDataset ra bss = either throwString pure $
+  case (ra, bss) of
+    (JSON, bs:|[]) ->  V.fromList <$> JSON.eitherDecode' bs
+    (CSVRecord hhdr opts, bs:|[]) -> V.convert <$> decodeWith opts hhdr bs
+    (CSVNamedRecord opts, bs:|[]) -> V.convert . snd <$> decodeByNameWith opts bs
+    (Parsable psr, bs:|[]) -> V.fromList <$> Atto.eitherResult (Atto.parse (Atto.many' psr) bs)
+
+    (ImageFolder labels, _) -> do
+      ds <- mapM (getImFiles labels) bss
+      pure $ V.fromList (toList ds)
+
+    _ -> Left $
+      if length bss > 1
+      then "Cannot parse more than one file for this data format"
+      else "impossible: logic has changed, please file this issue on dh-core"
+
+  where
+    getImFiles :: NonEmpty String -> BL.ByteString -> Either String (String, FilePath)
+    getImFiles labels bs' = Atto.eitherResult (Atto.parse (parseTaggedFile labels) bs')
+
+    parseTaggedFile :: NonEmpty String -> Atto.Parser (String, FilePath)
+    parseTaggedFile (l0:|ls) = do
+      lbl <- Atto.choice $ Atto.string . B8.pack <$> (l0:ls)
+      _ <- Atto.string "<<.>>"
+      fp <- Atto.takeByteString
+      pure (B8.unpack lbl, B8.unpack fp)
+
+
+-- | Get a temporary directory for a dataset.
+tempDirForDataset :: Dataset a -> IO FilePath
+tempDirForDataset = defaultTempDir . temporaryDirectory
+
+-- | Reify an optional temporary directory
+defaultTempDir :: Maybe FilePath -> IO FilePath
+defaultTempDir = \case
+  Nothing -> getTemporaryDirectory
+  Just tdir -> return tdir
+
+
 -- | A 'Dataset' source can be either a URL (for remotely-hosted datasets) or the filepath of a local file.
-data Source h = URL (Url h)
-              | File FilePath
+data Source
+  = forall h . URL (Url h)
+  | File FilePath
+  | ImgFolder FilePath (NonEmpty String)
 
 -- | A 'Dataset' contains metadata for loading, caching, preprocessing and parsing data.
-data Dataset h a = Dataset
-  { source :: Source h  -- ^ Dataset source
+data Dataset a = Dataset
+  { source :: Source    -- ^ Dataset source
   , temporaryDirectory :: Maybe FilePath  -- ^ Temporary directory (optional)
   , preProcess :: Maybe (BL.ByteString -> BL.ByteString)  -- ^ Dataset preprocessing function (optional)
   , readAs :: ReadAs a
@@ -132,6 +198,10 @@
   JSON :: FromJSON a => ReadAs a
   CSVRecord :: FromRecord a => HasHeader -> DecodeOptions -> ReadAs a
   CSVNamedRecord :: FromNamedRecord a => DecodeOptions -> ReadAs a
+  Parsable :: Atto.Parser a -> ReadAs a
+  ImageFolder
+    :: NonEmpty String           -- labels used as folders
+    -> ReadAs (String, FilePath) -- FilePaths representing images on disk, Strings are labels
 
 -- | A CSV record with default decoding options (i.e. columns are separated by commas)
 csvRecord :: FromRecord a => ReadAs a
@@ -140,37 +210,37 @@
 -- * Defining datasets
 
 -- | Define a dataset from a source for a CSV file
-csvDataset :: FromRecord a =>  Source h -> Dataset h a
-csvDataset src = Dataset src Nothing Nothing csvRecord 
+csvDataset :: FromRecord a =>  Source   -> Dataset a
+csvDataset src = Dataset src Nothing Nothing csvRecord
 
 -- | Define a dataset from a source for a CSV file, skipping the header line
-csvDatasetSkipHdr :: FromRecord a => Source h -> Dataset h a
+csvDatasetSkipHdr :: FromRecord a => Source -> Dataset a
 csvDatasetSkipHdr src = Dataset src Nothing Nothing $ CSVRecord HasHeader defaultDecodeOptions
 
 
 -- |Define a dataset from a source for a CSV file with a known header
-csvHdrDataset :: FromNamedRecord a => Source h -> Dataset h a
+csvHdrDataset :: FromNamedRecord a => Source -> Dataset a
 csvHdrDataset src = Dataset src Nothing Nothing $ CSVNamedRecord defaultDecodeOptions
 
 -- |Define a dataset from a source for a CSV file with a known header and separator
-csvHdrDatasetSep :: FromNamedRecord a => Char -> Source h -> Dataset h a
+csvHdrDatasetSep :: FromNamedRecord a => Char -> Source -> Dataset a
 csvHdrDatasetSep sepc src
    = Dataset src Nothing Nothing
        $ CSVNamedRecord defaultDecodeOptions { decDelimiter = fromIntegral (ord sepc)}
 
--- | Define a dataset from a source for a JSON file 
-jsonDataset :: FromJSON a => Source h -> Dataset h a
+-- | Define a dataset from a source for a JSON file
+jsonDataset :: FromJSON a => Source -> Dataset a
 jsonDataset src = Dataset src Nothing Nothing JSON
 
 
 -- * Modifying datasets
 
 -- | Include a preprocessing stage to a Dataset: each field in the raw data will be preprocessed with the given function.
-withPreprocess :: (BL8.ByteString -> BL8.ByteString) -> Dataset h a -> Dataset h a
+withPreprocess :: (BL8.ByteString -> BL8.ByteString) -> Dataset a -> Dataset a
 withPreprocess preF ds = ds { preProcess = Just preF}
 
 -- | Include a temporary directory for caching the dataset after this has been downloaded one first time.
-withTempDir :: FilePath -> Dataset h a -> Dataset h a
+withTempDir :: FilePath -> Dataset a -> Dataset a
 withTempDir dir ds = ds { temporaryDirectory = Just dir }
 
 
@@ -220,21 +290,19 @@
 
 -- | Filter out escaped double quotes from a field
 removeEscQuotes :: BL8.ByteString -> BL8.ByteString
-removeEscQuotes = BL8.filter (/= '\"')    
+removeEscQuotes = BL8.filter (/= '\"')
 
 -- * Helper functions for data analysis
 
 -- | Convert a fractional year to UTCTime with second-level precision (due to not taking into account leap seconds)
 yearToUTCTime :: Double -> UTCTime
-yearToUTCTime yearDbl =
-  let (yearn,yearFrac)  = properFraction yearDbl
-      dayYearBegin = fromGregorian yearn 1 1
-      (dayn, dayFrac) = properFraction $ yearFrac * (if isLeapYear yearn then 366 else 365)
-      day = addDays dayn dayYearBegin
-      dt = secondsToDiffTime $ round $ dayFrac * 86400
-  in UTCTime day dt
-
-
+yearToUTCTime yearDbl = UTCTime day dt
+  where
+    (yearn,yearFrac)  = properFraction yearDbl
+    dayYearBegin = fromGregorian yearn 1 1
+    (dayn, dayFrac) = properFraction $ yearFrac * (if isLeapYear yearn then 366 else 365)
+    day = addDays dayn dayYearBegin
+    dt = secondsToDiffTime $ round $ dayFrac * 86400
 
 -- * URLs
 
diff --git a/src/Numeric/Datasets/Abalone.hs b/src/Numeric/Datasets/Abalone.hs
--- a/src/Numeric/Datasets/Abalone.hs
+++ b/src/Numeric/Datasets/Abalone.hs
@@ -36,6 +36,6 @@
 
 instance FromRecord Abalone
 
-abalone :: Dataset 'Http Abalone
+abalone :: Dataset Abalone
 abalone = csvDataset $ URL $ umassMLDB /: "abalone" /: "abalone.data"
 
diff --git a/src/Numeric/Datasets/Adult.hs b/src/Numeric/Datasets/Adult.hs
--- a/src/Numeric/Datasets/Adult.hs
+++ b/src/Numeric/Datasets/Adult.hs
@@ -95,10 +95,10 @@
                         <*> v.!4 <*> v.!5 <*> (v.!6 <|> return Nothing) <*> v.!7 <*> v.!8
                         <*> v.!9 <*> v.!10 <*> v.!11 <*> v.!12 <*> v.!13 <*> v.!14
 
-adult :: Dataset 'Http Adult
+adult :: Dataset Adult
 adult = csvDataset $ URL $ umassMLDB /: "adult" /: "adult.data"
 
-adultTestSet :: Dataset 'Http Adult
+adultTestSet :: Dataset Adult
 adultTestSet = withPreprocess (dropLines 1) $ csvDataset $ URL $ umassMLDB /: "adult" /: "adult.test"
 
 -- "http://mlr.cs.umass.edu/ml/machine-learning-databases/adult/adult.test"
diff --git a/src/Numeric/Datasets/BostonHousing.hs b/src/Numeric/Datasets/BostonHousing.hs
--- a/src/Numeric/Datasets/BostonHousing.hs
+++ b/src/Numeric/Datasets/BostonHousing.hs
@@ -58,7 +58,7 @@
              intToBool 0 = False
              intToBool 1 = True
              intToBool _ = error "intToBool"
-             
-bostonHousing :: Dataset 'Http BostonHousing
-bostonHousing = withPreprocess fixedWidthToCSV $ 
+
+bostonHousing :: Dataset BostonHousing
+bostonHousing = withPreprocess fixedWidthToCSV $
             csvDataset $ URL $ umassMLDB /: "housing" /: "housing.data"
diff --git a/src/Numeric/Datasets/BreastCancerWisconsin.hs b/src/Numeric/Datasets/BreastCancerWisconsin.hs
--- a/src/Numeric/Datasets/BreastCancerWisconsin.hs
+++ b/src/Numeric/Datasets/BreastCancerWisconsin.hs
@@ -43,7 +43,7 @@
 instance FromRecord BreastCancerEntry where
   parseRecord v = BreastCancerEntry <$> v .! 0 <*> v .! 1 <*> v .! 2 <*> v .! 3 <*> v .! 4 <*> v .! 5 <*> (v .! 6 <|> return Nothing) <*> v .! 7  <*> v .! 8  <*> v .! 9  <*> (intToDiagnosis <$> v .! 10)
 
-breastCancerDatabase :: Dataset 'Http BreastCancerEntry
+breastCancerDatabase :: Dataset BreastCancerEntry
 breastCancerDatabase = csvDataset
    $ URL $ umassMLDB /: "breast-cancer-wisconsin" /: "breast-cancer-wisconsin.data"
 
@@ -90,10 +90,10 @@
 instance FromRecord CellFeatures where
   parseRecord v = CellFeatures <$> v .! 2 <*> v .! 3 <*> v .! 4 <*> v .! 5 <*> v .! 6  <*> v .! 7  <*> v .! 8  <*> v .! 9  <*> v .! 10
 
-diagnosticBreastCancer :: Dataset 'Http DiagnosticBreastCancer
+diagnosticBreastCancer :: Dataset DiagnosticBreastCancer
 diagnosticBreastCancer = csvDataset
    $ URL $ umassMLDB /: "breast-cancer-wisconsin" /: "wdbc.data"
 
-prognosticBreastCancer :: Dataset 'Http PrognosticBreastCancer
+prognosticBreastCancer :: Dataset PrognosticBreastCancer
 prognosticBreastCancer = csvDataset
    $ URL $ umassMLDB /: "breast-cancer-wisconsin" /: "wpbc.data"
diff --git a/src/Numeric/Datasets/CIFAR10.hs b/src/Numeric/Datasets/CIFAR10.hs
new file mode 100644
--- /dev/null
+++ b/src/Numeric/Datasets/CIFAR10.hs
@@ -0,0 +1,175 @@
+-------------------------------------------------------------------------------
+-- |
+-- Module    :  Numeric.Datasets.CIFAR10
+-- License   :  BSD-3-Clause
+-- Stability :  experimental
+-- Portability: non-portable
+--
+-- The binary version contains the files data_batch_1.bin, data_batch_2.bin,
+-- ..., data_batch_5.bin, as well as test_batch.bin. Each of these files is
+-- formatted as follows:
+--
+--     <1 x label><3072 x pixel>
+--     ...
+--     <1 x label><3072 x pixel>
+--
+-- In other words, the first byte is the label of the first image, which is a
+-- number in the range 0-9. The next 3072 bytes are the values of the pixels of
+-- the image. The first 1024 bytes are the red channel values, the next 1024
+-- the green, and the final 1024 the blue. The values are stored in row-major
+-- order, so the first 32 bytes are the red channel values of the first row of
+-- the image.
+-------------------------------------------------------------------------------
+{-# LANGUAGE CPP #-}
+{-# LANGUAGE DataKinds #-}
+{-# LANGUAGE DeriveAnyClass #-}
+{-# LANGUAGE DeriveGeneric #-}
+#if MIN_VERSION_GLASGOW_HASKELL(8,2,0,0)
+{-# LANGUAGE DerivingStrategies #-}
+#endif
+{-# LANGUAGE GeneralizedNewtypeDeriving #-}
+{-# LANGUAGE OverloadedStrings #-}
+{-# LANGUAGE ScopedTypeVariables #-}
+{-# LANGUAGE TupleSections #-}
+#if MIN_VERSION_JuicyPixels(3,3,0)
+#else
+{-# LANGUAGE UndecidableInstances #-}
+#endif
+module Numeric.Datasets.CIFAR10
+  ( Label(..)
+  , CIFARImage(..), height, width, image, label
+  , cifarURL
+  , cifar10
+  , parseCifar
+  ) where
+
+import Codec.Picture (Image, PixelRGB8(PixelRGB8), Pixel8, writePixel)
+import Codec.Picture.Types (newMutableImage, freezeImage)
+import Control.DeepSeq
+import Control.Exception (throw)
+import Control.Monad.ST (runST)
+import Data.List (zipWith4)
+import GHC.Generics (Generic)
+import Network.HTTP.Req (Url, (/:), https, Scheme(..))
+import qualified Codec.Archive.Tar as Tar
+import qualified Codec.Compression.GZip as GZip
+import qualified Data.Attoparsec.ByteString.Lazy as Atto
+import qualified Data.ByteString as BS
+import qualified Data.ByteString.Lazy as BL
+
+import Numeric.Datasets
+
+#if MIN_VERSION_JuicyPixels(3,3,0)
+#else
+import Foreign.Storable (Storable)
+import qualified Codec.Picture as Compat
+#endif
+-- ========================================================================= --
+
+-- | labels of CIFAR-10 dataset. Enum corresponds to binary-based uint8 label.
+data Label
+  = Airplane
+  | Automobile
+  | Bird
+  | Cat
+  | Deer
+  | Dog
+  | Frog
+  | Horse
+  | Ship
+  | Truck
+  deriving (Show, Eq, Generic, Bounded, Enum, Read, NFData)
+
+#if MIN_VERSION_JuicyPixels(3,3,0)
+#else
+instance (Eq (Compat.PixelBaseComponent a), Storable (Compat.PixelBaseComponent a))
+    => Eq (Image a) where
+  a == b = Compat.imageWidth  a == Compat.imageWidth  b &&
+           Compat.imageHeight a == Compat.imageHeight b &&
+           Compat.imageData   a == Compat.imageData   b
+#endif
+
+-- | Data representation of a CIFAR image is a 32x32 RGB image
+newtype CIFARImage = CIFARImage { getXY :: (Image PixelRGB8, Label) }
+#if MIN_VERSION_GLASGOW_HASKELL(8,2,0,0)
+  deriving newtype (Eq, NFData)
+#else
+  deriving (Eq, Generic, NFData)
+#endif
+
+instance Show CIFARImage where
+  show im = "CIFARImage{Height: 32, Width: 32, Pixel: RGB8, Label: " ++ show (label im) ++ "}"
+
+-- | height of 'CIFARImage'
+height :: Int
+height = 32
+
+-- | width of 'CIFARImage'
+width :: Int
+width = 32
+
+-- | extract the JuicyPixel representation from a CIFAR datapoint
+image :: CIFARImage -> Image PixelRGB8
+image = fst . getXY
+
+-- | extract the label from a CIFAR datapoint
+label :: CIFARImage -> Label
+label = snd . getXY
+
+-- | Source URL for cifar-10 and cifar-100
+cifarURL :: Url 'Https
+cifarURL = https "www.cs.toronto.edu" /: "~kriz"
+
+-------------------------------------------------------------------------------
+tempdir :: Maybe FilePath
+tempdir = Nothing
+
+-- | Define a dataset from a source for a CSV file
+cifar10 :: Dataset CIFARImage
+cifar10 = Dataset
+  (URL $ cifarURL /: "cifar-10-binary.tar.gz")
+  tempdir
+  (Just unzipCifar)
+  (Parsable parseCifar)
+
+-- cifar10Sha256 = "c4a38c50a1bc5f3a1c5537f2155ab9d68f9f25eb1ed8d9ddda3db29a59bca1dd"
+
+-- | parser for a cifar binary
+parseCifar :: Atto.Parser CIFARImage
+parseCifar = do
+  lbl :: Label <- toEnum . fromIntegral <$> Atto.anyWord8
+  rs :: [Pixel8] <- BS.unpack <$> Atto.take 1024
+  gs :: [Pixel8] <- BS.unpack <$> Atto.take 1024
+  bs :: [Pixel8] <- BS.unpack <$> Atto.take 1024
+  let ipixels = zipWith4 (\ix r g b -> (ix, PixelRGB8 r g b)) ixs rs gs bs
+  pure $ CIFARImage (newImage ipixels, lbl)
+  where
+    newImage :: [((Int, Int), PixelRGB8)] -> Image PixelRGB8
+    newImage ipixels = runST $ do
+      mim <- newMutableImage height width
+      mapM_ (\((x, y), rgb) -> writePixel mim x y rgb) ipixels
+      freezeImage mim
+
+    ixs :: [(Int, Int)]
+    ixs = concat $ zipWith (\(row::Int) cols -> (row,) <$> cols) [0..] (replicate height [0..width - 1])
+
+-- | how to unpack the tarball
+--
+-- FIXME: this should be in MonadThrow
+unzipCifar :: BL.ByteString -> BL.ByteString
+unzipCifar zipbs = do
+  either (throw . fst) (BL.concat) $ Tar.foldlEntries go [] entries
+  where
+    entries :: Tar.Entries Tar.FormatError
+    entries = Tar.read $ GZip.decompress zipbs
+
+    go :: [BL.ByteString] -> Tar.Entry -> [BL.ByteString]
+    go agg entry =
+      case Tar.entryContent entry of
+        Tar.NormalFile ps fs ->
+          -- Each file is exactly 30730000 bytes long. All other files are metadata. See https://www.cs.toronto.edu/~kriz/cifar.html
+          if fs == 30730000
+          then ps:agg
+          else agg
+        _ -> agg
+
diff --git a/src/Numeric/Datasets/CO2.hs b/src/Numeric/Datasets/CO2.hs
--- a/src/Numeric/Datasets/CO2.hs
+++ b/src/Numeric/Datasets/CO2.hs
@@ -26,7 +26,7 @@
 
 instance FromNamedRecord CO2
 
-maunaLoaCO2 :: Dataset 'Https CO2
+maunaLoaCO2 :: Dataset CO2
 maunaLoaCO2 = csvHdrDataset
    $ URL $ https "raw.githubusercontent.com" /: "vincentarelbundock" /: "Rdatasets" /: "master" /: "csv" /: "datasets" /: "CO2.csv"
 
diff --git a/src/Numeric/Datasets/Car.hs b/src/Numeric/Datasets/Car.hs
--- a/src/Numeric/Datasets/Car.hs
+++ b/src/Numeric/Datasets/Car.hs
@@ -66,6 +66,6 @@
 
 instance FromRecord Car
 
-car :: Dataset 'Http Car
+car :: Dataset Car
 car = csvDataset
           $ URL $ umassMLDB /: "car" /: "car.data"
diff --git a/src/Numeric/Datasets/Coal.hs b/src/Numeric/Datasets/Coal.hs
--- a/src/Numeric/Datasets/Coal.hs
+++ b/src/Numeric/Datasets/Coal.hs
@@ -28,6 +28,6 @@
 instance FromRecord Coal where
   parseRecord v = Coal <$> v .! 1
 
-coal :: Dataset 'Http Coal
+coal :: Dataset Coal
 coal = let src = URL $ http "vincentarelbundock.github.io" /: "Rdatasets" /: "csv" /: "boot" /: "coal.csv"
        in Dataset src Nothing Nothing $ CSVRecord HasHeader defaultDecodeOptions
diff --git a/src/Numeric/Datasets/Gapminder.hs b/src/Numeric/Datasets/Gapminder.hs
--- a/src/Numeric/Datasets/Gapminder.hs
+++ b/src/Numeric/Datasets/Gapminder.hs
@@ -41,6 +41,6 @@
           where roundIt :: Double -> Integer
                 roundIt = round
 
-gapminder :: Dataset 'Https Gapminder
+gapminder :: Dataset Gapminder
 gapminder = csvHdrDataset
    $ URL $ https "raw.githubusercontent.com" /: "plotly" /: "datasets" /: "master" /: "gapminderDataFiveYear.csv"
diff --git a/src/Numeric/Datasets/Internal/Streaming.hs b/src/Numeric/Datasets/Internal/Streaming.hs
new file mode 100644
--- /dev/null
+++ b/src/Numeric/Datasets/Internal/Streaming.hs
@@ -0,0 +1,99 @@
+{-# LANGUAGE FlexibleContexts #-}
+{-# LANGUAGE GADTs #-}
+{-# LANGUAGE LambdaCase #-}
+{-# LANGUAGE OverloadedStrings #-}
+{-# LANGUAGE ScopedTypeVariables #-}
+{-# OPTIONS_GHC -Wno-unused-top-binds #-}
+module Numeric.Datasets.Internal.Streaming
+    ( streamDataset
+    , streamByteString
+    ) where
+
+import Control.Exception.Safe (MonadThrow, Exception, throwString, throwM)
+import Control.Monad.Error.Class (MonadError)
+import Control.Monad.IO.Class (MonadIO, liftIO)
+import Data.Attoparsec.ByteString.Lazy (Parser)
+import Data.Maybe (fromMaybe)
+import Data.Text.Encoding (decodeUtf8)
+import Data.List.NonEmpty (NonEmpty, toList)
+import Streaming (Stream, Of((:>)), lift)
+import qualified Data.ByteString.Streaming as BS (fromLazy, ByteString, null)
+import qualified Data.ByteString      as B' (pack)
+import qualified Data.ByteString.Lazy as B (ByteString, concat)
+import qualified Data.List as L (intercalate)
+import qualified Data.Attoparsec.ByteString.Streaming as Atto (parse)
+import qualified Data.Attoparsec.ByteString.Lazy as Atto (anyWord8)
+import qualified Streaming         as S (hoist, unfold)
+import qualified Streaming.Cassava as S (decodeWith, decodeByNameWith, CsvParseException)
+import qualified Streaming.Prelude as S (print, maps)
+
+import Numeric.Datasets
+import Streaming.Instances ()
+
+-- | Stream a dataset
+streamDataset
+  :: forall io a . (MonadThrow io, MonadIO io)
+  => Dataset a
+  -> Stream (Of a) io ()
+streamDataset ds = do
+  folder <- liftIO $ defaultTempDir (temporaryDirectory ds)
+  files  <- liftIO $ getFileFromSource folder (source ds)
+  streamByteString (readAs ds) (fromMaybe id (preProcess ds) <$> files)
+
+
+-- | Stream a ByteString into a Haskell value
+streamByteString
+  :: forall m a
+  .  (MonadThrow m)
+  => ReadAs a
+  -> NonEmpty B.ByteString
+  -> Stream (Of a) m ()
+streamByteString ra bs = _streamDataset ra (BS.fromLazy $ B.concat $ toList bs)
+
+
+-- private function which uses the streaming interface of bytestring
+_streamDataset
+  :: forall mt a e
+  .  (MonadThrow mt, Exception e)
+  => (MonadError S.CsvParseException (Either e))
+  => ReadAs a
+  -> BS.ByteString (Either e) ()
+  -> Stream (Of a) mt ()
+_streamDataset ra bs =
+  case ra of
+    JSON -> lift $ throwString "Not implemented: JSON streaming"
+    CSVRecord hhdr opts -> S.hoist either2Throw $ S.decodeWith opts hhdr bs
+    CSVNamedRecord opts -> S.hoist either2Throw $ S.decodeByNameWith opts bs
+    Parsable psr -> parseStream psr (S.hoist either2Throw bs)
+    ImageFolder _ -> lift $ throwString "Not implemented: Image Folder streaming, use Dataloader"
+  where
+    either2Throw :: MonadThrow m => (forall x e . Exception e => Either e x -> m x)
+    either2Throw = \case
+      Left e -> throwM e
+      Right r -> pure r
+
+
+-- private function to generate a stream from a parser
+parseStream
+  :: forall m a . MonadThrow m => Parser a -> BS.ByteString m () -> Stream (Of a) m ()
+parseStream psr = S.unfold go
+  where
+    go :: BS.ByteString m () -> m (Either () (Of a (BS.ByteString m ())))
+    go bs = do
+      (eea, rest) <- Atto.parse psr bs
+      BS.null rest >>= \(empty :> _) ->
+        if empty
+        then pure $ Left ()
+        else case eea of
+          Left (es, lst) -> throwString (lst ++ "\n" ++ L.intercalate "\n" es)
+          Right a -> pure $ Right (a :> rest)
+
+
+-- make this a real test
+test :: IO ()
+test = do
+  S.print $ S.maps render $ parseStream Atto.anyWord8 (BS.fromLazy "1")
+  S.print $ S.maps render $ parseStream Atto.anyWord8 (BS.fromLazy "1 2 3 4")
+  where
+    render (a:>b) = (decodeUtf8 (B'.pack [a]) :> b)
+
diff --git a/src/Numeric/Datasets/Mushroom.hs b/src/Numeric/Datasets/Mushroom.hs
--- a/src/Numeric/Datasets/Mushroom.hs
+++ b/src/Numeric/Datasets/Mushroom.hs
@@ -354,9 +354,9 @@
   'u' -> Urban
   'w' -> Waste
   'd' -> Woods
-  x -> error $ unwords ["Unexpected feature value :", show x]    
+  x -> error $ unwords ["Unexpected feature value :", show x]
 
 
-mushroom :: Dataset 'Https MushroomEntry
+mushroom :: Dataset MushroomEntry
 mushroom = csvDataset
    $ URL $ uciMLDB /: "mushroom" /: "agaricus-lepiota.data"
diff --git a/src/Numeric/Datasets/OldFaithful.hs b/src/Numeric/Datasets/OldFaithful.hs
--- a/src/Numeric/Datasets/OldFaithful.hs
+++ b/src/Numeric/Datasets/OldFaithful.hs
@@ -30,7 +30,7 @@
 instance FromRecord OldFaithful where
   parseRecord v = OldFaithful <$> v .! 2 <*> v.! 1
 
-oldFaithful :: Dataset 'Https OldFaithful
+oldFaithful :: Dataset OldFaithful
 oldFaithful
   = let src = URL $ https "raw.githubusercontent.com" /: "vincentarelbundock" /: "Rdatasets" /: "master" /: "csv" /: "datasets" /: "faithful.csv"
     in Dataset src Nothing Nothing $ CSVRecord HasHeader defaultDecodeOptions
diff --git a/src/Numeric/Datasets/Quakes.hs b/src/Numeric/Datasets/Quakes.hs
--- a/src/Numeric/Datasets/Quakes.hs
+++ b/src/Numeric/Datasets/Quakes.hs
@@ -27,6 +27,6 @@
 
 instance FromNamedRecord Quake
 
-quakes :: Dataset 'Http Quake
+quakes :: Dataset Quake
 quakes = csvHdrDataset
    $ URL $ http "vincentarelbundock.github.io" /: "Rdatasets" /: "csv" /: "datasets" /: "quakes.csv"
diff --git a/src/Numeric/Datasets/States.hs b/src/Numeric/Datasets/States.hs
--- a/src/Numeric/Datasets/States.hs
+++ b/src/Numeric/Datasets/States.hs
@@ -40,7 +40,7 @@
                          m .: "dollars"  <*>
                          m .: "pay"
 
-states :: Dataset 'Http StateEdu
+states :: Dataset StateEdu
 states = csvHdrDataset
    $ URL $ http "vincentarelbundock.github.io" /: "Rdatasets" /: "csv" /: "car" /: "States.csv"
 
diff --git a/src/Numeric/Datasets/Sunspots.hs b/src/Numeric/Datasets/Sunspots.hs
--- a/src/Numeric/Datasets/Sunspots.hs
+++ b/src/Numeric/Datasets/Sunspots.hs
@@ -30,7 +30,7 @@
                          m .: "time" <*>
                          m .: "value"
 
-sunspots :: Dataset 'Http Sunspot
+sunspots :: Dataset Sunspot
 sunspots = csvHdrDataset
    $ URL $ http "vincentarelbundock.github.io" /: "Rdatasets" /: "csv" /: "datasets" /: "sunspot.month.csv"
 
diff --git a/src/Numeric/Datasets/Titanic.hs b/src/Numeric/Datasets/Titanic.hs
--- a/src/Numeric/Datasets/Titanic.hs
+++ b/src/Numeric/Datasets/Titanic.hs
@@ -68,20 +68,20 @@
 parseSex = \case
   "female" -> Female
   "male" -> Male
-  x -> error $ unwords ["Unexpected feature value :", show x]      
+  x -> error $ unwords ["Unexpected feature value :", show x]
 
 parseBool :: String -> Bool
 parseBool = \case
   "1" -> True
   "0" -> False
-  x -> error $ unwords ["Unexpected feature value :", show x]   
+  x -> error $ unwords ["Unexpected feature value :", show x]
 
 -- | The Titanic dataset, to be downloaded from <https://raw.githubusercontent.com/JackStat/6003Data/master/Titanic.txt>
-titanicRemote :: Dataset 'Https TitanicEntry
+titanicRemote :: Dataset TitanicEntry
 titanicRemote = withPreprocess removeEscQuotes $ csvHdrDatasetSep '\t' $ URL $ https "raw.githubusercontent.com" /: "JackStat" /: "6003Data" /: "master" /: "Titanic.txt"
 
 -- | The Titanic dataset, parsed from a local copy
-titanicLocal :: Dataset h TitanicEntry
+titanicLocal :: Dataset TitanicEntry
 titanicLocal = withPreprocess removeEscQuotes $ csvHdrDatasetSep '\t' $ File "datafiles/titanic2_full.tsv"
 
 
diff --git a/src/Numeric/Datasets/UN.hs b/src/Numeric/Datasets/UN.hs
--- a/src/Numeric/Datasets/UN.hs
+++ b/src/Numeric/Datasets/UN.hs
@@ -29,7 +29,7 @@
                          (m .: "infant.mortality" <|> return Nothing) <*>
                          (m .: "gdp" <|> return Nothing)
 
-gdpMortalityUN :: Dataset 'Http GdpMortality
+gdpMortalityUN :: Dataset GdpMortality
 gdpMortalityUN = csvHdrDataset
    $ URL $ http "vincentarelbundock.github.io" /: "Rdatasets" /: "csv" /: "car" /: "UN.csv"
 
diff --git a/src/Numeric/Datasets/Vocabulary.hs b/src/Numeric/Datasets/Vocabulary.hs
--- a/src/Numeric/Datasets/Vocabulary.hs
+++ b/src/Numeric/Datasets/Vocabulary.hs
@@ -31,6 +31,6 @@
 
 instance FromNamedRecord Vocab
 
-vocab :: Dataset 'Http Vocab
+vocab :: Dataset Vocab
 vocab = csvHdrDataset
    $ URL $ http "vincentarelbundock.github.io" /: "Rdatasets" /: "csv" /: "car" /: "Vocab.csv"
diff --git a/src/Numeric/Datasets/Wine.hs b/src/Numeric/Datasets/Wine.hs
--- a/src/Numeric/Datasets/Wine.hs
+++ b/src/Numeric/Datasets/Wine.hs
@@ -35,6 +35,6 @@
 
 instance FromRecord Wine
 
-wine :: Dataset 'Http Wine
+wine :: Dataset Wine
 wine = withPreprocess fixAmericanDecimals $
           csvDataset $ URL $ umassMLDB /: "wine" /: "wine.data"
diff --git a/src/Numeric/Datasets/WineQuality.hs b/src/Numeric/Datasets/WineQuality.hs
--- a/src/Numeric/Datasets/WineQuality.hs
+++ b/src/Numeric/Datasets/WineQuality.hs
@@ -48,7 +48,7 @@
                          m .: "alcohol" <*>
                          m .: "quality"
 
-redWineQuality, whiteWineQuality :: Dataset 'Http WineQuality
+redWineQuality, whiteWineQuality :: Dataset WineQuality
 redWineQuality = csvHdrDatasetSep ';'
    $ URL $ umassMLDB /: "wine-quality" /: "winequality-red.csv"
 
diff --git a/src/Streaming/Instances.hs b/src/Streaming/Instances.hs
new file mode 100644
--- /dev/null
+++ b/src/Streaming/Instances.hs
@@ -0,0 +1,21 @@
+{-# LANGUAGE TypeSynonymInstances #-}
+{-# LANGUAGE FlexibleInstances #-}
+{-# OPTIONS_GHC -Wno-orphans #-}
+module Streaming.Instances () where
+
+import Streaming (Stream, lift)
+import Data.Attoparsec.ByteString.Streaming (Errors)
+import Control.Monad.Catch (MonadThrow(throwM), Exception)
+
+instance (Functor f, MonadThrow m) => MonadThrow (Stream f m) where
+  throwM e = lift (throwM e)
+
+instance Exception Errors where
+
+
+
+
+
+
+
+
diff --git a/test/Spec.hs b/test/Spec.hs
new file mode 100644
--- /dev/null
+++ b/test/Spec.hs
@@ -0,0 +1,15 @@
+module Main where
+
+import Test.Hspec
+import Test.Hspec.QuickCheck
+import Test.QuickCheck
+
+
+main :: IO ()
+main = hspec spec
+
+spec :: Spec
+spec = do
+  describe "Spec" $ 
+    it "works" $ do
+      41 + 1 `shouldBe` 42