csv-conduit 0.4.1 → 0.5.0

raw patch · 5 files changed

+74/−72 lines, 5 filesdep +data-defaultdep ~conduitPVP ok

version bump matches the API change (PVP)

Dependencies added: data-default

Dependency ranges changed: conduit

API changes (from Hackage documentation)

Files

• README.markdown +32/−7
• csv-conduit.cabal +3/−2
• src/Data/CSV/Conduit.hs +26/−43
• src/Data/CSV/Conduit/Types.hs +12/−18
• test/Test.hs +1/−2

@@ -6,24 +6,49 @@ particularly when dealing with enterprise application or disparate database systems. -While there are a number of csv libraries in Haskell, at the time of this-project's start in 2010, there wasn't one that provided all of the following:+While there are a number of csv libraries in Haskell, at the time of+this project's start, there wasn't one that provided all of the+following:  * Full flexibility in quote characters, separators, input/output * Constant space operation * Robust parsing and error resiliency+* Battle-tested reliability in real-world datasets * Fast operation * Convenient interface that supports a variety of use cases -This library is an attempt to close these gaps.+Over time, people created other plausible CSV packages like cassava.+The major benefit from this library remains to be: +* Direct participation in the conduit ecosystem, which is now quite+  large, and all the benefits that come with it.+* Flexibility in CSV format definition.+* Resiliency to errors in the input data. + ## This package -csv-conduit is a conduits based CSV parsing library that is easy to-use, flexible and fast. Furthermore, it provides ways to use-constant-space during operation, which is absolutely critical in many-real world use cases.+csv-conduit is a conduit-based CSV parsing library that is easy to+use, flexible and fast. It leverages the conduit infrastructure to+provide constant-space operation, which is quite critical in many real+world use cases.++For example, you can use http-conduit to download a CSV file from the+internet and plug its Source into intoCSV to stream-convert the+download into the Row data type and do something with it as the data+streams, that is without having to download the entire file to disk+first.+++## Author & Contributors++- Ozgun Ataman (@ozataman)+- Daniel Bergey (@bergey)+- BJTerry (@BJTerry)+- Mike Craig (@mkscrg)+- Daniel Corson (@dancor)+- Dmitry Dzhus (@dzhus)+- Niklas Hambüchen (@nh2)   ### Introduction

@@ -1,5 +1,5 @@ Name:                csv-conduit-Version:             0.4.1+Version:             0.5.0 Synopsis:            A flexible, fast, conduit-based CSV parser library for Haskell. Homepage:            http://github.com/ozataman/csv-conduit License:             BSD3@@ -74,10 +74,11 @@     , attoparsec-conduit >= 0.5.0.2     , base >= 4 && < 5     , bytestring-    , conduit == 0.5.*+    , conduit >= 1.0 && < 2.0     , containers >= 0.3     , monad-control     , text+    , data-default   ghc-options: -funbox-strict-fields   ghc-prof-options: -fprof-auto 

@@ -99,13 +99,13 @@   -----------------------------------------------------------------------------   -- | Turn a stream of 's' into a stream of CSV row type. An example   -- would be parsing a ByteString stream as rows of 'MapRow' 'Text'.-  intoCSV :: (MonadThrow m) => CSVSettings -> GLInfConduit s m r+  intoCSV :: (MonadThrow m) => CSVSettings -> Conduit s m r    -----------------------------------------------------------------------------   -- | Turn a stream of CSV row type back into a stream of 's'. An   -- example would be rendering a stream of 'Row' 'ByteString' rows as   -- 'Text'.-  fromCSV :: Monad m => CSVSettings -> GInfConduit r m s+  fromCSV :: Monad m => CSVSettings -> Conduit r m s   @@ -116,8 +116,8 @@ instance CSV ByteString (Row ByteString) where   rowToStr s !r =     let-      sep = B.pack [c2w (csvOutputColSep s)]-      wrapField !f = case csvOutputQuoteChar s of+      sep = B.pack [c2w (csvSep s)]+      wrapField !f = case csvQuoteChar s of         Just !x -> (x `B8.cons` escape x f) `B8.snoc` x         _ -> f       escape c str = B8.intercalate (B8.pack [c,c]) $ B8.split c str@@ -132,8 +132,8 @@ instance CSV Text (Row Text) where   rowToStr s !r =     let-      sep = T.pack [csvOutputColSep s]-      wrapField !f = case csvOutputQuoteChar s of+      sep = T.pack [csvSep s]+      wrapField !f = case csvQuoteChar s of         Just !x -> x `T.cons` escape x f `T.snoc` x         _ -> f       escape c str = T.intercalate (T.pack [c,c]) $ T.split (== c) str@@ -147,8 +147,8 @@ -- | 'Row' instance using 'Text' based on 'ByteString' stream instance CSV ByteString (Row Text) where     rowToStr s r = T.encodeUtf8 $ rowToStr s r-    intoCSV set = intoCSV set >+> C.map (map T.decodeUtf8)-    fromCSV set = fromCSV set >+> C.map T.encodeUtf8+    intoCSV set = intoCSV set =$= C.map (map T.decodeUtf8)+    fromCSV set = fromCSV set =$= C.map T.encodeUtf8   @@ -158,32 +158,27 @@ -- lots of unnecessary overhead. Included for convenience. instance CSV ByteString (Row String) where     rowToStr s r = rowToStr s $ map B8.pack r-    intoCSV set = intoCSV set >+> C.map (map B8.unpack)-    fromCSV set = C.map (map B8.pack) >+> fromCSV set+    intoCSV set = intoCSV set =$= C.map (map B8.unpack)+    fromCSV set = C.map (map B8.pack) =$= fromCSV set    ------------------------------------------------------------------------------- fromCSVRow :: (Monad m, IsString s, CSV s r)-           => CSVSettings -> GInfConduit r m s-fromCSVRow set = do-  erow <- awaitE-  case erow of-    Left ures -> return ures-    Right row -> mapM_ yield [rowToStr set row, "\n"] >> fromCSVRow set+           => CSVSettings -> Conduit r m s+fromCSVRow set = awaitForever $ \row -> do+    mapM_ yield [rowToStr set row, "\n"]+    fromCSVRow set   ------------------------------------------------------------------------------- intoCSVRow :: (MonadThrow m, AttoparsecInput i)-           => Parser i (Maybe o) -> GLInfConduit i m o-intoCSVRow p = parse >+> puller+           => Parser i (Maybe o) -> Conduit i m o+intoCSVRow p = parse =$= puller   where     parse = {-# SCC "conduitParser_p" #-} conduitParser p-    puller = {-# SCC "puller" #-} do-      emrow <- awaitE-      case emrow of-        Left ures -> return ures-        Right (_, mrow) ->+    puller = {-# SCC "puller" #-}+      awaitForever $ \ (_, mrow) ->           case mrow of             Just row -> yield row >> puller             Nothing -> puller@@ -201,8 +196,8 @@  ------------------------------------------------------------------------------- intoCSVMap :: (Ord a, MonadThrow m, CSV s [a])-           => CSVSettings -> GLInfConduit s m (MapRow a)-intoCSVMap set = intoCSV set >+> (headers >>= converter)+           => CSVSettings -> Conduit s m (MapRow a)+intoCSVMap set = intoCSV set =$= (headers >>= converter)   where     headers = do       mrow <- await@@ -210,22 +205,14 @@         Nothing -> return []         Just [] -> headers         Just hs -> return hs-    converter hs = do-      erow <- awaitE-      case erow of-        Left ures -> return ures-        Right row -> yield (toMapCSV hs row) >> converter hs+    converter hs = awaitForever $ \row -> yield (toMapCSV hs row) >> converter hs     toMapCSV !hs !fs = M.fromList $ zip hs fs   ------------------------------------------------------------------------------- fromCSVMap :: (Monad m, IsString s, CSV s [a])-           => CSVSettings -> GInfConduit (M.Map k a) m s-fromCSVMap set = do-  erow <- awaitE-  case erow of-    Left ures -> return ures-    Right row -> push row >> fromCSVMap set+           => CSVSettings -> Conduit (M.Map k a) m s+fromCSVMap set = awaitForever $ \row -> push row >> fromCSVMap set   where     push r = mapM_ yield [rowToStr set (M.elems r), "\n"] @@ -238,7 +225,7 @@ writeHeaders     :: (Monad m, CSV s (Row r), IsString s)     => CSVSettings-    -> GConduit (MapRow r) m s+    -> Conduit (MapRow r) m s writeHeaders set = do   mrow <- await   case mrow of@@ -256,9 +243,6 @@  ------------------------------------------------------------------------------- -- | Read the entire contents of a CSV file into memory.------ An easy way to run this function would be 'runResourceT' after--- feeding it all the arguments. readCSVFile     :: (CSV ByteString a)     => CSVSettings@@ -269,7 +253,6 @@ readCSVFile set fp = runResourceT $ sourceFile fp $= intoCSV set $$ C.consume  - ------------------------------------------------------------------------------- -- | Write CSV data into file. writeCSVFile@@ -297,7 +280,7 @@ mapCSVFile     :: (MonadResource m, MonadThrow m, CSV ByteString a, CSV ByteString b)     => CSVSettings-    -- ^ Settings to use both for input and output+    -- ^ Settings to use both for both input and output     -> (a -> [b])     -- ^ A mapping function     -> FilePath@@ -324,7 +307,7 @@ transformCSV     :: (MonadThrow m, CSV s a, CSV s' b)     => CSVSettings-    -- ^ Settings to be used for input and output+    -- ^ Settings to be used for both input and output     -> Source m s     -- ^ A raw stream data source. Ex: 'sourceFile inFile'     -> Conduit a m b

@@ -1,10 +1,11 @@-{-# LANGUAGE OverloadedStrings #-}+{-# LANGUAGE OverloadedStrings    #-} {-# LANGUAGE TypeSynonymInstances #-}  module Data.CSV.Conduit.Types where  --------------------------------------------------------------------------------import qualified Data.Map as M+import           Data.Default+import qualified Data.Map     as M -------------------------------------------------------------------------------  @@ -12,41 +13,34 @@ -- | Settings for a CSV file. This library is intended to be flexible -- and offer a way to process the majority of text data files out -- there.-data CSVSettings = CSVS-  { +data CSVSettings = CSVSettings+  {     -- | Separator character to be used in between fields-    csvSep :: !Char          +    csvSep       :: !Char      -- | Quote character that may sometimes be present around fields.     -- If 'Nothing' is given, the library will never expect quotation     -- even if it is present.   , csvQuoteChar :: !(Maybe Char)-  -    -- | Quote character that should be used in the output.-  , csvOutputQuoteChar :: !(Maybe Char)-  -    -- | Field separator that should be used in the output.-  , csvOutputColSep :: !Char   } deriving (Read, Show, Eq)    ---------------------------------------------------------------------------------- | Default settings for a CSV file. +-- | Default settings for a CSV file. -- -- > csvSep = ',' -- > csvQuoteChar = Just '"'--- > csvOutputQuoteChar = Just '"'--- > csvOutputColSep = ',' -- defCSVSettings :: CSVSettings-defCSVSettings = CSVS+defCSVSettings = CSVSettings   { csvSep = ','   , csvQuoteChar = Just '"'-  , csvOutputQuoteChar = Just '"'-  , csvOutputColSep = ','-  } +  } ++instance Default CSVSettings where+    def = defCSVSettings  ------------------------------------------------------------------------------- -- | A 'Row' is just a list of fields

@@ -54,8 +54,7 @@   csvSettings :: CSVSettings-csvSettings = defCSVSettings { csvQuoteChar = Just '`'-                             , csvOutputQuoteChar = Just '`' }+csvSettings = defCSVSettings { csvQuoteChar = Just '`'}  testFile1, testFile2 :: FilePath testFile1 = "test/test.csv"