csv-conduit 0.2 → 0.2.1.1
raw patch · 4 files changed
+127/−101 lines, 4 filesPVP ok
version bump matches the API change (PVP)
API changes (from Hackage documentation)
+ Data.CSV.Conduit: writeHeaders :: (MonadResource m, CSV s (Row r), IsString s) => CSVSettings -> Conduit (MapRow r) m s
Files
- README.markdown +4/−2
- csv-conduit.cabal +1/−1
- src/Data/CSV/Conduit.hs +106/−81
- src/Data/CSV/Conduit/Parser/Text.hs +16/−17
README.markdown view
@@ -56,9 +56,10 @@ import Data.Conduit.Binary import Data.Conduit.List as CL import Data.CSV.Conduit+ import Data.Text (Text) -- Just reverse te columns- myProcessor :: Conduit (Row Text) m (Row Text)+ myProcessor :: Monad m => Conduit (Row Text) m (Row Text) myProcessor = CL.map reverse test :: IO ()@@ -76,7 +77,8 @@ import Data.Conduit import Data.Conduit.Binary import Data.CSV.Conduit- + import Data.Text (Text)+ myProcessor :: Conduit (Row Text) m (Row Text) myProcessor = undefined
csv-conduit.cabal view
@@ -1,5 +1,5 @@ Name: csv-conduit-Version: 0.2+Version: 0.2.1.1 Synopsis: A flexible, fast, conduit-based CSV parser library for Haskell. Homepage: http://github.com/ozataman/csv-conduit License: BSD3
src/Data/CSV/Conduit.hs view
@@ -1,57 +1,61 @@-{-# LANGUAGE OverloadedStrings, BangPatterns #-}-{-# LANGUAGE PackageImports #-}-{-# LANGUAGE FlexibleContexts #-}-{-# LANGUAGE RankNTypes #-}+{-# LANGUAGE BangPatterns #-}+{-# LANGUAGE FlexibleContexts #-}+{-# LANGUAGE FlexibleInstances #-} {-# LANGUAGE MultiParamTypeClasses #-}-{-# LANGUAGE TypeSynonymInstances, FlexibleInstances #-}+{-# LANGUAGE OverloadedStrings #-}+{-# LANGUAGE PackageImports #-}+{-# LANGUAGE RankNTypes #-}+{-# LANGUAGE TypeSynonymInstances #-} module Data.CSV.Conduit- ( - - -- * Fundamental Types+ (++ -- * Key Operations CSV (..)+ , writeHeaders+ , readCSVFile+ , transformCSV+ , mapCSVFile++ -- * Important Types , CSVSettings (..) , defCSVSettings , MapRow , Row- -- * Convenience Functions- , readCSVFile- , transformCSV- , mapCSVFile- + -- * Re-exported For Convenience , runResourceT ) where --------------------------------------------------------------------------------import Control.Applicative hiding (many)-import Control.Exception (bracket, SomeException)-import Control.Monad (mzero, mplus, foldM, when, liftM)-import Control.Monad.IO.Class (liftIO, MonadIO)+import Control.Applicative hiding (many)+import Control.Exception (SomeException, bracket)+import Control.Monad (foldM, liftM, mplus, mzero, when)+import Control.Monad.IO.Class (MonadIO, liftIO) import Control.Monad.Trans.Control-import Data.Attoparsec as P hiding (take)-import qualified Data.Attoparsec.Char8 as C8-import qualified Data.ByteString as B-import Data.ByteString.Char8 (ByteString)-import qualified Data.ByteString.Char8 as B8-import Data.ByteString.Internal (c2w)-import Data.Conduit as C+import Data.Attoparsec as P hiding (take)+import qualified Data.Attoparsec.Char8 as C8+import qualified Data.ByteString as B+import Data.ByteString.Char8 (ByteString)+import qualified Data.ByteString.Char8 as B8+import Data.ByteString.Internal (c2w)+import Data.Conduit as C import Data.Conduit.Attoparsec-import Data.Conduit.Binary (sourceFile, sinkFile)-import qualified Data.Conduit.List as C+import Data.Conduit.Binary (sinkFile, sourceFile)+import qualified Data.Conduit.List as C import Data.Conduit.Text-import qualified Data.Map as M+import qualified Data.Map as M import Data.String-import Data.Text (Text)-import qualified Data.Text as T-import qualified Data.Text.Encoding as T-import Data.Word (Word8)-import Safe (headMay)+import Data.Text (Text)+import qualified Data.Text as T+import qualified Data.Text.Encoding as T+import Data.Word (Word8)+import Safe (headMay) import System.Directory-import System.PosixCompat.Files (getFileStatus, fileSize)+import System.PosixCompat.Files (fileSize, getFileStatus) ------------------------------------------------------------------------------- import qualified Data.CSV.Conduit.Parser.ByteString as BSP-import qualified Data.CSV.Conduit.Parser.Text as TP+import qualified Data.CSV.Conduit.Parser.Text as TP import Data.CSV.Conduit.Types ------------------------------------------------------------------------------- @@ -60,9 +64,9 @@ -- | Represents types 'r' that are CSV-like and can be converted -- to/from an underlying stream of type 's'. ----- +-- -- Example #1: Basics Using Convenience API--- +-- -- @ -- import Data.Conduit -- import Data.Conduit.Binary@@ -71,30 +75,30 @@ -- -- myProcessor :: Conduit (Row Text) m (Row Text) -- myProcessor = CL.map reverse--- --- test = runResourceT $ --- transformCSV defCSVSettings --- (sourceFile "input.csv") +--+-- test = runResourceT $+-- transformCSV defCSVSettings+-- (sourceFile "input.csv") -- myProcessor -- (sinkFile "output.csv") -- @ -- -- -- Example #2: Basics Using Conduit API--- +-- -- @ -- import Data.Conduit -- import Data.Conduit.Binary -- import Data.CSV.Conduit ----- myProcessor :: Conduit (Row Text) m (Row Text)+-- myProcessor :: Conduit (MapRow Text) m (MapRow Text) -- myProcessor = undefined ----- test = runResourceT $ --- sourceFile "test/BigFile.csv" $= +-- test = runResourceT $+-- sourceFile "test/BigFile.csv" $= -- intoCSV defCSVSettings $= -- myProcessor $=--- fromCSV defCSVSettings $$+-- (writeHeaders defCSVSettings >> fromCSV defCSVSettings) $$ -- sinkFile "test/BigFileOut.csv" -- @ class CSV s r where@@ -115,12 +119,15 @@ fromCSV :: MonadResource m => CSVSettings -> Conduit r m s +++ ------------------------------------------------------------------------------ -- | 'Row' instance using 'ByteString' instance CSV ByteString (Row ByteString) where- rowToStr s !r = - let - sep = B.pack [c2w (csvOutputColSep s)] + rowToStr s !r =+ let+ sep = B.pack [c2w (csvOutputColSep s)] wrapField !f = case (csvOutputQuoteChar s) of Just !x -> x `B8.cons` escape x f `B8.snoc` x otherwise -> f@@ -134,9 +141,9 @@ ------------------------------------------------------------------------------ -- | 'Row' instance using 'Text' instance CSV Text (Row Text) where- rowToStr s !r = - let - sep = T.pack [(csvOutputColSep s)] + rowToStr s !r =+ let+ sep = T.pack [(csvOutputColSep s)] wrapField !f = case (csvOutputQuoteChar s) of Just !x -> x `T.cons` escape x f `T.snoc` x otherwise -> f@@ -194,9 +201,9 @@ ------------------------------------------------------------------------------- intoCSVMap set = intoCSV set =$= converter where- converter = conduitState Nothing push close + converter = conduitState Nothing push close where- push Nothing row = + push Nothing row = case row of [] -> return $ StateProducing Nothing [] xs -> return $ StateProducing (Just xs) []@@ -206,16 +213,32 @@ --------------------------------------------------------------------------------fromCSVMap set = conduitState False push close+fromCSVMap set = do+ r <- C.await+ case r of+ Nothing -> return ()+ Just r' -> push r' >> fromCSVMap set+ where- push False r = return $ StateProducing True - [rowToStr set (M.keys r), "\n", rowToStr set (M.elems r), "\n"]- push True r = return $ StateProducing True- [rowToStr set (M.elems r), "\n"]- close _ = return []+ push r = mapM_ C.yield [rowToStr set (M.elems r), "\n"] +-------------------------------------------------------------------------------+-- | Write headers AND the row into the output stream, once. Just+-- chain this using the 'Monad' instance in your pipeline:+--+-- > ... =$= writeHeaders settings >> fromCSV settings( $$ sinkFile "..."+writeHeaders+ :: (MonadResource m, CSV s (Row r), IsString s)+ => CSVSettings+ -> Conduit (MapRow r) m s+writeHeaders set = do+ r <- C.await+ case r of+ Nothing -> return ()+ Just r' -> mapM_ yield [rowToStr set (M.keys r'), "\n", rowToStr set (M.elems r'), "\n"] + --------------------------- -- Convenience Functions -- ---------------------------@@ -226,10 +249,10 @@ -- -- An easy way to run this function would be 'runResourceT' after -- feeding it all the arguments.-readCSVFile - :: (MonadResource m, CSV ByteString a) - => CSVSettings - -> FilePath +readCSVFile+ :: (MonadResource m, CSV ByteString a)+ => CSVSettings+ -> FilePath -- ^ Input file -> m [a] readCSVFile set fp = sourceFile fp $= intoCSV set $$ C.consume@@ -241,18 +264,18 @@ -- -- An easy way to run this function would be 'runResourceT' after -- feeding it all the arguments.-mapCSVFile - :: (MonadResource m, CSV ByteString a, CSV ByteString b) - => CSVSettings +mapCSVFile+ :: (MonadResource m, CSV ByteString a, CSV ByteString b)+ => CSVSettings -- ^ Settings to use both for input and output- -> (a -> [b]) + -> (a -> [b]) -- ^ A mapping function- -> FilePath + -> FilePath -- ^ Input file- -> FilePath + -> FilePath -- ^ Output file -> m ()-mapCSVFile set f fi fo = +mapCSVFile set f fi fo = transformCSV set (sourceFile fi) (C.concatMap f) (sinkFile fo) @@ -266,11 +289,11 @@ -- feeding it all the arguments. -- -- Example - map a function over the rows of a CSV file:--- +-- -- > transformCSV set (sourceFile inFile) (C.map f) (sinkFile outFile)-transformCSV - :: (MonadResource m, CSV s a, CSV s' b) - => CSVSettings +transformCSV+ :: (MonadResource m, CSV s a, CSV s' b)+ => CSVSettings -- ^ Settings to be used for input and output -> Source m s -- ^ A raw stream data source. Ex: 'sourceFile inFile'@@ -279,25 +302,27 @@ -> Sink s' m () -- ^ A raw stream data sink. Ex: 'sinkFile outFile' -> m ()-transformCSV set source c sink = +transformCSV set source c sink = source $= intoCSV set $= c $= fromCSV set $$ sink- + ----------------- -- Simple Test -- ----------------- + test :: IO ()-test = runResourceT $ - sourceFile "test/BigFile.csv" $= +test = runResourceT $+ sourceFile "test/BigFile.csv" $= decode utf8 $=- (intoCSV defCSVSettings - :: forall m. MonadResource m => Conduit Text m (MapRow Text)) $= - fromCSV defCSVSettings $=+ (intoCSV defCSVSettings+ :: forall m. MonadResource m => Conduit Text m (MapRow Text)) $=+ C.map (id :: MapRow Text -> MapRow Text) $=+ (writeHeaders defCSVSettings >> fromCSV defCSVSettings) $= encode utf8 $$ sinkFile "test/BigFileOut.csv"
src/Data/CSV/Conduit/Parser/Text.hs view
@@ -1,4 +1,4 @@-{-| +{-| This module exports the underlying Attoparsec row parser. This is helpful if you want to do some ad-hoc CSV string parsing.@@ -14,14 +14,14 @@ ------------------------------------------------------------------------------- import Control.Applicative-import Control.Monad (mzero, mplus, foldM, when, liftM)-import Control.Monad.IO.Class (liftIO, MonadIO)-import Data.Attoparsec.Text as P hiding (take)-import qualified Data.Attoparsec.Text as T-import qualified Data.Map as M-import Data.Text (Text)-import qualified Data.Text as T-import Data.Word (Word8)+import Control.Monad (foldM, liftM, mplus, mzero, when)+import Control.Monad.IO.Class (MonadIO, liftIO)+import Data.Attoparsec.Text as P hiding (take)+import qualified Data.Attoparsec.Text as T+import qualified Data.Map as M+import Data.Text (Text)+import qualified Data.Text as T+import Data.Word (Word8) ------------------------------------------------------------------------------- import Data.CSV.Conduit.Types -------------------------------------------------------------------------------@@ -63,11 +63,11 @@ badrow :: Parser (Maybe (Row Text))-badrow = P.takeWhile (not . T.isEndOfLine) *> +badrow = P.takeWhile (not . T.isEndOfLine) *> (T.endOfLine <|> T.endOfInput) *> return Nothing csvrow :: CSVSettings -> Parser (Maybe (Row Text))-csvrow c = +csvrow c = let rowbody = (quotedField' <|> (field c)) `sepBy` T.char (csvSep c) properrow = rowbody <* (T.endOfLine <|> P.endOfInput) quotedField' = case csvQuoteChar c of@@ -78,22 +78,21 @@ return $ Just res field :: CSVSettings -> Parser Text-field s = P.takeWhile (isFieldChar s) +field s = P.takeWhile (isFieldChar s) isFieldChar s = notInClass xs' where xs = csvSep s : "\n\r"- xs' = case csvQuoteChar s of + xs' = case csvQuoteChar s of Nothing -> xs Just x -> x : xs quotedField :: Char -> Parser Text-quotedField c = +quotedField c = do let quoted = string dbl *> return c dbl = T.pack [c,c]- in do- T.char c + T.char c f <- many (T.notChar c <|> quoted)- T.char c + T.char c return $ T.pack f