diff --git a/README.markdown b/README.markdown --- a/README.markdown +++ b/README.markdown @@ -56,9 +56,10 @@ import Data.Conduit.Binary import Data.Conduit.List as CL import Data.CSV.Conduit + import Data.Text (Text) -- Just reverse te columns - myProcessor :: Conduit (Row Text) m (Row Text) + myProcessor :: Monad m => Conduit (Row Text) m (Row Text) myProcessor = CL.map reverse test :: IO () @@ -76,7 +77,8 @@ import Data.Conduit import Data.Conduit.Binary import Data.CSV.Conduit - + import Data.Text (Text) + myProcessor :: Conduit (Row Text) m (Row Text) myProcessor = undefined diff --git a/csv-conduit.cabal b/csv-conduit.cabal --- a/csv-conduit.cabal +++ b/csv-conduit.cabal @@ -1,5 +1,5 @@ Name: csv-conduit -Version: 0.2 +Version: 0.2.1.1 Synopsis: A flexible, fast, conduit-based CSV parser library for Haskell. Homepage: http://github.com/ozataman/csv-conduit License: BSD3 diff --git a/src/Data/CSV/Conduit.hs b/src/Data/CSV/Conduit.hs --- a/src/Data/CSV/Conduit.hs +++ b/src/Data/CSV/Conduit.hs @@ -1,57 +1,61 @@ -{-# LANGUAGE OverloadedStrings, BangPatterns #-} -{-# LANGUAGE PackageImports #-} -{-# LANGUAGE FlexibleContexts #-} -{-# LANGUAGE RankNTypes #-} +{-# LANGUAGE BangPatterns #-} +{-# LANGUAGE FlexibleContexts #-} +{-# LANGUAGE FlexibleInstances #-} {-# LANGUAGE MultiParamTypeClasses #-} -{-# LANGUAGE TypeSynonymInstances, FlexibleInstances #-} +{-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE PackageImports #-} +{-# LANGUAGE RankNTypes #-} +{-# LANGUAGE TypeSynonymInstances #-} module Data.CSV.Conduit - ( - - -- * Fundamental Types + ( + + -- * Key Operations CSV (..) + , writeHeaders + , readCSVFile + , transformCSV + , mapCSVFile + + -- * Important Types , CSVSettings (..) , defCSVSettings , MapRow , Row - -- * Convenience Functions - , readCSVFile - , transformCSV - , mapCSVFile - + -- * Re-exported For Convenience , runResourceT ) where ------------------------------------------------------------------------------- -import Control.Applicative hiding (many) -import Control.Exception (bracket, SomeException) -import Control.Monad (mzero, mplus, foldM, when, liftM) -import Control.Monad.IO.Class (liftIO, MonadIO) +import Control.Applicative hiding (many) +import Control.Exception (SomeException, bracket) +import Control.Monad (foldM, liftM, mplus, mzero, when) +import Control.Monad.IO.Class (MonadIO, liftIO) import Control.Monad.Trans.Control -import Data.Attoparsec as P hiding (take) -import qualified Data.Attoparsec.Char8 as C8 -import qualified Data.ByteString as B -import Data.ByteString.Char8 (ByteString) -import qualified Data.ByteString.Char8 as B8 -import Data.ByteString.Internal (c2w) -import Data.Conduit as C +import Data.Attoparsec as P hiding (take) +import qualified Data.Attoparsec.Char8 as C8 +import qualified Data.ByteString as B +import Data.ByteString.Char8 (ByteString) +import qualified Data.ByteString.Char8 as B8 +import Data.ByteString.Internal (c2w) +import Data.Conduit as C import Data.Conduit.Attoparsec -import Data.Conduit.Binary (sourceFile, sinkFile) -import qualified Data.Conduit.List as C +import Data.Conduit.Binary (sinkFile, sourceFile) +import qualified Data.Conduit.List as C import Data.Conduit.Text -import qualified Data.Map as M +import qualified Data.Map as M import Data.String -import Data.Text (Text) -import qualified Data.Text as T -import qualified Data.Text.Encoding as T -import Data.Word (Word8) -import Safe (headMay) +import Data.Text (Text) +import qualified Data.Text as T +import qualified Data.Text.Encoding as T +import Data.Word (Word8) +import Safe (headMay) import System.Directory -import System.PosixCompat.Files (getFileStatus, fileSize) +import System.PosixCompat.Files (fileSize, getFileStatus) ------------------------------------------------------------------------------- import qualified Data.CSV.Conduit.Parser.ByteString as BSP -import qualified Data.CSV.Conduit.Parser.Text as TP +import qualified Data.CSV.Conduit.Parser.Text as TP import Data.CSV.Conduit.Types ------------------------------------------------------------------------------- @@ -60,9 +64,9 @@ -- | Represents types 'r' that are CSV-like and can be converted -- to/from an underlying stream of type 's'. -- --- +-- -- Example #1: Basics Using Convenience API --- +-- -- @ -- import Data.Conduit -- import Data.Conduit.Binary @@ -71,30 +75,30 @@ -- -- myProcessor :: Conduit (Row Text) m (Row Text) -- myProcessor = CL.map reverse --- --- test = runResourceT $ --- transformCSV defCSVSettings --- (sourceFile "input.csv") +-- +-- test = runResourceT $ +-- transformCSV defCSVSettings +-- (sourceFile "input.csv") -- myProcessor -- (sinkFile "output.csv") -- @ -- -- -- Example #2: Basics Using Conduit API --- +-- -- @ -- import Data.Conduit -- import Data.Conduit.Binary -- import Data.CSV.Conduit -- --- myProcessor :: Conduit (Row Text) m (Row Text) +-- myProcessor :: Conduit (MapRow Text) m (MapRow Text) -- myProcessor = undefined -- --- test = runResourceT $ --- sourceFile "test/BigFile.csv" $= +-- test = runResourceT $ +-- sourceFile "test/BigFile.csv" $= -- intoCSV defCSVSettings $= -- myProcessor $= --- fromCSV defCSVSettings $$ +-- (writeHeaders defCSVSettings >> fromCSV defCSVSettings) $$ -- sinkFile "test/BigFileOut.csv" -- @ class CSV s r where @@ -115,12 +119,15 @@ fromCSV :: MonadResource m => CSVSettings -> Conduit r m s + + + ------------------------------------------------------------------------------ -- | 'Row' instance using 'ByteString' instance CSV ByteString (Row ByteString) where - rowToStr s !r = - let - sep = B.pack [c2w (csvOutputColSep s)] + rowToStr s !r = + let + sep = B.pack [c2w (csvOutputColSep s)] wrapField !f = case (csvOutputQuoteChar s) of Just !x -> x `B8.cons` escape x f `B8.snoc` x otherwise -> f @@ -134,9 +141,9 @@ ------------------------------------------------------------------------------ -- | 'Row' instance using 'Text' instance CSV Text (Row Text) where - rowToStr s !r = - let - sep = T.pack [(csvOutputColSep s)] + rowToStr s !r = + let + sep = T.pack [(csvOutputColSep s)] wrapField !f = case (csvOutputQuoteChar s) of Just !x -> x `T.cons` escape x f `T.snoc` x otherwise -> f @@ -194,9 +201,9 @@ ------------------------------------------------------------------------------- intoCSVMap set = intoCSV set =$= converter where - converter = conduitState Nothing push close + converter = conduitState Nothing push close where - push Nothing row = + push Nothing row = case row of [] -> return $ StateProducing Nothing [] xs -> return $ StateProducing (Just xs) [] @@ -206,16 +213,32 @@ ------------------------------------------------------------------------------- -fromCSVMap set = conduitState False push close +fromCSVMap set = do + r <- C.await + case r of + Nothing -> return () + Just r' -> push r' >> fromCSVMap set + where - push False r = return $ StateProducing True - [rowToStr set (M.keys r), "\n", rowToStr set (M.elems r), "\n"] - push True r = return $ StateProducing True - [rowToStr set (M.elems r), "\n"] - close _ = return [] + push r = mapM_ C.yield [rowToStr set (M.elems r), "\n"] +------------------------------------------------------------------------------- +-- | Write headers AND the row into the output stream, once. Just +-- chain this using the 'Monad' instance in your pipeline: +-- +-- > ... =$= writeHeaders settings >> fromCSV settings( $$ sinkFile "..." +writeHeaders + :: (MonadResource m, CSV s (Row r), IsString s) + => CSVSettings + -> Conduit (MapRow r) m s +writeHeaders set = do + r <- C.await + case r of + Nothing -> return () + Just r' -> mapM_ yield [rowToStr set (M.keys r'), "\n", rowToStr set (M.elems r'), "\n"] + --------------------------- -- Convenience Functions -- --------------------------- @@ -226,10 +249,10 @@ -- -- An easy way to run this function would be 'runResourceT' after -- feeding it all the arguments. -readCSVFile - :: (MonadResource m, CSV ByteString a) - => CSVSettings - -> FilePath +readCSVFile + :: (MonadResource m, CSV ByteString a) + => CSVSettings + -> FilePath -- ^ Input file -> m [a] readCSVFile set fp = sourceFile fp $= intoCSV set $$ C.consume @@ -241,18 +264,18 @@ -- -- An easy way to run this function would be 'runResourceT' after -- feeding it all the arguments. -mapCSVFile - :: (MonadResource m, CSV ByteString a, CSV ByteString b) - => CSVSettings +mapCSVFile + :: (MonadResource m, CSV ByteString a, CSV ByteString b) + => CSVSettings -- ^ Settings to use both for input and output - -> (a -> [b]) + -> (a -> [b]) -- ^ A mapping function - -> FilePath + -> FilePath -- ^ Input file - -> FilePath + -> FilePath -- ^ Output file -> m () -mapCSVFile set f fi fo = +mapCSVFile set f fi fo = transformCSV set (sourceFile fi) (C.concatMap f) (sinkFile fo) @@ -266,11 +289,11 @@ -- feeding it all the arguments. -- -- Example - map a function over the rows of a CSV file: --- +-- -- > transformCSV set (sourceFile inFile) (C.map f) (sinkFile outFile) -transformCSV - :: (MonadResource m, CSV s a, CSV s' b) - => CSVSettings +transformCSV + :: (MonadResource m, CSV s a, CSV s' b) + => CSVSettings -- ^ Settings to be used for input and output -> Source m s -- ^ A raw stream data source. Ex: 'sourceFile inFile' @@ -279,25 +302,27 @@ -> Sink s' m () -- ^ A raw stream data sink. Ex: 'sinkFile outFile' -> m () -transformCSV set source c sink = +transformCSV set source c sink = source $= intoCSV set $= c $= fromCSV set $$ sink - + ----------------- -- Simple Test -- ----------------- + test :: IO () -test = runResourceT $ - sourceFile "test/BigFile.csv" $= +test = runResourceT $ + sourceFile "test/BigFile.csv" $= decode utf8 $= - (intoCSV defCSVSettings - :: forall m. MonadResource m => Conduit Text m (MapRow Text)) $= - fromCSV defCSVSettings $= + (intoCSV defCSVSettings + :: forall m. MonadResource m => Conduit Text m (MapRow Text)) $= + C.map (id :: MapRow Text -> MapRow Text) $= + (writeHeaders defCSVSettings >> fromCSV defCSVSettings) $= encode utf8 $$ sinkFile "test/BigFileOut.csv" diff --git a/src/Data/CSV/Conduit/Parser/Text.hs b/src/Data/CSV/Conduit/Parser/Text.hs --- a/src/Data/CSV/Conduit/Parser/Text.hs +++ b/src/Data/CSV/Conduit/Parser/Text.hs @@ -1,4 +1,4 @@ -{-| +{-| This module exports the underlying Attoparsec row parser. This is helpful if you want to do some ad-hoc CSV string parsing. @@ -14,14 +14,14 @@ ------------------------------------------------------------------------------- import Control.Applicative -import Control.Monad (mzero, mplus, foldM, when, liftM) -import Control.Monad.IO.Class (liftIO, MonadIO) -import Data.Attoparsec.Text as P hiding (take) -import qualified Data.Attoparsec.Text as T -import qualified Data.Map as M -import Data.Text (Text) -import qualified Data.Text as T -import Data.Word (Word8) +import Control.Monad (foldM, liftM, mplus, mzero, when) +import Control.Monad.IO.Class (MonadIO, liftIO) +import Data.Attoparsec.Text as P hiding (take) +import qualified Data.Attoparsec.Text as T +import qualified Data.Map as M +import Data.Text (Text) +import qualified Data.Text as T +import Data.Word (Word8) ------------------------------------------------------------------------------- import Data.CSV.Conduit.Types ------------------------------------------------------------------------------- @@ -63,11 +63,11 @@ badrow :: Parser (Maybe (Row Text)) -badrow = P.takeWhile (not . T.isEndOfLine) *> +badrow = P.takeWhile (not . T.isEndOfLine) *> (T.endOfLine <|> T.endOfInput) *> return Nothing csvrow :: CSVSettings -> Parser (Maybe (Row Text)) -csvrow c = +csvrow c = let rowbody = (quotedField' <|> (field c)) `sepBy` T.char (csvSep c) properrow = rowbody <* (T.endOfLine <|> P.endOfInput) quotedField' = case csvQuoteChar c of @@ -78,22 +78,21 @@ return $ Just res field :: CSVSettings -> Parser Text -field s = P.takeWhile (isFieldChar s) +field s = P.takeWhile (isFieldChar s) isFieldChar s = notInClass xs' where xs = csvSep s : "\n\r" - xs' = case csvQuoteChar s of + xs' = case csvQuoteChar s of Nothing -> xs Just x -> x : xs quotedField :: Char -> Parser Text -quotedField c = +quotedField c = do let quoted = string dbl *> return c dbl = T.pack [c,c] - in do - T.char c + T.char c f <- many (T.notChar c <|> quoted) - T.char c + T.char c return $ T.pack f