pipes-key-value-csv (empty) → 0.0.0.0
raw patch · 26 files changed
+1434/−0 lines, 26 filesdep +QuickCheckdep +basedep +containerssetup-changed
Dependencies added: QuickCheck, base, containers, contravariant, data-default-class, lens, mtl, pipes, pipes-bytestring, pipes-group, pipes-key-value-csv, pipes-parse, pipes-safe, pipes-text, reflection, text, transformers, validation, vinyl, vinyl-utils
Files
- LICENSE +30/−0
- README.md +2/−0
- Setup.hs +2/−0
- pipes-key-value-csv.cabal +87/−0
- src/Pipes/KeyValueCsv.hs +73/−0
- src/Pipes/KeyValueCsv/Cell.hs +57/−0
- src/Pipes/KeyValueCsv/Common.hs +105/−0
- src/Pipes/KeyValueCsv/Csv.hs +50/−0
- src/Pipes/KeyValueCsv/IO.hs +74/−0
- src/Pipes/KeyValueCsv/Internal.hs +82/−0
- src/Pipes/KeyValueCsv/Internal/Csv.hs +100/−0
- src/Pipes/KeyValueCsv/Internal/KeyValue.hs +122/−0
- src/Pipes/KeyValueCsv/Internal/Names.hs +30/−0
- src/Pipes/KeyValueCsv/Internal/Types.hs +36/−0
- src/Pipes/KeyValueCsv/KeyValue.hs +84/−0
- src/Pipes/KeyValueCsv/Names.hs +35/−0
- src/Pipes/KeyValueCsv/Types.hs +54/−0
- src/Pipes/KeyValueCsv/Types/Common.hs +40/−0
- src/Pipes/KeyValueCsv/Types/Csv.hs +52/−0
- src/Pipes/KeyValueCsv/Types/KeyValue.hs +59/−0
- test/Test/Common.hs +46/−0
- test/Test/Csv.hs +58/−0
- test/Test/Instances.hs +30/−0
- test/Test/KeyValue.hs +67/−0
- test/Test/Types.hs +42/−0
- test/test-kvc.hs +17/−0
+ LICENSE view
@@ -0,0 +1,30 @@+Copyright (c) 2015, Marcin Mrotek + +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials provided + with the distribution. + + * Neither the name of Marcin Mrotek nor the names of other + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ README.md view
@@ -0,0 +1,2 @@+# key-value-csv +Processing CSV files preceded by key-value pairs.
+ Setup.hs view
@@ -0,0 +1,2 @@+import Distribution.Simple +main = defaultMain
+ pipes-key-value-csv.cabal view
@@ -0,0 +1,87 @@+name: pipes-key-value-csv +version: 0.0.0.0 +synopsis: Streaming processing of CSV files preceded by key-value pairs. +description: Nonstandard CSV files, with the table preceded by a header consisting of key-value pairs are often the output format of varius test equipment, such as digital oscilloscopes. +homepage: https://github.com/marcinmrotek/key-value-csv +license: BSD3 +license-file: LICENSE +author: Marcin Mrotek +maintainer: marcin.jan.mrotek@gmail.com +copyright: Marcin Mrotek, 2015 +category: Text, Pipes, CSV +build-type: Simple +extra-source-files: README.md +cabal-version: >=1.10 + +Flag devel + description: Development mode (-Werror). + default: False + manual: True + +library + exposed-modules: Pipes.KeyValueCsv + , Pipes.KeyValueCsv.Cell + , Pipes.KeyValueCsv.Common + , Pipes.KeyValueCsv.Csv + , Pipes.KeyValueCsv.KeyValue + , Pipes.KeyValueCsv.IO + , Pipes.KeyValueCsv.Types + , Pipes.KeyValueCsv.Types.Common + , Pipes.KeyValueCsv.Types.Csv + , Pipes.KeyValueCsv.Types.KeyValue + , Pipes.KeyValueCsv.Names + , Pipes.KeyValueCsv.Internal + , Pipes.KeyValueCsv.Internal.Csv + , Pipes.KeyValueCsv.Internal.KeyValue + , Pipes.KeyValueCsv.Internal.Names + , Pipes.KeyValueCsv.Internal.Types + build-depends: base >= 4.8 && < 4.9 + , containers >= 0.5.6 && < 0.6 + , data-default-class >= 0.0.1 && < 0.2 + , lens >= 4.12 && < 4.14 + , mtl >= 2.2 && < 2.3 + , pipes >= 4.1 && < 4.2 + , pipes-bytestring >= 2.1 && < 2.2 + , pipes-group >= 1.0 && < 1.1 + , pipes-parse >= 3.0 && < 3.1 + , pipes-safe >= 2.2 && < 2.3 + , pipes-text >= 0.0 && < 0.1 + , reflection >= 1.5 && < 2.2 + , text >= 1.2 && < 1.3 + , validation >= 0.5.2 && < 0.6 + , vinyl >= 0.5 && < 0.6 + , vinyl-utils >= 0.2.0.1 && < 0.3 + hs-source-dirs: src + default-language: Haskell2010 + ghc-options: -Wall + if flag(devel) + ghc-options: -Werror + +test-suite test-kvc + type: exitcode-stdio-1.0 + main-is: test-kvc.hs + other-modules: Test.Common + , Test.Csv + , Test.Instances + , Test.KeyValue + , Test.Types + build-depends: base >= 4.8 && < 4.9 + , contravariant >= 1.2 && < 1.5 + , pipes-key-value-csv + , data-default-class + , lens + , QuickCheck >= 2.8 && < 2.9 + , pipes + , pipes-text + , reflection + , text + , transformers + , validation + , vinyl + , vinyl-utils + hs-source-dirs: test + default-language: Haskell2010 + ghc-options: -Wall + if flag(devel) + ghc-options: -Werror +
+ src/Pipes/KeyValueCsv.hs view
@@ -0,0 +1,73 @@+{-| +Module : Pipes.KeyValueCsv +Copyright : (c) Marcin Mrotek, 2015 +License : BSD3 +Maintainer : marcin.jan.mrotek@gmail.com +Stability : experimental + +Parse CSV files with key-value headers. +-} + +{-# LANGUAGE + DataKinds + , ExplicitForAll + , PolyKinds + , TypeOperators + #-} + +module Pipes.KeyValueCsv + ( breakLines + , parseKeyValueCsv + , module Pipes.KeyValueCsv.Csv + , module Pipes.KeyValueCsv.KeyValue + , module Pipes.KeyValueCsv.Types + , Record (..) + -- * Re-exports + , AccValidation(..) + ) where + +import Prelude hiding (lines) + +import Pipes.KeyValueCsv.Internal +import Pipes.KeyValueCsv.Internal.Types +import Pipes.KeyValueCsv.Common +import Pipes.KeyValueCsv.Csv +import Pipes.KeyValueCsv.KeyValue +import Pipes.KeyValueCsv.Types + +import Control.Lens +import Data.Validation +import Data.Vinyl +import Data.Vinyl.Functor +import Data.Vinyl.Utils.Proxy + +breakLines + :: Monad m + => (Text -> Bool) + -> Lines m r + -> Lines m (Lines m r) +{-^ +Break a stream of lines into two parts, on a line that satisfies the given predicate. +All input lines up to the breaking one will be fully read, and each (not including the breaking one) will be re-'yield'ed. +-} +breakLines p (Lines l) = Lines $ Lines <$> breakLines' p l + +parseKeyValueCsv + :: forall (m :: * -> *) (f :: k -> *) (g :: j -> *) (hs :: [k]) (rs :: [j]) (r :: *) + . ( Monad m + , Record hs + ) + => Options m f g hs rs + -> Producer Text m r + -> m + ( Rec (WithKeyValueError :. f) hs + , Producer (Rec (WithCsvError :. g) rs) m r + ) +-- ^Read a CSV file preceded by key-value pairs. +parseKeyValueCsv options producer = useDelimiter (options^.delimiter) $ do + (hdr, remaining) + <- parseKeyValues (options^.kvOptions) + . breakLines (options^.predicate) + $ lines producer + pure (hdr, parseCsv (options^.csvOptions) remaining) +
+ src/Pipes/KeyValueCsv/Cell.hs view
@@ -0,0 +1,57 @@+{-| +Module : Pipes.KeyValueCsv.Cell +Copyright : (c) Marcin Mrotek, 2015 +License : BSD3 +Maintainer : marcin.jan.mrotek@gmail.com +Stability : experimental + +Automated parsing of single cells. +-} + +{-# LANGUAGE + FlexibleContexts + , FlexibleInstances + , GeneralizedNewtypeDeriving + , PolyKinds + , RankNTypes + , TypeSynonymInstances + #-} + +module Pipes.KeyValueCsv.Cell where + +import Pipes.KeyValueCsv.Internal + +import Data.Text (Text) +import qualified Data.Text.Lazy as Lazy +import Data.Text.Lazy.Read +import Data.Vinyl.Functor +import Pipes.Parse (Parser) +import Pipes.Parse.Tutorial() + +-- |Parse a single cell. For more information about parsing with Pipes, see "Pipes.Parse.Tutorial". +class FromCell c where + fromCell :: Monad m => Parser Text m (Either String c) + +instance FromCell Text where + fromCell = Right . Lazy.toStrict <$> drawText + +instance FromCell String where + fromCell = Right . Lazy.unpack <$> drawText + +instance FromCell Integer where + fromCell = fmap fst . signed decimal <$> (skipSpace >> build) + +instance FromCell Int where + fromCell = fmap fst . signed decimal <$> (skipSpace >> build) + +instance FromCell Double where + fromCell = fmap fst . double <$> (skipSpace >> build) + +instance FromCell a => FromCell (Identity a) where + fromCell = fmap Identity <$> fromCell + +instance FromCell (f (g a)) => FromCell (Compose f g a) where + fromCell = fmap Compose <$> fromCell + +instance FromCell a => FromCell (Thunk a) where + fromCell = fmap Thunk <$> fromCell
+ src/Pipes/KeyValueCsv/Common.hs view
@@ -0,0 +1,105 @@+{-| +Module : Pipes.KeyValueCsv.Common +Copyright : (c) Marcin Mrotek, 2015 +License : BSD3 +Maintainer : marcin.jan.mrotek@gmail.com +Stability : experimental + +Utility functions. +-} + +{-# LANGUAGE + DataKinds + , GADTs + , FlexibleContexts + , PolyKinds + , RankNTypes + , ScopedTypeVariables + , TypeOperators + #-} + +module Pipes.KeyValueCsv.Common + ( module Pipes.KeyValueCsv.Common + , Lines + , Cells + , Producer + , Text + ) where + +import Pipes.KeyValueCsv.Cell +import Pipes.KeyValueCsv.Internal.Types +import Pipes.KeyValueCsv.Types.Common + +import Control.Lens +import Data.Reflection +import Data.Vinyl +import Data.Vinyl.Functor +import Data.Vinyl.TypeLevel +import Data.Vinyl.Utils.Proxy +import Data.Text (Text) +import Pipes hiding (Proxy) +import qualified Pipes.Group as Group +import Pipes.Parse (Parser) +import qualified Pipes.Text as PipesText + +lines + :: Monad m + => Producer Text m r + -> Lines m r +-- ^Split a 'Text' stream into lines. +lines = + Lines + . Group.maps Line + . view PipesText.lines + +cells + :: ( Monad m + , Given Delimiter + ) + => Line m r + -> Cells m r +-- ^Split a line into delimited cells. +cells = + Cells + . Group.maps Cell + . view (PipesText.splits delim) + . unLine + where delim = getDelimiter given + +useDelimiter :: Char -> (Given Delimiter => a) -> a +{-^ +Provide a delimiter for a computation. +Do not use this function if there is already a 'Given' 'Delimiter' in scope. +-} +useDelimiter = give . Delimiter + +wrapParser + :: forall m f r . Functor m + => Parser Text m (Either String (f r)) + -> CellParser m f r +-- ^Wrap a parser to use it as a type constructor for 'Rec'. +wrapParser p = Compose wrapped + where + wrapped :: WrapParser Text m ((Either String :. f) r) + wrapped = WrapParser composed + composed :: Parser Text m ((Either String :. f) r) + composed = Compose <$> p + +defaultParser' + :: ( RecAll f rs FromCell + , Monad m + ) + => Rec Proxy rs + -> Rec (CellParser m f) rs +defaultParser' RNil = RNil +defaultParser' (Proxy :& ps) = wrapParser fromCell :& defaultParser' ps + +defaultParser + :: forall (m :: * -> *) (f :: k -> *) (rs :: [k]) + . ( RecAll f rs FromCell + , Monad m + , Record rs + ) + => Rec (CellParser m f) rs +defaultParser = defaultParser' proxyRecord +
+ src/Pipes/KeyValueCsv/Csv.hs view
@@ -0,0 +1,50 @@+{-| +Module : Pipes.KeyValueCsv.Csv +Copyright : (c) Marcin Mrotek, 2015 +License : BSD3 +Maintainer : marcin.jan.mrotek@gmail.com +Stability : experimental + +Parsing the main CSV body. +-} + +{-# LANGUAGE + DataKinds + , FlexibleContexts + , PolyKinds + , RankNTypes + , TypeOperators + #-} + +module Pipes.KeyValueCsv.Csv + ( parseCsv + , WrapParser(..) + , wrapParser + ) where + +import Pipes.KeyValueCsv.Internal.Csv +import Pipes.KeyValueCsv.Internal.Types +import Pipes.KeyValueCsv.Common +import Pipes.KeyValueCsv.Types.Common +import Pipes.KeyValueCsv.Types.Csv + +import Control.Lens +import Data.Reflection +import Data.Vinyl +import Data.Vinyl.Functor +import qualified Pipes.Group as Group + +parseCsv + :: forall (m :: * -> *) (f :: k -> *) (rs :: [k]) (r :: *) + . ( Given Delimiter + , Monad m + ) + => CsvOptions m f rs + -> Lines m r + -> Producer (Rec (WithCsvError :. f) rs) m r +-- ^Parse a stream of lines. +parseCsv options = + Group.concats + . Group.maps (parseLine $ options^.csvParser) + . unLines +
+ src/Pipes/KeyValueCsv/IO.hs view
@@ -0,0 +1,74 @@+{-| +Module : Pipes.KeyValueCsv.IO +Copyright : (c) Marcin Mrotek, 2015 +License : BSD3 +Maintainer : marcin.jan.mrotek@gmail.com +Stability : experimental + +File input and output. +-} + +{-# LANGUAGE + DataKinds + , FlexibleContexts + , PolyKinds + , RankNTypes + , TypeOperators + #-} + +module Pipes.KeyValueCsv.IO + ( streamIn + , streamInBS + -- * Re-exports + , SafeT + ) where + +import Prelude hiding (lines) + +import Pipes.KeyValueCsv + +import Data.Reflection +import Data.Vinyl +import Data.Vinyl.Functor +import Pipes +import Pipes.ByteString (ByteString) +import qualified Pipes.ByteString as Pipes +import Pipes.Safe (SafeT, MonadMask) +import qualified Pipes.Safe as Pipes +import Pipes.Text.Encoding +import System.IO + +streamInBS + :: ( MonadIO m + , MonadMask m + ) + => FilePath + -> Producer ByteString (SafeT m) () +-- ^Read a 'ByteString' stream from file. +streamInBS path = + Pipes.bracket + ( liftIO $ openFile path ReadMode ) + ( liftIO . hClose ) + Pipes.fromHandle + +streamIn + :: forall (m :: * -> *) (f :: k -> *) (g :: j -> *) (hs :: [k]) (rs :: [j]) + . ( Given Delimiter + , MonadIO m + , MonadMask m + , Record hs + ) + => Codec -- ^File encoding. + -> FilePath -- ^Input file. + -> Options (SafeT m) f g hs rs -- ^Parsing options. + -> SafeT m + ( Rec (WithKeyValueError :. f) hs + , Producer (Rec (WithCsvError :. g) rs) (SafeT m) + ( Producer ByteString (SafeT m) () ) + ) +-- ^Read a CSV file. +streamIn codec path options = + parseKeyValueCsv options + . decode codec + $ streamInBS path +
+ src/Pipes/KeyValueCsv/Internal.hs view
@@ -0,0 +1,82 @@+{-| +Module : Pipes.KeyValueCsv.Cell +Copyright : (c) Marcin Mrotek, 2015 +License : BSD3 +Maintainer : marcin.jan.mrotek@gmail.com +Stability : experimental + +Helper functions + +-} + +{-# LANGUAGE RankNTypes #-} + +module Pipes.KeyValueCsv.Internal where + +import Pipes.KeyValueCsv.Internal.Types + +import Control.Lens +import Control.Monad.State.Strict +import Data.Char +import Data.Monoid +import Data.Text (Text) +import qualified Data.Text.Lazy as Lazy +import Data.Text.Lazy.Builder +import Pipes +import qualified Pipes.Prelude as Pipes +import Pipes.Group (FreeT(..), FreeF(..)) +import Pipes.Parse (Parser) +import qualified Pipes.Parse as Parse +import qualified Pipes.Text as PipesText + +build :: Monad m => Parser Text m Lazy.Text +-- ^Build a lazy 'Text' from chunks obtained from a pipe. +build = + Parse.foldAll + (\b t -> b <> fromText t) + mempty + toLazyText + +skipSpace :: Monad m => Parser Text m () +skipSpace = zoom (PipesText.span isSpace) Parse.skipAll + +drawText :: Monad m => Parser Text m Lazy.Text +{-^ +Draw text that may be enclosed in quotes. +Skips initial whitespace. +If the first non-white character is not a quote mark, the function draws only the part of the text upto the next white character. +-} +drawText = do + skipSpace + quote'm <- PipesText.peekChar + case quote'm of + Just '"' -> do + drawn <- Parse.draw *> zoom (PipesText.break isQuote) build + pure drawn + Just _ -> zoom (PipesText.break isSpace) build + Nothing -> pure $ Lazy.empty + where + isQuote = (== '"') + +breakLines' + :: Monad m + => (Text -> Bool) + -> FreeT (Line m) m r + -> FreeT (Line m) m (FreeT (Line m) m r) +{-^ +Break a 'FreeT'-delimited stream of lines into two parts, on a line that satisfies the given predicate. +All input lines up to the breaking one will be fully read, and each (not including the breaking one) will be re-'yield'ed. +-} +breakLines' predicate ls = FreeT $ do + ft <- runFreeT ls + case ft of + p@(Pure _) -> pure . Pure . FreeT $ pure p + Free (Line line) -> do + (text, leftovers) <- runStateT (Lazy.toStrict <$> build) line + r <- runEffect $ leftovers >-> Pipes.drain + pure $ if predicate text + then Pure r + else Free . Line $ do + yield text + pure $ breakLines' predicate r +
+ src/Pipes/KeyValueCsv/Internal/Csv.hs view
@@ -0,0 +1,100 @@+{-| +Module : Pipes.KeyValueCsv.Internal.Csv +Copyright : (c) Marcin Mrotek, 2015 +License : BSD3 +Maintainer : marcin.jan.mrotek@gmail.com +Stability : experimental + +Helper classes and functions. +-} + +{-# LANGUAGE + DataKinds + , FlexibleContexts + , FlexibleInstances + , GADTs + , MultiParamTypeClasses + , PolyKinds + , RankNTypes + , TypeOperators + #-} + +module Pipes.KeyValueCsv.Internal.Csv where + +import Pipes.KeyValueCsv.Internal.Types +import Pipes.KeyValueCsv.Common +import Pipes.KeyValueCsv.Types.Common +import Pipes.KeyValueCsv.Types.Csv + +import Control.Monad.State.Strict +import Data.Reflection (Given(..)) +import Data.Validation +import Data.Vinyl +import Data.Vinyl.Functor +import Pipes +import qualified Pipes.Prelude as Pipes +import Pipes.Group (FreeT(..), FreeF(..)) +import qualified Pipes.Group as Group +import Pipes.Parse.Tutorial() + +parseLine + :: forall (m :: * -> *) (f :: k -> *) (rs :: [k]) (r :: *) + . ( Monad m + , Given Delimiter + ) + => Rec (CellParser m f) rs + -> Line m r + -> Producer (Rec (WithCsvError :. f) rs) m r +-- ^Parse a single row from a stream of text, discard the remaining characters. +parseLine parser = parseCells parser . cells + +parseCells + :: forall (m :: * -> *) (f :: k -> *) (rs :: [k]) (x :: *) + . Monad m + => Rec (CellParser m f) rs -- ^Parser record. + -> Cells m x -- ^Stream of cells. + -> Producer (Rec (WithCsvError :. f) rs) m x +-- ^Parse a row of cells and pack it into a one-shot 'Producer'. +parseCells parser stream = do + (result, end) <- lift $ fromCells parser stream + yield result + pure end + +validateCell :: (Either String :. f) a -> (AccValidation [CsvError] :. f) a +validateCell (Compose e) = Compose r + where + r = case e of + Left err -> AccFailure [CellParseError err] + Right a -> AccSuccess a + +buildFromCells + :: Monad m + => Rec (CellParser m f) (r ': rs) -- ^Parser record. + -> Cells m x -- ^Stream of cells. + -> m ( Rec (WithCsvError :. f) rs -> Rec (WithCsvError :. f) (r ': rs) + , Cells m x + ) +-- ^A helper function for the 'FromCells' type class. Build a 'Rec' incrementally from 'Cells'. +buildFromCells (Compose (WrapParser parser) :& _) (Cells fs) = do + ft <- runFreeT fs + case ft of + Pure r -> pure (\rs -> Compose (AccFailure [MissingCell]) :& rs, Cells $ pure r) + Free (Cell cell) -> do + (result,leftovers) <- runStateT parser cell + end <- runEffect $ leftovers >-> Pipes.drain + pure (\rs -> validateCell result :& rs, Cells end) + +fromCells + :: Monad m + => Rec (CellParser m f) rs -- ^Parser record. + -> Cells m x -- ^Stream of cells. + -> m (Rec (WithCsvError :. f) rs, x) +-- ^Parse a row of cells. For more information about parsing with Pipes, see "Pipes.Parse.Tutorial". +fromCells RNil (Cells fs) = do + end <- runEffect $ Group.concats ( Group.maps unCell fs) >-> Pipes.drain + pure (RNil, end) +fromCells parsers@(_ :& ps) stream = do + (run, remaining) <- buildFromCells parsers stream + (result, end) <- fromCells ps remaining + pure (run result, end) +
+ src/Pipes/KeyValueCsv/Internal/KeyValue.hs view
@@ -0,0 +1,122 @@+{-| +Module : Pipes.KeyValueCsv.Internal.KeyValue +Copyright : (c) Marcin Mrotek, 2015 +License : BSD3 +Maintainer : marcin.jan.mrotek@gmail.com +Stability : experimental + +-} + +{-# LANGUAGE + DataKinds + , FlexibleContexts + , FlexibleInstances + , GADTs + , MonadComprehensions + , MultiParamTypeClasses + , PolyKinds + , ScopedTypeVariables + , TypeOperators + #-} + +module Pipes.KeyValueCsv.Internal.KeyValue + ( module Pipes.KeyValueCsv.Internal.KeyValue + , Record + ) where + +import Pipes.KeyValueCsv.Internal +import Pipes.KeyValueCsv.Internal.Types +import Pipes.KeyValueCsv.Common +import Pipes.KeyValueCsv.Types.Common +import Pipes.KeyValueCsv.Types.KeyValue + +import Control.Monad.State.Strict +import Data.Reflection +import Data.Set (Set) +import qualified Data.Set as Set +import qualified Data.Text.Lazy as Lazy +import Data.Validation +import Data.Vinyl +import Data.Vinyl.Functor +import Data.Vinyl.Utils.Proxy +import Pipes hiding (Proxy) +import qualified Pipes.Prelude as Pipes +import Pipes.Group (FreeT(..), FreeF(..)) +import qualified Pipes.Group as Group + +drawCell + :: Monad m + => StateT (Cells m r) m (Maybe (Lazy.Text)) +-- ^Draw a single cell from a 'FreeT'-delimited stream. +drawCell = StateT $ \(Cells cs) -> do + ft <- runFreeT cs + case ft of + Pure r -> pure (Nothing, Cells $ pure r) + Free (Cell cell) -> do + (text, remaining) <- runStateT drawText cell + r <- runEffect $ remaining >-> Pipes.drain + pure (Just text, Cells r) + +missing :: forall (f :: k -> *) (rs :: [k]). Record rs => Rec (WithKeyValueError :. f) rs +-- ^An empty record filled with 'AccFailure' 'MissingValue'. +missing = recPure (Compose $ AccFailure [MissingValue]) --missing' proxyRecord + +parseKeyValue + :: forall (m :: * -> *) (f :: k -> *) (rs :: [k]) (r :: *) + . ( Given Delimiter + , Monad m + , Record rs + ) + => Rec (CellParser m f) rs -- ^Parser record. + -> Rec (Const Text) rs -- ^Key name record. + -> Set Text -- ^Name set. + -> Line m r -- ^Line to parse. + -> Producer (Rec (WithKeyValueError :. f) rs) m r +-- ^Parse a single line into an one-shot producer of records with at most one 'AccSuccess' field. +parseKeyValue parser names names'set line = do + (key'm, Cells cs) <- lift $ runStateT (fmap Lazy.toStrict <$> drawCell) $ cells line + remaining'f <- lift $ runFreeT cs + case remaining'f of + Pure r -> pure r + cell@(Free (Cell remaining)) -> do + leftovers <- + case [k | k <- key'm, Set.member k names'set] of + Just key -> do + (result, leftovers) <- lift $ fromKeyValues (key, remaining) parser names missing + yield result + pure leftovers + Nothing -> do + yield missing + pure . FreeT $ pure cell + lift + . runEffect + $ (Group.concats . Group.maps unCell) leftovers >-> Pipes.drain + +fromKeyValues + :: forall (m :: * -> *) (f :: k -> *) (rs :: [k]) (x :: *) + . Monad m + => (Text, Producer Text m x) -- ^Key-value pair. + -> Rec (CellParser m f) rs -- ^Parser record. + -> Rec (Const Text) rs -- ^Key name record. + -> Rec (WithKeyValueError :. f) rs -- ^Current parsing result. + -> m (Rec (WithKeyValueError :. f) rs, x) +-- ^Parse a key-value pair into a record with at most one 'AccSuccess' field. +fromKeyValues (_,p) RNil RNil RNil = do + x <- runEffect $ p >-> Pipes.drain + pure (RNil, x) +fromKeyValues _ RNil _ _ = error "impossible due to GADT constraints on Rec" +fromKeyValues pair@(key,value) (Compose (WrapParser parser) :& ps) (Const expected :& ks) (r :& rs) = do + if key == expected + then do + (Compose parsed, leftovers) <- runStateT parser value + result <- runEffect $ leftovers >-> Pipes.drain + let validated = + case parsed of + Left err -> AccFailure [ValueParsingError err] + Right a -> AccSuccess a + pure (Compose validated :& rs, result) + else do + (remaining, result) <- fromKeyValues pair ps ks rs + pure (r :& remaining, result) +fromKeyValues _ (_ :& _) _ _ = error "impossible due to GADT constraints on Rec" +
+ src/Pipes/KeyValueCsv/Internal/Names.hs view
@@ -0,0 +1,30 @@+{-| +Module : Pipes.KeyValueCsv.Internal.Names +Copyright : (c) Marcin Mrotek, 2015 +License : BSD3 +Maintainer : marcin.jan.mrotek@gmail.com +Stability : experimental + +Default key name generation by printing the field label type names. +-} + +{-# LANGUAGE + DataKinds + , GADTs + , PolyKinds + #-} + +module Pipes.KeyValueCsv.Internal.Names where + +import Data.Vinyl +import Data.Vinyl.Functor +import Data.Vinyl.Utils.Field +import Data.Text (Text) +import qualified Data.Text as Text +import Data.Typeable + +names :: Rec (DictProxy Typeable) rs -> Rec (Const Text) rs +-- ^Construct a record of 'Text' from a record of 'Proxy'ies enriched with 'Typeable' constraints. +names RNil = RNil +names (p@DictProxy :& ps) = (Const . Text.pack . show . typeRep $ getProxy p) :& names ps +
+ src/Pipes/KeyValueCsv/Internal/Types.hs view
@@ -0,0 +1,36 @@+{-| +Module : Pipes.KeyValueCsv.Csv +Copyright : (c) Marcin Mrotek, 2015 +License : BSD3 +Maintainer : marcin.jan.mrotek@gmail.com +Stability : experimental + +Abstract types. +-} + +{-# LANGUAGE GeneralizedNewtypeDeriving #-} + +module Pipes.KeyValueCsv.Internal.Types where + +import Data.Text (Text) +import Pipes +import Pipes.Group (FreeT) + +-- |A wrapped 'Char' that has the explicit puropse of delimiting cells. +newtype Delimiter = Delimiter { getDelimiter :: Char } + +-- |A single line. Internally, a stream of text. +newtype Line m r = Line { unLine :: Producer Text m r } + deriving (Functor, Applicative, Monad) +-- |A stream of lines. +newtype Lines m r = Lines { unLines :: FreeT (Line m) m r } + deriving (Functor, Applicative, Monad) + +-- |A single cell. Internally, a stream of text. +newtype Cell m r = Cell { unCell :: Producer Text m r } + deriving (Functor, Applicative, Monad) +-- |A stream of cells. +newtype Cells m r = Cells { unCells :: FreeT (Cell m) m r } + deriving (Functor, Applicative, Monad) + +
+ src/Pipes/KeyValueCsv/KeyValue.hs view
@@ -0,0 +1,84 @@+{-| +Module : Pipes.KeyValueCsv.KeyValue +Copyright : (c) Marcin Mrotek, 2015 +License : BSD3 +Maintainer : marcin.jan.mrotek@gmail.com +Stability : experimental + +Parse key-value pairs. +-} + +{-# LANGUAGE + DataKinds + , ExplicitForAll + , FlexibleContexts + , GADTs + , PolyKinds + , TypeOperators + #-} + +module Pipes.KeyValueCsv.KeyValue + ( module Pipes.KeyValueCsv.KeyValue + , Record + , KeyValueError(..) + ) where + +import Pipes.KeyValueCsv.Internal.KeyValue +import Pipes.KeyValueCsv.Internal.Types +import Pipes.KeyValueCsv.Types.KeyValue + +import Control.Lens +import Data.Reflection +import qualified Data.Set as Set +import Data.Validation +import Data.Vinyl +import Data.Vinyl.Functor +import Pipes +import qualified Pipes.Prelude as Pipes +import qualified Pipes.Group as Group + +parseKeyValues + :: forall (m :: * -> *) (f :: k -> *) (rs :: [k]) (r :: *) + . ( Given Delimiter + , Monad m + , Record rs + ) + => KeyValueOptions m f rs -- ^Options + -> Lines m r -- ^Stream of lines + -> m (Rec (WithKeyValueError :. f) rs, r) +-- ^Parse lines into a stream of records with at most one 'AccSuccess' field. +parseKeyValues options = + foldHeader + . Group.concats + . Group.maps (parseKeyValue (options^.kvParser) names names'set) + . unLines + where + names = options^.keyNames + names'set = Set.fromList $ recordToList names + +foldHeader + :: ( Monad m + , Record rs + ) + => Producer (Rec (WithKeyValueError :. f) rs) m r + -> m (Rec (WithKeyValueError :. f) rs, r) +-- ^Fold a stream of key-value records into a single record. +foldHeader = Pipes.fold' joinRecs missing id + +joinRecs :: Rec (WithKeyValueError :. f) rs -> Rec (WithKeyValueError :. f) rs -> Rec (WithKeyValueError :. f) rs +-- ^Join two 'Rec's, replacing missing values with successes and accumulating failures. +joinRecs RNil RNil = RNil +joinRecs RNil _ = error "impossible" +joinRecs _ RNil= error "impossible" +joinRecs (Compose va :& as) (Compose vb :& bs) = Compose new :& joinRecs as bs + where + new = case (va, vb) of + (AccSuccess a, AccFailure [MissingValue]) -> AccSuccess a + (AccFailure [MissingValue], AccSuccess b) -> AccSuccess b + (AccSuccess _, AccSuccess _) -> AccFailure [MultipleValues] + (AccSuccess _, AccFailure f) -> AccFailure f + (AccFailure f, AccSuccess _) -> AccFailure f + (AccFailure [MissingValue], AccFailure f) -> AccFailure f + (AccFailure f, AccFailure [MissingValue]) -> AccFailure f + (AccFailure f1, AccFailure f2) -> AccFailure $ MultipleValues : (f1++f2) +
+ src/Pipes/KeyValueCsv/Names.hs view
@@ -0,0 +1,35 @@+{-| +Module : Pipes.KeyValueCsv.Names +Copyright : (c) Marcin Mrotek, 2015 +License : BSD3 +Maintainer : marcin.jan.mrotek@gmail.com +Stability : experimental + +Default key name generation by printing the field label type names. +-} + +{-# LANGUAGE + DataKinds + , ExplicitForAll + , PolyKinds + #-} + +module Pipes.KeyValueCsv.Names + ( module Pipes.KeyValueCsv.Names + , FieldAll + , Record + ) where + +import qualified Pipes.KeyValueCsv.Internal.Names as Internal + +import Data.Proxy +import Data.Vinyl +import Data.Vinyl.Functor +import Data.Vinyl.Utils.Field +import Data.Vinyl.Utils.Proxy +import Data.Text (Text) +import Data.Typeable + +names :: forall (rs :: [k]). (FieldAll rs Typeable, Record rs) => Rec (Const Text) rs +names = Internal.names $ reifyFieldConstraint (Proxy :: Proxy Typeable) proxyRecord +
+ src/Pipes/KeyValueCsv/Types.hs view
@@ -0,0 +1,54 @@+{-| +Module : Pipes.KeyValueCsv.Types +Copyright : (c) Marcin Mrotek, 2015 +License : BSD3 +Maintainer : marcin.jan.mrotek@gmail.com +Stability : experimental + +Types used by the key-vale + CSV processing part. +-} + +{-# LANGUAGE + DataKinds + , FlexibleContexts + , PolyKinds + , RankNTypes + , TemplateHaskell + #-} + +module Pipes.KeyValueCsv.Types + ( module Pipes.KeyValueCsv.Types + , module Pipes.KeyValueCsv.Types.Common + , module Pipes.KeyValueCsv.Types.Csv + , module Pipes.KeyValueCsv.Types.KeyValue + ) where + + +import Pipes.KeyValueCsv.Types.Common +import Pipes.KeyValueCsv.Types.Csv +import Pipes.KeyValueCsv.Types.KeyValue + +import Control.Lens +import Data.Default.Class +import Data.Text (Text) + +-- |Options for the whole processing. +data Options (m :: * -> *) (f :: k -> *) (g :: j -> *) (hs :: [k]) (rs :: [j]) = Options + { _kvOptions :: KeyValueOptions m f hs + , _csvOptions :: CsvOptions m g rs + , _predicate :: Text -> Bool + , _delimiter :: Char + } + +makeLenses ''Options + +instance ( Default (KeyValueOptions m f hs) + , Default (CsvOptions m g rs) + ) => Default (Options m f g hs rs) where + def = Options + { _kvOptions = def + , _csvOptions = def + , _predicate = const False + , _delimiter = ',' + } +
+ src/Pipes/KeyValueCsv/Types/Common.hs view
@@ -0,0 +1,40 @@+{-| +Module : Pipes.KeyValueCsv.Types.Common +Copyright : (c) Marcin Mrotek, 2015 +License : BSD3 +Maintainer : marcin.jan.mrotek@gmail.com +Stability : experimental + +Types common to both CSV and key-value part of the library, and reexports of abstract types. +-} + +{-# LANGUAGE + DataKinds + , GADTs + , FlexibleContexts + , ScopedTypeVariables + , PolyKinds + , RankNTypes + , TypeOperators + #-} + +module Pipes.KeyValueCsv.Types.Common + ( Line + , Lines + , Cell + , Cells + , Delimiter + , module Pipes.KeyValueCsv.Types.Common + ) where + +import Pipes.KeyValueCsv.Internal.Types + +import Data.Vinyl.Functor +import Data.Text (Text) +import Pipes.Parse (Parser) + +-- |As 'Parser' is a type synonym, it can't be partially applied as an argument for 'Rec'. +newtype WrapParser (a :: *) (m :: * -> *) (r :: *) = WrapParser { unwrapParser :: Parser a m r } + +-- |Shorthand for a 'Parser' of 'Text' that can return a 'String' error a'la "Data.Text.Read". +type CellParser (m :: * -> *) (f :: k -> *) = WrapParser Text m :. (Either String :. f)
+ src/Pipes/KeyValueCsv/Types/Csv.hs view
@@ -0,0 +1,52 @@+{-| +Module : Pipes.KeyValueCsv.Types.Csv +Copyright : (c) Marcin Mrotek, 2015 +License : BSD3 +Maintainer : marcin.jan.mrotek@gmail.com +Stability : experimental + +Types used by the CSV part of the library. +-} + +{-# LANGUAGE + DataKinds + , GADTs + , RankNTypes + , PolyKinds + , UndecidableInstances + #-} + +module Pipes.KeyValueCsv.Types.Csv where + +import Pipes.KeyValueCsv.Cell +import Pipes.KeyValueCsv.Common +import Pipes.KeyValueCsv.Types.Common + +import Control.Lens +import Data.Vinyl +import Data.Vinyl.TypeLevel +import Data.Vinyl.Utils.Proxy +import Data.Default.Class +import Data.Validation + +data CsvOptions (m :: * -> *) (f :: k -> *) (rs :: [k]) = CsvOptions + { _csvParser :: Rec (CellParser m f) rs + } + +csvParser + :: forall (m :: * -> *) (f :: k -> *) (rs :: [k]) + . Lens' (CsvOptions m f rs) (Rec (CellParser m f) rs) +csvParser = lens _csvParser $ \o p -> o {_csvParser = p} + +instance (RecAll f rs FromCell, Record rs, Monad m) => Default (CsvOptions m f rs) where + def = CsvOptions defaultParser + +-- |An error that occurred during the parsing of a row. +data CsvError + = CellParseError String -- ^Error as returned by "Text.Read". + | MissingCell + deriving (Show, Eq) + +-- |Shorthand for functions that can return 'CsvError's. +type WithCsvError = AccValidation [CsvError] +
+ src/Pipes/KeyValueCsv/Types/KeyValue.hs view
@@ -0,0 +1,59 @@+{-| +Module : Pipes.KeyValueCsv.Types.KeyValue +Copyright : (c) Marcin Mrotek, 2015 +License : BSD3 +Maintainer : marcin.jan.mrotek@gmail.com +Stability : experimental + +Types used by the key-value part of the library. +-} + +{-# LANGUAGE + DataKinds + , PolyKinds + , TemplateHaskell + , UndecidableInstances + #-} + +module Pipes.KeyValueCsv.Types.KeyValue where + +import Pipes.KeyValueCsv.Cell +import Pipes.KeyValueCsv.Common +import Pipes.KeyValueCsv.Names +import Pipes.KeyValueCsv.Types.Common + +import Control.Lens hiding (Const) +import Data.Default.Class +import Data.Typeable +import Data.Vinyl +import Data.Vinyl.Functor +import Data.Vinyl.TypeLevel +import Data.Validation + +-- |An error that occured during the parsing of the header. +data KeyValueError + = MissingValue -- ^A value is missing in the header. + | MultipleValues -- ^A value is given multiple times. + | ValueParsingError String -- ^Parsing error as returned by "Data.Text.Lazy.Read". + deriving (Show, Eq) + +-- |Shorthand for functions that can return 'KeyValueError's. +type WithKeyValueError = AccValidation [KeyValueError] + +data KeyValueOptions (m :: * -> *) (f :: k -> *) (rs :: [k]) = KeyValueOptions + { _kvParser :: Rec (CellParser m f) rs + , _keyNames :: Rec (Const Text) rs + } + +makeLenses ''KeyValueOptions + +instance ( RecAll f rs FromCell + , FieldAll rs Typeable + , Record rs + , Monad m + ) => Default (KeyValueOptions m f rs) where + def = KeyValueOptions + { _kvParser = defaultParser + , _keyNames = names + } +
+ test/Test/Common.hs view
@@ -0,0 +1,46 @@+{-# LANGUAGE + DataKinds + , FlexibleContexts + , FlexibleInstances + , TypeOperators + #-} + +module Test.Common where + +import Data.Functor.Contravariant +import Data.List +import Data.Text.Lazy.Builder +import Data.Vinyl +import Data.Vinyl.Functor +import Data.Vinyl.Utils.Operator + +type Id = Identity + +buildDelimited + :: Buildable rs + => Char + -> Rec Id rs + -> Builder +buildDelimited delim = + mconcat + . intersperse (singleton delim) + . build + +build + :: Buildable rs + => Rec Id rs + -> [Builder] +build r = + map getIdentity + . recordToList + $ builder \$\ r + +class Buildable rs where + builder :: Rec (Op Builder) rs + +instance Buildable '[] where + builder = RNil + +instance (Buildable rs, Show r) => Buildable (r ': rs) where + builder = Op (fromString . show) :& builder +
+ test/Test/Csv.hs view
@@ -0,0 +1,58 @@+{-# LANGUAGE + DataKinds + , FlexibleContexts + , TemplateHaskell + #-} + +module Test.Csv where + +import Prelude hiding (lines) +import Test.Common +import Test.Instances () +import Test.Types + +import Pipes.KeyValueCsv.Common +import Pipes.KeyValueCsv.Csv +import Pipes.KeyValueCsv.Types + +import Control.Lens hiding (Const) +import qualified Data.Functor.Identity as Transformers +import Data.Default.Class +import Data.List hiding (lines) +import qualified Data.Text as Text +import Data.Text.Lazy.Builder +import Data.Reflection +import Data.Validation +import Data.Vinyl +import Data.Vinyl.Utils.Compose +import Test.QuickCheck +import qualified Pipes.Prelude as Pipes +import Pipes.Text (fromLazy) + +delim :: Char +delim = ',' + +type TestFields = '[Int, Alphabetic, Quoted] + +prop_read :: [Rec Id TestFields] -> Property +prop_read rs = map pure rs === useDelimiter delim parsed + & counterexample (concat. map Text.unpack $ Pipes.toList pipe) + where + pipe :: Producer Text Transformers.Identity () + pipe = + fromLazy + . toLazyText + . mconcat + . intersperse (singleton '\n') + $ map (buildDelimited delim) rs + parsed :: Given Delimiter => [AccValidation [CsvError] (Rec Id TestFields)] + parsed = + map rtraverse1 + . Pipes.toList + . parseCsv def + $ lines pipe + +return [] + +test :: IO Bool +test = $quickCheckAll
+ test/Test/Instances.hs view
@@ -0,0 +1,30 @@+{-# LANGUAGE + DataKinds + , FlexibleContexts + , FlexibleInstances + , StandaloneDeriving + , TypeOperators + #-} + +{-# OPTIONS_GHC -fno-warn-orphans #-} + +module Test.Instances where + +import Data.Vinyl +import Data.Vinyl.Functor +import Test.QuickCheck + +instance Arbitrary (Rec f '[]) where + arbitrary = pure RNil + shrink = shrinkNothing + +instance (Arbitrary (Rec f rs), Arbitrary (f r)) => Arbitrary (Rec f (r ': rs)) where + arbitrary = (:&) <$> arbitrary <*> arbitrary + shrink = shrinkNothing + +deriving instance Eq a => Eq (Identity a) + +instance Arbitrary a => Arbitrary (Identity a) where + arbitrary = Identity <$> arbitrary + shrink (Identity a) = Identity <$> shrink a +
+ test/Test/KeyValue.hs view
@@ -0,0 +1,67 @@+{-# LANGUAGE + DataKinds + , FlexibleContexts + , OverloadedStrings + , TemplateHaskell + #-} + +module Test.KeyValue where + +import Prelude hiding (lines) +import Test.Common +import Test.Instances () +import Test.Types + +import Pipes.KeyValueCsv.Common +import Pipes.KeyValueCsv.KeyValue +import Pipes.KeyValueCsv.Types + +import Control.Lens hiding (Const) +import qualified Data.Functor.Identity as Transformers +import Data.Default.Class +import Data.List hiding (lines) +import Data.Monoid +import qualified Data.Text as Text +import Data.Text.Lazy.Builder +import Data.Reflection +import Data.Validation +import Data.Vinyl +import Data.Vinyl.Functor +import Data.Vinyl.Utils.Compose +import Test.QuickCheck +import qualified Pipes.Prelude as Pipes +import Pipes.Text (fromLazy) + +delim :: Char +delim = ',' + +type TestFields = '[Int, Alphabetic, Quoted] + +prop_read :: Rec Id TestFields -> Property +prop_read r = pure r === useDelimiter delim parsed + & counterexample (concat. map Text.unpack $ Pipes.toList pipe) + where + names :: Rec (Const Text) TestFields + names = Const "foo" :& Const "bar" :& Const "baz" :& RNil + pipe :: Producer Text Transformers.Identity () + pipe = + fromLazy + . toLazyText + . mconcat + . intersperse (singleton '\n') + . zipWith keyValue (map fromText $ recordToList names) + $ build r + keyValue k v = k <> singleton delim <> v + parsed :: Given Delimiter => AccValidation [KeyValueError] (Rec Id TestFields) + parsed = + rtraverse1 + . fst + . Transformers.runIdentity + . parseKeyValues (def & keyNames .~ names) + $ lines pipe + +return [] + +test :: IO Bool +test = $quickCheckAll +
+ test/Test/Types.hs view
@@ -0,0 +1,42 @@+module Test.Types where + +import Pipes.KeyValueCsv.Cell + +import Data.Char +import Test.QuickCheck + +newtype Alphabetic = Alphabetic { getAlphabetic :: String } + deriving (Eq) + +instance Show Alphabetic where + show (Alphabetic a) = show a + +alphabetLower :: String +alphabetLower = "abcdefghijklmnopqrstuvwxyz" + +alphabet :: String +alphabet = alphabetLower ++ map toUpper alphabetLower + +instance FromCell Alphabetic where + fromCell = fmap Alphabetic <$> fromCell + +instance Arbitrary Alphabetic where + arbitrary = Alphabetic <$> listOf1 (elements alphabet) + shrink (Alphabetic a) = Alphabetic <$> shrink a + +newtype Quoted = Quoted { quoted :: String } + deriving (Eq) + +instance Show Quoted where + show (Quoted q) = q + +quote :: String -> String +quote s = "\"" ++ s ++ "\"" + +instance Arbitrary Quoted where + arbitrary = Quoted . quote <$> listOf1 (elements $ alphabet ++ " ") + shrink (Quoted q) = Quoted . quote <$> shrink contents + where contents = init $ tail q + +instance FromCell Quoted where + fromCell = fmap (Quoted . quote) <$> fromCell
+ test/test-kvc.hs view
@@ -0,0 +1,17 @@+module Main where + +import qualified Test.Csv as Csv +import qualified Test.KeyValue as KeyValue + +import System.Exit + +main :: IO () +main = do + tests <- sequence + [ Csv.test + , KeyValue.test + ] + if all id tests + then exitSuccess + else exitFailure +