Frames-dsv (empty) → 0.1.0.0

raw patch · 8 files changed

+234/−0 lines, 8 filesdep +Framesdep +Frames-dsvdep +basesetup-changed

Dependencies added: Frames, Frames-dsv, base, bytestring, hspec, hw-dsv, pipes, template-haskell, text, vector, vinyl

Files

• CHANGELOG.md +3/−0
• Frames-dsv.cabal +45/−0
• LICENSE +30/−0
• README.md +1/−0
• Setup.hs +2/−0
• src/Frames/Dsv.hs +127/−0
• test/Spec.hs +18/−0
• test/data/multiline.csv +8/−0

@@ -0,0 +1,3 @@+# 0.1++First release of an `hw-dsv`-based CSV parser for use with the `Frames` package.

@@ -0,0 +1,45 @@+name:                Frames-dsv+version:             0.1.0.0+synopsis:            Alternative CSV parser for the Frames package+description: Alternative CSV parser for the Frames package. In cases+             where the built-in Frames CSV parser does not work+             (e.g. when dealing with data files that include newlines+             embedded in data values), the parsers exposed here may+             help.+homepage:            https://github.com/acowley/Frames-dsv#readme+license:             BSD3+license-file:        LICENSE+author:              Anthony Cowley+maintainer:          acowley@gmail.com+copyright:           2018 Anthony Cowley+category:            Data+build-type:          Simple+extra-source-files:  README.md+                     CHANGELOG.md+                     test/data/multiline.csv+cabal-version:       >=1.10++library+  hs-source-dirs:      src+  exposed-modules:     Frames.Dsv+  build-depends:       base >= 4.7 && < 5+                     , bytestring+                     , Frames >= 0.6+                     , vinyl+                     , hw-dsv >= 0.3 && < 0.4+                     , pipes >= 4.1 && < 5+                     , text+                     , vector+                     , template-haskell+  default-language:    Haskell2010++source-repository head+  type:     git+  location: https://github.com/acowley/Frames-dsv++test-suite spec+  type:                     exitcode-stdio-1.0+  hs-source-dirs:           test+  main-is:                  Spec.hs+  build-depends:            base, Frames, Frames-dsv, pipes, hspec+  default-language:         Haskell2010

@@ -0,0 +1,30 @@+Copyright Author name here (c) 2018++All rights reserved.++Redistribution and use in source and binary forms, with or without+modification, are permitted provided that the following conditions are met:++    * Redistributions of source code must retain the above copyright+      notice, this list of conditions and the following disclaimer.++    * Redistributions in binary form must reproduce the above+      copyright notice, this list of conditions and the following+      disclaimer in the documentation and/or other materials provided+      with the distribution.++    * Neither the name of Author name here nor the names of other+      contributors may be used to endorse or promote products derived+      from this software without specific prior written permission.++THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

@@ -0,0 +1,1 @@+Use the [hw-dsv](https://hackage.haskell.org/package/hw-dsv) package to parse CSV data into the [Frames](https://hackage.haskell.org/package/Frames) Haskell data frames library.

@@ -0,0 +1,2 @@+import Distribution.Simple+main = defaultMain

@@ -0,0 +1,127 @@+{-# LANGUAGE TypeOperators #-}+module Frames.Dsv where+import Control.Monad (when)+import qualified Data.ByteString as BS+import qualified Data.ByteString.Lazy as LBS+import Data.Maybe (isNothing)+import Data.Text (Text)+import qualified Data.Text as T+import qualified Data.Text.Encoding as T+import qualified Data.Vector as V+import Data.Vinyl+import Data.Vinyl.Functor ((:.), Compose(..))+import Data.Word (Word8)++import Frames (recMaybe, Record)+import Frames.CSV (defaultParser, pipeTableMaybeOpt)+import Frames.CSV (ReadRec(..), Separator, ParserOptions(..))+import Frames.TH (rowGen, RowGen(..), tableTypes')++import qualified HaskellWorks.Data.Dsv.Lazy.Cursor as SVL++import Pipes (MonadIO, Producer, (>->), yield)+import qualified Pipes as P+import qualified Pipes.Prelude as P++import Language.Haskell.TH++-- * Row Reading++-- | Produce one DSV row at a time.+rowLoop :: Monad m => SVL.DsvCursor -> Producer [BS.ByteString] m ()+rowLoop c =+  if SVL.dsvCursorPosition d > SVL.dsvCursorPosition c && not (SVL.atEnd c)+  then do yield (V.toList (SVL.getRowBetweenStrict c d dEnd))+          rowLoop (SVL.trim d)+  else return ()+  where nr = SVL.nextRow c+        d = SVL.nextPosition nr+        dEnd = SVL.atEnd nr++-- | Produce rows of raw 'LBS.ByteString' values.+dsvRowsByte :: MonadIO m => FilePath -> Word8 -> Producer [BS.ByteString] m ()+dsvRowsByte fp columnSeparator =+  do bs <- P.liftIO (LBS.readFile fp)+     rowLoop (SVL.makeCursor columnSeparator bs)++-- | Produce rows of UTF-8 encoded 'Text' values.+dsvRows' :: MonadIO m => FilePath -> Word8 -> Producer [Text] m ()+dsvRows' fp = (>-> P.map (map T.decodeUtf8)) . dsvRowsByte fp++-- | Produce rows of Latin-1 (aka ISO-8859-1) encoded 'Text' values.+dsvRowsLatin1' :: MonadIO m => FilePath -> Word8 -> Producer [Text] m ()+dsvRowsLatin1' fp = (>-> P.map (map T.decodeLatin1)) . dsvRowsByte fp++-- | Call 'error' indicating the problem with a separator intended for+-- use with the @hw-dsv@ library.+dsvSepErr :: DsvParserOptionsError -> a+dsvSepErr = error . ("DSV separator must be a single character: "++) . show++-- | Produce rows of UTF-8 encoded 'Text' values.+dsvRows :: MonadIO m => FilePath -> Separator -> Producer [Text] m ()+dsvRows fp = either dsvSepErr (dsvRows' fp) . separatorWord8++-- | Produce rows of Latin-1 (aka ISO-8859-1) encoded 'Text' values.+dsvRowsLatin1 :: MonadIO m => FilePath -> Separator -> Producer [Text] m ()+dsvRowsLatin1 fp = either dsvSepErr (dsvRowsLatin1' fp) . separatorWord8++-- | The ways in which an arbitrary 'Text' value may be unsuitable for+-- use as a separator for the @hw-dsv@ package.+data DsvParserOptionsError = SeparatorIsNull+                           | SeparatorCharIsMoreThanOneByte+                           | SeparatorIsMoreThanOneChar+  deriving (Eq, Show)++-- | The @Frames@ library supports column separators that can be+-- arbitrary 'Text' values, but the @hw-dsv@ library requires a single+-- byte be used to demarcate values. If the given 'Text' can be+-- losslessly represented as a single byte, we sue it, otherwise we+-- return an error indicating the problem.+separatorWord8 :: Separator -> Either DsvParserOptionsError Word8+separatorWord8 sep =+  case T.uncons sep of+    Nothing -> Left SeparatorIsNull+    Just (h, t)+      | T.null t -> let i = fromEnum h+                    in if i < 256+                       then Right (fromIntegral i)+                       else Left SeparatorCharIsMoreThanOneByte+      | otherwise -> Left SeparatorIsMoreThanOneChar++-- * Whole Table Reading++-- | Produce rows where any given entry can fail to parse.+readDsvTableMaybeOpt :: (MonadIO m, ReadRec rs, RMap rs)+                     => ParserOptions+                     -> FilePath+                     -> P.Producer (Rec (Maybe :. ElField) rs) m ()+readDsvTableMaybeOpt opts csvFile =+  dsvRows csvFile (columnSeparator opts)  >-> pipeTableMaybeOpt opts+{-# INLINE readDsvTableMaybeOpt #-}++-- | Returns a producer of rows for which each column was successfully+-- parsed.+readDsvTableOpt :: (MonadIO m, ReadRec rs, RMap rs)+                => ParserOptions -> FilePath -> P.Producer (Record rs) m ()+readDsvTableOpt opts csvFile = readDsvTableMaybeOpt opts csvFile P.>-> go+  where go = P.await >>= maybe go (\x -> P.yield x >> go) . recMaybe+{-# INLINE readDsvTableOpt #-}++-- | Returns a producer of rows for which each column was successfully+-- parsed.+readDsvTable :: (MonadIO m, ReadRec rs, RMap rs)+             => FilePath -> P.Producer (Record rs) m ()+readDsvTable = readDsvTableOpt defaultParser+{-# INLINE readDsvTable #-}++-- * Template Haskell++-- | Like 'tableType', but additionally generates a type synonym for+-- each column, and a proxy value of that type. If the CSV file has+-- column names \"foo\", \"bar\", and \"baz\", then this will declare+-- @type Foo = "foo" :-> Int@, for example, @foo = rlens \@Foo@, and+-- @foo' = rlens' \@Foo@.+dsvTableTypes :: String -> FilePath -> DecsQ+dsvTableTypes n fp =+  tableTypes' (rowGen fp) { rowTypeName = n+                          , lineReader = dsvRows fp }

@@ -0,0 +1,18 @@+{-# LANGUAGE DataKinds, FlexibleContexts, TemplateHaskell,+             TypeApplications #-}+module Main (main) where+import Data.Foldable (toList)+import Test.Hspec as H+import Frames+import Frames.Dsv++dsvTableTypes "Multi" "test/data/multiline.csv"++main :: IO ()+main = hspec $ do+  describe "Embedded newlines" $ do+    frame <- H.runIO $+      (inCoreAoS (readDsvTable "test/data/multiline.csv")+       :: IO (Frame Multi))+    it "Parses the file" $+      rget @X (toList frame !! 2) `shouldBe` 30

@@ -0,0 +1,8 @@+RowNum,Description,X,Y+1,"simple",10,10+2,"""quoted""",20,20+3,"multi+line+text+field",30,30+4,"simple again",40,40