csv-conduit 1.0.0.1 → 1.0.1.0

raw patch · 9 files changed

+67/−34 lines, 9 filesdep ~conduitbinary-addedPVP: major bump suggested

API removals or changes: PVP suggests a major version bump

Dependency ranges changed: conduit

API changes (from Hackage documentation)

Files

• README.md +3/−3
• changelog.md +8/−0
• csv-conduit.cabal +4/−2
• src/Data/CSV/Conduit.hs +18/−18
• src/Data/CSV/Conduit/Parser/ByteString.hs +4/−4
• src/Data/CSV/Conduit/Parser/Text.hs +4/−2
• test/Test.hs +25/−5
• test/test-mac-excel.csv +1/−0
• test/test.xls binary

@@ -1,7 +1,7 @@ # README [![cabal-build](https://github.com/dmvianna/csv-conduit/actions/workflows/cabal.yml/badge.svg)](https://github.com/dmvianna/csv-conduit/actions)-[![stack build](https://github.com/dmvianna/csv-conduit/actions/workflows/stack.yml/badge.svg)](https://github.com/dmvianna/csv-conduit/actions)+build](https://github.com/ozataman/csv-conduit/actions/workflows/cabal.yml/badge.svg)](https://github.com/ozataman/csv-conduit/actions)+[![stack build](https://github.com/ozataman/csv-conduit/actions/workflows/stack.yml/badge.svg)](https://github.com/ozataman/csv-conduit/actions)  ## CSV Files and Haskell @@ -53,7 +53,7 @@ - Dmitry Dzhus (@dzhus) - Niklas Hambüchen (@nh2) - Facundo Domínguez (@facundominguez)-+- Daniel Vianna (@dmvianna)  ### Introduction 

@@ -1,3 +1,11 @@+1.0.1.0+* Use ConduitT instead of ConduitM (prettier type inference with newer conduit imports)++1.0.0.2+* Fixed [#17](https://github.com/ozataman/csv-conduit/issues/17),+  where CSV created with Excel in Mac OS failed to parse due to its+  newline characters.+ 1.0.0.1 * Removed dependencies: mmorph, monad-control, mtl,   unordered-containers, primitive

@@ -1,5 +1,5 @@ name:               csv-conduit-version:            1.0.0.1+version:            1.0.1.0 synopsis:   A flexible, fast, conduit-based CSV parser library for Haskell. @@ -57,8 +57,10 @@ extra-source-files:   changelog.md   README.md+  test/test-mac-excel.csv   test/test.csv   test/Test.hs+  test/test.xls  flag lib-Werror   default: False@@ -89,7 +91,7 @@     , base                >=4       && <5     , blaze-builder     , bytestring-    , conduit             >=1.2.8+    , conduit             >=1.3.0   && <2.0     , conduit-extra     , containers          >=0.3     , data-default

@@ -135,13 +135,13 @@   -----------------------------------------------------------------------------   -- | Turn a stream of 's' into a stream of CSV row type. An example   -- would be parsing a ByteString stream as rows of 'MapRow' 'Text'.-  intoCSV :: (MonadThrow m) => CSVSettings -> ConduitM s r m ()+  intoCSV :: (MonadThrow m) => CSVSettings -> ConduitT s r m ()    -----------------------------------------------------------------------------   -- | Turn a stream of CSV row type back into a stream of 's'. An   -- example would be rendering a stream of 'Row' 'ByteString' rows as   -- 'Text'.-  fromCSV :: Monad m => CSVSettings -> ConduitM r s m ()+  fromCSV :: Monad m => CSVSettings -> ConduitT r s m ()   @@ -213,13 +213,13 @@  ------------------------------------------------------------------------------- fromCSVRow :: (Monad m, IsString s, CSV s r)-           => CSVSettings -> ConduitM r s m ()+           => CSVSettings -> ConduitT r s m () fromCSVRow set = awaitForever $ \row -> mapM_ yield [rowToStr set row, "\n"]    --------------------------------------------------------------------------------intoCSVRow :: (MonadThrow m, AttoparsecInput i) => Parser i (Maybe o) -> ConduitM i o m ()+intoCSVRow :: (MonadThrow m, AttoparsecInput i) => Parser i (Maybe o) -> ConduitT i o m () intoCSVRow p = parse .| puller   where     parse = {-# SCC "conduitParser_p" #-} conduitParser p@@ -243,7 +243,7 @@  ------------------------------------------------------------------------------- intoCSVMap :: (Ord a, MonadThrow m, CSV s [a])-           => CSVSettings -> ConduitM s (MapRow a) m ()+           => CSVSettings -> ConduitT s (MapRow a) m () intoCSVMap set = intoCSV set .| (headers >>= converter)   where     headers = do@@ -298,7 +298,7 @@  ------------------------------------------------------------------------------- fromCSVMap :: (Monad m, IsString s, CSV s [a])-           => CSVSettings -> ConduitM (M.Map k a) s m ()+           => CSVSettings -> ConduitT (M.Map k a) s m () fromCSVMap set = awaitForever push   where     push r = mapM_ yield [rowToStr set (M.elems r), "\n"]@@ -321,7 +321,7 @@ writeHeaders     :: (Monad m, CSV s (Row r), IsString s)     => CSVSettings-    -> ConduitM (MapRow r) s m ()+    -> ConduitT (MapRow r) s m () writeHeaders set = do   mrow <- await   case mrow of@@ -380,14 +380,14 @@     -> Either SomeException (v a) decodeCSV set bs = runST $ runExceptT pipeline   where-    src :: ConduitM () s (ExceptT SomeException (ST s1)) ()+    src :: ConduitT () s (ExceptT SomeException (ST s1)) ()     src = C.sourceList [bs]-    csvConvert :: ConduitM s a (ExceptT SomeException (ST s1)) ()+    csvConvert :: ConduitT s a (ExceptT SomeException (ST s1)) ()     csvConvert = transPipe (ExceptT . runCatchT) csvConvert'-    csvConvert' :: ConduitM s a (CatchT (ST s1)) ()+    csvConvert' :: ConduitT s a (CatchT (ST s1)) ()     csvConvert' = intoCSV set     growthFactor = 10-    sink :: ConduitM a Void.Void (ExceptT SomeException (ST s1)) (v a)+    sink :: ConduitT a Void.Void (ExceptT SomeException (ST s1)) (v a)     sink = sinkVector growthFactor     pipeline :: ExceptT SomeException (ST s1) (v a)     pipeline = runConduit (src .| csvConvert .| sink)@@ -447,11 +447,11 @@     :: (MonadThrow m, CSV s a, CSV s' b)     => CSVSettings     -- ^ Settings to be used for both input and output-    -> ConduitM () s m ()+    -> ConduitT () s m ()     -- ^ A raw stream data source. Ex: 'sourceFile inFile'-    -> ConduitM a b m ()+    -> ConduitT a b m ()     -- ^ A transforming conduit-    -> ConduitM s' Void.Void m ()+    -> ConduitT s' Void.Void m ()     -- ^ A raw stream data sink. Ex: 'sinkFile outFile'     -> m () transformCSV set = transformCSV' set set@@ -475,11 +475,11 @@     -- ^ Settings to be used for input     -> CSVSettings     -- ^ Settings to be used for output-    -> ConduitM () s m ()+    -> ConduitT () s m ()     -- ^ A raw stream data source. Ex: 'sourceFile inFile'-    -> ConduitM a b m ()+    -> ConduitT a b m ()     -- ^ A transforming conduit-    -> ConduitM s' Void.Void m ()+    -> ConduitT s' Void.Void m ()     -- ^ A raw stream data sink. Ex: 'sinkFile outFile'     -> m () transformCSV' setIn setOut source c sink = runConduit $@@ -500,7 +500,7 @@  ------------------------------------------------------------------------------- -- | An efficient sink that incrementally grows a vector from the input stream-sinkVector :: (PrimMonad m, GV.Vector v a) => Int -> ConduitM a o m (v a)+sinkVector :: (PrimMonad m, GV.Vector v a) => Int -> ConduitT a o m (v a) sinkVector by = do     v <- lift $ GMV.new by     go 0 v

@@ -58,15 +58,17 @@ row :: CSVSettings -> Parser (Maybe (Row ByteString)) row csvs = csvrow csvs <|> badrow +csvEndOfLine :: Parser ()+csvEndOfLine = (word8 10 >> return ()) <|> (word8 13 >> return ())  badrow :: Parser (Maybe (Row ByteString)) badrow = P.takeWhile (not . C8.isEndOfLine) *>-         (C8.endOfLine <|> C8.endOfInput) *> return Nothing+         (csvEndOfLine <|> C8.endOfInput) *> return Nothing  csvrow :: CSVSettings -> Parser (Maybe (Row ByteString)) csvrow c =   let rowbody = (quotedField' <|> field c) `sepBy` C8.char (csvSep c)-      properrow = rowbody <* (C8.endOfLine <|> P.endOfInput)+      properrow = rowbody <* (csvEndOfLine <|> P.endOfInput)       quotedField' = case csvQuoteChar c of           Nothing -> mzero           Just q' -> try (quotedField q')@@ -93,5 +95,3 @@   f <- many (C8.notChar c <|> quoted)   _ <- C8.char c   return $ B8.pack f--

@@ -58,15 +58,17 @@ row :: CSVSettings -> Parser (Maybe (Row Text)) row csvs = csvrow csvs <|> badrow +csvEndOfLine :: Parser ()+csvEndOfLine = (char '\n' >> return ()) <|> (string (T.pack "\r\n") >> return ()) <|> (char '\r' >> return ())  badrow :: Parser (Maybe (Row Text)) badrow = P.takeWhile (not . T.isEndOfLine) *>-         (T.endOfLine <|> T.endOfInput) *> return Nothing+         (csvEndOfLine <|> T.endOfInput) *> return Nothing  csvrow :: CSVSettings -> Parser (Maybe (Row Text)) csvrow c =   let rowbody = (quotedField' <|> field c) `sepBy` T.char (csvSep c)-      properrow = rowbody <* (T.endOfLine <|> P.endOfInput)+      properrow = rowbody <* (csvEndOfLine <|> P.endOfInput)       quotedField' = case csvQuoteChar c of           Nothing -> mzero           Just q' -> try (quotedField q')

@@ -32,7 +32,9 @@ baseTests :: [Test] baseTests =   [ testCase "mapping with id works" test_identityMap,-    testCase "simple parsing works" test_simpleParse,+    testCase "simple parsing works" (test_simpleParse testFile1),+    testCase "simple parsing works for Mac-Excel" (test_simpleParse testFile3),+    testCase "fails parsing gracefully" test_parseFail,     testCase "OrderedMap" test_orderedMap   ] @@ -74,9 +76,9 @@     f :: Row Text -> [Row Text]     f = return -test_simpleParse :: IO ()-test_simpleParse = do-  (d :: V.Vector (MapRow B.ByteString)) <- readCSVFile csvSettings testFile1+test_simpleParse :: FilePath -> IO ()+test_simpleParse fp = do+  (d :: V.Vector (MapRow B.ByteString)) <- readCSVFile csvSettings fp   V.mapM_ assertRow d   where     assertRow r = v3 @=? (v1 + v2)@@ -85,6 +87,20 @@         v2 = readBS $ r Map.! "Col3"         v3 = readBS $ r Map.! "Sum" +test_parseFail :: IO ()+test_parseFail = do+  (d :: V.Vector (MapRow B.ByteString)) <- readCSVFile csvSettings testXLS+  errored <- catch (V.mapM_ assertRow d >> pure False) handler+  if errored then pure () else assertFailure "readCSVFile shouldn't read XLS"+  where+    handler :: ErrorCall -> IO Bool+    handler _ = pure True+    assertRow r = v3 @=? (v1 + v2)+      where+        v1 = readBS $ r Map.! "Col2"+        v2 = readBS $ r Map.! "Col3"+        v3 = readBS $ r Map.! "Sum"+ test_orderedMap :: IO () test_orderedMap = do   unorderedRes <-@@ -109,9 +125,13 @@ csvSettings :: CSVSettings csvSettings = defCSVSettings {csvQuoteCharAndStyle = Just ('`', DontQuoteEmpty)} -testFile1, testFile2 :: FilePath+testFile1, testFile2, testFile3 :: FilePath testFile1 = "test/test.csv" testFile2 = "test/test.csv"+testFile3 = "test/test-mac-excel.csv"++testXLS :: FilePath+testXLS = "test/test.xls"  readBS :: B.ByteString -> Int readBS = read . B.unpack

@@ -0,0 +1,1 @@+`Col1`,`Col2`,`Col3`,`Sum`
`A`,`2`,`3`,`5`
`B`,`3`,`4`,`7`
`Field using the quote char ``this is the in-quoted value```,`4`,`5`,`9`

binary file changed (absent → 6144 bytes)