hw-dsv 0.3.3 → 0.3.4

raw patch · 3 files changed

+47/−12 lines, 3 files

Files

• hw-dsv.cabal +1/−1
• src/HaskellWorks/Data/Dsv/Internal/Vector.hs +4/−4
• test/HaskellWorks/Data/Dsv/Lazy/CursorSpec.hs +42/−7

@@ -3,7 +3,7 @@ -- see: https://github.com/sol/hpack  name:           hw-dsv-version:        0.3.3+version:        0.3.4 synopsis:       Unbelievably fast streaming DSV file parser description:    Please see the README on Github at <https://github.com/haskell-works/hw-dsv#readme> category:       Text, CSV, SIMD, Succinct Data Structures, Data Structures

@@ -57,10 +57,10 @@           let enters = pdep (oddsMask .<. (0x1 .&.      pc)) qq           let leaves = pdep (oddsMask .<. (0x1 .&. comp pc)) qq -          let compLeaves = comp leaves-          let preQuoteMask = enters + compLeaves-          let quoteMask = preQuoteMask + carry-          let newCarry = preQuoteMask `ltWord` enters+          let compLeaves    = comp leaves+          let preQuoteMask  = enters + compLeaves+          let quoteMask     = preQuoteMask + carry+          let newCarry      = quoteMask `ltWord` (enters .|. compLeaves .|. carry)            DVSM.unsafeWrite tmks i ((nl .|. mk) .&. quoteMask)           DVSM.unsafeWrite tnls i ( nl         .&. quoteMask)

@@ -4,13 +4,14 @@ module HaskellWorks.Data.Dsv.Lazy.CursorSpec (spec) where  import Data.Semigroup                      ((<>))+import HaskellWorks.Data.Bits.BitShown import HaskellWorks.Data.Dsv.Internal.Char (comma) import HaskellWorks.Hspec.Hedgehog import Hedgehog import Test.Hspec  import qualified Data.ByteString.Lazy              as LBS-import qualified Data.Vector                       as V+import qualified Data.Vector                       as DV import qualified HaskellWorks.Data.Dsv.Lazy.Cursor as SVL  {-# ANN module ("HLint: ignore Redundant do"        :: String) #-}@@ -18,7 +19,7 @@ {-# ANN module ("HLint: ignore Redundant bracket"   :: String) #-}  subjectEmpty, subjectSS, subjectSM, subjectMS, subjectMM :: LBS.ByteString-expectedEmpty, expectedSS, expectedSM, expectedMS, expectedMM :: [V.Vector LBS.ByteString]+expectedEmpty, expectedSS, expectedSM, expectedMS, expectedMM :: [DV.Vector LBS.ByteString] subjectEmpty = "" expectedEmpty = mkExpected [] subjectSS = "hello"@@ -30,18 +31,18 @@ subjectMM = "hello,goodbye\nyes,no" expectedMM = mkExpected [["hello","goodbye"],["yes","no"]] -testToListVector :: LBS.ByteString -> [V.Vector LBS.ByteString]+testToListVector :: LBS.ByteString -> [DV.Vector LBS.ByteString] testToListVector = SVL.toListVector . SVL.makeCursor comma  testToListList :: LBS.ByteString -> [[LBS.ByteString]]-testToListList = fmap V.toList . testToListVector+testToListList = fmap DV.toList . testToListVector  -- Adds a terminal newline to the file-testToListVector' :: LBS.ByteString -> [V.Vector LBS.ByteString]+testToListVector' :: LBS.ByteString -> [DV.Vector LBS.ByteString] testToListVector' = testToListVector . (<> "\n") -mkExpected :: [[LBS.ByteString]] -> [V.Vector LBS.ByteString]-mkExpected = fmap V.fromList+mkExpected :: [[LBS.ByteString]] -> [DV.Vector LBS.ByteString]+mkExpected = fmap DV.fromList  spec :: Spec spec = describe "HaskellWorks.Data.Dsv.Lazy.CursorSpec" $ do@@ -205,3 +206,37 @@           , "1.0e-2"           ]         ]+  describe "single quoted field with newline at 127-129" $ do+    it "newline at 127 bytes" $ requireTest $ do+      let newlinePos = 127+      let text = "\"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\n\""+      let expected = [["\"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\n\""]]+      let cursor = SVL.makeCursor 44 text+      _ <- forAll $ pure $ BitShown (head (SVL.dsvCursorMarkers  cursor))+      _ <- forAll $ pure $ BitShown (head (SVL.dsvCursorNewlines cursor))+      let actual = fmap DV.toList (DV.toList (SVL.toVectorVector cursor))+      LBS.length text === newlinePos + 2+      LBS.index text newlinePos === 10+      actual === expected+    it "newline at 128 bytes" $ requireTest $ do+      let newlinePos = 128+      let text = "\"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\n\""+      let expected = [["\"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\n\""]]+      let cursor = SVL.makeCursor 44 text+      _ <- forAll $ pure $ BitShown (head (SVL.dsvCursorMarkers  cursor))+      _ <- forAll $ pure $ BitShown (head (SVL.dsvCursorNewlines cursor))+      let actual = fmap DV.toList (DV.toList (SVL.toVectorVector cursor))+      LBS.length text === newlinePos + 2+      LBS.index text newlinePos === 10+      actual === expected+    it "newline at 129 bytes" $ requireTest $ do+      let newlinePos = 129+      let text = "\"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\n\""+      let expected = [["\"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\n\""]]+      let cursor = SVL.makeCursor 44 text+      _ <- forAll $ pure $ BitShown (head (SVL.dsvCursorMarkers  cursor))+      _ <- forAll $ pure $ BitShown (head (SVL.dsvCursorNewlines cursor))+      let actual = fmap DV.toList (DV.toList (SVL.toVectorVector cursor))+      LBS.length text === newlinePos + 2+      LBS.index text newlinePos === 10+      actual === expected