hw-dsv 0.3.0 → 0.3.1
raw patch · 3 files changed
+150/−9 lines, 3 filesPVP ok
version bump matches the API change (PVP)
API changes (from Hackage documentation)
Files
- hw-dsv.cabal +2/−2
- src/HaskellWorks/Data/Dsv/Lazy/Cursor.hs +4/−1
- test/HaskellWorks/Data/Dsv/Lazy/CursorSpec.hs +144/−6
hw-dsv.cabal view
@@ -3,7 +3,7 @@ -- see: https://github.com/sol/hpack name: hw-dsv-version: 0.3.0+version: 0.3.1 synopsis: Unbelievably fast streaming DSV file parser description: Please see the README on Github at <https://github.com/haskell-works/hw-dsv#readme> category: Text, CSV, SIMD, Succinct Data Structures, Data Structures@@ -14,7 +14,7 @@ copyright: 2018 John Ky license: BSD3 license-file: LICENSE-tested-with: GHC == 8.4.3, GHC == 8.2.2, GHC == 8.0.2, GHC == 7.10.3+tested-with: GHC == 8.6.1, GHC == 8.4.3, GHC == 8.2.2, GHC == 8.0.2, GHC == 7.10.3 build-type: Simple cabal-version: >= 1.10
src/HaskellWorks/Data/Dsv/Lazy/Cursor.hs view
@@ -33,12 +33,15 @@ import qualified HaskellWorks.Data.Dsv.Internal.Vector as DVS import qualified HaskellWorks.Data.Simd.Comparison as DVS +empty64 :: DVS.Vector Word64+empty64 = DVS.replicate 64 0+ makeIndexes :: [DVS.Vector Word64] -> [DVS.Vector Word64] -> [DVS.Vector Word64] -> ([DVS.Vector Word64], [DVS.Vector Word64]) makeIndexes ds ns qs = unzip $ go 0 0 ds ns qs where go pc carry (dv:dvs) (nv:nvs) (qv:qvs) = let (dv', nv', pc', carry') = DVS.indexCsvChunk pc carry dv nv qv in (dv', nv'):go pc' carry' dvs nvs qvs- go _ _ [] [] [] = []+ go _ _ [] [] [] = [(empty64, empty64)] go _ _ _ _ _ = error "Unbalanced inputs" makeCursor :: Word8 -> LBS.ByteString -> DsvCursor
test/HaskellWorks/Data/Dsv/Lazy/CursorSpec.hs view
@@ -1,18 +1,17 @@-{-# LANGUAGE BangPatterns #-} {-# LANGUAGE OverloadedStrings #-} {-# LANGUAGE ScopedTypeVariables #-} module HaskellWorks.Data.Dsv.Lazy.CursorSpec (spec) where -import Data.Semigroup ((<>))+import Data.Semigroup ((<>))+import HaskellWorks.Data.Dsv.Internal.Char (comma) import HaskellWorks.Hspec.Hedgehog import Hedgehog import Test.Hspec -import qualified Data.ByteString.Lazy as LBS-import qualified Data.Vector as V-import qualified HaskellWorks.Data.Dsv.Lazy.Cursor as SVL-import HaskellWorks.Data.Dsv.Internal.Char (comma)+import qualified Data.ByteString.Lazy as LBS+import qualified Data.Vector as V+import qualified HaskellWorks.Data.Dsv.Lazy.Cursor as SVL {-# ANN module ("HLint: ignore Redundant do" :: String) #-} {-# ANN module ("HLint: ignore Reduce duplication" :: String) #-}@@ -34,6 +33,9 @@ testToListVector :: LBS.ByteString -> [V.Vector LBS.ByteString] testToListVector = SVL.toListVector . SVL.makeCursor comma +testToListList :: LBS.ByteString -> [[LBS.ByteString]]+testToListList = fmap V.toList . testToListVector+ -- Adds a terminal newline to the file testToListVector' :: LBS.ByteString -> [V.Vector LBS.ByteString] testToListVector' = testToListVector . (<> "\n")@@ -66,4 +68,140 @@ testToListVector' subjectMS === expectedMS it "Case 10" $ requireTest $ do testToListVector' subjectMM === expectedMM+ describe "boundaries" $ do+ it "513 bytes" $ requireTest $ do+ let actual = testToListList $ LBS.intercalate "\n"+ [ "id,group_id,arm,first_name,last_name,email,contact_number,start_date,status,dose_smart_id,fitbit_id,glyco_id,sms_reminder,email_reminder,inactive_reminder,starting_weight,starting_bmi,starting_hba1c"+ , "10,aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa,1,aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa,aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa,aaaaaaa,aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa,2018-01-01 00:00:00 UTC,Active,,,,Never,Weekly,Never,100.01,10.01,1.0e-2"+ ]+ actual ===+ [ [ "id"+ , "group_id"+ , "arm"+ , "first_name"+ , "last_name"+ , "email"+ , "contact_number"+ , "start_date"+ , "status"+ , "dose_smart_id"+ , "fitbit_id"+ , "glyco_id"+ , "sms_reminder"+ , "email_reminder"+ , "inactive_reminder"+ , "starting_weight"+ , "starting_bmi"+ , "starting_hba1c"+ ]+ , [ "10"+ , "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"+ , "1"+ , "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"+ , "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"+ , "aaaaaaa"+ , "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"+ , "2018-01-01 00:00:00 UTC"+ , "Active"+ , ""+ , ""+ , ""+ , "Never"+ , "Weekly"+ , "Never"+ , "100.01"+ , "10.01"+ , "1.0e-2"+ ]+ ]+ it "512 bytes" $ requireTest $ do+ let actual = testToListList $ LBS.intercalate "\n"+ [ "id,group_id,arm,first_name,last_name,email,contact_number,start_date,status,dose_smart_id,fitbit_id,glyco_id,sms_reminder,email_reminder,inactive_reminder,starting_weight,starting_bmi,starting_hba1c"+ , "10,aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa,1,aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa,aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa,aaaaaaa,aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa,2018-01-01 00:00:00 UTC,Active,,,,Never,Weekly,Never,100.01,10.01,1.0e-2"+ ]+ actual ===+ [ [ "id"+ , "group_id"+ , "arm"+ , "first_name"+ , "last_name"+ , "email"+ , "contact_number"+ , "start_date"+ , "status"+ , "dose_smart_id"+ , "fitbit_id"+ , "glyco_id"+ , "sms_reminder"+ , "email_reminder"+ , "inactive_reminder"+ , "starting_weight"+ , "starting_bmi"+ , "starting_hba1c"+ ]+ , [ "10"+ , "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"+ , "1"+ , "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"+ , "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"+ , "aaaaaaa"+ , "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"+ , "2018-01-01 00:00:00 UTC"+ , "Active"+ , ""+ , ""+ , ""+ , "Never"+ , "Weekly"+ , "Never"+ , "100.01"+ , "10.01"+ , "1.0e-2"+ ]+ ] + it "511 bytes" $ requireTest $ do+ let actual = testToListList $ LBS.intercalate "\n"+ [ "id,group_id,arm,first_name,last_name,email,contact_number,start_date,status,dose_smart_id,fitbit_id,glyco_id,sms_reminder,email_reminder,inactive_reminder,starting_weight,starting_bmi,starting_hba1c"+ , "10,aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa,1,aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa,aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa,aaaaaaa,aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa,2018-01-01 00:00:00 UTC,Active,,,,Never,Weekly,Never,100.01,10.01,1.0e-2"+ ]+ actual ===+ [ [ "id"+ , "group_id"+ , "arm"+ , "first_name"+ , "last_name"+ , "email"+ , "contact_number"+ , "start_date"+ , "status"+ , "dose_smart_id"+ , "fitbit_id"+ , "glyco_id"+ , "sms_reminder"+ , "email_reminder"+ , "inactive_reminder"+ , "starting_weight"+ , "starting_bmi"+ , "starting_hba1c"+ ]+ , [ "10"+ , "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"+ , "1"+ , "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"+ , "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"+ , "aaaaaaa"+ , "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"+ , "2018-01-01 00:00:00 UTC"+ , "Active"+ , ""+ , ""+ , ""+ , "Never"+ , "Weekly"+ , "Never"+ , "100.01"+ , "10.01"+ , "1.0e-2"+ ]+ ]