diff --git a/hw-dsv.cabal b/hw-dsv.cabal --- a/hw-dsv.cabal +++ b/hw-dsv.cabal @@ -3,7 +3,7 @@ -- see: https://github.com/sol/hpack name: hw-dsv -version: 0.3.0 +version: 0.3.1 synopsis: Unbelievably fast streaming DSV file parser description: Please see the README on Github at category: Text, CSV, SIMD, Succinct Data Structures, Data Structures @@ -14,7 +14,7 @@ copyright: 2018 John Ky license: BSD3 license-file: LICENSE -tested-with: GHC == 8.4.3, GHC == 8.2.2, GHC == 8.0.2, GHC == 7.10.3 +tested-with: GHC == 8.6.1, GHC == 8.4.3, GHC == 8.2.2, GHC == 8.0.2, GHC == 7.10.3 build-type: Simple cabal-version: >= 1.10 diff --git a/src/HaskellWorks/Data/Dsv/Lazy/Cursor.hs b/src/HaskellWorks/Data/Dsv/Lazy/Cursor.hs --- a/src/HaskellWorks/Data/Dsv/Lazy/Cursor.hs +++ b/src/HaskellWorks/Data/Dsv/Lazy/Cursor.hs @@ -33,12 +33,15 @@ import qualified HaskellWorks.Data.Dsv.Internal.Vector as DVS import qualified HaskellWorks.Data.Simd.Comparison as DVS +empty64 :: DVS.Vector Word64 +empty64 = DVS.replicate 64 0 + makeIndexes :: [DVS.Vector Word64] -> [DVS.Vector Word64] -> [DVS.Vector Word64] -> ([DVS.Vector Word64], [DVS.Vector Word64]) makeIndexes ds ns qs = unzip $ go 0 0 ds ns qs where go pc carry (dv:dvs) (nv:nvs) (qv:qvs) = let (dv', nv', pc', carry') = DVS.indexCsvChunk pc carry dv nv qv in (dv', nv'):go pc' carry' dvs nvs qvs - go _ _ [] [] [] = [] + go _ _ [] [] [] = [(empty64, empty64)] go _ _ _ _ _ = error "Unbalanced inputs" makeCursor :: Word8 -> LBS.ByteString -> DsvCursor diff --git a/test/HaskellWorks/Data/Dsv/Lazy/CursorSpec.hs b/test/HaskellWorks/Data/Dsv/Lazy/CursorSpec.hs --- a/test/HaskellWorks/Data/Dsv/Lazy/CursorSpec.hs +++ b/test/HaskellWorks/Data/Dsv/Lazy/CursorSpec.hs @@ -1,18 +1,17 @@ -{-# LANGUAGE BangPatterns #-} {-# LANGUAGE OverloadedStrings #-} {-# LANGUAGE ScopedTypeVariables #-} module HaskellWorks.Data.Dsv.Lazy.CursorSpec (spec) where -import Data.Semigroup ((<>)) +import Data.Semigroup ((<>)) +import HaskellWorks.Data.Dsv.Internal.Char (comma) import HaskellWorks.Hspec.Hedgehog import Hedgehog import Test.Hspec -import qualified Data.ByteString.Lazy as LBS -import qualified Data.Vector as V -import qualified HaskellWorks.Data.Dsv.Lazy.Cursor as SVL -import HaskellWorks.Data.Dsv.Internal.Char (comma) +import qualified Data.ByteString.Lazy as LBS +import qualified Data.Vector as V +import qualified HaskellWorks.Data.Dsv.Lazy.Cursor as SVL {-# ANN module ("HLint: ignore Redundant do" :: String) #-} {-# ANN module ("HLint: ignore Reduce duplication" :: String) #-} @@ -34,6 +33,9 @@ testToListVector :: LBS.ByteString -> [V.Vector LBS.ByteString] testToListVector = SVL.toListVector . SVL.makeCursor comma +testToListList :: LBS.ByteString -> [[LBS.ByteString]] +testToListList = fmap V.toList . testToListVector + -- Adds a terminal newline to the file testToListVector' :: LBS.ByteString -> [V.Vector LBS.ByteString] testToListVector' = testToListVector . (<> "\n") @@ -66,4 +68,140 @@ testToListVector' subjectMS === expectedMS it "Case 10" $ requireTest $ do testToListVector' subjectMM === expectedMM + describe "boundaries" $ do + it "513 bytes" $ requireTest $ do + let actual = testToListList $ LBS.intercalate "\n" + [ "id,group_id,arm,first_name,last_name,email,contact_number,start_date,status,dose_smart_id,fitbit_id,glyco_id,sms_reminder,email_reminder,inactive_reminder,starting_weight,starting_bmi,starting_hba1c" + , "10,aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa,1,aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa,aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa,aaaaaaa,aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa,2018-01-01 00:00:00 UTC,Active,,,,Never,Weekly,Never,100.01,10.01,1.0e-2" + ] + actual === + [ [ "id" + , "group_id" + , "arm" + , "first_name" + , "last_name" + , "email" + , "contact_number" + , "start_date" + , "status" + , "dose_smart_id" + , "fitbit_id" + , "glyco_id" + , "sms_reminder" + , "email_reminder" + , "inactive_reminder" + , "starting_weight" + , "starting_bmi" + , "starting_hba1c" + ] + , [ "10" + , "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + , "1" + , "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + , "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + , "aaaaaaa" + , "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + , "2018-01-01 00:00:00 UTC" + , "Active" + , "" + , "" + , "" + , "Never" + , "Weekly" + , "Never" + , "100.01" + , "10.01" + , "1.0e-2" + ] + ] + it "512 bytes" $ requireTest $ do + let actual = testToListList $ LBS.intercalate "\n" + [ "id,group_id,arm,first_name,last_name,email,contact_number,start_date,status,dose_smart_id,fitbit_id,glyco_id,sms_reminder,email_reminder,inactive_reminder,starting_weight,starting_bmi,starting_hba1c" + , "10,aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa,1,aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa,aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa,aaaaaaa,aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa,2018-01-01 00:00:00 UTC,Active,,,,Never,Weekly,Never,100.01,10.01,1.0e-2" + ] + actual === + [ [ "id" + , "group_id" + , "arm" + , "first_name" + , "last_name" + , "email" + , "contact_number" + , "start_date" + , "status" + , "dose_smart_id" + , "fitbit_id" + , "glyco_id" + , "sms_reminder" + , "email_reminder" + , "inactive_reminder" + , "starting_weight" + , "starting_bmi" + , "starting_hba1c" + ] + , [ "10" + , "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + , "1" + , "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + , "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + , "aaaaaaa" + , "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + , "2018-01-01 00:00:00 UTC" + , "Active" + , "" + , "" + , "" + , "Never" + , "Weekly" + , "Never" + , "100.01" + , "10.01" + , "1.0e-2" + ] + ] + it "511 bytes" $ requireTest $ do + let actual = testToListList $ LBS.intercalate "\n" + [ "id,group_id,arm,first_name,last_name,email,contact_number,start_date,status,dose_smart_id,fitbit_id,glyco_id,sms_reminder,email_reminder,inactive_reminder,starting_weight,starting_bmi,starting_hba1c" + , "10,aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa,1,aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa,aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa,aaaaaaa,aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa,2018-01-01 00:00:00 UTC,Active,,,,Never,Weekly,Never,100.01,10.01,1.0e-2" + ] + actual === + [ [ "id" + , "group_id" + , "arm" + , "first_name" + , "last_name" + , "email" + , "contact_number" + , "start_date" + , "status" + , "dose_smart_id" + , "fitbit_id" + , "glyco_id" + , "sms_reminder" + , "email_reminder" + , "inactive_reminder" + , "starting_weight" + , "starting_bmi" + , "starting_hba1c" + ] + , [ "10" + , "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + , "1" + , "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + , "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + , "aaaaaaa" + , "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + , "2018-01-01 00:00:00 UTC" + , "Active" + , "" + , "" + , "" + , "Never" + , "Weekly" + , "Never" + , "100.01" + , "10.01" + , "1.0e-2" + ] + ]