diff --git a/ChangeLog.md b/ChangeLog.md --- a/ChangeLog.md +++ b/ChangeLog.md @@ -1,10 +1,32 @@ # Changelog for hw-dsv -# 0.2.1 +## 0.4.0 +* Resolve some ambiguity to whether functions are lazy by moving them to designated modules. +* Fix names of functions which were lying about the types they were dealing with. In particular: + + * ```haskell + selectListVector :: [Int] -> DsvCursor -> [[LBS.ByteString]] + ``` + + should be + + ```haskell + selectListList :: [Int] -> DsvCursor -> [[LBS.ByteString]] + ``` + +* Remove deprecated functions. + +## 0.3.7 + +* Deprecate some functions. Some of these functions have been moved to another module or renamed. + See implementation of deprecated functions to see how downstream code should be refactor. + +## 0.2.1 + * Add instance NFData Lazy.DsvCursor -# 0.2 +## 0.2 * Change delimiter types from Char to Word8, since only 8-bit chars work. * Support GHC 7.10 diff --git a/README.md b/README.md --- a/README.md +++ b/README.md @@ -30,7 +30,7 @@ use the `hw-dsv` binary. For best performance, add the `bmi2` and `avx2` flags to target the BMI2 and -AVS2 instruction are specified in the `cabal.project` file. +AVX2 instruction are specified in the `cabal.project` file. For slightly older CPUs, remove `avx2` flags from the `cabal.project` file to target only the BMI2 instruction set. diff --git a/app/App/Char.hs b/app/App/Char.hs --- a/app/App/Char.hs +++ b/app/App/Char.hs @@ -1,7 +1,6 @@ module App.Char where import Data.Char -import Data.Semigroup ((<>)) import Data.Word import Options.Applicative (ReadM, eitherReader) diff --git a/app/App/Commands.hs b/app/App/Commands.hs --- a/app/App/Commands.hs +++ b/app/App/Commands.hs @@ -6,7 +6,6 @@ import App.Commands.QueryLazy import App.Commands.QueryStrict import App.Commands.RangeJoin -import Data.Semigroup ((<>)) import Options.Applicative commands :: Parser (IO ()) diff --git a/app/App/Commands/CreateIndex.hs b/app/App/Commands/CreateIndex.hs --- a/app/App/Commands/CreateIndex.hs +++ b/app/App/Commands/CreateIndex.hs @@ -10,7 +10,6 @@ import Control.Lens import Control.Monad import Data.Generics.Product.Any -import Data.Semigroup ((<>)) import HaskellWorks.Data.ByteString.Lazy import Options.Applicative hiding (columns) diff --git a/app/App/Commands/Generate.hs b/app/App/Commands/Generate.hs --- a/app/App/Commands/Generate.hs +++ b/app/App/Commands/Generate.hs @@ -8,7 +8,6 @@ import Control.Lens import Data.Generics.Product.Any import Data.List -import Data.Semigroup ((<>)) import Options.Applicative hiding (columns) import qualified App.Commands.Options.Type as Z diff --git a/app/App/Commands/IndexWord8s.hs b/app/App/Commands/IndexWord8s.hs --- a/app/App/Commands/IndexWord8s.hs +++ b/app/App/Commands/IndexWord8s.hs @@ -7,7 +7,6 @@ import Control.Lens import Data.Generics.Product.Any -import Data.Semigroup ((<>)) import HaskellWorks.Data.Vector.AsVector64 import Options.Applicative diff --git a/app/App/Commands/Options/Parse.hs b/app/App/Commands/Options/Parse.hs --- a/app/App/Commands/Options/Parse.hs +++ b/app/App/Commands/Options/Parse.hs @@ -6,7 +6,6 @@ , rangeJoinColumn ) where -import Data.Semigroup ((<>)) import Data.Text import Options.Applicative import Text.Read (readEither) diff --git a/app/App/Commands/QueryLazy.hs b/app/App/Commands/QueryLazy.hs --- a/app/App/Commands/QueryLazy.hs +++ b/app/App/Commands/QueryLazy.hs @@ -17,7 +17,6 @@ import Control.Monad.Trans.Resource import Data.Generics.Product.Any import Data.List -import Data.Semigroup ((<>)) import Options.Applicative hiding (columns) import qualified App.Commands.Options.Type as Z diff --git a/app/App/Commands/QueryStrict.hs b/app/App/Commands/QueryStrict.hs --- a/app/App/Commands/QueryStrict.hs +++ b/app/App/Commands/QueryStrict.hs @@ -17,7 +17,6 @@ import Control.Monad.Trans.Resource import Data.Generics.Product.Any import Data.List -import Data.Semigroup ((<>)) import Options.Applicative import qualified App.Commands.Options.Type as Z diff --git a/app/App/Commands/RangeJoin.hs b/app/App/Commands/RangeJoin.hs --- a/app/App/Commands/RangeJoin.hs +++ b/app/App/Commands/RangeJoin.hs @@ -18,7 +18,6 @@ import Data.Generics.Product.Any import Data.List import Data.Maybe (catMaybes) -import Data.Semigroup ((<>)) import Data.Word import Options.Applicative hiding (columns) import Text.Read (readMaybe) diff --git a/app/Main.hs b/app/Main.hs --- a/app/Main.hs +++ b/app/Main.hs @@ -2,7 +2,6 @@ import App.Commands import Control.Monad -import Data.Semigroup ((<>)) import Options.Applicative main :: IO () diff --git a/bench/Main.hs b/bench/Main.hs --- a/bench/Main.hs +++ b/bench/Main.hs @@ -7,7 +7,6 @@ import Criterion.Main import Data.ByteString (ByteString) import Data.List -import Data.Monoid ((<>)) import Data.Vector (Vector) import Data.Word import HaskellWorks.Data.Dsv.Internal.Char (comma, pipe) diff --git a/hw-dsv.cabal b/hw-dsv.cabal --- a/hw-dsv.cabal +++ b/hw-dsv.cabal @@ -1,7 +1,7 @@ cabal-version: 2.2 name: hw-dsv -version: 0.4.0 +version: 0.4.1.0 synopsis: Unbelievably fast streaming DSV file parser description: Please see the README on Github at category: Text, CSV, SIMD, Succinct Data Structures, Data Structures @@ -9,10 +9,10 @@ bug-reports: https://github.com/haskell-works/hw-dsv/issues author: John Ky maintainer: newhoggy@gmail.com -copyright: 2018-2019 John Ky +copyright: 2018-2020 John Ky license: BSD-3-Clause license-file: LICENSE -tested-with: GHC == 8.8.1, GHC == 8.6.5, GHC == 8.4.4, GHC == 8.2.2 +tested-with: GHC == 8.10.1, GHC == 8.8.3, GHC == 8.6.5, GHC == 8.4.4 build-type: Simple extra-source-files: README.md ChangeLog.md @@ -37,37 +37,36 @@ manual: False default: True -common base { build-depends: base >= 4.7 && < 5 } - -common appar { build-depends: appar >= 0.1.8 && < 0.2 } -common bits-extra { build-depends: bits-extra >= 0.0.1.2 && < 0.1 } -common bytestring { build-depends: bytestring >= 0.10 && < 0.11 } -common cassava { build-depends: cassava >= 0.5.1.0 && < 0.6 } -common criterion { build-depends: criterion >= 1.4.1.0 && < 1.6 } -common deepseq { build-depends: deepseq >= 1.4 && < 1.5 } -common directory { build-depends: directory >= 1.2.2 && < 1.4 } -common doctest { build-depends: doctest >= 0.16.2 && < 0.17 } -common doctest-discover { build-depends: doctest-discover >= 0.2 && < 0.3 }common generic-lens { build-depends: generic-lens >= 1.2.0.1 && < 1.3 } -common ghc-prim { build-depends: ghc-prim >= 0.4 && < 0.6 } -common hedgehog { build-depends: hedgehog >= 0.5 && < 1.1 } -common hspec { build-depends: hspec >= 2.4 && < 3 } -common hw-bits { build-depends: hw-bits >= 0.7.0.2 && < 0.8 } -common hw-ip { build-depends: hw-ip >= 2.3.4.2 && < 2.4 } -common hw-hspec-hedgehog { build-depends: hw-hspec-hedgehog >= 0.1.0.4 && < 0.2 } -common hw-prim { build-depends: hw-prim >= 0.6.2.14 && < 0.7 } -common hw-rankselect { build-depends: hw-rankselect >= 0.12.0.2 && < 0.14 } -common hw-rankselect-base { build-depends: hw-rankselect-base >= 0.3.2.0 && < 0.4 } -common hw-simd { build-depends: hw-simd >= 0.1.1.3 && < 0.2 } -common lens { build-depends: lens >= 4.15 && < 5 } -common mmap { build-depends: mmap >= 0.5.9 && < 0.6 } -common optparse-applicative { build-depends: optparse-applicative >= 0.13 && < 0.16 } -common resourcet { build-depends: resourcet >= 1.1 && < 1.3 } -common text { build-depends: text >= 1.2.2 && < 2.0 } -common vector { build-depends: vector >= 0.12.0.1 && < 0.13 } -common weigh { build-depends: weigh >= 0.0.6 && < 0.1 } +common base { build-depends: base >= 4.11 && < 5 } -common semigroups { if (!impl(ghc >= 8.0.1)) { build-depends: semigroups >= 0.8.4 && < 0.19 } } -common transformers { if (!impl(ghc >= 8.0.1)) { build-depends: transformers >= 0.4 && < 0.6 } } +common appar { build-depends: appar >= 0.1.8 && < 0.2 } +common bits-extra { build-depends: bits-extra >= 0.0.1.2 && < 0.1 } +common bytestring { build-depends: bytestring >= 0.10 && < 0.11 } +common cassava { build-depends: cassava >= 0.5.1.0 && < 0.6 } +common criterion { build-depends: criterion >= 1.4.1.0 && < 1.6 } +common deepseq { build-depends: deepseq >= 1.4 && < 1.5 } +common directory { build-depends: directory >= 1.2.2 && < 1.4 } +common doctest { build-depends: doctest >= 0.16.2 && < 0.17 } +common doctest-discover { build-depends: doctest-discover >= 0.2 && < 0.3 } +common generic-lens { build-depends: generic-lens >= 1.2.0.1 && < 1.3 } +common ghc-prim { build-depends: ghc-prim >= 0.4 && < 0.7 } +common hedgehog { build-depends: hedgehog >= 0.5 && < 1.1 } +common hspec { build-depends: hspec >= 2.4 && < 3 } +common hw-bits { build-depends: hw-bits >= 0.7.0.2 && < 0.8 } +common hw-ip { build-depends: hw-ip >= 2.3.4.2 && < 2.5 } +common hw-hspec-hedgehog { build-depends: hw-hspec-hedgehog >= 0.1.0.4 && < 0.2 } +common hw-prim { build-depends: hw-prim >= 0.6.2.14 && < 0.7 } +common hw-rankselect { build-depends: hw-rankselect >= 0.12.0.2 && < 0.14 } +common hw-rankselect-base { build-depends: hw-rankselect-base >= 0.3.2.0 && < 0.4 } +common hw-simd { build-depends: hw-simd >= 0.1.1.3 && < 0.2 } +common lens { build-depends: lens >= 4.15 && < 5 } +common mmap { build-depends: mmap >= 0.5.9 && < 0.6 } +common optparse-applicative { build-depends: optparse-applicative >= 0.13 && < 0.16 } +common resourcet { build-depends: resourcet >= 1.1 && < 1.3 } +common text { build-depends: text >= 1.2.2 && < 2.0 } +common transformers { build-depends: transformers >= 0.4 && < 0.6 } +common vector { build-depends: vector >= 0.12.0.1 && < 0.13 } +common weigh { build-depends: weigh >= 0.0.6 && < 0.1 } common config default-language: Haskell2010 @@ -101,7 +100,6 @@ , hw-rankselect , hw-rankselect-base , hw-simd - , semigroups , transformers , vector exposed-modules: @@ -141,7 +139,6 @@ , lens , optparse-applicative , resourcet - , semigroups , text , transformers , vector diff --git a/src/HaskellWorks/Data/Dsv/Internal/Bits.hs b/src/HaskellWorks/Data/Dsv/Internal/Bits.hs --- a/src/HaskellWorks/Data/Dsv/Internal/Bits.hs +++ b/src/HaskellWorks/Data/Dsv/Internal/Bits.hs @@ -15,10 +15,10 @@ {-# INLINE testWord8s #-} zipOr :: DVS.Vector Word64 -> DVS.Vector Word64 -> DVS.Vector Word64 -zipOr as bs = DVS.constructN (DVS.length as `max` DVS.length bs) go - where go :: DVS.Vector Word64 -> Word64 +zipOr as bs = DVS.generate (DVS.length as `max` DVS.length bs) go + where go :: Int -> Word64 go u = - let ui = DVS.length u + let ui = fromIntegral u in if ui < DVS.length as && ui < DVS.length bs then DVS.unsafeIndex as ui .|. DVS.unsafeIndex bs ui else error "Different sized vectors" @@ -32,10 +32,10 @@ {-# INLINE zip2Or #-} zipAnd :: DVS.Vector Word64 -> DVS.Vector Word64 -> DVS.Vector Word64 -zipAnd as bs = DVS.constructN (DVS.length as `max` DVS.length bs) go - where go :: DVS.Vector Word64 -> Word64 +zipAnd as bs = DVS.generate (DVS.length as `max` DVS.length bs) go + where go :: Int -> Word64 go u = - let ui = DVS.length u + let ui = fromIntegral u in if ui < DVS.length as && ui < DVS.length bs then DVS.unsafeIndex as ui .&. DVS.unsafeIndex bs ui else error "Different sized vectors" diff --git a/src/HaskellWorks/Data/Dsv/Strict/Cursor/Internal/Reference.hs b/src/HaskellWorks/Data/Dsv/Strict/Cursor/Internal/Reference.hs --- a/src/HaskellWorks/Data/Dsv/Strict/Cursor/Internal/Reference.hs +++ b/src/HaskellWorks/Data/Dsv/Strict/Cursor/Internal/Reference.hs @@ -5,7 +5,6 @@ module HaskellWorks.Data.Dsv.Strict.Cursor.Internal.Reference where import Data.Bits (popCount) -import Data.Semigroup import Data.Word import HaskellWorks.Data.AtIndex import HaskellWorks.Data.Bits.BitWise @@ -45,9 +44,9 @@ -- n: Number of rank select bit string words since beginning -- returns: dquote interest bits in high part and other interest bits in low part mkDsvRawBitsByWord64s :: Word64 -> Word64 -> Word64 -> DVS.Vector Word64 -> DVS.Vector Word64 -mkDsvRawBitsByWord64s rdqs rnls rdls v = DVS.constructN (((DVS.length v + 7) `div` 8) * 2) go - where go :: DVS.Vector Word64 -> Word64 - go u = let vi = dvsLength u * 4 in +mkDsvRawBitsByWord64s rdqs rnls rdls v = DVS.generate (((DVS.length v + 7) `div` 8) * 2) go + where go :: Int -> Word64 + go u = let vi = fromIntegral u * 4 in if dvsLength v - vi >= 4 then let w0 = unsafeIndex v vi w0Dqs = testWord8s (w0 .^. rdqs) @@ -106,9 +105,9 @@ in atIndexOr2 0 u (ui - 1) + fromIntegral (popCount w) mkIbVector' :: DVS.Vector Word64 -> DVS.Vector Word64 -> DVS.Vector Word64 -> DVS.Vector Word64 -mkIbVector' rawBits cpcs v = DVS.constructN ((DVS.length v + 7) `div` 8) go - where go :: DVS.Vector Word64 -> Word64 - go u = let ui = dvsLength u in if ui > 1 +mkIbVector' rawBits cpcs v = DVS.generate ((DVS.length v + 7) `div` 8) go + where go :: Int -> Word64 + go u = let ui = fromIntegral u in if ui > 1 then let vi = ui * 2 cpc = unsafeIndex cpcs (ui - 1) w0 = unsafeIndex rawBits vi @@ -141,11 +140,11 @@ -- n: Number of rank select bit string words since beginning -- returns: dquote interest bits in high part and other interest bits in low part mkStripes :: Word64 -> Word64 -> Word64 -> DVS.Vector Word64 -> DVS.Vector Word64 -mkStripes rdqs rnls rdls v = DVS.constructN (((DVS.length v + 7) `div` 8) * 3) go +mkStripes rdqs rnls rdls v = DVS.generate (((DVS.length v + 7) `div` 8) * 3) go where stripePatterns = DVS.fromList [rdqs, rnls, rdls] - go :: DVS.Vector Word64 -> Word64 + go :: Int -> Word64 go u = - let ui = dvsLength u + let ui = fromIntegral u si = ui `mod` 3 vi = (ui `div` 3) * 8 ws = unsafeIndex stripePatterns si @@ -182,9 +181,9 @@ in unsafeIndex u (ui - 1) + fromIntegral (popCount w) mkDsvIbNlFromStriped :: DVS.Vector Word64 -> DVS.Vector Word64 -> DVS.Vector Word64 -mkDsvIbNlFromStriped sv cpcs = DVS.constructN ((DVS.length sv) `div` 3) go - where go :: DVS.Vector Word64 -> Word64 - go u = let ui = dvsLength u in if ui > 1 +mkDsvIbNlFromStriped sv cpcs = DVS.generate ((DVS.length sv) `div` 3) go + where go :: Int -> Word64 + go u = let ui = fromIntegral u in if ui > 1 then let svi = ui * 2 cpc = unsafeIndex cpcs (ui - 1) wdq = unsafeIndex sv svi @@ -199,9 +198,9 @@ in wnl .&. m mkDsvIbDlFromStriped :: DVS.Vector Word64 -> DVS.Vector Word64 -> DVS.Vector Word64 -mkDsvIbDlFromStriped sv cpcs = DVS.constructN ((DVS.length sv) `div` 3) go - where go :: DVS.Vector Word64 -> Word64 - go u = let ui = dvsLength u in if ui > 1 +mkDsvIbDlFromStriped sv cpcs = DVS.generate ((DVS.length sv) `div` 3) go + where go :: Int -> Word64 + go u = let ui = fromIntegral u in if ui > 1 then let svi = ui * 2 cpc = unsafeIndex cpcs (ui - 1) wdq = unsafeIndex sv svi diff --git a/test/HaskellWorks/Data/Dsv/Lazy/CursorSpec.hs b/test/HaskellWorks/Data/Dsv/Lazy/CursorSpec.hs --- a/test/HaskellWorks/Data/Dsv/Lazy/CursorSpec.hs +++ b/test/HaskellWorks/Data/Dsv/Lazy/CursorSpec.hs @@ -3,7 +3,6 @@ module HaskellWorks.Data.Dsv.Lazy.CursorSpec (spec) where -import Data.Semigroup ((<>)) import HaskellWorks.Data.Bits.BitShown import HaskellWorks.Data.Dsv.Internal.Char (comma) import HaskellWorks.Hspec.Hedgehog diff --git a/test/HaskellWorks/Data/Dsv/Strict/Cursor/InternalSpec.hs b/test/HaskellWorks/Data/Dsv/Strict/Cursor/InternalSpec.hs --- a/test/HaskellWorks/Data/Dsv/Strict/Cursor/InternalSpec.hs +++ b/test/HaskellWorks/Data/Dsv/Strict/Cursor/InternalSpec.hs @@ -9,7 +9,6 @@ import Data.ByteString (ByteString) import Data.Char import Data.List (isSuffixOf) -import Data.Monoid ((<>)) import HaskellWorks.Data.Bits.PopCount.PopCount1 import HaskellWorks.Data.Dsv.Internal.Char (pipe) import HaskellWorks.Data.FromByteString