hw-dsv 0.4.0 → 0.4.1.0
raw patch · 18 files changed
+78/−73 lines, 18 filesdep −semigroupsdep ~basedep ~bytestringdep ~doctestPVP ok
version bump matches the API change (PVP)
Dependencies removed: semigroups
Dependency ranges changed: base, bytestring, doctest, generic-lens, ghc-prim, lens, optparse-applicative, transformers
API changes (from Hackage documentation)
Files
- ChangeLog.md +24/−2
- README.md +1/−1
- app/App/Char.hs +0/−1
- app/App/Commands.hs +0/−1
- app/App/Commands/CreateIndex.hs +0/−1
- app/App/Commands/Generate.hs +0/−1
- app/App/Commands/IndexWord8s.hs +0/−1
- app/App/Commands/Options/Parse.hs +0/−1
- app/App/Commands/QueryLazy.hs +0/−1
- app/App/Commands/QueryStrict.hs +0/−1
- app/App/Commands/RangeJoin.hs +0/−1
- app/Main.hs +0/−1
- bench/Main.hs +0/−1
- hw-dsv.cabal +32/−35
- src/HaskellWorks/Data/Dsv/Internal/Bits.hs +6/−6
- src/HaskellWorks/Data/Dsv/Strict/Cursor/Internal/Reference.hs +15/−16
- test/HaskellWorks/Data/Dsv/Lazy/CursorSpec.hs +0/−1
- test/HaskellWorks/Data/Dsv/Strict/Cursor/InternalSpec.hs +0/−1
ChangeLog.md view
@@ -1,10 +1,32 @@ # Changelog for hw-dsv -# 0.2.1+## 0.4.0 +* Resolve some ambiguity to whether functions are lazy by moving them to designated modules.+* Fix names of functions which were lying about the types they were dealing with. In particular:++ * ```haskell+ selectListVector :: [Int] -> DsvCursor -> [[LBS.ByteString]]+ ```++ should be++ ```haskell+ selectListList :: [Int] -> DsvCursor -> [[LBS.ByteString]]+ ```++* Remove deprecated functions.++## 0.3.7++* Deprecate some functions. Some of these functions have been moved to another module or renamed.+ See implementation of deprecated functions to see how downstream code should be refactor.++## 0.2.1+ * Add instance NFData Lazy.DsvCursor -# 0.2+## 0.2 * Change delimiter types from Char to Word8, since only 8-bit chars work. * Support GHC 7.10
README.md view
@@ -30,7 +30,7 @@ use the `hw-dsv` binary. For best performance, add the `bmi2` and `avx2` flags to target the BMI2 and-AVS2 instruction are specified in the `cabal.project` file.+AVX2 instruction are specified in the `cabal.project` file. For slightly older CPUs, remove `avx2` flags from the `cabal.project` file to target only the BMI2 instruction set.
app/App/Char.hs view
@@ -1,7 +1,6 @@ module App.Char where import Data.Char-import Data.Semigroup ((<>)) import Data.Word import Options.Applicative (ReadM, eitherReader)
app/App/Commands.hs view
@@ -6,7 +6,6 @@ import App.Commands.QueryLazy import App.Commands.QueryStrict import App.Commands.RangeJoin-import Data.Semigroup ((<>)) import Options.Applicative commands :: Parser (IO ())
app/App/Commands/CreateIndex.hs view
@@ -10,7 +10,6 @@ import Control.Lens import Control.Monad import Data.Generics.Product.Any-import Data.Semigroup ((<>)) import HaskellWorks.Data.ByteString.Lazy import Options.Applicative hiding (columns)
app/App/Commands/Generate.hs view
@@ -8,7 +8,6 @@ import Control.Lens import Data.Generics.Product.Any import Data.List-import Data.Semigroup ((<>)) import Options.Applicative hiding (columns) import qualified App.Commands.Options.Type as Z
app/App/Commands/IndexWord8s.hs view
@@ -7,7 +7,6 @@ import Control.Lens import Data.Generics.Product.Any-import Data.Semigroup ((<>)) import HaskellWorks.Data.Vector.AsVector64 import Options.Applicative
app/App/Commands/Options/Parse.hs view
@@ -6,7 +6,6 @@ , rangeJoinColumn ) where -import Data.Semigroup ((<>)) import Data.Text import Options.Applicative import Text.Read (readEither)
app/App/Commands/QueryLazy.hs view
@@ -17,7 +17,6 @@ import Control.Monad.Trans.Resource import Data.Generics.Product.Any import Data.List-import Data.Semigroup ((<>)) import Options.Applicative hiding (columns) import qualified App.Commands.Options.Type as Z
app/App/Commands/QueryStrict.hs view
@@ -17,7 +17,6 @@ import Control.Monad.Trans.Resource import Data.Generics.Product.Any import Data.List-import Data.Semigroup ((<>)) import Options.Applicative import qualified App.Commands.Options.Type as Z
app/App/Commands/RangeJoin.hs view
@@ -18,7 +18,6 @@ import Data.Generics.Product.Any import Data.List import Data.Maybe (catMaybes)-import Data.Semigroup ((<>)) import Data.Word import Options.Applicative hiding (columns) import Text.Read (readMaybe)
app/Main.hs view
@@ -2,7 +2,6 @@ import App.Commands import Control.Monad-import Data.Semigroup ((<>)) import Options.Applicative main :: IO ()
bench/Main.hs view
@@ -7,7 +7,6 @@ import Criterion.Main import Data.ByteString (ByteString) import Data.List-import Data.Monoid ((<>)) import Data.Vector (Vector) import Data.Word import HaskellWorks.Data.Dsv.Internal.Char (comma, pipe)
hw-dsv.cabal view
@@ -1,7 +1,7 @@ cabal-version: 2.2 name: hw-dsv-version: 0.4.0+version: 0.4.1.0 synopsis: Unbelievably fast streaming DSV file parser description: Please see the README on Github at <https://github.com/haskell-works/hw-dsv#readme> category: Text, CSV, SIMD, Succinct Data Structures, Data Structures@@ -9,10 +9,10 @@ bug-reports: https://github.com/haskell-works/hw-dsv/issues author: John Ky maintainer: newhoggy@gmail.com-copyright: 2018-2019 John Ky+copyright: 2018-2020 John Ky license: BSD-3-Clause license-file: LICENSE-tested-with: GHC == 8.8.1, GHC == 8.6.5, GHC == 8.4.4, GHC == 8.2.2+tested-with: GHC == 8.10.1, GHC == 8.8.3, GHC == 8.6.5, GHC == 8.4.4 build-type: Simple extra-source-files: README.md ChangeLog.md@@ -37,37 +37,36 @@ manual: False default: True -common base { build-depends: base >= 4.7 && < 5 }--common appar { build-depends: appar >= 0.1.8 && < 0.2 }-common bits-extra { build-depends: bits-extra >= 0.0.1.2 && < 0.1 }-common bytestring { build-depends: bytestring >= 0.10 && < 0.11 }-common cassava { build-depends: cassava >= 0.5.1.0 && < 0.6 }-common criterion { build-depends: criterion >= 1.4.1.0 && < 1.6 }-common deepseq { build-depends: deepseq >= 1.4 && < 1.5 }-common directory { build-depends: directory >= 1.2.2 && < 1.4 }-common doctest { build-depends: doctest >= 0.16.2 && < 0.17 }-common doctest-discover { build-depends: doctest-discover >= 0.2 && < 0.3 }common generic-lens { build-depends: generic-lens >= 1.2.0.1 && < 1.3 }-common ghc-prim { build-depends: ghc-prim >= 0.4 && < 0.6 }-common hedgehog { build-depends: hedgehog >= 0.5 && < 1.1 }-common hspec { build-depends: hspec >= 2.4 && < 3 }-common hw-bits { build-depends: hw-bits >= 0.7.0.2 && < 0.8 }-common hw-ip { build-depends: hw-ip >= 2.3.4.2 && < 2.4 }-common hw-hspec-hedgehog { build-depends: hw-hspec-hedgehog >= 0.1.0.4 && < 0.2 }-common hw-prim { build-depends: hw-prim >= 0.6.2.14 && < 0.7 }-common hw-rankselect { build-depends: hw-rankselect >= 0.12.0.2 && < 0.14 }-common hw-rankselect-base { build-depends: hw-rankselect-base >= 0.3.2.0 && < 0.4 }-common hw-simd { build-depends: hw-simd >= 0.1.1.3 && < 0.2 }-common lens { build-depends: lens >= 4.15 && < 5 }-common mmap { build-depends: mmap >= 0.5.9 && < 0.6 }-common optparse-applicative { build-depends: optparse-applicative >= 0.13 && < 0.16 }-common resourcet { build-depends: resourcet >= 1.1 && < 1.3 }-common text { build-depends: text >= 1.2.2 && < 2.0 }-common vector { build-depends: vector >= 0.12.0.1 && < 0.13 }-common weigh { build-depends: weigh >= 0.0.6 && < 0.1 }+common base { build-depends: base >= 4.11 && < 5 } -common semigroups { if (!impl(ghc >= 8.0.1)) { build-depends: semigroups >= 0.8.4 && < 0.19 } }-common transformers { if (!impl(ghc >= 8.0.1)) { build-depends: transformers >= 0.4 && < 0.6 } }+common appar { build-depends: appar >= 0.1.8 && < 0.2 }+common bits-extra { build-depends: bits-extra >= 0.0.1.2 && < 0.1 }+common bytestring { build-depends: bytestring >= 0.10 && < 0.11 }+common cassava { build-depends: cassava >= 0.5.1.0 && < 0.6 }+common criterion { build-depends: criterion >= 1.4.1.0 && < 1.6 }+common deepseq { build-depends: deepseq >= 1.4 && < 1.5 }+common directory { build-depends: directory >= 1.2.2 && < 1.4 }+common doctest { build-depends: doctest >= 0.16.2 && < 0.17 }+common doctest-discover { build-depends: doctest-discover >= 0.2 && < 0.3 }+common generic-lens { build-depends: generic-lens >= 1.2.0.1 && < 1.3 }+common ghc-prim { build-depends: ghc-prim >= 0.4 && < 0.7 }+common hedgehog { build-depends: hedgehog >= 0.5 && < 1.1 }+common hspec { build-depends: hspec >= 2.4 && < 3 }+common hw-bits { build-depends: hw-bits >= 0.7.0.2 && < 0.8 }+common hw-ip { build-depends: hw-ip >= 2.3.4.2 && < 2.5 }+common hw-hspec-hedgehog { build-depends: hw-hspec-hedgehog >= 0.1.0.4 && < 0.2 }+common hw-prim { build-depends: hw-prim >= 0.6.2.14 && < 0.7 }+common hw-rankselect { build-depends: hw-rankselect >= 0.12.0.2 && < 0.14 }+common hw-rankselect-base { build-depends: hw-rankselect-base >= 0.3.2.0 && < 0.4 }+common hw-simd { build-depends: hw-simd >= 0.1.1.3 && < 0.2 }+common lens { build-depends: lens >= 4.15 && < 5 }+common mmap { build-depends: mmap >= 0.5.9 && < 0.6 }+common optparse-applicative { build-depends: optparse-applicative >= 0.13 && < 0.16 }+common resourcet { build-depends: resourcet >= 1.1 && < 1.3 }+common text { build-depends: text >= 1.2.2 && < 2.0 }+common transformers { build-depends: transformers >= 0.4 && < 0.6 }+common vector { build-depends: vector >= 0.12.0.1 && < 0.13 }+common weigh { build-depends: weigh >= 0.0.6 && < 0.1 } common config default-language: Haskell2010@@ -101,7 +100,6 @@ , hw-rankselect , hw-rankselect-base , hw-simd- , semigroups , transformers , vector exposed-modules:@@ -141,7 +139,6 @@ , lens , optparse-applicative , resourcet- , semigroups , text , transformers , vector
src/HaskellWorks/Data/Dsv/Internal/Bits.hs view
@@ -15,10 +15,10 @@ {-# INLINE testWord8s #-} zipOr :: DVS.Vector Word64 -> DVS.Vector Word64 -> DVS.Vector Word64-zipOr as bs = DVS.constructN (DVS.length as `max` DVS.length bs) go- where go :: DVS.Vector Word64 -> Word64+zipOr as bs = DVS.generate (DVS.length as `max` DVS.length bs) go+ where go :: Int -> Word64 go u =- let ui = DVS.length u+ let ui = fromIntegral u in if ui < DVS.length as && ui < DVS.length bs then DVS.unsafeIndex as ui .|. DVS.unsafeIndex bs ui else error "Different sized vectors"@@ -32,10 +32,10 @@ {-# INLINE zip2Or #-} zipAnd :: DVS.Vector Word64 -> DVS.Vector Word64 -> DVS.Vector Word64-zipAnd as bs = DVS.constructN (DVS.length as `max` DVS.length bs) go- where go :: DVS.Vector Word64 -> Word64+zipAnd as bs = DVS.generate (DVS.length as `max` DVS.length bs) go+ where go :: Int -> Word64 go u =- let ui = DVS.length u+ let ui = fromIntegral u in if ui < DVS.length as && ui < DVS.length bs then DVS.unsafeIndex as ui .&. DVS.unsafeIndex bs ui else error "Different sized vectors"
src/HaskellWorks/Data/Dsv/Strict/Cursor/Internal/Reference.hs view
@@ -5,7 +5,6 @@ module HaskellWorks.Data.Dsv.Strict.Cursor.Internal.Reference where import Data.Bits (popCount)-import Data.Semigroup import Data.Word import HaskellWorks.Data.AtIndex import HaskellWorks.Data.Bits.BitWise@@ -45,9 +44,9 @@ -- n: Number of rank select bit string words since beginning -- returns: dquote interest bits in high part and other interest bits in low part mkDsvRawBitsByWord64s :: Word64 -> Word64 -> Word64 -> DVS.Vector Word64 -> DVS.Vector Word64-mkDsvRawBitsByWord64s rdqs rnls rdls v = DVS.constructN (((DVS.length v + 7) `div` 8) * 2) go- where go :: DVS.Vector Word64 -> Word64- go u = let vi = dvsLength u * 4 in+mkDsvRawBitsByWord64s rdqs rnls rdls v = DVS.generate (((DVS.length v + 7) `div` 8) * 2) go+ where go :: Int -> Word64+ go u = let vi = fromIntegral u * 4 in if dvsLength v - vi >= 4 then let w0 = unsafeIndex v vi w0Dqs = testWord8s (w0 .^. rdqs)@@ -106,9 +105,9 @@ in atIndexOr2 0 u (ui - 1) + fromIntegral (popCount w) mkIbVector' :: DVS.Vector Word64 -> DVS.Vector Word64 -> DVS.Vector Word64 -> DVS.Vector Word64-mkIbVector' rawBits cpcs v = DVS.constructN ((DVS.length v + 7) `div` 8) go- where go :: DVS.Vector Word64 -> Word64- go u = let ui = dvsLength u in if ui > 1+mkIbVector' rawBits cpcs v = DVS.generate ((DVS.length v + 7) `div` 8) go+ where go :: Int -> Word64+ go u = let ui = fromIntegral u in if ui > 1 then let vi = ui * 2 cpc = unsafeIndex cpcs (ui - 1) w0 = unsafeIndex rawBits vi@@ -141,11 +140,11 @@ -- n: Number of rank select bit string words since beginning -- returns: dquote interest bits in high part and other interest bits in low part mkStripes :: Word64 -> Word64 -> Word64 -> DVS.Vector Word64 -> DVS.Vector Word64-mkStripes rdqs rnls rdls v = DVS.constructN (((DVS.length v + 7) `div` 8) * 3) go+mkStripes rdqs rnls rdls v = DVS.generate (((DVS.length v + 7) `div` 8) * 3) go where stripePatterns = DVS.fromList [rdqs, rnls, rdls]- go :: DVS.Vector Word64 -> Word64+ go :: Int -> Word64 go u =- let ui = dvsLength u+ let ui = fromIntegral u si = ui `mod` 3 vi = (ui `div` 3) * 8 ws = unsafeIndex stripePatterns si@@ -182,9 +181,9 @@ in unsafeIndex u (ui - 1) + fromIntegral (popCount w) mkDsvIbNlFromStriped :: DVS.Vector Word64 -> DVS.Vector Word64 -> DVS.Vector Word64-mkDsvIbNlFromStriped sv cpcs = DVS.constructN ((DVS.length sv) `div` 3) go- where go :: DVS.Vector Word64 -> Word64- go u = let ui = dvsLength u in if ui > 1+mkDsvIbNlFromStriped sv cpcs = DVS.generate ((DVS.length sv) `div` 3) go+ where go :: Int -> Word64+ go u = let ui = fromIntegral u in if ui > 1 then let svi = ui * 2 cpc = unsafeIndex cpcs (ui - 1) wdq = unsafeIndex sv svi@@ -199,9 +198,9 @@ in wnl .&. m mkDsvIbDlFromStriped :: DVS.Vector Word64 -> DVS.Vector Word64 -> DVS.Vector Word64-mkDsvIbDlFromStriped sv cpcs = DVS.constructN ((DVS.length sv) `div` 3) go- where go :: DVS.Vector Word64 -> Word64- go u = let ui = dvsLength u in if ui > 1+mkDsvIbDlFromStriped sv cpcs = DVS.generate ((DVS.length sv) `div` 3) go+ where go :: Int -> Word64+ go u = let ui = fromIntegral u in if ui > 1 then let svi = ui * 2 cpc = unsafeIndex cpcs (ui - 1) wdq = unsafeIndex sv svi
test/HaskellWorks/Data/Dsv/Lazy/CursorSpec.hs view
@@ -3,7 +3,6 @@ module HaskellWorks.Data.Dsv.Lazy.CursorSpec (spec) where -import Data.Semigroup ((<>)) import HaskellWorks.Data.Bits.BitShown import HaskellWorks.Data.Dsv.Internal.Char (comma) import HaskellWorks.Hspec.Hedgehog
test/HaskellWorks/Data/Dsv/Strict/Cursor/InternalSpec.hs view
@@ -9,7 +9,6 @@ import Data.ByteString (ByteString) import Data.Char import Data.List (isSuffixOf)-import Data.Monoid ((<>)) import HaskellWorks.Data.Bits.PopCount.PopCount1 import HaskellWorks.Data.Dsv.Internal.Char (pipe) import HaskellWorks.Data.FromByteString