diff --git a/app/App/Commands/Demo.hs b/app/App/Commands/Demo.hs --- a/app/App/Commands/Demo.hs +++ b/app/App/Commands/Demo.hs @@ -22,16 +22,13 @@ import HaskellWorks.Data.MQuery import HaskellWorks.Data.MQuery.Micro import HaskellWorks.Data.RankSelect.CsPoppy -import HaskellWorks.Data.Vector.AsVector8 +import HaskellWorks.Data.Vector.Storable import Options.Applicative hiding (columns) import qualified App.Commands.Types as Z -import qualified Data.ByteString as BS import qualified Data.ByteString.Internal as BSI import qualified Data.ByteString.Lazy as LBS import qualified Data.DList as DL -import qualified Data.Vector.Storable as DVS -import qualified Data.Vector.Storable.Mutable as DVSM import qualified HaskellWorks.Data.BalancedParens.RangeMin as RM import qualified HaskellWorks.Data.ByteString.Lazy as LBS import qualified HaskellWorks.Data.Json.Simd.Index.Standard as S @@ -39,28 +36,6 @@ import qualified System.IO as IO import qualified System.IO.MMap as IO -constructUnzipN :: Int -> [(BS.ByteString, BS.ByteString)] -> (DVS.Vector Word64, DVS.Vector Word64) -constructUnzipN nBytes xs = (DVS.unsafeCast ibv, DVS.unsafeCast bpv) - where [ibv, bpv] = DVS.createT $ do - let nW64s = (nBytes + 7) `div` 8 - let capacity = nW64s * 8 - ibmv <- DVSM.new capacity - bpmv <- DVSM.new capacity - (ibmvRemaining, bpmvRemaining) <- go ibmv bpmv xs - return - [ DVSM.take (((DVSM.length ibmv - ibmvRemaining) `div` 8) * 8) ibmv - , DVSM.take (((DVSM.length bpmv - bpmvRemaining) `div` 8) * 8) bpmv - ] - go :: DVSM.MVector s Word8 -> DVSM.MVector s Word8 -> [(BS.ByteString, BS.ByteString)] -> ST s (Int, Int) - go ibmv bpmv ((ib, bp):ys) = do - DVS.copy (DVSM.take (BS.length ib) ibmv) (asVector8 ib) - DVS.copy (DVSM.take (BS.length bp) bpmv) (asVector8 bp) - go (DVSM.drop (BS.length ib) ibmv) (DVSM.drop (BS.length bp) bpmv) ys - go ibmv bpmv [] = do - DVSM.set (DVSM.take 8 ibmv) 0 - DVSM.set (DVSM.take 8 bpmv) 0 - return (DVSM.length (DVSM.drop 8 ibmv), DVSM.length (DVSM.drop 8 bpmv)) - runDemo :: Z.DemoOptions -> IO () runDemo opts = do let filePath = opts ^. the @"filePath" @@ -76,7 +51,7 @@ let !bs = BSI.fromForeignPtr (castForeignPtr fptr) offset size case S.makeStandardJsonIbBps (LBS.resegmentPadded 512 (LBS.fromStrict bs)) of Right ibBps -> do - let (!ib, !bp) = constructUnzipN size ibBps + let (!ib, !bp) = construct64UnzipN size ibBps let !cursor = GenericCursor bs (makeCsPoppy ib) (RM.mkRangeMin bp) 1 let !json = lightJsonAt cursor let q = MQuery (DL.singleton json) diff --git a/hw-json.cabal b/hw-json.cabal --- a/hw-json.cabal +++ b/hw-json.cabal @@ -1,7 +1,7 @@ cabal-version: 2.2 name: hw-json -version: 1.3.0.1 +version: 1.3.1.0 synopsis: Memory efficient JSON parser description: Memory efficient JSON parser. Please see README.md category: Data @@ -9,7 +9,7 @@ bug-reports: https://github.com/haskell-works/hw-json/issues author: John Ky maintainer: newhoggy@gmail.com -copyright: 2016 - 2019 John Ky +copyright: 2016-2019 John Ky license: BSD-3-Clause license-file: LICENSE build-type: Simple @@ -59,7 +59,7 @@ common hw-json-standard-cursor { build-depends: hw-json-standard-cursor >= 0.2.1.0 && < 0.3 } common hw-mquery { build-depends: hw-mquery >= 0.2.0.0 && < 0.3 } common hw-parser { build-depends: hw-parser >= 0.1 && < 0.2 } -common hw-prim { build-depends: hw-prim >= 0.6.2.28 && < 0.7 } +common hw-prim { build-depends: hw-prim >= 0.6.2.32 && < 0.7 } common hw-rankselect { build-depends: hw-rankselect >= 0.13 && < 0.14 } common hw-rankselect-base { build-depends: hw-rankselect-base >= 0.3.2.1 && < 0.4 } common hw-simd { build-depends: hw-simd >= 0.1.1.2 && < 0.2 } @@ -75,8 +75,8 @@ common semigroups { if impl(ghc < 8 ) { build-depends: semigroups >= 0.16 && < 0.19 } } common config - default-language: Haskell2010 - ghc-options: -Wall -O2 -msse4.2 + default-language: Haskell2010 + ghc-options: -Wall -O2 -msse4.2 if flag(sse42) ghc-options: -msse4.2 if flag(bmi2) && impl(ghc >= 8.4.1) @@ -150,10 +150,10 @@ , semigroups , text , vector - main-is: Main.hs - hs-source-dirs: app - ghc-options: -threaded -rtsopts -with-rtsopts=-N - build-depends: hw-json + main-is: Main.hs + hs-source-dirs: app + ghc-options: -threaded -rtsopts -with-rtsopts=-N + build-depends: hw-json other-modules: App.Commands App.Commands.CreateIndex @@ -180,12 +180,12 @@ , text , transformers , vector - type: exitcode-stdio-1.0 - main-is: Spec.hs - build-depends: hw-json - hs-source-dirs: test - ghc-options: -threaded -rtsopts -with-rtsopts=-N - build-tools: hspec-discover + type: exitcode-stdio-1.0 + main-is: Spec.hs + build-depends: hw-json + hs-source-dirs: test + ghc-options: -threaded -rtsopts -with-rtsopts=-N + build-tool-depends: hspec-discover:hspec-discover other-modules: HaskellWorks.Data.Json.LightJsonSpec HaskellWorks.Data.Json.Simple.CursorSpec @@ -204,8 +204,8 @@ , hw-json-standard-cursor , mmap , semigroups - type: exitcode-stdio-1.0 - main-is: Main.hs - hs-source-dirs: bench - build-depends: hw-json - other-modules: Paths_hw_json + type: exitcode-stdio-1.0 + main-is: Main.hs + hs-source-dirs: bench + build-depends: hw-json + other-modules: Paths_hw_json