diff --git a/hw-json-standard-cursor.cabal b/hw-json-standard-cursor.cabal --- a/hw-json-standard-cursor.cabal +++ b/hw-json-standard-cursor.cabal @@ -1,7 +1,7 @@ cabal-version: 2.2 name: hw-json-standard-cursor -version: 0.2.2.0 +version: 0.2.3.0 synopsis: Memory efficient JSON parser description: Memory efficient JSON parser. Please see README.md category: Data @@ -46,10 +46,10 @@ common directory { build-depends: directory >= 1.3 && < 1.4 } common doctest { build-depends: doctest >= 0.16.2 && < 0.17 } common doctest-discover { build-depends: doctest-discover >= 0.2 && < 0.3 } -common generic-lens { build-depends: generic-lens >= 1.1.0.0 && < 1.3 } +common generic-lens { build-depends: generic-lens >= 1.1.0.0 && < 2.1 } common hedgehog { build-depends: hedgehog >= 0.6 && < 1.1 } common hspec { build-depends: hspec >= 2.4 && < 3 } -common hw-balancedparens { build-depends: hw-balancedparens >= 0.3.0.0 && < 0.4 } +common hw-balancedparens { build-depends: hw-balancedparens >= 0.3.0.0 && < 0.5 } common hw-bits { build-depends: hw-bits >= 0.7.0.5 && < 0.8 } common hw-hspec-hedgehog { build-depends: hw-hspec-hedgehog >= 0.1.0.4 && < 0.2 } common hw-json-simd { build-depends: hw-json-simd >= 0.1.0.2 && < 0.2 } @@ -92,10 +92,10 @@ exposed-modules: HaskellWorks.Data.Json.Standard.Cursor HaskellWorks.Data.Json.Standard.Cursor.Fast HaskellWorks.Data.Json.Standard.Cursor.Generic + HaskellWorks.Data.Json.Standard.Cursor.IbBp HaskellWorks.Data.Json.Standard.Cursor.Index HaskellWorks.Data.Json.Standard.Cursor.Internal.Blank HaskellWorks.Data.Json.Standard.Cursor.Internal.BlankedJson - HaskellWorks.Data.Json.Standard.Cursor.Internal.IbBp HaskellWorks.Data.Json.Standard.Cursor.Internal.MakeIndex HaskellWorks.Data.Json.Standard.Cursor.Internal.StateMachine HaskellWorks.Data.Json.Standard.Cursor.Internal.ToBalancedParens64 @@ -167,14 +167,14 @@ hs-source-dirs: bench build-depends: hw-json-standard-cursor -test-suite hw-json-standard-cursor-doctest +test-suite doctest import: base, config , doctest , doctest-discover , bits-extra default-language: Haskell2010 type: exitcode-stdio-1.0 - ghc-options: -threaded + ghc-options: -threaded -rtsopts -with-rtsopts=-N main-is: DoctestDriver.hs HS-Source-Dirs: doctest build-tool-depends: doctest-discover:doctest-discover diff --git a/src/HaskellWorks/Data/Json/Standard/Cursor/Fast.hs b/src/HaskellWorks/Data/Json/Standard/Cursor/Fast.hs --- a/src/HaskellWorks/Data/Json/Standard/Cursor/Fast.hs +++ b/src/HaskellWorks/Data/Json/Standard/Cursor/Fast.hs @@ -13,16 +13,16 @@ import Foreign.ForeignPtr import HaskellWorks.Data.Json.Standard.Cursor.Generic -import HaskellWorks.Data.Json.Standard.Cursor.Internal.IbBp +import HaskellWorks.Data.Json.Standard.Cursor.IbBp import HaskellWorks.Data.Json.Standard.Cursor.Specific import HaskellWorks.Data.RankSelect.CsPoppy1 -import qualified Data.ByteString as BS -import qualified Data.ByteString.Char8 as BSC -import qualified Data.ByteString.Internal as BSI -import qualified HaskellWorks.Data.BalancedParens.RangeMin as RM -import qualified HaskellWorks.Data.FromForeignRegion as F -import qualified HaskellWorks.Data.Json.Standard.Cursor.Internal.IbBp as J +import qualified Data.ByteString as BS +import qualified Data.ByteString.Char8 as BSC +import qualified Data.ByteString.Internal as BSI +import qualified HaskellWorks.Data.BalancedParens.RangeMin as RM +import qualified HaskellWorks.Data.FromForeignRegion as F +import qualified HaskellWorks.Data.Json.Standard.Cursor.IbBp as J data Fast diff --git a/src/HaskellWorks/Data/Json/Standard/Cursor/IbBp.hs b/src/HaskellWorks/Data/Json/Standard/Cursor/IbBp.hs new file mode 100644 --- /dev/null +++ b/src/HaskellWorks/Data/Json/Standard/Cursor/IbBp.hs @@ -0,0 +1,47 @@ +{-# LANGUAGE BangPatterns #-} + +module HaskellWorks.Data.Json.Standard.Cursor.IbBp + ( IbBp(..) + , slowToIbBp + , simdToIbBp) where + +import Control.Monad +import Control.Monad.ST (ST) +import Data.Word +import HaskellWorks.Data.Vector.AsVector64 + +import qualified Data.ByteString as BS +import qualified Data.ByteString.Lazy as LBS +import qualified Data.Vector.Storable as DVS +import qualified Data.Vector.Storable.Mutable as DVSM +import qualified HaskellWorks.Data.ByteString.Lazy as LBS +import qualified HaskellWorks.Data.Json.Simd.Index.Standard as STSI +import qualified HaskellWorks.Data.Json.Standard.Cursor.Internal.BlankedJson as J +import qualified HaskellWorks.Data.Json.Standard.Cursor.Internal.ToBalancedParens64 as J +import qualified HaskellWorks.Data.Json.Standard.Cursor.Internal.ToInterestBits64 as J +import qualified HaskellWorks.Data.Vector.Storable as DVS + +data IbBp = IbBp + { ib :: DVS.Vector Word64 + , bp :: DVS.Vector Word64 + } + +slowToIbBp :: BS.ByteString -> IbBp +slowToIbBp bs = IbBp + { ib = J.toInterestBits64 blankedJson + , bp = J.toBalancedParens64 blankedJson + } + where blankedJson = J.toBlankedJsonTyped bs + +simdToIbBp :: BS.ByteString -> IbBp +simdToIbBp bs = case STSI.makeStandardJsonIbBps (LBS.rechunkPadded chunkSize (LBS.fromStrict bs)) of + Left msg -> error msg + Right chunks -> uncurry IbBp $ DVS.construct2N maxSize (go . fst) maxSize (go . snd) chunks + where chunkSize = 16384 + maxSize = (BS.length bs + chunkSize - 1) `div` 8 + go :: BS.ByteString -> DVSM.MVector s Word64 -> ST s Int + go bytes mv = do + let source = asVector64 bytes + let target = DVSM.take (DVS.length source) mv + when (DVS.length source > 0) $ DVS.copy target source + return (DVS.length source) diff --git a/src/HaskellWorks/Data/Json/Standard/Cursor/Internal/IbBp.hs b/src/HaskellWorks/Data/Json/Standard/Cursor/Internal/IbBp.hs deleted file mode 100644 --- a/src/HaskellWorks/Data/Json/Standard/Cursor/Internal/IbBp.hs +++ /dev/null @@ -1,47 +0,0 @@ -{-# LANGUAGE BangPatterns #-} - -module HaskellWorks.Data.Json.Standard.Cursor.Internal.IbBp - ( IbBp(..) - , slowToIbBp - , simdToIbBp) where - -import Control.Monad -import Control.Monad.ST (ST) -import Data.Word -import HaskellWorks.Data.Vector.AsVector64 - -import qualified Data.ByteString as BS -import qualified Data.ByteString.Lazy as LBS -import qualified Data.Vector.Storable as DVS -import qualified Data.Vector.Storable.Mutable as DVSM -import qualified HaskellWorks.Data.ByteString.Lazy as LBS -import qualified HaskellWorks.Data.Json.Simd.Index.Standard as STSI -import qualified HaskellWorks.Data.Json.Standard.Cursor.Internal.BlankedJson as J -import qualified HaskellWorks.Data.Json.Standard.Cursor.Internal.ToBalancedParens64 as J -import qualified HaskellWorks.Data.Json.Standard.Cursor.Internal.ToInterestBits64 as J -import qualified HaskellWorks.Data.Vector.Storable as DVS - -data IbBp = IbBp - { ib :: DVS.Vector Word64 - , bp :: DVS.Vector Word64 - } - -slowToIbBp :: BS.ByteString -> IbBp -slowToIbBp bs = IbBp - { ib = J.toInterestBits64 blankedJson - , bp = J.toBalancedParens64 blankedJson - } - where blankedJson = J.toBlankedJsonTyped bs - -simdToIbBp :: BS.ByteString -> IbBp -simdToIbBp bs = case STSI.makeStandardJsonIbBps (LBS.rechunkPadded chunkSize (LBS.fromStrict bs)) of - Left msg -> error msg - Right chunks -> uncurry IbBp $ DVS.construct2N maxSize (go . fst) maxSize (go . snd) chunks - where chunkSize = 16384 - maxSize = (BS.length bs + chunkSize - 1) `div` 8 - go :: BS.ByteString -> DVSM.MVector s Word64 -> ST s Int - go bytes mv = do - let source = asVector64 bytes - let target = DVSM.take (DVS.length source) mv - when (DVS.length source > 0) $ DVS.copy target source - return (DVS.length source) diff --git a/src/HaskellWorks/Data/Json/Standard/Cursor/Slow.hs b/src/HaskellWorks/Data/Json/Standard/Cursor/Slow.hs --- a/src/HaskellWorks/Data/Json/Standard/Cursor/Slow.hs +++ b/src/HaskellWorks/Data/Json/Standard/Cursor/Slow.hs @@ -2,6 +2,7 @@ module HaskellWorks.Data.Json.Standard.Cursor.Slow ( Cursor + , fromBsIbBp , fromByteString , fromForeignRegion , fromString @@ -10,15 +11,16 @@ import Data.Word import Foreign.ForeignPtr import HaskellWorks.Data.Json.Standard.Cursor.Generic +import HaskellWorks.Data.Json.Standard.Cursor.IbBp import HaskellWorks.Data.Json.Standard.Cursor.Specific -import qualified Data.ByteString as BS -import qualified Data.ByteString.Char8 as BSC -import qualified Data.ByteString.Internal as BSI -import qualified Data.Vector.Storable as DVS -import qualified HaskellWorks.Data.BalancedParens as BP -import qualified HaskellWorks.Data.FromForeignRegion as F -import qualified HaskellWorks.Data.Json.Standard.Cursor.Internal.IbBp as J +import qualified Data.ByteString as BS +import qualified Data.ByteString.Char8 as BSC +import qualified Data.ByteString.Internal as BSI +import qualified Data.Vector.Storable as DVS +import qualified HaskellWorks.Data.BalancedParens as BP +import qualified HaskellWorks.Data.FromForeignRegion as F +import qualified HaskellWorks.Data.Json.Standard.Cursor.IbBp as J data Slow @@ -27,14 +29,17 @@ type Cursor = GenericCursor BS.ByteString (DVS.Vector Word64) (BP.SimpleBalancedParens (DVS.Vector Word64)) +fromBsIbBp :: BS.ByteString -> IbBp -> Cursor +fromBsIbBp bs ibBp = GenericCursor + { cursorText = bs + , interests = ibPart + , balancedParens = BP.SimpleBalancedParens bpPart + , cursorRank = 1 + } + where J.IbBp ibPart bpPart = ibBp + fromByteString :: BS.ByteString -> Cursor -fromByteString bs = GenericCursor - { cursorText = bs - , interests = ib - , balancedParens = BP.SimpleBalancedParens bp - , cursorRank = 1 - } - where J.IbBp ib bp = J.slowToIbBp bs +fromByteString bs = fromBsIbBp bs (J.slowToIbBp bs) fromForeignRegion :: F.ForeignRegion -> Cursor fromForeignRegion (fptr, offset, size) = fromByteString (BSI.fromForeignPtr (castForeignPtr fptr) offset size)