diff --git a/README.md b/README.md --- a/README.md +++ b/README.md @@ -1,13 +1,10 @@ # hw-json -[![0.0-branch](https://circleci.com/gh/haskell-works/hw-json/tree/0.0-branch.svg?style=svg)](https://circleci.com/gh/haskell-works/hw-json/tree/0-branch) - -Conduits for tokenizing streams. +[![master](https://circleci.com/gh/haskell-works/hw-json/tree/master.svg?style=svg)](https://circleci.com/gh/haskell-works/hw-json/tree/master) -`hw-json` is a succinct JSON parsing library. It uses succinct data-structures to allow traversal of +`hw-json` is a succinct JSON parsing library. +It uses succinct data-structures to allow traversal of large JSON strings with minimal memory overhead. -It is currently considered experimental. - For an example, see [`app/Main.hs`](../master/app/Main.hs) ## Prerequisites @@ -34,8 +31,8 @@ Mem (MB) CMD -------- --------------------------------------------------------- - 54 import Control.DeepSeq - 64 import Data.Aeson + 54 import Control.DeepSeq + 64 import Data.Aeson 64 import qualified Data.ByteString.Lazy as BSL 66 json78m <- BSL.readFile "../data/78mb.json" 167 let !x = deepseq json78m json78m @@ -141,7 +138,7 @@ !cursor <- loadJsonPartial "../data/78mb.json" !cursor <- loadJsonWithIndex "../data/78mb.json" !cursor <- loadJson "../data/78mb.json" -!cursor <- loadJsonWithPoppy512SMinMaxIndex "../data/78mb.json" +!cursor <- loadJsonWithCsPoppyIndex "../data/78mb.json" let !json = jsonPartialJsonValueAt cursor let q = MQuery (DL.singleton json) ``` @@ -177,7 +174,7 @@ !cursor <- loadJsonPartial "../data/78mb.json" !cursor <- loadJsonWithIndex "../data/78mb.json" !cursor <- loadJson "../data/78mb.json" -!cursor <- loadJsonWithPoppy512SMinMaxIndex "../data/78mb.json" +!cursor <- loadJsonWithCsPoppyIndex "../data/78mb.json" let !json = lightJsonAt cursor let q = MQuery (DL.singleton json) ``` diff --git a/app/Main.hs b/app/Main.hs --- a/app/Main.hs +++ b/app/Main.hs @@ -4,22 +4,23 @@ module Main where -import qualified Data.ByteString as BS -import qualified Data.DList as DL -import Data.Function -import qualified Data.Vector.Storable as DVS -import Data.Word -import HaskellWorks.Data.BalancedParens.Simple -import HaskellWorks.Data.Bits.BitShown -import HaskellWorks.Data.FromByteString -import HaskellWorks.Data.Json.Succinct.Cursor -import HaskellWorks.Data.Json.LightJson -import HaskellWorks.Data.Json.LoadCursor -import HaskellWorks.Data.Micro -import HaskellWorks.Data.MQuery -import HaskellWorks.Diagnostics.Time -import Control.Monad +import Control.Monad +import Data.Function +import Data.Word +import HaskellWorks.Data.BalancedParens.Simple +import HaskellWorks.Data.Bits.BitShown +import HaskellWorks.Data.FromByteString +import HaskellWorks.Data.Json.LightJson +import HaskellWorks.Data.Json.LoadCursor +import HaskellWorks.Data.Json.Succinct.Cursor +import HaskellWorks.Data.Micro +import HaskellWorks.Data.MQuery +import HaskellWorks.Diagnostics.Time +import qualified Data.ByteString as BS +import qualified Data.DList as DL +import qualified Data.Vector.Storable as DVS + readJson :: String -> IO (JsonCursor BS.ByteString (BitShown (DVS.Vector Word64)) (SimpleBalancedParens (DVS.Vector Word64))) readJson path = do bs <- BS.readFile path @@ -43,7 +44,7 @@ main :: IO () main = do - !cursor <- loadJsonWithPoppy512SMinMaxIndex "../data/78mb.json" + !cursor <- loadJsonWithCsPoppyIndex "../data/78mb.json" -- let ranks = explore (balancedParens cursor) (cursorRank cursor) -- putStrLn $ "Ranks: " ++ show (length (ranks [])) diff --git a/hw-json.cabal b/hw-json.cabal --- a/hw-json.cabal +++ b/hw-json.cabal @@ -1,166 +1,190 @@ -name: hw-json -version: 0.6.0.0 -synopsis: Memory efficient JSON parser -description: Please see README.md -homepage: http://github.com/haskell-works/hw-json#readme -license: BSD3 -license-file: LICENSE -author: John Ky -maintainer: newhoggy@gmail.com -copyright: 2016 John Ky -category: Data, Conduit -build-type: Simple -extra-source-files: README.md, - corpus/5000B.bp, - corpus/5000B.ib, - corpus/5000B.json, - corpus/issue-0001.bp, - corpus/issue-0001.ib, - corpus/issue-0001.json, - corpus/issue-0001.md -cabal-version: >= 1.22 +-- This file has been generated from package.yaml by hpack version 0.20.0. +-- +-- see: https://github.com/sol/hpack +-- +-- hash: b4bd849a21612a66cf5c9004ea3b8843cdb14954a1181ced3bbf953a50752f99 -executable hw-json-example - hs-source-dirs: app - main-is: Main.hs - ghc-options: -threaded -rtsopts -with-rtsopts=-N -O2 -Wall -msse4.2 - build-depends: base >= 4 && < 5 - , array - , ansi-wl-pprint - , attoparsec - , bytestring - , conduit - , containers - , criterion - , dlist - , hw-balancedparens >= 0.1.0.0 - , hw-bits >= 0.4.0.0 - , hw-conduit >= 0.1.0.0 - , hw-diagnostics >= 0.0.0.5 - , hw-json - , hw-mquery >= 0.1.0.0 - , hw-parser >= 0.0.0.1 - , hw-prim >= 0.4.0.0 - , hw-rankselect >= 0.8.0.0 - , hw-rankselect-base >= 0.2.0.0 - , mmap - , mono-traversable - , resourcet - , text - , vector - , word8 - default-language: Haskell2010 +name: hw-json +version: 0.7.0.1 +synopsis: Memory efficient JSON parser +description: Memory efficient JSON parser. Please see README.md +category: Data, Conduit +homepage: http://github.com/haskell-works/hw-json#readme +bug-reports: https://github.com/haskell-works/hw-json/issues +author: John Ky +maintainer: newhoggy@gmail.com +copyright: 2016 John Ky +license: BSD3 +license-file: LICENSE +build-type: Simple +cabal-version: >= 1.10 -library - hs-source-dirs: src - exposed-modules: HaskellWorks.Data.Json - , HaskellWorks.Data.Json.CharLike - , HaskellWorks.Data.Json.Conduit - , HaskellWorks.Data.Json.Conduit.Blank - , HaskellWorks.Data.Json.Conduit.Words - , HaskellWorks.Data.Json.DecodeError - , HaskellWorks.Data.Json.FromValue - , HaskellWorks.Data.Json.LightJson - , HaskellWorks.Data.Json.Load - , HaskellWorks.Data.Json.LoadCursor - , HaskellWorks.Data.Json.PartialValue - , HaskellWorks.Data.Json.Succinct - , HaskellWorks.Data.Json.Succinct.Cursor - , HaskellWorks.Data.Json.Succinct.Cursor.BalancedParens - , HaskellWorks.Data.Json.Succinct.Cursor.BlankedJson - , HaskellWorks.Data.Json.Succinct.Cursor.InterestBits - , HaskellWorks.Data.Json.Succinct.Cursor.Internal - , HaskellWorks.Data.Json.Succinct.Cursor.Token - , HaskellWorks.Data.Json.Succinct.Index - , HaskellWorks.Data.Json.Succinct.PartialIndex - , HaskellWorks.Data.Json.Token.Tokenize - , HaskellWorks.Data.Json.Token.Types - , HaskellWorks.Data.Json.Token - , HaskellWorks.Data.Json.Type - , HaskellWorks.Data.Json.Value - , HaskellWorks.Data.Json.Value.Internal +extra-source-files: + corpus/5000B.bp + corpus/5000B.ib + corpus/5000B.json + corpus/issue-0001.bp + corpus/issue-0001.ib + corpus/issue-0001.json + corpus/issue-0001.md + README.md - build-depends: base >= 4 && < 5 - , array - , ansi-wl-pprint - , attoparsec - , bytestring - , conduit - , containers - , dlist - , hw-balancedparens >= 0.1.0.0 - , hw-bits >= 0.4.0.0 - , hw-conduit >= 0.1.0.0 - , hw-mquery >= 0.1.0.0 - , hw-parser >= 0.0.0.1 - , hw-prim >= 0.4.0.0 - , hw-rankselect >= 0.8.0.0 - , hw-rankselect-base >= 0.2.0.0 - , mmap - , mono-traversable - , resourcet - , text - , vector - , word8 +source-repository head + type: git + location: https://github.com/haskell-works/hw-json - default-language: Haskell2010 - ghc-options: -Wall -O2 -msse4.2 +library + hs-source-dirs: + src + ghc-options: -Wall -O2 -msse4.2 + build-depends: + ansi-wl-pprint + , array + , attoparsec + , base >=4 && <5 + , bytestring + , conduit + , containers + , dlist + , hw-balancedparens >=0.1.0.0 + , hw-bits >=0.4.0.0 + , hw-conduit >=0.1.0.0 + , hw-mquery >=0.1.0.0 + , hw-parser >=0.0.0.1 + , hw-prim >=0.4.0.0 + , hw-rankselect >=0.8.0.0 + , hw-rankselect-base >=0.2.0.0 + , mmap + , mono-traversable + , resourcet + , text + , vector + , word8 + exposed-modules: + HaskellWorks.Data.Json + HaskellWorks.Data.Json.CharLike + HaskellWorks.Data.Json.Conduit + HaskellWorks.Data.Json.Conduit.Blank + HaskellWorks.Data.Json.Conduit.Words + HaskellWorks.Data.Json.DecodeError + HaskellWorks.Data.Json.FromValue + HaskellWorks.Data.Json.LightJson + HaskellWorks.Data.Json.Load + HaskellWorks.Data.Json.LoadCursor + HaskellWorks.Data.Json.PartialValue + HaskellWorks.Data.Json.Succinct + HaskellWorks.Data.Json.Succinct.Cursor + HaskellWorks.Data.Json.Succinct.Cursor.BalancedParens + HaskellWorks.Data.Json.Succinct.Cursor.BlankedJson + HaskellWorks.Data.Json.Succinct.Cursor.InterestBits + HaskellWorks.Data.Json.Succinct.Cursor.Internal + HaskellWorks.Data.Json.Succinct.Cursor.Token + HaskellWorks.Data.Json.Succinct.Index + HaskellWorks.Data.Json.Succinct.PartialIndex + HaskellWorks.Data.Json.Token.Tokenize + HaskellWorks.Data.Json.Token.Types + HaskellWorks.Data.Json.Token + HaskellWorks.Data.Json.Type + HaskellWorks.Data.Json.Value + HaskellWorks.Data.Json.Value.Internal + other-modules: + Paths_hw_json + default-language: Haskell2010 -test-suite hw-json-test - type: exitcode-stdio-1.0 - hs-source-dirs: test - main-is: Spec.hs - other-modules: HaskellWorks.Data.Json.Conduit.BlankSpec - , HaskellWorks.Data.Json.CorpusSpec - , HaskellWorks.Data.Json.Succinct.Cursor.BalancedParensSpec - , HaskellWorks.Data.Json.Succinct.Cursor.InterestBitsSpec - , HaskellWorks.Data.Json.Succinct.CursorSpec - , HaskellWorks.Data.Json.Token.TokenizeSpec - , HaskellWorks.Data.Json.TypeSpec - , HaskellWorks.Data.Json.ValueSpec - build-depends: base >= 4 && < 5 - , attoparsec - , bytestring - , containers - , conduit - , hspec - , hw-balancedparens >= 0.1.0.0 - , hw-bits >= 0.4.0.0 - , hw-conduit >= 0.1.0.0 - , hw-json - , hw-prim >= 0.4.0.0 - , hw-rankselect >= 0.8.0.0 - , hw-rankselect-base >= 0.2.0.0 - , mmap - , parsec - , QuickCheck - , resourcet - , transformers - , vector - ghc-options: -threaded -rtsopts -with-rtsopts=-N -Wall - default-language: Haskell2010 +executable hw-json-example + main-is: Main.hs + hs-source-dirs: + app + ghc-options: -Wall -threaded -rtsopts -with-rtsopts=-N -O2 -msse4.2 + build-depends: + ansi-wl-pprint + , array + , attoparsec + , base >=4 && <5 + , bytestring + , conduit + , containers + , criterion + , dlist + , hw-balancedparens >=0.1.0.0 + , hw-bits >=0.4.0.0 + , hw-conduit >=0.1.0.0 + , hw-diagnostics >=0.0.0.5 + , hw-json + , hw-mquery >=0.1.0.0 + , hw-parser >=0.0.0.1 + , hw-prim >=0.4.0.0 + , hw-rankselect >=0.8.0.0 + , hw-rankselect-base >=0.2.0.0 + , mmap + , mono-traversable + , resourcet + , text + , vector + , word8 + other-modules: + Paths_hw_json + default-language: Haskell2010 -source-repository head - type: git - location: https://github.com/haskell-works/hw-json +test-suite hw-json-test + type: exitcode-stdio-1.0 + main-is: Spec.hs + hs-source-dirs: + test + ghc-options: -Wall -threaded -rtsopts -with-rtsopts=-N + build-depends: + QuickCheck + , attoparsec + , base >=4 && <5 + , bytestring + , conduit + , containers + , hspec + , hw-balancedparens >=0.1.0.0 + , hw-bits >=0.4.0.0 + , hw-conduit >=0.1.0.0 + , hw-json + , hw-prim >=0.4.0.0 + , hw-rankselect >=0.8.0.0 + , hw-rankselect-base >=0.2.0.0 + , mmap + , parsec + , resourcet + , transformers + , vector + other-modules: + HaskellWorks.Data.Json.Conduit.BlankSpec + HaskellWorks.Data.Json.CorpusSpec + HaskellWorks.Data.Json.Succinct.Cursor.BalancedParensSpec + HaskellWorks.Data.Json.Succinct.Cursor.InterestBitsSpec + HaskellWorks.Data.Json.Succinct.CursorSpec + HaskellWorks.Data.Json.Token.TokenizeSpec + HaskellWorks.Data.Json.TypeSpec + HaskellWorks.Data.Json.ValueSpec + Paths_hw_json + default-language: Haskell2010 benchmark bench - Type: exitcode-stdio-1.0 - HS-Source-Dirs: bench - Main-Is: Main.hs - GHC-Options: -O2 -Wall -msse4.2 - Default-Language: Haskell2010 - Build-Depends: base >= 4 && < 5 - , bytestring - , conduit - , criterion - , hw-balancedparens >= 0.1.0.0 - , hw-bits >= 0.4.0.0 - , hw-conduit >= 0.1.0.0 - , hw-json - , hw-prim >= 0.4.0.0 - , hw-rankselect >= 0.8.0.0 - , hw-rankselect-base >= 0.2.0.0 - , mmap - , resourcet - , vector + type: exitcode-stdio-1.0 + main-is: Main.hs + hs-source-dirs: + bench + ghc-options: -Wall -O2 -msse4.2 + build-depends: + base >=4 && <5 + , bytestring + , conduit + , criterion + , hw-balancedparens >=0.1.0.0 + , hw-bits >=0.4.0.0 + , hw-conduit >=0.1.0.0 + , hw-json + , hw-prim >=0.4.0.0 + , hw-rankselect >=0.8.0.0 + , hw-rankselect-base >=0.2.0.0 + , mmap + , resourcet + , vector + other-modules: + Paths_hw_json + default-language: Haskell2010 diff --git a/src/HaskellWorks/Data/Json/Conduit.hs b/src/HaskellWorks/Data/Json/Conduit.hs --- a/src/HaskellWorks/Data/Json/Conduit.hs +++ b/src/HaskellWorks/Data/Json/Conduit.hs @@ -29,7 +29,7 @@ else 0) | w <- [0 .. 255]] -blankedJsonToInterestBits :: Monad m => Conduit BS.ByteString m BS.ByteString +blankedJsonToInterestBits :: Monad m => ConduitT BS.ByteString BS.ByteString m () blankedJsonToInterestBits = blankedJsonToInterestBits' "" padRight :: Word8 -> Int -> BS.ByteString -> BS.ByteString @@ -39,7 +39,7 @@ Just (c, ds) -> Just (c, ds) Nothing -> Just (w, BS.empty) -blankedJsonToInterestBits' :: Monad m => BS.ByteString -> Conduit BS.ByteString m BS.ByteString +blankedJsonToInterestBits' :: Monad m => BS.ByteString -> ConduitT BS.ByteString BS.ByteString m () blankedJsonToInterestBits' rs = do mbs <- await case mbs of @@ -59,14 +59,14 @@ , BS.drop 8 as ) -blankedJsonToBalancedParens :: Monad m => Conduit BS.ByteString m Bool +blankedJsonToBalancedParens :: Monad m => ConduitT BS.ByteString Bool m () blankedJsonToBalancedParens = do mbs <- await case mbs of Just bs -> blankedJsonToBalancedParens' bs Nothing -> return () -blankedJsonToBalancedParens' :: Monad m => BS.ByteString -> Conduit BS.ByteString m Bool +blankedJsonToBalancedParens' :: Monad m => BS.ByteString -> ConduitT BS.ByteString Bool m () blankedJsonToBalancedParens' bs = case BS.uncons bs of Just (c, cs) -> do case c of @@ -90,10 +90,10 @@ abLen = BS.length abBS cLen = (abLen `div` 8) * 8 -compressWordAsBit :: Monad m => Conduit BS.ByteString m BS.ByteString +compressWordAsBit :: Monad m => ConduitT BS.ByteString BS.ByteString m () compressWordAsBit = compressWordAsBit' BS.empty -compressWordAsBit' :: Monad m => BS.ByteString -> Conduit BS.ByteString m BS.ByteString +compressWordAsBit' :: Monad m => BS.ByteString -> ConduitT BS.ByteString BS.ByteString m () compressWordAsBit' aBS = do mbBS <- await case mbBS of @@ -112,7 +112,7 @@ , BS.drop 8 xs ) -blankedJsonToBalancedParens2 :: Monad m => Conduit BS.ByteString m BS.ByteString +blankedJsonToBalancedParens2 :: Monad m => ConduitT BS.ByteString BS.ByteString m () blankedJsonToBalancedParens2 = do mbs <- await case mbs of @@ -148,7 +148,7 @@ d | d == _n -> MiniTF _ -> MiniN -yieldBitsOfWord8 :: Monad m => Word8 -> Conduit BS.ByteString m Bool +yieldBitsOfWord8 :: Monad m => Word8 -> ConduitT BS.ByteString Bool m () yieldBitsOfWord8 w = do yield ((w .&. BITS.bit 0) /= 0) yield ((w .&. BITS.bit 1) /= 0) @@ -159,10 +159,10 @@ yield ((w .&. BITS.bit 6) /= 0) yield ((w .&. BITS.bit 7) /= 0) -yieldBitsofWord8s :: Monad m => [Word8] -> Conduit BS.ByteString m Bool +yieldBitsofWord8s :: Monad m => [Word8] -> ConduitT BS.ByteString Bool m () yieldBitsofWord8s = P.foldr ((>>) . yieldBitsOfWord8) (return ()) -byteStringToBits :: Monad m => Conduit BS.ByteString m Bool +byteStringToBits :: Monad m => ConduitT BS.ByteString Bool m () byteStringToBits = do mbs <- await case mbs of diff --git a/src/HaskellWorks/Data/Json/Conduit/Blank.hs b/src/HaskellWorks/Data/Json/Conduit/Blank.hs --- a/src/HaskellWorks/Data/Json/Conduit/Blank.hs +++ b/src/HaskellWorks/Data/Json/Conduit/Blank.hs @@ -1,4 +1,4 @@ -{-# LANGUAGE BangPatterns #-} +{-# LANGUAGE BangPatterns #-} module HaskellWorks.Data.Json.Conduit.Blank ( blankJson @@ -20,10 +20,10 @@ | InNumber | InIdent -blankJson :: MonadThrow m => Conduit BS.ByteString m BS.ByteString +blankJson :: MonadThrow m => ConduitT BS.ByteString BS.ByteString m () blankJson = blankJson' InJson -blankJson' :: MonadThrow m => BlankState -> Conduit BS.ByteString m BS.ByteString +blankJson' :: MonadThrow m => BlankState -> ConduitT BS.ByteString BS.ByteString m () blankJson' lastState = do mbs <- await case mbs of @@ -38,25 +38,25 @@ Just (!c, !cs) | isLeadingDigit c -> Just (_1 , (InNumber , cs)) Just (!c, !cs) | c == _quotedbl -> Just (_parenleft , (InString , cs)) Just (!c, !cs) | isAlphabetic c -> Just (c , (InIdent , cs)) - Just (!c, !cs) -> Just (c , (InJson , cs)) - Nothing -> Nothing + Just (!c, !cs) -> Just (c , (InJson , cs)) + Nothing -> Nothing blankByteString (InString, bs) = case BS.uncons bs of Just (!c, !cs) | c == _backslash -> Just (_space , (Escaped , cs)) Just (!c, !cs) | c == _quotedbl -> Just (_parenright , (InJson , cs)) - Just (_ , !cs) -> Just (_space , (InString , cs)) - Nothing -> Nothing + Just (_ , !cs) -> Just (_space , (InString , cs)) + Nothing -> Nothing blankByteString (Escaped, bs) = case BS.uncons bs of - Just (_, !cs) -> Just (_space, (InString, cs)) - Nothing -> Nothing + Just (_, !cs) -> Just (_space, (InString, cs)) + Nothing -> Nothing blankByteString (InNumber, bs) = case BS.uncons bs of Just (!c, !cs) | isTrailingDigit c -> Just (_0 , (InNumber , cs)) Just (!c, !cs) | c == _quotedbl -> Just (_parenleft , (InString , cs)) Just (!c, !cs) | isAlphabetic c -> Just (c , (InIdent , cs)) - Just (!c, !cs) -> Just (c , (InJson , cs)) - Nothing -> Nothing + Just (!c, !cs) -> Just (c , (InJson , cs)) + Nothing -> Nothing blankByteString (InIdent, bs) = case BS.uncons bs of Just (!c, !cs) | isAlphabetic c -> Just (_underscore , (InIdent , cs)) Just (!c, !cs) | isLeadingDigit c -> Just (_1 , (InNumber , cs)) Just (!c, !cs) | c == _quotedbl -> Just (_parenleft , (InString , cs)) - Just (!c, !cs) -> Just (c , (InJson , cs)) - Nothing -> Nothing + Just (!c, !cs) -> Just (c , (InJson , cs)) + Nothing -> Nothing diff --git a/src/HaskellWorks/Data/Json/Load.hs b/src/HaskellWorks/Data/Json/Load.hs --- a/src/HaskellWorks/Data/Json/Load.hs +++ b/src/HaskellWorks/Data/Json/Load.hs @@ -12,10 +12,6 @@ , loadJsonWithIndex , loadJsonWithPoppy512Index , loadJsonWithPoppy512Index2 - , loadJsonWithPoppy512SIndex - , loadJsonWithPoppy512SIndex2 - , loadJsonWithPoppy512SMinMaxIndex - , loadJsonWithPoppy512SMinMax2Index ) where import Control.Monad @@ -23,8 +19,6 @@ import Foreign.ForeignPtr import Foreign.Ptr import Foreign.Storable -import HaskellWorks.Data.BalancedParens.RangeMinMax -import HaskellWorks.Data.BalancedParens.RangeMinMax2 import HaskellWorks.Data.BalancedParens.Simple import HaskellWorks.Data.Bits.BitShown import HaskellWorks.Data.FromByteString @@ -37,7 +31,6 @@ import HaskellWorks.Data.Json.Value import HaskellWorks.Data.RankSelect.CsPoppy import HaskellWorks.Data.RankSelect.Poppy512 -import HaskellWorks.Data.RankSelect.Poppy512S import System.IO import System.IO.MMap @@ -101,29 +94,6 @@ let !jsonResult = jsonPartialJsonValueAt (jsonPartialIndexAt cursor) return jsonResult -loadJsonWithPoppy512SIndex :: String -> IO JsonPartialValue -loadJsonWithPoppy512SIndex filename = do - (jsonBS, jsonIb, jsonBp) <- loadJsonRawWithIndex filename - let cursor = JsonCursor jsonBS (makePoppy512S jsonIb) (SimpleBalancedParens jsonBp) 1 - let !jsonResult = jsonPartialJsonValueAt (jsonPartialIndexAt cursor) - return jsonResult - -loadJsonWithPoppy512SMinMaxIndex :: String -> IO JsonPartialValue -loadJsonWithPoppy512SMinMaxIndex filename = do - (jsonBS, jsonIb, jsonBp) <- loadJsonRawWithIndex filename - let !rangeMinMax = mkRangeMinMax jsonBp - let cursorL3 = JsonCursor jsonBS (makePoppy512S jsonIb) rangeMinMax 1 - let !jsonResult = jsonPartialJsonValueAt (jsonPartialIndexAt cursorL3) - return jsonResult - -loadJsonWithPoppy512SMinMax2Index :: String -> IO JsonPartialValue -loadJsonWithPoppy512SMinMax2Index filename = do - (jsonBS, jsonIb, jsonBp) <- loadJsonRawWithIndex filename - let !rangeMinMax2 = mkRangeMinMax2 jsonBp - let cursorMinMaxL2 = JsonCursor jsonBS (makePoppy512S jsonIb) rangeMinMax2 1 - let !jsonResult = jsonPartialJsonValueAt (jsonPartialIndexAt cursorMinMaxL2) - return jsonResult - loadJsonWithCsPoppyIndex :: String -> IO JsonPartialValue loadJsonWithCsPoppyIndex filename = do (jsonBS, jsonIb, jsonBp) <- loadJsonRawWithIndex filename @@ -139,14 +109,6 @@ let !jsonResult = jsonPartialJsonValueAt (jsonPartialIndexAt cursor) return jsonResult -loadJsonWithPoppy512SIndex2 :: String -> IO JsonPartialValue -loadJsonWithPoppy512SIndex2 filename = do - (jsonBS, jsonIb, jsonBp) <- loadJsonRawWithIndex filename - let cursor = JsonCursor jsonBS (makePoppy512S jsonIb) (SimpleBalancedParens (makePoppy512S jsonBp)) 1 - :: JsonCursor BSI.ByteString Poppy512S (SimpleBalancedParens Poppy512S) - let !jsonResult = jsonPartialJsonValueAt (jsonPartialIndexAt cursor) - return jsonResult - indexJson :: String -> IO () indexJson filename = do JsonCursor _ (BitShown ib) (SimpleBalancedParens bp) _ <- readJson filename @@ -156,7 +118,7 @@ writeVector (filename ++ ".bp") wbp loadJson :: String -> IO JsonPartialValue -loadJson = loadJsonWithPoppy512SMinMaxIndex +loadJson = loadJsonWithCsPoppyIndex loadJsonPartial :: String -> IO JsonPartialValue loadJsonPartial filename = do diff --git a/src/HaskellWorks/Data/Json/LoadCursor.hs b/src/HaskellWorks/Data/Json/LoadCursor.hs --- a/src/HaskellWorks/Data/Json/LoadCursor.hs +++ b/src/HaskellWorks/Data/Json/LoadCursor.hs @@ -10,10 +10,6 @@ , loadJsonWithIndex , loadJsonWithPoppy512Index , loadJsonWithPoppy512Index2 - , loadJsonWithPoppy512SIndex - , loadJsonWithPoppy512SIndex2 - , loadJsonWithPoppy512SMinMaxIndex - , loadJsonWithPoppy512SMinMax2Index ) where import Control.Monad @@ -21,8 +17,6 @@ import Foreign.ForeignPtr import Foreign.Ptr import Foreign.Storable -import HaskellWorks.Data.BalancedParens.RangeMinMax -import HaskellWorks.Data.BalancedParens.RangeMinMax2 import HaskellWorks.Data.BalancedParens.Simple import HaskellWorks.Data.Bits.BitShown import HaskellWorks.Data.FromByteString @@ -33,7 +27,6 @@ import HaskellWorks.Data.Json.Value import HaskellWorks.Data.RankSelect.CsPoppy import HaskellWorks.Data.RankSelect.Poppy512 -import HaskellWorks.Data.RankSelect.Poppy512S import System.IO import System.IO.MMap @@ -95,26 +88,6 @@ let cursor = JsonCursor jsonBS (makePoppy512 jsonIb) (SimpleBalancedParens jsonBp) 1 return cursor -loadJsonWithPoppy512SIndex :: String -> IO (JsonCursor BSI.ByteString Poppy512S (SimpleBalancedParens (DVS.Vector Word64))) -loadJsonWithPoppy512SIndex filename = do - (jsonBS, jsonIb, jsonBp) <- loadJsonRawWithIndex filename - let cursor = JsonCursor jsonBS (makePoppy512S jsonIb) (SimpleBalancedParens jsonBp) 1 - return cursor - -loadJsonWithPoppy512SMinMaxIndex :: String -> IO (JsonCursor BSI.ByteString Poppy512S (RangeMinMax Poppy512S)) -loadJsonWithPoppy512SMinMaxIndex filename = do - (jsonBS, jsonIb, jsonBp) <- loadJsonRawWithIndex filename - let !rangeMinMax = mkRangeMinMax (makePoppy512S jsonBp) - let cursorL3 = JsonCursor jsonBS (makePoppy512S jsonIb) rangeMinMax 1 - return cursorL3 - -loadJsonWithPoppy512SMinMax2Index :: String -> IO (JsonCursor BSI.ByteString Poppy512S (RangeMinMax2 Poppy512S)) -loadJsonWithPoppy512SMinMax2Index filename = do - (jsonBS, jsonIb, jsonBp) <- loadJsonRawWithIndex filename - let !rangeMinMax2 = mkRangeMinMax2 (makePoppy512S jsonBp) - let cursorMinMaxL2 = JsonCursor jsonBS (makePoppy512S jsonIb) rangeMinMax2 1 - return cursorMinMaxL2 - loadJsonWithCsPoppyIndex :: String -> IO (JsonCursor BSI.ByteString CsPoppy (SimpleBalancedParens (DVS.Vector Word64))) loadJsonWithCsPoppyIndex filename = do (jsonBS, jsonIb, jsonBp) <- loadJsonRawWithIndex filename @@ -126,13 +99,6 @@ (jsonBS, jsonIb, jsonBp) <- loadJsonRawWithIndex filename let cursor = JsonCursor jsonBS (makePoppy512 jsonIb) (SimpleBalancedParens (makePoppy512 jsonBp)) 1 :: JsonCursor BSI.ByteString Poppy512 (SimpleBalancedParens Poppy512) - return cursor - -loadJsonWithPoppy512SIndex2 :: String -> IO (JsonCursor BSI.ByteString Poppy512S (SimpleBalancedParens Poppy512S)) -loadJsonWithPoppy512SIndex2 filename = do - (jsonBS, jsonIb, jsonBp) <- loadJsonRawWithIndex filename - let cursor = JsonCursor jsonBS (makePoppy512S jsonIb) (SimpleBalancedParens (makePoppy512S jsonBp)) 1 - :: JsonCursor BSI.ByteString Poppy512S (SimpleBalancedParens Poppy512S) return cursor indexJsonCursor :: String -> IO () diff --git a/src/HaskellWorks/Data/Json/Succinct/Cursor/BalancedParens.hs b/src/HaskellWorks/Data/Json/Succinct/Cursor/BalancedParens.hs --- a/src/HaskellWorks/Data/Json/Succinct/Cursor/BalancedParens.hs +++ b/src/HaskellWorks/Data/Json/Succinct/Cursor/BalancedParens.hs @@ -33,20 +33,20 @@ instance FromBlankedJson (JsonBalancedParens (SimpleBalancedParens (DVS.Vector Word8))) where fromBlankedJson bj = JsonBalancedParens (SimpleBalancedParens (DVS.unsafeCast (DVS.unfoldrN newLen genBitWordsForever bpBS))) - where bpBS = BS.concat (runListConduit (blankedJsonToBalancedParens2 =$= compressWordAsBit) (getBlankedJson bj)) + where bpBS = BS.concat (runListConduit (blankedJsonToBalancedParens2 .| compressWordAsBit) (getBlankedJson bj)) newLen = (BS.length bpBS + 7) `div` 8 * 8 instance FromBlankedJson (JsonBalancedParens (SimpleBalancedParens (DVS.Vector Word16))) where fromBlankedJson bj = JsonBalancedParens (SimpleBalancedParens (DVS.unsafeCast (DVS.unfoldrN newLen genBitWordsForever bpBS))) - where bpBS = BS.concat (runListConduit (blankedJsonToBalancedParens2 =$= compressWordAsBit) (getBlankedJson bj)) + where bpBS = BS.concat (runListConduit (blankedJsonToBalancedParens2 .| compressWordAsBit) (getBlankedJson bj)) newLen = (BS.length bpBS + 7) `div` 8 * 8 instance FromBlankedJson (JsonBalancedParens (SimpleBalancedParens (DVS.Vector Word32))) where fromBlankedJson bj = JsonBalancedParens (SimpleBalancedParens (DVS.unsafeCast (DVS.unfoldrN newLen genBitWordsForever bpBS))) - where bpBS = BS.concat (runListConduit (blankedJsonToBalancedParens2 =$= compressWordAsBit) (getBlankedJson bj)) + where bpBS = BS.concat (runListConduit (blankedJsonToBalancedParens2 .| compressWordAsBit) (getBlankedJson bj)) newLen = (BS.length bpBS + 7) `div` 8 * 8 instance FromBlankedJson (JsonBalancedParens (SimpleBalancedParens (DVS.Vector Word64))) where fromBlankedJson bj = JsonBalancedParens (SimpleBalancedParens (DVS.unsafeCast (DVS.unfoldrN newLen genBitWordsForever bpBS))) - where bpBS = BS.concat (runListConduit (blankedJsonToBalancedParens2 =$= compressWordAsBit) (getBlankedJson bj)) + where bpBS = BS.concat (runListConduit (blankedJsonToBalancedParens2 .| compressWordAsBit) (getBlankedJson bj)) newLen = (BS.length bpBS + 7) `div` 8 * 8 diff --git a/test/HaskellWorks/Data/Json/Succinct/Cursor/BalancedParensSpec.hs b/test/HaskellWorks/Data/Json/Succinct/Cursor/BalancedParensSpec.hs --- a/test/HaskellWorks/Data/Json/Succinct/Cursor/BalancedParensSpec.hs +++ b/test/HaskellWorks/Data/Json/Succinct/Cursor/BalancedParensSpec.hs @@ -23,5 +23,5 @@ bp `shouldBe` fromString "11111111 11111111 00000000 11111111 00000000 00000000" it "Blanking JSON should not contain strange characters 2" $ do let blankedJson = BlankedJson ["[ [],", "[]]"] - let bp = BitShown $ BS.concat (runListConduit (blankedJsonToBalancedParens2 =$= compressWordAsBit) (getBlankedJson blankedJson)) + let bp = BitShown $ BS.concat (runListConduit (blankedJsonToBalancedParens2 .| compressWordAsBit) (getBlankedJson blankedJson)) bp `shouldBe` fromString "11010000" diff --git a/test/HaskellWorks/Data/Json/Succinct/CursorSpec.hs b/test/HaskellWorks/Data/Json/Succinct/CursorSpec.hs --- a/test/HaskellWorks/Data/Json/Succinct/CursorSpec.hs +++ b/test/HaskellWorks/Data/Json/Succinct/CursorSpec.hs @@ -96,9 +96,7 @@ , BalancedParens u , TestBit u , FromForeignRegion (JsonCursor BS.ByteString t u) - , IsString (JsonCursor BS.ByteString t u) - , JsonIndexAt (JsonCursor BS.ByteString t u) - ) + , IsString (JsonCursor BS.ByteString t u)) => String -> (JsonCursor BS.ByteString t u) -> SpecWith () genSpec t _ = do describe ("Json cursor of type " ++ t) $ do diff --git a/test/HaskellWorks/Data/Json/TypeSpec.hs b/test/HaskellWorks/Data/Json/TypeSpec.hs --- a/test/HaskellWorks/Data/Json/TypeSpec.hs +++ b/test/HaskellWorks/Data/Json/TypeSpec.hs @@ -20,7 +20,6 @@ import HaskellWorks.Data.Bits.BitWise import HaskellWorks.Data.FromForeignRegion import HaskellWorks.Data.Json.Succinct.Cursor as C -import HaskellWorks.Data.Json.Succinct.Index import HaskellWorks.Data.Json.Type import HaskellWorks.Data.RankSelect.Base.Rank0 import HaskellWorks.Data.RankSelect.Base.Rank1 @@ -95,9 +94,7 @@ , BalancedParens u , TestBit u , FromForeignRegion (JsonCursor BS.ByteString t u) - , IsString (JsonCursor BS.ByteString t u) - , JsonIndexAt (JsonCursor BS.ByteString t u) - ) + , IsString (JsonCursor BS.ByteString t u)) => String -> (JsonCursor BS.ByteString t u) -> SpecWith () genSpec t _ = do describe ("Json cursor of type " ++ t) $ do diff --git a/test/HaskellWorks/Data/Json/ValueSpec.hs b/test/HaskellWorks/Data/Json/ValueSpec.hs --- a/test/HaskellWorks/Data/Json/ValueSpec.hs +++ b/test/HaskellWorks/Data/Json/ValueSpec.hs @@ -49,8 +49,13 @@ genSpec "DVS.Vector Word64" (undefined :: JsonCursor BS.ByteString (BitShown (DVS.Vector Word64)) (SimpleBalancedParens (DVS.Vector Word64))) genSpec "Poppy512" (undefined :: JsonCursor BS.ByteString Poppy512 (SimpleBalancedParens (DVS.Vector Word64))) -jsonValueVia :: JsonIndexAt (JsonCursor BS.ByteString t u) - => Maybe (JsonCursor BS.ByteString t u) -> Either DecodeError JsonValue +jsonValueVia :: + ( BalancedParens u + , Rank0 u + , Rank1 u + , Select1 t + , TestBit u) + => Maybe (JsonCursor BS.ByteString t u) -> Either DecodeError JsonValue jsonValueVia mk = case mk of Just k -> (jsonIndexAt >=> jsonValueAt) k Nothing -> Left (DecodeError "No such element") @@ -66,9 +71,7 @@ , BalancedParens u , TestBit u , FromForeignRegion (JsonCursor BS.ByteString t u) - , IsString (JsonCursor BS.ByteString t u) - , JsonIndexAt (JsonCursor BS.ByteString t u) - ) + , IsString (JsonCursor BS.ByteString t u)) => String -> (JsonCursor BS.ByteString t u) -> SpecWith () genSpec t _ = do describe ("Json cursor of type " ++ t) $ do