hw-json 1.0.0.0 → 1.0.0.1
raw patch · 14 files changed
+115/−185 lines, 14 filesPVP ok
version bump matches the API change (PVP)
API changes (from Hackage documentation)
Files
- README.md +98/−166
- corpus/5000B.bp +0/−1
- corpus/5000B.ib +0/−1
- corpus/5000B.json.bp.idx +1/−0
- corpus/5000B.json.ib.idx +1/−0
- corpus/issue-0001.bp +0/−1
- corpus/issue-0001.ib +0/−1
- corpus/issue-0001.json.bp.idx +1/−0
- corpus/issue-0001.json.ib.idx +1/−0
- hw-json.cabal +5/−5
- src/HaskellWorks/Data/Json/Backend/Standard/Cursor.hs +0/−2
- src/HaskellWorks/Data/Json/Backend/Standard/Index.hs +2/−2
- src/HaskellWorks/Data/Json/Backend/Standard/Load/Raw.hs +2/−2
- test/HaskellWorks/Data/Json/CorpusSpec.hs +4/−4
README.md view
@@ -2,117 +2,98 @@ [](https://circleci.com/gh/haskell-works/hw-json/tree/master) `hw-json` is a succinct JSON parsing library.-It uses succinct data-structures to allow traversal of-large JSON strings with minimal memory overhead. +It uses succinct data-structures to allow traversal of large JSON strings with minimal memory overhead.+ For an example, see [`app/Main.hs`](../master/app/Main.hs) ## Prerequisites-* Install `haskell-stack`.-* Install `hlint` (eg. `stack install hlint`) +* `cabal` version `2.2` or later+ ## Memory benchmark ### Parsing large Json files in Scala with Argonaut - S0U EU OU MU CCSU CMD- --------- --------- ----------- -------- -------- ---------------------------------------------------------------- 0.0 80,526.3 76,163.6 72,338.6 13,058.6 sbt console- 0.0 536,660.4 76,163.6 72,338.6 13,058.6 import java.io._, argonaut._, Argonaut._- 0.0 552,389.1 76,163.6 72,338.6 13,058.6 val file = new File("/Users/jky/Downloads/78mbs.json"- 0.0 634,066.5 76,163.6 72,338.6 13,058.6 val array = new Array[Byte](file.length.asInstanceOf[Int])- 0.0 644,552.3 76,163.6 72,338.6 13,058.6 val is = new FileInputStream("/Users/jky/Downloads/78mbs.json")- 0.0 655,038.1 76,163.6 72,338.6 13,058.6 is.read(array)- 294,976.0 160,159.7 1,100,365.0 79,310.8 13,748.1 val json = new String(array)- 285,182.9 146,392.6 1,956,264.5 82,679.8 14,099.6 val data = Parse.parse(json)- ***********+```text+ S0U EU OU MU CCSU CMD+--------- --------- ----------- -------- -------- ---------------------------------------------------------------+ 0.0 80,526.3 76,163.6 72,338.6 13,058.6 sbt console+ 0.0 536,660.4 76,163.6 72,338.6 13,058.6 import java.io._, argonaut._, Argonaut._+ 0.0 552,389.1 76,163.6 72,338.6 13,058.6 val file = new File("/Users/jky/Downloads/78mbs.json"+ 0.0 634,066.5 76,163.6 72,338.6 13,058.6 val array = new Array[Byte](file.length.asInstanceOf[Int])+ 0.0 644,552.3 76,163.6 72,338.6 13,058.6 val is = new FileInputStream("/Users/jky/Downloads/78mbs.json")+ 0.0 655,038.1 76,163.6 72,338.6 13,058.6 is.read(array)+294,976.0 160,159.7 1,100,365.0 79,310.8 13,748.1 val json = new String(array)+285,182.9 146,392.6 1,956,264.5 82,679.8 14,099.6 val data = Parse.parse(json)+ ***********+``` ### Parsing large Json files in Haskell with Aeson - Mem (MB) CMD- -------- ---------------------------------------------------------- 54 import Control.DeepSeq- 64 import Data.Aeson- 64 import qualified Data.ByteString.Lazy as BSL- 66 json78m <- BSL.readFile "../data/78mb.json"- 167 let !x = deepseq json78m json78m- 1207 let !y = decode json78m :: Maybe Value-+```haskell+-- CMD -- Mem (MB)+---------------------------------------------------------- -- --------+import Control.DeepSeq -- 94+import Data.Aeson -- 100+import qualified Data.ByteString.Lazy as BSL -- 104+bs <- BSL.readFile "../corpus/bench/hospitalisation.json" -- 105+let !x = deepseq bs bs -- 146+let !y = decode json78m :: Maybe Value -- 669+``` ### Parsing large Json files in Haskell with hw-json - Mem (MB) CMD- -------- ---------------------------------------------------------- 57 import Foreign- 58 import Control.Monad- 59 import qualified Data.ByteString as BS- 76 import qualified Data.Vector.Storable as DVS- 77 import Data.Word- 77 import HaskellWorks.Data.BalancedParens.Simple- 77 import HaskellWorks.Data.Bits.BitShown- 77 import HaskellWorks.Data.FromForeignRegion- 81 import HaskellWorks.Data.Json.Succinct.Cursor- 82 import HaskellWorks.Diagnostics- 83 import System.IO.MMap- 88 (fptr :: ForeignPtr Word8, offset, size) <- mmapFileForeignPtr "../data/78mbs.json" ReadOnly Nothing- 276 cursor <- measure (fromForeignRegion (fptr, offset, size) :: JsonCursor BS.ByteString (DVS.Vector Word64) (SimpleBalancedParens (DVS.Vector Word64)))+```haskell+-- CMD -- Mem (MB)+--------------------------------------------------------------------- -- --------+import Foreign -- 93+import Control.Monad -- 95+import Data.Word -- 96+import HaskellWorks.Data.BalancedParens.Simple -- 97+import HaskellWorks.Data.Bits.BitShown -- 98+import HaskellWorks.Data.FromForeignRegion -- 99+import HaskellWorks.Data.Json.Backend.Standard.Cursor -- 106+import System.IO.MMap -- 109+import qualified Data.ByteString as BS -- 110+import qualified Data.Vector.Storable as DVS -- 111+import qualified HaskellWorks.Data.ByteString as BS -- 112+import qualified HaskellWorks.Data.Json.Backend.Standard.Fast as FAST -- 114+bs <- BS.mmap "../corpus/bench/hospitalisation.json" -- 115+let !cursor = FAST.makeCursor bs -- 203+``` ## Examples -### Performance example-(This section out of date)-- import Foreign- import qualified Data.Vector.Storable as DVS- import qualified Data.ByteString as BS- import qualified Data.ByteString.Internal as BSI- import System.IO.MMap- import Data.Word- import System.CPUTime- (fptr :: ForeignPtr Word8, offset, size) <- mmapFileForeignPtr "/Users/jky/Downloads/78mbs.json" ReadOnly Nothing- cursor <- measure (fromForeignRegion (fptr, offset, size) :: JsonCursor BS.ByteString (DVS.Vector Word64) (SimpleBalancedParens (DVS.Vector Word64)))- let !bs = BSI.fromForeignPtr (castForeignPtr fptr) offset size- x <- measure $ jsonBsToInterestBs bs- let !y = BS.concat (unescape' "" [bs])-- import Foreign- import qualified Data.Vector.Storable as DVS- import qualified Data.ByteString as BS- import qualified Data.ByteString.Internal as BSI- import System.IO.MMap- import Data.Word- import System.CPUTime- (fptr :: ForeignPtr Word8, offset, size) <- mmapFileForeignPtr "/Users/jky/Downloads/part40.json" ReadOnly Nothing- let !bs = BSI.fromForeignPtr (castForeignPtr fptr) offset size- x <- measure $ BS.concat $ blankedJsonToInterestBits $ blankJson [bs]- x <- measure $ jsonBsToInterestBs bs+### Navigation example - jsonTokenAt $ J.nextSibling $ J.firstChild $ J.nextSibling $ J.firstChild $ J.firstChild cursor+```haskell+import Control.Monad+import Data.String+import Data.Word+import HaskellWorks.Data.BalancedParens.Simple+import HaskellWorks.Data.Bits.BitShow+import HaskellWorks.Data.Bits.BitShown+import HaskellWorks.Data.FromForeignRegion+import HaskellWorks.Data.Json.Backend.Standard.Cursor+import HaskellWorks.Data.Json.Internal.Token.Types+import HaskellWorks.Data.RankSelect.Base.Rank0+import HaskellWorks.Data.RankSelect.Base.Rank1+import HaskellWorks.Data.RankSelect.Base.Select1+import HaskellWorks.Data.RankSelect.Poppy512+import System.IO.MMap -### Navigation example+import qualified Data.ByteString as BS+import qualified Data.Vector.Storable as DVS+import qualified HaskellWorks.Data.Json.Backend.Standard.Cursor as C+import qualified HaskellWorks.Data.Json.Backend.Standard.Fast as FAST+import qualified HaskellWorks.Data.TreeCursor as TC -```-import qualified Data.ByteString as BS-import Data.String-import qualified Data.Vector.Storable as DVS-import Data.Word-import HaskellWorks.Data.Bits.BitShow-import HaskellWorks.Data.Bits.BitShown-import HaskellWorks.Data.FromForeignRegion-import HaskellWorks.Data.Json.Succinct.Cursor as C-import HaskellWorks.Data.Json.Token-import HaskellWorks.Data.BalancedParens.Simple-import HaskellWorks.Data.RankSelect.Base.Rank0-import HaskellWorks.Data.RankSelect.Base.Rank1-import HaskellWorks.Data.RankSelect.Base.Select1-import HaskellWorks.Data.RankSelect.Poppy512-import qualified HaskellWorks.Data.TreeCursor as TC-import System.IO.MMap let fc = TC.firstChild let ns = TC.nextSibling let pn = TC.parent let ss = TC.subtreeSize-let cursor = "[null, {\"field\": 1}]" :: JsonCursor BS.ByteString (DVS.Vector Word64) (SimpleBalancedParens (DVS.Vector Word64))+let cursor = FAST.makeCursor "[null, {\"field\": 1}]" cursor fc cursor (fc >=> ns) cursor@@ -120,106 +101,57 @@ ### Querying example -```+```haskell import Control.Monad import Data.Function import Data.List-import HaskellWorks.Data.Json.Backend.Standard.Load-import HaskellWorks.Data.Json.LoadCursor+import HaskellWorks.Data.Json.Backend.Standard.Load.Cursor+import HaskellWorks.Data.Json.Backend.Standard.Load.Partial+import HaskellWorks.Data.Json.Backend.Standard.Load.Raw import HaskellWorks.Data.Json.PartialValue import HaskellWorks.Data.MQuery import HaskellWorks.Data.MQuery.Micro import HaskellWorks.Data.MQuery.Row-import HaskellWorks.Diagnostics import qualified Data.DList as DL-``` -```-!cursor <- loadJsonPartial "../data/78mb.json"-!cursor <- loadJsonWithIndex "../data/78mb.json"-!cursor <- loadJson "../data/78mb.json"-!cursor <- loadJsonWithCsPoppyIndex "../data/78mb.json"+!cursor <- loadPartial "../corpus/bench/78mb.json"+!cursor <- loadCursorWithIndex "../corpus/bench/78mb.json"+!cursor <- loadCursor "../corpus/bench/78mb.json"+!cursor <- loadCursorWithCsPoppyIndex "../corpus/bench/78mb.json" let !json = jsonPartialJsonValueAt cursor let q = MQuery (DL.singleton json)-``` -```-measureIO $ putPretty $ q >>= item & limit 10-measureIO $ putPretty $ q >>= item & page 10 1-measureIO $ putPretty $ q >>= item >>= hasKV "founded_year" (JsonPartialNumber 2005) & limit 10-measureIO $ putPretty $ q >>= item >>= entry-measureIO $ putPretty $ q >>= item >>= entry >>= named "name" & limit 10-measureIO $ putPretty $ q >>= (item >=> entry >=> named "acquisition" >=> entry >=> named "price_currency_code")-measureIO $ putPretty $ q >>= (item >=> entry >=> named "acquisition" >=> entry >=> named "price_currency_code") & onList (uniq . sort)-measureIO $ putPretty $ q >>= (item >=> entry >=> named "acquisition" >=> entry >=> named "price_currency_code" >=> asString >=> valueOf "USD") & limit 10-measureIO $ putPretty $ q >>= (item >=> entry >=> named "acquisition" >=> having (entry >=> named "price_currency_code" >=> asString >=> valueOf "USD") >=> entry >=> named "price_amount") & limit 10-measureIO $ putPretty $ q >>= (item >=> entry >=> named "acquisition" >=> having (entry >=> named "price_currency_code" >=> asString >=> valueOf "USD") >=> entry >=> named "price_amount" >=> castAsInteger ) & limit 10-measureIO $ putPretty $ q >>= (item >=> entry >=> named "acquisition" >=> having (entry >=> named "price_currency_code" >=> asString >=> valueOf "USD") >=> entry >=> named "price_amount" >=> castAsInteger ) & aggregate sum-```--```-import Control.Monad-import qualified Data.DList as DL-import Data.Function-import Data.List-import HaskellWorks.Data.Json.LoadCursor-import HaskellWorks.Data.Micro-import HaskellWorks.Data.MQuery-import HaskellWorks.Data.Json.LightJson-import HaskellWorks.Data.Row-import HaskellWorks.Diagnostics-```--```-!cursor <- loadJsonPartial "../data/78mb.json"-!cursor <- loadJsonWithIndex "../data/78mb.json"-!cursor <- loadJson "../data/78mb.json"-!cursor <- loadJsonWithCsPoppyIndex "../data/78mb.json"-let !json = lightJsonAt cursor-let q = MQuery (DL.singleton json)-```--```-measureIO $ putPretty $ q >>= item & limit 10-measureIO $ putPretty $ q >>= item & page 10 1-measureIO $ putPretty $ q >>= item >>= entry-measureIO $ putPretty $ q >>= item >>= entry >>= named "name" & limit 10-measureIO $ putPretty $ q >>= (item >=> entry >=> named "acquisition" >=> entry >=> named "price_currency_code" >=> asString)-measureIO $ putPretty $ q >>= (item >=> entry >=> named "acquisition" >=> entry >=> named "price_currency_code" >=> asString) & onList (uniq . sort)-measureIO $ putPretty $ q >>= (item >=> entry >=> named "acquisition" >=> entry >=> named "price_currency_code" >=> asString >=> valueOf "USD") & limit 10-measureIO $ putPretty $ q >>= (item >=> entry >=> named "acquisition" >=> having (entry >=> named "price_currency_code" >=> asString >=> valueOf "USD") >=> entry >=> named "price_amount") & limit 10-measureIO $ putPretty $ q >>= (item >=> entry >=> named "acquisition" >=> having (entry >=> named "price_currency_code" >=> asString >=> valueOf "USD") >=> entry >=> named "price_amount" >=> castAsInteger ) & limit 10-measureIO $ putPretty $ q >>= (item >=> entry >=> named "acquisition" >=> having (entry >=> named "price_currency_code" >=> asString >=> valueOf "USD") >=> entry >=> named "price_amount" >=> castAsInteger ) & aggregate sum-```--### Decoding-#### Line separated base 64 encoded gzipped json-while read in; do echo "$in" | base64 --decode | gunzip; echo ""; done < file.lgz > firehose.json--### Profiling with stack traces-```-mafia build -p-cabal repl --ghc-options='-fexternal-interpreter -prof'-```+putPretty $ q >>= item & limit 10+putPretty $ q >>= item & page 10 1+putPretty $ q >>= item >>= hasKV "founded_year" (JsonPartialNumber 2005) & limit 10+putPretty $ q >>= item >>= entry+putPretty $ q >>= item >>= entry >>= named "name" & limit 10+putPretty $ q >>= (item >=> entry >=> named "acquisition" >=> entry >=> named "price_currency_code")+putPretty $ q >>= (item >=> entry >=> named "acquisition" >=> entry >=> named "price_currency_code") & onList (uniq . sort)+putPretty $ q >>= (item >=> entry >=> named "acquisition" >=> entry >=> named "price_currency_code" >=> asString >=> valueOf "USD") & limit 10+putPretty $ q >>= (item >=> entry >=> named "acquisition" >=> having (entry >=> named "price_currency_code" >=> asString >=> valueOf "USD") >=> entry >=> named "price_amount") & limit 10+putPretty $ q >>= (item >=> entry >=> named "acquisition" >=> having (entry >=> named "price_currency_code" >=> asString >=> valueOf "USD") >=> entry >=> named "price_amount" >=> castAsInteger ) & limit 10+putPretty $ q >>= (item >=> entry >=> named "acquisition" >=> having (entry >=> named "price_currency_code" >=> asString >=> valueOf "USD") >=> entry >=> named "price_amount" >=> castAsInteger ) & aggregate sum -```-import HaskellWorks.Data.Succinct.BalancedParens-import HaskellWorks.Data.Succinct.RankSelect.Binary.Poppy512-import HaskellWorks.Data.Positioning-import qualified Data.Vector.Storable as DVS-import HaskellWorks.Data.IndexedSeq-(jsonBS, jsonIb, jsonBp) <- loadRawWithIndex "firehose.json"-let bp1 = SimpleBalancedParens jsonBp-let bp2 = SimpleBalancedParens (makePoppy512 jsonBp)-let bp3 = makePoppy512 jsonBp+putPretty $ q >>= item & limit 10+putPretty $ q >>= item & page 10 1+putPretty $ q >>= item >>= entry+putPretty $ q >>= item >>= entry >>= named "name" & limit 10+putPretty $ q >>= (item >=> entry >=> named "acquisition" >=> entry >=> named "price_currency_code" >=> asString)+putPretty $ q >>= (item >=> entry >=> named "acquisition" >=> entry >=> named "price_currency_code" >=> asString) & onList (uniq . sort)+putPretty $ q >>= (item >=> entry >=> named "acquisition" >=> entry >=> named "price_currency_code" >=> asString >=> valueOf "USD") & limit 10+putPretty $ q >>= (item >=> entry >=> named "acquisition" >=> having (entry >=> named "price_currency_code" >=> asString >=> valueOf "USD") >=> entry >=> named "price_amount") & limit 10+putPretty $ q >>= (item >=> entry >=> named "acquisition" >=> having (entry >=> named "price_currency_code" >=> asString >=> valueOf "USD") >=> entry >=> named "price_amount" >=> castAsInteger ) & limit 10+putPretty $ q >>= (item >=> entry >=> named "acquisition" >=> having (entry >=> named "price_currency_code" >=> asString >=> valueOf "USD") >=> entry >=> named "price_amount" >=> castAsInteger ) & aggregate sum ``` ## References+ * [Semi-Indexing Semi-Structured Data in Tiny Space](http://www.di.unipi.it/~ottavian/files/semi_index_cikm.pdf) * [Succinct Data Structures talk by Edward Kmett](https://www.youtube.com/watch?v=uA0Z7_4J7u8) * [Typed Tagless Final Interpreters](http://okmij.org/ftp/tagless-final/course/lecture.pdf) - ## Special mentions+ * [Sydney Paper Club](http://www.meetup.com/Sydney-Paper-Club/)
− corpus/5000B.bp
@@ -1,1 +0,0 @@-111011010010101010101010101010101010101010101010101010101010101010101010101010101010110100101010101011011110100100111010010011101001000010111010101001101010100010111010101010110101010101000110101010101101010101010001101010101011010101010100011010101010110101010101000110101010101101010101010001101010101011010101010100011010101010110101010101000110101010101101010101010001101010101011010101010100011010101010110101010101000110101010101101010101010001101010101011010101010100011010101010110101010101000110101010101101010101010001101010101011010101010100011010101010110101010101000110101010101101010101010000101110110101010001101101010100011011010101000110110101010001101101010100011011010101000000
− corpus/5000B.ib
@@ -1,1 +0,0 @@-10101000000010100000000100000000000000000000000000000100000000100000000000100000000000001000000010000000000000000001000000000000000000000000000000000000000000000100000000000000001000000000000000000000000001000000000000100000000000000000000000000000010000000000000000010000000000000000000000000000000000010000000000000000000010000000000000000001000000000000000001000000100000000000000000000000100010000000000000000100000100000000000000000100010000000000000001000100000000000000000001001000000000000100000000000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000001000100000000000000000100000000000000000000100000000000000001000000000000000100000000000000010000000000000000000000000000001000000000000001010000000001000000000000000010000000000000010000000000000000000000000000000100000000000010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000001010000000000000000000101010100001000001000000000000000000000000000000000000000000000000000000000000101010000100000010000000000000000000000000000000000000000000000000000000000001010100001000000100000000000000000000000000000000000000000000000000000000000000001000000000000101010000000010000000000000000000100000000000001000000000000000000101000000001000000000000000000000000000000000000001000000000000010000000000000000000000000000000000000000001000000000000000001010100000000000100000010000000001000000000000000000000000000000000000000000000000000001000000000010100000000000000100000000001000000000000010000000001000000000000010000000000000000000001010000000000010000001000000000100000000000000000000000000000000000010000000000101000000000000001000000100000000000001000000000010000000000000100000000000000000010100000000000100000010000000001000000000000000000000000010000000000101000000000000001000000100000000000001000000001000000000000010000000000000000101000000000001000000100000000010000000000000000000000000000000000000000010000000000101000000000000001000000001000000000000010000000001000000000000010000000000000000000101000000000001000000100000000010000000010000000000101000000000000001000000000001000000000000010000000000010000000000000100000000000000000000000010100000000000100000010000000001000000000000000100000000001010000000000000010000001000000000000010000001000000000000010000000000000010100000000000100000010000000001000000001000000000010100000000000000100000010000000000000100000000010000000000000100000000000000000101000000000001000000100000000010000000000000000000000000000000001000000000010100000000000000100000001000000000000010000000000001000000000000010000000000000000000001010000000000010000010000000001000000100000000001010000000000000010000000100000000000001000000001000000000000010000000000000000010100000000000100000100000000010000000000010000000000101000000000000001000000001000000000000010000000000100000000000001000000000000000000001010000000000010000010000000001000000000000000000000000000000000001000000000010100000000000000100000010000000000000100000000100000000000001000000000000000010100000000000100000100000000010000000000000000000000000000010000000000101000000000000001000000000100000000000001000000001000000000000010000000000000000000101000000000001000001000000000100000000000000010000000000101000000000000001000000001000000000000010000000000010000000000000100000000000000000000010100000000000100000100000000010000000000000000000001000000000010100000000000000100000001000000000000010000000100000000000001000000000000000010100000000000100000100000000010000000000000000010000000000101000000000000001000000001000000000000010000000000100000000000001000000000000000000001010000000000010000010000000001000000000000000000010000000000101000000000000001000000001000000000000010000000000100000000000001000000000000000000001010000000000010000010000000001000000000000000000000000001000000000010100000000000000100000000100000000000001000000000100000000000001000000000000000000000100000000000000001010100000000000000101000000001000000001000000000000010000000000001010000000000000010100000000100000000001000000000000010000000000000010100000000000000101000000001000000000000010000000000000100000000000000000101000000000000001010000000010000000000000000000001000000000000010000000000010100000000000000101000000001000000000100000000000001000000000000010100000000000000101000001000010000100000
+ corpus/5000B.json.bp.idx view
@@ -0,0 +1,1 @@+111011010010101010101010101010101010101010101010101010101010101010101010101010101010110100101010101011011110100100111010010011101001000010111010101001101010100010111010101010110101010101000110101010101101010101010001101010101011010101010100011010101010110101010101000110101010101101010101010001101010101011010101010100011010101010110101010101000110101010101101010101010001101010101011010101010100011010101010110101010101000110101010101101010101010001101010101011010101010100011010101010110101010101000110101010101101010101010001101010101011010101010100011010101010110101010101000110101010101101010101010000101110110101010001101101010100011011010101000110110101010001101101010100011011010101000000
+ corpus/5000B.json.ib.idx view
@@ -0,0 +1,1 @@+10101000000010100000000100000000000000000000000000000100000000100000000000100000000000001000000010000000000000000001000000000000000000000000000000000000000000000100000000000000001000000000000000000000000001000000000000100000000000000000000000000000010000000000000000010000000000000000000000000000000000010000000000000000000010000000000000000001000000000000000001000000100000000000000000000000100010000000000000000100000100000000000000000100010000000000000001000100000000000000000001001000000000000100000000000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000001000100000000000000000100000000000000000000100000000000000001000000000000000100000000000000010000000000000000000000000000001000000000000001010000000001000000000000000010000000000000010000000000000000000000000000000100000000000010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000001010000000000000000000101010100001000001000000000000000000000000000000000000000000000000000000000000101010000100000010000000000000000000000000000000000000000000000000000000000001010100001000000100000000000000000000000000000000000000000000000000000000000000001000000000000101010000000010000000000000000000100000000000001000000000000000000101000000001000000000000000000000000000000000000001000000000000010000000000000000000000000000000000000000001000000000000000001010100000000000100000010000000001000000000000000000000000000000000000000000000000000001000000000010100000000000000100000000001000000000000010000000001000000000000010000000000000000000001010000000000010000001000000000100000000000000000000000000000000000010000000000101000000000000001000000100000000000001000000000010000000000000100000000000000000010100000000000100000010000000001000000000000000000000000010000000000101000000000000001000000100000000000001000000001000000000000010000000000000000101000000000001000000100000000010000000000000000000000000000000000000000010000000000101000000000000001000000001000000000000010000000001000000000000010000000000000000000101000000000001000000100000000010000000010000000000101000000000000001000000000001000000000000010000000000010000000000000100000000000000000000000010100000000000100000010000000001000000000000000100000000001010000000000000010000001000000000000010000001000000000000010000000000000010100000000000100000010000000001000000001000000000010100000000000000100000010000000000000100000000010000000000000100000000000000000101000000000001000000100000000010000000000000000000000000000000001000000000010100000000000000100000001000000000000010000000000001000000000000010000000000000000000001010000000000010000010000000001000000100000000001010000000000000010000000100000000000001000000001000000000000010000000000000000010100000000000100000100000000010000000000010000000000101000000000000001000000001000000000000010000000000100000000000001000000000000000000001010000000000010000010000000001000000000000000000000000000000000001000000000010100000000000000100000010000000000000100000000100000000000001000000000000000010100000000000100000100000000010000000000000000000000000000010000000000101000000000000001000000000100000000000001000000001000000000000010000000000000000000101000000000001000001000000000100000000000000010000000000101000000000000001000000001000000000000010000000000010000000000000100000000000000000000010100000000000100000100000000010000000000000000000001000000000010100000000000000100000001000000000000010000000100000000000001000000000000000010100000000000100000100000000010000000000000000010000000000101000000000000001000000001000000000000010000000000100000000000001000000000000000000001010000000000010000010000000001000000000000000000010000000000101000000000000001000000001000000000000010000000000100000000000001000000000000000000001010000000000010000010000000001000000000000000000000000001000000000010100000000000000100000000100000000000001000000000100000000000001000000000000000000000100000000000000001010100000000000000101000000001000000001000000000000010000000000001010000000000000010100000000100000000001000000000000010000000000000010100000000000000101000000001000000000000010000000000000100000000000000000101000000000000001010000000010000000000000000000001000000000000010000000000010100000000000000101000000001000000000100000000000001000000000000010100000000000000101000001000010000100000
− corpus/issue-0001.bp
@@ -1,1 +0,0 @@-111010101010101011010101010101010101010101010101010110101010001010101110101010101010101010101010101010101001101010101010101010101010101010101010001011101010101010101010101010101010101010101010101010101101010100011010101010101010101010101010101010101010101010101011010101000010101011101111010010011101001001110100100010100010111010101001101010100110101010011010101001101010100110101010011010101001101010100110101010000100
− corpus/issue-0001.ib
@@ -1,1 +0,0 @@-1010100000000000000000000001000000000100000000000000000010001000000000000000100010000000000000001010000000000000000100000000010000000000000000000000010000001000000000000010000000000000000010000000000000010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000000100000000000000000000000000000000000000000000000000000000000000000000010000000000000000010000010000000000000000001000100000000000000001000100000000000000000000010100000000100000000010000000000000100000000000001000000000000000010001000000000001010100000000000000010001000000000000100000000000000000001000000000000100000000000001000000000000100000000100000000100000000001000000000000001000001000000000000000010000001000000000000100000000001000000000000010000000000000010100000000000000010001000000000000100000000000000000000000100000000000010000000000001000000000000100000000100000000100000000000100000000000000100000100000000000000001000000100000000000010000000000010000000000000100000000000000001000000000000001010100000010000010000000000000001000000000000000000000000000000000000000000000000000000000000010000000000000001000001000000000000000010010000000000000010010000000000000010000000000000000000000000000000000000000000000001000000000000000100000100000000000000000000001000000000000000000000000000000000000000000000000000000001000000000000000000100000000001000000000000000010000010000000000000000000001000001000000000000000000010000010000000000000101000000001000000000001000000000000010000000000000001010000001000001000000000000000100000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000001000001000000000000000010010000000000000010010000000000000010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000001000001000000000000000000000010000000000000001000000000000000000100000000001000000000000000010000010000000000000000000001000001000000000000000000010000010000000000000101000000001000000000001000000000000010000000000000000010000000000000000100010000000000000001010100000000000000000001010101000010000010000000000000000000000000000000000000000000000000000000000000101010000100000010000000000000000000000000000000000000000000000000000000000000101010000100000010000000000000000000000000000000000000000000000000000000000000001000000000000000100000000010000000000000000001010100000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010100000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000000001000000000100000000000000000000000000000000000000000000000010100000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001000000000100000000000000000000000000000000000000000000000010100000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000001000000000000000000000000000000000000000000010100000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001000000000100000000000000000000000000000000000000000000000000000010100000000000000001000000000000000000000000000000000000000000000000000010000000001000000000000000000000000000000000000010100000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000001000000000000000000000000000000000010100000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100000000010000000000000000000000000000000000000000000000000000000000001010000000000000000100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000001000000000000000000000000000000000000001000000000000000000000000000000
+ corpus/issue-0001.json.bp.idx view
@@ -0,0 +1,1 @@+111010101010101011010101010101010101010101010101010110101010001010101110101010101010101010101010101010101001101010101010101010101010101010101010001011101010101010101010101010101010101010101010101010101101010100011010101010101010101010101010101010101010101010101011010101000010101011101111010010011101001001110100100010100010111010101001101010100110101010011010101001101010100110101010011010101001101010100110101010000100
+ corpus/issue-0001.json.ib.idx view
@@ -0,0 +1,1 @@+1010100000000000000000000001000000000100000000000000000010001000000000000000100010000000000000001010000000000000000100000000010000000000000000000000010000001000000000000010000000000000000010000000000000010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000000100000000000000000000000000000000000000000000000000000000000000000000010000000000000000010000010000000000000000001000100000000000000001000100000000000000000000010100000000100000000010000000000000100000000000001000000000000000010001000000000001010100000000000000010001000000000000100000000000000000001000000000000100000000000001000000000000100000000100000000100000000001000000000000001000001000000000000000010000001000000000000100000000001000000000000010000000000000010100000000000000010001000000000000100000000000000000000000100000000000010000000000001000000000000100000000100000000100000000000100000000000000100000100000000000000001000000100000000000010000000000010000000000000100000000000000001000000000000001010100000010000010000000000000001000000000000000000000000000000000000000000000000000000000000010000000000000001000001000000000000000010010000000000000010010000000000000010000000000000000000000000000000000000000000000001000000000000000100000100000000000000000000001000000000000000000000000000000000000000000000000000000001000000000000000000100000000001000000000000000010000010000000000000000000001000001000000000000000000010000010000000000000101000000001000000000001000000000000010000000000000001010000001000001000000000000000100000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000001000001000000000000000010010000000000000010010000000000000010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000001000001000000000000000000000010000000000000001000000000000000000100000000001000000000000000010000010000000000000000000001000001000000000000000000010000010000000000000101000000001000000000001000000000000010000000000000000010000000000000000100010000000000000001010100000000000000000001010101000010000010000000000000000000000000000000000000000000000000000000000000101010000100000010000000000000000000000000000000000000000000000000000000000000101010000100000010000000000000000000000000000000000000000000000000000000000000001000000000000000100000000010000000000000000001010100000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010100000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000000001000000000100000000000000000000000000000000000000000000000010100000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001000000000100000000000000000000000000000000000000000000000010100000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000001000000000000000000000000000000000000000000010100000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001000000000100000000000000000000000000000000000000000000000000000010100000000000000001000000000000000000000000000000000000000000000000000010000000001000000000000000000000000000000000000010100000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000001000000000000000000000000000000000010100000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100000000010000000000000000000000000000000000000000000000000000000000001010000000000000000100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000001000000000000000000000000000000000000001000000000000000000000000000000
hw-json.cabal view
@@ -1,7 +1,7 @@ cabal-version: 2.2 name: hw-json-version: 1.0.0.0+version: 1.0.0.1 synopsis: Memory efficient JSON parser description: Memory efficient JSON parser. Please see README.md category: Data@@ -15,12 +15,12 @@ build-type: Simple extra-source-files: README.md- corpus/5000B.bp- corpus/5000B.ib corpus/5000B.json- corpus/issue-0001.bp- corpus/issue-0001.ib+ corpus/5000B.json.bp.idx+ corpus/5000B.json.ib.idx corpus/issue-0001.json+ corpus/issue-0001.json.bp.idx+ corpus/issue-0001.json.ib.idx corpus/issue-0001.md source-repository head
src/HaskellWorks/Data/Json/Backend/Standard/Cursor.hs view
@@ -171,5 +171,3 @@ instance (BP.BalancedParens w, Rank0 w, Rank1 w, Select1 v, TestBit w) => JsonPartialValueAt (JsonCursor BS.ByteString v w) where jsonPartialJsonValueAt = jsonPartialJsonValueAt . jsonPartialIndexAt--
src/HaskellWorks/Data/Json/Backend/Standard/Index.hs view
@@ -16,5 +16,5 @@ JsonCursor _ ib (SimpleBalancedParens bp) _ <- L.loadCursor filename let wib = DVS.unsafeCast ib :: DVS.Vector Word8 let wbp = DVS.unsafeCast bp :: DVS.Vector Word8- BS.writeFile (filename ++ ".ib") (BS.toByteString wib)- BS.writeFile (filename ++ ".bp") (BS.toByteString wbp)+ BS.writeFile (filename ++ ".ib.idx") (BS.toByteString wib)+ BS.writeFile (filename ++ ".bp.idx") (BS.toByteString wbp)
src/HaskellWorks/Data/Json/Backend/Standard/Load/Raw.hs view
@@ -12,8 +12,8 @@ loadRawWithIndex :: String -> IO (BS.ByteString, DVS.Vector Word64, DVS.Vector Word64) loadRawWithIndex filename = do jsonFr <- mmapFileForeignPtr filename ReadOnly Nothing- jsonIbFr <- mmapFileForeignPtr (filename ++ ".ib") ReadOnly Nothing- jsonBpFr <- mmapFileForeignPtr (filename ++ ".bp") ReadOnly Nothing+ jsonIbFr <- mmapFileForeignPtr (filename ++ ".ib.idx") ReadOnly Nothing+ jsonBpFr <- mmapFileForeignPtr (filename ++ ".bp.idx") ReadOnly Nothing let jsonBS = fromForeignRegion jsonFr :: BS.ByteString let jsonIb = fromForeignRegion jsonIbFr :: DVS.Vector Word64 let jsonBp = fromForeignRegion jsonBpFr :: DVS.Vector Word64
test/HaskellWorks/Data/Json/CorpusSpec.hs view
@@ -31,8 +31,8 @@ spec = describe "HaskellWorks.Data.Json.Corpus" $ do it "Corpus 5000B loads properly" $ requireTest $ do inJsonBS <- liftIO $ BS.readFile "corpus/5000B.json"- inInterestBitsBS <- liftIO $ BS.readFile "corpus/5000B.ib"- inInterestBalancedParensBS <- liftIO $ BS.readFile "corpus/5000B.bp"+ inInterestBitsBS <- liftIO $ BS.readFile "corpus/5000B.json.ib.idx"+ inInterestBalancedParensBS <- liftIO $ BS.readFile "corpus/5000B.json.bp.idx" let inInterestBits = fromBitTextByteString inInterestBitsBS let inInterestBalancedParens = fromBitTextByteString inInterestBalancedParensBS let !cursor = SLOW.makeCursor inJsonBS@@ -44,8 +44,8 @@ bp === inInterestBalancedParens it "issue-0001 loads properly" $ requireTest $ do inJsonBS <- liftIO $ BS.readFile "corpus/issue-0001.json"- inInterestBitsBS <- liftIO $ BS.readFile "corpus/issue-0001.ib"- inInterestBalancedParensBS <- liftIO $ BS.readFile "corpus/issue-0001.bp"+ inInterestBitsBS <- liftIO $ BS.readFile "corpus/issue-0001.json.ib.idx"+ inInterestBalancedParensBS <- liftIO $ BS.readFile "corpus/issue-0001.json.bp.idx" let inInterestBits = fromBitTextByteString inInterestBitsBS let inInterestBalancedParens = fromBitTextByteString inInterestBalancedParensBS let !cursor = SLOW.makeCursor inJsonBS