diff --git a/README.md b/README.md --- a/README.md +++ b/README.md @@ -2,117 +2,98 @@ [![master](https://circleci.com/gh/haskell-works/hw-json/tree/master.svg?style=svg)](https://circleci.com/gh/haskell-works/hw-json/tree/master) `hw-json` is a succinct JSON parsing library. -It uses succinct data-structures to allow traversal of -large JSON strings with minimal memory overhead. +It uses succinct data-structures to allow traversal of large JSON strings with minimal memory overhead. + For an example, see [`app/Main.hs`](../master/app/Main.hs) ## Prerequisites -* Install `haskell-stack`. -* Install `hlint` (eg. `stack install hlint`) +* `cabal` version `2.2` or later + ## Memory benchmark ### Parsing large Json files in Scala with Argonaut - S0U EU OU MU CCSU CMD - --------- --------- ----------- -------- -------- --------------------------------------------------------------- - 0.0 80,526.3 76,163.6 72,338.6 13,058.6 sbt console - 0.0 536,660.4 76,163.6 72,338.6 13,058.6 import java.io._, argonaut._, Argonaut._ - 0.0 552,389.1 76,163.6 72,338.6 13,058.6 val file = new File("/Users/jky/Downloads/78mbs.json" - 0.0 634,066.5 76,163.6 72,338.6 13,058.6 val array = new Array[Byte](file.length.asInstanceOf[Int]) - 0.0 644,552.3 76,163.6 72,338.6 13,058.6 val is = new FileInputStream("/Users/jky/Downloads/78mbs.json") - 0.0 655,038.1 76,163.6 72,338.6 13,058.6 is.read(array) - 294,976.0 160,159.7 1,100,365.0 79,310.8 13,748.1 val json = new String(array) - 285,182.9 146,392.6 1,956,264.5 82,679.8 14,099.6 val data = Parse.parse(json) - *********** +```text + S0U EU OU MU CCSU CMD +--------- --------- ----------- -------- -------- --------------------------------------------------------------- + 0.0 80,526.3 76,163.6 72,338.6 13,058.6 sbt console + 0.0 536,660.4 76,163.6 72,338.6 13,058.6 import java.io._, argonaut._, Argonaut._ + 0.0 552,389.1 76,163.6 72,338.6 13,058.6 val file = new File("/Users/jky/Downloads/78mbs.json" + 0.0 634,066.5 76,163.6 72,338.6 13,058.6 val array = new Array[Byte](file.length.asInstanceOf[Int]) + 0.0 644,552.3 76,163.6 72,338.6 13,058.6 val is = new FileInputStream("/Users/jky/Downloads/78mbs.json") + 0.0 655,038.1 76,163.6 72,338.6 13,058.6 is.read(array) +294,976.0 160,159.7 1,100,365.0 79,310.8 13,748.1 val json = new String(array) +285,182.9 146,392.6 1,956,264.5 82,679.8 14,099.6 val data = Parse.parse(json) + *********** +``` ### Parsing large Json files in Haskell with Aeson - Mem (MB) CMD - -------- --------------------------------------------------------- - 54 import Control.DeepSeq - 64 import Data.Aeson - 64 import qualified Data.ByteString.Lazy as BSL - 66 json78m <- BSL.readFile "../data/78mb.json" - 167 let !x = deepseq json78m json78m - 1207 let !y = decode json78m :: Maybe Value - +```haskell +-- CMD -- Mem (MB) +---------------------------------------------------------- -- -------- +import Control.DeepSeq -- 94 +import Data.Aeson -- 100 +import qualified Data.ByteString.Lazy as BSL -- 104 +bs <- BSL.readFile "../corpus/bench/hospitalisation.json" -- 105 +let !x = deepseq bs bs -- 146 +let !y = decode json78m :: Maybe Value -- 669 +``` ### Parsing large Json files in Haskell with hw-json - Mem (MB) CMD - -------- --------------------------------------------------------- - 57 import Foreign - 58 import Control.Monad - 59 import qualified Data.ByteString as BS - 76 import qualified Data.Vector.Storable as DVS - 77 import Data.Word - 77 import HaskellWorks.Data.BalancedParens.Simple - 77 import HaskellWorks.Data.Bits.BitShown - 77 import HaskellWorks.Data.FromForeignRegion - 81 import HaskellWorks.Data.Json.Succinct.Cursor - 82 import HaskellWorks.Diagnostics - 83 import System.IO.MMap - 88 (fptr :: ForeignPtr Word8, offset, size) <- mmapFileForeignPtr "../data/78mbs.json" ReadOnly Nothing - 276 cursor <- measure (fromForeignRegion (fptr, offset, size) :: JsonCursor BS.ByteString (DVS.Vector Word64) (SimpleBalancedParens (DVS.Vector Word64))) +```haskell +-- CMD -- Mem (MB) +--------------------------------------------------------------------- -- -------- +import Foreign -- 93 +import Control.Monad -- 95 +import Data.Word -- 96 +import HaskellWorks.Data.BalancedParens.Simple -- 97 +import HaskellWorks.Data.Bits.BitShown -- 98 +import HaskellWorks.Data.FromForeignRegion -- 99 +import HaskellWorks.Data.Json.Backend.Standard.Cursor -- 106 +import System.IO.MMap -- 109 +import qualified Data.ByteString as BS -- 110 +import qualified Data.Vector.Storable as DVS -- 111 +import qualified HaskellWorks.Data.ByteString as BS -- 112 +import qualified HaskellWorks.Data.Json.Backend.Standard.Fast as FAST -- 114 +bs <- BS.mmap "../corpus/bench/hospitalisation.json" -- 115 +let !cursor = FAST.makeCursor bs -- 203 +``` ## Examples -### Performance example -(This section out of date) - - import Foreign - import qualified Data.Vector.Storable as DVS - import qualified Data.ByteString as BS - import qualified Data.ByteString.Internal as BSI - import System.IO.MMap - import Data.Word - import System.CPUTime - (fptr :: ForeignPtr Word8, offset, size) <- mmapFileForeignPtr "/Users/jky/Downloads/78mbs.json" ReadOnly Nothing - cursor <- measure (fromForeignRegion (fptr, offset, size) :: JsonCursor BS.ByteString (DVS.Vector Word64) (SimpleBalancedParens (DVS.Vector Word64))) - let !bs = BSI.fromForeignPtr (castForeignPtr fptr) offset size - x <- measure $ jsonBsToInterestBs bs - let !y = BS.concat (unescape' "" [bs]) - - import Foreign - import qualified Data.Vector.Storable as DVS - import qualified Data.ByteString as BS - import qualified Data.ByteString.Internal as BSI - import System.IO.MMap - import Data.Word - import System.CPUTime - (fptr :: ForeignPtr Word8, offset, size) <- mmapFileForeignPtr "/Users/jky/Downloads/part40.json" ReadOnly Nothing - let !bs = BSI.fromForeignPtr (castForeignPtr fptr) offset size - x <- measure $ BS.concat $ blankedJsonToInterestBits $ blankJson [bs] - x <- measure $ jsonBsToInterestBs bs +### Navigation example - jsonTokenAt $ J.nextSibling $ J.firstChild $ J.nextSibling $ J.firstChild $ J.firstChild cursor +```haskell +import Control.Monad +import Data.String +import Data.Word +import HaskellWorks.Data.BalancedParens.Simple +import HaskellWorks.Data.Bits.BitShow +import HaskellWorks.Data.Bits.BitShown +import HaskellWorks.Data.FromForeignRegion +import HaskellWorks.Data.Json.Backend.Standard.Cursor +import HaskellWorks.Data.Json.Internal.Token.Types +import HaskellWorks.Data.RankSelect.Base.Rank0 +import HaskellWorks.Data.RankSelect.Base.Rank1 +import HaskellWorks.Data.RankSelect.Base.Select1 +import HaskellWorks.Data.RankSelect.Poppy512 +import System.IO.MMap -### Navigation example +import qualified Data.ByteString as BS +import qualified Data.Vector.Storable as DVS +import qualified HaskellWorks.Data.Json.Backend.Standard.Cursor as C +import qualified HaskellWorks.Data.Json.Backend.Standard.Fast as FAST +import qualified HaskellWorks.Data.TreeCursor as TC -``` -import qualified Data.ByteString as BS -import Data.String -import qualified Data.Vector.Storable as DVS -import Data.Word -import HaskellWorks.Data.Bits.BitShow -import HaskellWorks.Data.Bits.BitShown -import HaskellWorks.Data.FromForeignRegion -import HaskellWorks.Data.Json.Succinct.Cursor as C -import HaskellWorks.Data.Json.Token -import HaskellWorks.Data.BalancedParens.Simple -import HaskellWorks.Data.RankSelect.Base.Rank0 -import HaskellWorks.Data.RankSelect.Base.Rank1 -import HaskellWorks.Data.RankSelect.Base.Select1 -import HaskellWorks.Data.RankSelect.Poppy512 -import qualified HaskellWorks.Data.TreeCursor as TC -import System.IO.MMap let fc = TC.firstChild let ns = TC.nextSibling let pn = TC.parent let ss = TC.subtreeSize -let cursor = "[null, {\"field\": 1}]" :: JsonCursor BS.ByteString (DVS.Vector Word64) (SimpleBalancedParens (DVS.Vector Word64)) +let cursor = FAST.makeCursor "[null, {\"field\": 1}]" cursor fc cursor (fc >=> ns) cursor @@ -120,106 +101,57 @@ ### Querying example -``` +```haskell import Control.Monad import Data.Function import Data.List -import HaskellWorks.Data.Json.Backend.Standard.Load -import HaskellWorks.Data.Json.LoadCursor +import HaskellWorks.Data.Json.Backend.Standard.Load.Cursor +import HaskellWorks.Data.Json.Backend.Standard.Load.Partial +import HaskellWorks.Data.Json.Backend.Standard.Load.Raw import HaskellWorks.Data.Json.PartialValue import HaskellWorks.Data.MQuery import HaskellWorks.Data.MQuery.Micro import HaskellWorks.Data.MQuery.Row -import HaskellWorks.Diagnostics import qualified Data.DList as DL -``` -``` -!cursor <- loadJsonPartial "../data/78mb.json" -!cursor <- loadJsonWithIndex "../data/78mb.json" -!cursor <- loadJson "../data/78mb.json" -!cursor <- loadJsonWithCsPoppyIndex "../data/78mb.json" +!cursor <- loadPartial "../corpus/bench/78mb.json" +!cursor <- loadCursorWithIndex "../corpus/bench/78mb.json" +!cursor <- loadCursor "../corpus/bench/78mb.json" +!cursor <- loadCursorWithCsPoppyIndex "../corpus/bench/78mb.json" let !json = jsonPartialJsonValueAt cursor let q = MQuery (DL.singleton json) -``` -``` -measureIO $ putPretty $ q >>= item & limit 10 -measureIO $ putPretty $ q >>= item & page 10 1 -measureIO $ putPretty $ q >>= item >>= hasKV "founded_year" (JsonPartialNumber 2005) & limit 10 -measureIO $ putPretty $ q >>= item >>= entry -measureIO $ putPretty $ q >>= item >>= entry >>= named "name" & limit 10 -measureIO $ putPretty $ q >>= (item >=> entry >=> named "acquisition" >=> entry >=> named "price_currency_code") -measureIO $ putPretty $ q >>= (item >=> entry >=> named "acquisition" >=> entry >=> named "price_currency_code") & onList (uniq . sort) -measureIO $ putPretty $ q >>= (item >=> entry >=> named "acquisition" >=> entry >=> named "price_currency_code" >=> asString >=> valueOf "USD") & limit 10 -measureIO $ putPretty $ q >>= (item >=> entry >=> named "acquisition" >=> having (entry >=> named "price_currency_code" >=> asString >=> valueOf "USD") >=> entry >=> named "price_amount") & limit 10 -measureIO $ putPretty $ q >>= (item >=> entry >=> named "acquisition" >=> having (entry >=> named "price_currency_code" >=> asString >=> valueOf "USD") >=> entry >=> named "price_amount" >=> castAsInteger ) & limit 10 -measureIO $ putPretty $ q >>= (item >=> entry >=> named "acquisition" >=> having (entry >=> named "price_currency_code" >=> asString >=> valueOf "USD") >=> entry >=> named "price_amount" >=> castAsInteger ) & aggregate sum -``` - -``` -import Control.Monad -import qualified Data.DList as DL -import Data.Function -import Data.List -import HaskellWorks.Data.Json.LoadCursor -import HaskellWorks.Data.Micro -import HaskellWorks.Data.MQuery -import HaskellWorks.Data.Json.LightJson -import HaskellWorks.Data.Row -import HaskellWorks.Diagnostics -``` - -``` -!cursor <- loadJsonPartial "../data/78mb.json" -!cursor <- loadJsonWithIndex "../data/78mb.json" -!cursor <- loadJson "../data/78mb.json" -!cursor <- loadJsonWithCsPoppyIndex "../data/78mb.json" -let !json = lightJsonAt cursor -let q = MQuery (DL.singleton json) -``` - -``` -measureIO $ putPretty $ q >>= item & limit 10 -measureIO $ putPretty $ q >>= item & page 10 1 -measureIO $ putPretty $ q >>= item >>= entry -measureIO $ putPretty $ q >>= item >>= entry >>= named "name" & limit 10 -measureIO $ putPretty $ q >>= (item >=> entry >=> named "acquisition" >=> entry >=> named "price_currency_code" >=> asString) -measureIO $ putPretty $ q >>= (item >=> entry >=> named "acquisition" >=> entry >=> named "price_currency_code" >=> asString) & onList (uniq . sort) -measureIO $ putPretty $ q >>= (item >=> entry >=> named "acquisition" >=> entry >=> named "price_currency_code" >=> asString >=> valueOf "USD") & limit 10 -measureIO $ putPretty $ q >>= (item >=> entry >=> named "acquisition" >=> having (entry >=> named "price_currency_code" >=> asString >=> valueOf "USD") >=> entry >=> named "price_amount") & limit 10 -measureIO $ putPretty $ q >>= (item >=> entry >=> named "acquisition" >=> having (entry >=> named "price_currency_code" >=> asString >=> valueOf "USD") >=> entry >=> named "price_amount" >=> castAsInteger ) & limit 10 -measureIO $ putPretty $ q >>= (item >=> entry >=> named "acquisition" >=> having (entry >=> named "price_currency_code" >=> asString >=> valueOf "USD") >=> entry >=> named "price_amount" >=> castAsInteger ) & aggregate sum -``` - -### Decoding -#### Line separated base 64 encoded gzipped json -while read in; do echo "$in" | base64 --decode | gunzip; echo ""; done < file.lgz > firehose.json - -### Profiling with stack traces -``` -mafia build -p -cabal repl --ghc-options='-fexternal-interpreter -prof' -``` +putPretty $ q >>= item & limit 10 +putPretty $ q >>= item & page 10 1 +putPretty $ q >>= item >>= hasKV "founded_year" (JsonPartialNumber 2005) & limit 10 +putPretty $ q >>= item >>= entry +putPretty $ q >>= item >>= entry >>= named "name" & limit 10 +putPretty $ q >>= (item >=> entry >=> named "acquisition" >=> entry >=> named "price_currency_code") +putPretty $ q >>= (item >=> entry >=> named "acquisition" >=> entry >=> named "price_currency_code") & onList (uniq . sort) +putPretty $ q >>= (item >=> entry >=> named "acquisition" >=> entry >=> named "price_currency_code" >=> asString >=> valueOf "USD") & limit 10 +putPretty $ q >>= (item >=> entry >=> named "acquisition" >=> having (entry >=> named "price_currency_code" >=> asString >=> valueOf "USD") >=> entry >=> named "price_amount") & limit 10 +putPretty $ q >>= (item >=> entry >=> named "acquisition" >=> having (entry >=> named "price_currency_code" >=> asString >=> valueOf "USD") >=> entry >=> named "price_amount" >=> castAsInteger ) & limit 10 +putPretty $ q >>= (item >=> entry >=> named "acquisition" >=> having (entry >=> named "price_currency_code" >=> asString >=> valueOf "USD") >=> entry >=> named "price_amount" >=> castAsInteger ) & aggregate sum -``` -import HaskellWorks.Data.Succinct.BalancedParens -import HaskellWorks.Data.Succinct.RankSelect.Binary.Poppy512 -import HaskellWorks.Data.Positioning -import qualified Data.Vector.Storable as DVS -import HaskellWorks.Data.IndexedSeq -(jsonBS, jsonIb, jsonBp) <- loadRawWithIndex "firehose.json" -let bp1 = SimpleBalancedParens jsonBp -let bp2 = SimpleBalancedParens (makePoppy512 jsonBp) -let bp3 = makePoppy512 jsonBp +putPretty $ q >>= item & limit 10 +putPretty $ q >>= item & page 10 1 +putPretty $ q >>= item >>= entry +putPretty $ q >>= item >>= entry >>= named "name" & limit 10 +putPretty $ q >>= (item >=> entry >=> named "acquisition" >=> entry >=> named "price_currency_code" >=> asString) +putPretty $ q >>= (item >=> entry >=> named "acquisition" >=> entry >=> named "price_currency_code" >=> asString) & onList (uniq . sort) +putPretty $ q >>= (item >=> entry >=> named "acquisition" >=> entry >=> named "price_currency_code" >=> asString >=> valueOf "USD") & limit 10 +putPretty $ q >>= (item >=> entry >=> named "acquisition" >=> having (entry >=> named "price_currency_code" >=> asString >=> valueOf "USD") >=> entry >=> named "price_amount") & limit 10 +putPretty $ q >>= (item >=> entry >=> named "acquisition" >=> having (entry >=> named "price_currency_code" >=> asString >=> valueOf "USD") >=> entry >=> named "price_amount" >=> castAsInteger ) & limit 10 +putPretty $ q >>= (item >=> entry >=> named "acquisition" >=> having (entry >=> named "price_currency_code" >=> asString >=> valueOf "USD") >=> entry >=> named "price_amount" >=> castAsInteger ) & aggregate sum ``` ## References + * [Semi-Indexing Semi-Structured Data in Tiny Space](http://www.di.unipi.it/~ottavian/files/semi_index_cikm.pdf) * [Succinct Data Structures talk by Edward Kmett](https://www.youtube.com/watch?v=uA0Z7_4J7u8) * [Typed Tagless Final Interpreters](http://okmij.org/ftp/tagless-final/course/lecture.pdf) - ## Special mentions + * [Sydney Paper Club](http://www.meetup.com/Sydney-Paper-Club/) diff --git a/corpus/5000B.bp b/corpus/5000B.bp deleted file mode 100644 --- a/corpus/5000B.bp +++ /dev/null @@ -1,1 +0,0 @@ -111011010010101010101010101010101010101010101010101010101010101010101010101010101010110100101010101011011110100100111010010011101001000010111010101001101010100010111010101010110101010101000110101010101101010101010001101010101011010101010100011010101010110101010101000110101010101101010101010001101010101011010101010100011010101010110101010101000110101010101101010101010001101010101011010101010100011010101010110101010101000110101010101101010101010001101010101011010101010100011010101010110101010101000110101010101101010101010001101010101011010101010100011010101010110101010101000110101010101101010101010000101110110101010001101101010100011011010101000110110101010001101101010100011011010101000000 diff --git a/corpus/5000B.ib b/corpus/5000B.ib deleted file mode 100644 --- a/corpus/5000B.ib +++ /dev/null @@ -1,1 +0,0 @@ -10101000000010100000000100000000000000000000000000000100000000100000000000100000000000001000000010000000000000000001000000000000000000000000000000000000000000000100000000000000001000000000000000000000000001000000000000100000000000000000000000000000010000000000000000010000000000000000000000000000000000010000000000000000000010000000000000000001000000000000000001000000100000000000000000000000100010000000000000000100000100000000000000000100010000000000000001000100000000000000000001001000000000000100000000000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000001000100000000000000000100000000000000000000100000000000000001000000000000000100000000000000010000000000000000000000000000001000000000000001010000000001000000000000000010000000000000010000000000000000000000000000000100000000000010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000001010000000000000000000101010100001000001000000000000000000000000000000000000000000000000000000000000101010000100000010000000000000000000000000000000000000000000000000000000000001010100001000000100000000000000000000000000000000000000000000000000000000000000001000000000000101010000000010000000000000000000100000000000001000000000000000000101000000001000000000000000000000000000000000000001000000000000010000000000000000000000000000000000000000001000000000000000001010100000000000100000010000000001000000000000000000000000000000000000000000000000000001000000000010100000000000000100000000001000000000000010000000001000000000000010000000000000000000001010000000000010000001000000000100000000000000000000000000000000000010000000000101000000000000001000000100000000000001000000000010000000000000100000000000000000010100000000000100000010000000001000000000000000000000000010000000000101000000000000001000000100000000000001000000001000000000000010000000000000000101000000000001000000100000000010000000000000000000000000000000000000000010000000000101000000000000001000000001000000000000010000000001000000000000010000000000000000000101000000000001000000100000000010000000010000000000101000000000000001000000000001000000000000010000000000010000000000000100000000000000000000000010100000000000100000010000000001000000000000000100000000001010000000000000010000001000000000000010000001000000000000010000000000000010100000000000100000010000000001000000001000000000010100000000000000100000010000000000000100000000010000000000000100000000000000000101000000000001000000100000000010000000000000000000000000000000001000000000010100000000000000100000001000000000000010000000000001000000000000010000000000000000000001010000000000010000010000000001000000100000000001010000000000000010000000100000000000001000000001000000000000010000000000000000010100000000000100000100000000010000000000010000000000101000000000000001000000001000000000000010000000000100000000000001000000000000000000001010000000000010000010000000001000000000000000000000000000000000001000000000010100000000000000100000010000000000000100000000100000000000001000000000000000010100000000000100000100000000010000000000000000000000000000010000000000101000000000000001000000000100000000000001000000001000000000000010000000000000000000101000000000001000001000000000100000000000000010000000000101000000000000001000000001000000000000010000000000010000000000000100000000000000000000010100000000000100000100000000010000000000000000000001000000000010100000000000000100000001000000000000010000000100000000000001000000000000000010100000000000100000100000000010000000000000000010000000000101000000000000001000000001000000000000010000000000100000000000001000000000000000000001010000000000010000010000000001000000000000000000010000000000101000000000000001000000001000000000000010000000000100000000000001000000000000000000001010000000000010000010000000001000000000000000000000000001000000000010100000000000000100000000100000000000001000000000100000000000001000000000000000000000100000000000000001010100000000000000101000000001000000001000000000000010000000000001010000000000000010100000000100000000001000000000000010000000000000010100000000000000101000000001000000000000010000000000000100000000000000000101000000000000001010000000010000000000000000000001000000000000010000000000010100000000000000101000000001000000000100000000000001000000000000010100000000000000101000001000010000100000 diff --git a/corpus/5000B.json.bp.idx b/corpus/5000B.json.bp.idx new file mode 100644 --- /dev/null +++ b/corpus/5000B.json.bp.idx @@ -0,0 +1,1 @@ +111011010010101010101010101010101010101010101010101010101010101010101010101010101010110100101010101011011110100100111010010011101001000010111010101001101010100010111010101010110101010101000110101010101101010101010001101010101011010101010100011010101010110101010101000110101010101101010101010001101010101011010101010100011010101010110101010101000110101010101101010101010001101010101011010101010100011010101010110101010101000110101010101101010101010001101010101011010101010100011010101010110101010101000110101010101101010101010001101010101011010101010100011010101010110101010101000110101010101101010101010000101110110101010001101101010100011011010101000110110101010001101101010100011011010101000000 diff --git a/corpus/5000B.json.ib.idx b/corpus/5000B.json.ib.idx new file mode 100644 --- /dev/null +++ b/corpus/5000B.json.ib.idx @@ -0,0 +1,1 @@ +10101000000010100000000100000000000000000000000000000100000000100000000000100000000000001000000010000000000000000001000000000000000000000000000000000000000000000100000000000000001000000000000000000000000001000000000000100000000000000000000000000000010000000000000000010000000000000000000000000000000000010000000000000000000010000000000000000001000000000000000001000000100000000000000000000000100010000000000000000100000100000000000000000100010000000000000001000100000000000000000001001000000000000100000000000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000001000100000000000000000100000000000000000000100000000000000001000000000000000100000000000000010000000000000000000000000000001000000000000001010000000001000000000000000010000000000000010000000000000000000000000000000100000000000010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000001010000000000000000000101010100001000001000000000000000000000000000000000000000000000000000000000000101010000100000010000000000000000000000000000000000000000000000000000000000001010100001000000100000000000000000000000000000000000000000000000000000000000000001000000000000101010000000010000000000000000000100000000000001000000000000000000101000000001000000000000000000000000000000000000001000000000000010000000000000000000000000000000000000000001000000000000000001010100000000000100000010000000001000000000000000000000000000000000000000000000000000001000000000010100000000000000100000000001000000000000010000000001000000000000010000000000000000000001010000000000010000001000000000100000000000000000000000000000000000010000000000101000000000000001000000100000000000001000000000010000000000000100000000000000000010100000000000100000010000000001000000000000000000000000010000000000101000000000000001000000100000000000001000000001000000000000010000000000000000101000000000001000000100000000010000000000000000000000000000000000000000010000000000101000000000000001000000001000000000000010000000001000000000000010000000000000000000101000000000001000000100000000010000000010000000000101000000000000001000000000001000000000000010000000000010000000000000100000000000000000000000010100000000000100000010000000001000000000000000100000000001010000000000000010000001000000000000010000001000000000000010000000000000010100000000000100000010000000001000000001000000000010100000000000000100000010000000000000100000000010000000000000100000000000000000101000000000001000000100000000010000000000000000000000000000000001000000000010100000000000000100000001000000000000010000000000001000000000000010000000000000000000001010000000000010000010000000001000000100000000001010000000000000010000000100000000000001000000001000000000000010000000000000000010100000000000100000100000000010000000000010000000000101000000000000001000000001000000000000010000000000100000000000001000000000000000000001010000000000010000010000000001000000000000000000000000000000000001000000000010100000000000000100000010000000000000100000000100000000000001000000000000000010100000000000100000100000000010000000000000000000000000000010000000000101000000000000001000000000100000000000001000000001000000000000010000000000000000000101000000000001000001000000000100000000000000010000000000101000000000000001000000001000000000000010000000000010000000000000100000000000000000000010100000000000100000100000000010000000000000000000001000000000010100000000000000100000001000000000000010000000100000000000001000000000000000010100000000000100000100000000010000000000000000010000000000101000000000000001000000001000000000000010000000000100000000000001000000000000000000001010000000000010000010000000001000000000000000000010000000000101000000000000001000000001000000000000010000000000100000000000001000000000000000000001010000000000010000010000000001000000000000000000000000001000000000010100000000000000100000000100000000000001000000000100000000000001000000000000000000000100000000000000001010100000000000000101000000001000000001000000000000010000000000001010000000000000010100000000100000000001000000000000010000000000000010100000000000000101000000001000000000000010000000000000100000000000000000101000000000000001010000000010000000000000000000001000000000000010000000000010100000000000000101000000001000000000100000000000001000000000000010100000000000000101000001000010000100000 diff --git a/corpus/issue-0001.bp b/corpus/issue-0001.bp deleted file mode 100644 --- a/corpus/issue-0001.bp +++ /dev/null @@ -1,1 +0,0 @@ -111010101010101011010101010101010101010101010101010110101010001010101110101010101010101010101010101010101001101010101010101010101010101010101010001011101010101010101010101010101010101010101010101010101101010100011010101010101010101010101010101010101010101010101011010101000010101011101111010010011101001001110100100010100010111010101001101010100110101010011010101001101010100110101010011010101001101010100110101010000100 diff --git a/corpus/issue-0001.ib b/corpus/issue-0001.ib deleted file mode 100644 --- a/corpus/issue-0001.ib +++ /dev/null @@ -1,1 +0,0 @@ -1010100000000000000000000001000000000100000000000000000010001000000000000000100010000000000000001010000000000000000100000000010000000000000000000000010000001000000000000010000000000000000010000000000000010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000000100000000000000000000000000000000000000000000000000000000000000000000010000000000000000010000010000000000000000001000100000000000000001000100000000000000000000010100000000100000000010000000000000100000000000001000000000000000010001000000000001010100000000000000010001000000000000100000000000000000001000000000000100000000000001000000000000100000000100000000100000000001000000000000001000001000000000000000010000001000000000000100000000001000000000000010000000000000010100000000000000010001000000000000100000000000000000000000100000000000010000000000001000000000000100000000100000000100000000000100000000000000100000100000000000000001000000100000000000010000000000010000000000000100000000000000001000000000000001010100000010000010000000000000001000000000000000000000000000000000000000000000000000000000000010000000000000001000001000000000000000010010000000000000010010000000000000010000000000000000000000000000000000000000000000001000000000000000100000100000000000000000000001000000000000000000000000000000000000000000000000000000001000000000000000000100000000001000000000000000010000010000000000000000000001000001000000000000000000010000010000000000000101000000001000000000001000000000000010000000000000001010000001000001000000000000000100000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000001000001000000000000000010010000000000000010010000000000000010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000001000001000000000000000000000010000000000000001000000000000000000100000000001000000000000000010000010000000000000000000001000001000000000000000000010000010000000000000101000000001000000000001000000000000010000000000000000010000000000000000100010000000000000001010100000000000000000001010101000010000010000000000000000000000000000000000000000000000000000000000000101010000100000010000000000000000000000000000000000000000000000000000000000000101010000100000010000000000000000000000000000000000000000000000000000000000000001000000000000000100000000010000000000000000001010100000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010100000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000000001000000000100000000000000000000000000000000000000000000000010100000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001000000000100000000000000000000000000000000000000000000000010100000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000001000000000000000000000000000000000000000000010100000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001000000000100000000000000000000000000000000000000000000000000000010100000000000000001000000000000000000000000000000000000000000000000000010000000001000000000000000000000000000000000000010100000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000001000000000000000000000000000000000010100000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100000000010000000000000000000000000000000000000000000000000000000000001010000000000000000100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000001000000000000000000000000000000000000001000000000000000000000000000000 diff --git a/corpus/issue-0001.json.bp.idx b/corpus/issue-0001.json.bp.idx new file mode 100644 --- /dev/null +++ b/corpus/issue-0001.json.bp.idx @@ -0,0 +1,1 @@ +111010101010101011010101010101010101010101010101010110101010001010101110101010101010101010101010101010101001101010101010101010101010101010101010001011101010101010101010101010101010101010101010101010101101010100011010101010101010101010101010101010101010101010101011010101000010101011101111010010011101001001110100100010100010111010101001101010100110101010011010101001101010100110101010011010101001101010100110101010000100 diff --git a/corpus/issue-0001.json.ib.idx b/corpus/issue-0001.json.ib.idx new file mode 100644 --- /dev/null +++ b/corpus/issue-0001.json.ib.idx @@ -0,0 +1,1 @@ +1010100000000000000000000001000000000100000000000000000010001000000000000000100010000000000000001010000000000000000100000000010000000000000000000000010000001000000000000010000000000000000010000000000000010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000000100000000000000000000000000000000000000000000000000000000000000000000010000000000000000010000010000000000000000001000100000000000000001000100000000000000000000010100000000100000000010000000000000100000000000001000000000000000010001000000000001010100000000000000010001000000000000100000000000000000001000000000000100000000000001000000000000100000000100000000100000000001000000000000001000001000000000000000010000001000000000000100000000001000000000000010000000000000010100000000000000010001000000000000100000000000000000000000100000000000010000000000001000000000000100000000100000000100000000000100000000000000100000100000000000000001000000100000000000010000000000010000000000000100000000000000001000000000000001010100000010000010000000000000001000000000000000000000000000000000000000000000000000000000000010000000000000001000001000000000000000010010000000000000010010000000000000010000000000000000000000000000000000000000000000001000000000000000100000100000000000000000000001000000000000000000000000000000000000000000000000000000001000000000000000000100000000001000000000000000010000010000000000000000000001000001000000000000000000010000010000000000000101000000001000000000001000000000000010000000000000001010000001000001000000000000000100000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000001000001000000000000000010010000000000000010010000000000000010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000001000001000000000000000000000010000000000000001000000000000000000100000000001000000000000000010000010000000000000000000001000001000000000000000000010000010000000000000101000000001000000000001000000000000010000000000000000010000000000000000100010000000000000001010100000000000000000001010101000010000010000000000000000000000000000000000000000000000000000000000000101010000100000010000000000000000000000000000000000000000000000000000000000000101010000100000010000000000000000000000000000000000000000000000000000000000000001000000000000000100000000010000000000000000001010100000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010100000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000000001000000000100000000000000000000000000000000000000000000000010100000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001000000000100000000000000000000000000000000000000000000000010100000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000001000000000000000000000000000000000000000000010100000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001000000000100000000000000000000000000000000000000000000000000000010100000000000000001000000000000000000000000000000000000000000000000000010000000001000000000000000000000000000000000000010100000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000001000000000000000000000000000000000010100000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100000000010000000000000000000000000000000000000000000000000000000000001010000000000000000100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000001000000000000000000000000000000000000001000000000000000000000000000000 diff --git a/hw-json.cabal b/hw-json.cabal --- a/hw-json.cabal +++ b/hw-json.cabal @@ -1,7 +1,7 @@ cabal-version: 2.2 name: hw-json -version: 1.0.0.0 +version: 1.0.0.1 synopsis: Memory efficient JSON parser description: Memory efficient JSON parser. Please see README.md category: Data @@ -15,12 +15,12 @@ build-type: Simple extra-source-files: README.md - corpus/5000B.bp - corpus/5000B.ib corpus/5000B.json - corpus/issue-0001.bp - corpus/issue-0001.ib + corpus/5000B.json.bp.idx + corpus/5000B.json.ib.idx corpus/issue-0001.json + corpus/issue-0001.json.bp.idx + corpus/issue-0001.json.ib.idx corpus/issue-0001.md source-repository head diff --git a/src/HaskellWorks/Data/Json/Backend/Standard/Cursor.hs b/src/HaskellWorks/Data/Json/Backend/Standard/Cursor.hs --- a/src/HaskellWorks/Data/Json/Backend/Standard/Cursor.hs +++ b/src/HaskellWorks/Data/Json/Backend/Standard/Cursor.hs @@ -171,5 +171,3 @@ instance (BP.BalancedParens w, Rank0 w, Rank1 w, Select1 v, TestBit w) => JsonPartialValueAt (JsonCursor BS.ByteString v w) where jsonPartialJsonValueAt = jsonPartialJsonValueAt . jsonPartialIndexAt - - diff --git a/src/HaskellWorks/Data/Json/Backend/Standard/Index.hs b/src/HaskellWorks/Data/Json/Backend/Standard/Index.hs --- a/src/HaskellWorks/Data/Json/Backend/Standard/Index.hs +++ b/src/HaskellWorks/Data/Json/Backend/Standard/Index.hs @@ -16,5 +16,5 @@ JsonCursor _ ib (SimpleBalancedParens bp) _ <- L.loadCursor filename let wib = DVS.unsafeCast ib :: DVS.Vector Word8 let wbp = DVS.unsafeCast bp :: DVS.Vector Word8 - BS.writeFile (filename ++ ".ib") (BS.toByteString wib) - BS.writeFile (filename ++ ".bp") (BS.toByteString wbp) + BS.writeFile (filename ++ ".ib.idx") (BS.toByteString wib) + BS.writeFile (filename ++ ".bp.idx") (BS.toByteString wbp) diff --git a/src/HaskellWorks/Data/Json/Backend/Standard/Load/Raw.hs b/src/HaskellWorks/Data/Json/Backend/Standard/Load/Raw.hs --- a/src/HaskellWorks/Data/Json/Backend/Standard/Load/Raw.hs +++ b/src/HaskellWorks/Data/Json/Backend/Standard/Load/Raw.hs @@ -12,8 +12,8 @@ loadRawWithIndex :: String -> IO (BS.ByteString, DVS.Vector Word64, DVS.Vector Word64) loadRawWithIndex filename = do jsonFr <- mmapFileForeignPtr filename ReadOnly Nothing - jsonIbFr <- mmapFileForeignPtr (filename ++ ".ib") ReadOnly Nothing - jsonBpFr <- mmapFileForeignPtr (filename ++ ".bp") ReadOnly Nothing + jsonIbFr <- mmapFileForeignPtr (filename ++ ".ib.idx") ReadOnly Nothing + jsonBpFr <- mmapFileForeignPtr (filename ++ ".bp.idx") ReadOnly Nothing let jsonBS = fromForeignRegion jsonFr :: BS.ByteString let jsonIb = fromForeignRegion jsonIbFr :: DVS.Vector Word64 let jsonBp = fromForeignRegion jsonBpFr :: DVS.Vector Word64 diff --git a/test/HaskellWorks/Data/Json/CorpusSpec.hs b/test/HaskellWorks/Data/Json/CorpusSpec.hs --- a/test/HaskellWorks/Data/Json/CorpusSpec.hs +++ b/test/HaskellWorks/Data/Json/CorpusSpec.hs @@ -31,8 +31,8 @@ spec = describe "HaskellWorks.Data.Json.Corpus" $ do it "Corpus 5000B loads properly" $ requireTest $ do inJsonBS <- liftIO $ BS.readFile "corpus/5000B.json" - inInterestBitsBS <- liftIO $ BS.readFile "corpus/5000B.ib" - inInterestBalancedParensBS <- liftIO $ BS.readFile "corpus/5000B.bp" + inInterestBitsBS <- liftIO $ BS.readFile "corpus/5000B.json.ib.idx" + inInterestBalancedParensBS <- liftIO $ BS.readFile "corpus/5000B.json.bp.idx" let inInterestBits = fromBitTextByteString inInterestBitsBS let inInterestBalancedParens = fromBitTextByteString inInterestBalancedParensBS let !cursor = SLOW.makeCursor inJsonBS @@ -44,8 +44,8 @@ bp === inInterestBalancedParens it "issue-0001 loads properly" $ requireTest $ do inJsonBS <- liftIO $ BS.readFile "corpus/issue-0001.json" - inInterestBitsBS <- liftIO $ BS.readFile "corpus/issue-0001.ib" - inInterestBalancedParensBS <- liftIO $ BS.readFile "corpus/issue-0001.bp" + inInterestBitsBS <- liftIO $ BS.readFile "corpus/issue-0001.json.ib.idx" + inInterestBalancedParensBS <- liftIO $ BS.readFile "corpus/issue-0001.json.bp.idx" let inInterestBits = fromBitTextByteString inInterestBitsBS let inInterestBalancedParens = fromBitTextByteString inInterestBalancedParensBS let !cursor = SLOW.makeCursor inJsonBS