diff --git a/README.md b/README.md --- a/README.md +++ b/README.md @@ -1,150 +1,11 @@ -# hw-json -[![master](https://circleci.com/gh/haskell-works/hw-json/tree/master.svg?style=svg)](https://circleci.com/gh/haskell-works/hw-json/tree/master) +# hw-json-standard-cursor +[![master](https://circleci.com/gh/haskell-works/hw-json-standard-cursor/tree/master.svg?style=svg)](https://circleci.com/gh/haskell-works/hw-json-standard-cursor/tree/master) -`hw-json` is a succinct JSON parsing library. +`hw-json-standard-cursor` is support library for `hw-json`, a succinct JSON parsing library. It uses succinct data-structures to allow traversal of large JSON strings with minimal memory overhead. -For an example, see [`app/Main.hs`](../master/app/Main.hs) - -## Prerequisites - -* `cabal` version `2.2` or later - -## Memory benchmark - -### Parsing large Json files in Scala with Argonaut - -```text - S0U EU OU MU CCSU CMD ---------- --------- ----------- -------- -------- --------------------------------------------------------------- - 0.0 80,526.3 76,163.6 72,338.6 13,058.6 sbt console - 0.0 536,660.4 76,163.6 72,338.6 13,058.6 import java.io._, argonaut._, Argonaut._ - 0.0 552,389.1 76,163.6 72,338.6 13,058.6 val file = new File("/Users/jky/Downloads/78mbs.json" - 0.0 634,066.5 76,163.6 72,338.6 13,058.6 val array = new Array[Byte](file.length.asInstanceOf[Int]) - 0.0 644,552.3 76,163.6 72,338.6 13,058.6 val is = new FileInputStream("/Users/jky/Downloads/78mbs.json") - 0.0 655,038.1 76,163.6 72,338.6 13,058.6 is.read(array) -294,976.0 160,159.7 1,100,365.0 79,310.8 13,748.1 val json = new String(array) -285,182.9 146,392.6 1,956,264.5 82,679.8 14,099.6 val data = Parse.parse(json) - *********** -``` - -### Parsing large Json files in Haskell with Aeson - -```haskell --- CMD -- Mem (MB) ----------------------------------------------------------- -- -------- -import Control.DeepSeq -- 94 -import Data.Aeson -- 100 -import qualified Data.ByteString.Lazy as BSL -- 104 -bs <- BSL.readFile "../corpus/bench/hospitalisation.json" -- 105 -let !x = deepseq bs bs -- 146 -let !y = decode json78m :: Maybe Value -- 669 -``` - -### Parsing large Json files in Haskell with hw-json - -```haskell --- CMD -- Mem (MB) ---------------------------------------------------------------------- -- -------- -import Foreign -- 93 -import Control.Monad -- 95 -import Data.Word -- 96 -import HaskellWorks.Data.BalancedParens.Simple -- 97 -import HaskellWorks.Data.Bits.BitShown -- 98 -import HaskellWorks.Data.FromForeignRegion -- 99 -import HaskellWorks.Data.Json.Backend.Standard.Cursor -- 106 -import System.IO.MMap -- 109 -import qualified Data.ByteString as BS -- 110 -import qualified Data.Vector.Storable as DVS -- 111 -import qualified HaskellWorks.Data.ByteString as BS -- 112 -import qualified HaskellWorks.Data.Json.Backend.Standard.Fast as FAST -- 114 -bs <- BS.mmap "../corpus/bench/hospitalisation.json" -- 115 -let !cursor = FAST.makeCursor bs -- 203 -``` - -## Examples - -### Navigation example - -```haskell -import Control.Monad -import Data.String -import Data.Word -import HaskellWorks.Data.BalancedParens.Simple -import HaskellWorks.Data.Bits.BitShow -import HaskellWorks.Data.Bits.BitShown -import HaskellWorks.Data.FromForeignRegion -import HaskellWorks.Data.Json.Backend.Standard.Cursor -import HaskellWorks.Data.Json.Internal.Token.Types -import HaskellWorks.Data.RankSelect.Base.Rank0 -import HaskellWorks.Data.RankSelect.Base.Rank1 -import HaskellWorks.Data.RankSelect.Base.Select1 -import HaskellWorks.Data.RankSelect.CsPoppy -import System.IO.MMap - -import qualified Data.ByteString as BS -import qualified Data.Vector.Storable as DVS -import qualified HaskellWorks.Data.Json.Backend.Standard.Cursor as C -import qualified HaskellWorks.Data.Json.Backend.Standard.Fast as FAST -import qualified HaskellWorks.Data.TreeCursor as TC - -let fc = TC.firstChild -let ns = TC.nextSibling -let pn = TC.parent -let ss = TC.subtreeSize -let cursor = FAST.makeCursor "[null, {\"field\": 1}]" -cursor -fc cursor -(fc >=> ns) cursor -``` - -### Querying example - -```haskell -import Control.Monad -import Data.Function -import Data.List -import HaskellWorks.Data.Json.Backend.Standard.Load.Cursor -import HaskellWorks.Data.Json.Backend.Standard.Load.Partial -import HaskellWorks.Data.Json.Backend.Standard.Load.Raw -import HaskellWorks.Data.Json.PartialValue -import HaskellWorks.Data.MQuery -import HaskellWorks.Data.MQuery.Micro -import HaskellWorks.Data.MQuery.Row - -import qualified Data.DList as DL - -!cursor <- loadPartial "../corpus/bench/78mb.json" -!cursor <- loadCursorWithIndex "../corpus/bench/78mb.json" -!cursor <- loadCursor "../corpus/bench/78mb.json" -!cursor <- loadCursorWithCsPoppyIndex "../corpus/bench/78mb.json" -let !json = jsonPartialJsonValueAt cursor -let q = MQuery (DL.singleton json) - -putPretty $ q >>= item & limit 10 -putPretty $ q >>= item & page 10 1 -putPretty $ q >>= item >>= hasKV "founded_year" (JsonPartialNumber 2005) & limit 10 -putPretty $ q >>= item >>= entry -putPretty $ q >>= item >>= entry >>= named "name" & limit 10 -putPretty $ q >>= (item >=> entry >=> named "acquisition" >=> entry >=> named "price_currency_code") -putPretty $ q >>= (item >=> entry >=> named "acquisition" >=> entry >=> named "price_currency_code") & onList (uniq . sort) -putPretty $ q >>= (item >=> entry >=> named "acquisition" >=> entry >=> named "price_currency_code" >=> asString >=> valueOf "USD") & limit 10 -putPretty $ q >>= (item >=> entry >=> named "acquisition" >=> having (entry >=> named "price_currency_code" >=> asString >=> valueOf "USD") >=> entry >=> named "price_amount") & limit 10 -putPretty $ q >>= (item >=> entry >=> named "acquisition" >=> having (entry >=> named "price_currency_code" >=> asString >=> valueOf "USD") >=> entry >=> named "price_amount" >=> castAsInteger ) & limit 10 -putPretty $ q >>= (item >=> entry >=> named "acquisition" >=> having (entry >=> named "price_currency_code" >=> asString >=> valueOf "USD") >=> entry >=> named "price_amount" >=> castAsInteger ) & aggregate sum - -putPretty $ q >>= item & limit 10 -putPretty $ q >>= item & page 10 1 -putPretty $ q >>= item >>= entry -putPretty $ q >>= item >>= entry >>= named "name" & limit 10 -putPretty $ q >>= (item >=> entry >=> named "acquisition" >=> entry >=> named "price_currency_code" >=> asString) -putPretty $ q >>= (item >=> entry >=> named "acquisition" >=> entry >=> named "price_currency_code" >=> asString) & onList (uniq . sort) -putPretty $ q >>= (item >=> entry >=> named "acquisition" >=> entry >=> named "price_currency_code" >=> asString >=> valueOf "USD") & limit 10 -putPretty $ q >>= (item >=> entry >=> named "acquisition" >=> having (entry >=> named "price_currency_code" >=> asString >=> valueOf "USD") >=> entry >=> named "price_amount") & limit 10 -putPretty $ q >>= (item >=> entry >=> named "acquisition" >=> having (entry >=> named "price_currency_code" >=> asString >=> valueOf "USD") >=> entry >=> named "price_amount" >=> castAsInteger ) & limit 10 -putPretty $ q >>= (item >=> entry >=> named "acquisition" >=> having (entry >=> named "price_currency_code" >=> asString >=> valueOf "USD") >=> entry >=> named "price_amount" >=> castAsInteger ) & aggregate sum -``` +For more information see [`hw-json`](https://github.com/haskell-works/hw-json). ## References diff --git a/hw-json-standard-cursor.cabal b/hw-json-standard-cursor.cabal --- a/hw-json-standard-cursor.cabal +++ b/hw-json-standard-cursor.cabal @@ -1,7 +1,7 @@ cabal-version: 2.2 name: hw-json-standard-cursor -version: 0.1.0.0 +version: 0.1.0.1 synopsis: Memory efficient JSON parser description: Memory efficient JSON parser. Please see README.md category: Data @@ -39,14 +39,11 @@ common base { build-depends: base >= 4 && < 5 } -common ansi-wl-pprint { build-depends: ansi-wl-pprint >= 0.6.8.2 && < 0.7 } common array { build-depends: array >= 0.5 && < 0.6 } -common attoparsec { build-depends: attoparsec >= 0.13 && < 0.14 } common bits-extra { build-depends: bits-extra >= 0.0.1.3 && < 0.1 } common bytestring { build-depends: bytestring >= 0.10.6 && < 0.11 } common criterion { build-depends: criterion >= 1.4 && < 1.6 } common directory { build-depends: directory >= 1.3 && < 1.4 } -common dlist { build-depends: dlist >= 0.8 && < 0.9 } common generic-lens { build-depends: generic-lens >= 1.1.0.0 && < 1.2 } common hedgehog { build-depends: hedgehog >= 0.6 && < 1.1 } common hspec { build-depends: hspec >= 2.4 && < 3 } @@ -54,17 +51,13 @@ common hw-bits { build-depends: hw-bits >= 0.7.0.5 && < 0.8 } common hw-hspec-hedgehog { build-depends: hw-hspec-hedgehog >= 0.1.0.4 && < 0.2 } common hw-json-simd { build-depends: hw-json-simd >= 0.1.0.2 && < 0.2 } -common hw-mquery { build-depends: hw-mquery >= 0.2.0.0 && < 0.3 } -common hw-parser { build-depends: hw-parser >= 0.1 && < 0.2 } common hw-prim { build-depends: hw-prim >= 0.6.2.28 && < 0.7 } common hw-rankselect { build-depends: hw-rankselect >= 0.13 && < 0.14 } common hw-rankselect-base { build-depends: hw-rankselect-base >= 0.3.2.1 && < 0.4 } -common hw-simd { build-depends: hw-simd >= 0.1.1.2 && < 0.2 } common lens { build-depends: lens >= 4 && < 5 } common mmap { build-depends: mmap >= 0.5 && < 0.6 } common optparse-applicative { build-depends: optparse-applicative >= 0.14 && < 0.15 } common text { build-depends: text >= 1.2 && < 1.3 } -common transformers { build-depends: transformers >= 0.4 && < 0.6 } common vector { build-depends: vector >= 0.12 && < 0.13 } common word8 { build-depends: word8 >= 0.1 && < 0.2 } @@ -117,11 +110,9 @@ executable hw-json-standard-cursor import: base, config , bytestring - , dlist , generic-lens , hw-balancedparens , hw-json-simd - , hw-mquery , hw-prim , hw-rankselect , hw-rankselect-base @@ -142,7 +133,6 @@ test-suite hw-json-standard-cursor-test import: base, config - , attoparsec , bytestring , hedgehog , hspec @@ -152,7 +142,6 @@ , hw-prim , hw-rankselect , hw-rankselect-base - , transformers , vector type: exitcode-stdio-1.0 main-is: Spec.hs