diff --git a/README.md b/README.md --- a/README.md +++ b/README.md @@ -1,101 +1,4 @@ -# hw-succinct -[![Circle CI](https://circleci.com/gh/haskell-works/hw-succinct.svg?style=svg)](https://circleci.com/gh/haskell-works/hw-succinct) -Conduits for tokenizing streams. - -`hw-succinct` is a succinct JSON parsing library. It uses succinct data-structures to allow traversal of -large JSON strings with minimal memory overhead. - -It is currently considered experimental. - -For an example, see [`app/Main.hs`](../master/app/Main.hs) - -## Prerequisites -* Install `haskell-stack`. -* Install `hlint` (eg. `stack install hlint`) - -## Building - -Run the following in the shell: - - git clone git@github.com:haskell-works/hw-succinct.git - cd hw-succinct - stack setup - stack build - stack test - stack ghci --ghc-options -XOverloadedStrings \ - --main-is hw-succinct:exe:hw-succinct-example - -## Memory benchmark - -### Parsing large Json files in Scala with Argonaut - - S0U EU OU MU CCSU CMD - --------- --------- ----------- -------- -------- --------------------------------------------------------------- - 0.0 80,526.3 76,163.6 72,338.6 13,058.6 sbt console - 0.0 536,660.4 76,163.6 72,338.6 13,058.6 import java.io._, argonaut._, Argonaut._ - 0.0 552,389.1 76,163.6 72,338.6 13,058.6 val file = new File("/Users/jky/Downloads/78mbs.json" - 0.0 634,066.5 76,163.6 72,338.6 13,058.6 val array = new Array[Byte](file.length.asInstanceOf[Int]) - 0.0 644,552.3 76,163.6 72,338.6 13,058.6 val is = new FileInputStream("/Users/jky/Downloads/78mbs.json") - 0.0 655,038.1 76,163.6 72,338.6 13,058.6 is.read(array) - 294,976.0 160,159.7 1,100,365.0 79,310.8 13,748.1 val json = new String(array) - 285,182.9 146,392.6 1,956,264.5 82,679.8 14,099.6 val data = Parse.parse(json) - *********** - -### Parsing large Json files in Haskell with Aeson - - Mem (MB) CMD - -------- --------------------------------------------------------- - 302 import Data.Aeson - 302 import qualified Data.ByteString.Lazy as BSL - 302 json78m <- BSL.readFile "/Users/jky/Downloads/78mbs.json" - 1400 let !x = decode json78m :: Maybe Value - -### Parsing large Json files in Haskell with hw-succinct - - Mem (MB) CMD - -------- --------------------------------------------------------- - 274 import Foreign - 274 import qualified Data.Vector.Storable as DVS - 274 import qualified Data.ByteString as BS - 274 import System.IO.MMap - 274 import Data.Word - 274 (fptr :: ForeignPtr Word8, offset, size) <- mmapFileForeignPtr "/Users/jky/Downloads/78mbs.json" ReadOnly Nothing - 601 cursor <- measure (fromForeignRegion (fptr, offset, size) :: JsonCursor BS.ByteString (BitShown (DVS.Vector Word64)) (SimpleBalancedParens (DVS.Vector Word64))) - -## Examples - - import Foreign - import qualified Data.Vector.Storable as DVS - import qualified Data.ByteString as BS - import qualified Data.ByteString.Internal as BSI - import System.IO.MMap - import Data.Word - import System.CPUTime - (fptr :: ForeignPtr Word8, offset, size) <- mmapFileForeignPtr "/Users/jky/Downloads/78mbs.json" ReadOnly Nothing - cursor <- measure (fromForeignRegion (fptr, offset, size) :: JsonCursor BS.ByteString (BitShown (DVS.Vector Word64)) (SimpleBalancedParens (DVS.Vector Word64))) - let !bs = BSI.fromForeignPtr (castForeignPtr fptr) offset size - x <- measure $ jsonBsToInterestBs bs - let !y = runListConduit [bs] (unescape' "") - - import Foreign - import qualified Data.Vector.Storable as DVS - import qualified Data.ByteString as BS - import qualified Data.ByteString.Internal as BSI - import System.IO.MMap - import Data.Word - import System.CPUTime - (fptr :: ForeignPtr Word8, offset, size) <- mmapFileForeignPtr "/Users/jky/Downloads/part40.json" ReadOnly Nothing - let !bs = BSI.fromForeignPtr (castForeignPtr fptr) offset size - x <- measure $ BS.concat $ runListConduit [bs] (blankJson =$= blankedJsonToInterestBits) - x <- measure $ jsonBsToInterestBs bs - - jsonTokenAt $ J.nextSibling $ J.firstChild $ J.nextSibling $ J.firstChild $ J.firstChild cursor - -## References -* [Succinct Data Structures talk by Edward Kmett](https://www.youtube.com/watch?v=uA0Z7_4J7u8) -* [Typed Tagless Final Interpreters](http://okmij.org/ftp/tagless-final/course/lecture.pdf) -* [Conduit Overview](https://www.schoolofhaskell.com/school/to-infinity-and-beyond/pick-of-the-week/conduit-overview) - +# hw-bits +[![v0.0-branch](https://circleci.com/gh/haskell-works/hw-bits/tree/v0.0-branch.svg?style=svg)](https://circleci.com/gh/haskell-works/hw-bits/tree/v0.0-branch) -## Special mentions -* [Sydney Paper Club](http://www.meetup.com/Sydney-Paper-Club/) +Facilities for manipulating bits. diff --git a/hw-bits.cabal b/hw-bits.cabal --- a/hw-bits.cabal +++ b/hw-bits.cabal @@ -1,5 +1,5 @@ name: hw-bits -version: 0.0.0.5 +version: 0.0.0.6 synopsis: Conduits for tokenizing streams. description: Please see README.md homepage: http://github.com/haskell-works/hw-bits#readme @@ -19,9 +19,7 @@ hs-source-dirs: app main-is: Main.hs ghc-options: -threaded -rtsopts -with-rtsopts=-N -Wall -msse4.2 - build-depends: base >= 4 && < 5 - , bytestring - , conduit + build-depends: base >= 4 && < 5 , criterion , hw-bits , mmap @@ -46,25 +44,11 @@ , HaskellWorks.Data.Bits.Types.Broadword , HaskellWorks.Data.Bits.Types.Builtin , HaskellWorks.Data.Bits.Word - build-depends: base >= 4.7 && < 5 - , array - , attoparsec + build-depends: base >= 4.7 && < 5 , bytestring - , conduit - , deepseq - , ghc-prim , hw-prim - , lens - , mmap - , mono-traversable , parsec - , QuickCheck - , random - , resourcet - , safe - , text , vector - , word8 default-language: Haskell2010 ghc-options: -rtsopts -with-rtsopts=-N -Wall -O2 -msse4.2 @@ -76,16 +60,9 @@ other-modules: HaskellWorks.Data.Bits.BitReadSpec , HaskellWorks.Data.Bits.BitWiseSpec build-depends: base - , attoparsec - , bytestring - , conduit , hspec , hw-bits - , mmap - , parsec , QuickCheck - , resourcet - , transformers , vector ghc-options: -threaded -rtsopts -with-rtsopts=-N -Wall default-language: Haskell2010 @@ -100,12 +77,8 @@ Main-Is: Main.hs GHC-Options: -Wall -O2 -msse4.2 Default-Language: Haskell2010 - Build-Depends: base >= 4 && < 5 - , bytestring - , conduit + Build-Depends: base >= 4 && < 5 , criterion , hw-bits , hw-prim - , mmap - , resourcet , vector