hw-diagnostics 0.0.0.1 → 0.0.0.2
raw patch · 3 files changed
+10/−130 lines, 3 filesdep −arraydep −attoparsecdep −bytestringdep ~basePVP ok
version bump matches the API change (PVP)
Dependencies removed: array, attoparsec, bytestring, conduit, criterion, deepseq, ghc-prim, lens, mmap, mono-traversable, parsec, random, resourcet, safe, text, vector, word8
Dependency ranges changed: base
API changes (from Hackage documentation)
Files
- README.md +3/−100
- hw-diagnostics.cabal +6/−29
- test/HaskellWorks/DiagnosticsSpec.hs +1/−1
README.md view
@@ -1,101 +1,4 @@-# hw-succinct-[](https://circleci.com/gh/haskell-works/hw-succinct)-Conduits for tokenizing streams.--`hw-succinct` is a succinct JSON parsing library. It uses succinct data-structures to allow traversal of-large JSON strings with minimal memory overhead.--It is currently considered experimental.--For an example, see [`app/Main.hs`](../master/app/Main.hs)--## Prerequisites-* Install `haskell-stack`.-* Install `hlint` (eg. `stack install hlint`)--## Building--Run the following in the shell:-- git clone git@github.com:haskell-works/hw-succinct.git- cd hw-succinct- stack setup- stack build- stack test- stack ghci --ghc-options -XOverloadedStrings \- --main-is hw-succinct:exe:hw-succinct-example--## Memory benchmark--### Parsing large Json files in Scala with Argonaut-- S0U EU OU MU CCSU CMD- --------- --------- ----------- -------- -------- ---------------------------------------------------------------- 0.0 80,526.3 76,163.6 72,338.6 13,058.6 sbt console- 0.0 536,660.4 76,163.6 72,338.6 13,058.6 import java.io._, argonaut._, Argonaut._- 0.0 552,389.1 76,163.6 72,338.6 13,058.6 val file = new File("/Users/jky/Downloads/78mbs.json"- 0.0 634,066.5 76,163.6 72,338.6 13,058.6 val array = new Array[Byte](file.length.asInstanceOf[Int])- 0.0 644,552.3 76,163.6 72,338.6 13,058.6 val is = new FileInputStream("/Users/jky/Downloads/78mbs.json")- 0.0 655,038.1 76,163.6 72,338.6 13,058.6 is.read(array)- 294,976.0 160,159.7 1,100,365.0 79,310.8 13,748.1 val json = new String(array)- 285,182.9 146,392.6 1,956,264.5 82,679.8 14,099.6 val data = Parse.parse(json)- ***********--### Parsing large Json files in Haskell with Aeson-- Mem (MB) CMD- -------- ---------------------------------------------------------- 302 import Data.Aeson- 302 import qualified Data.ByteString.Lazy as BSL- 302 json78m <- BSL.readFile "/Users/jky/Downloads/78mbs.json"- 1400 let !x = decode json78m :: Maybe Value--### Parsing large Json files in Haskell with hw-succinct-- Mem (MB) CMD- -------- ---------------------------------------------------------- 274 import Foreign- 274 import qualified Data.Vector.Storable as DVS- 274 import qualified Data.ByteString as BS- 274 import System.IO.MMap- 274 import Data.Word- 274 (fptr :: ForeignPtr Word8, offset, size) <- mmapFileForeignPtr "/Users/jky/Downloads/78mbs.json" ReadOnly Nothing- 601 cursor <- measure (fromForeignRegion (fptr, offset, size) :: JsonCursor BS.ByteString (BitShown (DVS.Vector Word64)) (SimpleBalancedParens (DVS.Vector Word64)))--## Examples-- import Foreign- import qualified Data.Vector.Storable as DVS- import qualified Data.ByteString as BS- import qualified Data.ByteString.Internal as BSI- import System.IO.MMap- import Data.Word- import System.CPUTime- (fptr :: ForeignPtr Word8, offset, size) <- mmapFileForeignPtr "/Users/jky/Downloads/78mbs.json" ReadOnly Nothing- cursor <- measure (fromForeignRegion (fptr, offset, size) :: JsonCursor BS.ByteString (BitShown (DVS.Vector Word64)) (SimpleBalancedParens (DVS.Vector Word64)))- let !bs = BSI.fromForeignPtr (castForeignPtr fptr) offset size- x <- measure $ jsonBsToInterestBs bs- let !y = runListConduit [bs] (unescape' "")-- import Foreign- import qualified Data.Vector.Storable as DVS- import qualified Data.ByteString as BS- import qualified Data.ByteString.Internal as BSI- import System.IO.MMap- import Data.Word- import System.CPUTime- (fptr :: ForeignPtr Word8, offset, size) <- mmapFileForeignPtr "/Users/jky/Downloads/part40.json" ReadOnly Nothing- let !bs = BSI.fromForeignPtr (castForeignPtr fptr) offset size- x <- measure $ BS.concat $ runListConduit [bs] (blankJson =$= blankedJsonToInterestBits)- x <- measure $ jsonBsToInterestBs bs- - jsonTokenAt $ J.nextSibling $ J.firstChild $ J.nextSibling $ J.firstChild $ J.firstChild cursor--## References-* [Succinct Data Structures talk by Edward Kmett](https://www.youtube.com/watch?v=uA0Z7_4J7u8)-* [Typed Tagless Final Interpreters](http://okmij.org/ftp/tagless-final/course/lecture.pdf)-* [Conduit Overview](https://www.schoolofhaskell.com/school/to-infinity-and-beyond/pick-of-the-week/conduit-overview)-+# hw-diagnostics+[](https://circleci.com/gh/haskell-works/hw-diagnostics/tree/v0.0-branch) -## Special mentions-* [Sydney Paper Club](http://www.meetup.com/Sydney-Paper-Club/)+Simple facilities for debugging code
hw-diagnostics.cabal view
@@ -1,5 +1,5 @@ name: hw-diagnostics-version: 0.0.0.1+version: 0.0.0.2 synopsis: Conduits for tokenizing streams. description: Please see README.md homepage: http://github.com/haskell-works/hw-diagnostics#readme@@ -16,42 +16,19 @@ executable hw-diagnostics-example hs-source-dirs: app main-is: Main.hs- ghc-options: -threaded -rtsopts -with-rtsopts=-N -Wall -msse4.2- build-depends: base >= 4 && < 5 ,- bytestring ,- conduit ,- criterion >= 1.1.0.0 && < 1.2 ,- hw-diagnostics ,- mmap ,- resourcet ,- vector >= 0.6 && < 0.12+ ghc-options: -threaded -rtsopts -with-rtsopts=-N -Wall+ build-depends: base >= 4 && < 5+ , hw-diagnostics default-language: Haskell2010 library hs-source-dirs: src exposed-modules: HaskellWorks.Diagnostics , HaskellWorks.Diagnostics.Time- build-depends: base >= 4.7 && < 5- , array- , attoparsec >= 0.10- , bytestring- , conduit >= 1.1 && < 1.3- , deepseq < 1.5- , ghc-prim- , lens- , mmap- , mono-traversable- , parsec- , QuickCheck- , random- , resourcet >= 1.1- , safe- , text- , vector- , word8+ build-depends: base >= 4 && < 5 default-language: Haskell2010- ghc-options: -rtsopts -with-rtsopts=-N -Wall -msse4.2+ ghc-options: -rtsopts -with-rtsopts=-N -Wall test-suite hw-diagnostics-test type: exitcode-stdio-1.0
test/HaskellWorks/DiagnosticsSpec.hs view
@@ -7,4 +7,4 @@ spec :: Spec spec = describe "HaskellWorks.DiagnosticsSpec" $ do it "Not a test" $- 1 `shouldBe` 1+ (1 :: Int) `shouldBe` 1