diff --git a/bench/Main.hs b/bench/Main.hs --- a/bench/Main.hs +++ b/bench/Main.hs @@ -33,8 +33,7 @@ . Char8ByteString.length . encodeWithJsonifier sample sampleName sampleData = - "- " <> TextBuilder.text sampleName <> ": " - <> sampleDataSize sampleData + "- " <> TextBuilder.text sampleName <> ": " <> sampleDataSize sampleData in "Input data sizes report:\n" <> sample "twitter with 1 objects" twitter1Data <> "\n" diff --git a/cbits/json_allocation.c b/cbits/json_allocation.c --- a/cbits/json_allocation.c +++ b/cbits/json_allocation.c @@ -11,7 +11,9 @@ static const int allocation_by_septet[128] = {6,6,6,6,6,6,6,6,6,2,2,6,6,2,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,1,1,2,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,2,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1}; -int count_string_allocation +// UTF-16 + +int measure_utf16_text_ ( const uint16_t *src_ptr, const uint16_t *end_ptr @@ -42,7 +44,7 @@ return allocation; } -int count_string_allocation_off_len +int measure_utf16_text_off_len ( const uint16_t *src_ptr, size_t src_off, @@ -52,5 +54,49 @@ src_ptr += src_off; const uint16_t *end_ptr = src_ptr + src_len; - return count_string_allocation(src_ptr, end_ptr); + return measure_utf16_text_(src_ptr, end_ptr); +} + +// UTF-8 + +int measure_utf8_text_ +( + const uint8_t *src_ptr, + const uint8_t *end_ptr +) +{ + size_t allocation = 0; + + while (src_ptr < end_ptr) { + uint8_t x = *src_ptr; + + if (x < 0x80) { + allocation += allocation_by_septet[x]; + src_ptr += 1; + } else if (x < 0xE0) { + allocation += 2; + src_ptr += 2; + } else if (x < 0xF0) { + allocation += 3; + src_ptr += 3; + } else { + allocation += 4; + src_ptr += 4; + } + } + + return allocation; +} + +int measure_utf8_text_off_len +( + const uint8_t *src_ptr, + size_t src_off, + size_t src_len +) +{ + src_ptr += src_off; + const uint8_t *end_ptr = src_ptr + src_len; + + return measure_utf8_text_(src_ptr, end_ptr); } diff --git a/cbits/json_encoding.c b/cbits/json_encoding.c --- a/cbits/json_encoding.c +++ b/cbits/json_encoding.c @@ -31,7 +31,7 @@ static const uint16_t two_byte_seq_by_septet[128] = {0,0,0,0,0,0,0,0,0,slash_t_seq_def,slash_n_seq_def,0,0,slash_r_seq_def,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,slash_doublequote_seq_def,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,slash_slash_seq_def}; -uint8_t* encode_utf16_as_string +uint8_t* encode_utf16_text ( uint8_t *dest, const uint16_t *src, @@ -86,6 +86,72 @@ *dest++ = ((c >> 12) & 0x3F) | 0x80; *dest++ = ((c >> 6) & 0x3F) | 0x80; *dest++ = (c & 0x3F) | 0x80; + } + } + + *dest++ = 34; + + return dest; +} + +uint8_t* encode_utf8_text +( + uint8_t *dest, + const uint8_t *src, + size_t src_offset, + size_t src_length +) +{ + + src += src_offset; + + const uint8_t *src_end = src + src_length; + + // Write double quote + *dest++ = 34; + + while (src < src_end) { + uint8_t x = *src; + + if (x < 0x80) { + if (pass_through_by_septet[x]) { + *dest++ = x; + } else { + uint16_t two_byte_seq = two_byte_seq_by_septet[x]; + if (two_byte_seq) { + *((uint16_t*) dest) = two_byte_seq; + dest += 2; + } else { + // \u + *((uint16_t*) dest) = slash_u_seq; + + // hex encoding of 4 nibbles + *(dest + 2) = digits[x >> 12 & 0xF]; + *(dest + 3) = digits[x >> 8 & 0xF]; + *(dest + 4) = digits[x >> 4 & 0xF]; + *(dest + 5) = digits[x & 0xF]; + dest += 6; + } + } + src++; + } else if (x < 0xE0) { + *dest = x; + *(dest + 1) = *(src + 1); + dest += 2; + src += 2; + } else if (x < 0xF0) { + *dest = x; + *(dest + 1) = *(src + 1); + *(dest + 2) = *(src + 2); + dest += 3; + src += 3; + } else { + *dest = x; + *(dest + 1) = *(src + 1); + *(dest + 2) = *(src + 2); + *(dest + 3) = *(src + 3); + dest += 4; + src += 4; } } diff --git a/demo/Main.hs b/demo/Main.hs --- a/demo/Main.hs +++ b/demo/Main.hs @@ -3,6 +3,7 @@ import qualified Data.ByteString.Char8 import qualified Jsonifier as J +import Prelude -- | -- Outputs the following: diff --git a/jsonifier.cabal b/jsonifier.cabal --- a/jsonifier.cabal +++ b/jsonifier.cabal @@ -1,92 +1,137 @@ -name: jsonifier -version: 0.2.1.1 -synopsis: Fast and simple JSON encoding toolkit +cabal-version: 3.0 +name: jsonifier +version: 0.2.1.2 +synopsis: Fast and simple JSON encoding toolkit description: Minimalistic library for encoding JSON directly to strict bytestring, which is up to 3x faster than \"aeson\". . For introduction, benchmark results and demo please skip to Readme. -category: JSON -homepage: https://github.com/nikita-volkov/jsonifier -bug-reports: https://github.com/nikita-volkov/jsonifier/issues -author: Nikita Volkov -maintainer: Nikita Volkov -copyright: (c) 2020 Nikita Volkov -license: MIT -license-file: LICENSE -build-type: Simple -cabal-version: >=1.10 + +category: JSON +homepage: https://github.com/nikita-volkov/jsonifier +bug-reports: https://github.com/nikita-volkov/jsonifier/issues +author: Nikita Volkov +maintainer: Nikita Volkov +copyright: (c) 2020 Nikita Volkov +license: MIT +license-file: LICENSE extra-source-files: README.md samples/*.json source-repository head - type: git + type: git location: git://github.com/nikita-volkov/jsonifier.git +common base-settings + default-language: Haskell2010 + default-extensions: + NoImplicitPrelude + NoMonomorphismRestriction + BangPatterns + ConstraintKinds + DataKinds + DefaultSignatures + DeriveDataTypeable + DeriveFoldable + DeriveFunctor + DeriveGeneric + DeriveTraversable + EmptyDataDecls + FlexibleContexts + FlexibleInstances + FunctionalDependencies + GADTs + GeneralizedNewtypeDeriving + InstanceSigs + LambdaCase + LiberalTypeSynonyms + MagicHash + MultiParamTypeClasses + MultiWayIf + OverloadedStrings + ParallelListComp + PatternGuards + QuasiQuotes + RankNTypes + RecordWildCards + ScopedTypeVariables + StandaloneDeriving + StrictData + TemplateHaskell + TupleSections + TypeApplications + TypeFamilies + TypeOperators + UnboxedTuples + ViewPatterns + +common executable-settings + import: base-settings + ghc-options: -O2 -threaded "-with-rtsopts=-N -I0 -qg" + library - hs-source-dirs: library - default-extensions: BangPatterns, ConstraintKinds, DataKinds, DefaultSignatures, DeriveDataTypeable, DeriveFoldable, DeriveFunctor, DeriveGeneric, DeriveTraversable, EmptyDataDecls, FlexibleContexts, FlexibleInstances, FunctionalDependencies, GADTs, GeneralizedNewtypeDeriving, InstanceSigs, LambdaCase, LiberalTypeSynonyms, MagicHash, MultiParamTypeClasses, MultiWayIf, NoImplicitPrelude, NoMonomorphismRestriction, OverloadedStrings, PatternGuards, ParallelListComp, QuasiQuotes, RankNTypes, RecordWildCards, ScopedTypeVariables, StandaloneDeriving, StrictData, TemplateHaskell, TupleSections, TypeApplications, TypeFamilies, TypeOperators, UnboxedTuples, ViewPatterns - default-language: Haskell2010 - exposed-modules: - Jsonifier + import: base-settings + hs-source-dirs: library + exposed-modules: Jsonifier other-modules: - Jsonifier.Size Jsonifier.Ffi Jsonifier.Poke Jsonifier.Prelude + Jsonifier.Size Jsonifier.Text Jsonifier.Write + c-sources: cbits/json_allocation.c cbits/json_encoding.c + build-depends: - base >=4.11 && <5, - bytestring >=0.10.10 && <0.12, - ptr-poker >=0.1.2.2 && <0.2, - scientific >=0.3.6.2 && <0.4, - text >=1 && <2 + , base >=4.11 && <5 + , bytestring >=0.10.10 && <0.12 + , ptr-poker ^>=0.1.2.2 + , scientific ^>=0.3.6.2 + , text >=1 && <3 test-suite demo - type: exitcode-stdio-1.0 - hs-source-dirs: demo - main-is: Main.hs + import: base-settings + type: exitcode-stdio-1.0 + hs-source-dirs: demo + main-is: Main.hs default-language: Haskell2010 build-depends: - jsonifier, - rerebase + , jsonifier + , rerebase test-suite test - type: exitcode-stdio-1.0 + import: base-settings + type: exitcode-stdio-1.0 hs-source-dirs: test - default-extensions: BangPatterns, ConstraintKinds, DataKinds, DefaultSignatures, DeriveDataTypeable, DeriveFoldable, DeriveFunctor, DeriveGeneric, DeriveTraversable, EmptyDataDecls, FlexibleContexts, FlexibleInstances, FunctionalDependencies, GADTs, GeneralizedNewtypeDeriving, InstanceSigs, LambdaCase, LiberalTypeSynonyms, MagicHash, MultiParamTypeClasses, MultiWayIf, NoImplicitPrelude, NoMonomorphismRestriction, OverloadedStrings, PatternGuards, ParallelListComp, QuasiQuotes, RankNTypes, RecordWildCards, ScopedTypeVariables, StandaloneDeriving, StrictData, TemplateHaskell, TupleSections, TypeApplications, TypeFamilies, TypeOperators, UnboxedTuples, ViewPatterns - default-language: Haskell2010 - main-is: Main.hs - other-modules: - Main.Util.HedgehogGens + main-is: Main.hs + other-modules: Main.Util.HedgehogGens build-depends: - aeson >=2 && <3, - hedgehog >=1.0.3 && <2, - jsonifier, - numeric-limits >=0.1 && <0.2, - rerebase >=1.10.0.1 && <2 + , aeson >=2 && <3 + , hedgehog >=1.0.3 && <2 + , jsonifier + , numeric-limits ^>=0.1 + , rerebase >=1.10.0.1 && <2 benchmark bench - type: exitcode-stdio-1.0 + import: executable-settings + type: exitcode-stdio-1.0 hs-source-dirs: bench - main-is: Main.hs + main-is: Main.hs other-modules: Main.Aeson Main.BufferBuilder Main.Jsonifier Main.Model - ghc-options: -O2 -threaded "-with-rtsopts=-N" - default-extensions: BangPatterns, ConstraintKinds, DataKinds, DefaultSignatures, DeriveDataTypeable, DeriveFoldable, DeriveFunctor, DeriveGeneric, DeriveTraversable, EmptyDataDecls, FlexibleContexts, FlexibleInstances, FunctionalDependencies, GADTs, GeneralizedNewtypeDeriving, InstanceSigs, LambdaCase, LiberalTypeSynonyms, MagicHash, MultiParamTypeClasses, MultiWayIf, NoImplicitPrelude, NoMonomorphismRestriction, OverloadedStrings, PatternGuards, ParallelListComp, QuasiQuotes, RankNTypes, RecordWildCards, ScopedTypeVariables, StandaloneDeriving, StrictData, TemplateHaskell, TupleSections, TypeApplications, TypeFamilies, TypeOperators, UnboxedTuples, ViewPatterns - default-language: Haskell2010 + build-depends: - aeson >=2 && <3, - buffer-builder >=0.2.4.7 && <0.3, - gauge >=0.2.5 && <0.3, - jsonifier, - rerebase >=1.10.0.1 && <2, - text-builder >=0.6.6.1 && <0.7 + , aeson >=2 && <3 + , buffer-builder ^>=0.2.4.7 + , gauge ^>=0.2.5 + , jsonifier + , rerebase >=1.10.0.1 && <2 + , text-builder ^>=0.6.6.1 diff --git a/library/Jsonifier/Ffi.hs b/library/Jsonifier/Ffi.hs --- a/library/Jsonifier/Ffi.hs +++ b/library/Jsonifier/Ffi.hs @@ -1,3 +1,4 @@ +{-# LANGUAGE CPP #-} {-# LANGUAGE UnliftedFFITypes #-} module Jsonifier.Ffi where @@ -6,8 +7,20 @@ import GHC.Base (ByteArray#, MutableByteArray#) import Jsonifier.Prelude -foreign import ccall unsafe "static count_string_allocation_off_len" - countStringAllocationSize :: ByteArray# -> CSize -> CSize -> IO CInt +#if MIN_VERSION_text (2, 0, 0) -foreign import ccall unsafe "static encode_utf16_as_string" - encodeString :: Ptr Word8 -> ByteArray# -> CSize -> CSize -> IO (Ptr Word8) +foreign import ccall unsafe "static measure_utf8_text_off_len" + countTextEncoding :: ByteArray# -> CSize -> CSize -> IO CInt + +foreign import ccall unsafe "static encode_utf8_text" + encodeText :: Ptr Word8 -> ByteArray# -> CSize -> CSize -> IO (Ptr Word8) + +#else + +foreign import ccall unsafe "static measure_utf16_text_off_len" + countTextEncoding :: ByteArray# -> CSize -> CSize -> IO CInt + +foreign import ccall unsafe "static encode_utf16_text" + encodeText :: Ptr Word8 -> ByteArray# -> CSize -> CSize -> IO (Ptr Word8) + +#endif diff --git a/library/Jsonifier/Poke.hs b/library/Jsonifier/Poke.hs --- a/library/Jsonifier/Poke.hs +++ b/library/Jsonifier/Poke.hs @@ -27,7 +27,7 @@ string = Text.destruct $ \arr off len -> Poke $ \ptr -> - Ffi.encodeString ptr arr (fromIntegral off) (fromIntegral len) + Ffi.encodeText ptr arr (fromIntegral off) (fromIntegral len) -- | -- > "key":value diff --git a/library/Jsonifier/Size.hs b/library/Jsonifier/Size.hs --- a/library/Jsonifier/Size.hs +++ b/library/Jsonifier/Size.hs @@ -36,7 +36,7 @@ stringBody :: Text -> Int stringBody = Text.destruct $ \arr off len -> - Ffi.countStringAllocationSize + Ffi.countTextEncoding arr (fromIntegral off) (fromIntegral len)