json-stream 0.4.4.0 → 0.4.4.1

raw patch · 9 files changed

+62/−18 lines, 9 filesPVP: minor bump suggested

API additions: PVP suggests at least a minor version bump

API changes (from Hackage documentation)

Files

• Data/JsonStream/CLexer.hs +1/−1
• Data/JsonStream/Parser.hs +18/−6
• Data/JsonStream/TokenParser.hs +1/−1
• Data/JsonStream/Unescape.hs +19/−4
• README.md +2/−2
• c_lib/lexer.c +6/−3
• changelog.md +6/−0
• json-stream.cabal +1/−1
• test/ParserSpec.hs +8/−0

@@ -218,7 +218,7 @@                        Nothing -> TokFailed         | resType == resString ->           if | resAddData == -1 || resAddData == 0 -> -- One-part string without escaped characters; with escaped-                PartialResult (StringRaw textSection) next+                PartialResult (StringRaw textSection (resAddData == -1)) next              | otherwise -> PartialResult (StringContent textSection) -- Final part of partial strings                             (PartialResult StringEnd next)         | resType == resStringPartial ->

@@ -67,6 +67,7 @@   , objectWithKey   , objectItems   , objectValues+  , objectKeyValues   , arrayOf   , arrayWithIndexOf   , indexedArrayOf@@ -111,6 +112,7 @@  import           Data.JsonStream.CLexer import           Data.JsonStream.TokenParser+import Data.JsonStream.Unescape (unsafeDecodeASCII)  -- | Limit for the size of an object key objectKeyStringLimit :: Int@@ -323,7 +325,9 @@     nextitem _ _ (ObjectEnd ctx) ntok = Done ctx ntok     nextitem yielded _ (JValue (AE.String key)) ntok =       objcontent yielded (callParse (valparse key) ntok)-    nextitem yielded _ (StringRaw bs) ntok = +    nextitem yielded _ (StringRaw bs True) ntok = +        objcontent yielded (callParse (valparse (unsafeDecodeASCII bs)) ntok)+    nextitem yielded _ (StringRaw bs False) ntok =        case unescapeText bs of         Right t -> objcontent yielded (callParse (valparse t) ntok)         Left e -> Failed (show e)@@ -370,6 +374,12 @@ objectValues :: Parser a -> Parser a objectValues valparse = object' False (const valparse) +-- | Match all key-value pairs of an object, and parse the value based on the key.+-- If the source object defines same key multiple times, all values+-- are matched.+objectKeyValues :: (T.Text -> Parser a) -> Parser a+objectKeyValues = object' False+ -- | Match only specific key of an object. -- This function will return only the first matched value in an object even -- if the source JSON defines the key multiple times (in violation of the specification).@@ -394,7 +404,8 @@         JValue val -> Yield val (Done "" ntok)         JInteger val -> Yield (AE.Number $ fromIntegral val) (Done "" ntok)         StringContent _ -> callParse (AE.String <$> longString Nothing) tok-        StringRaw bs -> case unescapeText bs of+        StringRaw bs True -> Yield (AE.String (unsafeDecodeASCII bs)) (Done "" ntok)+        StringRaw bs False -> case unescapeText bs of               Right t -> Yield (AE.String t) (Done "" ntok)               Left e -> Failed (show e)         ArrayBegin -> AE.Array . Vec.fromList <$> callParse (many (arrayOf aeValue)) tok@@ -434,7 +445,7 @@     handle acc !len tok el ntok =       case el of         JValue (AE.String _) -> Failed "INTERNAL ERROR! - got decoded JValue instead of string"-        StringRaw bs -> Yield bs (Done "" ntok)+        StringRaw bs _ -> Yield bs (Done "" ntok)         StringContent str           | (Just bounds) <- mbounds, len > bounds -- If the string exceeds bounds, discard it                           -> callParse (ignoreStrRestThen (Parser $ Done "")) ntok@@ -462,7 +473,8 @@     handle acc !len tok el ntok =       case el of         JValue (AE.String str) -> Yield str (Done "" ntok)-        StringRaw bs -> +        StringRaw bs True -> Yield (unsafeDecodeASCII bs) (Done "" ntok)+        StringRaw bs False ->            case unescapeText bs of             Right t -> Yield t (Done "" ntok)             Left e -> Failed (show e)@@ -607,7 +619,7 @@      handleTok :: Int -> TokenResult -> Element -> TokenResult -> ParseResult a     handleTok 0 _ (JValue _) ntok = Done "" ntok-    handleTok 0 _ (StringRaw _) ntok = Done "" ntok+    handleTok 0 _ (StringRaw _ _) ntok = Done "" ntok     handleTok 0 _ (JInteger _) ntok = Done "" ntok     handleTok 0 _ (ArrayEnd _) _ = Failed "ArrayEnd in ignoreval on 0 level"     handleTok 0 _ (ObjectEnd _) _ = Failed "ObjectEnd in ignoreval on 0 level"@@ -618,7 +630,7 @@         JValue _ -> moreData (handleTok level) ntok         JInteger _ -> moreData (handleTok level) ntok         StringContent _ -> moreData (handleLongString level) ntok-        StringRaw _ -> moreData (handleTok level) ntok+        StringRaw _ _ -> moreData (handleTok level) ntok         ArrayEnd _ -> moreData (handleTok (level - 1)) ntok         ObjectEnd _ -> moreData (handleTok (level - 1)) ntok         ArrayBegin -> moreData (handleTok (level + 1)) ntok

@@ -15,7 +15,7 @@   | ObjectBegin   | ObjectEnd !BS.ByteString   | StringContent !BS.ByteString-  | StringRaw !BS.ByteString -- Allow raw strings to go into parser as bytestring+  | StringRaw !BS.ByteString !Bool -- Allow raw strings to go into parser as bytestring/ isAscii   | StringEnd   | JValue !AE.Value   | JInteger !CLong

@@ -4,9 +4,11 @@ {-# LANGUAGE UnliftedFFITypes         #-} {-# LANGUAGE BangPatterns #-} {-# LANGUAGE MultiWayIf   #-}+{-# LANGUAGE PatternGuards   #-}  module Data.JsonStream.Unescape (-  unescapeText+    unescapeText+  , unsafeDecodeASCII ) where  import           Data.ByteString            as B@@ -18,16 +20,17 @@ import           Foreign.ForeignPtr         (withForeignPtr) import           Foreign.Ptr                (Ptr, plusPtr) import           Foreign.Storable           (peek)+import qualified Data.Text as T  #if MIN_VERSION_text(2,0,0)  import qualified Data.Primitive           as P-import qualified Data.Text.Array          as T+import qualified Data.Text.Array          as TA import qualified Data.Text.Internal       as T import           Data.Bits                (shiftL, shiftR, (.&.), (.|.)) import           Control.Exception        (try, throwIO) import Foreign.ForeignPtr (ForeignPtr)-import GHC.ForeignPtr (plusForeignPtr)+import qualified Data.ByteString.Short.Internal as SBS  #else @@ -38,9 +41,21 @@ import qualified Data.Text.Array            as A import           GHC.Base                   (MutableByteArray#) import           Foreign.C.Types            (CInt (..), CSize (..))+import qualified Data.Text.Encoding as TE  #endif +unsafeDecodeASCII :: ByteString -> T.Text++#if MIN_VERSION_text(2,0,0)+unsafeDecodeASCII bs = withBS bs $ \_fp len -> if len == 0 then T.empty else+  let !(SBS.SBS arr) = SBS.toShort bs in T.Text (TA.ByteArray arr) 0 len++#else+unsafeDecodeASCII = TE.decodeLatin1+#endif++ #if !MIN_VERSION_text(2,0,0)  foreign import ccall unsafe "_jstream_decode_string" c_js_decode@@ -429,7 +444,7 @@             P.shrinkMutablePrimArray arr out             frozenArr <- P.unsafeFreezePrimArray arr             return $ case frozenArr of-              P.PrimArray ba -> T.Text (T.ByteArray ba) 0 out+              P.PrimArray ba -> T.Text (TA.ByteArray ba) 0 out            | otherwise = do             w8 <- peek inp

@@ -7,7 +7,7 @@ - use [aeson](https://hackage.haskell.org/package/aeson) if you can; compile aeson with `cffi` flag if you need better performance - use `json-stream` if you   - need streaming-  - need every bit of performance (do profile; aeson is quite fast these days)+  - need every bit of performance (do profile; the best course could be using the aeson `value` parser with json-stream)   - do not care that parsing may not fail on malformed JSON data   - do not need advanced error reporting; json-stream tends to skip data that     doesn't fit parsing rules (this might be implemented better in the future)@@ -87,7 +87,7 @@   (the `cffi` flag of aeson enables fast text decoding borrowed from json-stream) - parsing only subset of big JSON structures -Json-stream is in streaming mode is also much friendlier to the GC.+Json-stream in streaming mode is also much friendlier to the GC.  Using json-stream parser instead of aeson `value` evades the need to build the structure using aeson `Value` and then converting it to the user-requested structure. Instead

@@ -186,7 +186,9 @@     char ch;     int hasspecialchar = 0; -    for (ch=input[lexer->position]; lexer->position < lexer->length; ch = input[++lexer->position])+    for (ch=input[lexer->position]; lexer->position < lexer->length; ch = input[++lexer->position]) {+      if (ch < 32 || ch > 126)+        hasspecialchar = 1;       if (lexer->state_data_2)         lexer->state_data_2 = 0;       else if (ch == '\\') {@@ -194,6 +196,7 @@         hasspecialchar = 1;       } else if (ch == '"')         break;+    }      struct lexer_result *res = &result[lexer->result_num];     res->startpos = startposition;@@ -203,9 +206,9 @@       if (lexer->state_data)         res->adddata = 1; // Indicate that we are final portion of the string       else if (hasspecialchar)-        res->adddata = 0; // Indicate that the string contains escaped characters+        res->adddata = 0; // Indicate that the string contains escaped/UTF-8 characters       else-        res->adddata = -1; // Indicate that the stirng is clean UTF-8 (optimization)+        res->adddata = -1; // Indicate that the stirng is clean ASCII (optimization)        lexer->result_num++;       lexer->current_state = STATE_BASE;

@@ -1,3 +1,9 @@+# 0.4.4.1++- added objectKeyValues (Dylan Simon)+- optimization for reading ASCII strings++ # 0.4.4.0  - added text 2.0 compatibility

@@ -1,5 +1,5 @@ name:                json-stream-version:             0.4.4.0+version:             0.4.4.1 synopsis:            Incremental applicative JSON parser description:         Easy to use JSON parser fully supporting incremental parsing.                      Parsing grammar in applicative form.

@@ -79,6 +79,14 @@         msg = parse (arrayOf $ (,) <$> objectWithKey "name" value <*> objectWithKey "age" value) test :: [(T.Text,Int)]     msg `shouldBe` [("John",20),("Frank",30)] +  it "objectKeyValues works" $ do+    let test = "[{'name': 'John', 'age': 20}, {'age': 30, 'name': 'Frank' } ]"+        item "name" = Left <$> string+        item "age" = Right <$> integer+        item _ = empty+        msg = parse (arrayOf $ objectKeyValues item) test :: [Either T.Text Int]+    msg `shouldBe` [Left "John",Right 20,Right 30,Left "Frank"]+   it "yield test 1" $ do     let test = "[{'key1': [1,2,3], 'key2': [5,6,7]}]"         msg1 = parse (arrayOf $ objectItems value) test :: [(T.Text, [Int])]