json-stream 0.4.2.4 → 0.4.3.0

raw patch · 6 files changed

+89/−24 lines, 6 filesdep ~aesonPVP: major bump suggested

API removals or changes: PVP suggests a major version bump

Dependency ranges changed: aeson

API changes (from Hackage documentation)

Files

• Data/JsonStream/CLexer.hs +5/−12
• Data/JsonStream/Parser.hs +59/−5
• Data/JsonStream/TokenParser.hs +12/−5
• changelog.md +5/−0
• json-stream.cabal +2/−2
• test/ParserSpec.hs +6/−0

@@ -20,7 +20,6 @@ import qualified Data.ByteString.Char8       as BS import           Data.ByteString.Unsafe      (unsafeUseAsCString) import           Data.Scientific             (Scientific, scientific)-import           Data.Text.Encoding          (decodeUtf8') import           Data.Text.Internal.Unsafe   (inlinePerformIO) import           Foreign import           Foreign.C.Types@@ -28,7 +27,7 @@  import           Data.JsonStream.CLexType import           Data.JsonStream.TokenParser (Element (..), TokenResult (..))-import           Data.JsonStream.Unescape+import           Data.JsonStream.Unescape (unescapeText)  -- | Limit for maximum size of a number; fail if larger number is found -- this is needed to make this constant-space, otherwise we would eat@@ -174,7 +173,7 @@  -- | Parse particular result parseResults :: TempData -> (CInt, Header, Int, ResultPtr) -> TokenResult-parseResults (TempData {tmpNumbers=tmpNumbers, tmpBuffer=bs}) (err, hdr, rescount, resptr) = parse 0+parseResults TempData{tmpNumbers=tmpNumbers, tmpBuffer=bs} (err, hdr, rescount, resptr) = parse 0   where     newtemp = TempData bs hdr (err /= 0)     -- We iterate the items from CNT to 1, 1 is the last element, CNT is the first@@ -218,14 +217,8 @@                        Just num -> PartialResult (JValue (AE.Number num)) next                        Nothing -> TokFailed         | resType == resString ->-          if | resAddData == -1 -> -- One-part string without escaped characters-                case decodeUtf8' textSection  of-                  Right ctext -> PartialResult (JValue (AE.String ctext)) next-                  Left _ -> TokFailed-             | resAddData == 0 -> -- One-part string with escaped characters-                case unescapeText textSection of-                  Right ctext -> PartialResult (JValue (AE.String ctext)) next-                  _ -> TokFailed+          if | resAddData == -1 || resAddData == 0 -> -- One-part string without escaped characters; with escaped+                PartialResult (StringRaw textSection) next              | otherwise -> PartialResult (StringContent textSection) -- Final part of partial strings                             (PartialResult StringEnd next)         | resType == resStringPartial ->@@ -237,7 +230,7 @@ estResultLimit dta = fromIntegral $ 20 + BS.length dta `quot` 5  getNextResult :: TempData -> TokenResult-getNextResult tmp@(TempData {..})+getNextResult tmp@TempData{..}   | tmpError = TokFailed   | hdrPosition tmpHeader < hdrLength tmpHeader = parseResults tmp (callLex tmpBuffer tmpHeader)   | otherwise = TokMoreData newdata

@@ -48,6 +48,7 @@     -- * FromJSON parser   , value   , string+  , byteString     -- * Constant space parsers   , safeString   , number@@ -55,6 +56,7 @@   , real   , bool   , jNull+  , safeByteString     -- * Structure operators   , (.:)   , (.:?)@@ -91,7 +93,13 @@ import qualified Data.ByteString.Lazy.Char8  as BL import qualified Data.ByteString.Lazy.Internal as BL import           Data.Char                   (isSpace)+#if MIN_VERSION_aeson(2,0,0)+import qualified Data.Aeson.KeyMap           as AEK+import qualified Data.Aeson.Key              as AEK+import           Data.Bifunctor              (first)+#else import qualified Data.HashMap.Strict         as HMap+#endif import           Data.Scientific             (Scientific, isInteger,                                               toBoundedInteger, toRealFloat) import qualified Data.Text                   as T@@ -307,12 +315,17 @@     (PartialResult ObjectBegin ntp) -> moreData (nextitem False) ntp     (PartialResult _ _) -> callParse ignoreVal tp -- Run ignoreval parser on the same output we got     (TokMoreData ntok) -> MoreData (object' once valparse, ntok)-    (TokFailed) -> Failed "Array - token failed"+    TokFailed -> Failed "Array - token failed"   where     nextitem _ _ (ObjectEnd ctx) ntok = Done ctx ntok-    nextitem yielded _ (JValue (AE.String key)) ntok = objcontent yielded (callParse (valparse key) ntok)+    nextitem yielded _ (JValue (AE.String key)) ntok =+      objcontent yielded (callParse (valparse key) ntok)+    nextitem yielded _ (StringRaw bs) ntok = +      case unescapeText bs of+        Right t -> objcontent yielded (callParse (valparse t) ntok)+        Left e -> Failed (show e)     nextitem yielded _ (StringContent str) ntok =-          objcontent yielded $ moreData (getLongKey [str] (BS.length str)) ntok+      objcontent yielded $ moreData (getLongKey [str] (BS.length str)) ntok     nextitem _ _ el _ = Failed $ "Object - unexpected item: " ++ show el      -- If we already yielded and should yield once, ignore the rest of the object@@ -368,13 +381,21 @@ aeValue :: Parser AE.Value aeValue = Parser $ moreData value'   where+#if MIN_VERSION_aeson(2,0,0)+    tomap = AEK.fromList . map (first AEK.fromText)+#else+    tomap = HMap.fromList+#endif     value' tok el ntok =       case el of         JValue val -> Yield val (Done "" ntok)         JInteger val -> Yield (AE.Number $ fromIntegral val) (Done "" ntok)         StringContent _ -> callParse (AE.String <$> longString Nothing) tok+        StringRaw bs -> case unescapeText bs of+              Right t -> Yield (AE.String t) (Done "" ntok)+              Left e -> Failed (show e)         ArrayBegin -> AE.Array . Vec.fromList <$> callParse (many (arrayOf aeValue)) tok-        ObjectBegin -> AE.Object . HMap.fromList <$> callParse (manyReverse (objectItems aeValue)) tok+        ObjectBegin -> AE.Object . tomap <$> callParse (manyReverse (objectItems aeValue)) tok         _ -> Failed ("aeValue - unexpected token: " ++ show el)  -- | Optimized function for aeson objects - evades reversing the objects@@ -402,13 +423,44 @@         _ -> callParse ignoreVal tok  +longByteString :: Maybe Int -> Parser BS.ByteString+longByteString mbounds = Parser $ moreData (handle id 0)+  where+    handle acc !len tok el ntok =+      case el of+        JValue (AE.String _) -> Failed "INTERNAL ERROR! - got decoded JValue instead of string"+        StringRaw bs -> Yield bs (Done "" ntok)+        StringContent str+          | (Just bounds) <- mbounds, len > bounds -- If the string exceeds bounds, discard it+                          -> callParse (ignoreStrRestThen (Parser $ Done "")) ntok+          | otherwise     -> moreData (handle (acc . (str:)) (len + BS.length str)) ntok+        StringEnd -> Yield (BS.concat (acc [])) (Done "" ntok)+        _ ->  callParse ignoreVal tok+++-- | Parse raw bytestring value (json string expected), skip parsing otherwise.+-- The returned value is not unescaped.+byteString :: Parser BS.ByteString+byteString = longByteString Nothing++-- | Stops parsing string after the limit is reached. The string will not be matched+-- if it exceeds the size. The size is the size of escaped string including escape+-- characters. +-- The return value is not unescaped.+safeByteString :: Int -> Parser BS.ByteString+safeByteString limit = longByteString (Just limit)+ -- | Match a possibly bounded string roughly limited by a limit longString :: Maybe Int -> Parser T.Text-longString mbounds = Parser $ moreData (handle (BS.empty :) 0)+longString mbounds = Parser $ moreData (handle id 0)   where     handle acc !len tok el ntok =       case el of         JValue (AE.String str) -> Yield str (Done "" ntok)+        StringRaw bs -> +          case unescapeText bs of+            Right t -> Yield t (Done "" ntok)+            Left e -> Failed (show e)         StringContent str           | (Just bounds) <- mbounds, len > bounds -- If the string exceeds bounds, discard it                           -> callParse (ignoreStrRestThen (Parser $ Done "")) ntok@@ -546,6 +598,7 @@      handleTok :: Int -> TokenResult -> Element -> TokenResult -> ParseResult a     handleTok 0 _ (JValue _) ntok = Done "" ntok+    handleTok 0 _ (StringRaw _) ntok = Done "" ntok     handleTok 0 _ (JInteger _) ntok = Done "" ntok     handleTok 0 _ (ArrayEnd _) _ = Failed "ArrayEnd in ignoreval on 0 level"     handleTok 0 _ (ObjectEnd _) _ = Failed "ObjectEnd in ignoreval on 0 level"@@ -556,6 +609,7 @@         JValue _ -> moreData (handleTok level) ntok         JInteger _ -> moreData (handleTok level) ntok         StringContent _ -> moreData (handleLongString level) ntok+        StringRaw _ -> moreData (handleTok level) ntok         ArrayEnd _ -> moreData (handleTok (level - 1)) ntok         ObjectEnd _ -> moreData (handleTok (level - 1)) ntok         ArrayBegin -> moreData (handleTok (level + 1)) ntok

@@ -9,14 +9,21 @@ import qualified Data.ByteString.Char8 as BS import           Foreign.C.Types -data Element = ArrayBegin | ArrayEnd BS.ByteString | ObjectBegin | ObjectEnd BS.ByteString-               | StringContent BS.ByteString | StringEnd-               | JValue AE.Value | JInteger CLong-               deriving (Show, Eq)+data Element = +    ArrayBegin+  | ArrayEnd !BS.ByteString+  | ObjectBegin+  | ObjectEnd !BS.ByteString+  | StringContent !BS.ByteString+  | StringRaw !BS.ByteString -- Allow raw strings to go into parser as bytestring+  | StringEnd+  | JValue !AE.Value+  | JInteger !CLong+  deriving (Show, Eq)  -- | Public interface for parsing JSON tokens. data TokenResult =  TokMoreData (BS.ByteString -> TokenResult)-                  | PartialResult Element (TokenResult)+                  | PartialResult Element TokenResult                   -- ^ found element, continuation, actual parsing view - so that we can report the unparsed                   -- data when the parsing finishes.                   | TokFailed

@@ -1,3 +1,8 @@+# 0.4.3.0++- Aeson 2.0 compatibility+- Added support for raw bytestring+ # 0.4.2.4  Fix compiling with new ghc.

@@ -1,5 +1,5 @@ name:                json-stream-version:             0.4.2.4+version:             0.4.3.0 synopsis:            Incremental applicative JSON parser description:         Easy to use JSON parser fully supporting incremental parsing.                      Parsing grammar in applicative form.@@ -41,7 +41,7 @@   build-depends:         base >=4.7 && <5                        , bytestring                        , text-                       , aeson >= 0.7+                       , aeson >= 0.7 && < 2.1                        , vector                        , unordered-containers                        , scientific

@@ -118,6 +118,12 @@         msg = parseLazyByteString parser (BL.fromChunks test) :: [Int]     msg `shouldBe` [10,1,20] +  it "Has working byteString parser" $ do+    let test = ["[\"abcd\\n\\rxyz\"]"]+        parser = arrayOf byteString :: Parser BS.ByteString+        msg = parseLazyByteString parser (BL.fromChunks test) :: [BS.ByteString]+    msg `shouldBe` ["abcd\\n\\rxyz"]+ specEdge :: Spec specEdge = describe "Edge cases" $ do   it "Correct incremental parsing 1" $ do