protocol-buffers 1.5.0 → 1.6.0
raw patch · 7 files changed
+381/−137 lines, 7 files
Files
- TODO +130/−0
- Text/ProtocolBuffers/Extensions.hs +173/−60
- Text/ProtocolBuffers/Header.hs +5/−3
- Text/ProtocolBuffers/Reflections.hs +23/−5
- Text/ProtocolBuffers/Unknown.hs +35/−59
- Text/ProtocolBuffers/WireMessage.hs +14/−9
- protocol-buffers.cabal +1/−1
TODO view
@@ -1,3 +1,133 @@+make changes to align with protocol-buffer 2.3.0+http://protobuf.googlecode.com/svn/trunk/CHANGES.txt++ General+ * Parsers for repeated numeric fields now always accept both packed and+ unpacked input. The [packed=true] option only affects serializers.+ Therefore, it is possible to switch a field to packed format without+ breaking backwards-compatibility -- as long as all parties are using+ protobuf 2.3.0 or above, at least.++Should be possible to do this.+Reflection:+ add "FieldInfo.packedTag :: Maybe WireTag"+ need to make FieldInfo.wireTag always unpacked+ note that FieldInfo.wireTagLength is the same for both++Above may work except for loading previously unknown packed extensions +Need to change Unknown.loadUnknown to tolerate the same field'Number with different wire types++MakeReflections:+ toFieldInfo'++Gen: allowed'wire'Tags gets both packed and unpacked codes for repeated+ update'Self must switch on wire'Tag not field'Number+ and have both packed and unpacked cases for repeated++Gen: Clean up the generated wireGet code.++currently:+ wireGet ft'+ = case ft' of+ 10 -> P'.getBareMessageWith check'allowed+ 11 -> P'.getMessageWith check'allowed+ _ -> P'.wireGetErr ft'+ where+ update'Self wire'Tag old'Self+ = case wire'Tag of+ 90 -> P'.wireGetKey UnittestProto.packed_int32_extension old'Self+ _ -> P'.unknownField old'Self (P'.fieldIdOf wire'Tag)++ check'allowed wire'Tag field'Number wire'Type old'Self+ = P'.catchError+ (if P'.member wire'Tag allowed'wire'Tags then update'Self wire'Tag old'Self else+ if P'.or [1 <= field'Number && field'Number <= 18999, 20000 <= field'Number] then+ P'.loadExtension (P'.fieldIdOf wire'Tag) wire'Type old'Self else+ P'.unknown (P'.fieldIdOf wire'Tag) wire'Type old'Self)+ (\ _ -> P'.loadUnknown (P'.fieldIdOf wire'Tag) wire'Type old'Self)++desired: + update'Self wire'Tag old'Self+ = case wire'Tag of+ 90 -> P'.wireGetKey UnittestProto.packed_int32_extension old'Self+ _ -> let (field'Number,wire'Type) = splitWireTag wire'Tag+ in if P'.or [1 <= field'Number && field'Number <= 18999, 20000 <= field'Number]+ then P'.loadExtension field'Number wire'Type old'Self+ else P'.unknown field'Number wire'Type old'Self+ check'allowed wire'Tag old'Self+ = P'.catchError (update'Self wire'Tag old'Self) (\ _ -> P'.loadUnknown wire'Tag old'Self)++without --unknown the code was++ check'allowed wire'Tag field'Number wire'Type old'Self+ = if P'.member wire'Tag allowed'wire'Tags then update'Self wire'Tag old'Self else+ if P'.or [1 <= field'Number && field'Number <= 18999, 20000 <= field'Number] then+ P'.loadExtension (P'.fieldIdOf wire'Tag) wire'Type old'Self else P'.unknown (P'.fieldIdOf wire'Tag) wire'Type old'Self+ +and should become ++ _ -> let (field'Number,wire'Type) = splitWireTag wire'Tag+ in if P'.or [1 <= field'Number && field'Number <= 18999, 20000 <= field'Number]+ then P'.loadExtension field'Number wire'Type old'Self+ else P'.unknown field'Number wire'Type old'Self++ check'allowed = update'Self++Note that "unknownField", which was impossible, is no longer present.+check'allowed and update'Self and loadUnknown just takes the wire'Tag and old'Self,+while loadExtension and unknown are unchanged.+And allowed'wire'Tags can be dropped altogether.++The check'allowed is no longer needed when there is no catchError!+Thus rename to catch'Unknown ? Still generate or put in library with INLINE ?+Put in library, now it will be++ wireGet ft'+ = case ft' of+ 10 -> P'.getBareMessageWith (catch'Unknown update'Self)+ 11 -> P'.getMessageWith (catch'Unknown update'Self)+ _ -> P'.wireGetErr ft'+++ * The generic RPC service code generated by the C++, Java, and Python+ generators can be disabled via file options:+ option cc_generic_services = false;+ option java_generic_services = false;+ option py_generic_services = false;+ This allows plugins to generate alternative code, possibly specific to some+ particular RPC implementation.++Heh. It is always false for hprotoc++ protoc+ * Now supports a plugin system for code generators. Plugins can generate+ code for new languages or inject additional code into the output of other+ code generators. Plugins are just binaries which accept a protocol buffer+ on stdin and write a protocol buffer to stdout, so they may be written in+ any language. See src/google/protobuf/compiler/plugin.proto.+ **WARNING**: Plugins are experimental. The interface may change in a+ future version.++Nope.++ * If the output location ends in .zip or .jar, protoc will write its output+ to a zip/jar archive instead of a directory. For example:+ protoc --java_out=myproto_srcs.jar --python_out=myproto.zip myproto.proto+ Currently the archive contents are not compressed, though this could change+ in the future.++Nope.++ * inf, -inf, and nan can now be used as default values for float and double+ fields.++Should be possible to do this.+Done; required changes to the Lexer.x to recognize "-inf" as a discrete symbol.+NOT TESTED++----++ Add even more of the documentation for the public API. delete commented out code add strictness annotations to internal data types.
Text/ProtocolBuffers/Extensions.hs view
@@ -21,10 +21,11 @@ getKeyFieldId,getKeyFieldType,getKeyDefaultValue -- * External types and classes , Key(..),ExtKey(..),MessageAPI(..)- , PackedSeq(..)+ , PackedSeq(..), EP(..) -- * Internal types, functions, and classes , wireSizeExtField,wirePutExtField,loadExtension,notExtension- , GPB,ExtField(..),ExtendMessage(..),ExtFieldValue(..)+ , wireGetKeyToUnPacked, wireGetKeyToPacked+ , GPB,ExtField(..),ExtendMessage(..),ExtFieldValue(..), ) where import Control.Monad.Error.Class(throwError)@@ -130,12 +131,15 @@ -- | The WireType is used to ensure the Seq is homogenous. -- The ByteString is the unparsed input after the tag.-data ExtFieldValue = ExtFromWire !WireType !(Seq ByteString)+data ExtFieldValue = ExtFromWire !(Seq EP) -- XXX must store wiretype with ByteString | ExtOptional !FieldType !GPDyn | ExtRepeated !FieldType !GPDynSeq | ExtPacked !FieldType !GPDynSeq deriving (Typeable,Ord,Show) +data EP = EP {-# UNPACK #-} !WireType {-# UNPACK #-} !ByteString+ deriving (Typeable,Eq,Ord,Show)+ data DummyMessageType deriving (Typeable) instance ExtendMessage DummyMessageType where@@ -152,36 +156,36 @@ -- were interpreted by the same Key that their resulting values would -- compare True. instance Eq ExtFieldValue where- (==) (ExtFromWire a b) (ExtFromWire a' b') = a==a' && b==b'+ (==) (ExtFromWire b) (ExtFromWire b') = b==b' (==) (ExtOptional a b) (ExtOptional a' b') = a==a' && b==b' (==) (ExtRepeated a b) (ExtRepeated a' b') = a==a' && b==b' (==) (ExtPacked a b) (ExtPacked a' b') = a==a' && b==b'- (==) x@(ExtOptional ft (GPDyn w@GPWitness _)) (ExtFromWire wt' s') =+ (==) x@(ExtOptional ft (GPDyn w@GPWitness _)) (ExtFromWire s') = let wt = toWireType ft- in wt==wt' && (let makeKeyType :: GPWitness a -> Key Maybe DummyMessageType a- makeKeyType = undefined- key = Key 0 ft Nothing `asTypeOf` makeKeyType w- in case parseWireExtMaybe key wt s' of- Right (_,y) -> x==y- _ -> False)+ makeKeyType :: GPWitness a -> Key Maybe DummyMessageType a+ makeKeyType = undefined+ key = Key 0 ft Nothing `asTypeOf` makeKeyType w+ in case parseWireExtMaybe key wt s' of+ Right (_,y) -> x==y+ _ -> False (==) y@(ExtFromWire {}) x@(ExtOptional {}) = x == y- (==) x@(ExtRepeated ft (GPDynSeq w@GPWitness _)) (ExtFromWire wt' s') =+ (==) x@(ExtRepeated ft (GPDynSeq w@GPWitness _)) (ExtFromWire s') = let wt = toWireType ft- in wt==wt' && (let makeKeyType :: GPWitness a -> Key Seq DummyMessageType a- makeKeyType = undefined- key = Key 0 ft Nothing `asTypeOf` makeKeyType w- in case parseWireExtSeq key wt s' of- Right (_,y) -> x==y- _ -> False)+ makeKeyType :: GPWitness a -> Key Seq DummyMessageType a+ makeKeyType = undefined+ key = Key 0 ft Nothing `asTypeOf` makeKeyType w+ in case parseWireExtSeq key wt s' of+ Right (_,y) -> x==y+ _ -> False (==) y@(ExtFromWire {}) x@(ExtRepeated {}) = x == y- (==) x@(ExtPacked ft (GPDynSeq w@GPWitness _)) (ExtFromWire wt' s') =+ (==) x@(ExtPacked ft (GPDynSeq w@GPWitness _)) (ExtFromWire s') = let wt = 2 -- all packed types have wire type 2, length delimited- in wt == wt' && (let makeKeyType :: GPWitness a -> Key PackedSeq DummyMessageType a- makeKeyType = undefined- key = Key 0 ft Nothing `asTypeOf` makeKeyType w- in case parseWireExtPackedSeq key wt s' of- Right (_,y) -> x==y- _ -> False)+ makeKeyType :: GPWitness a -> Key PackedSeq DummyMessageType a+ makeKeyType = undefined+ key = Key 0 ft Nothing `asTypeOf` makeKeyType w+ in case parseWireExtPackedSeq key wt s' of+ Right (_,y) -> x==y+ _ -> False (==) y@(ExtFromWire {}) x@(ExtPacked {}) = x == y (==) _ _ = False @@ -200,6 +204,68 @@ putExtField :: ExtField -> msg -> msg validExtRanges :: msg -> [(FieldId,FieldId)] ++-- wireKeyToUnPacked is used to load a repeated packed format into a repeated non-packed extension key+-- wireKetToPacked is used to load a repeated unpacked format into a repeated packed extension key+wireGetKeyToUnPacked :: (ExtendMessage msg,GPB v) => Key Seq msg v -> msg -> Get msg+wireGetKeyToUnPacked k@(Key i t mv) msg = do+ let myCast :: Maybe a -> Get (Seq a)+ myCast = undefined+ vv <- wireGetPacked t `asTypeOf` (myCast mv)+ let (ExtField ef) = getExtField msg+ v' <- case M.lookup i ef of+ Nothing -> return $ ExtRepeated t (GPDynSeq GPWitness vv)+ Just (ExtRepeated t' (GPDynSeq GPWitness s)) | t/=t' ->+ fail $ "wireGetKeyToUnPacked: Key mismatch! found wrong field type: "++show (k,t,t')+ | otherwise ->+ case cast s of+ Nothing -> fail $ "wireGetKeyToUnPacked: previous Seq value cast failed: "++show (k,typeOf s)+ Just s' -> return $ ExtRepeated t (GPDynSeq GPWitness (s' >< vv))+ Just (ExtFromWire raw) ->+ case parseWireExtSeq k (toWireType t) raw of -- was wt from ExtFromWire+ Left errMsg -> fail $ "wireGetKeyToUnPacked: Could not parseWireExtSeq: "++show k++"\n"++errMsg+ Right (_,ExtRepeated t' (GPDynSeq GPWitness s)) | t/=t' ->+ fail $ "wireGetKeyToUnPacked:: Key mismatch! parseWireExtSeq returned wrong field type: "++show (k,t,t')+ | otherwise ->+ case cast s of+ Nothing -> fail $ "wireGetKey Seq: previous Seq value cast failed: "++show (k,typeOf s)+ Just s' -> return $ ExtRepeated t (GPDynSeq GPWitness (s' >< vv))+ wtf -> fail $ "wireGetKeyToUnPacked: Weird parseWireExtSeq return value: "++show (k,wtf)+ Just wtf@(ExtOptional {}) -> fail $ "wireGetKeyToUnPacked: ExtOptional found when ExtRepeated expected: "++show (k,wtf)+ Just wtf@(ExtPacked {}) -> fail $ "wireGetKeyToUnPacked: ExtPacked found when ExtRepeated expected: "++show (k,wtf)+ let ef' = M.insert i v' ef+ seq v' $ seq ef' $ return (putExtField (ExtField ef') msg)++wireGetKeyToPacked :: (ExtendMessage msg,GPB v) => Key PackedSeq msg v -> msg -> Get msg+wireGetKeyToPacked k@(Key i t mv) msg = do+ let wt = toWireType t+ myCast :: Maybe a -> Get a+ myCast = undefined+ v <- wireGet t `asTypeOf` (myCast mv)+ let (ExtField ef) = getExtField msg+ v' <- case M.lookup i ef of+ Nothing -> return $ ExtPacked t (GPDynSeq GPWitness (Seq.singleton v))+ Just (ExtPacked t' (GPDynSeq GPWitness s)) | t/=t' ->+ fail $ "wireGetKeyToPacked: Key mismatch! found wrong field type: "++show (k,t,t')+ | otherwise ->+ case cast s of+ Nothing -> fail $ "wireGetKeyToPacked: previous Seq value cast failed: "++show (k,typeOf s)+ Just s' -> return $ ExtRepeated t (GPDynSeq GPWitness (s' |> v))+ Just (ExtFromWire raw) ->+ case parseWireExtPackedSeq k wt raw of+ Left errMsg -> fail $ "wireGetKeyToPacked: Could not parseWireExtPackedSeq: "++show k++"\n"++errMsg+ Right (_,ExtPacked t' (GPDynSeq GPWitness s)) | t/=t' ->+ fail $ "wireGetKeyToPacked: Key mismatch! parseWireExtPackedSeq returned wrong field type: "++show (k,t,t')+ | otherwise ->+ case cast s of+ Nothing -> fail $ "wireGetKeyToPacked: previous Seq value cast failed: "++show (k,typeOf s)+ Just s' -> return $ ExtRepeated t (GPDynSeq GPWitness (s' |> v))+ wtf -> fail $ "wireGetKeyToPacked: Weird parseWireExtPackedSeq return value: "++show (k,wtf)+ Just wtf@(ExtOptional {}) -> fail $ "wireGetKeyToPacked: ExtOptional found when ExtPacked expected: "++show (k,wtf)+ Just wtf@(ExtRepeated {}) -> fail $ "wireGetKeyToPacked: ExtRepeated found when ExtPacked expected: "++show (k,wtf)+ let ef' = M.insert i v' ef+ seq v' $ seq ef' $ return (putExtField (ExtField ef') msg)+ -- | The 'ExtKey' class has three functions for user of the API: -- 'putExt', 'getExt', and 'clearExt'. The 'wireGetKey' is used in -- generated code.@@ -262,9 +328,7 @@ mergeAppend (ExtField m1) (ExtField m2) = ExtField (M.unionWith mergeExtFieldValue m1 m2) mergeExtFieldValue :: ExtFieldValue -> ExtFieldValue -> ExtFieldValue-mergeExtFieldValue (ExtFromWire wt1 s1) (ExtFromWire wt2 s2) =- if wt1 /= wt2 then err $ "mergeExtFieldValue : ExtFromWire WireType mismatch " ++ show (wt1,wt2)- else ExtFromWire wt2 (mappend s1 s2)+mergeExtFieldValue (ExtFromWire s1) (ExtFromWire s2) = ExtFromWire (mappend s1 s2) mergeExtFieldValue (ExtOptional ft1 (GPDyn GPWitness d1)) (ExtOptional ft2 (GPDyn GPWitness d2)) =@@ -403,10 +467,11 @@ in seq ef' (putExtField (ExtField ef') msg) getExt k@(Key i t _) msg =- let (ExtField ef) = getExtField msg+ let wt = toWireType t+ (ExtField ef) = getExtField msg in case M.lookup i ef of Nothing -> Right Nothing- Just (ExtFromWire wt raw) -> either Left (getExt' . snd) (parseWireExtMaybe k wt raw)+ Just (ExtFromWire raw) -> either Left (getExt' . snd) (parseWireExtMaybe k wt raw) Just x -> getExt' x where getExt' (ExtRepeated t' _) = Left $ "getExt Maybe: ExtField has repeated type: "++show (k,t') getExt' (ExtPacked t' _) = Left $ "getExt Maybe: ExtField has packed type: "++show (k,t')@@ -419,7 +484,8 @@ getExt' (ExtFromWire {}) = err $ "Impossible? getExt.getExt' Maybe should not have ExtFromWire case (after parseWireExt)!" wireGetKey k@(Key i t mv) msg = do- let myCast :: Maybe a -> Get a+ let wt = toWireType t+ myCast :: Maybe a -> Get a myCast = undefined v <- wireGet t `asTypeOf` (myCast mv) let (ExtField ef) = getExtField msg@@ -431,12 +497,12 @@ case cast vOld of Nothing -> fail $ "wireGetKey Maybe: previous Maybe value case failed: "++show (k,typeOf vOld) Just vOld' -> return $ ExtOptional t (GPDyn GPWitness (mergeAppend vOld' v))- Just (ExtFromWire wt raw) ->+ Just (ExtFromWire raw) -> case parseWireExtMaybe k wt raw of Left errMsg -> fail $ "wireGetKey Maybe: Could not parseWireExtMaybe: "++show k++"\n"++errMsg Right (_,ExtOptional t' (GPDyn GPWitness vOld)) | t/=t' -> fail $ "wireGetKey Maybe: Key mismatch! found wrong field type: "++show (k,t,t')- | otherwise ->+ | otherwise -> case cast vOld of Nothing -> fail $ "wireGetKey Maybe: previous Maybe value case failed: "++show (k,typeOf vOld) Just vOld' -> return $ ExtOptional t (GPDyn GPWitness (mergeAppend vOld' v))@@ -447,18 +513,30 @@ seq v' $ seq ef' $ return (putExtField (ExtField ef') msg) -- | used by 'getVal' and 'wireGetKey' for the 'Maybe' instance-parseWireExtMaybe :: Key Maybe msg v -> WireType -> Seq ByteString -> Either String (FieldId,ExtFieldValue)+parseWireExtMaybe :: Key Maybe msg v -> WireType -> Seq EP -> Either String (FieldId,ExtFieldValue) parseWireExtMaybe k@(Key fi ft mv) wt raw | wt /= toWireType ft = Left $ "parseWireExt Maybe: Key's FieldType does not match ExtField's wire type: "++show (k,toWireType ft,wt) | otherwise = do let mkWitType :: Maybe a -> GPWitness a mkWitType = undefined witness = GPWitness `asTypeOf` (mkWitType mv)- parsed = map (applyGet (wireGet ft)) . F.toList $ raw+-- parsed = map (applyGet (wireGet ft)) . F.toList $ raw+ parsed = map (chooseGet ft) . F.toList $ raw errs = [ m | Left m <- parsed ]- if null errs then Right (fi,(ExtOptional ft (GPDyn witness (mergeConcat [ a | Right a <- parsed ]))))+ if null errs then Right (fi,(ExtOptional ft (GPDyn witness (mergeConcat . mconcat $ [ a | Right a <- parsed ])))) else Left (unlines errs) +-- 'chooseGet' is an intermediate handler between parseWireExt* and applyGet. This does not know+-- whether the EP will result in a single r or repeat r, so it always returns a Seq. It may also+-- realize that there is a mismatch between the derired FieldType and the WireType+chooseGet :: (Wire r) => FieldType -> EP -> Either String (Seq r)+chooseGet ft (EP wt bsIn) =+ if (2==wt) && (isValidPacked ft)+ then applyGet (wireGetPacked ft) bsIn+ else if (wt == toWireType ft)+ then applyGet (fmap Seq.singleton $ wireGet ft) bsIn+ else Left $ "Text.ProtocolBuffers.Extensions.chooseGet: wireType mismatch "++show(wt,ft)+ -- | Converts the the 'Result' into an 'Either' type and enforces -- consumption of entire 'ByteString'. Used by 'parseWireExtMaybe' -- and 'parseWireExtSeq' to process raw wire input that has been@@ -485,10 +563,11 @@ in seq ef' (putExtField (ExtField ef') msg) getExt k@(Key i t _) msg =- let (ExtField ef) = getExtField msg+ let wt = toWireType t+ (ExtField ef) = getExtField msg in case M.lookup i ef of Nothing -> Right Seq.empty- Just (ExtFromWire wt raw) -> either Left (getExt' . snd) (parseWireExtSeq k wt raw)+ Just (ExtFromWire raw) -> either Left (getExt' . snd) (parseWireExtSeq k wt raw) Just x -> getExt' x where getExt' (ExtOptional t' _) = Left $ "getExt Seq: ExtField has optional type: "++show (k,t') getExt' (ExtPacked t' _) = Left $ "getExt Seq: ExtField has packed type: "++show (k,t')@@ -505,7 +584,8 @@ -- to. All sanity checks are included below. TODO: do enough -- testing to be confident in removing some checks. wireGetKey k@(Key i t mv) msg = do- let myCast :: Maybe a -> Get a+ let wt = toWireType t+ myCast :: Maybe a -> Get a myCast = undefined v <- wireGet t `asTypeOf` (myCast mv) let (ExtField ef) = getExtField msg@@ -517,7 +597,7 @@ case cast s of Nothing -> fail $ "wireGetKey Seq: previous Seq value cast failed: "++show (k,typeOf s) Just s' -> return $ ExtRepeated t (GPDynSeq GPWitness (s' |> v))- Just (ExtFromWire wt raw) ->+ Just (ExtFromWire raw) -> case parseWireExtSeq k wt raw of Left errMsg -> fail $ "wireGetKey Seq: Could not parseWireExtSeq: "++show k++"\n"++errMsg Right (_,ExtRepeated t' (GPDynSeq GPWitness s)) | t/=t' ->@@ -533,16 +613,17 @@ seq v' $ seq ef' $ return (putExtField (ExtField ef') msg) -- | used by 'getVal' and 'wireGetKey' for the 'Seq' instance-parseWireExtSeq :: Key Seq msg v -> WireType -> Seq ByteString -> Either String (FieldId,ExtFieldValue)+parseWireExtSeq :: Key Seq msg v -> WireType -> Seq EP -> Either String (FieldId,ExtFieldValue) parseWireExtSeq k@(Key i t mv) wt raw | wt /= toWireType t = Left $ "parseWireExtSeq: Key mismatch! Key's FieldType does not match ExtField's wire type: "++show (k,toWireType t,wt) | otherwise = do let mkWitType :: Maybe a -> GPWitness a mkWitType = undefined witness = GPWitness `asTypeOf` (mkWitType mv)- parsed = map (applyGet (wireGet t)) . F.toList $ raw+-- parsed = map (applyGet (wireGet t)) . F.toList $ raw+ parsed = map (chooseGet t) . F.toList $ raw errs = [ m | Left m <- parsed ]- if null errs then Right (i,(ExtRepeated t (GPDynSeq witness (Seq.fromList [ a | Right a <- parsed ]))))+ if null errs then Right (i,(ExtRepeated t (GPDynSeq witness (mconcat [ a | Right a <- parsed ])))) else Left (unlines errs) instance ExtKey PackedSeq where@@ -559,10 +640,11 @@ in seq ef' (putExtField (ExtField ef') msg) getExt k@(Key i t _) msg =- let (ExtField ef) = getExtField msg+ let wt = toWireType t+ (ExtField ef) = getExtField msg in case M.lookup i ef of Nothing -> Right (PackedSeq Seq.empty)- Just (ExtFromWire wt raw) -> either Left (getExt' . snd) (parseWireExtPackedSeq k wt raw)+ Just (ExtFromWire raw) -> either Left (getExt' . snd) (parseWireExtPackedSeq k wt raw) Just x -> getExt' x where getExt' (ExtOptional t' _) = Left $ "getExt PackedSeq: ExtField has optional type: "++show (k,t') getExt' (ExtRepeated t' _) = Left $ "getExt PackedSeq: ExtField has repeated type: "++show (k,t')@@ -575,7 +657,8 @@ getExt' (ExtFromWire {}) = err $ "Impossible? getExt.getExt' PackedSeq should not have ExtFromWire case (after parseWireExtSeq)!" wireGetKey k@(Key i t mv) msg = do- let myCast :: Maybe a -> Get (Seq a)+ let wt = toWireType t+ myCast :: Maybe a -> Get (Seq a) myCast = undefined vv <- wireGetPacked t `asTypeOf` (myCast mv) let (ExtField ef) = getExtField msg@@ -587,7 +670,7 @@ case cast s of Nothing -> fail $ "wireGetKey PackedSeq: previous Seq value cast failed: "++show (k,typeOf s) Just s' -> return $ ExtRepeated t (GPDynSeq GPWitness (s' >< vv))- Just (ExtFromWire wt raw) ->+ Just (ExtFromWire raw) -> case parseWireExtPackedSeq k wt raw of Left errMsg -> fail $ "wireGetKey PackedSeq: Could not parseWireExtPackedSeq: "++show k++"\n"++errMsg Right (_,ExtPacked t' (GPDynSeq GPWitness s)) | t/=t' ->@@ -598,18 +681,20 @@ Just s' -> return $ ExtRepeated t (GPDynSeq GPWitness (s' >< vv)) wtf -> fail $ "wireGetKey PackedSeq: Weird parseWireExtPackedSeq return value: "++show (k,wtf) Just wtf@(ExtOptional {}) -> fail $ "wireGetKey PackedSeq: ExtOptional found when ExtPacked expected: "++show (k,wtf)+-- XXX XXX XXX 2.3.0 need to add handling to the next line? Just wtf@(ExtRepeated {}) -> fail $ "wireGetKey PackedSeq: ExtRepeated found when ExtPacked expected: "++show (k,wtf) let ef' = M.insert i v' ef seq v' $ seq ef' $ return (putExtField (ExtField ef') msg) -parseWireExtPackedSeq :: Key PackedSeq msg v -> WireType -> Seq ByteString -> Either String (FieldId,ExtFieldValue)+parseWireExtPackedSeq :: Key PackedSeq msg v -> WireType -> Seq EP -> Either String (FieldId,ExtFieldValue) parseWireExtPackedSeq k@(Key i t mv) wt raw | wt /= 2 {- packed wire type is 2, length delimited -} = Left $ "parseWireExtPackedSeq: Key mismatch! Key's FieldType does not match ExtField's wire type: "++show (k,toWireType t,wt) | otherwise = do let mkWitType :: Maybe a -> GPWitness a mkWitType = undefined witness = GPWitness `asTypeOf` (mkWitType mv)- parsed = map (applyGet (wireGetPacked t)) . F.toList $ raw+-- parsed = map (applyGet (wireGetPacked t)) . F.toList $ raw+ parsed = map (chooseGet t) . F.toList $ raw errs = [ m | Left m <- parsed ] if null errs then Right (i,(ExtPacked t (GPDynSeq witness (mconcat [ a | Right a <- parsed ])))) else Left (unlines errs)@@ -617,9 +702,14 @@ -- | This is used by the generated code wireSizeExtField :: ExtField -> WireSize wireSizeExtField (ExtField m) = F.foldl' aSize 0 (M.assocs m) where- aSize old (fi,(ExtFromWire wt raw)) = old ++ aSize old (fi,(ExtFromWire raw)) =+ let toSize (EP wt bs) = (size'Varint (getWireTag (mkWireTag fi wt))) + L.length bs+ in F.foldl' (\oldVal new -> oldVal + toSize new) old raw+{-+ aSize old (fi,(ExtFromWire raw)) = old + let tagSize = size'Varint (getWireTag (mkWireTag fi wt)) in F.foldl' (\oldVal new -> oldVal + L.length new) (fromIntegral (Seq.length raw) * tagSize) raw+-} aSize old (fi,(ExtOptional ft (GPDyn GPWitness d))) = old + let tagSize = size'Varint (getWireTag (toWireTag fi ft)) in wireSizeReq tagSize ft d@@ -634,7 +724,7 @@ -- order of increasing field number. wirePutExtField :: ExtField -> Put wirePutExtField (ExtField m) = mapM_ aPut (M.assocs m) where- aPut (fi,(ExtFromWire wt raw)) = F.mapM_ (\bs -> putVarUInt (getWireTag $ mkWireTag fi wt) >> putLazyByteString bs) raw+ aPut (fi,(ExtFromWire raw)) = F.mapM_ (\(EP wt bs) -> putVarUInt (getWireTag $ mkWireTag fi wt) >> putLazyByteString bs) raw aPut (fi,(ExtOptional ft (GPDyn GPWitness d))) = wirePutOpt (toWireTag fi ft) ft (Just d) aPut (fi,(ExtRepeated ft (GPDynSeq GPWitness s))) = wirePutRep (toWireTag fi ft) ft s aPut (fi,(ExtPacked ft (GPDynSeq GPWitness s))) = wirePutPacked (toPackedWireTag fi) ft s@@ -642,10 +732,10 @@ notExtension :: (ReflectDescriptor a, ExtendMessage a,Typeable a) => FieldId -> WireType -> a -> Get a notExtension fieldId _wireType msg = throwError ("Field id "++show fieldId++" is not a valid extension field id for "++show (typeOf (undefined `asTypeOf` msg))) --- | get a value from the wire into the message's ExtField. This is--- used by 'getMessageExt' and 'getBareMessageExt' above.+-- | get a value from the wire into the message's ExtField. This is used by generated code for+-- extensions that were not known at compile time. loadExtension :: (ReflectDescriptor a, ExtendMessage a) => FieldId -> WireType -> a -> Get a---loadExtension fieldId wireType msg | isValidExt fieldId msg = do -- XXX+--loadExtension fieldId wireType msg | isValidExt fieldId msg = do -- XXX check moved to generated code --loadExtension fieldId wireType msg = unknown fieldId wireType msg -- XXX loadExtension fieldId wireType msg = do let (ExtField ef) = getExtField msg@@ -655,13 +745,12 @@ case M.lookup fieldId ef of Nothing -> do bs <- wireGetFromWire fieldId wireType- let v' = ExtFromWire wireType (Seq.singleton bs)+ let v' = ExtFromWire (Seq.singleton (EP wireType bs)) ef' = M.insert fieldId v' ef seq v' $ seq ef' $ return $ putExtField (ExtField ef') msg- Just (ExtFromWire wt raw) | wt /= wireType -> badwt wt- | otherwise -> do+ Just (ExtFromWire raw) -> do bs <- wireGetFromWire fieldId wireType- let v' = ExtFromWire wt (raw |> bs)+ let v' = ExtFromWire (raw |> (EP wireType bs)) ef' = M.insert fieldId v' ef seq v' $ seq ef' $ return (putExtField (ExtField ef') msg) Just (ExtOptional ft (GPDyn x@GPWitness a)) | toWireType ft /= wireType -> badwt (toWireType ft)@@ -670,13 +759,27 @@ let v' = ExtOptional ft (GPDyn x (mergeAppend a b)) ef' = M.insert fieldId v' ef seq v' $ seq ef' $ return (putExtField (ExtField ef') msg)- Just (ExtRepeated ft (GPDynSeq x@GPWitness s)) | toWireType ft /= wireType -> badwt (toWireType ft)+-- handle wireType of "2" when toWireType ft is not "2" but ft could be packed by using wireGetPacked ft+ Just (ExtRepeated ft (GPDynSeq x@GPWitness s)) | toWireType ft /= wireType -> if (wireType==2) && (isValidPacked ft) + then do+ aa <- wireGetPacked ft+ let v' = ExtRepeated ft (GPDynSeq x (s >< aa))+ ef' = M.insert fieldId v' ef+ seq v' $ seq ef' $ return (putExtField (ExtField ef') msg)+ else badwt (toWireType ft) | otherwise -> do a <- wireGet ft let v' = ExtRepeated ft (GPDynSeq x (s |> a)) ef' = M.insert fieldId v' ef seq v' $ seq ef' $ return (putExtField (ExtField ef') msg)- Just (ExtPacked ft (GPDynSeq x@GPWitness s)) | 2 /= wireType -> badwt 2 {- packed uses length delimited: 2 -}+-- handle wireType of NOT "2" when wireType is good match for ft by using wireGet ft+ Just (ExtPacked ft (GPDynSeq x@GPWitness s)) | 2 /= wireType -> if (toWireType ft) == wireType+ then do+ a <- wireGet ft+ let v' = ExtPacked ft (GPDynSeq x (s |> a))+ ef' = M.insert fieldId v' ef+ seq v' $ seq ef' $ return (putExtField (ExtField ef') msg)+ else badwt 2 {- packed uses length delimited: 2 -} | otherwise -> do aa <- wireGetPacked ft let v' = ExtPacked ft (GPDynSeq x (s >< aa))@@ -748,3 +851,13 @@ instance MessageAPI msg (msg -> Int64) Int64 where getVal m f = f m instance MessageAPI msg (msg -> Word32) Word32 where getVal m f = f m instance MessageAPI msg (msg -> Word64) Word64 where getVal m f = f m++-- Must keep synchronized with Parser.isValidPacked+isValidPacked :: FieldType -> Bool+isValidPacked fieldType =+ case fieldType of+ 9 -> False+ 10 -> False+ 11 -> False -- Impossible value for typeCode from parseType, but here for completeness+ 12 -> False+ _ -> True
Text/ProtocolBuffers/Header.hs view
@@ -30,6 +30,7 @@ import Text.ProtocolBuffers.Default() import Text.ProtocolBuffers.Extensions ( wireSizeExtField,wirePutExtField,loadExtension,notExtension+ , wireGetKeyToUnPacked, wireGetKeyToPacked , GPB,Key(..),ExtField,ExtendMessage(..),MessageAPI(..),ExtKey(wireGetKey),PackedSeq ) import Text.ProtocolBuffers.Identifiers(FIName(..),MName(..),FName(..)) import Text.ProtocolBuffers.Mergeable()@@ -37,14 +38,15 @@ ( ReflectDescriptor(..),ReflectEnum(..),EnumInfo(..),ProtoName(..) , GetMessageInfo(GetMessageInfo),DescriptorInfo(extRanges),makePNF ) import Text.ProtocolBuffers.Unknown- ( UnknownField,UnknownMessage(..),wireSizeUnknownField,wirePutUnknownField,loadUnknown )+ ( UnknownField,UnknownMessage(..),wireSizeUnknownField,wirePutUnknownField,catch'Unknown ) import Text.ProtocolBuffers.WireMessage- ( prependMessageSize,putSize+ ( prependMessageSize,putSize,splitWireTag , wireSizeReq,wireSizeOpt,wireSizeRep , wirePutReq,wirePutOpt,wirePutRep , getMessageWith,getBareMessageWith,wireGetEnum,wireGetPackedEnum , wireSizeErr,wirePutErr,wireGetErr- , unknown,unknownField)+ , unknown,unknownField+ , fieldIdOf) {-# INLINE append #-} append :: Seq a -> a -> Seq a
Text/ProtocolBuffers/Reflections.hs view
@@ -12,9 +12,9 @@ -- is put into the top level module created by hprotoc. module Text.ProtocolBuffers.Reflections ( ProtoName(..),ProtoFName(..),ProtoInfo(..),DescriptorInfo(..),FieldInfo(..),KeyInfo- , HsDefault(..),EnumInfo(..),EnumInfoApp+ , HsDefault(..),SomeRealFloat(..),EnumInfo(..),EnumInfoApp , ReflectDescriptor(..),ReflectEnum(..),GetMessageInfo(..)- , makePNF+ , makePNF, toRF, fromRF ) where import Text.ProtocolBuffers.Basic@@ -94,11 +94,13 @@ data FieldInfo = FieldInfo { fieldName :: ProtoFName , fieldNumber :: FieldId- , wireTag :: WireTag+ , wireTag :: WireTag -- ^ Used for writing and reading if packedTag is Nothing+ , packedTag :: Maybe (WireTag,WireTag) -- ^ used for reading when Just {} instead of wireTag , wireTagLength :: WireSize -- ^ Bytes required in the Varint formatted wireTag , isPacked :: Bool , isRequired :: Bool- , canRepeat :: Bool+ , canRepeat :: Bool -- ^ True if repeated is the field type+ , mightPack :: Bool -- ^ True if packed would be valid for this field type , typeCode :: FieldType -- ^ fromEnum of Text.DescriptorProtos.FieldDescriptorProto.Type , typeName :: Maybe ProtoName -- ^ Set for Messages,Groups,and Enums , hsRawDefault :: Maybe ByteString -- ^ crappy, but not escaped, thing@@ -114,10 +116,26 @@ -- 'ByteString' here as this is sufficient for code generation. data HsDefault = HsDef'Bool Bool | HsDef'ByteString ByteString- | HsDef'Rational Rational+ | HsDef'RealFloat SomeRealFloat | HsDef'Integer Integer | HsDef'Enum String deriving (Show,Read,Eq,Ord,Data,Typeable)++-- | 'SomeRealFloat' projects Double/Float to Rational or a special IEEE type.+-- This is needed to track protobuf-2.3.0 which allows nan and inf and -inf default values.+data SomeRealFloat = SRF'Rational Rational | SRF'nan | SRF'ninf | SRF'inf+ deriving (Show,Read,Eq,Ord,Data,Typeable)++toRF :: (RealFloat a, Fractional a) => SomeRealFloat -> a+toRF (SRF'Rational r) = fromRational r+toRF SRF'nan = (0/0)+toRF SRF'ninf = (-1/0)+toRF SRF'inf = (1/0)++fromRF :: (RealFloat a, Fractional a) => a -> SomeRealFloat+fromRF x | isNaN x = SRF'nan+ | isInfinite x = if 0 < x then SRF'inf else SRF'ninf+ | otherwise = SRF'Rational (toRational x) data EnumInfo = EnumInfo { enumName :: ProtoName , enumFilePath :: [FilePath]
Text/ProtocolBuffers/Unknown.hs view
@@ -1,91 +1,67 @@--- | This module add unknown field support to the library. There are--- no user API things here.---- This should support--- 1) Storing unknown bytestrings in messages--- a) Mergeable--- b) Default--- c) Show--- 2) loading the unknown bytestrings into a (Map FieldId) from wire--- a) If wiretypes differ this is an error so report it--- b) Take extra care to ensure a _copy_ of the input is kept (?)--- 3) save unknown bytestring back to the wire--- 4) API ?--- a) Provide ability to "wireGet" the data as a real type--- b) clear the data--- c) has any unkown data ?--- 5) Extend reflection to indicate presence of support for unkown data--- 6) Extend Options and command line to flag this--- 7) Extend hprotoc to add in this field+-- | This module add unknown field support to the library. There are no user API things here,+-- except for advanced spelunking into the data structures which can and have changed with no+-- notice. Importer beware. module Text.ProtocolBuffers.Unknown- ( UnknownField(..),UnknownMessage(..),UnknownFieldValue(..),wireSizeUnknownField,wirePutUnknownField,loadUnknown+ ( UnknownField(..),UnknownMessage(..),UnknownFieldValue(..)+ , wireSizeUnknownField,wirePutUnknownField,catch'Unknown ) where import qualified Data.ByteString.Lazy as L import qualified Data.Foldable as F import Data.Generics-import Data.Map(Map)-import qualified Data.Map as M-import Data.Monoid(mappend)+import Data.Monoid(mempty,mappend) import Data.Sequence(Seq,(|>)) import qualified Data.Sequence as Seq import Data.Typeable+import Control.Monad.Error.Class(catchError) import Text.ProtocolBuffers.Basic import Text.ProtocolBuffers.WireMessage-import Text.ProtocolBuffers.Get as Get (Get,bytesRead)+import Text.ProtocolBuffers.Get as Get (Get) -err :: String -> b-err msg = error $ "Text.ProtocolBuffers.Unknown error\n"++msg+-- err :: String -> b+-- err msg = error $ "Text.ProtocolBuffers.Unknown error\n"++msg +-- | Messages that can store unknown fields implement this interface.+-- UnknownField is a supposedly opaque type. class UnknownMessage msg where getUnknownField :: msg -> UnknownField putUnknownField :: UnknownField -> msg -> msg -newtype UnknownField = UnknownField (Map FieldId UnknownFieldValue)+-- | This is a suposedly opaque type+newtype UnknownField = UnknownField (Seq UnknownFieldValue) deriving (Eq,Ord,Show,Read,Data,Typeable) -data UnknownFieldValue = UFV !WireType !(Seq ByteString)+data UnknownFieldValue = UFV {-# UNPACK #-} !WireTag {-# UNPACK #-} !ByteString deriving (Eq,Ord,Show,Read,Data,Typeable) instance Mergeable UnknownField where- mergeEmpty = UnknownField M.empty- mergeAppend (UnknownField m1) (UnknownField m2) = UnknownField (M.unionWith mergeUnknownFieldValue m1 m2)--mergeUnknownFieldValue :: UnknownFieldValue -> UnknownFieldValue -> UnknownFieldValue-mergeUnknownFieldValue (UFV wt1 s1) (UFV wt2 s2) =- if wt1 /= wt2 then err $ "mergeUnknownFieldValue: WireType mismatch "++show (wt1,wt2)- else UFV wt2 (mappend s1 s2)+ mergeEmpty = UnknownField mempty+ mergeAppend (UnknownField m1) (UnknownField m2) = UnknownField (mappend m1 m2) instance Default UnknownField where- defaultValue = UnknownField M.empty+ defaultValue = UnknownField mempty -- | This is used by the generated code wireSizeUnknownField :: UnknownField -> WireSize-wireSizeUnknownField (UnknownField m) = F.foldl' aSize 0 (M.assocs m) where- aSize old (fi,(UFV wt raw)) = old +- let tagSize = size'Varint (getWireTag (mkWireTag fi wt))- in F.foldl' (\oldVal new -> oldVal + L.length new) (fromIntegral (Seq.length raw) * tagSize) raw+wireSizeUnknownField (UnknownField m) = F.foldl' aSize 0 m where+ aSize old (UFV tag bs) = old + size'Varint (getWireTag tag) + L.length bs -- | This is used by the generated code wirePutUnknownField :: UnknownField -> Put-wirePutUnknownField (UnknownField m) = mapM_ aPut (M.assocs m) where- aPut (fi,(UFV wt raw)) = F.mapM_ (\bs -> putVarUInt (getWireTag $ mkWireTag fi wt) >> putLazyByteString bs) raw+wirePutUnknownField (UnknownField m) = F.mapM_ aPut m where+ aPut (UFV tag bs) = putVarUInt (getWireTag tag) >> putLazyByteString bs -loadUnknown :: (Typeable a, UnknownMessage a) => FieldId -> WireType -> a -> Get a-loadUnknown fieldId wireType msg = do- let (UnknownField uf) = getUnknownField msg- badwt wt = do here <- bytesRead- fail $ "Conflicting wire types at byte position "++show here ++ " for unknown field of message: "++show (typeOf msg,fieldId,wireType,wt)- case M.lookup fieldId uf of- Nothing -> do- bs <- wireGetFromWire fieldId wireType- let v' = UFV wireType (Seq.singleton bs)- uf' = M.insert fieldId v' uf- seq v' $ seq uf' $ return $ putUnknownField (UnknownField uf') msg- Just (UFV wt raw) | wt /= wireType -> badwt wt- | otherwise -> do- bs <- wireGetFromWire fieldId wireType- let v' = UFV wt (raw |> bs)- uf' = M.insert fieldId v' uf- seq v' $ seq uf' $ return $ putUnknownField (UnknownField uf') msg+{-# INLINE catch'Unknown #-}+-- | This is used by the generated code+catch'Unknown :: (Typeable a, UnknownMessage a) => (WireTag -> a -> Get a) -> (WireTag -> a -> Get a)+catch'Unknown update'Self = \wire'Tag old'Self -> catchError (update'Self wire'Tag old'Self) (\_ -> loadUnknown wire'Tag old'Self) + where loadUnknown :: (Typeable a, UnknownMessage a) => WireTag -> a -> Get a+ loadUnknown tag msg = do+ let (fieldId,wireType) = splitWireTag tag+ (UnknownField uf) = getUnknownField msg+ bs <- wireGetFromWire fieldId wireType+ let v' = UFV tag bs+ uf' = uf |> v'+ seq v' $ seq uf' $ return $ putUnknownField (UnknownField uf') msg+
Text/ProtocolBuffers/WireMessage.hs view
@@ -26,7 +26,7 @@ , Wire(..) -- * The internal exports, for use by generated code and the "Text.ProtcolBuffer.Extensions" module , size'Varint,toWireType,toWireTag,toPackedWireTag,mkWireTag- , prependMessageSize,putSize,putVarUInt,getVarInt,putLazyByteString,splitWireTag+ , prependMessageSize,putSize,putVarUInt,getVarInt,putLazyByteString,splitWireTag,fieldIdOf , wireSizeReq,wireSizeOpt,wireSizeRep,wireSizePacked , wirePutReq,wirePutOpt,wirePutRep,wirePutPacked , wireSizeErr,wirePutErr,wireGetErr@@ -258,6 +258,9 @@ splitWireTag (WireTag wireTag) = ( FieldId . fromIntegral $ wireTag `shiftR` 3 , WireType . fromIntegral $ wireTag .&. 7 ) +fieldIdOf :: WireTag -> FieldId+fieldIdOf = fst . splitWireTag+ {-# INLINE wireGetPackedEnum #-} wireGetPackedEnum :: (Typeable e,Enum e) => (Int -> Maybe e) -> Get (Seq e) wireGetPackedEnum toMaybe'Enum = do@@ -304,7 +307,8 @@ -- getMessageWith assumes the wireTag for the message, if it existed, has already been read. -- getMessageWith assumes that it still needs to read the Varint encoded length of the message. getMessageWith :: (Mergeable message, ReflectDescriptor message)- => (WireTag -> FieldId -> WireType -> message -> Get message)+-- => (WireTag -> FieldId -> WireType -> message -> Get message)+ => (WireTag -> message -> Get message) -> Get message getMessageWith updater = do messageLength <- getVarInt@@ -319,9 +323,9 @@ LT -> tooMuchData messageLength start here GT -> do wireTag <- fmap WireTag getVarInt -- get tag off wire- let (fieldId,wireType) = splitWireTag wireTag+ let -- (fieldId,wireType) = splitWireTag wireTag reqs' = Set.delete wireTag reqs- updater wireTag fieldId wireType message >>= go reqs'+ updater wireTag {- fieldId wireType -} message >>= go reqs' go' message = do here <- bytesRead case compare stop here of@@ -329,8 +333,8 @@ LT -> tooMuchData messageLength start here GT -> do wireTag <- fmap WireTag getVarInt -- get tag off wire- let (fieldId,wireType) = splitWireTag wireTag- updater wireTag fieldId wireType message >>= go'+-- let (fieldId,wireType) = splitWireTag wireTag+ updater wireTag {- fieldId wireType -} message >>= go' go required initialMessage where initialMessage = mergeEmpty@@ -350,7 +354,8 @@ -- getBareMessageWith will consume the entire ByteString it is operating on, or until it -- finds any STOP_GROUP tag (wireType == 4) getBareMessageWith :: (Mergeable message, ReflectDescriptor message)- => (WireTag -> FieldId -> WireType -> message -> Get message) -- handle wireTags that are unknown or produce errors+-- => (WireTag -> FieldId -> WireType -> message -> Get message) -- handle wireTags that are unknown or produce errors+ => (WireTag -> message -> Get message) -- handle wireTags that are unknown or produce errors -> Get message getBareMessageWith updater = go required initialMessage where@@ -363,7 +368,7 @@ let (fieldId,wireType) = splitWireTag wireTag if wireType == 4 then notEnoughData -- END_GROUP too soon else let reqs' = Set.delete wireTag reqs- in updater wireTag fieldId wireType message >>= go reqs'+ in updater wireTag {- fieldId wireType -} message >>= go reqs' go' message = do done <- isReallyEmpty if done then return message@@ -371,7 +376,7 @@ wireTag <- fmap WireTag getVarInt -- get tag off wire let (fieldId,wireType) = splitWireTag wireTag if wireType == 4 then return message- else updater wireTag fieldId wireType message >>= go'+ else updater wireTag {- fieldId wireType -} message >>= go' initialMessage = mergeEmpty (GetMessageInfo {requiredTags=required}) = getMessageInfo initialMessage notEnoughData = throwError ("Text.ProtocolBuffers.WireMessage.getBareMessageWith: Required fields missing when processing "
protocol-buffers.cabal view
@@ -1,5 +1,5 @@ name: protocol-buffers-version: 1.5.0+version: 1.6.0 cabal-version: >= 1.6 build-type: Simple license: BSD3