avro 0.3.0.0 → 0.3.0.1
raw patch · 12 files changed
+232/−72 lines, 12 files
Files
- avro.cabal +50/−46
- src/Data/Avro/Deconflict.hs +40/−20
- src/Data/Avro/Deriving.hs +1/−1
- src/Data/Avro/Schema.hs +2/−2
- test/Avro/Deconflict/Reader.hs +9/−0
- test/Avro/Deconflict/Writer.hs +9/−0
- test/Avro/DeconflictSpec.hs +40/−0
- test/Avro/DefaultsSpec.hs +2/−1
- test/Avro/THUnionSpec.hs +44/−0
- test/data/deconflict/reader.avsc +16/−0
- test/data/deconflict/writer.avsc +15/−0
- test/data/unions.avsc +4/−2
avro.cabal view
@@ -1,11 +1,11 @@--- This file has been generated from package.yaml by hpack version 0.20.0.+-- This file has been generated from package.yaml by hpack version 0.28.2. -- -- see: https://github.com/sol/hpack ----- hash: b4a4d7f0413845a9e4530632fc5abaf13e59faf4ba490991cd165e3ad78684be+-- hash: ba78df8186da1a2bbab610186df6edbf105eaf622ce7e3f1681252bb60941fb9 name: avro-version: 0.3.0.0+version: 0.3.0.1 synopsis: Avro serialization support for Haskell description: Avro serialization and deserialization support for Haskell category: Data@@ -17,9 +17,10 @@ license-file: LICENSE build-type: Simple cabal-version: >= 1.10- extra-source-files: ChangeLog.md+ test/data/deconflict/reader.avsc+ test/data/deconflict/writer.avsc test/data/enums-object.json test/data/enums.avsc test/data/maybe.avsc@@ -44,6 +45,25 @@ default: True library+ exposed-modules:+ Data.Avro+ Data.Avro.Decode+ Data.Avro.DecodeRaw+ Data.Avro.Deconflict+ Data.Avro.Encode+ Data.Avro.Deriving+ Data.Avro.EncodeRaw+ Data.Avro.JSON+ Data.Avro.Schema+ Data.Avro.Types+ Data.Avro.Zag+ Data.Avro.Zig+ Data.Avro.HasAvroSchema+ Data.Avro.FromAvro+ Data.Avro.ToAvro+ other-modules:+ Data.Avro.Deriving.NormSchema+ Paths_avro hs-source-dirs: src other-extensions: OverloadedStrings@@ -74,30 +94,37 @@ template-haskell >=2.4 if flag(dev) ghc-options: -Wall -Werror- exposed-modules:- Data.Avro- Data.Avro.Decode- Data.Avro.DecodeRaw- Data.Avro.Deconflict- Data.Avro.Encode- Data.Avro.Deriving- Data.Avro.EncodeRaw- Data.Avro.JSON- Data.Avro.Schema- Data.Avro.Types- Data.Avro.Zag- Data.Avro.Zig- Data.Avro.HasAvroSchema- Data.Avro.FromAvro- Data.Avro.ToAvro- other-modules:- Data.Avro.Deriving.NormSchema- Paths_avro default-language: Haskell2010 test-suite test type: exitcode-stdio-1.0 main-is: Spec.hs+ other-modules:+ Avro.Codec.ArraySpec+ Avro.Codec.BoolSpec+ Avro.Codec.CodecRawSpec+ Avro.Codec.DoubleSpec+ Avro.Codec.FloatSpec+ Avro.Codec.Int64Spec+ Avro.Codec.MaybeSpec+ Avro.Codec.NestedSpec+ Avro.Codec.TextSpec+ Avro.Codec.ZigZagSpec+ Avro.Deconflict.Reader+ Avro.Deconflict.Writer+ Avro.DeconflictSpec+ Avro.DefaultsSpec+ Avro.EncodeRawSpec+ Avro.JSONSpec+ Avro.THEncodeContainerSpec+ Avro.THEnumSpec+ Avro.THReusedSpec+ Avro.THSimpleSpec+ Avro.THUnionSpec+ Avro.ToAvroSpec+ DecodeContainer+ Example1+ Paths_avro hs-source-dirs: test ghc-options: -threaded@@ -134,27 +161,4 @@ template-haskell if flag(dev) ghc-options: -Wall -Werror- other-modules:- Avro.Codec.ArraySpec- Avro.Codec.BoolSpec- Avro.Codec.CodecRawSpec- Avro.Codec.DoubleSpec- Avro.Codec.FloatSpec- Avro.Codec.Int64Spec- Avro.Codec.MaybeSpec- Avro.Codec.NestedSpec- Avro.Codec.TextSpec- Avro.Codec.ZigZagSpec- Avro.DefaultsSpec- Avro.EncodeRawSpec- Avro.JSONSpec- Avro.THEncodeContainerSpec- Avro.THEnumSpec- Avro.THReusedSpec- Avro.THSimpleSpec- Avro.THUnionSpec- Avro.ToAvroSpec- DecodeContainer- Example1- Paths_avro default-language: Haskell2010
src/Data/Avro/Deconflict.hs view
@@ -3,12 +3,16 @@ ( deconflict ) where +import Control.Applicative ((<|>)) import Data.Avro.Schema as S import Data.Avro.Types as T import Data.HashMap.Strict (HashMap) import qualified Data.HashMap.Strict as HashMap-import qualified Data.List.NonEmpty as NE-import Data.List.NonEmpty (NonEmpty(..))+import Data.List (find)+import Data.List.NonEmpty (NonEmpty (..))+import qualified Data.List.NonEmpty as NE+import qualified Data.Map as M+import qualified Data.Set as Set import Data.Text (Text) import qualified Data.Text as Text import qualified Data.Text.Encoding as Text@@ -16,25 +20,27 @@ -- | @deconflict writer reader val@ will convert a value that was -- encoded/decoded with the writer's schema into the form specified by the -- reader's schema.-deconflict :: Schema -> Schema -> T.Value Type -> Either String (T.Value Type)-deconflict writerType readerType val =- resolveSchema writerType readerType val+deconflict :: Schema -- ^ Writer schema+ -> Schema -- ^ Reader schema+ -> T.Value Type+ -> Either String (T.Value Type)+deconflict = resolveSchema resolveSchema :: Type -> Type -> T.Value Type -> Either String (T.Value Type)-resolveSchema e d v- | e == d = Right v- | otherwise = go e d v+resolveSchema writerSchema readerSchema v+ | writerSchema == readerSchema = Right v+ | otherwise = go writerSchema readerSchema v where go :: Type -> Type -> T.Value Type -> Either String (T.Value Type) go (S.Array aTy) (S.Array bTy) (T.Array vec) = T.Array <$> mapM (go aTy bTy) vec go (S.Map aTy) (S.Map bTy) (T.Map mp) = T.Map <$> mapM (go aTy bTy) mp- go a@(S.Enum {}) b@(S.Enum {}) val+ go a@S.Enum {} b@S.Enum {} val | name a == name b = resolveEnum a b val- go a@(S.Fixed {}) b@(S.Fixed {}) val+ go a@S.Fixed {} b@S.Fixed {} val | name a == name b && size a == size b = Right val- go a@(S.Record {}) b@(S.Record {}) val+ go a@S.Record {} b@S.Record {} val | name a == name b = resolveRecord a b val go (S.Union _ _) (S.Union ys _) val = resolveTwoUnions ys val@@ -69,16 +75,16 @@ let hdl [] = Left "Impossible: empty non-empty list." hdl (d:rest) = case resolveSchema e d val of- Right v -> Right (T.Union ds d v)- Left _ -> hdl rest+ Right v -> Right (T.Union ds d v)+ Left _ -> hdl rest in hdl (NE.toList ds) resolveWriterUnion :: Type -> T.Value Type -> Either String (T.Value Type) resolveWriterUnion reader (T.Union _ ty val) = resolveSchema ty reader val resolveRecord :: Type -> Type -> T.Value Type -> Either String (T.Value Type)-resolveRecord eRec dRec (T.Record ty fldVals) =- T.Record ty . HashMap.fromList <$> mapM (resolveFields fldVals (fields eRec)) (fields dRec)+resolveRecord writerSchema readerSchema (T.Record ty fldVals) =+ T.Record ty . HashMap.fromList <$> mapM (resolveFields fldVals (fields writerSchema)) (fields readerSchema) -- For each field of the decoders, lookup the field in the hash map -- 1) If the field exists, call 'resolveSchema'@@ -87,8 +93,22 @@ -- -- XXX: Consider aliases in the writer schema, use those to retry on failed lookup. resolveFields :: HashMap Text (T.Value Type) -> [Field] -> Field -> Either String (Text,T.Value Type)-resolveFields hm eFlds d =- case (HashMap.lookup (fldName d) hm, fldDefault d) of- (Just x,_) -> Right (fldName d, x)- (_,Just def) -> Right (fldName d,def)- (Nothing,Nothing) -> Left $ "No field and no default for " ++ show (fldName d)+resolveFields hm writerFields readerField =+ let+ mbWriterField = findField readerField writerFields+ mbValue = HashMap.lookup (fldName readerField) hm+ in case (mbWriterField, mbValue, fldDefault readerField) of+ (Just w, Just x,_) -> (fldName readerField,) <$> resolveSchema (fldType w) (fldType readerField) x+ (_, Just x,_) -> Right (fldName readerField, x)+ (_, _,Just def) -> Right (fldName readerField, def)+ (_,Nothing,Nothing) -> Left $ "No field and no default for " ++ show (fldName readerField)++findField :: Field -> [Field] -> Maybe Field+findField f fs =+ let+ byName = find (\x -> fldName x == fldName f) fs+ allNames fld = Set.fromList (fldName fld : fldAliases fld)+ fNames = allNames f+ sameField = not . Set.null . Set.intersection fNames . allNames+ byAliases = find sameField fs+ in byName <|> byAliases
src/Data/Avro/Deriving.hs view
@@ -224,7 +224,7 @@ AT.Array vec -> [e| AT.Array $ V.fromList $(ListE <$> mapM mkDefaultValue (V.toList vec)) |] AT.Map m -> [e| AT.Map $ $(mkMap m) |] AT.Record s m -> [e| AT.Record $(mkSchema s) $(mkMap m) |]- AT.Union _ _ v -> mkDefaultValue v+ AT.Union ts t v -> [e| AT.Union $(mkNE ts) $(mkSchema t) $(mkDefaultValue v) |] AT.Fixed s bs -> [e| AT.Fixed $(mkSchema s) $(mkByteString bs) |] AT.Enum s n sym -> [e| AT.Enum $(mkSchema s) $(litE $ IntegerL $ fromIntegral n) $(mkText sym) |]
src/Data/Avro/Schema.hs view
@@ -383,8 +383,8 @@ -- of the first type in the union. parseFieldDefault :: (Text -> Maybe Type) -> Type -> A.Value -> Result (Ty.Value Type) parseFieldDefault env schema value = parseAvroJSON defaultUnion env schema value- where defaultUnion (Union (t :| _) _) val = parseFieldDefault env t val- defaultUnion _ _ = error "Impossible: not Union."+ where defaultUnion (Union ts@(t :| _) _) val = Ty.Union ts t <$> parseFieldDefault env t val+ defaultUnion _ _ = error "Impossible: not Union." -- | Parse JSON-encoded avro data. parseAvroJSON :: (Type -> A.Value -> Result (Ty.Value Type))
+ test/Avro/Deconflict/Reader.hs view
@@ -0,0 +1,9 @@+{-# LANGUAGE OverloadedStrings #-}+{-# LANGUAGE TemplateHaskell #-}+module Avro.Deconflict.Reader+where++import Data.Avro.Deconflict+import Data.Avro.Deriving++deriveAvro "test/data/deconflict/reader.avsc"
+ test/Avro/Deconflict/Writer.hs view
@@ -0,0 +1,9 @@+{-# LANGUAGE OverloadedStrings #-}+{-# LANGUAGE TemplateHaskell #-}+module Avro.Deconflict.Writer+where++import Data.Avro.Deconflict+import Data.Avro.Deriving++deriveAvro "test/data/deconflict/writer.avsc"
+ test/Avro/DeconflictSpec.hs view
@@ -0,0 +1,40 @@+{-# LANGUAGE OverloadedStrings #-}+{-# LANGUAGE ScopedTypeVariables #-}+module Avro.DeconflictSpec+where++import Data.Avro as A+import qualified Data.Avro.Decode as A (decodeAvro)+import Data.Avro.Deconflict+import Data.Avro.Deriving+import Data.Avro.Schema+import qualified Data.Avro.Types as Ty+import Data.Either+import Data.List.NonEmpty (NonEmpty (..))++import qualified Avro.Deconflict.Reader as R+import qualified Avro.Deconflict.Writer as W++import Test.Hspec++{-# ANN module ("HLint: ignore Redundant do" :: String) #-}++writerMessage :: W.Outer+writerMessage = W.Outer "Peone" (W.Inner 3)++spec :: Spec+spec = describe "Avro.DeconflictSpec" $ do+ it "should deconflict simple message" $ do+ let payload = A.encode $ W.Inner 3+ let Right decodedAvro = A.decodeAvro W.schema'Inner payload+ let Right deconflicted = deconflict W.schema'Inner R.schema'Inner decodedAvro+ fromAvro deconflicted `shouldBe` Success (R.Inner 3 Nothing)++ it "should deconflict nested message" $ do+ let payload = A.encode writerMessage+ let Right decodedAvro = A.decodeAvro W.schema'Outer payload+ let Right deconflicted = deconflict W.schema'Outer R.schema'Outer decodedAvro++ fromAvro deconflicted `shouldBe` Success (R.Outer "Peone" (R.Inner 3 Nothing))++
test/Avro/DefaultsSpec.hs view
@@ -10,6 +10,7 @@ import Data.Avro.Schema import qualified Data.Avro.Types as Ty import qualified Data.HashMap.Strict as M+import Data.List.NonEmpty (NonEmpty (..)) import qualified Data.Vector as V import Test.Hspec@@ -33,7 +34,7 @@ msgSchema = schemaOf (undefined :: MaybeTest) fixedSchema = schemaOf (undefined :: FixedTag) defaults = fldDefault <$> fields msgSchema- in defaults `shouldBe` [ Just Ty.Null+ in defaults `shouldBe` [ Just $ Ty.Union (Null :| [String]) Null Ty.Null , Just $ Ty.Fixed fixedSchema "\0\42\255" , Just $ Ty.Bytes "\0\37\255" ]
test/Avro/THUnionSpec.hs view
@@ -4,11 +4,21 @@ module Avro.THUnionSpec where +import qualified Data.List.NonEmpty as NE++import qualified Data.Aeson as Aeson import Data.Avro import Data.Avro.Deriving+import qualified Data.Avro.Schema as Schema+import qualified Data.Avro.Types as Avro+import qualified Data.ByteString.Lazy as LBS +import System.Directory (doesFileExist)+ import Test.Hspec +import Paths_avro+ deriveAvro "test/data/unions.avsc" spec :: Spec@@ -27,6 +37,40 @@ } , unionsSameFields = Right $ NotFoo { notFooStuff = "different from Foo" } }++ field name schema def = Schema.Field name [] Nothing (Just Schema.Ascending) schema def+ record name namespace fields =+ Schema.Record name namespace [] Nothing (Just Schema.Ascending) fields+ named = Schema.NamedType . Schema.TN++ expectedSchema = record "Unions" (Just "haskell.avro.example")+ [ field "scalars" (Schema.mkUnion (NE.fromList [Schema.String, Schema.Long])) scalarsDefault+ , field "nullable" (Schema.mkUnion (NE.fromList [Schema.Null, Schema.Int])) nullableDefault+ , field "records" (Schema.mkUnion (NE.fromList [fooSchema, barSchema])) Nothing+ , field "sameFields" (Schema.mkUnion (NE.fromList [named "Foo", notFooSchema])) Nothing+ ]+ scalarsDefault = Just $ Avro.Union (NE.fromList [Schema.String, Schema.Long]) Schema.String (Avro.String "foo")+ nullableDefault = Just $ Avro.Union (NE.fromList [Schema.Null, Schema.Int]) Schema.Null Avro.Null++ fooSchema = record "Foo" Nothing [field "stuff" Schema.String Nothing]+ barSchema = record "Bar" Nothing+ [ field "stuff" Schema.String Nothing+ , field "things" (named "Foo") Nothing+ ]+ notFooSchema = record "NotFoo" Nothing [field "stuff" Schema.String Nothing]++ unionsSchemaFile <- runIO $ getFileName "test/data/unions.avsc" >>= LBS.readFile+ let Just unionsSchemaFromJSON = Aeson.decode unionsSchemaFile++ it "produces valid schemas" $ do+ schema'Unions `shouldBe` expectedSchema+ unionsSchemaFromJSON `shouldBe` expectedSchema it "records with unions should roundtrip" $ do fromAvro (toAvro objA) `shouldBe` pure objA fromAvro (toAvro objB) `shouldBe` pure objB++getFileName :: FilePath -> IO FilePath+getFileName p = do+ path <- getDataFileName p+ isOk <- doesFileExist path+ pure $ if isOk then path else p
+ test/data/deconflict/reader.avsc view
@@ -0,0 +1,16 @@+{+ "type": "record",+ "name": "Outer",+ "fields": [+ { "name": "name", "type": "string" },+ { "name": "inner", "type": {+ "type": "record",+ "name": "Inner",+ "fields": [+ { "name": "id", "type": "int" },+ { "name": "smell", "type": ["null", "string"], "default": null }+ ]+ }+ }+ ]+}
+ test/data/deconflict/writer.avsc view
@@ -0,0 +1,15 @@+{+ "type": "record",+ "name": "Outer",+ "fields": [+ { "name": "name", "type": "string" },+ { "name": "inner", "type": {+ "type": "record",+ "name": "Inner",+ "fields": [+ { "name": "id", "type": "int" }+ ]+ }+ }+ ]+}
test/data/unions.avsc view
@@ -3,10 +3,12 @@ "namespace" : "haskell.avro.example", "fields" : [ { "name" : "scalars",- "type" : ["string", "long"]+ "type" : ["string", "long"],+ "default" : "foo" }, { "name" : "nullable",- "type" : ["null", "int"]+ "type" : ["null", "int"],+ "default" : null }, { "name" : "records", "type" : [