diff --git a/avro.cabal b/avro.cabal --- a/avro.cabal +++ b/avro.cabal @@ -1,133 +1,160 @@ --- Initial avro.cabal generated by cabal init. For further documentation, --- see http://haskell.org/cabal/users-guide/ +-- This file has been generated from package.yaml by hpack version 0.20.0. +-- +-- see: https://github.com/sol/hpack +-- +-- hash: b4a4d7f0413845a9e4530632fc5abaf13e59faf4ba490991cd165e3ad78684be -name: avro -version: 0.2.1.1 -synopsis: Avro serialization support for Haskell -description: Avro serialization and deserialization support for Haskell -homepage: https://github.com/GaloisInc/avro.git -license: BSD3 -license-file: LICENSE -author: Thomas M. DuBuisson -maintainer: Alexey Raga --- copyright: -category: Data -build-type: Simple -extra-source-files: ChangeLog.md, test/data/reused.avsc, test/data/small.avsc, test/data/enums.avsc, test/data/maybe.avsc -cabal-version: >=1.10 +name: avro +version: 0.3.0.0 +synopsis: Avro serialization support for Haskell +description: Avro serialization and deserialization support for Haskell +category: Data +homepage: https://github.com/GaloisInc/avro.git#readme +bug-reports: https://github.com/GaloisInc/avro.git/issues +author: Thomas M. DuBuisson +maintainer: Alexey Raga +license: BSD3 +license-file: LICENSE +build-type: Simple +cabal-version: >= 1.10 +extra-source-files: + ChangeLog.md + test/data/enums-object.json + test/data/enums.avsc + test/data/maybe.avsc + test/data/reused.avsc + test/data/small.avsc + test/data/unions-object-a.json + test/data/unions-object-b.json + test/data/unions.avsc + source-repository head - type: git + type: git location: https://github.com/GaloisInc/avro.git flag dev - default: False - manual: True description: Use development GHC flags + manual: True + default: False -flag templateHaskell - Description: Build Avro.Deriving, which uses Template Haskell. - Default: True +flag templatehaskell + description: Build Avro.Deriving, which uses Template Haskell. + manual: False + default: True library - exposed-modules: Data.Avro, - Data.Avro.Decode, - Data.Avro.DecodeRaw, - Data.Avro.Deconflict, - Data.Avro.Encode, - Data.Avro.EncodeRaw, - Data.Avro.Schema, - Data.Avro.Types, - Data.Avro.Zag, - Data.Avro.Zig, - Data.Avro.HasAvroSchema, - Data.Avro.FromAvro, - Data.Avro.ToAvro - other-extensions: OverloadedStrings - build-depends: base >=4.8 && <5.0, - aeson, - array, - base16-bytestring, - binary, - bytestring, - containers, - data-binary-ieee754, - entropy, - fail, - hashable, - mtl, - pure-zlib, - scientific, - semigroups, - tagged, - text, - unordered-containers, - vector - - if flag(templateHaskell) - build-depends: template-haskell >= 2.4 + hs-source-dirs: + src + other-extensions: OverloadedStrings + ghc-options: -O2 + build-depends: + aeson + , array + , base >=4.8 && <5.0 + , base16-bytestring + , binary + , bytestring + , containers + , data-binary-ieee754 + , entropy + , fail + , hashable + , mtl + , pure-zlib + , scientific + , semigroups + , tagged + , text + , unordered-containers + , vector + if flag(templatehaskell) other-extensions: TemplateHaskell - exposed-modules: Data.Avro.Deriving - other-modules: Data.Avro.Deriving.NormSchema - - hs-source-dirs: src - default-language: Haskell2010 - ghc-options: -O2 + build-depends: + template-haskell >=2.4 if flag(dev) ghc-options: -Wall -Werror + exposed-modules: + Data.Avro + Data.Avro.Decode + Data.Avro.DecodeRaw + Data.Avro.Deconflict + Data.Avro.Encode + Data.Avro.Deriving + Data.Avro.EncodeRaw + Data.Avro.JSON + Data.Avro.Schema + Data.Avro.Types + Data.Avro.Zag + Data.Avro.Zig + Data.Avro.HasAvroSchema + Data.Avro.FromAvro + Data.Avro.ToAvro + other-modules: + Data.Avro.Deriving.NormSchema + Paths_avro + default-language: Haskell2010 test-suite test - type: exitcode-stdio-1.0 - default-language: Haskell2010 - hs-source-dirs: test - other-modules: Avro.Codec.ArraySpec - , Avro.Codec.BoolSpec - , Avro.Codec.CodecRawSpec - , Avro.Codec.DoubleSpec - , Avro.Codec.FloatSpec - , Avro.Codec.Int64Spec - , Avro.Codec.MaybeSpec - , Avro.Codec.NestedSpec - , Avro.Codec.TextSpec - , Avro.Codec.ZigZagSpec - , Avro.EncodeRawSpec - , Avro.ToAvroSpec - - build-depends: base >=4.6 && < 5 - , aeson - , lens-aeson - , array - , avro - , base16-bytestring - , binary - , bytestring - , containers - , entropy - , extra - , fail - , hashable - , hspec - , lens - , mtl - , pure-zlib - , QuickCheck - , scientific - , semigroups - , tagged - , text - , transformers - , unordered-containers - , vector - - if flag(templateHaskell) - build-depends: template-haskell + type: exitcode-stdio-1.0 + main-is: Spec.hs + hs-source-dirs: + test + ghc-options: -threaded + build-depends: + QuickCheck + , aeson + , array + , avro + , base >=4.6 && <5 + , base16-bytestring + , binary + , bytestring + , containers + , directory + , entropy + , extra + , fail + , hashable + , hspec + , lens + , lens-aeson + , mtl + , pure-zlib + , scientific + , semigroups + , tagged + , text + , transformers + , unordered-containers + , vector + if flag(templatehaskell) other-extensions: TemplateHaskell - other-modules: Avro.THEnumSpec - , Avro.THReusedSpec - , Avro.THSimpleSpec - , Avro.THEncodeContainerSpec - - main-is: Spec.hs - ghc-options: -threaded + build-depends: + template-haskell if flag(dev) ghc-options: -Wall -Werror + other-modules: + Avro.Codec.ArraySpec + Avro.Codec.BoolSpec + Avro.Codec.CodecRawSpec + Avro.Codec.DoubleSpec + Avro.Codec.FloatSpec + Avro.Codec.Int64Spec + Avro.Codec.MaybeSpec + Avro.Codec.NestedSpec + Avro.Codec.TextSpec + Avro.Codec.ZigZagSpec + Avro.DefaultsSpec + Avro.EncodeRawSpec + Avro.JSONSpec + Avro.THEncodeContainerSpec + Avro.THEnumSpec + Avro.THReusedSpec + Avro.THSimpleSpec + Avro.THUnionSpec + Avro.ToAvroSpec + DecodeContainer + Example1 + Paths_avro + default-language: Haskell2010 diff --git a/src/Data/Avro.hs b/src/Data/Avro.hs --- a/src/Data/Avro.hs +++ b/src/Data/Avro.hs @@ -1,7 +1,7 @@ -{-# LANGUAGE FlexibleInstances #-} -{-# LANGUAGE ConstraintKinds #-} +{-# LANGUAGE ConstraintKinds #-} +{-# LANGUAGE FlexibleInstances #-} +{-# LANGUAGE MultiWayIf #-} {-# LANGUAGE ScopedTypeVariables #-} -{-# LANGUAGE MultiWayIf #-} -- | Avro encoding and decoding routines. -- -- This library provides a high level interface for encoding (and decoding) @@ -86,34 +86,34 @@ , schemaOf ) where -import Prelude as P -import Control.Arrow (first) -import qualified Data.Avro.Decode as D -import Data.Avro.Deconflict as C -import qualified Data.Avro.Encode as E -import Data.Avro.Schema as S -import Data.Avro.Types as T -import qualified Data.Binary.Get as G -import qualified Data.Binary.Put as P -import qualified Data.ByteString as B -import Data.ByteString.Lazy (ByteString) -import qualified Data.ByteString.Lazy as BL -import Data.Foldable (toList) -import qualified Data.HashMap.Strict as HashMap +import Control.Arrow (first) +import qualified Data.Avro.Decode as D +import Data.Avro.Deconflict as C +import qualified Data.Avro.Encode as E +import Data.Avro.Schema as S +import Data.Avro.Types as T +import qualified Data.Binary.Get as G +import qualified Data.Binary.Put as P +import qualified Data.ByteString as B +import Data.ByteString.Lazy (ByteString) +import qualified Data.ByteString.Lazy as BL +import Data.Foldable (toList) +import qualified Data.HashMap.Strict as HashMap import Data.Int -import Data.List.NonEmpty (NonEmpty(..)) -import qualified Data.Map as Map -import Data.Monoid ((<>)) -import Data.Text (Text) -import qualified Data.Text as Text -import qualified Data.Text.Lazy as TL +import Data.List.NonEmpty (NonEmpty (..)) +import qualified Data.Map as Map +import Data.Monoid ((<>)) import Data.Tagged -import qualified Data.Vector as V +import Data.Text (Text) +import qualified Data.Text as Text +import qualified Data.Text.Lazy as TL +import qualified Data.Vector as V import Data.Word +import Prelude as P -import Data.Avro.FromAvro -import Data.Avro.ToAvro -import Data.Avro.HasAvroSchema +import Data.Avro.FromAvro +import Data.Avro.HasAvroSchema +import Data.Avro.ToAvro type Avro a = (FromAvro a, ToAvro a) @@ -167,7 +167,7 @@ decodeContainerBytes bs = case D.decodeContainerWith schemaBytes bs of Right (writerSchema, val) -> val - Left e -> error $ "Could not decode container: " <> e + Left e -> error $ "Could not decode container: " <> e where schemaBytes sch = do start <- G.bytesRead diff --git a/src/Data/Avro/Deriving.hs b/src/Data/Avro/Deriving.hs --- a/src/Data/Avro/Deriving.hs +++ b/src/Data/Avro/Deriving.hs @@ -1,6 +1,9 @@ -{-# LANGUAGE CPP #-} -{-# LANGUAGE TemplateHaskell #-} -{-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE CPP #-} +{-# LANGUAGE LambdaCase #-} +{-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE RecordWildCards #-} +{-# LANGUAGE TemplateHaskell #-} +{-# LANGUAGE ViewPatterns #-} module Data.Avro.Deriving ( deriveAvro @@ -9,29 +12,33 @@ ) where -import Control.Monad (join) -import Data.Aeson (eitherDecode) -import Data.Char (isAlphaNum) -import qualified Data.Aeson as J -import Data.Avro hiding (decode, encode) -import Data.Avro.Schema as S -import qualified Data.Avro.Types as AT -import Data.ByteString (ByteString) +import Control.Monad (join) +import Data.Aeson (eitherDecode) +import qualified Data.Aeson as J +import Data.Avro hiding (decode, encode) +import Data.Avro.Schema as S +import qualified Data.Avro.Types as AT +import Data.ByteString (ByteString) +import qualified Data.ByteString as B +import Data.Char (isAlphaNum) import Data.Int -import Data.List.NonEmpty (NonEmpty( (:|) )) -import Data.Map (Map) -import Data.Maybe (fromMaybe) -import Data.Semigroup ((<>)) -import Language.Haskell.TH as TH +import Data.List.NonEmpty (NonEmpty ((:|))) +import qualified Data.List.NonEmpty as NE +import Data.Map (Map) +import Data.Maybe (fromMaybe) +import Data.Semigroup ((<>)) +import Language.Haskell.TH as TH import Language.Haskell.TH.Syntax -import Data.Avro.Deriving.NormSchema +import Data.Avro.Deriving.NormSchema -import qualified Data.ByteString as BS -import qualified Data.ByteString.Lazy as LBS -import qualified Data.ByteString.Lazy.Char8 as LBSC8 -import Data.Text (Text) -import qualified Data.Text as T +import qualified Data.ByteString as BS +import qualified Data.ByteString.Lazy as LBS +import qualified Data.ByteString.Lazy.Char8 as LBSC8 +import qualified Data.HashMap.Strict as HM +import Data.Text (Text) +import qualified Data.Text as T +import qualified Data.Vector as V -- | Derives Avro from a given schema file. -- Generates data types, FromAvro and ToAvro instances. @@ -62,8 +69,8 @@ qAddDependentFile p mbSchema <- runIO $ decodeSchema p case mbSchema of - Left err -> fail $ "Unable to generate AVRO for " <> p <> ": " <> err - Right sch -> pure sch + Left err -> fail $ "Unable to generate AVRO for " <> p <> ": " <> err + Right sch -> pure sch genFromAvro :: Schema -> Q [Dec] genFromAvro (S.Enum n _ _ _ _ _) = @@ -145,18 +152,99 @@ schemaDef sname sch = setName sname $ [d| x :: Schema - x = fromMaybe undefined (J.decode (LBSC8.pack $(mkLit (LBSC8.unpack $ J.encode sch)))) + x = $(mkSchema sch) |] + where mkSchema = \case + Null -> [e| Null |] + Boolean -> [e| Boolean |] + Int -> [e| Int |] + Long -> [e| Long |] + Float -> [e| Float |] + Double -> [e| Double |] + Bytes -> [e| Bytes |] + String -> [e| String |] + Array item -> [e| Array $(mkSchema item) |] + Map values -> [e| Map $(mkSchema values) |] + NamedType name -> [e| NamedType $(mkName name) |] + Record {..} -> [e| Record { name = $(mkName name) + , namespace = $(mkMaybeText namespace) + , aliases = $(ListE <$> mapM mkName aliases) + , doc = $(mkMaybeText doc) + , order = $(mkOrder order) + , fields = $(ListE <$> mapM mkField fields) + } + |] + Enum {..} -> [e| mkEnum $(mkName name) + $(ListE <$> mapM mkName aliases) + $(mkMaybeText namespace) + $(mkMaybeText doc) + $(ListE <$> mapM mkText symbols) + |] + Union {..} -> [e| mkUnion $(mkNE options) |] + Fixed {..} -> [e| Fixed { name = $(mkName name) + , namespace = $(mkMaybeText namespace) + , aliases = $(ListE <$> mapM mkName aliases) + , size = $(litE $ IntegerL $ fromIntegral size) + } + |] + mkText text = [e| T.pack $(mkTextLit text) |] + + mkName (TN name) = [e| TN $(mkText name) |] + + mkMaybeText (Just text) = [e| Just $(mkText text) |] + mkMaybeText Nothing = [e| Nothing |] + + mkOrder (Just Ascending) = [e| Just Ascending |] + mkOrder (Just Descending) = [e| Just Descending |] + mkOrder (Just Ignore) = [e| Just Ignore |] + mkOrder Nothing = [e| Nothing |] + + mkField Field {..} = + [e| Field { fldName = $(mkText fldName) + , fldAliases = $(ListE <$> mapM mkText fldAliases) + , fldDoc = $(mkMaybeText fldDoc) + , fldOrder = $(mkOrder fldOrder) + , fldType = $(mkSchema fldType) + , fldDefault = $(fromMaybe [e|Nothing|] $ mkJust . mkDefaultValue <$> fldDefault) + } + |] + + mkJust exp = [e|Just $(exp)|] + + mkDefaultValue = \case + AT.Null -> [e| AT.Null |] + AT.Boolean b -> [e| AT.Boolean $(if b then [e|True|] else [e|False|]) |] + AT.Int n -> [e| AT.Int $(litE $ IntegerL $ fromIntegral n) |] + AT.Long n -> [e| AT.Long $(litE $ IntegerL $ fromIntegral n) |] + AT.Float f -> [e| AT.Long $(litE $ FloatPrimL $ realToFrac f) |] + AT.Double f -> [e| AT.Long $(litE $ FloatPrimL $ realToFrac f) |] + AT.Bytes bs -> [e| AT.Bytes $(mkByteString bs) |] + AT.String s -> [e| AT.String $(mkText s) |] + AT.Array vec -> [e| AT.Array $ V.fromList $(ListE <$> mapM mkDefaultValue (V.toList vec)) |] + AT.Map m -> [e| AT.Map $ $(mkMap m) |] + AT.Record s m -> [e| AT.Record $(mkSchema s) $(mkMap m) |] + AT.Union _ _ v -> mkDefaultValue v + AT.Fixed s bs -> [e| AT.Fixed $(mkSchema s) $(mkByteString bs) |] + AT.Enum s n sym -> [e| AT.Enum $(mkSchema s) $(litE $ IntegerL $ fromIntegral n) $(mkText sym) |] + + mkByteString bs = [e| B.pack $(ListE <$> mapM numericLit (B.unpack bs)) |] + where numericLit = litE . IntegerL . fromIntegral + + mkMap (HM.toList -> xs) = [e| HM.fromList $(ListE <$> mapM mkKVPair xs) |] + mkKVPair (k, v) = [e| ($(mkText k), $(mkDefaultValue v)) e|] + + mkNE (NE.toList -> xs) = [e| NE.fromList $(ListE <$> mapM mkSchema xs) |] + -- | A hack around TemplateHaskell limitation: -- It is currently not possible to splice variable name in QQ. -- This function allows to replace hardcoded name into the specified one. setName :: Name -> Q [Dec] -> Q [Dec] setName = fmap . map . sn where - sn n (SigD _ t) = SigD n t + sn n (SigD _ t) = SigD n t sn n (ValD (VarP _) x y) = ValD (VarP n) x y - sn _ d = d + sn _ d = d genType :: Schema -> Q [Dec] genType (S.Record n _ _ _ _ fs) = do diff --git a/src/Data/Avro/FromAvro.hs b/src/Data/Avro/FromAvro.hs --- a/src/Data/Avro/FromAvro.hs +++ b/src/Data/Avro/FromAvro.hs @@ -40,10 +40,12 @@ Just v -> fromAvro v instance (FromAvro a, FromAvro b) => FromAvro (Either a b) where - fromAvro e@(T.Union _ v x) = - if | v == untag (schema :: Tagged a Type) -> Left <$> fromAvro x - | v == untag (schema :: Tagged b Type) -> Right <$> fromAvro x - | otherwise -> badValue e "either" + fromAvro e@(T.Union _ branch x) + | S.matches branch schemaA = Left <$> fromAvro x + | S.matches branch schemaB = Right <$> fromAvro x + | otherwise = badValue e "either" + where Tagged schemaA = schema :: Tagged a Type + Tagged schemaB = schema :: Tagged b Type fromAvro x = badValue x "either" instance FromAvro Bool where fromAvro (T.Boolean b) = pure b diff --git a/src/Data/Avro/JSON.hs b/src/Data/Avro/JSON.hs new file mode 100644 --- /dev/null +++ b/src/Data/Avro/JSON.hs @@ -0,0 +1,118 @@ +{-# LANGUAGE ScopedTypeVariables #-} +-- | Avro supports a JSON representation of Avro objects alongside the +-- Avro binary format. An Avro schema can be used to generate and +-- validate JSON representations of Avro objects. +-- +-- The JSON format is the same format as used for default values in +-- schemas except unions are encoded differently. Non-union values are +-- encoded as follows: +-- +-- +--------------+----------+----------+ +-- |Avro Type |JSON Type |Example | +-- +==============+==========+==========+ +-- |null |null |null | +-- +--------------+----------+----------+ +-- |boolean |boolean |true | +-- +--------------+----------+----------+ +-- |int, long |integer |1 | +-- +--------------+----------+----------+ +-- |float, double |number |1.1 | +-- +--------------+----------+----------+ +-- |bytes |string |"\u00FF" | +-- +--------------+----------+----------+ +-- |string |string |"foo" | +-- +--------------+----------+----------+ +-- |record |object |{"a":1} | +-- +--------------+----------+----------+ +-- |enum |string |"FOO" | +-- +--------------+----------+----------+ +-- |array |array |[1] | +-- +--------------+----------+----------+ +-- |map |object |{"a":1} | +-- +--------------+----------+----------+ +-- |fixed |string |"\u00FF" | +-- +--------------+----------+----------+ +-- +-- (Table from the Avro 1.8.2 specification: +-- ) +-- +-- Bytes and fixed are encoded as JSON strings where each byte is +-- translated into the corresponding Unicode codepoint between 0–255, +-- which includes non-printable characters. Note that this encoding +-- happens at the Unicode code-point level, meaning it is independent +-- of text encoding. (JSON is, by definition, encoded in UTF8.) +-- +-- Unions are encoded as an object with a single field that specifies +-- the "branch" of the union. If the branch is a primitive type like +-- @"string"@, the name of the primitive type is used: +-- +-- @ +-- { "string" : "foo" } +-- @ +-- +-- For named types (record, enum and fixed), the name of the type is +-- used: +-- +-- @ +-- { "MyRecord" : { ... } } +-- @ +module Data.Avro.JSON where + +import Data.Semigroup ((<>)) + +import qualified Data.Aeson as Aeson +import Data.ByteString.Lazy (ByteString) +import Data.HashMap.Strict ((!)) +import qualified Data.HashMap.Strict as HashMap +import Data.List.NonEmpty (NonEmpty (..)) +import qualified Data.List.NonEmpty as NE +import Data.Tagged +import qualified Data.Text as Text + +import Data.Avro (FromAvro (..), Result (..), ToAvro (..)) +import qualified Data.Avro as Avro +import Data.Avro.Schema (Schema, parseAvroJSON) +import qualified Data.Avro.Schema as Schema +import qualified Data.Avro.Types as Avro + +-- | Convert a 'Aeson.Value' into a type that has an Avro schema. The +-- schema is used to validate the JSON and will return an 'Error' if +-- the JSON object is not encoded correctly or does not match the schema. +fromJSON :: forall a. (FromAvro a) => Aeson.Value -> Result a +fromJSON json = parseAvroJSON union env schema json >>= fromAvro + where schema = untag (Avro.schema :: Tagged a Schema) + env = Schema.buildTypeEnvironment missing schema . Schema.TN + missing name = fail $ "Type " <> show name <> " not in schema." + + union (Schema.Union schemas _) Aeson.Null + | Schema.Null `elem` schemas = pure $ Avro.Union schemas Schema.Null Avro.Null + | otherwise = fail "Null not in union." + union (Schema.Union schemas _) (Aeson.Object obj) + | length obj < 1 = fail "Invalid encoding of union: empty object ({})." + | length obj > 1 = fail "Invalid encoding of union: object with too many fields." + | otherwise = + let branch = head $ HashMap.keys obj + names = HashMap.fromList [(Schema.typeName t, t) | t <- NE.toList schemas] + in case HashMap.lookup branch names of + Just t -> do + nested <- parseAvroJSON union env t $ obj ! branch + return $ Avro.Union schemas t nested + Nothing -> fail $ "Type '" <> Text.unpack branch <> "' not in union: " <> show schemas + union Schema.Union{} _ = + Avro.Error "Invalid JSON representation for union: has to be a JSON object with exactly one field." + union _ _ = error "Impossible: function given non-union schema." + +-- | Parse a 'ByteString' as JSON and convert it to a type with an +-- Avro schema. Will return 'Error' if the input is not valid JSON or +-- the JSON does not convert with the specified schema. +parseJSON :: forall a. (FromAvro a) => ByteString -> Result a +parseJSON input = case Aeson.eitherDecode input of + Left msg -> Error msg + Right value -> fromJSON value + +-- | Convert an object with an Avro schema to JSON using that schema. +-- +-- We always need the schema to /encode/ to JSON because representing +-- unions requires using the names of named types. +toJSON :: forall a. (ToAvro a) => a -> Aeson.Value +toJSON = Aeson.toJSON . toAvro diff --git a/src/Data/Avro/Schema.hs b/src/Data/Avro/Schema.hs --- a/src/Data/Avro/Schema.hs +++ b/src/Data/Avro/Schema.hs @@ -1,9 +1,12 @@ {-# LANGUAGE FlexibleInstances #-} {-# LANGUAGE MultiParamTypeClasses #-} +{-# LANGUAGE NamedFieldPuns #-} {-# LANGUAGE OverloadedStrings #-} {-# LANGUAGE RecordWildCards #-} {-# LANGUAGE TupleSections #-} {-# LANGUAGE TypeSynonymInstances #-} +{-# LANGUAGE ViewPatterns #-} + -- | Avro 'Schema's, represented here as values of type 'Schema', -- describe the serialization and de-serialization of values. -- @@ -22,6 +25,13 @@ , typeName , buildTypeEnvironment , Result(..) + + , matches + + , parseBytes + , serializeBytes + + , parseAvroJSON ) where import Control.Applicative @@ -33,7 +43,10 @@ import qualified Data.Aeson as A import Data.Aeson.Types (Parser, typeMismatch) import qualified Data.Avro.Types as Ty +import qualified Data.ByteString as B import qualified Data.ByteString.Base16 as Base16 +import qualified Data.Char as Char +import Data.Function (on) import Data.Hashable import qualified Data.HashMap.Strict as HashMap import Data.Int @@ -50,8 +63,9 @@ import Data.Text.Encoding as T import qualified Data.Vector as V import Prelude as P -import Text.Show.Functions +import Text.Show.Functions () + -- |An Avro schema is either -- * A "JSON object in the form `{"type":"typeName" ...` -- * A "JSON string, naming a defined type" (basic type w/o free variables/names) @@ -112,7 +126,8 @@ Array ty == Array ty2 = ty == ty2 Map ty == Map ty2 = ty == ty2 NamedType t == NamedType t2 = t == t2 - Record _ _ _ _ _ fs == Record _ _ _ _ _ fs2 = fs == fs2 + Record name1 ns1 _ _ _ fs1 == Record name2 ns2 _ _ _ fs2 = + and [name1 == name2, ns1 == ns2, fs1 == fs2] Enum _ _ _ _ s _ == Enum _ _ _ _ s2 _ = s == s2 Union a _ == Union b _ = a == b Fixed _ _ _ s == Fixed _ _ _ s2 = s == s2 @@ -282,7 +297,7 @@ ty <- o .: "type" let err = fail "Haskell Avro bindings does not support default for aliased or recursive types at this time." defM <- o .:! "default" - def <- case parseAvroJSON err ty <$> defM of + def <- case parseFieldDefault err ty <$> defM of Just (Success x) -> return (Just x) Just (Error e) -> fail e Nothing -> return Nothing @@ -314,14 +329,14 @@ Ty.Long i -> A.Number (fromIntegral i) Ty.Float f -> A.Number (realToFrac f) Ty.Double d -> A.Number (realToFrac d) - Ty.Bytes bs -> A.String ("\\u" <> T.decodeUtf8 (Base16.encode bs)) + Ty.Bytes bs -> A.String (serializeBytes bs) Ty.String t -> A.String t Ty.Array vec -> A.Array (V.map toJSON vec) Ty.Map mp -> A.Object (HashMap.map toJSON mp) Ty.Record _ flds -> A.Object (HashMap.map toJSON flds) Ty.Union _ _ Ty.Null -> A.Null Ty.Union _ ty val -> object [ typeName ty .= val ] - Ty.Fixed _ bs -> A.String ("\\u" <> T.decodeUtf8 (Base16.encode bs)) -- XXX the example wasn't literal - this should be an actual bytestring... somehow. + Ty.Fixed _ bs -> A.String (serializeBytes bs) Ty.Enum _ _ txt -> A.String txt data Result a = Success a | Error String @@ -363,25 +378,49 @@ traverse _ (Error err) = pure (Error err) traverse f (Success v) = Success <$> f v --- |Parse JSON-encoded avro data. -parseAvroJSON :: (Text -> Maybe Type) -> Type -> A.Value -> Result (Ty.Value Type) -parseAvroJSON env (NamedType (TN tn)) av = +-- | Field defaults are in the normal Avro JSON format except for +-- unions. Default values for unions are specified as JSON encodings +-- of the first type in the union. +parseFieldDefault :: (Text -> Maybe Type) -> Type -> A.Value -> Result (Ty.Value Type) +parseFieldDefault env schema value = parseAvroJSON defaultUnion env schema value + where defaultUnion (Union (t :| _) _) val = parseFieldDefault env t val + defaultUnion _ _ = error "Impossible: not Union." + +-- | Parse JSON-encoded avro data. +parseAvroJSON :: (Type -> A.Value -> Result (Ty.Value Type)) + -- ^ How to handle unions. The way unions are + -- formatted in JSON depends on whether we're parsing + -- a normal Avro object or we're parsing a default + -- declaration in a schema. + -- + -- This function will only ever be passed 'Union' + -- schemas. It /should/ error out if this is not the + -- case—it represents a bug in this code. + -> (Text -> Maybe Type) + -> Type + -> A.Value + -> Result (Ty.Value Type) +parseAvroJSON union env (NamedType (TN tn)) av = case env tn of Nothing -> fail $ "Could not resolve type name for " <> show tn - Just t -> parseAvroJSON env t av -parseAvroJSON env ty av = + Just t -> parseAvroJSON union env t av +parseAvroJSON union _ u@Union{} av = union u av +parseAvroJSON union env ty av = case av of - A.String s -> + A.String s -> case ty of - String -> return $ Ty.String s - Enum {..} -> + String -> return $ Ty.String s + Enum {..} -> if s `elem` symbols then return $ Ty.Enum ty (maybe (error "IMPOSSIBLE BUG") id $ lookup s (zip symbols [0..])) s else fail $ "JSON string is not one of the expected symbols for enum '" <> show name <> "': " <> T.unpack s - Union tys _ -> do - f <- tryAllTypes env tys av - maybe (fail $ "No match for String in union '" <> show (typeName ty) <> "'.") pure f - _ -> avroTypeMismatch ty "string" + Bytes -> Ty.Bytes <$> parseBytes s + Fixed {..} -> do + bytes <- parseBytes s + let len = B.length bytes + when (len /= size) $ + fail $ "Fixed string wrong size. Expected " <> show size <> " but got " <> show len + return $ Ty.Fixed ty bytes A.Bool b -> case ty of Boolean -> return $ Ty.Boolean b _ -> avroTypeMismatch ty "boolean" @@ -391,42 +430,42 @@ Long -> return $ Ty.Long (floor i) Float -> return $ Ty.Float (realToFrac i) Double -> return $ Ty.Double (realToFrac i) - Union tys _ -> do - f <- tryAllTypes env tys av - maybe (fail $ "No match for Number in union '" <> show (typeName ty) <> "'.") pure f _ -> avroTypeMismatch ty "number" A.Array vec -> case ty of - Array t -> Ty.Array <$> V.mapM (parseAvroJSON env t) vec - Union tys _ -> do - f <- tryAllTypes env tys av - maybe (fail $ "No match for Array in union '" <> show (typeName ty) <> "'.") pure f + Array t -> Ty.Array <$> V.mapM (parseAvroJSON union env t) vec _ -> avroTypeMismatch ty "array" A.Object obj -> case ty of - Map mTy -> Ty.Map <$> mapM (parseAvroJSON env mTy) obj + Map mTy -> Ty.Map <$> mapM (parseAvroJSON union env mTy) obj Record {..} -> do let lkAndParse f = case HashMap.lookup (fldName f) obj of Nothing -> case fldDefault f of Just v -> return v Nothing -> fail $ "Decode failure: No record field '" <> T.unpack (fldName f) <> "' and no default in schema." - Just v -> parseAvroJSON env (fldType f) v + Just v -> parseAvroJSON union env (fldType f) v Ty.Record ty . HashMap.fromList <$> mapM (\f -> (fldName f,) <$> lkAndParse f) fields - Union tys _ -> do - f <- tryAllTypes env tys av - maybe (fail $ "No match for given record in union '" <> show (typeName ty) <> "'.") pure f _ -> avroTypeMismatch ty "object" A.Null -> case ty of Null -> return Ty.Null - Union us _ | Null `elem` NE.toList us -> return $ Ty.Union us Null Ty.Null _ -> avroTypeMismatch ty "null" -tryAllTypes :: (Text -> Maybe Type) -> NonEmpty Type -> A.Value -> Result (Maybe (Ty.Value Type)) -tryAllTypes env tys av = - getFirst <$> foldMap (\t -> First . Just <$> parseAvroJSON env t av) (NE.toList tys) - `catchError` (\_ -> return mempty) +-- | Parses a string literal into a bytestring in the format expected +-- for bytes and fixed values. Will fail if every character does not +-- have a codepoint between 0 and 255. +parseBytes :: Text -> Result B.ByteString +parseBytes bytes = case T.find (not . inRange) bytes of + Just badChar -> fail $ "Invalid character in bytes or fixed string representation: " <> show badChar + Nothing -> return $ B.pack $ fromIntegral . Char.ord <$> T.unpack bytes + where inRange (Char.ord -> c) = c >= 0x00 && c <= 0xFF +-- | Turn a 'ByteString' into a 'Text' that matches the format Avro +-- expects from bytes and fixed literals in JSON. Each byte is mapped +-- to a single Unicode codepoint between 0 and 255. +serializeBytes :: B.ByteString -> Text +serializeBytes = T.pack . map (Char.chr . fromIntegral) . B.unpack + avroTypeMismatch :: Type -> Text -> Result a avroTypeMismatch expected actual = fail $ "Could not resolve type '" <> T.unpack actual <> "' with expected type: " <> show expected @@ -487,3 +526,22 @@ Fixed {..} -> mk name aliases namespace Array {..} -> go item _ -> [] + +-- | Checks that two schemas match. This is like equality of schemas, +-- except 'NamedTypes' match against other types /with the same name/. +-- +-- This extends recursively: two records match if they have the same +-- name, the same number of fields and the fields all match. +matches :: Type -> Type -> Bool +matches (NamedType (TN n)) t = n == typeName t +matches t (NamedType (TN n)) = typeName t == n +matches (Array itemA) (Array itemB) = matches itemA itemB +matches a@Record{} b@Record{} = + and [ name a == name b + , namespace a == namespace b + , length (fields a) == length (fields b) + , and $ zipWith fieldMatches (fields a) (fields b) + ] + where fieldMatches = matches `on` fldType +matches a@Union{} b@Union{} = and $ NE.zipWith matches (options a) (options b) +matches t1 t2 = t1 == t2 diff --git a/test/Avro/DefaultsSpec.hs b/test/Avro/DefaultsSpec.hs new file mode 100644 --- /dev/null +++ b/test/Avro/DefaultsSpec.hs @@ -0,0 +1,47 @@ +{-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE ScopedTypeVariables #-} +{-# LANGUAGE TemplateHaskell #-} +module Avro.DefaultsSpec +where + +import qualified Data.Aeson as J +import Data.Avro +import Data.Avro.Deriving +import Data.Avro.Schema +import qualified Data.Avro.Types as Ty +import qualified Data.HashMap.Strict as M +import qualified Data.Vector as V + +import Test.Hspec + +{-# ANN module ("HLint: ignore Redundant do" :: String) #-} + +deriveAvro "test/data/maybe.avsc" + +spec :: Spec +spec = describe "Avro.DefaultsSpec: Schema with named types" $ do + it "should decode value" $ + let msg = MaybeTest (Just "value") (FixedTag "\0\42\255") "\0\37\255" + in fromAvro (toAvro msg) `shouldBe` pure msg + + it "should decode no value" $ + let msg = MaybeTest Nothing (FixedTag "\0\42\255") "\0\37\255" + in fromAvro (toAvro msg) `shouldBe` pure msg + + it "should read default from Schema" $ + let + msgSchema = schemaOf (undefined :: MaybeTest) + fixedSchema = schemaOf (undefined :: FixedTag) + defaults = fldDefault <$> fields msgSchema + in defaults `shouldBe` [ Just Ty.Null + , Just $ Ty.Fixed fixedSchema "\0\42\255" + , Just $ Ty.Bytes "\0\37\255" + ] + + it "should encode schema with default" $ + let + msgSchema = schemaOf (undefined :: MaybeTest) + (J.Object jSchema) = J.toJSON msgSchema + (Just (J.Array flds)) = M.lookup "fields" jSchema + (J.Object jFld) = V.head flds + in M.lookup "default" jFld `shouldBe` Just J.Null diff --git a/test/Avro/JSONSpec.hs b/test/Avro/JSONSpec.hs new file mode 100644 --- /dev/null +++ b/test/Avro/JSONSpec.hs @@ -0,0 +1,95 @@ +{-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE ScopedTypeVariables #-} +{-# LANGUAGE TemplateHaskell #-} +module Avro.JSONSpec where + +import Control.Monad (forM_) + +import qualified Data.Aeson as Aeson +import qualified Data.ByteString.Lazy as LBS + +import Data.Avro.Deriving +import Data.Avro.JSON + +import Test.Hspec + +import Paths_avro + +import System.Directory (doesFileExist, getCurrentDirectory) +import System.Environment (setEnv) + +deriveAvro "test/data/enums.avsc" +deriveAvro "test/data/reused.avsc" +deriveAvro "test/data/small.avsc" +deriveAvro "test/data/unions.avsc" + +spec :: Spec +spec = describe "Avro.JSONSpec: JSON serialization/parsing" $ do + it "should do roundtrip (enums)" $ do + let msg = EnumWrapper + { enumWrapperId = 42 + , enumWrapperName = "Text" + , enumWrapperReason = EnumReasonBecause + } + parseJSON (Aeson.encode (toJSON msg)) `shouldBe` pure msg + it "should do roundtrip (reused)" $ do + let msg = ReusedWrapper + { reusedWrapperFull = ReusedChild 42 + , reusedWrapperInner = ContainerChild + { containerChildFstIncluded = ReusedChild 37 + , containerChildSndIncluded = ReusedChild 64 + } + } + parseJSON (Aeson.encode (toJSON msg)) `shouldBe` pure msg + it "should do roundtrip (small)" $ do + let msgs = + [ Endpoint + { endpointIps = ["192.168.1.1", "127.0.0.1"] + , endpointPorts = [PortRange 1 10, PortRange 11 20] + , endpointOpaque = Opaque "16-b-long-string" + , endpointCorrelation = Opaque "opaq-correlation" + , endpointTag = Left 14 + } + , Endpoint + { endpointIps = [] + , endpointPorts = [PortRange 1 10, PortRange 11 20] + , endpointOpaque = Opaque "opaque-long-text" + , endpointCorrelation = Opaque "correlation-data" + , endpointTag = Right "first-tag" + } + ] + forM_ msgs $ \ msg -> + parseJSON (Aeson.encode (toJSON msg)) `shouldBe` pure msg + + enumsExampleJSON <- runIO $ getFileName "test/data/enums-object.json" >>= LBS.readFile + it "should parse (enums)" $ do + let expected = EnumWrapper 37 "blarg" EnumReasonInstead + parseJSON enumsExampleJSON `shouldBe` pure expected + let unionsExampleA = Unions + { unionsScalars = Left "blarg" + , unionsNullable = Nothing + , unionsRecords = Left $ Foo { fooStuff = "stuff" } + , unionsSameFields = Left $ Foo { fooStuff = "foo stuff" } + } + unionsExampleB = Unions + { unionsScalars = Right 37 + , unionsNullable = Just 42 + , unionsRecords = Right $ Bar { barStuff = "stuff" + , barThings = Foo "things" + } + , unionsSameFields = Right $ NotFoo { notFooStuff = "not foo stuff" } + } + it "should roundtrip (unions)" $ do + forM_ [unionsExampleA, unionsExampleB] $ \ msg -> + parseJSON (Aeson.encode (toJSON msg)) `shouldBe` pure msg + unionsJsonA <- runIO $ getFileName "test/data/unions-object-a.json" >>= LBS.readFile + unionsJsonB <- runIO $ getFileName "test/data/unions-object-b.json" >>= LBS.readFile + it "should parse (unions)" $ do + parseJSON unionsJsonA `shouldBe` pure unionsExampleA + parseJSON unionsJsonB `shouldBe` pure unionsExampleB + +getFileName :: FilePath -> IO FilePath +getFileName p = do + path <- getDataFileName p + isOk <- doesFileExist path + pure $ if isOk then path else p diff --git a/test/Avro/THSimpleSpec.hs b/test/Avro/THSimpleSpec.hs --- a/test/Avro/THSimpleSpec.hs +++ b/test/Avro/THSimpleSpec.hs @@ -4,21 +4,23 @@ module Avro.THSimpleSpec where -import Control.Monad import Control.Lens +import Control.Monad + +import qualified Data.Aeson as J import Data.Aeson.Lens +import qualified Data.ByteString as B +import qualified Data.Char as Char +import Data.Monoid ((<>)) +import Data.Text (Text) +import qualified Data.Text as T + +import Test.Hspec + import Data.Avro import Data.Avro.Deriving import Data.Avro.Schema -import Data.Monoid ((<>)) -import qualified Data.Aeson as J -import Data.Text (Text) -import qualified Data.Text as T -import qualified Data.Text.Encoding as T -import qualified Data.ByteString.Base16 as Base16 -import Test.Hspec - {-# ANN module ("HLint: ignore Redundant do" :: String) #-} deriveAvro "test/data/small.avsc" @@ -63,5 +65,5 @@ json ^? key "tag" . _Value . key "string" . _String `shouldBe` endpointTag msg ^? _Right where encodeOpaque :: Opaque -> Text - encodeOpaque v = "\\u" <> T.decodeUtf8 (Base16.encode $ unOpaque v) + encodeOpaque (Opaque bs) = T.pack $ Char.chr . fromIntegral <$> B.unpack bs diff --git a/test/Avro/THUnionSpec.hs b/test/Avro/THUnionSpec.hs new file mode 100644 --- /dev/null +++ b/test/Avro/THUnionSpec.hs @@ -0,0 +1,32 @@ +{-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE ScopedTypeVariables #-} +{-# LANGUAGE TemplateHaskell #-} +module Avro.THUnionSpec +where + +import Data.Avro +import Data.Avro.Deriving + +import Test.Hspec + +deriveAvro "test/data/unions.avsc" + +spec :: Spec +spec = describe "Avro.THUnionSpec: Schema with unions." $ do + let objA = Unions + { unionsScalars = Left "foo" + , unionsNullable = Nothing + , unionsRecords = Left $ Foo { fooStuff = "stuff" } + , unionsSameFields = Left $ Foo { fooStuff = "more stuff" } + } + objB = Unions + { unionsScalars = Right 42 + , unionsNullable = Just 37 + , unionsRecords = Right $ Bar { barStuff = "stuff" + , barThings = Foo { fooStuff = "things" } + } + , unionsSameFields = Right $ NotFoo { notFooStuff = "different from Foo" } + } + it "records with unions should roundtrip" $ do + fromAvro (toAvro objA) `shouldBe` pure objA + fromAvro (toAvro objB) `shouldBe` pure objB diff --git a/test/DecodeContainer.hs b/test/DecodeContainer.hs new file mode 100644 --- /dev/null +++ b/test/DecodeContainer.hs @@ -0,0 +1,25 @@ +module DecodeContainer +where + +import qualified Data.Aeson as A +import Data.Avro.Decode +import Data.Avro.Deconflict as D +import qualified Data.ByteString.Lazy as BL +import System.Environment + +main :: IO () +main = + do (file:rest) <- getArgs + cont <- BL.readFile file + case decodeContainer cont of + Left e -> print e + Right (s,v) -> + do putStrLn $ "Schema: " ++ show s + putStrLn "--------------------------------------------------" + print v + case rest of + [schFile] -> do + putStrLn "---- DECONFLICTED ------" + Just readerSchema <- A.decode <$> BL.readFile schFile + print (map (map (D.deconflict s readerSchema)) v) + _ -> return () diff --git a/test/Example1.hs b/test/Example1.hs new file mode 100644 --- /dev/null +++ b/test/Example1.hs @@ -0,0 +1,84 @@ +{-# LANGUAGE OverloadedStrings #-} +module Example1 +where +import Data.Avro +import Data.Avro.Schema +import qualified Data.Avro.Types as Ty +import qualified Data.ByteString.Lazy as BL +import Data.List.NonEmpty (NonEmpty (..)) +import Data.Map as M +import Data.Text (Text) +import qualified Data.Text as Text + +data MyEnum = A | B | C | D deriving (Eq,Ord,Show,Enum) +data MyStruct = MyStruct (Either MyEnum Text) Int deriving (Eq,Ord,Show) + +-- Explicit 'Schema' types that specify the Avro encoding of a structure. +-- Schema's often come from an external JSON definition (.avsc files) or +-- embedded in object files. +meSchema :: Schema +meSchema = mkEnum "MyEnum" [] Nothing Nothing ["A","B","C","D"] + +msSchema :: Schema +msSchema = + Record "MyStruct" Nothing [] Nothing Nothing + [ fld "enumOrString" eOrS (Just $ Ty.String "The Default") + , fld "intvalue" Long Nothing + ] + where + fld nm ty def = Field nm [] Nothing Nothing ty def + eOrS = mkUnion (meSchema :| [String]) + +-- Encoding data, via the ToAvro class, requires both the routine that encodes +-- data as well as the schema under which it is encoded. The encoding and +-- schema must match, though there is no type or programmatic routine that enforces +-- this law. +instance HasAvroSchema MyEnum where + schema = pure meSchema + +instance ToAvro MyEnum where + toAvro x = Ty.Enum meSchema (fromEnum x) (Text.pack $ show x) + -- schema = pure meSchema + +instance HasAvroSchema MyStruct where + schema = pure msSchema + +instance ToAvro MyStruct where + toAvro (MyStruct ab i) = + record msSchema + [ "enumOrString" .= ab + , "intvalue" .= i + ] + -- schema = pure msSchema + +-- Much like Aeson, decoding data is involves pattern matching the value +-- constructor then building the ADT. +instance FromAvro MyStruct where + fromAvro (Ty.Record _fs r) = + MyStruct <$> r .: "enumOrString" + <*> r .: "intvalue" + fromAvro v = badValue v "MyStruct" + +instance FromAvro MyEnum where + fromAvro (Ty.Enum _ i _) = pure (toEnum i) + fromAvro v = badValue v "MyEnum" + +main = do + let valR = MyStruct (Right "Hello") 1 + encR = toAvro valR + valL = MyStruct (Left C) (negate 1944) + encL = toAvro valL + putStrLn "----------- MS Right value -------------" + print (fromAvro encR `asTypeOf` Success valR) + print (fromAvro encR == Success valR) + putStrLn "----------- MS Left value --------------" + print (fromAvro encL `asTypeOf` Success valL) + print (fromAvro encL == Success valL) + putStrLn "----------- MS Right full bytestring enc/dec--------------" + print (BL.unpack $ encode valR) + print (decode (encode valR) `asTypeOf` Success valR) + print (decode (encode valR) == Success valR) + putStrLn "----------- MS Left full bytestring enc/dec--------------" + print (BL.unpack $ encode valL) + print (decode (encode valL) `asTypeOf` Success valL) + print (decode (encode valL) == Success valL) diff --git a/test/data/enums-object.json b/test/data/enums-object.json new file mode 100644 --- /dev/null +++ b/test/data/enums-object.json @@ -0,0 +1,1 @@ +{"id":37,"name":"blarg","reason":"Instead"} diff --git a/test/data/maybe.avsc b/test/data/maybe.avsc --- a/test/data/maybe.avsc +++ b/test/data/maybe.avsc @@ -1,7 +1,19 @@ { - "type": "record", - "name": "MaybeTest", - "fields": [ - { "name": "tag", "type": ["null", "string"], "default": null } - ] - } + "type": "record", + "name": "MaybeTest", + "fields": [ + { "name": "tag", "type": ["null", "string"], "default": null }, + { "name": "fixedTag", + "type": { + "type": "fixed", + "name": "FixedTag", + "size": 3 + }, + "default": "\u0000\u002a\u00ff" + }, + { "name": "bytesTag", + "type": "bytes", + "default": "\u0000\u0025\u00ff" + } + ] +} diff --git a/test/data/unions-object-a.json b/test/data/unions-object-a.json new file mode 100644 --- /dev/null +++ b/test/data/unions-object-a.json @@ -0,0 +1,16 @@ +{ + "scalars": { + "string": "blarg" + }, + "nullable": null, + "records": { + "Foo": { + "stuff": "stuff" + } + }, + "sameFields": { + "Foo": { + "stuff": "foo stuff" + } + } +} diff --git a/test/data/unions-object-b.json b/test/data/unions-object-b.json new file mode 100644 --- /dev/null +++ b/test/data/unions-object-b.json @@ -0,0 +1,21 @@ +{ + "scalars": { + "long": 37 + }, + "nullable": { + "int": 42 + }, + "records": { + "Bar": { + "stuff": "stuff", + "things": { + "stuff": "things" + } + } + }, + "sameFields": { + "NotFoo": { + "stuff": "not foo stuff" + } + } +} diff --git a/test/data/unions.avsc b/test/data/unions.avsc new file mode 100644 --- /dev/null +++ b/test/data/unions.avsc @@ -0,0 +1,46 @@ +{ "type" : "record", + "name" : "Unions", + "namespace" : "haskell.avro.example", + "fields" : [ + { "name" : "scalars", + "type" : ["string", "long"] + }, + { "name" : "nullable", + "type" : ["null", "int"] + }, + { "name" : "records", + "type" : [ + { "type" : "record", + "name" : "Foo", + "fields" : [ + { "name" : "stuff", + "type" : "string" + } + ] + }, + { "type" : "record", + "name" : "Bar", + "fields" : [ + { "name" : "stuff", + "type" : "string" + }, + { "name" : "things", + "type" : "Foo" + } + ] + } + ] + }, + { "name" : "sameFields", + "type" : [ + "Foo", + { "type" : "record", + "name" : "NotFoo", + "fields" : [ + { "name" : "stuff", "type" : "string" } + ] + } + ] + } + ] +}