diff --git a/ChangeLog.md b/ChangeLog.md --- a/ChangeLog.md +++ b/ChangeLog.md @@ -1,9 +1,24 @@ # Revision history for avro -## HEAD -- 2018-11-07 +## 0.4.1.1 -* Fixed an omitted data fixture from the cabal sdist +- Fixed bugs in handling of namespaces when parsing and printing avro types +- Fixed a schema overlay test +## 0.4.1.0 + +- Fixed an omitted data fixture from the cabal sdist +- Improvements on experimental lazy decoding (up to 25% faster on our tests) +- Useful instances for EitherN + +## 0.4.0.0 + +- Technical release to respect potentially breaking changes introduced earlier. + +## 0.3.6.1 + +- Fixed Data.Avro.Schema.extractBindings by @TikhonJelvis + ## 0.1.0.0 -- YYYY-mm-dd -* First version. Released on an unsuspecting world. +- First version. Released on an unsuspecting world. diff --git a/README.md b/README.md new file mode 100644 --- /dev/null +++ b/README.md @@ -0,0 +1,146 @@ +# Native Haskell implementation of Avro + +This is a Haskell [Avro](https://avro.apache.org/) library useful for decoding +and encoding Avro data structures. Avro can be thought of as a serialization +format and RPC specification which induces three separable tasks: + +* *Serialization*/*Deserialization* - This library has been used "in anger" for: + - Deserialization of avro container files + - Serialization/deserialization Avro messages to/from Kafka topics +* *RPC* - There is currently no support for Avro RPC in this library. + +This library also provides functionality for automatically generating Avro-related data types and instances from Avro schemas (using TemplateHaskell). + +# Quickstart + +This library provides the following conversions between Haskell types and Avro types: + +| Haskell type | Avro type | +|:------------------|:--------------------------------| +| () | "null" | +| Bool | "boolean" | +| Int, Int64 | "long" | +| Int32 | "int" | +| Double | "double" | +| Text | "string" | +| ByteString | "bytes" | +| Maybe a | ["null", "a"] | +| Either a b | ["a", "b"] | +| Map Text a | {"type": "map", "value": "a"} | +| Map String a | {"type": "map", "value": "a"} | +| HashMap Text a | {"type": "map", "value": "a"} | +| HashMap String a | {"type": "map", "value": "a"} | +| [a] | {"type": "array", "value": "a"} | + +User defined data types should provide `HasAvroSchema`/`ToAvro`/`FromAvro` instances to be encoded/decoded to/from Avro. + +## Defining types and `HasAvroSchema` / `FromAvro` / `ToAvro` manually + +Typically these imports are useful: +``` +import Data.Avro +import Data.Avro.Schema as S +import qualified Data.Avro.Types as AT +``` + +Assuming there is a data type to be encoded/decoded from/to Avro: +``` +data Gender = Male | Female deriving (Eq, Ord, Show, Enum) +data Person = Person + { fullName :: Text + , age :: Int32 + , gender :: Gender + , ssn :: Maybe Text + } deriving (Show, Eq) +``` + +Avro schema for this type can be defined as: +``` +genderSchema :: Schema +genderSchema = mkEnum "Gender" [] Nothing Nothing ["Male", "Female"] + +personSchema :: Schema +personSchema = + Record "Person" Nothing [] Nothing Nothing + [ fld "name" String Nothing + , fld "age" Int Nothing + , fld "gender" genderSchema Nothing + , fld "ssn" (mkUnion $ Null :| [String]) Nothing + ] + where + fld nm ty def = Field nm [] Nothing Nothing ty def + +instance HasAvroSchema Person where + schema = pure personSchema +``` + +`ToAvro` instance for `Person` can be defined as: +``` +instance ToAvro Person where + schema = pure personSchema + toAvro p = record personSchema + [ "name" .= fullName p + , "age" .= age p + , "gender" .= gender p + , "ssn" .= ssn p + ] +``` + +`FromAvro` instance for `Person` can be defined as: +``` +instance FromAvro Person where + fromAvro (AT.Record _ r) = + Person <$> r .: "name" + <*> r .: "age" + <*> r .: "gender" + <*> r .: "ssn" + fromAvro r = badValue r "Person" +``` + +## Defining types and `HasAvroSchema` / `FromAvro` / `ToAvro` "automatically" +This library provides functionality to derive Haskell data types and `HasAvroSchema`/`FromAvro`/`ToAvro` instances "automatically" from already existing Avro schemas (using TemplateHaskell). + +### Examples + +`deriveAvro` will derive data types, `FromAvro` and `ToAvro` instances from a provided Avro schema file: +``` +{-# LANGUAGE TemplateHaskell #-} +{-# LANGUAGE DeriveGeneric #-} +import Data.Avro.Deriving + +deriveAvro "schemas/contract.avsc" +``` + +Similarly, `deriveFromAvro` can be used to only derive data types and `FromAvro`, but not `ToAvro` instances. + +If you prefer defining Avro schema in Haskell and not in `avsc`, then `deriveAvro'` can be used instead of `deriveAvro`. + +### Conventions +When Haskell data types are generated, these conventions are followed: + +- Type and field names are "sanitized": +all the charachers except `[a-z,A-Z,',_]` are removed from names +- Field names are prefixed with the name of the record they are declared in. + +For example, if Avro schema defines `Person` record as: +``` +{ "type": "record", + "name": "Person", + "fields": [ + { "name": "name", "type": "string"} + ] +} +``` + +then generated Haskell type will look like: +``` +data Person = Person + { personName :: Text + } deriving (Show, Eq) +``` + +### Limitations +Two-parts unions like `["null", "MyType"]` or `["MyType", "YourType"]` are supported (as Haskell's `Maybe MyType` and `Either MyType YourType`), but multi-parts unions are currently _not_ supported. +It is not due to any fundamental problems but because it has not been done yet. PRs are welcomed! :) +# TODO +Please see the [TODO](TODO) diff --git a/avro.cabal b/avro.cabal --- a/avro.cabal +++ b/avro.cabal @@ -1,11 +1,13 @@ --- This file has been generated from package.yaml by hpack version 0.28.2. +cabal-version: 1.12 + +-- This file has been generated from package.yaml by hpack version 0.31.1. -- -- see: https://github.com/sol/hpack -- --- hash: e0011cbcce3a1d5f4de5830d2b9f6acb6f32dcacc90e79bf61dbec2530bbf88d +-- hash: 0368e22fa6552aa4d9b3ada7a95e8210af8b40090f3244fe99d68ca30f606e8d name: avro -version: 0.4.1.0 +version: 0.4.1.1 synopsis: Avro serialization support for Haskell description: Avro serialization and deserialization support for Haskell category: Data @@ -16,27 +18,27 @@ license: BSD3 license-file: LICENSE build-type: Simple -cabal-version: >= 1.10 extra-source-files: + README.md ChangeLog.md - test/data/deconflict/reader.avsc - test/data/deconflict/writer.avsc - test/data/enums-object.json test/data/enums.avsc test/data/internal-bindings.avsc test/data/karma.avsc test/data/logical.avsc test/data/maybe.avsc - test/data/namespace-inference.json - test/data/overlay/composite.avsc - test/data/overlay/expectation.avsc - test/data/overlay/primitives.avsc test/data/record.avsc test/data/reused.avsc test/data/small.avsc + test/data/unions.avsc + test/data/enums-object.json + test/data/namespace-inference.json test/data/unions-object-a.json test/data/unions-object-b.json - test/data/unions.avsc + test/data/deconflict/reader.avsc + test/data/deconflict/writer.avsc + test/data/overlay/composite.avsc + test/data/overlay/expectation.avsc + test/data/overlay/primitives.avsc source-repository head type: git diff --git a/src/Data/Avro/Schema.hs b/src/Data/Avro/Schema.hs --- a/src/Data/Avro/Schema.hs +++ b/src/Data/Avro/Schema.hs @@ -189,6 +189,9 @@ -- as @"com.example.Foo"@. -- -- Fullnames have to be globally unique inside an Avro schema. +-- +-- A namespace of @[]@ or @[""]@ is the "null namespace". In avro +-- an explicitly null-namespaced identifier is written as ".Foo" data TypeName = TN { baseName :: T.Text , namespace :: [T.Text] } @@ -205,8 +208,14 @@ -- > renderFullname (TN "Foo" ["com", "example"]) -- "com.example.Foo" -- @ +-- +-- @ +-- > renderFullname (TN "Foo" []) +-- ".Foo" +-- @ renderFullname :: TypeName -> T.Text -renderFullname TN { baseName, namespace } = T.intercalate "." $ namespace <> [baseName] +renderFullname TN { baseName, namespace } = + T.intercalate "." namespace <> "." <> baseName -- | Parses a fullname into a 'TypeName', assuming the string -- representation is valid. @@ -216,10 +225,10 @@ -- TN { baseName = "Foo", components = ["com", "example"] } -- @ parseFullname :: T.Text -> TypeName -parseFullname (T.splitOn "." -> components) = TN - { baseName = last components - , namespace = init components - } +parseFullname (T.splitOn "." -> components) = TN { baseName, namespace } + where + baseName = last components + namespace = filter (/= "") (init components) -- | Build a type name out of the @name@ and @namespace@ fields of an -- Avro record, enum or fixed definition. diff --git a/test/data/overlay/composite.avsc b/test/data/overlay/composite.avsc --- a/test/data/overlay/composite.avsc +++ b/test/data/overlay/composite.avsc @@ -20,13 +20,17 @@ }, { "name": "foo-bar-array", - "type": "array", - "items": "avro.test.data.overlay.primitive.foo-bar" + "type": { + "type": "array", + "items": "avro.test.data.overlay.primitive.foo-bar" + } }, { "name": "foo-bar-map", - "type": "map", - "values": "avro.test.data.overlay.primitive.foo-bar" + "type": { + "type": "map", + "values": "avro.test.data.overlay.primitive.foo-bar" + } }, { "name": "foo-bar-int-union",