diff --git a/avro.cabal b/avro.cabal --- a/avro.cabal +++ b/avro.cabal @@ -2,10 +2,10 @@ -- -- see: https://github.com/sol/hpack -- --- hash: ba78df8186da1a2bbab610186df6edbf105eaf622ce7e3f1681252bb60941fb9 +-- hash: d00924760f916aaff838af8858aa0535e6b78f32409c94cd4ca2258f7b68f921 name: avro -version: 0.3.0.1 +version: 0.3.0.2 synopsis: Avro serialization support for Haskell description: Avro serialization and deserialization support for Haskell category: Data diff --git a/src/Data/Avro.hs b/src/Data/Avro.hs --- a/src/Data/Avro.hs +++ b/src/Data/Avro.hs @@ -79,6 +79,7 @@ , decode , decodeWithSchema , decodeContainer + , decodeContainerWithSchema , decodeContainerBytes , encode , encodeContainer @@ -117,24 +118,35 @@ type Avro a = (FromAvro a, ToAvro a) --- |Decode a lazy bytestring using a given Schema. +-- | Decode a lazy bytestring using a Schema for the return type. decode :: forall a. FromAvro a => ByteString -> Result a decode bytes = case D.decodeAvro (untag (schema :: Tagged a Type)) bytes of Right val -> fromAvro val Left err -> Error err +-- | Decode a lazy bytestring with a provided schema decodeWithSchema :: FromAvro a => Schema -> ByteString -> Result a decodeWithSchema sch bytes = case D.decodeAvro sch bytes of Right val -> fromAvro val Left err -> Error err +-- | Decode a container and de-conflict the writer schema with +-- a reader schema for a return type. +-- Like in 'decodeContainerWithSchema' +-- exceptions are thrown instead of a 'Result' type to +-- allow this function to be read lazy (to be done in some later version). +decodeContainer :: forall a. FromAvro a => ByteString -> [[a]] +decodeContainer bs = + let readerSchema = untag (schema :: Tagged a Schema) + in decodeContainerWithSchema readerSchema bs + -- |Decode a container and de-conflict the writer schema with a given -- reader-schema. Exceptions are thrown instead of a 'Result' type to -- allow this function to be read lazy (to be done in some later version). -decodeContainer :: FromAvro a => Schema -> ByteString -> [[a]] -decodeContainer readerSchema bs = +decodeContainerWithSchema :: FromAvro a => Schema -> ByteString -> [[a]] +decodeContainerWithSchema readerSchema bs = case D.decodeContainer bs of Right (writerSchema,val) -> let err e = error $ "Could not deconflict reader and writer schema." <> e @@ -147,15 +159,25 @@ in P.map (P.map dec) val Left err -> error err +-- | Encodes a value to a lazy ByteString encode :: ToAvro a => a -> BL.ByteString encode = E.encodeAvro . toAvro -encodeContainer :: ToAvro a => [[a]] -> IO BL.ByteString -encodeContainer = E.encodeContainer . map (map toAvro) +-- | Encode chunks of objects into a container, using 16 random bytes for +-- the synchronization markers. +encodeContainer :: forall a. ToAvro a => [[a]] -> IO BL.ByteString +encodeContainer = + let sch = untag (schema :: Tagged a Schema) + in E.encodeContainer sch . map (map toAvro) -encodeContainerWithSync :: ToAvro a => (Word64,Word64,Word64,Word64) -> [[a]] -> BL.ByteString -encodeContainerWithSync (a,b,c,d) = E.encodeContainerWithSync s . map (map toAvro) - where s = P.runPut $ mapM_ P.putWord64le [a,b,c,d] +-- | Encode chunks of objects into a container, using the provided +-- ByteString as the synchronization markers +encodeContainerWithSync :: forall a. ToAvro a => (Word64,Word64,Word64,Word64) -> [[a]] -> BL.ByteString +encodeContainerWithSync (a,b,c,d) = + let + sch = untag (schema :: Tagged a Schema) + syncBytes = P.runPut $ mapM_ P.putWord64le [a,b,c,d] + in E.encodeContainerWithSync sch syncBytes . map (map toAvro) -- |Like 'decodeContainer' but returns the avro-encoded bytes for each -- object in the container instead of the Haskell type. diff --git a/src/Data/Avro/Encode.hs b/src/Data/Avro/Encode.hs --- a/src/Data/Avro/Encode.hs +++ b/src/Data/Avro/Encode.hs @@ -1,9 +1,9 @@ -{-# LANGUAGE TypeSynonymInstances #-} -{-# LANGUAGE RecordWildCards #-} -{-# LANGUAGE ScopedTypeVariables #-} {-# LANGUAGE FlexibleInstances #-} {-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE RecordWildCards #-} +{-# LANGUAGE ScopedTypeVariables #-} {-# LANGUAGE TypeFamilies #-} +{-# LANGUAGE TypeSynonymInstances #-} module Data.Avro.Encode ( -- * High level interface @@ -16,64 +16,64 @@ , putAvro ) where -import Prelude as P -import qualified Data.Aeson as A -import qualified Data.Array as Ar -import Data.Ix (Ix) +import qualified Data.Aeson as A +import qualified Data.Array as Ar +import qualified Data.Binary.IEEE754 as IEEE import Data.Bits -import Data.ByteString.Lazy as BL -import qualified Data.Binary.IEEE754 as IEEE -import Data.ByteString.Lazy.Char8 () -import qualified Data.ByteString as B +import qualified Data.ByteString as B import Data.ByteString.Builder -import qualified Data.Foldable as F -import Data.HashMap.Strict (HashMap) -import qualified Data.HashMap.Strict as HashMap +import Data.ByteString.Lazy as BL +import Data.ByteString.Lazy.Char8 () +import qualified Data.Foldable as F +import Data.HashMap.Strict (HashMap) +import qualified Data.HashMap.Strict as HashMap import Data.Int -import Data.List as DL -import Data.List.NonEmpty (NonEmpty(..)) -import qualified Data.List.NonEmpty as NE +import Data.Ix (Ix) +import Data.List as DL +import Data.List.NonEmpty (NonEmpty (..)) +import qualified Data.List.NonEmpty as NE +import Data.Maybe (catMaybes, mapMaybe) import Data.Monoid -import Data.Maybe (catMaybes, mapMaybe) -import qualified Data.Set as S -import Data.Text (Text) -import qualified Data.Text as T -import qualified Data.Text.Encoding as T -import qualified Data.Text.Lazy as TL -import qualified Data.Text.Lazy.Encoding as TL -import qualified Data.Vector as V -import qualified Data.Vector.Unboxed as U -import Data.Word import Data.Proxy -import System.Entropy (getEntropy) +import qualified Data.Set as S +import Data.Text (Text) +import qualified Data.Text as T +import qualified Data.Text.Encoding as T +import qualified Data.Text.Lazy as TL +import qualified Data.Text.Lazy.Encoding as TL +import qualified Data.Vector as V +import qualified Data.Vector.Unboxed as U +import Data.Word +import Prelude as P +import System.Entropy (getEntropy) -import Data.Avro.EncodeRaw -import Data.Avro.Schema as S -import Data.Avro.Types as T -import Data.Avro.Zag -import Data.Avro.Zig +import Data.Avro.EncodeRaw +import Data.Avro.HasAvroSchema +import Data.Avro.Schema as S +import Data.Avro.Types as T +import Data.Avro.Zag +import Data.Avro.Zig encodeAvro :: EncodeAvro a => a -> BL.ByteString encodeAvro = toLazyByteString . putAvro -- |Encode chunks of objects into a container, using 16 random bytes for -- the synchronization markers. -encodeContainer :: EncodeAvro a => [[a]] -> IO BL.ByteString -encodeContainer xss = +encodeContainer :: EncodeAvro a => Schema -> [[a]] -> IO BL.ByteString +encodeContainer sch xss = do sync <- getEntropy 16 - return $ encodeContainerWithSync (BL.fromStrict sync) xss + return $ encodeContainerWithSync sch (BL.fromStrict sync) xss -- |Encode chunks of objects into a container, using the provided -- ByteString as the synchronization markers. -encodeContainerWithSync :: EncodeAvro a => BL.ByteString -> [[a]] -> BL.ByteString -encodeContainerWithSync syncBytes xss = +encodeContainerWithSync :: EncodeAvro a => Schema -> BL.ByteString -> [[a]] -> BL.ByteString +encodeContainerWithSync sch syncBytes xss = toLazyByteString $ lazyByteString avroMagicBytes <> - putAvro (HashMap.fromList [("avro.schema", A.encode objSchema), ("avro.codec","null")] :: HashMap Text BL.ByteString) <> + putAvro (HashMap.fromList [("avro.schema", A.encode sch), ("avro.codec","null")] :: HashMap Text BL.ByteString) <> lazyByteString syncBytes <> foldMap putBlocks xss where - objSchema = getSchema (P.head (P.head xss)) putBlocks ys = let nrObj = P.length ys nrBytes = BL.length theBytes diff --git a/src/Data/Avro/HasAvroSchema.hs b/src/Data/Avro/HasAvroSchema.hs --- a/src/Data/Avro/HasAvroSchema.hs +++ b/src/Data/Avro/HasAvroSchema.hs @@ -1,8 +1,10 @@ -{-# LANGUAGE FlexibleInstances #-} -{-# LANGUAGE ConstraintKinds #-} -{-# LANGUAGE ScopedTypeVariables #-} +{-# LANGUAGE ConstraintKinds #-} +{-# LANGUAGE FlexibleInstances #-} +{-# LANGUAGE ScopedTypeVariables #-} +{-# LANGUAGE TypeSynonymInstances #-} module Data.Avro.HasAvroSchema where +import qualified Data.Array as Ar import Data.Avro.Schema as S import Data.Avro.Types as T import qualified Data.ByteString as B @@ -10,16 +12,19 @@ import qualified Data.ByteString.Lazy as BL import qualified Data.HashMap.Strict as HashMap import Data.Int +import Data.Ix (Ix) +import Data.List.NonEmpty (NonEmpty (..)) import qualified Data.Map as Map import Data.Monoid ((<>)) +import Data.Proxy +import qualified Data.Set as S +import Data.Tagged import Data.Text (Text) import qualified Data.Text as Text -import Data.List.NonEmpty (NonEmpty(..)) import qualified Data.Text.Lazy as TL -import Data.Tagged import qualified Data.Vector as V +import qualified Data.Vector.Unboxed as U import Data.Word -import Data.Proxy class HasAvroSchema a where schema :: Tagged a Type @@ -27,6 +32,18 @@ schemaOf :: (HasAvroSchema a) => a -> Type schemaOf = witness schema +instance HasAvroSchema Word8 where + schema = Tagged S.Int + +instance HasAvroSchema Word16 where + schema = Tagged S.Int + +instance HasAvroSchema Word32 where + schema = Tagged S.Long + +instance HasAvroSchema Word64 where + schema = Tagged S.Long + instance HasAvroSchema Bool where schema = Tagged S.Boolean @@ -36,6 +53,12 @@ instance HasAvroSchema Int where schema = Tagged S.Long +instance HasAvroSchema Int8 where + schema = Tagged S.Int + +instance HasAvroSchema Int16 where + schema = Tagged S.Int + instance HasAvroSchema Int32 where schema = Tagged S.Int @@ -85,6 +108,18 @@ schema = Tagged $ mkUnion (S.Null:| [untag (schema :: Tagged a Type)]) instance (HasAvroSchema a) => HasAvroSchema [a] where + schema = wrapTag S.Array (schema :: Tagged a Type) + +instance (HasAvroSchema a, Ix i) => HasAvroSchema (Ar.Array i a) where + schema = wrapTag S.Array (schema :: Tagged a Type) + +instance HasAvroSchema a => HasAvroSchema (V.Vector a) where + schema = wrapTag S.Array (schema :: Tagged a Type) + +instance HasAvroSchema a => HasAvroSchema (U.Vector a) where + schema = wrapTag S.Array (schema :: Tagged a Type) + +instance HasAvroSchema a => HasAvroSchema (S.Set a) where schema = wrapTag S.Array (schema :: Tagged a Type) wrapTag :: (Type -> Type) -> Tagged a Type -> Tagged b Type