avro 0.3.0.1 → 0.3.0.2
raw patch · 4 files changed
+113/−56 lines, 4 files
Files
- avro.cabal +2/−2
- src/Data/Avro.hs +30/−8
- src/Data/Avro/Encode.hs +40/−40
- src/Data/Avro/HasAvroSchema.hs +41/−6
avro.cabal view
@@ -2,10 +2,10 @@ -- -- see: https://github.com/sol/hpack ----- hash: ba78df8186da1a2bbab610186df6edbf105eaf622ce7e3f1681252bb60941fb9+-- hash: d00924760f916aaff838af8858aa0535e6b78f32409c94cd4ca2258f7b68f921 name: avro-version: 0.3.0.1+version: 0.3.0.2 synopsis: Avro serialization support for Haskell description: Avro serialization and deserialization support for Haskell category: Data
src/Data/Avro.hs view
@@ -79,6 +79,7 @@ , decode , decodeWithSchema , decodeContainer+ , decodeContainerWithSchema , decodeContainerBytes , encode , encodeContainer@@ -117,24 +118,35 @@ type Avro a = (FromAvro a, ToAvro a) --- |Decode a lazy bytestring using a given Schema.+-- | Decode a lazy bytestring using a Schema for the return type. decode :: forall a. FromAvro a => ByteString -> Result a decode bytes = case D.decodeAvro (untag (schema :: Tagged a Type)) bytes of Right val -> fromAvro val Left err -> Error err +-- | Decode a lazy bytestring with a provided schema decodeWithSchema :: FromAvro a => Schema -> ByteString -> Result a decodeWithSchema sch bytes = case D.decodeAvro sch bytes of Right val -> fromAvro val Left err -> Error err +-- | Decode a container and de-conflict the writer schema with+-- a reader schema for a return type.+-- Like in 'decodeContainerWithSchema'+-- exceptions are thrown instead of a 'Result' type to+-- allow this function to be read lazy (to be done in some later version).+decodeContainer :: forall a. FromAvro a => ByteString -> [[a]]+decodeContainer bs =+ let readerSchema = untag (schema :: Tagged a Schema)+ in decodeContainerWithSchema readerSchema bs+ -- |Decode a container and de-conflict the writer schema with a given -- reader-schema. Exceptions are thrown instead of a 'Result' type to -- allow this function to be read lazy (to be done in some later version).-decodeContainer :: FromAvro a => Schema -> ByteString -> [[a]]-decodeContainer readerSchema bs =+decodeContainerWithSchema :: FromAvro a => Schema -> ByteString -> [[a]]+decodeContainerWithSchema readerSchema bs = case D.decodeContainer bs of Right (writerSchema,val) -> let err e = error $ "Could not deconflict reader and writer schema." <> e@@ -147,15 +159,25 @@ in P.map (P.map dec) val Left err -> error err +-- | Encodes a value to a lazy ByteString encode :: ToAvro a => a -> BL.ByteString encode = E.encodeAvro . toAvro -encodeContainer :: ToAvro a => [[a]] -> IO BL.ByteString-encodeContainer = E.encodeContainer . map (map toAvro)+-- | Encode chunks of objects into a container, using 16 random bytes for+-- the synchronization markers.+encodeContainer :: forall a. ToAvro a => [[a]] -> IO BL.ByteString+encodeContainer =+ let sch = untag (schema :: Tagged a Schema)+ in E.encodeContainer sch . map (map toAvro) -encodeContainerWithSync :: ToAvro a => (Word64,Word64,Word64,Word64) -> [[a]] -> BL.ByteString-encodeContainerWithSync (a,b,c,d) = E.encodeContainerWithSync s . map (map toAvro)- where s = P.runPut $ mapM_ P.putWord64le [a,b,c,d]+-- | Encode chunks of objects into a container, using the provided+-- ByteString as the synchronization markers+encodeContainerWithSync :: forall a. ToAvro a => (Word64,Word64,Word64,Word64) -> [[a]] -> BL.ByteString+encodeContainerWithSync (a,b,c,d) =+ let+ sch = untag (schema :: Tagged a Schema)+ syncBytes = P.runPut $ mapM_ P.putWord64le [a,b,c,d]+ in E.encodeContainerWithSync sch syncBytes . map (map toAvro) -- |Like 'decodeContainer' but returns the avro-encoded bytes for each -- object in the container instead of the Haskell type.
src/Data/Avro/Encode.hs view
@@ -1,9 +1,9 @@-{-# LANGUAGE TypeSynonymInstances #-}-{-# LANGUAGE RecordWildCards #-}-{-# LANGUAGE ScopedTypeVariables #-} {-# LANGUAGE FlexibleInstances #-} {-# LANGUAGE OverloadedStrings #-}+{-# LANGUAGE RecordWildCards #-}+{-# LANGUAGE ScopedTypeVariables #-} {-# LANGUAGE TypeFamilies #-}+{-# LANGUAGE TypeSynonymInstances #-} module Data.Avro.Encode ( -- * High level interface@@ -16,64 +16,64 @@ , putAvro ) where -import Prelude as P-import qualified Data.Aeson as A-import qualified Data.Array as Ar-import Data.Ix (Ix)+import qualified Data.Aeson as A+import qualified Data.Array as Ar+import qualified Data.Binary.IEEE754 as IEEE import Data.Bits-import Data.ByteString.Lazy as BL-import qualified Data.Binary.IEEE754 as IEEE-import Data.ByteString.Lazy.Char8 ()-import qualified Data.ByteString as B+import qualified Data.ByteString as B import Data.ByteString.Builder-import qualified Data.Foldable as F-import Data.HashMap.Strict (HashMap)-import qualified Data.HashMap.Strict as HashMap+import Data.ByteString.Lazy as BL+import Data.ByteString.Lazy.Char8 ()+import qualified Data.Foldable as F+import Data.HashMap.Strict (HashMap)+import qualified Data.HashMap.Strict as HashMap import Data.Int-import Data.List as DL-import Data.List.NonEmpty (NonEmpty(..))-import qualified Data.List.NonEmpty as NE+import Data.Ix (Ix)+import Data.List as DL+import Data.List.NonEmpty (NonEmpty (..))+import qualified Data.List.NonEmpty as NE+import Data.Maybe (catMaybes, mapMaybe) import Data.Monoid-import Data.Maybe (catMaybes, mapMaybe)-import qualified Data.Set as S-import Data.Text (Text)-import qualified Data.Text as T-import qualified Data.Text.Encoding as T-import qualified Data.Text.Lazy as TL-import qualified Data.Text.Lazy.Encoding as TL-import qualified Data.Vector as V-import qualified Data.Vector.Unboxed as U-import Data.Word import Data.Proxy-import System.Entropy (getEntropy)+import qualified Data.Set as S+import Data.Text (Text)+import qualified Data.Text as T+import qualified Data.Text.Encoding as T+import qualified Data.Text.Lazy as TL+import qualified Data.Text.Lazy.Encoding as TL+import qualified Data.Vector as V+import qualified Data.Vector.Unboxed as U+import Data.Word+import Prelude as P+import System.Entropy (getEntropy) -import Data.Avro.EncodeRaw-import Data.Avro.Schema as S-import Data.Avro.Types as T-import Data.Avro.Zag-import Data.Avro.Zig+import Data.Avro.EncodeRaw+import Data.Avro.HasAvroSchema+import Data.Avro.Schema as S+import Data.Avro.Types as T+import Data.Avro.Zag+import Data.Avro.Zig encodeAvro :: EncodeAvro a => a -> BL.ByteString encodeAvro = toLazyByteString . putAvro -- |Encode chunks of objects into a container, using 16 random bytes for -- the synchronization markers.-encodeContainer :: EncodeAvro a => [[a]] -> IO BL.ByteString-encodeContainer xss =+encodeContainer :: EncodeAvro a => Schema -> [[a]] -> IO BL.ByteString+encodeContainer sch xss = do sync <- getEntropy 16- return $ encodeContainerWithSync (BL.fromStrict sync) xss+ return $ encodeContainerWithSync sch (BL.fromStrict sync) xss -- |Encode chunks of objects into a container, using the provided -- ByteString as the synchronization markers.-encodeContainerWithSync :: EncodeAvro a => BL.ByteString -> [[a]] -> BL.ByteString-encodeContainerWithSync syncBytes xss =+encodeContainerWithSync :: EncodeAvro a => Schema -> BL.ByteString -> [[a]] -> BL.ByteString+encodeContainerWithSync sch syncBytes xss = toLazyByteString $ lazyByteString avroMagicBytes <>- putAvro (HashMap.fromList [("avro.schema", A.encode objSchema), ("avro.codec","null")] :: HashMap Text BL.ByteString) <>+ putAvro (HashMap.fromList [("avro.schema", A.encode sch), ("avro.codec","null")] :: HashMap Text BL.ByteString) <> lazyByteString syncBytes <> foldMap putBlocks xss where- objSchema = getSchema (P.head (P.head xss)) putBlocks ys = let nrObj = P.length ys nrBytes = BL.length theBytes
src/Data/Avro/HasAvroSchema.hs view
@@ -1,8 +1,10 @@-{-# LANGUAGE FlexibleInstances #-}-{-# LANGUAGE ConstraintKinds #-}-{-# LANGUAGE ScopedTypeVariables #-}+{-# LANGUAGE ConstraintKinds #-}+{-# LANGUAGE FlexibleInstances #-}+{-# LANGUAGE ScopedTypeVariables #-}+{-# LANGUAGE TypeSynonymInstances #-} module Data.Avro.HasAvroSchema where +import qualified Data.Array as Ar import Data.Avro.Schema as S import Data.Avro.Types as T import qualified Data.ByteString as B@@ -10,16 +12,19 @@ import qualified Data.ByteString.Lazy as BL import qualified Data.HashMap.Strict as HashMap import Data.Int+import Data.Ix (Ix)+import Data.List.NonEmpty (NonEmpty (..)) import qualified Data.Map as Map import Data.Monoid ((<>))+import Data.Proxy+import qualified Data.Set as S+import Data.Tagged import Data.Text (Text) import qualified Data.Text as Text-import Data.List.NonEmpty (NonEmpty(..)) import qualified Data.Text.Lazy as TL-import Data.Tagged import qualified Data.Vector as V+import qualified Data.Vector.Unboxed as U import Data.Word-import Data.Proxy class HasAvroSchema a where schema :: Tagged a Type@@ -27,6 +32,18 @@ schemaOf :: (HasAvroSchema a) => a -> Type schemaOf = witness schema +instance HasAvroSchema Word8 where+ schema = Tagged S.Int++instance HasAvroSchema Word16 where+ schema = Tagged S.Int++instance HasAvroSchema Word32 where+ schema = Tagged S.Long++instance HasAvroSchema Word64 where+ schema = Tagged S.Long+ instance HasAvroSchema Bool where schema = Tagged S.Boolean @@ -36,6 +53,12 @@ instance HasAvroSchema Int where schema = Tagged S.Long +instance HasAvroSchema Int8 where+ schema = Tagged S.Int++instance HasAvroSchema Int16 where+ schema = Tagged S.Int+ instance HasAvroSchema Int32 where schema = Tagged S.Int @@ -85,6 +108,18 @@ schema = Tagged $ mkUnion (S.Null:| [untag (schema :: Tagged a Type)]) instance (HasAvroSchema a) => HasAvroSchema [a] where+ schema = wrapTag S.Array (schema :: Tagged a Type)++instance (HasAvroSchema a, Ix i) => HasAvroSchema (Ar.Array i a) where+ schema = wrapTag S.Array (schema :: Tagged a Type)++instance HasAvroSchema a => HasAvroSchema (V.Vector a) where+ schema = wrapTag S.Array (schema :: Tagged a Type)++instance HasAvroSchema a => HasAvroSchema (U.Vector a) where+ schema = wrapTag S.Array (schema :: Tagged a Type)++instance HasAvroSchema a => HasAvroSchema (S.Set a) where schema = wrapTag S.Array (schema :: Tagged a Type) wrapTag :: (Type -> Type) -> Tagged a Type -> Tagged b Type