avro 0.4.2.0 → 0.4.3.0

raw patch · 14 files changed

+415/−121 lines, 14 filesdep +deepseqdep +gaugedep +randomdep ~basenew-uploaderPVP: major bump suggested

API removals or changes: PVP suggests a major version bump

Dependencies added: deepseq, gauge, random, zlib

Dependency ranges changed: base

API changes (from Hackage documentation)

Files

• avro.cabal +33/−10
• bench/Time.hs +121/−0
• src/Data/Avro.hs +4/−3
• src/Data/Avro/Codec.hs +116/−0
• src/Data/Avro/Decode.hs +7/−14
• src/Data/Avro/Decode/Get.hs +13/−11
• src/Data/Avro/Decode/Lazy.hs +7/−4
• src/Data/Avro/DecodeRaw.hs +1/−1
• src/Data/Avro/Encode.hs +41/−32
• src/Data/Avro/Schema.hs +32/−26
• src/Data/Avro/Types/Value.hs +7/−1
• test/Avro/Decode/Lazy/ContainerSpec.hs +22/−9
• test/Avro/Decode/Lazy/RawBlocksSpec.hs +3/−2
• test/Avro/JSONSpec.hs +8/−8

@@ -1,7 +1,7 @@ cabal-version: 1.12  name:           avro-version:        0.4.2.0+version:        0.4.3.0 synopsis:       Avro serialization support for Haskell description:    Avro serialization and deserialization support for Haskell category:       Data@@ -52,6 +52,7 @@ library   exposed-modules:       Data.Avro+      Data.Avro.Codec       Data.Avro.Decode       Data.Avro.Decode.Get       Data.Avro.Decode.Lazy@@ -77,11 +78,9 @@       Data.Avro.Types.Value       Data.Avro.Zag       Data.Avro.Zig-  other-modules:-      Paths_avro-  hs-source-dirs:-      src+  hs-source-dirs: src   other-extensions: OverloadedStrings+  default-language: Haskell2010   build-depends:       aeson     , array@@ -92,10 +91,10 @@     , bytestring     , containers     , data-binary-ieee754+    , deepseq     , fail     , hashable     , mtl-    , pure-zlib     , scientific     , semigroups     , tagged@@ -103,17 +102,20 @@     , tf-random     , unordered-containers     , vector+    , zlib   if flag(templatehaskell)     other-extensions: TemplateHaskell     build-depends:         template-haskell >=2.4   if flag(dev)     ghc-options: -Wall -Werror-  default-language: Haskell2010  test-suite test   type: exitcode-stdio-1.0+  ghc-options: -threaded+  default-language: Haskell2010   main-is: Spec.hs+  build-tool-depends: hspec-discover:hspec-discover   other-modules:       Avro.Codec.ArraySpec       Avro.Codec.BoolSpec@@ -148,9 +150,7 @@       DecodeContainer       Example1       Paths_avro-  hs-source-dirs:-      test-  ghc-options: -threaded+  hs-source-dirs: test   build-depends:       QuickCheck     , aeson@@ -185,4 +185,27 @@         template-haskell   if flag(dev)     ghc-options: -Wall -Werror+++benchmark bench-time   default-language: Haskell2010+  type: exitcode-stdio-1.0+  main-is: Time.hs+  hs-source-dirs: bench+  build-depends:+      avro+    , base >=4.6 && <5++    , aeson+    , bytestring+    , containers+    , hashable+    , mtl+    , text+    , random+    , transformers+    , unordered-containers+    , vector++    -- benchmarking-specific libraries+    , gauge

@@ -0,0 +1,121 @@+{-# LANGUAGE NumDecimals         #-}+{-# LANGUAGE OverloadedLists     #-}+{-# LANGUAGE OverloadedStrings   #-}+{-# LANGUAGE ScopedTypeVariables #-}+module Main where++import           Control.Monad         (replicateM)++import qualified Data.ByteString.Lazy  as LBS+import           Data.HashMap.Strict   (HashMap)+import qualified Data.HashMap.Strict   as HashMap+import           Data.Text             (Text)+import           Data.Vector           (Vector)+import qualified Data.Vector           as Vector++import           Gauge++import           GHC.Int               (Int32, Int64)++import qualified System.Random         as Random++import qualified Data.Avro             as Avro+import qualified Data.Avro.Decode      as Decode+import qualified Data.Avro.Encode      as Encode+import           Data.Avro.Schema      (Schema)+import qualified Data.Avro.Schema      as Schema+import           Data.Avro.Types.Value (Value)+import qualified Data.Avro.Types.Value as Value++main :: IO ()+main = defaultMain [encode, decode]++-- * Encoding to binary++encode :: Benchmark+encode = bgroup "encode" [ encodeArray ]++encodeArray :: Benchmark+encodeArray = env randoms $ \ ~(bools, ints, longs, records) ->+  bgroup "array"+  [ bench "bools" $+      nf encodeAvro $ Value.Array $ Value.Boolean <$> bools+  , bench "ints" $+      nf encodeAvro $ Value.Array $ Value.Int <$> ints+  , bench "longs" $+      nf encodeAvro $ Value.Array $ Value.Long <$> longs+  , bench "records" $+      nf encodeAvro $ Value.Array records+  ]+  where randoms = do+          bools <- array+          ints  <- array+          longs <- array+          pure (bools, ints, longs, records bools ints longs)++        array :: (Bounded r, Random.Random r) => IO (Vector r)+        array = Vector.replicateM 1e5 (Random.randomRIO (minBound, maxBound))++        records bools ints longs =+          Vector.zipWith3 record bools ints longs+        record bool int long = Value.Record recordSchema+          [ ("b", Value.Boolean bool)+          , ("i", Value.Int int)+          , ("l", Value.Long long)+          ]+        recordSchema = Schema.Record "Rec" [] Nothing Nothing+          [ Schema.Field "b" [] Nothing Nothing Schema.Boolean Nothing+          , Schema.Field "i" [] Nothing Nothing Schema.Int Nothing+          , Schema.Field "l" [] Nothing Nothing Schema.Long Nothing+          ]++encodeAvro :: Value Schema -> LBS.ByteString+encodeAvro = Encode.encodeAvro++-- * Decoding from binary++decode :: Benchmark+decode = bgroup "decode" [ decodeArray ]++decodeArray :: Benchmark+decodeArray = env randoms $ \ ~(bools, ints, longs, records) ->+  bgroup "array"+  [ bench "bools" $+      nf (decodeAvro $ Schema.Array Schema.Boolean) bools+  , bench "ints" $+      nf (decodeAvro $ Schema.Array Schema.Int) ints+  , bench "longs" $+      nf (decodeAvro $ Schema.Array Schema.Long) longs+  , bench "records" $+      nf (decodeAvro $ Schema.Array recordSchema) records+  ]+  where randoms = do+          bools <- array+          ints  <- array+          longs <- array+          pure ( encodeAvro $ Value.Array $ Value.Boolean <$> bools+               , encodeAvro $ Value.Array $ Value.Int <$> ints+               , encodeAvro $ Value.Array $ Value.Long <$> longs+               , encodeAvro $ Value.Array $ records bools ints longs+               )++        array :: (Bounded r, Random.Random r) => IO (Vector r)+        array = Vector.replicateM 1e5 (Random.randomRIO (minBound, maxBound))++        records bools ints longs =+          Vector.zipWith3 record bools ints longs+        record bool int long = Value.Record recordSchema+          [ ("b", Value.Boolean bool)+          , ("i", Value.Int int)+          , ("l", Value.Long long)+          ]+        recordSchema = Schema.Record "Rec" [] Nothing Nothing+          [ Schema.Field "b" [] Nothing Nothing Schema.Boolean Nothing+          , Schema.Field "i" [] Nothing Nothing Schema.Int Nothing+          , Schema.Field "l" [] Nothing Nothing Schema.Long Nothing+          ]++decodeAvro :: Schema -> LBS.ByteString -> Value Schema+decodeAvro schema bytes = case Decode.decodeAvro schema bytes of+  Left err  -> error err+  Right res -> res

@@ -114,6 +114,7 @@ import           Data.Word import           Prelude               as P +import Data.Avro.Codec (nullCodec) import Data.Avro.FromAvro import Data.Avro.HasAvroSchema import Data.Avro.ToAvro@@ -170,16 +171,16 @@ encodeContainer :: forall a. ToAvro a => [[a]] -> IO BL.ByteString encodeContainer =   let sch = untag (schema :: Tagged a Schema)-  in E.encodeContainer sch . map (map toAvro)+  in E.encodeContainer nullCodec sch . map (map toAvro)  -- | Encode chunks of objects into a container, using the provided--- ByteString as the synchronization markers+-- ByteString as the synchronization markers. encodeContainerWithSync :: forall a. ToAvro a => (Word64,Word64,Word64,Word64) -> [[a]] -> BL.ByteString encodeContainerWithSync (a,b,c,d) =   let     sch = untag (schema :: Tagged a Schema)     syncBytes = P.runPut $ mapM_ P.putWord64le [a,b,c,d]-  in E.encodeContainerWithSync sch syncBytes . map (map toAvro)+  in E.encodeContainerWithSync nullCodec sch syncBytes . map (map toAvro)  -- |Like 'decodeContainer' but returns the avro-encoded bytes for each -- object in the container instead of the Haskell type.

@@ -0,0 +1,116 @@+{-# LANGUAGE BangPatterns        #-}+{-# LANGUAGE OverloadedStrings   #-}+{-# LANGUAGE RankNTypes          #-}+{-# LANGUAGE ScopedTypeVariables #-}+module Data.Avro.Codec (+    Codec(..)+  , Decompress+  , nullCodec+  , deflateCodec+  ) where++import           Codec.Compression.Zlib.Internal as Zlib+import qualified Data.Binary.Get                 as G+import           Data.ByteString                 (ByteString)+import qualified Data.ByteString                 as BS+import qualified Data.ByteString.Lazy            as LBS++-- | Block decompression function for blocks of Avro.+type Decompress a = LBS.ByteString -> G.Get a -> Either String a++-- | A `Codec` allows for compression/decompression of a block in an+-- Avro container according to the Avro spec.+data Codec = Codec+  {+    -- | The name of the codec according to the Avro spec.+    codecName       :: ByteString+    -- | Execute a `G.Get` over a chunk of bytes possibly decompressing+    -- the chunk incrementally.+    --+    -- The API is somewhat more complex than say `codecCompress` to allow+    -- interleaving of decompression and binary decoding while still allowing+    -- proper error handling without resorting to async exceptions.+  , codecDecompress :: forall a. Decompress a++    -- | Compresses a lazy stream of bytes.+  , codecCompress   :: LBS.ByteString -> LBS.ByteString+  }++-- | `nullCodec` specifies @null@ required by Avro spec.+-- (see https://avro.apache.org/docs/1.8.1/spec.html#null)+nullCodec :: Codec+nullCodec =+  Codec+    {+      codecName = "null"+    , codecDecompress = \input parser ->+        case G.runGetOrFail parser input of+          Right (_, _, x)  -> Right x+          Left (_, _, err) -> Left err+    , codecCompress   = id+    }++-- | `deflateCodec` specifies @deflate@ codec required by Avro spec.+-- (see https://avro.apache.org/docs/1.8.1/spec.html#deflate)+deflateCodec :: Codec+deflateCodec =+  Codec+    {+      codecName       = "deflate"+    , codecDecompress = deflateDecompress+    , codecCompress   = deflateCompress+    }++deflateCompress :: LBS.ByteString -> LBS.ByteString+deflateCompress =+  Zlib.compress Zlib.rawFormat Zlib.defaultCompressParams+++-- | Internal type to help construct a lazy list of+-- decompressed bytes interleaved with errors if any.+data Chunk+  = ChunkRest LBS.ByteString+  | ChunkBytes ByteString+  | ChunkError Zlib.DecompressError+++deflateDecompress :: forall a. LBS.ByteString -> G.Get a -> Either String a+deflateDecompress bytes parser = do+  let+    -- N.B. this list is lazily created which allows us to+    -- interleave decompression and binary decoding.+    chunks :: [Chunk]+    chunks =+      Zlib.foldDecompressStreamWithInput+        (\x xs -> ChunkBytes x : xs)+        (\rest -> [ChunkRest rest])+        (\err -> [ChunkError err])+        (Zlib.decompressST Zlib.rawFormat Zlib.defaultDecompressParams)+        bytes++    decode :: G.Decoder a -> [Chunk] -> Either String (G.Decoder a)+    decode !dec@G.Fail{} _ =+      -- short circuit if decoding failed+      pure dec+    decode !dec [] =+      pure dec+    decode !dec (inchunk : inchunks) =+      case inchunk of+        ChunkBytes x ->+          decode (G.pushChunk dec x) inchunks+        ChunkError err ->+          Left (show err)+        ChunkRest rest -> do+          let+            dec' = G.pushEndOfInput dec+          pure $ G.pushChunks dec' rest++  dec <- decode (G.runGetIncremental parser) chunks++  case dec of+    G.Fail _ _ err ->+      Left err+    G.Partial{} ->+      Left "deflate: Not enough input"+    G.Done _ _ x ->+      Right x

@@ -1,6 +1,7 @@ {-# LANGUAGE FlexibleInstances   #-} {-# LANGUAGE MultiWayIf          #-} {-# LANGUAGE OverloadedStrings   #-}+{-# LANGUAGE RankNTypes          #-} {-# LANGUAGE RecordWildCards     #-} {-# LANGUAGE ScopedTypeVariables #-} {-# LANGUAGE TupleSections       #-}@@ -39,6 +40,7 @@ import qualified Data.Vector                      as V import           Prelude                          as P +import           Data.Avro.Codec import           Data.Avro.Decode.Get import           Data.Avro.DecodeRaw import           Data.Avro.Schema                 as S@@ -70,7 +72,7 @@    do ContainerHeader {..} <- getAvro       (containedSchema,) <$> getBlocks (schemaToGet containedSchema) syncBytes decompress   where-  getBlocks :: Get a -> BL.ByteString -> (BL.ByteString -> Get BL.ByteString) -> Get [[a]]+  getBlocks :: Get a -> BL.ByteString -> (forall x. Decompress x) -> Get [[a]]   getBlocks getValue sync decompress = do     isEmpty <- G.isEmpty     if isEmpty@@ -78,19 +80,10 @@       else do         nrObj    <- sFromIntegral =<< getLong         nrBytes  <- getLong-        bytes    <- decompress =<< G.getLazyByteString nrBytes-        r        <- case runGetOrFail (replicateM nrObj getValue) bytes of-                      Right (_,_,x) -> return x-                      Left (_,_,s)  -> fail s+        bytes    <- G.getLazyByteString nrBytes+        r        <- case decompress bytes (replicateM nrObj getValue) of+          Left err -> fail err+          Right x -> pure x         marker   <- G.getLazyByteString nrSyncBytes         when (marker /= sync) (fail "Invalid marker, does not match sync bytes.")         (r :) <$> getBlocks getValue sync decompress--getCodec :: Monad m => Maybe BL.ByteString -> m (BL.ByteString -> m BL.ByteString)-getCodec code | Just "null"    <- code =-                     return return-              | Just "deflate" <- code =-                     return (either (fail . show) return . Z.decompress)-              | Just x <- code =-                     fail ("Unrecognized codec: " <> BC.unpack x)-              | otherwise = return return

@@ -1,6 +1,7 @@ {-# LANGUAGE FlexibleInstances   #-} {-# LANGUAGE MultiWayIf          #-} {-# LANGUAGE OverloadedStrings   #-}+{-# LANGUAGE RankNTypes          #-} {-# LANGUAGE ScopedTypeVariables #-}  module Data.Avro.Decode.Get@@ -31,6 +32,7 @@ import qualified Data.Vector                as V import           Prelude                    as P +import           Data.Avro.Codec import           Data.Avro.DecodeRaw import           Data.Avro.Schema           as S import           Data.Avro.Zag@@ -80,7 +82,7 @@  data ContainerHeader = ContainerHeader   { syncBytes       :: !BL.ByteString-  , decompress      :: BL.ByteString -> Get BL.ByteString+  , decompress      :: forall a. Decompress a   , containedSchema :: !Schema   } @@ -100,7 +102,10 @@                   Just s  -> case A.eitherDecode' s of                                 Left e  -> fail ("Can not decode container schema: " <> e)                                 Right x -> return x-      return ContainerHeader { syncBytes = sync, decompress = codec, containedSchema = schema }+      return ContainerHeader { syncBytes = sync+                             , decompress = codecDecompress codec+                             , containedSchema = schema+                             }    where avroMagicSize :: Integral a => a          avroMagicSize = 4 @@ -111,14 +116,11 @@          getFixed = G.getByteString  -getCodec :: Monad m => Maybe BL.ByteString -> m (BL.ByteString -> m BL.ByteString)-getCodec code | Just "null"    <- code =-                     return return-              | Just "deflate" <- code =-                     return (either (fail . show) return . Z.decompress)-              | Just x <- code =-                     fail ("Unrecognized codec: " <> BC.unpack x)-              | otherwise = return return+getCodec :: Monad m => Maybe BL.ByteString -> m Codec+getCodec (Just "null") = pure nullCodec+getCodec (Just "deflate") = pure deflateCodec+getCodec (Just x) = fail $ "Unrecognized codec: " <> BC.unpack x+getCodec Nothing = pure nullCodec   --------------------------------------------------------------------------------@@ -127,7 +129,7 @@ getBoolean :: Get Bool getBoolean =  do w <- G.getWord8-    return (w == 0x01)+    return $! (w == 0x01)  -- |Get a 32-bit int (zigzag encoded, max of 5 bytes) getInt :: Get Int32

@@ -29,7 +29,6 @@   , badValue   ) where -import qualified Codec.Compression.Zlib     as Z import           Control.Monad              (foldM, replicateM, when) import qualified Data.Aeson                 as A import qualified Data.Array                 as Array@@ -56,6 +55,7 @@ import qualified Data.Vector                as V import           Prelude                    as P +import           Data.Avro.Codec (Decompress) import qualified Data.Avro.Decode.Lazy.LazyValue as T import           Data.Avro.DecodeRaw import           Data.Avro.HasAvroSchema         (schema)@@ -179,7 +179,7 @@         Left err                           -> Just (Left err, Nothing)  getNextBlock :: BL.ByteString-             -> (BL.ByteString -> Get BL.ByteString)+             -> Decompress BL.ByteString              -> BL.ByteString              -> Either String (Maybe (Int, BL.ByteString, BL.ByteString)) getNextBlock sync decompress bs =@@ -192,11 +192,14 @@           Left err   -> Left err           Right rest -> Right $ Just (nrObj, bytes, rest)   where-    getRawBlock :: (BL.ByteString -> Get BL.ByteString) -> Get (Int, BL.ByteString)+    getRawBlock :: Decompress BL.ByteString -> Get (Int, BL.ByteString)     getRawBlock decompress = do       nrObj    <- getLong >>= sFromIntegral       nrBytes  <- getLong-      bytes    <- G.getLazyByteString nrBytes >>= decompress+      compressed <- G.getLazyByteString nrBytes+      bytes <- case decompress compressed G.getRemainingLazyByteString of+        Right x -> pure x+        Left err -> fail err       pure (nrObj, bytes)      checkMarker :: BL.ByteString -> BL.ByteString -> Either String BL.ByteString

@@ -12,7 +12,7 @@ getNonNegative :: (Bits i, Integral i) => Get i getNonNegative = do   orig <- getWord8s-  return (foldl' (\a x -> (a `shiftL` 7) + fromIntegral x) 0 (reverse orig))+  return $! (foldl' (\a x -> (a `shiftL` 7) + fromIntegral x) 0 (reverse orig))  getWord8s :: Get [Word8] getWord8s = do

@@ -56,6 +56,7 @@ import           System.Random.TF.Instances (randoms) import           Prelude                    as P +import Data.Avro.Codec import Data.Avro.EncodeRaw import Data.Avro.HasAvroSchema import Data.Avro.Schema        as S@@ -71,34 +72,41 @@ newSyncBytes = BL.pack . DL.take 16 . randoms <$> initTFGen  -- |Encode chunks of objects into a container, using 16 random bytes for--- the synchronization markers.-encodeContainer :: EncodeAvro a => Schema -> [[a]] -> IO BL.ByteString-encodeContainer sch xss =+-- the synchronization markers. Blocks are compressed (or not) according+-- to the given `Codec` (`nullCodec` or `deflateCodec`).+encodeContainer :: EncodeAvro a => Codec -> Schema -> [[a]] -> IO BL.ByteString+encodeContainer codec sch xss =   do sync <- newSyncBytes-     return $ encodeContainerWithSync sch sync xss+     return $ encodeContainerWithSync codec sch sync xss  -- | Creates an Avro container header for a given schema.-containerHeaderWithSync :: Schema -> BL.ByteString -> Builder-containerHeaderWithSync sch syncBytes =-  lazyByteString avroMagicBytes <>-  putAvro (HashMap.fromList [("avro.schema", A.encode sch), ("avro.codec","null")] :: HashMap Text BL.ByteString) <>-  lazyByteString syncBytes+containerHeaderWithSync :: Codec -> Schema -> BL.ByteString -> Builder+containerHeaderWithSync codec sch syncBytes =+  lazyByteString avroMagicBytes <> putAvro headers <> lazyByteString syncBytes   where     avroMagicBytes :: BL.ByteString     avroMagicBytes = "Obj" <> BL.pack [1] +    headers :: HashMap Text BL.ByteString+    headers =+      HashMap.fromList+        [+          ("avro.schema", A.encode sch)+        , ("avro.codec", BL.fromStrict (codecName codec))+        ]+ -- |Encode chunks of objects into a container, using the provided -- ByteString as the synchronization markers.-encodeContainerWithSync :: EncodeAvro a => Schema -> BL.ByteString -> [[a]] -> BL.ByteString-encodeContainerWithSync sch syncBytes xss =+encodeContainerWithSync :: EncodeAvro a => Codec -> Schema -> BL.ByteString -> [[a]] -> BL.ByteString+encodeContainerWithSync codec sch syncBytes xss =  toLazyByteString $-  containerHeaderWithSync sch syncBytes <>+  containerHeaderWithSync codec sch syncBytes <>   foldMap putBlocks xss  where   putBlocks ys =     let nrObj    = P.length ys         nrBytes  = BL.length theBytes-        theBytes = toLazyByteString $ foldMap putAvro ys+        theBytes = codecCompress codec $ toLazyByteString $ foldMap putAvro ys     in putAvro nrObj <>        putAvro nrBytes <>        lazyByteString theBytes <>@@ -106,47 +114,48 @@  -- | Packs a new container from a list of already encoded Avro blocks. -- Each block is denoted as a pair of a number of objects within that block and the block content.-packContainerBlocks :: Schema -> [(Int, BL.ByteString)] -> IO BL.ByteString-packContainerBlocks sch blocks = do+packContainerBlocks :: Codec -> Schema -> [(Int, BL.ByteString)] -> IO BL.ByteString+packContainerBlocks codec sch blocks = do   sync <- newSyncBytes-  pure $ packContainerBlocksWithSync sch sync blocks+  pure $ packContainerBlocksWithSync codec sch sync blocks  -- | Packs a new container from a list of already encoded Avro blocks. -- Each block is denoted as a pair of a number of objects within that block and the block content.-packContainerBlocksWithSync :: Schema -> BL.ByteString -> [(Int, BL.ByteString)] -> BL.ByteString-packContainerBlocksWithSync sch syncBytes blocks =+packContainerBlocksWithSync :: Codec -> Schema -> BL.ByteString -> [(Int, BL.ByteString)] -> BL.ByteString+packContainerBlocksWithSync codec sch syncBytes blocks =   toLazyByteString $-    containerHeaderWithSync sch syncBytes <>+    containerHeaderWithSync codec sch syncBytes <>     foldMap putBlock blocks   where     putBlock (nrObj, bytes) =-      putAvro nrObj <>-      putAvro (BL.length bytes) <>-      lazyByteString bytes <>-      lazyByteString syncBytes+      let compressed = codecCompress codec bytes in+        putAvro nrObj <>+        putAvro (BL.length compressed) <>+        lazyByteString compressed <>+        lazyByteString syncBytes  -- | Packs a container from a given list of already encoded Avro values -- Each bytestring should represent exactly one one value serialised to Avro.-packContainerValues :: Schema -> [[BL.ByteString]] -> IO BL.ByteString-packContainerValues sch values = do+packContainerValues :: Codec -> Schema -> [[BL.ByteString]] -> IO BL.ByteString+packContainerValues codec sch values = do   sync <- newSyncBytes-  pure $ packContainerValuesWithSync sch sync values+  pure $ packContainerValuesWithSync codec sch sync values  -- | Packs a container from a given list of already encoded Avro values -- Each bytestring should represent exactly one one value serialised to Avro.-packContainerValuesWithSync :: Schema -> BL.ByteString -> [[BL.ByteString]] -> BL.ByteString-packContainerValuesWithSync sch syncBytes values =+packContainerValuesWithSync :: Codec -> Schema -> BL.ByteString -> [[BL.ByteString]] -> BL.ByteString+packContainerValuesWithSync codec sch syncBytes values =   toLazyByteString $-    containerHeaderWithSync sch syncBytes <>+    containerHeaderWithSync codec sch syncBytes <>     foldMap putBlock values   where     putBlock ys =       let nrObj = P.length ys-          nrBytes = DL.foldl' (\a b -> BL.length b + a) 0 ys-          theBytes = mconcat $ lazyByteString <$> ys+          nrBytes = BL.length theBytes+          theBytes = codecCompress codec $ toLazyByteString $ mconcat $ lazyByteString <$> ys       in putAvro nrObj <>          putAvro nrBytes <>-         theBytes <>+         lazyByteString theBytes <>          lazyByteString syncBytes  putAvro :: EncodeAvro a => a -> Builder

@@ -1,3 +1,5 @@+{-# LANGUAGE DeriveAnyClass        #-}+{-# LANGUAGE DeriveGeneric         #-} {-# LANGUAGE FlexibleInstances     #-} {-# LANGUAGE LambdaCase            #-} {-# LANGUAGE MultiParamTypeClasses #-}@@ -46,38 +48,42 @@   ) where  import           Control.Applicative+import           Control.DeepSeq            (NFData) import           Control.Monad.Except import qualified Control.Monad.Fail         as MF import           Control.Monad.State.Strict -import           Data.Aeson             (FromJSON (..), ToJSON (..), object, (.!=), (.:), (.:!), (.:?), (.=))-import qualified Data.Aeson             as A-import           Data.Aeson.Types       (Parser, typeMismatch)-import qualified Data.Avro.Types        as Ty-import qualified Data.ByteString        as B-import qualified Data.ByteString.Base16 as Base16-import qualified Data.Char              as Char-import           Data.Function          (on)+import           Data.Aeson                 (FromJSON (..), ToJSON (..), object,+                                             (.!=), (.:), (.:!), (.:?), (.=))+import qualified Data.Aeson                 as A+import           Data.Aeson.Types           (Parser, typeMismatch)+import qualified Data.Avro.Types            as Ty+import qualified Data.ByteString            as B+import qualified Data.ByteString.Base16     as Base16+import qualified Data.Char                  as Char+import           Data.Function              (on) import           Data.Hashable-import qualified Data.HashMap.Strict    as HashMap+import qualified Data.HashMap.Strict        as HashMap import           Data.Int-import qualified Data.IntMap            as IM-import qualified Data.List              as L-import           Data.List.NonEmpty     (NonEmpty (..))-import qualified Data.List.NonEmpty     as NE-import           Data.Maybe             (catMaybes, fromMaybe, isJust)-import           Data.Monoid            (First (..))+import qualified Data.IntMap                as IM+import qualified Data.List                  as L+import           Data.List.NonEmpty         (NonEmpty (..))+import qualified Data.List.NonEmpty         as NE+import           Data.Maybe                 (catMaybes, fromMaybe, isJust)+import           Data.Monoid                (First (..)) import           Data.Semigroup-import qualified Data.Set               as S+import qualified Data.Set                   as S import           Data.String-import           Data.Text              (Text)-import qualified Data.Text              as T-import           Data.Text.Encoding     as T-import qualified Data.Vector            as V-import           Prelude                as P+import           Data.Text                  (Text)+import qualified Data.Text                  as T+import           Data.Text.Encoding         as T+import qualified Data.Vector                as V+import           Prelude                    as P -import Text.Show.Functions ()+import           GHC.Generics               (Generic) +import           Text.Show.Functions        ()+ -- |An Avro schema is either -- * A "JSON object in the form `{"type":"typeName" ...` -- * A "JSON string, naming a defined type" (basic type w/o free variables/names)@@ -121,7 +127,7 @@               , aliases :: [TypeName]               , size    :: Int               }-    deriving (Show)+    deriving (Show, Generic, NFData)  instance Eq Type where   Null == Null = True@@ -195,7 +201,7 @@ data TypeName = TN { baseName  :: T.Text                    , namespace :: [T.Text]                    }-  deriving (Eq, Ord)+  deriving (Eq, Ord, Generic, NFData)  -- | Show the 'TypeName' as a string literal compatible with its -- 'IsString' instance.@@ -302,10 +308,10 @@                    , fldType    :: Type                    , fldDefault :: Maybe (Ty.Value Type)                    }-  deriving (Eq, Show)+  deriving (Eq, Show, Generic, NFData)  data Order = Ascending | Descending | Ignore-  deriving (Eq, Ord, Show)+  deriving (Eq, Ord, Show, Generic, NFData)  instance FromJSON Type where   parseJSON = parseSchemaJSON Nothing

@@ -1,5 +1,9 @@+{-# LANGUAGE DeriveAnyClass #-}+{-# LANGUAGE DeriveGeneric  #-} module Data.Avro.Types.Value where +import           Control.DeepSeq     (NFData)+ import           Data.ByteString import           Data.HashMap.Strict (HashMap) import           Data.Int@@ -7,6 +11,8 @@ import           Data.Text import           Data.Vector +import           GHC.Generics        (Generic)+ data Value f       = Null       | Boolean !Bool@@ -22,4 +28,4 @@       | Union (NonEmpty f) f (Value f) -- ^ Set of union options, schema for selected option, and the actual value.       | Fixed f {-# UNPACK #-} !ByteString       | Enum f {-# UNPACK #-} !Int Text  -- ^ An enum is a set of the possible symbols (the schema) and the selected symbol-  deriving (Eq, Show)+  deriving (Eq, Show, Generic, NFData)

@@ -6,13 +6,16 @@ where  import           Data.Avro                     as A+import           Data.Avro.Codec               as A import           Data.Avro.Decode.Lazy         as DL import           Data.Avro.Decode.Lazy.Convert as TC import           Data.Avro.Deriving+import           Data.Avro.Encode              as E import           Data.Either                   (isLeft) import           Data.List                     (unfoldr) import           Data.Semigroup                ((<>)) import           Data.Text                     (pack)+import Data.ByteString.Char8 (unpack)  import           Test.Hspec @@ -21,35 +24,45 @@ deriveAvro "test/data/small.avsc"  spec :: Spec-spec = describe "Avro.Decode.Lazy.ContainerSpec" $ do+spec = do+  containerSpec A.nullCodec+  containerSpec A.deflateCodec +containerSpec :: A.Codec -> Spec+containerSpec codec = describe title $ do+   it "should decode empty container" $-    encodeThenDecode ([] :: [[Endpoint]]) >>= (`shouldBe` [])+    encodeThenDecode codec ([] :: [[Endpoint]]) >>= (`shouldBe` [])    it "should decode container with one block" $ do     let msg = mkEndpoint 1-    res <- encodeThenDecode [[msg]]+    res <- encodeThenDecode codec [[msg]]     sequence res `shouldBe` Right [msg]    it "should decode container with empty blocks" $ do     let msg = mkEndpoint 1-    res <- encodeThenDecode [[msg], [], []]+    res <- encodeThenDecode codec [[msg], [], []]     sequence res `shouldBe` Right [msg]    it "should decode container with empty blocks in between" $ do     let (msg1, msg2) = (mkEndpoint 1, mkEndpoint 2)-    res <- encodeThenDecode [[msg1], [], [], [msg2]]+    res <- encodeThenDecode codec [[msg1], [], [], [msg2]]     sequence res `shouldBe` Right [msg1, msg2]    it "should decode container with multiple blocks" $ do     let msgs = mkEndpoint <$> [1..10]     let chunks = chunksOf 4 msgs-    res <- encodeThenDecode chunks+    res <- encodeThenDecode codec chunks     sequence res `shouldBe` Right msgs+  where+    title =+      "Avro.Decode.Lazy.ContainerSpec (" ++ unpack (A.codecName codec) ++ ")" -encodeThenDecode :: (FromLazyAvro a, ToAvro a) => [[a]] -> IO [Either String a]-encodeThenDecode as =-  DL.decodeContainer <$> A.encodeContainer as++encodeThenDecode :: forall a. (FromLazyAvro a, ToAvro a) => A.Codec -> [[a]] -> IO [Either String a]+encodeThenDecode codec as =+  DL.decodeContainer <$>+    E.encodeContainer codec (schemaOf (undefined :: a)) (fmap (fmap toAvro) as)  mkEndpoint :: Int -> Endpoint mkEndpoint i =

@@ -6,6 +6,7 @@ where  import Data.Avro                     as A+import Data.Avro.Codec               (nullCodec) import Data.Avro.Decode.Lazy         as DL import Data.Avro.Decode.Lazy.Convert as TC import Data.Avro.Deriving@@ -46,7 +47,7 @@     let srcValues = mkEndpoint <$> [1..19]     srcContainer <- A.encodeContainer (chunksOf 4 srcValues)     let Right (s, bs) = DL.decodeRawBlocks srcContainer-    tgtContainer <- packContainerBlocks s (rights bs)+    tgtContainer <- packContainerBlocks nullCodec s (rights bs)     let tgtValues = DL.decodeContainer tgtContainer     let allTgtValues = rights tgtValues     allTgtValues `shouldBe` srcValues@@ -54,7 +55,7 @@   it "should pack container with individual values" $ do     let srcValues = mkEndpoint <$> [1..19]     let values = A.encode <$> srcValues-    container <- packContainerValues schema'Endpoint (chunksOf 4 values)+    container <- packContainerValues nullCodec schema'Endpoint (chunksOf 4 values)      let Right (s, bs) = DL.decodeRawBlocks container     s `shouldBe` schema'Endpoint

@@ -4,22 +4,22 @@ {-# LANGUAGE TemplateHaskell     #-} module Avro.JSONSpec where -import           Control.Monad        (forM_)+import Control.Monad (forM_)  import qualified Data.Aeson           as Aeson import qualified Data.ByteString.Lazy as LBS import qualified Data.Map             as Map -import           Data.Avro.Deriving-import           Data.Avro.EitherN-import           Data.Avro.JSON+import Data.Avro.Deriving+import Data.Avro.EitherN+import Data.Avro.JSON -import           Test.Hspec+import Test.Hspec -import           Paths_avro+import Paths_avro -import           System.Directory     (doesFileExist, getCurrentDirectory)-import           System.Environment   (setEnv)+import System.Directory   (doesFileExist, getCurrentDirectory)+import System.Environment (setEnv)  deriveAvro "test/data/enums.avsc" deriveAvro "test/data/reused.avsc"