bytestring-encoding 0.1.0.0 → 0.1.1.0
raw patch · 5 files changed
+37/−19 lines, 5 filesdep +deepseqPVP ok
version bump matches the API change (PVP)
Dependencies added: deepseq
API changes (from Hackage documentation)
Files
- ChangeLog.md +4/−0
- README.md +6/−2
- bytestring-encoding.cabal +7/−5
- src/Data/ByteString/Lazy/Encoding/Internal.hs +10/−12
- test/Base.hs +10/−0
ChangeLog.md view
@@ -1,3 +1,7 @@ # Changelog for bytestring-encoding ## Unreleased changes++## 0.1.1.0++* [Fix memory corruption bug](https://github.com/msakai/bytestring-encoding/pull/3)
README.md view
@@ -2,16 +2,20 @@ [](https://travis-ci.org/msakai/bytestring-encoding) [](https://ci.appveyor.com/project/msakai/bytestring-encoding/branch/master)+[](https://github.com/msakai/bytestring-encoding/actions) [](https://coveralls.io/github/msakai/bytestring-encoding?branch=master)+[](https://hackage.haskell.org/package/bytestring-encoding)+[](https://packdeps.haskellers.com/feed?needle=bytestring-encoding)+[](https://opensource.org/licenses/BSD-3-Clause) These library provides converter between `ByteString` and `Text` based on `GHC.IO.Encoding`. Compared to the [text-icu](http://hackage.haskell.org/package/text-icu)-package, it has only limited feature and platform dependent, but it is+package, it has only limited feature and is platform dependent, but is light-weight and consistent with conversion by `System.IO`. ## Limitations and Known issues * There are some cases that conversion can produce incomplete results due to the problem of `GHC.IO.Encoding` API.- see https://ghc.haskell.org/trac/ghc/ticket/15553 for details.+ see https://gitlab.haskell.org/ghc/ghc/-/issues/15553 for details.
bytestring-encoding.cabal view
@@ -1,11 +1,13 @@--- This file has been generated from package.yaml by hpack version 0.28.2.+cabal-version: 1.12++-- This file has been generated from package.yaml by hpack version 0.33.0. -- -- see: https://github.com/sol/hpack ----- hash: 99947b66971c69e6b7b235634501b6daf5d8231b122ddf14addced7506a7acbe+-- hash: 07afcc60ba7089a37925641b27313f86f9aa4e44f8ce1f4565cee5a74e09e58d name: bytestring-encoding-version: 0.1.0.0+version: 0.1.1.0 synopsis: ByteString ↔ Text converter based on GHC.IO.Encoding description: Please see the README on GitHub at <https://github.com/msakai/bytestring-encoding#readme> category: Data, Text@@ -17,10 +19,9 @@ license: BSD3 license-file: LICENSE build-type: Simple-cabal-version: >= 1.10 extra-source-files:- ChangeLog.md README.md+ ChangeLog.md source-repository head type: git@@ -57,6 +58,7 @@ , base >=4.7 && <5 , bytestring , bytestring-encoding+ , deepseq , tasty >=0.10.1 , tasty-hunit , tasty-quickcheck
src/Data/ByteString/Lazy/Encoding/Internal.hs view
@@ -127,7 +127,7 @@ moveBytes (q `plusPtr` bufR buf) p (bufferAvailable buf) go (B.drop (bufferAvailable buf) b : bs) buf{ bufR = bufR buf + bufferAvailable buf } - flushOutBuf :: CharBuffer -> Ptr Word16 -> IO ([T.Text], CharBuffer)+ flushOutBuf :: CharBuffer -> ForeignPtr Word16 -> IO ([T.Text], CharBuffer) flushOutBuf buf workspace | isEmptyBuffer buf = return ([], buf{ bufL=0, bufR=0 }) | charSize==2 = do@@ -136,7 +136,8 @@ p' = castPtr p t <- T.fromPtr (p' `plusPtr` bufL buf) (fromIntegral (bufferElems buf)) return ([t], buf{ bufL=0, bufR=0 })- | otherwise = do+ | otherwise =+ withForeignPtr workspace $ \workspace' -> withBuffer buf $ \p -> do let p' :: Ptr Char p' = castPtr p@@ -145,18 +146,18 @@ | otherwise = do c <- liftM fromEnum $ peekElemOff p' i if c < 0x10000 then do- pokeElemOff workspace j (fromIntegral c)+ pokeElemOff workspace' j (fromIntegral c) f (i+1) (j+1) else do let c' = c - 0x10000- pokeElemOff workspace j (fromIntegral (c' `div` 0x400 + 0xd800))- pokeElemOff workspace (j+1) (fromIntegral (c' `mod` 0x400 + 0xdc00))+ pokeElemOff workspace' j (fromIntegral (c' `div` 0x400 + 0xd800))+ pokeElemOff workspace' (j+1) (fromIntegral (c' `mod` 0x400 + 0xdc00)) f (i+1) (j+2) n <- f (bufL buf) 0- t <- T.fromPtr workspace (fromIntegral n)+ t <- T.fromPtr workspace' (fromIntegral n) return ([t], buf{ bufL=0, bufR=0 }) - loop :: [B.ByteString] -> Buffer Word8 -> CharBuffer -> Ptr Word16 -> IO [T.Text]+ loop :: [B.ByteString] -> Buffer Word8 -> CharBuffer -> ForeignPtr Word16 -> IO [T.Text] loop bs inBuf outBuf workspace = do (bs', inBuf1) <- fillInBuf bs inBuf if isEmptyBuffer inBuf1 then do@@ -190,8 +191,5 @@ inBuf <- newByteBuffer inBufSize ReadBuffer outBuf <- newCharBuffer outBufSize WriteBuffer- if charSize == 2 then do- loop (BL.toChunks b) inBuf outBuf nullPtr- else do- allocaArray (outBufSize * 2) $ \workspace ->- loop (BL.toChunks b) inBuf outBuf workspace+ workspace <- if charSize == 2 then newForeignPtr_ nullPtr else mallocForeignPtrArray (outBufSize * 2)+ loop (BL.toChunks b) inBuf outBuf workspace
test/Base.hs view
@@ -4,14 +4,17 @@ {-# LANGUAGE TemplateHaskell #-} module Base (baseTestGroup) where +import Control.DeepSeq import qualified Data.ByteString.Lazy.Encoding as Enc import qualified Data.ByteString.Lazy.Encoding.Internal as Enc import qualified Data.ByteString.Lazy as BL+import qualified Data.ByteString as B import qualified Data.Text.Lazy as TL import qualified Data.Text.Lazy.Encoding as TL import Test.Tasty import Test.Tasty.QuickCheck+import Test.Tasty.HUnit import Test.Tasty.TH checkEncode :: Enc.TextEncoding -> (TL.Text -> BL.ByteString) -> Property@@ -61,6 +64,13 @@ prop_encode_decode_utf32be :: Property prop_encode_decode_utf32be = checkRoundTrip Enc.utf32be++case_decoding_long_chunked_string :: IO ()+case_decoding_long_chunked_string = do+ let ls = [32752, 32752, 32752, 32752, 32752, 32752, 32752, 32752, 32752, 18193]+ bs = BL.fromChunks [B.pack (replicate l (fromIntegral (fromEnum 'a'))) | l <- ls]+ let t = Enc.decode Enc.utf8 bs+ deepseq t $ return () -- ---------------------------------------------------------------------