diff --git a/ChangeLog.md b/ChangeLog.md --- a/ChangeLog.md +++ b/ChangeLog.md @@ -2,6 +2,10 @@ ## Unreleased changes +## 0.1.2.0 + +* [Support text package 2.0](https://github.com/msakai/bytestring-encoding/pull/8) + ## 0.1.1.0 * [Fix memory corruption bug](https://github.com/msakai/bytestring-encoding/pull/3) diff --git a/README.md b/README.md --- a/README.md +++ b/README.md @@ -1,8 +1,6 @@ # bytestring-encoding: ByteString ↔ Text converter based on GHC.IO.Encoding -[![Build Status (Travis CI)](https://travis-ci.org/msakai/bytestring-encoding.svg?branch=master)](https://travis-ci.org/msakai/bytestring-encoding) -[![Build status (AppVeyor)](https://ci.appveyor.com/api/projects/status/8pwtxsky05ge0ooc/branch/master?svg=true)](https://ci.appveyor.com/project/msakai/bytestring-encoding/branch/master) -[![Build Status (GitHub Actions)](https://github.com/msakai/bytestring-encoding/workflows/build/badge.svg)](https://github.com/msakai/bytestring-encoding/actions) +[![Build Status](https://github.com/msakai/bytestring-encoding/workflows/build/badge.svg)](https://github.com/msakai/bytestring-encoding/actions) [![Coverage Status](https://coveralls.io/repos/github/msakai/bytestring-encoding/badge.svg?branch=master)](https://coveralls.io/github/msakai/bytestring-encoding?branch=master) [![Hackage](https://img.shields.io/hackage/v/bytestring-encoding.svg)](https://hackage.haskell.org/package/bytestring-encoding) [![Hackage Deps](https://img.shields.io/hackage-deps/v/bytestring-encoding.svg)](https://packdeps.haskellers.com/feed?needle=bytestring-encoding) diff --git a/bytestring-encoding.cabal b/bytestring-encoding.cabal --- a/bytestring-encoding.cabal +++ b/bytestring-encoding.cabal @@ -1,13 +1,13 @@ cabal-version: 1.12 --- This file has been generated from package.yaml by hpack version 0.33.0. +-- This file has been generated from package.yaml by hpack version 0.34.4. -- -- see: https://github.com/sol/hpack -- --- hash: 07afcc60ba7089a37925641b27313f86f9aa4e44f8ce1f4565cee5a74e09e58d +-- hash: 16490c648a555bee319cb103d3456c72ddb655413e3e134ccdf466c100ccb146 name: bytestring-encoding -version: 0.1.1.0 +version: 0.1.2.0 synopsis: ByteString ↔ Text converter based on GHC.IO.Encoding description: Please see the README on GitHub at category: Data, Text @@ -19,6 +19,8 @@ license: BSD3 license-file: LICENSE build-type: Simple +tested-with: + GHC==7.10.3 GHC==8.0.2 GHC==8.2.2 GHC==8.4.4 GHC==8.6.5 GHC==8.8.4 GHC==8.10.7 GHC==9.0.2 extra-source-files: README.md ChangeLog.md @@ -37,9 +39,9 @@ hs-source-dirs: src build-depends: - base >=4.7 && <5 - , bytestring - , text + base >=4.8 && <5 + , bytestring >=0.10.6.0 && <0.12 + , text >=1.2.2.2 && <2.1 default-language: Haskell2010 test-suite bytestring-encoding-test @@ -55,13 +57,13 @@ ghc-options: -threaded -rtsopts -with-rtsopts=-N build-depends: QuickCheck - , base >=4.7 && <5 - , bytestring + , base >=4.8 && <5 + , bytestring >=0.10.6.0 && <0.12 , bytestring-encoding , deepseq , tasty >=0.10.1 , tasty-hunit , tasty-quickcheck , tasty-th - , text + , text >=1.2.2.2 && <2.1 default-language: Haskell2010 diff --git a/src/Data/ByteString/Encoding.hs b/src/Data/ByteString/Encoding.hs --- a/src/Data/ByteString/Encoding.hs +++ b/src/Data/ByteString/Encoding.hs @@ -44,7 +44,7 @@ encode :: TextEncoding -> T.Text -> B.ByteString encode enc = BL.toStrict . L.encode enc . TL.fromStrict --- | Decode a strict 'B.ByteString' to a strit 'T.Text' using a given 'Enc.TextEncoding'. +-- | Decode a strict 'B.ByteString' to a strict 'T.Text' using a given 'Enc.TextEncoding'. decode :: TextEncoding -> B.ByteString -> T.Text decode enc = TL.toStrict . L.decode enc . BL.fromStrict diff --git a/src/Data/ByteString/Lazy/Encoding/Internal.hs b/src/Data/ByteString/Lazy/Encoding/Internal.hs --- a/src/Data/ByteString/Lazy/Encoding/Internal.hs +++ b/src/Data/ByteString/Lazy/Encoding/Internal.hs @@ -1,6 +1,7 @@ {-# OPTIONS_GHC -Wall #-} {-# OPTIONS_HADDOCK hide #-} {-# LANGUAGE BangPatterns #-} +{-# LANGUAGE CPP #-} {-# LANGUAGE OverloadedStrings #-} {-# LANGUAGE RecordWildCards #-} module Data.ByteString.Lazy.Encoding.Internal @@ -15,6 +16,7 @@ import qualified Data.ByteString as B import qualified Data.ByteString.Unsafe as B import qualified Data.ByteString.Lazy as BL +import Data.Char (ord) import qualified Data.Text as T import qualified Data.Text.Foreign as T import qualified Data.Text.Lazy as TL @@ -127,6 +129,23 @@ moveBytes (q `plusPtr` bufR buf) p (bufferAvailable buf) go (B.drop (bufferAvailable buf) b : bs) buf{ bufR = bufR buf + bufferAvailable buf } +#if MIN_VERSION_text(2,0,0) + flushOutBuf :: CharBuffer -> ForeignPtr Word8 -> IO ([T.Text], CharBuffer) + flushOutBuf buf workspace + | isEmptyBuffer buf = return ([], buf{ bufL=0, bufR=0 }) + | otherwise = + withForeignPtr workspace $ \workspace' -> + withBuffer buf $ \p -> do + let f !i !j + | bufR buf <= i = return j + | otherwise = do + (c, i') <- readCharBufPtr (castPtr p) i + j' <- writeUTF8 workspace' j c + f i' j' + n <- f (bufL buf) 0 + t <- T.fromPtr workspace' (fromIntegral n) + return ([t], buf{ bufL=0, bufR=0 }) +#else flushOutBuf :: CharBuffer -> ForeignPtr Word16 -> IO ([T.Text], CharBuffer) flushOutBuf buf workspace | isEmptyBuffer buf = return ([], buf{ bufL=0, bufR=0 }) @@ -156,8 +175,13 @@ n <- f (bufL buf) 0 t <- T.fromPtr workspace' (fromIntegral n) return ([t], buf{ bufL=0, bufR=0 }) +#endif +#if MIN_VERSION_text(2,0,0) + loop :: [B.ByteString] -> Buffer Word8 -> CharBuffer -> ForeignPtr Word8 -> IO [T.Text] +#else loop :: [B.ByteString] -> Buffer Word8 -> CharBuffer -> ForeignPtr Word16 -> IO [T.Text] +#endif loop bs inBuf outBuf workspace = do (bs', inBuf1) <- fillInBuf bs inBuf if isEmptyBuffer inBuf1 then do @@ -191,5 +215,66 @@ inBuf <- newByteBuffer inBufSize ReadBuffer outBuf <- newCharBuffer outBufSize WriteBuffer +#if MIN_VERSION_text(2,0,0) + workspace <- mallocForeignPtrArray (outBufSize * 4) +#else workspace <- if charSize == 2 then newForeignPtr_ nullPtr else mallocForeignPtrArray (outBufSize * 2) +#endif loop (BL.toChunks b) inBuf outBuf workspace + + +#if MIN_VERSION_text(2,0,0) + +writeUTF8 :: Ptr Word8 -> Int -> Char -> IO Int +writeUTF8 p i c = do + let x = ord c + if x <= 0x7F then do + pokeElemOff p i (fromIntegral x) + return $! i+1 + else if x <= 0x07FF then do + let (c1,c2) = ord2 c + pokeElemOff p i c1 + pokeElemOff p (i+1) c2 + return $! i+2 + else if x <= 0xFFFF then do + let (c1,c2,c3) = ord3 c + pokeElemOff p i c1 + pokeElemOff p (i+1) c2 + pokeElemOff p (i+2) c3 + return $! i+3 + else do + let (c1,c2,c3,c4) = ord4 c + pokeElemOff p i c1 + pokeElemOff p (i+1) c2 + pokeElemOff p (i+2) c3 + pokeElemOff p (i+3) c4 + return $! i+4 + +-- ----------------------------------------------------------------------------- +-- UTF-8 primitives, lifted from Data.Text.Fusion.Utf8 + +ord2 :: Char -> (Word8,Word8) +ord2 c = assert (n >= 0x80 && n <= 0x07ff) (x1,x2) + where + n = ord c + x1 = fromIntegral $ (n `shiftR` 6) + 0xC0 + x2 = fromIntegral $ (n .&. 0x3F) + 0x80 + +ord3 :: Char -> (Word8,Word8,Word8) +ord3 c = assert (n >= 0x0800 && n <= 0xffff) (x1,x2,x3) + where + n = ord c + x1 = fromIntegral $ (n `shiftR` 12) + 0xE0 + x2 = fromIntegral $ ((n `shiftR` 6) .&. 0x3F) + 0x80 + x3 = fromIntegral $ (n .&. 0x3F) + 0x80 + +ord4 :: Char -> (Word8,Word8,Word8,Word8) +ord4 c = assert (n >= 0x10000) (x1,x2,x3,x4) + where + n = ord c + x1 = fromIntegral $ (n `shiftR` 18) + 0xF0 + x2 = fromIntegral $ ((n `shiftR` 12) .&. 0x3F) + 0x80 + x3 = fromIntegral $ ((n `shiftR` 6) .&. 0x3F) + 0x80 + x4 = fromIntegral $ (n .&. 0x3F) + 0x80 + +#endif