utf8-light 0.4.2 → 0.4.3
raw patch · 10 files changed
+322/−93 lines, 10 filesdep +hspecnew-uploader
Dependencies added: hspec
Files
- NEWS +8/−0
- README +7/−0
- comp-dep/ghc-eq-or-greater-9.2/Codec/Binary/UTF8/Light/Helper.hs +56/−0
- comp-dep/ghc-less-than-9.2/Codec/Binary/UTF8/Light/Helper.hs +52/−0
- comp-dep/non-ghc/Codec/Binary/UTF8/Light/Helper.hs +50/−0
- src/Codec/Binary/UTF8/Light.hs +29/−69
- test/Codec/Binary/UTF8/Light/HelperSpec.hs +37/−0
- test/Codec/Binary/UTF8/LightSpec.hs +20/−0
- test/Test.hs +1/−0
- utf8-light.cabal +62/−24
+ NEWS view
@@ -0,0 +1,8 @@+Release 0.4.3:+- released sab 26 nov 2022, 15:30:59.+- New maintainer: Francesco Ariis <fa-ml@ariis.it>.+- New repo: https://gitlab.com/fffaaa/utf8-light.+- Added minimal testuite.+- Added README and NEWS.+- utf8-light now should wordk with both GHC≥9.2 and GHC<9.2.+- Please report bug and performances issues.
+ README view
@@ -0,0 +1,7 @@+utf8-light+==========++lightweight UTF-8 handling.++repo: https://gitlab.com/fffaaa/utf8-light+maintainer: Francesco Ariis <fa-ml@ariis.it>
+ comp-dep/ghc-eq-or-greater-9.2/Codec/Binary/UTF8/Light/Helper.hs view
@@ -0,0 +1,56 @@+{-# Language CPP, MagicHash #-}++{- |+ Module : Codec.Binary.UTF8.Light.Helper+ Copyright : (c) Francesco Ariis 2022+ License : BSD3+ Maintainer : Francesco Ariis <fa-ml@ariis.it>+ Stability : provisional+ Portability : portable++ Lightweight UTF8 handling, helper functions.+-}++module Codec.Binary.UTF8.Light.Helper (+ c2w, w2c, i2w, w2i, cwrd, wh, toW8+) where++import GHC.Exts (Int(I#), Char(C#))+import GHC.Word+ (Word8(W8#), Word32(W32#))+import GHC.Prim+ (Word#, Word32#+ ,ord#,chr#,word2Int#,int2Word#+ ,intToInt32#,int2Word#,int32ToInt#,int32ToWord32#+ ,wordToWord8#,wordToWord32#,word32ToWord#,word32ToInt32#)++-----------------------------------------------------------------------------+-- Helper functions for GHC ≥ 9.2++{-# INLINE w2c #-}+w2c :: Word32 -> Char+w2c (W32# w) = C#(chr#(word2Int#(word32ToWord# w)))++{-# INLINE c2w #-}+c2w :: Char -> Word32+c2w (C# c) = W32#(int32ToWord32#(intToInt32#(ord# c)))++{-# INLINE i2w #-}+i2w :: Int -> Word32+i2w (I# i) = W32#(wordToWord32#(int2Word# i))++{-# INLINE w2i #-}+w2i :: Word32 -> Int+w2i (W32# w) = I#(int32ToInt#(word32ToInt32# w))++{-# INLINE wh #-}+wh :: Word32# -> Word8+wh w = W8#(wordToWord8#(word32ToWord# w))++{-# INLINE toW8 #-}+toW8 :: Word# -> Word8+toW8 w = W8#((wordToWord8# w))++{-# INLINE cwrd #-}+cwrd :: Word32# -> Word#+cwrd w = word32ToWord# w
+ comp-dep/ghc-less-than-9.2/Codec/Binary/UTF8/Light/Helper.hs view
@@ -0,0 +1,52 @@+{-# LANGUAGE CPP, MagicHash, TypeSynonymInstances, FlexibleInstances #-}++{- |+ Module : Codec.Binary.UTF8.Light.Helper+ Copyright : (c) Francesco Ariis 2022+ License : BSD3+ Maintainer : Francesco Ariis <fa-ml@ariis.it>+ Stability : provisional+ Portability : portable++ Lightweight UTF8 handling, helper functions.+-}++module Codec.Binary.UTF8.Light.Helper (+ c2w, w2c, i2w, w2i, cwrd, wh, toW8+) where++import GHC.Exts (Int(I#), Char(C#))+import GHC.Word (Word8(W8#), Word32(W32#))+import GHC.Prim (Word#, ord#, chr#, word2Int#, int2Word#)++-----------------------------------------------------------------------------+-- Helper functions for GHC < 9.2++{-# INLINE w2c #-}+w2c :: Word32 -> Char+w2c (W32# w) = C#(chr#(word2Int# w))++{-# INLINE c2w #-}+c2w :: Char -> Word32+c2w (C# c) = W32#(int2Word#(ord# c))++{-# INLINE i2w #-}+i2w :: Int -> Word32+i2w (I# i) = W32#(int2Word# i)++{-# INLINE w2i #-}+w2i :: Word32 -> Int+w2i (W32# w) = I#(word2Int# w)++{-# INLINE wh #-}+wh :: Word# -> Word8+wh w = W8# w++{-# INLINE toW8 #-}+toW8 :: Word# -> Word8+toW8 w = W8# w++{-# INLINE cwrd #-}+cwrd :: Word# -> Word#+cwrd w = w+
+ comp-dep/non-ghc/Codec/Binary/UTF8/Light/Helper.hs view
@@ -0,0 +1,50 @@+{-# LANGUAGE CPP, MagicHash, TypeSynonymInstances, FlexibleInstances #-}++{- |+ Module : Codec.Binary.UTF8.Light.Helper+ Copyright : (c) Francesco Ariis 2022+ License : BSD3+ Maintainer : Francesco Ariis <fa-ml@ariis.it>+ Stability : provisional+ Portability : portable++ Lightweight UTF8 handling, helper functions.+-}++module Codec.Binary.UTF8.Light.Helper (+ c2w, w2c, i2w, w2i, cwrd, wh, toW8+) where++import Data.Word+ (Word,Word8,Word16,Word32)+import Data.Int(Int32)++fi :: (Num b, Integral a) => a -> b+fi = fromIntegral++{-# INLINE w2c #-}+w2c :: Word32 -> Char+w2c = unsafeChr . fromIntegral++{-# INLINE c2w #-}+c2w :: Char -> Word32+c2w = fi . ord++{-# INLINE i2w #-}+i2w :: Int -> Word32+i2w = fi++{-# INLINE w2i #-}+w2i :: Word32 -> Int+w2i = fi++{-# INLINE wh #-}+wh w = W8# w++{-# INLINE toW8 #-}+toW8 w = W8# w++{-# INLINE cwrd #-}+cwrd :: Word# -> Word#+cwrd w = w+
src/Codec/Binary/UTF8/Light.hs view
@@ -2,9 +2,9 @@ {- | Module : Codec.Binary.UTF8.Light- Copyright : (c) Matt Morrow 2008+ Copyright : (c) Matt Morrow 2008, Francesco Ariis 2022 License : BSD3- Maintainer : Matt Morrow <mjm2002@gmail.com>+ Maintainer : Francesco Ariis <fa-ml@ariis.it> Stability : provisional Portability : portable @@ -31,10 +31,6 @@ , hGetUTF8Contents , hGetUTF8 , hGetUTF8NonBlocking- , c2w- , w2c- , i2w- , w2i , flipUTF8 , unflipUTF8 , flipTab@@ -49,31 +45,22 @@ import Data.Bits import Data.List(foldl') import Data.Char(chr,ord)-import Data.Monoid(Monoid(..)) import Data.ByteString(ByteString) import qualified Data.ByteString as B import qualified Data.ByteString.Char8 as B8 import qualified Data.ByteString.Internal as B import Data.ByteString.Unsafe import System.IO(Handle)+import Codec.Binary.UTF8.Light.Helper (c2w, w2c, i2w, w2i, cwrd, wh, toW8) #if defined(__GLASGOW_HASKELL__) import GHC.Exts- (Int(I#),Word(W#),Char(C#)- ,Ptr(Ptr),FunPtr(FunPtr))-import GHC.Int- (Int8(I8#),Int16(I16#),Int32(I32#))+ (Int(I#))+import GHC.Int (Int8, Int16, Int32) import GHC.Word- (Word8(W8#),Word16(W16#),Word32(W32#))+ (Word8, Word16, Word32(W32#)) import GHC.Prim- (Char#,Int#,Word#,Addr#- ,ord#,chr#,int2Word#,word2Int#- ,and#,or#,xor#,not#- ,gtWord#,geWord#,eqWord#- ,neWord#,ltWord#,leWord#- ,uncheckedShiftL#,uncheckedShiftRL#- ,narrow8Int#,narrow16Int#,narrow32Int#- ,narrow8Word#,narrow16Word#,narrow32Word#)+ (int2Word# ,and#, or#, ltWord#, uncheckedShiftRL#) #else import Data.Word (Word,Word8,Word16,Word32)@@ -176,7 +163,7 @@ -- | Lengths in Word8s countUTF8 :: ByteString -> [Int]-countUTF8 s = go 0 (B.length s) s+countUTF8 bs = go 0 (B.length bs) bs where go :: Int -> Int -> ByteString -> [Int] go i len s | len <= i = [] | otherwise = case lenUTF8 (unsafeIndex s i)@@ -244,6 +231,7 @@ #else + -- | Word32s not representing -- valid UTF8 chars are /dropped/. encodeUTF8' :: [Word32] -> [[Word8]]@@ -253,55 +241,59 @@ -- ==> 0xff00ff00## encodeUTF8' ((W32# w):xs) #if MIN_VERSION_base(4,7,0)- | I# (w`ltWord#`(int2Word# 0x80#)) /= 0 =+ | I# (ww`ltWord#`(int2Word# 0x80#)) /= 0 = #else | w`ltWord#`(int2Word# 0x80#) = #endif- [W8# w] : encodeUTF8' xs+ [wh w] : encodeUTF8' xs+ #if MIN_VERSION_base(4,7,0)- | I# (w`ltWord#`(int2Word# 0x800#)) /= 0 =+ | I# (ww`ltWord#`(int2Word# 0x800#)) /= 0 = #else | w`ltWord#`(int2Word# 0x800#) = #endif- [ W8#(w`uncheckedShiftRL#`6#+ [ toW8(ww`uncheckedShiftRL#`6# `or#`(int2Word# 0xc0#))- , W8#(w`and#`(int2Word# 0x3f#)+ , toW8(ww`and#`(int2Word# 0x3f#) `or#`(int2Word# 0x80#)) ] : encodeUTF8' xs #if MIN_VERSION_base(4,7,0)- | I# (w`ltWord#`(int2Word# 0xf0000#)) /= 0 =+ | I# (ww`ltWord#`(int2Word# 0xf0000#)) /= 0 = #else | w`ltWord#`(int2Word# 0xf0000#) = #endif- [ W8#(w`uncheckedShiftRL#`12#+ [ toW8(ww`uncheckedShiftRL#`12# `or#`(int2Word# 0xe0#))- , W8#(w`uncheckedShiftRL#`6#+ , toW8(ww`uncheckedShiftRL#`6# `and#`(int2Word# 0x3f#) `or#`(int2Word# 0x80#))- , W8#(w`and#`(int2Word# 0x3f#)+ , toW8(ww`and#`(int2Word# 0x3f#) `or#`(int2Word# 0x80#)) ] : encodeUTF8' xs #if MIN_VERSION_base(4,7,0)- | I# (w`ltWord#`(int2Word# 0xe00000#)) /= 0 =+ | I# (ww`ltWord#`(int2Word# 0xe00000#)) /= 0 = #else | w`ltWord#`(int2Word# 0xe00000#) = #endif- [ W8#(w`uncheckedShiftRL#`18#+ [ toW8(ww`uncheckedShiftRL#`18# `or#`(int2Word# 0xf0#))- , W8#(w`uncheckedShiftRL#`12#+ , toW8(ww`uncheckedShiftRL#`12# `and#`(int2Word# 0x3f#) `or#`(int2Word# 0x80#))- , W8#(w`uncheckedShiftRL#`6#+ , toW8(ww`uncheckedShiftRL#`6# `and#`(int2Word# 0x3f#) `or#`(int2Word# 0x80#))- , W8#(w`and#`(int2Word# 0x3f#)+ , toW8(ww`and#`(int2Word# 0x3f#) `or#`(int2Word# 0x80#)) ] : encodeUTF8' xs | otherwise = [] : encodeUTF8' xs + where+ ww = cwrd w+ -- TODO: ghc-ify decodeUTF8 decodeUTF8 :: ByteString -> [Word32]-decodeUTF8 s = go 0 (B.length s) s+decodeUTF8 bs = go 0 (B.length bs) bs where go :: Int -> Int -> ByteString -> [Word32] go i len s | len <= i = [] | otherwise = let c1 = unsafeIndex s i@@ -329,42 +321,10 @@ `xor`fi(c3.&.0x3f)`shiftL`6 `xor`fi(c4.&.0x3f) : go (i+4) len s+ _ -> error "decodeUTF8: len > 4" #endif --------------------------------------------------------------------------------w2c :: Word32 -> Char-{-# INLINE w2c #-}-#if defined(__GLASGOW_HASKELL__)-w2c (W32# w) = C#(chr#(word2Int# w))-#else-w2c = unsafeChr . fromIntegral-#endif--c2w :: Char -> Word32-{-# INLINE c2w #-}-#if defined(__GLASGOW_HASKELL__)-c2w (C# c) = W32#(int2Word#(ord# c))-#else-c2w = fromIntegral . ord-#endif--i2w :: Int -> Word32-{-# INLINE i2w #-}-#if defined(__GLASGOW_HASKELL__)-i2w (I# i) = W32#(int2Word# i)-#else-i2w = fi-#endif--w2i :: Word32 -> Int-{-# INLINE w2i #-}-#if defined(__GLASGOW_HASKELL__)-w2i (W32# w) = I#(word2Int# w)-#else-w2i = fi-#endif -----------------------------------------------------------------------------
+ test/Codec/Binary/UTF8/Light/HelperSpec.hs view
@@ -0,0 +1,37 @@+module Codec.Binary.UTF8.Light.HelperSpec where++import Test.Hspec++import Codec.Binary.UTF8.Light.Helper++import GHC.Word (Word32)+++main :: IO ()+main = hspec spec+++spec :: Spec+spec = do++ describe "w2c" $ do+ it "converts a Word32 to a Char" $+ w2c 97 `shouldBe` 'a'++ describe "c2w" $ do+ it "converts a Char to a Word32" $+ c2w 'b' `shouldBe` 98++ describe "i2w" $ do+ it "converts an Int to a Word32" $+ i2w 98 `shouldBe` (98 :: Word32)+ -- TODO [bug] Bug in the old implementation+ -- λ> i2w 4294967296+ -- 4294967296+ -- λ> 4294967296 :: Word32+ -- 0++ describe "w2i" $ do+ it "converts a Word32 to an Int" $+ w2i 18 `shouldBe` (18 :: Int)+
+ test/Codec/Binary/UTF8/LightSpec.hs view
@@ -0,0 +1,20 @@+module Codec.Binary.UTF8.LightSpec where++import Test.Hspec++import Codec.Binary.UTF8.Light+++main :: IO ()+main = hspec spec+++spec :: Spec+spec = do++ describe "encodeUTF8'" $ do+ it "encodes UTF-8" $+ encodeUTF8' [9212 :: Word32] `shouldBe` [[226,143,188] :: [Word8]]+ it "drops Word32 representing invalid UTF-8" $+ encodeUTF8' [921234567 :: Word32] `shouldBe` [[] :: [Word8]]+
+ test/Test.hs view
@@ -0,0 +1,1 @@+{-# OPTIONS_GHC -F -pgmF hspec-discover #-}
utf8-light.cabal view
@@ -1,24 +1,62 @@-name: utf8-light-version: 0.4.2-cabal-version: >= 1.6-build-type: Simple-license: BSD3-license-file: LICENSE-category: Codec-author: Matt Morrow-copyright: Matt Morrow-maintainer: Matt Morrow <mjm2002@gmail.com>-stability: experimental-synopsis: Unicode-description: Lightweight UTF8 handling.--library- build-depends: base < 6- , ghc-prim- , bytestring- hs-source-dirs: src- exposed-modules: Codec.Binary.UTF8.Light--source-repository head- type: git- location: https://github.com/parcs/utf8-light.git+cabal-version: 2.4 +name: utf8-light +version: 0.4.3 +build-type: Simple +license: BSD-3-Clause +license-file: LICENSE +category: Codec +author: Matt Morrow +copyright: Matt Morrow +maintainer: Francesco Ariis <fa-ml@ariis.it> +stability: experimental +synopsis: Unicode +description: Lightweight UTF8 handling +extra-source-files: README, NEWS + + +library + build-depends: base == 4.* + , ghc-prim + , bytestring + hs-source-dirs: src + exposed-modules: Codec.Binary.UTF8.Light + Codec.Binary.UTF8.Light.Helper + default-language: Haskell2010 + ghc-options: -Wall + + if !impl(ghc) + hs-source-dirs: comp-dep/non-ghc + if impl(ghc < 9.2) + hs-source-dirs: comp-dep/ghc-less-than-9.2 + if impl(ghc >= 9.2) + hs-source-dirs: comp-dep/ghc-eq-or-greater-9.2 + + +test-suite test + HS-Source-Dirs: test, src + main-is: Test.hs + build-depends: base == 4.* + , ghc-prim + , bytestring + -- same as above, plus hspec + , hspec >= 2.3 && < 2.11 + build-tool-depends: hspec-discover:hspec-discover + other-modules: Codec.Binary.UTF8.Light + Codec.Binary.UTF8.LightSpec + Codec.Binary.UTF8.Light.Helper + Codec.Binary.UTF8.Light.HelperSpec + type: exitcode-stdio-1.0 + default-language: Haskell2010 + ghc-options: -Wall + + if !impl(ghc) + hs-source-dirs: comp-dep/non-ghc + if impl(ghc < 9.2) + hs-source-dirs: comp-dep/ghc-less-than-9.2 + if impl(ghc >= 9.2) + hs-source-dirs: comp-dep/ghc-eq-or-greater-9.2 + + +source-repository head + type: git + location: https://gitlab.com/fffaaa/utf8-light