diff --git a/NEWS b/NEWS new file mode 100644 --- /dev/null +++ b/NEWS @@ -0,0 +1,8 @@ +Release 0.4.3: +- released sab 26 nov 2022, 15:30:59. +- New maintainer: Francesco Ariis . +- New repo: https://gitlab.com/fffaaa/utf8-light. +- Added minimal testuite. +- Added README and NEWS. +- utf8-light now should wordk with both GHC≥9.2 and GHC<9.2. +- Please report bug and performances issues. diff --git a/README b/README new file mode 100644 --- /dev/null +++ b/README @@ -0,0 +1,7 @@ +utf8-light +========== + +lightweight UTF-8 handling. + +repo: https://gitlab.com/fffaaa/utf8-light +maintainer: Francesco Ariis diff --git a/comp-dep/ghc-eq-or-greater-9.2/Codec/Binary/UTF8/Light/Helper.hs b/comp-dep/ghc-eq-or-greater-9.2/Codec/Binary/UTF8/Light/Helper.hs new file mode 100644 --- /dev/null +++ b/comp-dep/ghc-eq-or-greater-9.2/Codec/Binary/UTF8/Light/Helper.hs @@ -0,0 +1,56 @@ +{-# Language CPP, MagicHash #-} + +{- | + Module : Codec.Binary.UTF8.Light.Helper + Copyright : (c) Francesco Ariis 2022 + License : BSD3 + Maintainer : Francesco Ariis + Stability : provisional + Portability : portable + + Lightweight UTF8 handling, helper functions. +-} + +module Codec.Binary.UTF8.Light.Helper ( + c2w, w2c, i2w, w2i, cwrd, wh, toW8 +) where + +import GHC.Exts (Int(I#), Char(C#)) +import GHC.Word + (Word8(W8#), Word32(W32#)) +import GHC.Prim + (Word#, Word32# + ,ord#,chr#,word2Int#,int2Word# + ,intToInt32#,int2Word#,int32ToInt#,int32ToWord32# + ,wordToWord8#,wordToWord32#,word32ToWord#,word32ToInt32#) + +----------------------------------------------------------------------------- +-- Helper functions for GHC ≥ 9.2 + +{-# INLINE w2c #-} +w2c :: Word32 -> Char +w2c (W32# w) = C#(chr#(word2Int#(word32ToWord# w))) + +{-# INLINE c2w #-} +c2w :: Char -> Word32 +c2w (C# c) = W32#(int32ToWord32#(intToInt32#(ord# c))) + +{-# INLINE i2w #-} +i2w :: Int -> Word32 +i2w (I# i) = W32#(wordToWord32#(int2Word# i)) + +{-# INLINE w2i #-} +w2i :: Word32 -> Int +w2i (W32# w) = I#(int32ToInt#(word32ToInt32# w)) + +{-# INLINE wh #-} +wh :: Word32# -> Word8 +wh w = W8#(wordToWord8#(word32ToWord# w)) + +{-# INLINE toW8 #-} +toW8 :: Word# -> Word8 +toW8 w = W8#((wordToWord8# w)) + +{-# INLINE cwrd #-} +cwrd :: Word32# -> Word# +cwrd w = word32ToWord# w diff --git a/comp-dep/ghc-less-than-9.2/Codec/Binary/UTF8/Light/Helper.hs b/comp-dep/ghc-less-than-9.2/Codec/Binary/UTF8/Light/Helper.hs new file mode 100644 --- /dev/null +++ b/comp-dep/ghc-less-than-9.2/Codec/Binary/UTF8/Light/Helper.hs @@ -0,0 +1,52 @@ +{-# LANGUAGE CPP, MagicHash, TypeSynonymInstances, FlexibleInstances #-} + +{- | + Module : Codec.Binary.UTF8.Light.Helper + Copyright : (c) Francesco Ariis 2022 + License : BSD3 + Maintainer : Francesco Ariis + Stability : provisional + Portability : portable + + Lightweight UTF8 handling, helper functions. +-} + +module Codec.Binary.UTF8.Light.Helper ( + c2w, w2c, i2w, w2i, cwrd, wh, toW8 +) where + +import GHC.Exts (Int(I#), Char(C#)) +import GHC.Word (Word8(W8#), Word32(W32#)) +import GHC.Prim (Word#, ord#, chr#, word2Int#, int2Word#) + +----------------------------------------------------------------------------- +-- Helper functions for GHC < 9.2 + +{-# INLINE w2c #-} +w2c :: Word32 -> Char +w2c (W32# w) = C#(chr#(word2Int# w)) + +{-# INLINE c2w #-} +c2w :: Char -> Word32 +c2w (C# c) = W32#(int2Word#(ord# c)) + +{-# INLINE i2w #-} +i2w :: Int -> Word32 +i2w (I# i) = W32#(int2Word# i) + +{-# INLINE w2i #-} +w2i :: Word32 -> Int +w2i (W32# w) = I#(word2Int# w) + +{-# INLINE wh #-} +wh :: Word# -> Word8 +wh w = W8# w + +{-# INLINE toW8 #-} +toW8 :: Word# -> Word8 +toW8 w = W8# w + +{-# INLINE cwrd #-} +cwrd :: Word# -> Word# +cwrd w = w + diff --git a/comp-dep/non-ghc/Codec/Binary/UTF8/Light/Helper.hs b/comp-dep/non-ghc/Codec/Binary/UTF8/Light/Helper.hs new file mode 100644 --- /dev/null +++ b/comp-dep/non-ghc/Codec/Binary/UTF8/Light/Helper.hs @@ -0,0 +1,50 @@ +{-# LANGUAGE CPP, MagicHash, TypeSynonymInstances, FlexibleInstances #-} + +{- | + Module : Codec.Binary.UTF8.Light.Helper + Copyright : (c) Francesco Ariis 2022 + License : BSD3 + Maintainer : Francesco Ariis + Stability : provisional + Portability : portable + + Lightweight UTF8 handling, helper functions. +-} + +module Codec.Binary.UTF8.Light.Helper ( + c2w, w2c, i2w, w2i, cwrd, wh, toW8 +) where + +import Data.Word + (Word,Word8,Word16,Word32) +import Data.Int(Int32) + +fi :: (Num b, Integral a) => a -> b +fi = fromIntegral + +{-# INLINE w2c #-} +w2c :: Word32 -> Char +w2c = unsafeChr . fromIntegral + +{-# INLINE c2w #-} +c2w :: Char -> Word32 +c2w = fi . ord + +{-# INLINE i2w #-} +i2w :: Int -> Word32 +i2w = fi + +{-# INLINE w2i #-} +w2i :: Word32 -> Int +w2i = fi + +{-# INLINE wh #-} +wh w = W8# w + +{-# INLINE toW8 #-} +toW8 w = W8# w + +{-# INLINE cwrd #-} +cwrd :: Word# -> Word# +cwrd w = w + diff --git a/src/Codec/Binary/UTF8/Light.hs b/src/Codec/Binary/UTF8/Light.hs --- a/src/Codec/Binary/UTF8/Light.hs +++ b/src/Codec/Binary/UTF8/Light.hs @@ -2,9 +2,9 @@ {- | Module : Codec.Binary.UTF8.Light - Copyright : (c) Matt Morrow 2008 + Copyright : (c) Matt Morrow 2008, Francesco Ariis 2022 License : BSD3 - Maintainer : Matt Morrow + Maintainer : Francesco Ariis Stability : provisional Portability : portable @@ -31,10 +31,6 @@ , hGetUTF8Contents , hGetUTF8 , hGetUTF8NonBlocking - , c2w - , w2c - , i2w - , w2i , flipUTF8 , unflipUTF8 , flipTab @@ -49,31 +45,22 @@ import Data.Bits import Data.List(foldl') import Data.Char(chr,ord) -import Data.Monoid(Monoid(..)) import Data.ByteString(ByteString) import qualified Data.ByteString as B import qualified Data.ByteString.Char8 as B8 import qualified Data.ByteString.Internal as B import Data.ByteString.Unsafe import System.IO(Handle) +import Codec.Binary.UTF8.Light.Helper (c2w, w2c, i2w, w2i, cwrd, wh, toW8) #if defined(__GLASGOW_HASKELL__) import GHC.Exts - (Int(I#),Word(W#),Char(C#) - ,Ptr(Ptr),FunPtr(FunPtr)) -import GHC.Int - (Int8(I8#),Int16(I16#),Int32(I32#)) + (Int(I#)) +import GHC.Int (Int8, Int16, Int32) import GHC.Word - (Word8(W8#),Word16(W16#),Word32(W32#)) + (Word8, Word16, Word32(W32#)) import GHC.Prim - (Char#,Int#,Word#,Addr# - ,ord#,chr#,int2Word#,word2Int# - ,and#,or#,xor#,not# - ,gtWord#,geWord#,eqWord# - ,neWord#,ltWord#,leWord# - ,uncheckedShiftL#,uncheckedShiftRL# - ,narrow8Int#,narrow16Int#,narrow32Int# - ,narrow8Word#,narrow16Word#,narrow32Word#) + (int2Word# ,and#, or#, ltWord#, uncheckedShiftRL#) #else import Data.Word (Word,Word8,Word16,Word32) @@ -176,7 +163,7 @@ -- | Lengths in Word8s countUTF8 :: ByteString -> [Int] -countUTF8 s = go 0 (B.length s) s +countUTF8 bs = go 0 (B.length bs) bs where go :: Int -> Int -> ByteString -> [Int] go i len s | len <= i = [] | otherwise = case lenUTF8 (unsafeIndex s i) @@ -244,6 +231,7 @@ #else + -- | Word32s not representing -- valid UTF8 chars are /dropped/. encodeUTF8' :: [Word32] -> [[Word8]] @@ -253,55 +241,59 @@ -- ==> 0xff00ff00## encodeUTF8' ((W32# w):xs) #if MIN_VERSION_base(4,7,0) - | I# (w`ltWord#`(int2Word# 0x80#)) /= 0 = + | I# (ww`ltWord#`(int2Word# 0x80#)) /= 0 = #else | w`ltWord#`(int2Word# 0x80#) = #endif - [W8# w] : encodeUTF8' xs + [wh w] : encodeUTF8' xs + #if MIN_VERSION_base(4,7,0) - | I# (w`ltWord#`(int2Word# 0x800#)) /= 0 = + | I# (ww`ltWord#`(int2Word# 0x800#)) /= 0 = #else | w`ltWord#`(int2Word# 0x800#) = #endif - [ W8#(w`uncheckedShiftRL#`6# + [ toW8(ww`uncheckedShiftRL#`6# `or#`(int2Word# 0xc0#)) - , W8#(w`and#`(int2Word# 0x3f#) + , toW8(ww`and#`(int2Word# 0x3f#) `or#`(int2Word# 0x80#)) ] : encodeUTF8' xs #if MIN_VERSION_base(4,7,0) - | I# (w`ltWord#`(int2Word# 0xf0000#)) /= 0 = + | I# (ww`ltWord#`(int2Word# 0xf0000#)) /= 0 = #else | w`ltWord#`(int2Word# 0xf0000#) = #endif - [ W8#(w`uncheckedShiftRL#`12# + [ toW8(ww`uncheckedShiftRL#`12# `or#`(int2Word# 0xe0#)) - , W8#(w`uncheckedShiftRL#`6# + , toW8(ww`uncheckedShiftRL#`6# `and#`(int2Word# 0x3f#) `or#`(int2Word# 0x80#)) - , W8#(w`and#`(int2Word# 0x3f#) + , toW8(ww`and#`(int2Word# 0x3f#) `or#`(int2Word# 0x80#)) ] : encodeUTF8' xs #if MIN_VERSION_base(4,7,0) - | I# (w`ltWord#`(int2Word# 0xe00000#)) /= 0 = + | I# (ww`ltWord#`(int2Word# 0xe00000#)) /= 0 = #else | w`ltWord#`(int2Word# 0xe00000#) = #endif - [ W8#(w`uncheckedShiftRL#`18# + [ toW8(ww`uncheckedShiftRL#`18# `or#`(int2Word# 0xf0#)) - , W8#(w`uncheckedShiftRL#`12# + , toW8(ww`uncheckedShiftRL#`12# `and#`(int2Word# 0x3f#) `or#`(int2Word# 0x80#)) - , W8#(w`uncheckedShiftRL#`6# + , toW8(ww`uncheckedShiftRL#`6# `and#`(int2Word# 0x3f#) `or#`(int2Word# 0x80#)) - , W8#(w`and#`(int2Word# 0x3f#) + , toW8(ww`and#`(int2Word# 0x3f#) `or#`(int2Word# 0x80#)) ] : encodeUTF8' xs | otherwise = [] : encodeUTF8' xs + where + ww = cwrd w + -- TODO: ghc-ify decodeUTF8 decodeUTF8 :: ByteString -> [Word32] -decodeUTF8 s = go 0 (B.length s) s +decodeUTF8 bs = go 0 (B.length bs) bs where go :: Int -> Int -> ByteString -> [Word32] go i len s | len <= i = [] | otherwise = let c1 = unsafeIndex s i @@ -329,42 +321,10 @@ `xor`fi(c3.&.0x3f)`shiftL`6 `xor`fi(c4.&.0x3f) : go (i+4) len s + _ -> error "decodeUTF8: len > 4" #endif ------------------------------------------------------------------------------ - -w2c :: Word32 -> Char -{-# INLINE w2c #-} -#if defined(__GLASGOW_HASKELL__) -w2c (W32# w) = C#(chr#(word2Int# w)) -#else -w2c = unsafeChr . fromIntegral -#endif - -c2w :: Char -> Word32 -{-# INLINE c2w #-} -#if defined(__GLASGOW_HASKELL__) -c2w (C# c) = W32#(int2Word#(ord# c)) -#else -c2w = fromIntegral . ord -#endif - -i2w :: Int -> Word32 -{-# INLINE i2w #-} -#if defined(__GLASGOW_HASKELL__) -i2w (I# i) = W32#(int2Word# i) -#else -i2w = fi -#endif - -w2i :: Word32 -> Int -{-# INLINE w2i #-} -#if defined(__GLASGOW_HASKELL__) -w2i (W32# w) = I#(word2Int# w) -#else -w2i = fi -#endif ----------------------------------------------------------------------------- diff --git a/test/Codec/Binary/UTF8/Light/HelperSpec.hs b/test/Codec/Binary/UTF8/Light/HelperSpec.hs new file mode 100644 --- /dev/null +++ b/test/Codec/Binary/UTF8/Light/HelperSpec.hs @@ -0,0 +1,37 @@ +module Codec.Binary.UTF8.Light.HelperSpec where + +import Test.Hspec + +import Codec.Binary.UTF8.Light.Helper + +import GHC.Word (Word32) + + +main :: IO () +main = hspec spec + + +spec :: Spec +spec = do + + describe "w2c" $ do + it "converts a Word32 to a Char" $ + w2c 97 `shouldBe` 'a' + + describe "c2w" $ do + it "converts a Char to a Word32" $ + c2w 'b' `shouldBe` 98 + + describe "i2w" $ do + it "converts an Int to a Word32" $ + i2w 98 `shouldBe` (98 :: Word32) + -- TODO [bug] Bug in the old implementation + -- λ> i2w 4294967296 + -- 4294967296 + -- λ> 4294967296 :: Word32 + -- 0 + + describe "w2i" $ do + it "converts a Word32 to an Int" $ + w2i 18 `shouldBe` (18 :: Int) + diff --git a/test/Codec/Binary/UTF8/LightSpec.hs b/test/Codec/Binary/UTF8/LightSpec.hs new file mode 100644 --- /dev/null +++ b/test/Codec/Binary/UTF8/LightSpec.hs @@ -0,0 +1,20 @@ +module Codec.Binary.UTF8.LightSpec where + +import Test.Hspec + +import Codec.Binary.UTF8.Light + + +main :: IO () +main = hspec spec + + +spec :: Spec +spec = do + + describe "encodeUTF8'" $ do + it "encodes UTF-8" $ + encodeUTF8' [9212 :: Word32] `shouldBe` [[226,143,188] :: [Word8]] + it "drops Word32 representing invalid UTF-8" $ + encodeUTF8' [921234567 :: Word32] `shouldBe` [[] :: [Word8]] + diff --git a/test/Test.hs b/test/Test.hs new file mode 100644 --- /dev/null +++ b/test/Test.hs @@ -0,0 +1,1 @@ +{-# OPTIONS_GHC -F -pgmF hspec-discover #-} diff --git a/utf8-light.cabal b/utf8-light.cabal --- a/utf8-light.cabal +++ b/utf8-light.cabal @@ -1,24 +1,62 @@ -name: utf8-light -version: 0.4.2 -cabal-version: >= 1.6 -build-type: Simple -license: BSD3 -license-file: LICENSE -category: Codec -author: Matt Morrow -copyright: Matt Morrow -maintainer: Matt Morrow -stability: experimental -synopsis: Unicode -description: Lightweight UTF8 handling. - -library - build-depends: base < 6 - , ghc-prim - , bytestring - hs-source-dirs: src - exposed-modules: Codec.Binary.UTF8.Light - -source-repository head - type: git - location: https://github.com/parcs/utf8-light.git +cabal-version: 2.4 +name: utf8-light +version: 0.4.3 +build-type: Simple +license: BSD-3-Clause +license-file: LICENSE +category: Codec +author: Matt Morrow +copyright: Matt Morrow +maintainer: Francesco Ariis +stability: experimental +synopsis: Unicode +description: Lightweight UTF8 handling +extra-source-files: README, NEWS + + +library + build-depends: base == 4.* + , ghc-prim + , bytestring + hs-source-dirs: src + exposed-modules: Codec.Binary.UTF8.Light + Codec.Binary.UTF8.Light.Helper + default-language: Haskell2010 + ghc-options: -Wall + + if !impl(ghc) + hs-source-dirs: comp-dep/non-ghc + if impl(ghc < 9.2) + hs-source-dirs: comp-dep/ghc-less-than-9.2 + if impl(ghc >= 9.2) + hs-source-dirs: comp-dep/ghc-eq-or-greater-9.2 + + +test-suite test + HS-Source-Dirs: test, src + main-is: Test.hs + build-depends: base == 4.* + , ghc-prim + , bytestring + -- same as above, plus hspec + , hspec >= 2.3 && < 2.11 + build-tool-depends: hspec-discover:hspec-discover + other-modules: Codec.Binary.UTF8.Light + Codec.Binary.UTF8.LightSpec + Codec.Binary.UTF8.Light.Helper + Codec.Binary.UTF8.Light.HelperSpec + type: exitcode-stdio-1.0 + default-language: Haskell2010 + ghc-options: -Wall + + if !impl(ghc) + hs-source-dirs: comp-dep/non-ghc + if impl(ghc < 9.2) + hs-source-dirs: comp-dep/ghc-less-than-9.2 + if impl(ghc >= 9.2) + hs-source-dirs: comp-dep/ghc-eq-or-greater-9.2 + + +source-repository head + type: git + location: https://gitlab.com/fffaaa/utf8-light