diff --git a/ChangeLog.md b/ChangeLog.md --- a/ChangeLog.md +++ b/ChangeLog.md @@ -1,3 +1,18 @@ +## 0.1.6 + + * Drop support for GHC prior 8.6.5 + * Support GHC-9.10 (base-4.21) + +## 0.1.5 + + * text-2.0 support + +## 0.1.4 + + * Fix `fromString` for single character strings. + https://github.com/haskell-hvr/text-short/issues/20 + * Add Template Haskell `Lift ShortText` instance. + ## 0.1.3 * Add `Data ShortText` instance diff --git a/Setup.hs b/Setup.hs deleted file mode 100644 --- a/Setup.hs +++ /dev/null @@ -1,2 +0,0 @@ -import Distribution.Simple -main = defaultMain diff --git a/cbits/memcmp.c b/cbits/memcmp.c deleted file mode 100644 --- a/cbits/memcmp.c +++ /dev/null @@ -1,12 +0,0 @@ -#include - -int -hs_text_short_memcmp(const void *s1, const size_t s1ofs, const void *s2, const size_t s2ofs, const size_t n) -{ - if (!n) return 0; - - const void *s1_ = s1+s1ofs; - const void *s2_ = s2+s2ofs; - - return (s1_ == s2_) ? 0 : memcmp(s1_, s2_, n); -} diff --git a/src-ghc708/PrimOps.hs b/src-ghc708/PrimOps.hs deleted file mode 100644 --- a/src-ghc708/PrimOps.hs +++ /dev/null @@ -1,23 +0,0 @@ -{-# LANGUAGE MagicHash #-} -{-# LANGUAGE UnliftedFFITypes #-} -{-# LANGUAGE Unsafe #-} - -module PrimOps ( compareByteArrays# ) where - -import Foreign.C.Types (CInt (..), CSize (..)) -import GHC.Exts (Int (I#)) -import GHC.Exts (ByteArray#, Int#) -import System.IO.Unsafe (unsafeDupablePerformIO) - --- | Emulate GHC 8.4's 'GHC.Prim.compareByteArrays#' -compareByteArrays# :: ByteArray# -> Int# -> ByteArray# -> Int# -> Int# -> Int# -compareByteArrays# ba1# ofs1# ba2# ofs2# n# - = unI (fromIntegral (unsafeDupablePerformIO (c_memcmp ba1# ofs1 ba2# ofs2 n))) - where - unI (I# i#) = i# - ofs1 = fromIntegral (I# ofs1#) - ofs2 = fromIntegral (I# ofs2#) - n = fromIntegral (I# n#) - -foreign import ccall unsafe "hs_text_short_memcmp" - c_memcmp :: ByteArray# -> CSize -> ByteArray# -> CSize -> CSize -> IO CInt diff --git a/src-test/Tests.hs b/src-test/Tests.hs --- a/src-test/Tests.hs +++ b/src-test/Tests.hs @@ -1,6 +1,13 @@ +{-# LANGUAGE CPP #-} {-# LANGUAGE OverloadedLists #-} {-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE TemplateHaskell #-} +{-# OPTIONS_GHC -fno-warn-orphans #-} +#ifndef MIN_VERSION_GLASGOW_HASKELL +#define MIN_VERSION_GLASGOW_HASKELL(x,y,z,w) ((x*100 + y) >= __GLASGOW_HASKELL__) +#endif + module Main(main) where import Data.Binary @@ -8,11 +15,11 @@ import Data.Maybe import Data.Monoid import qualified Data.String as D.S +import qualified Data.ByteString as BS import qualified Data.Text as T import qualified Data.Text.Encoding as T import qualified Data.Text.Short as IUT import qualified Data.Text.Short.Partial as IUT -import Test.QuickCheck.Instances () import Test.Tasty import Test.Tasty.HUnit import Test.Tasty.QuickCheck as QC @@ -164,8 +171,11 @@ , testCase "IsString U+D800" $ "\xFFFD" @?= (IUT.fromString "\xD800") -- , testCase "IsString U+D800" $ (IUT.fromString "\xD800") @?= IUT.fromText ("\xD800" :: T.Text) +#if !(MIN_VERSION_bytestring(0,11,0) && MIN_VERSION_GLASGOW_HASKELL(9,0,1,0) && !MIN_VERSION_GLASGOW_HASKELL(9,0,2,0)) + -- https://gitlab.haskell.org/ghc/ghc/-/issues/19976 , testCase "Binary.encode" $ encode ("Hello \8364 & \171581!\NUL" :: IUT.ShortText) @?= "\NUL\NUL\NUL\NUL\NUL\NUL\NUL\DC2Hello \226\130\172 & \240\169\184\189!\NUL" , testCase "Binary.decode" $ decode ("\NUL\NUL\NUL\NUL\NUL\NUL\NUL\DC2Hello \226\130\172 & \240\169\184\189!\NUL") @?= ("Hello \8364 & \171581!\NUL" :: IUT.ShortText) +#endif , testCase "singleton" $ [ c | c <- [minBound..maxBound], IUT.singleton c /= IUT.fromText (T.singleton c) ] @?= [] , testCase "splitAtEnd" $ IUT.splitAtEnd 1 "€€" @?= ("€","€") @@ -188,6 +198,27 @@ , testCase "literal9" $ [] @?= ("" :: IUT.ShortText) , testCase "literal10" $ ['¤','€','$'] @?= ("¤€$" :: IUT.ShortText) , testCase "literal12" $ IUT.unpack ['\xD800','\xD7FF','\xDFFF','\xE000'] @?= ['\xFFFD','\xD7FF','\xFFFD','\xE000'] + + -- template haskell + , testCase "TH.Lift" $ do + testLit1 @?= $([| testLit1 |]) + testLit2 @?= $([| testLit2 |]) + testLit3 @?= $([| testLit3 |]) + testLit4 @?= $([| testLit4 |]) + testLit5 @?= $([| testLit5 |]) + testLit6 @?= $([| testLit6 |]) + testLit7 @?= $([| testLit7 |]) + testLit8 @?= $([| testLit8 |]) + + , testCase "TTH.Lift" $ do + testLit1 @?= $$([|| testLit1 ||]) + testLit2 @?= $$([|| testLit2 ||]) + testLit3 @?= $$([|| testLit3 ||]) + testLit4 @?= $$([|| testLit4 ||]) + testLit5 @?= $$([|| testLit5 ||]) + testLit6 @?= $$([|| testLit6 ||]) + testLit7 @?= $$([|| testLit7 ||]) + testLit8 @?= $$([|| testLit8 ||]) ] -- isScalar :: Char -> Bool @@ -229,3 +260,18 @@ {-# NOINLINE testLit8 #-} testLit8 :: IUT.ShortText testLit8 = "\x7f" + +------------------------------------------------------------------------------- +-- orphans +------------------------------------------------------------------------------- + +-- orphan instances to not depend on quickcheck-instances +-- which would cause cycles + +instance Arbitrary BS.ByteString where + arbitrary = BS.pack `fmap` arbitrary + shrink xs = BS.pack `fmap` shrink (BS.unpack xs) + +instance Arbitrary T.Text where + arbitrary = T.pack `fmap` arbitrary + shrink xs = T.pack `fmap` shrink (T.unpack xs) diff --git a/src/Data/Text/Short/Internal.hs b/src/Data/Text/Short/Internal.hs --- a/src/Data/Text/Short/Internal.hs +++ b/src/Data/Text/Short/Internal.hs @@ -4,6 +4,7 @@ {-# LANGUAGE GeneralizedNewtypeDeriving #-} {-# LANGUAGE MagicHash #-} {-# LANGUAGE RankNTypes #-} +{-# LANGUAGE TemplateHaskellQuotes #-} {-# LANGUAGE TypeFamilies #-} {-# LANGUAGE UnboxedTuples #-} {-# LANGUAGE UnliftedFFITypes #-} @@ -95,11 +96,18 @@ , BS.ByteString , T.Text , module Prelude + , module Data.Monoid -- ** Internals , isValidUtf8 ) where +import Prelude + (Bool(..), Ordering(..), Int, Char, String, Maybe(..), IO, Eq, Ord, Num, Read, + Show, ($), ($!), (.), (==), (/=), (+), (*), (-), (>>), (<=), (<), (>), (>=), + compare, show, showsPrec, readsPrec, abs, return, fmap, error, + undefined, otherwise, fromIntegral, max, min, not, fst, snd, map, seq, fail, maybe) + import Control.DeepSeq (NFData) import Control.Monad.ST (stToIO) import Data.Binary @@ -117,10 +125,10 @@ import Data.Typeable (Typeable) import qualified Data.List as List import Data.Maybe (fromMaybe, isNothing) +import Data.Monoid (Monoid, mempty, mconcat) import Data.Semigroup import qualified Data.String as S import qualified Data.Text as T -import qualified Data.Text.Encoding as T import Foreign.C import GHC.Base (assert, unsafeChr) import qualified GHC.CString as GHC @@ -131,16 +139,19 @@ import qualified GHC.Foreign as GHC import GHC.IO.Encoding import GHC.ST -import Prelude hiding (all, any, break, concat, - drop, dropWhile, filter, foldl, - foldl1, foldr, foldr1, head, - init, last, length, null, - replicate, reverse, span, - splitAt, tail, take, takeWhile) import System.IO.Unsafe import Text.Printf (PrintfArg, formatArg, formatString) +import qualified Language.Haskell.TH.Syntax as TH + +#if MIN_VERSION_text(2,0,0) +import qualified Data.Text.Internal as TI +import qualified Data.Text.Array as TA +#else +import qualified Data.Text.Encoding as T +#endif + import qualified PrimOps -- | A compact representation of Unicode strings. @@ -151,11 +162,12 @@ -- -- This type relates to 'T.Text' as 'ShortByteString' relates to 'BS.ByteString' by providing a more compact type. Please consult the documentation of "Data.ByteString.Short" for more information. -- --- Currently, a boxed unshared 'T.Text' has a memory footprint of 6 words (i.e. 48 bytes on 64-bit systems) plus 2 or 4 bytes per code-point (due to the internal UTF-16 representation). Each 'T.Text' value which can share its payload with another 'T.Text' requires only 4 words additionally. Unlike 'BS.ByteString', 'T.Text' use unpinned memory. +-- Currently, a boxed unshared 'T.Text' has a memory footprint of 6 words (i.e. 48 bytes on 64-bit systems) plus 1, 2, 3 or 4 bytes per code-point for text-2 (due to the internal UTF-8 representation) or 2 or 4 bytes per code-point for text-1 (due to the internal UTF-16 representation). Each 'T.Text' value which can share its payload with another 'T.Text' requires only 4 words additionally. Unlike 'BS.ByteString', 'T.Text' use unpinned memory. -- -- In comparison, the footprint of a boxed 'ShortText' is only 4 words (i.e. 32 bytes on 64-bit systems) plus 1, 2, 3, or 4 bytes per code-point (due to the internal UTF-8 representation). --- It can be shown that for realistic data . -- +-- It can be shown that for realistic data . +-- -- __NOTE__: The `Typeable` instance isn't defined for GHC 7.8 (and older) prior to @text-short-0.1.3@ -- -- @since 0.1 @@ -221,7 +233,6 @@ formatArg txt = formatString $ toString txt -- | The 'Binary' encoding matches the one for 'T.Text' -#if MIN_VERSION_binary(0,8,1) instance Binary ShortText where put = put . toShortByteString get = do @@ -229,15 +240,17 @@ case fromShortByteString sbs of Nothing -> fail "Binary.get(ShortText): Invalid UTF-8 stream" Just st -> return st -#else --- fallback via 'ByteString' instance -instance Binary ShortText where - put = put . toByteString - get = do - bs <- get - case fromByteString bs of - Nothing -> fail "Binary.get(ShortText): Invalid UTF-8 stream" - Just st -> return st + +-- | Since 0.1.3 +instance TH.Lift ShortText where + -- TODO: Use DeriveLift with bytestring-0.11.2.0 + lift t = [| fromString s |] + where s = toString t + +#if MIN_VERSION_template_haskell(2,17,0) + liftTyped = TH.unsafeCodeCoerce . TH.lift +#elif MIN_VERSION_template_haskell(2,16,0) + liftTyped = TH.unsafeTExpCoerce . TH.lift #endif -- | \(\mathcal{O}(1)\) Test whether a 'ShortText' is empty. @@ -633,12 +646,16 @@ -- -- prop> (toText . fromText) t == t -- --- This is currently not \(\mathcal{O}(1)\) because currently 'T.Text' uses UTF-16 as its internal representation. --- In the event that 'T.Text' will change its internal representation to UTF-8 this operation will become \(\mathcal{O}(1)\). +-- This is \(\mathcal{O}(1)\) with @text-2@. +-- Previously it wasn't because 'T.Text' used UTF-16 as its internal representation. -- -- @since 0.1 toText :: ShortText -> T.Text +#if MIN_VERSION_text(2,0,0) +toText (ShortText (BSSI.SBS ba)) = TI.Text (TA.ByteArray ba) 0 (I# (GHC.Exts.sizeofByteArray# ba)) +#else toText = T.decodeUtf8 . toByteString +#endif ---- @@ -657,21 +674,37 @@ -- -- @since 0.1 fromString :: String -> ShortText -fromString [] = mempty -fromString [c] = singleton c -fromString s = ShortText . encodeStringShort utf8 . map r $ s +fromString s = case s of + [] -> mempty + [c] -> singleton $ r c + _ -> ShortText . encodeStringShort utf8 . map r $ s where r c | isSurr (ord c) = '\xFFFD' | otherwise = c -- | \(\mathcal{O}(n)\) Construct 'ShortText' from 'T.Text' -- --- This is currently not \(\mathcal{O}(1)\) because currently 'T.Text' uses UTF-16 as its internal representation. --- In the event that 'T.Text' will change its internal representation to UTF-8 this operation will become \(\mathcal{O}(1)\). +-- This is \(\mathcal{O}(1)\) with @text-2@ when the 'T.Text' is not sliced. +-- Previously it wasn't because 'T.Text' used UTF-16 as its internal representation. -- -- @since 0.1 fromText :: T.Text -> ShortText +#if MIN_VERSION_text(2,0,0) +fromText (TI.Text (TA.ByteArray ba) off len) = + ShortText (BSSI.SBS (case sliceByteArray (TA.ByteArray ba) off len of TA.ByteArray ba' -> ba')) + +sliceByteArray :: TA.Array -> Int -> Int -> TA.Array +sliceByteArray ta@(TA.ByteArray ba) 0 len + | len == I# (GHC.Exts.sizeofByteArray# ba) + = ta +sliceByteArray ta off len = TA.run $ do + ma <- TA.new len + TA.copyI len ma 0 ta off + return ma + +#else fromText = fromByteStringUnsafe . T.encodeUtf8 +#endif -- | \(\mathcal{O}(n)\) Construct 'ShortText' from UTF-8 encoded 'ShortByteString' -- @@ -1271,7 +1304,12 @@ {-# INLINE writeWord8Array #-} writeWord8Array :: MBA s -> B -> Word -> ST s () writeWord8Array (MBA# mba#) (B (I# i#)) (W# w#) - = ST $ \s -> case GHC.Exts.writeWord8Array# mba# i# w# s of + = ST $ \s -> +#if __GLASGOW_HASKELL__ >= 902 + case GHC.Exts.writeWord8Array# mba# i# (GHC.Exts.wordToWord8# w#) s of +#else + case GHC.Exts.writeWord8Array# mba# i# w# s of +#endif s' -> (# s', () #) {- not needed yet {-# INLINE indexWord8Array #-} diff --git a/text-short.cabal b/text-short.cabal --- a/text-short.cabal +++ b/text-short.cabal @@ -1,90 +1,100 @@ -cabal-version: 1.18 +cabal-version: 1.18 +name: text-short +version: 0.1.6.1 +synopsis: Memory-efficient representation of Unicode text strings +license: BSD3 +license-file: LICENSE +author: Herbert Valerio Riedel +maintainer: hvr@gnu.org +bug-reports: https://github.com/hvr/text-short/issues +category: Data +build-type: Simple +description: + This package provides the 'ShortText' type which is suitable for keeping many short strings in memory. This is similiar to how 'ShortByteString' relates to 'ByteString'. + . + The main difference between 'Text' and 'ShortText' is that 'ShortText' doesn't support zero-copy slicing (thereby saving 2 words), and, compared to text-1.*, that it uses UTF-8 instead of UTF-16 internally. Consequently, the memory footprint of a (boxed) 'ShortText' value is 4 words (2 words when unboxed) plus the length of the UTF-8 encoded payload. -name: text-short -version: 0.1.3 -synopsis: Memory-efficient representation of Unicode text strings -license: BSD3 -license-file: LICENSE -author: Herbert Valerio Riedel -maintainer: hvr@gnu.org -bug-reports: https://github.com/hvr/text-short/issues -category: Data -build-type: Simple -description: This package provides the 'ShortText' type which is suitable for keeping many short strings in memory. This is similiar to how 'ShortByteString' relates to 'ByteString'. - . - The main difference between 'Text' and 'ShortText' is that 'ShortText' uses UTF-8 instead of UTF-16 internally and also doesn't support zero-copy slicing (thereby saving 2 words). Consequently, the memory footprint of a (boxed) 'ShortText' value is 4 words (2 words when unboxed) plus the length of the UTF-8 encoded payload. +tested-with: + GHC ==8.6.5 + || ==8.8.3 + || ==8.10.7 + || ==9.0.2 + || ==9.2.8 + || ==9.4.8 + || ==9.6.6 + || ==9.8.4 + || ==9.10.2 + || ==9.12.2 + || ==9.14.1 -tested-with: GHC==8.6.5, GHC==8.4.4, GHC==8.2.2, GHC==8.0.2, GHC==7.10.3, GHC==7.8.4 -extra-source-files: ChangeLog.md +extra-source-files: ChangeLog.md -Source-Repository head - Type: git - Location: https://github.com/hvr/text-short.git +source-repository head + type: git + location: https://github.com/hvr/text-short.git flag asserts description: Enable runtime-checks via @assert@ - default: False - manual: True + default: False + manual: True library - exposed-modules: Data.Text.Short - Data.Text.Short.Partial - Data.Text.Short.Unsafe - - other-modules: Data.Text.Short.Internal - - build-depends: base >= 4.7 && < 4.13 - , bytestring >= 0.10.4 && < 0.11 - , hashable >= 1.2.6 && < 1.4 - , deepseq >= 1.3 && < 1.5 - , text >= 1.0 && < 1.3 - , binary >= 0.7.1 && < 0.9 - , ghc-prim >= 0.3.1 && < 0.6 - - if !impl(ghc >= 8.0) - build-depends: semigroups >= 0.18.2 && < 0.20 - - -- GHC version specific PrimOps - if impl(ghc >= 8.4) - hs-source-dirs: src-ghc804 - else - c-sources: cbits/memcmp.c - hs-source-dirs: src-ghc708 - other-modules: PrimOps + exposed-modules: + Data.Text.Short + Data.Text.Short.Partial + Data.Text.Short.Unsafe - hs-source-dirs: src + other-modules: Data.Text.Short.Internal + build-depends: + base >=4.12 && <4.23 + , binary >=0.8.6.0 && <0.9 + , bytestring >=0.10.8.2 && <0.13 + , deepseq >=1.4.4.0 && <1.6 + , ghc-prim >=0.5.3 && <0.14 + , hashable >=1.4.4.0 && <1.6 + , template-haskell >=2.14.0.0 && <2.25 + , text >=1.2.3.1 && <1.3 || >=2.0 && <2.2 - default-language: Haskell2010 - other-extensions: CPP - , GeneralizedNewtypeDeriving - , MagicHash - , UnliftedFFITypes - , Trustworthy - , Unsafe + other-modules: PrimOps + hs-source-dirs: src src-ghc804 + default-language: Haskell2010 + other-extensions: + CPP + GeneralizedNewtypeDeriving + MagicHash + TemplateHaskellQuotes + Trustworthy + UnliftedFFITypes + Unsafe - c-sources: cbits/cbits.c + c-sources: cbits/cbits.c if flag(asserts) - ghc-options: -fno-ignore-asserts + ghc-options: -fno-ignore-asserts + else - cc-options: -DNDEBUG=1 + cc-options: -DNDEBUG=1 - ghc-options: -Wall - cc-options: -O3 -Wall + ghc-options: -Wall + cc-options: -Wall -test-suite tests - type: exitcode-stdio-1.0 - hs-source-dirs: src-test - main-is: Tests.hs +test-suite text-short-tests + type: exitcode-stdio-1.0 + hs-source-dirs: src-test + main-is: Tests.hs - build-depends: base - , binary - , text - , text-short - -- deps which don't inherit constraints from library stanza: - , tasty >= 1.0.0 && < 1.3 - , tasty-quickcheck >= 0.10 && < 0.11 - , tasty-hunit >= 0.10.0 && < 0.11 - , quickcheck-instances >= 0.3.14 && < 0.4 + -- bytestring dependency for cabal_macros.h + build-depends: + base + , binary + , bytestring + , text + , text-short - default-language: Haskell2010 + -- deps which don't inherit constraints from library stanza: + build-depends: + tasty >=1.4 && <1.6 + , tasty-hunit >=0.10.0 && <0.11 + , tasty-quickcheck >=0.10 && <0.12 + + default-language: Haskell2010