text-short 0.1.3 → 0.1.6.1
raw patch · 7 files changed
Files
- ChangeLog.md +15/−0
- Setup.hs +0/−2
- cbits/memcmp.c +0/−12
- src-ghc708/PrimOps.hs +0/−23
- src-test/Tests.hs +47/−1
- src/Data/Text/Short/Internal.hs +65/−27
- text-short.cabal +82/−72
ChangeLog.md view
@@ -1,3 +1,18 @@+## 0.1.6++ * Drop support for GHC prior 8.6.5+ * Support GHC-9.10 (base-4.21)++## 0.1.5++ * text-2.0 support++## 0.1.4++ * Fix `fromString` for single character strings.+ https://github.com/haskell-hvr/text-short/issues/20+ * Add Template Haskell `Lift ShortText` instance.+ ## 0.1.3 * Add `Data ShortText` instance
− Setup.hs
@@ -1,2 +0,0 @@-import Distribution.Simple-main = defaultMain
− cbits/memcmp.c
@@ -1,12 +0,0 @@-#include <string.h>--int-hs_text_short_memcmp(const void *s1, const size_t s1ofs, const void *s2, const size_t s2ofs, const size_t n)-{- if (!n) return 0;-- const void *s1_ = s1+s1ofs;- const void *s2_ = s2+s2ofs;-- return (s1_ == s2_) ? 0 : memcmp(s1_, s2_, n);-}
− src-ghc708/PrimOps.hs
@@ -1,23 +0,0 @@-{-# LANGUAGE MagicHash #-}-{-# LANGUAGE UnliftedFFITypes #-}-{-# LANGUAGE Unsafe #-}--module PrimOps ( compareByteArrays# ) where--import Foreign.C.Types (CInt (..), CSize (..))-import GHC.Exts (Int (I#))-import GHC.Exts (ByteArray#, Int#)-import System.IO.Unsafe (unsafeDupablePerformIO)---- | Emulate GHC 8.4's 'GHC.Prim.compareByteArrays#'-compareByteArrays# :: ByteArray# -> Int# -> ByteArray# -> Int# -> Int# -> Int#-compareByteArrays# ba1# ofs1# ba2# ofs2# n#- = unI (fromIntegral (unsafeDupablePerformIO (c_memcmp ba1# ofs1 ba2# ofs2 n)))- where- unI (I# i#) = i#- ofs1 = fromIntegral (I# ofs1#)- ofs2 = fromIntegral (I# ofs2#)- n = fromIntegral (I# n#)--foreign import ccall unsafe "hs_text_short_memcmp"- c_memcmp :: ByteArray# -> CSize -> ByteArray# -> CSize -> CSize -> IO CInt
src-test/Tests.hs view
@@ -1,6 +1,13 @@+{-# LANGUAGE CPP #-} {-# LANGUAGE OverloadedLists #-} {-# LANGUAGE OverloadedStrings #-}+{-# LANGUAGE TemplateHaskell #-}+{-# OPTIONS_GHC -fno-warn-orphans #-} +#ifndef MIN_VERSION_GLASGOW_HASKELL+#define MIN_VERSION_GLASGOW_HASKELL(x,y,z,w) ((x*100 + y) >= __GLASGOW_HASKELL__)+#endif+ module Main(main) where import Data.Binary@@ -8,11 +15,11 @@ import Data.Maybe import Data.Monoid import qualified Data.String as D.S+import qualified Data.ByteString as BS import qualified Data.Text as T import qualified Data.Text.Encoding as T import qualified Data.Text.Short as IUT import qualified Data.Text.Short.Partial as IUT-import Test.QuickCheck.Instances () import Test.Tasty import Test.Tasty.HUnit import Test.Tasty.QuickCheck as QC@@ -164,8 +171,11 @@ , testCase "IsString U+D800" $ "\xFFFD" @?= (IUT.fromString "\xD800") -- , testCase "IsString U+D800" $ (IUT.fromString "\xD800") @?= IUT.fromText ("\xD800" :: T.Text) +#if !(MIN_VERSION_bytestring(0,11,0) && MIN_VERSION_GLASGOW_HASKELL(9,0,1,0) && !MIN_VERSION_GLASGOW_HASKELL(9,0,2,0))+ -- https://gitlab.haskell.org/ghc/ghc/-/issues/19976 , testCase "Binary.encode" $ encode ("Hello \8364 & \171581!\NUL" :: IUT.ShortText) @?= "\NUL\NUL\NUL\NUL\NUL\NUL\NUL\DC2Hello \226\130\172 & \240\169\184\189!\NUL" , testCase "Binary.decode" $ decode ("\NUL\NUL\NUL\NUL\NUL\NUL\NUL\DC2Hello \226\130\172 & \240\169\184\189!\NUL") @?= ("Hello \8364 & \171581!\NUL" :: IUT.ShortText)+#endif , testCase "singleton" $ [ c | c <- [minBound..maxBound], IUT.singleton c /= IUT.fromText (T.singleton c) ] @?= [] , testCase "splitAtEnd" $ IUT.splitAtEnd 1 "€€" @?= ("€","€")@@ -188,6 +198,27 @@ , testCase "literal9" $ [] @?= ("" :: IUT.ShortText) , testCase "literal10" $ ['¤','€','$'] @?= ("¤€$" :: IUT.ShortText) , testCase "literal12" $ IUT.unpack ['\xD800','\xD7FF','\xDFFF','\xE000'] @?= ['\xFFFD','\xD7FF','\xFFFD','\xE000']++ -- template haskell+ , testCase "TH.Lift" $ do+ testLit1 @?= $([| testLit1 |])+ testLit2 @?= $([| testLit2 |])+ testLit3 @?= $([| testLit3 |])+ testLit4 @?= $([| testLit4 |])+ testLit5 @?= $([| testLit5 |])+ testLit6 @?= $([| testLit6 |])+ testLit7 @?= $([| testLit7 |])+ testLit8 @?= $([| testLit8 |])++ , testCase "TTH.Lift" $ do+ testLit1 @?= $$([|| testLit1 ||])+ testLit2 @?= $$([|| testLit2 ||])+ testLit3 @?= $$([|| testLit3 ||])+ testLit4 @?= $$([|| testLit4 ||])+ testLit5 @?= $$([|| testLit5 ||])+ testLit6 @?= $$([|| testLit6 ||])+ testLit7 @?= $$([|| testLit7 ||])+ testLit8 @?= $$([|| testLit8 ||]) ] -- isScalar :: Char -> Bool@@ -229,3 +260,18 @@ {-# NOINLINE testLit8 #-} testLit8 :: IUT.ShortText testLit8 = "\x7f"++-------------------------------------------------------------------------------+-- orphans+-------------------------------------------------------------------------------++-- orphan instances to not depend on quickcheck-instances+-- which would cause cycles++instance Arbitrary BS.ByteString where+ arbitrary = BS.pack `fmap` arbitrary+ shrink xs = BS.pack `fmap` shrink (BS.unpack xs)++instance Arbitrary T.Text where+ arbitrary = T.pack `fmap` arbitrary+ shrink xs = T.pack `fmap` shrink (T.unpack xs)
src/Data/Text/Short/Internal.hs view
@@ -4,6 +4,7 @@ {-# LANGUAGE GeneralizedNewtypeDeriving #-} {-# LANGUAGE MagicHash #-} {-# LANGUAGE RankNTypes #-}+{-# LANGUAGE TemplateHaskellQuotes #-} {-# LANGUAGE TypeFamilies #-} {-# LANGUAGE UnboxedTuples #-} {-# LANGUAGE UnliftedFFITypes #-}@@ -95,11 +96,18 @@ , BS.ByteString , T.Text , module Prelude+ , module Data.Monoid -- ** Internals , isValidUtf8 ) where +import Prelude+ (Bool(..), Ordering(..), Int, Char, String, Maybe(..), IO, Eq, Ord, Num, Read,+ Show, ($), ($!), (.), (==), (/=), (+), (*), (-), (>>), (<=), (<), (>), (>=),+ compare, show, showsPrec, readsPrec, abs, return, fmap, error,+ undefined, otherwise, fromIntegral, max, min, not, fst, snd, map, seq, fail, maybe)+ import Control.DeepSeq (NFData) import Control.Monad.ST (stToIO) import Data.Binary@@ -117,10 +125,10 @@ import Data.Typeable (Typeable) import qualified Data.List as List import Data.Maybe (fromMaybe, isNothing)+import Data.Monoid (Monoid, mempty, mconcat) import Data.Semigroup import qualified Data.String as S import qualified Data.Text as T-import qualified Data.Text.Encoding as T import Foreign.C import GHC.Base (assert, unsafeChr) import qualified GHC.CString as GHC@@ -131,16 +139,19 @@ import qualified GHC.Foreign as GHC import GHC.IO.Encoding import GHC.ST-import Prelude hiding (all, any, break, concat,- drop, dropWhile, filter, foldl,- foldl1, foldr, foldr1, head,- init, last, length, null,- replicate, reverse, span,- splitAt, tail, take, takeWhile) import System.IO.Unsafe import Text.Printf (PrintfArg, formatArg, formatString) +import qualified Language.Haskell.TH.Syntax as TH++#if MIN_VERSION_text(2,0,0)+import qualified Data.Text.Internal as TI+import qualified Data.Text.Array as TA+#else+import qualified Data.Text.Encoding as T+#endif+ import qualified PrimOps -- | A compact representation of Unicode strings.@@ -151,11 +162,12 @@ -- -- This type relates to 'T.Text' as 'ShortByteString' relates to 'BS.ByteString' by providing a more compact type. Please consult the documentation of "Data.ByteString.Short" for more information. ----- Currently, a boxed unshared 'T.Text' has a memory footprint of 6 words (i.e. 48 bytes on 64-bit systems) plus 2 or 4 bytes per code-point (due to the internal UTF-16 representation). Each 'T.Text' value which can share its payload with another 'T.Text' requires only 4 words additionally. Unlike 'BS.ByteString', 'T.Text' use unpinned memory.+-- Currently, a boxed unshared 'T.Text' has a memory footprint of 6 words (i.e. 48 bytes on 64-bit systems) plus 1, 2, 3 or 4 bytes per code-point for text-2 (due to the internal UTF-8 representation) or 2 or 4 bytes per code-point for text-1 (due to the internal UTF-16 representation). Each 'T.Text' value which can share its payload with another 'T.Text' requires only 4 words additionally. Unlike 'BS.ByteString', 'T.Text' use unpinned memory. -- -- In comparison, the footprint of a boxed 'ShortText' is only 4 words (i.e. 32 bytes on 64-bit systems) plus 1, 2, 3, or 4 bytes per code-point (due to the internal UTF-8 representation).--- It can be shown that for realistic data <http://utf8everywhere.org/#asian UTF-16 has a space overhead of 50% over UTF-8>. --+-- It can be shown that for realistic data <http://utf8everywhere.org/#asian UTF-16, which is used by text-1, has a space overhead of 50% over UTF-8>.+-- -- __NOTE__: The `Typeable` instance isn't defined for GHC 7.8 (and older) prior to @text-short-0.1.3@ -- -- @since 0.1@@ -221,7 +233,6 @@ formatArg txt = formatString $ toString txt -- | The 'Binary' encoding matches the one for 'T.Text'-#if MIN_VERSION_binary(0,8,1) instance Binary ShortText where put = put . toShortByteString get = do@@ -229,15 +240,17 @@ case fromShortByteString sbs of Nothing -> fail "Binary.get(ShortText): Invalid UTF-8 stream" Just st -> return st-#else--- fallback via 'ByteString' instance-instance Binary ShortText where- put = put . toByteString- get = do- bs <- get- case fromByteString bs of- Nothing -> fail "Binary.get(ShortText): Invalid UTF-8 stream"- Just st -> return st++-- | Since 0.1.3+instance TH.Lift ShortText where+ -- TODO: Use DeriveLift with bytestring-0.11.2.0+ lift t = [| fromString s |]+ where s = toString t++#if MIN_VERSION_template_haskell(2,17,0)+ liftTyped = TH.unsafeCodeCoerce . TH.lift+#elif MIN_VERSION_template_haskell(2,16,0)+ liftTyped = TH.unsafeTExpCoerce . TH.lift #endif -- | \(\mathcal{O}(1)\) Test whether a 'ShortText' is empty.@@ -633,12 +646,16 @@ -- -- prop> (toText . fromText) t == t ----- This is currently not \(\mathcal{O}(1)\) because currently 'T.Text' uses UTF-16 as its internal representation.--- In the event that 'T.Text' will change its internal representation to UTF-8 this operation will become \(\mathcal{O}(1)\).+-- This is \(\mathcal{O}(1)\) with @text-2@.+-- Previously it wasn't because 'T.Text' used UTF-16 as its internal representation. -- -- @since 0.1 toText :: ShortText -> T.Text+#if MIN_VERSION_text(2,0,0)+toText (ShortText (BSSI.SBS ba)) = TI.Text (TA.ByteArray ba) 0 (I# (GHC.Exts.sizeofByteArray# ba))+#else toText = T.decodeUtf8 . toByteString+#endif ---- @@ -657,21 +674,37 @@ -- -- @since 0.1 fromString :: String -> ShortText-fromString [] = mempty-fromString [c] = singleton c-fromString s = ShortText . encodeStringShort utf8 . map r $ s+fromString s = case s of+ [] -> mempty+ [c] -> singleton $ r c+ _ -> ShortText . encodeStringShort utf8 . map r $ s where r c | isSurr (ord c) = '\xFFFD' | otherwise = c -- | \(\mathcal{O}(n)\) Construct 'ShortText' from 'T.Text' ----- This is currently not \(\mathcal{O}(1)\) because currently 'T.Text' uses UTF-16 as its internal representation.--- In the event that 'T.Text' will change its internal representation to UTF-8 this operation will become \(\mathcal{O}(1)\).+-- This is \(\mathcal{O}(1)\) with @text-2@ when the 'T.Text' is not sliced.+-- Previously it wasn't because 'T.Text' used UTF-16 as its internal representation. -- -- @since 0.1 fromText :: T.Text -> ShortText+#if MIN_VERSION_text(2,0,0)+fromText (TI.Text (TA.ByteArray ba) off len) =+ ShortText (BSSI.SBS (case sliceByteArray (TA.ByteArray ba) off len of TA.ByteArray ba' -> ba'))++sliceByteArray :: TA.Array -> Int -> Int -> TA.Array+sliceByteArray ta@(TA.ByteArray ba) 0 len+ | len == I# (GHC.Exts.sizeofByteArray# ba)+ = ta+sliceByteArray ta off len = TA.run $ do+ ma <- TA.new len+ TA.copyI len ma 0 ta off+ return ma++#else fromText = fromByteStringUnsafe . T.encodeUtf8+#endif -- | \(\mathcal{O}(n)\) Construct 'ShortText' from UTF-8 encoded 'ShortByteString' --@@ -1271,7 +1304,12 @@ {-# INLINE writeWord8Array #-} writeWord8Array :: MBA s -> B -> Word -> ST s () writeWord8Array (MBA# mba#) (B (I# i#)) (W# w#)- = ST $ \s -> case GHC.Exts.writeWord8Array# mba# i# w# s of+ = ST $ \s ->+#if __GLASGOW_HASKELL__ >= 902+ case GHC.Exts.writeWord8Array# mba# i# (GHC.Exts.wordToWord8# w#) s of+#else+ case GHC.Exts.writeWord8Array# mba# i# w# s of+#endif s' -> (# s', () #) {- not needed yet {-# INLINE indexWord8Array #-}
text-short.cabal view
@@ -1,90 +1,100 @@-cabal-version: 1.18+cabal-version: 1.18+name: text-short+version: 0.1.6.1+synopsis: Memory-efficient representation of Unicode text strings+license: BSD3+license-file: LICENSE+author: Herbert Valerio Riedel+maintainer: hvr@gnu.org+bug-reports: https://github.com/hvr/text-short/issues+category: Data+build-type: Simple+description:+ This package provides the 'ShortText' type which is suitable for keeping many short strings in memory. This is similiar to how 'ShortByteString' relates to 'ByteString'.+ .+ The main difference between 'Text' and 'ShortText' is that 'ShortText' doesn't support zero-copy slicing (thereby saving 2 words), and, compared to text-1.*, that it uses UTF-8 instead of UTF-16 internally. Consequently, the memory footprint of a (boxed) 'ShortText' value is 4 words (2 words when unboxed) plus the length of the UTF-8 encoded payload. -name: text-short-version: 0.1.3-synopsis: Memory-efficient representation of Unicode text strings-license: BSD3-license-file: LICENSE-author: Herbert Valerio Riedel-maintainer: hvr@gnu.org-bug-reports: https://github.com/hvr/text-short/issues-category: Data-build-type: Simple-description: This package provides the 'ShortText' type which is suitable for keeping many short strings in memory. This is similiar to how 'ShortByteString' relates to 'ByteString'.- .- The main difference between 'Text' and 'ShortText' is that 'ShortText' uses UTF-8 instead of UTF-16 internally and also doesn't support zero-copy slicing (thereby saving 2 words). Consequently, the memory footprint of a (boxed) 'ShortText' value is 4 words (2 words when unboxed) plus the length of the UTF-8 encoded payload.+tested-with:+ GHC ==8.6.5+ || ==8.8.3+ || ==8.10.7+ || ==9.0.2+ || ==9.2.8+ || ==9.4.8+ || ==9.6.6+ || ==9.8.4+ || ==9.10.2+ || ==9.12.2+ || ==9.14.1 -tested-with: GHC==8.6.5, GHC==8.4.4, GHC==8.2.2, GHC==8.0.2, GHC==7.10.3, GHC==7.8.4-extra-source-files: ChangeLog.md+extra-source-files: ChangeLog.md -Source-Repository head- Type: git- Location: https://github.com/hvr/text-short.git+source-repository head+ type: git+ location: https://github.com/hvr/text-short.git flag asserts description: Enable runtime-checks via @assert@- default: False- manual: True+ default: False+ manual: True library- exposed-modules: Data.Text.Short- Data.Text.Short.Partial- Data.Text.Short.Unsafe-- other-modules: Data.Text.Short.Internal-- build-depends: base >= 4.7 && < 4.13- , bytestring >= 0.10.4 && < 0.11- , hashable >= 1.2.6 && < 1.4- , deepseq >= 1.3 && < 1.5- , text >= 1.0 && < 1.3- , binary >= 0.7.1 && < 0.9- , ghc-prim >= 0.3.1 && < 0.6-- if !impl(ghc >= 8.0)- build-depends: semigroups >= 0.18.2 && < 0.20-- -- GHC version specific PrimOps- if impl(ghc >= 8.4)- hs-source-dirs: src-ghc804- else- c-sources: cbits/memcmp.c- hs-source-dirs: src-ghc708- other-modules: PrimOps+ exposed-modules:+ Data.Text.Short+ Data.Text.Short.Partial+ Data.Text.Short.Unsafe - hs-source-dirs: src+ other-modules: Data.Text.Short.Internal+ build-depends:+ base >=4.12 && <4.23+ , binary >=0.8.6.0 && <0.9+ , bytestring >=0.10.8.2 && <0.13+ , deepseq >=1.4.4.0 && <1.6+ , ghc-prim >=0.5.3 && <0.14+ , hashable >=1.4.4.0 && <1.6+ , template-haskell >=2.14.0.0 && <2.25+ , text >=1.2.3.1 && <1.3 || >=2.0 && <2.2 - default-language: Haskell2010- other-extensions: CPP- , GeneralizedNewtypeDeriving- , MagicHash- , UnliftedFFITypes- , Trustworthy- , Unsafe+ other-modules: PrimOps+ hs-source-dirs: src src-ghc804+ default-language: Haskell2010+ other-extensions:+ CPP+ GeneralizedNewtypeDeriving+ MagicHash+ TemplateHaskellQuotes+ Trustworthy+ UnliftedFFITypes+ Unsafe - c-sources: cbits/cbits.c+ c-sources: cbits/cbits.c if flag(asserts)- ghc-options: -fno-ignore-asserts+ ghc-options: -fno-ignore-asserts+ else- cc-options: -DNDEBUG=1+ cc-options: -DNDEBUG=1 - ghc-options: -Wall- cc-options: -O3 -Wall+ ghc-options: -Wall+ cc-options: -Wall -test-suite tests- type: exitcode-stdio-1.0- hs-source-dirs: src-test- main-is: Tests.hs+test-suite text-short-tests+ type: exitcode-stdio-1.0+ hs-source-dirs: src-test+ main-is: Tests.hs - build-depends: base- , binary- , text- , text-short- -- deps which don't inherit constraints from library stanza:- , tasty >= 1.0.0 && < 1.3- , tasty-quickcheck >= 0.10 && < 0.11- , tasty-hunit >= 0.10.0 && < 0.11- , quickcheck-instances >= 0.3.14 && < 0.4+ -- bytestring dependency for cabal_macros.h+ build-depends:+ base+ , binary+ , bytestring+ , text+ , text-short - default-language: Haskell2010+ -- deps which don't inherit constraints from library stanza:+ build-depends:+ tasty >=1.4 && <1.6+ , tasty-hunit >=0.10.0 && <0.11+ , tasty-quickcheck >=0.10 && <0.12++ default-language: Haskell2010