packages feed

text 0.2 → 0.3

raw patch · 15 files changed

+927/−51 lines, 15 files

Files

Data/Text.hs view
@@ -55,6 +55,12 @@     , transpose     , reverse +    -- * Case conversion+    -- $case+    , toCaseFold+    , toLower+    , toUpper+     -- * Folds     , foldl     , foldl'@@ -380,6 +386,53 @@ reverse :: Text -> Text reverse t = S.reverse (stream t) {-# INLINE reverse #-}++-- ----------------------------------------------------------------------------+-- ** Case conversions (folds)++-- $case+--+-- With Unicode text, it is incorrect to use combinators like @map+-- toUpper@ to case convert each character of a string individually.+-- Instead, use the whole-string case conversion functions from this+-- module.  For correctness in different writing systems, these+-- functions may map one input character to two or three output+-- characters.++-- | /O(n)/ Convert a string to folded case.  This function is mainly+-- useful for performing caseless (or case insensitive) string+-- comparisons.+--+-- A string @x@ is a caseless match for a string @y@ if and only if:+--+-- @toCaseFold x == toCaseFold y@+--+-- The result string may be longer than the input string, and may+-- differ from applying 'toLower' to the input string.  For instance,+-- the Armenian small ligature men now (U+FB13) is case folded to the+-- bigram men now (U+0574 U+0576), while the micro sign (U+00B5) is+-- case folded to the Greek small letter letter mu (U+03BC) instead of+-- itself.+toCaseFold :: Text -> Text+toCaseFold t = unstream (S.toCaseFold (stream t))+{-# INLINE [0] toCaseFold #-}++-- | /O(n)/ Convert a string to lower case, using simple case+-- conversion.  The result string may be longer than the input string.+-- For instance, the Latin capital letter I with dot above (U+0130)+-- maps to the sequence Latin small letter i (U+0069) followed by+-- combining dot above (U+0307).+toLower :: Text -> Text+toLower t = unstream (S.toLower (stream t))+{-# INLINE toLower #-}++-- | /O(n)/ Convert a string to upper case, using simple case+-- conversion.  The result string may be longer than the input string.+-- For instance, the German eszett (U+00DF) maps to the two-letter+-- sequence SS.+toUpper :: Text -> Text+toUpper t = unstream (S.toUpper (stream t))+{-# INLINE toUpper #-}  -- | /O(n)/ The 'transpose' function transposes the rows and columns -- of its 'Text' argument.  Note that this function uses 'pack',
Data/Text/Encoding.hs view
@@ -25,6 +25,12 @@     , decodeUtf16BE     , decodeUtf32LE     , decodeUtf32BE+    -- ** Controllable error handling+    , decodeUtf8With+    , decodeUtf16LEWith+    , decodeUtf16BEWith+    , decodeUtf32LEWith+    , decodeUtf32BEWith      -- * Encoding Text to ByteStrings     , encodeUtf8@@ -36,6 +42,7 @@      import Data.ByteString (ByteString) import qualified Data.Text.Fusion as F+import Data.Text.Encoding.Error (OnDecodeError, strictDecode) import qualified Data.Text.Encoding.Fusion as E import Data.Text.Internal (Text) @@ -45,8 +52,13 @@ {-# INLINE decodeASCII #-}  -- | Decode a 'ByteString' containing UTF-8 encoded text.+decodeUtf8With :: OnDecodeError -> ByteString -> Text+decodeUtf8With onErr bs = F.unstream (E.streamUtf8 onErr bs)+{-# INLINE decodeUtf8With #-}++-- | Decode a 'ByteString' containing UTF-8 encoded text. decodeUtf8 :: ByteString -> Text-decodeUtf8 bs = F.unstream (E.streamUtf8 bs)+decodeUtf8 = decodeUtf8With strictDecode {-# INLINE decodeUtf8 #-}  -- | Encode text using UTF-8 encoding.@@ -55,13 +67,23 @@ {-# INLINE encodeUtf8 #-}  -- | Decode text from little endian UTF-16 encoding.+decodeUtf16LEWith :: OnDecodeError -> ByteString -> Text+decodeUtf16LEWith onErr bs = F.unstream (E.streamUtf16LE onErr bs)+{-# INLINE decodeUtf16LEWith #-}++-- | Decode text from little endian UTF-16 encoding. decodeUtf16LE :: ByteString -> Text-decodeUtf16LE bs = F.unstream (E.streamUtf16LE bs)+decodeUtf16LE = decodeUtf16LEWith strictDecode {-# INLINE decodeUtf16LE #-}  -- | Decode text from big endian UTF-16 encoding.+decodeUtf16BEWith :: OnDecodeError -> ByteString -> Text+decodeUtf16BEWith onErr bs = F.unstream (E.streamUtf16BE onErr bs)+{-# INLINE decodeUtf16BEWith #-}++-- | Decode text from big endian UTF-16 encoding. decodeUtf16BE :: ByteString -> Text-decodeUtf16BE bs = F.unstream (E.streamUtf16BE bs)+decodeUtf16BE = decodeUtf16BEWith strictDecode {-# INLINE decodeUtf16BE #-}  -- | Encode text using little endian UTF-16 encoding.@@ -75,13 +97,23 @@ {-# INLINE encodeUtf16BE #-}  -- | Decode text from little endian UTF-32 encoding.+decodeUtf32LEWith :: OnDecodeError -> ByteString -> Text+decodeUtf32LEWith onErr bs = F.unstream (E.streamUtf32LE onErr bs)+{-# INLINE decodeUtf32LEWith #-}++-- | Decode text from little endian UTF-32 encoding. decodeUtf32LE :: ByteString -> Text-decodeUtf32LE bs = F.unstream (E.streamUtf32LE bs)+decodeUtf32LE = decodeUtf32LEWith strictDecode {-# INLINE decodeUtf32LE #-}  -- | Decode text from big endian UTF-32 encoding.+decodeUtf32BEWith :: OnDecodeError -> ByteString -> Text+decodeUtf32BEWith onErr bs = F.unstream (E.streamUtf32BE onErr bs)+{-# INLINE decodeUtf32BEWith #-}++-- | Decode text from big endian UTF-32 encoding. decodeUtf32BE :: ByteString -> Text-decodeUtf32BE bs = F.unstream (E.streamUtf32BE bs)+decodeUtf32BE = decodeUtf32BEWith strictDecode {-# INLINE decodeUtf32BE #-}  -- | Encode text using little endian UTF-32 encoding.
+ Data/Text/Encoding/Error.hs view
@@ -0,0 +1,108 @@+{-# LANGUAGE DeriveDataTypeable #-}+-- |+-- Module      : Data.Text.Encoding.Error+-- Copyright   : (c) Bryan O'Sullivan 2009+--+-- License     : BSD-style+-- Maintainer  : bos@serpentine.com, rtharper@aftereternity.co.uk,+--               duncan@haskell.org+-- Stability   : experimental+-- Portability : GHC+--+-- Types and functions for dealing with encoding and decoding errors+-- in Unicode text.+--+-- The standard functions for encoding and decoding text are strict,+-- which is to say that they throw exceptions on invalid input.  This+-- is often unhelpful on real world input, so alternative functions+-- exist that accept custom handlers for dealing with invalid inputs.+-- These 'OnError' handlers are normal Haskell functions.  You can use+-- one of the presupplied functions in this module, or you can write a+-- custom handler of your own.++module Data.Text.Encoding.Error+    (+    -- * Error handling types+      UnicodeException(..)+    , OnError+    , OnDecodeError+    , OnEncodeError+    -- * Useful error handling functions+    , lenientDecode+    , strictDecode+    , strictEncode+    , ignore+    , replace+    ) where++import Control.Exception (Exception, throw)+import Data.Typeable (Typeable)+import Data.Word (Word8)+import Numeric (showHex)++-- | Function type for handling a coding error.  It is supplied with+-- two inputs:+--+-- * A 'String' that describes the error.+--+-- * The input value that caused the error.  If the error arose+--   because the end of input was reached or could not be identified+--   precisely, this value will be 'Nothing'.+--+-- If the handler returns a value wrapped with 'Just', that value will+-- be used in the output as the replacement for the invalid input.  If+-- it returns 'Nothing', no value will be used in the output.+--+-- Should the handler need to abort processing, it should use 'error'+-- or 'throw' an exception (preferably a 'UnicodeException').  It may+-- use the description provided to construct a more helpful error+-- report.+type OnError a b = String -> Maybe a -> Maybe b+type OnDecodeError = OnError Word8 Char+type OnEncodeError = OnError Char Word8++-- | An exception type for representing Unicode encoding errors.+data UnicodeException =+    DecodeError String (Maybe Word8)+    -- ^ Could not decode a byte sequence because it was invalid under+    -- the given encoding, or ran out of input in mid-decode.+  | EncodeError String (Maybe Char)+    -- ^ Tried to encode a character that could not be represented+    -- under the given encoding, or ran out of input in mid-encode.+    deriving (Typeable)++showUnicodeException :: UnicodeException -> String+showUnicodeException (DecodeError desc (Just w))+    = "Cannot decode byte '\\x" ++ showHex w ("': " ++ desc)+showUnicodeException (DecodeError desc Nothing)+    = "Cannot decode input: " ++ desc+showUnicodeException (EncodeError desc (Just c))+    = "Cannot encode character '\\x" ++ showHex (fromEnum c) ("': " ++ desc)+showUnicodeException (EncodeError desc Nothing)+    = "Cannot encode input: " ++ desc+                     +instance Show UnicodeException where+    show = showUnicodeException++instance Exception UnicodeException++-- | Throw a 'UnicodeException' if decoding fails.+strictDecode :: OnError Word8 Char+strictDecode desc c = throw (DecodeError desc c)++-- | Replace an invalid input byte with the Unicode replacement+-- character U+FFFD.+lenientDecode :: OnError Word8 Char+lenientDecode _ _ = Just '\xfffd'++-- | Throw a 'UnicodeException' if encoding fails.+strictEncode :: OnError Char Word8+strictEncode desc c = throw (EncodeError desc c)++-- | Ignore an invalid input, substituting nothing in the output.+ignore :: OnError a b+ignore _ _ = Nothing++-- | Replace an invalid input with a valid output.+replace :: b -> OnError a b+replace c _ _ = Just c
Data/Text/Encoding/Fusion.hs view
@@ -1,4 +1,4 @@-{-# LANGUAGE BangPatterns #-}+{-# LANGUAGE BangPatterns, Rank2Types #-}  -- | -- Module      : Data.Text.Encoding.Fusion@@ -32,12 +32,13 @@     ) where  import Control.Exception (assert)-import Data.Bits (shiftL) import Data.ByteString as B import Data.ByteString.Internal (ByteString(..), mallocByteString, memcpy) import Data.Text.Fusion (Step(..), Stream(..))+import Data.Text.Encoding.Error import Data.Text.Encoding.Fusion.Common import Data.Text.UnsafeChar (unsafeChr, unsafeChr8, unsafeChr32)+import Data.Text.UnsafeShift (shiftL) import Data.Word (Word8, Word16, Word32) import Foreign.ForeignPtr (withForeignPtr, ForeignPtr) import Foreign.Storable (pokeByteOff)@@ -62,8 +63,8 @@  -- | /O(n)/ Convert a 'ByteString' into a 'Stream Char', using UTF-8 -- encoding.-streamUtf8 :: ByteString -> Stream Char-streamUtf8 bs = Stream next 0 l+streamUtf8 :: OnDecodeError -> ByteString -> Stream Char+streamUtf8 onErr bs = Stream next 0 l     where       l = B.length bs       {-# INLINE next #-}@@ -73,8 +74,9 @@           | i+1 < l && U8.validate2 x1 x2 = Yield (U8.chr2 x1 x2) (i+2)           | i+2 < l && U8.validate3 x1 x2 x3 = Yield (U8.chr3 x1 x2 x3) (i+3)           | i+3 < l && U8.validate4 x1 x2 x3 x4 = Yield (U8.chr4 x1 x2 x3 x4) (i+4)-          | otherwise = encodingError "UTF-8"+          | otherwise = decodeError "streamUtf8" "UTF-8" onErr mx (i+1)           where+            mx = if i >= l then Nothing else Just x1             x1 = idx i             x2 = idx (i + 1)             x3 = idx (i + 2)@@ -84,8 +86,8 @@  -- | /O(n)/ Convert a 'ByteString' into a 'Stream Char', using little -- endian UTF-16 encoding.-streamUtf16LE :: ByteString -> Stream Char-streamUtf16LE bs = Stream next 0 l+streamUtf16LE :: OnDecodeError -> ByteString -> Stream Char+streamUtf16LE onErr bs = Stream next 0 l     where       l = B.length bs       {-# INLINE next #-}@@ -93,7 +95,7 @@           | i >= l                         = Done           | i+1 < l && U16.validate1 x1    = Yield (unsafeChr x1) (i+2)           | i+3 < l && U16.validate2 x1 x2 = Yield (U16.chr2 x1 x2) (i+4)-          | otherwise = encodingError "UTF-16LE"+          | otherwise = decodeError "streamUtf16LE" "UTF-16LE" onErr Nothing (i+1)           where             x1    = idx i       + (idx (i + 1) `shiftL` 8)             x2    = idx (i + 2) + (idx (i + 3) `shiftL` 8)@@ -102,8 +104,8 @@  -- | /O(n)/ Convert a 'ByteString' into a 'Stream Char', using big -- endian UTF-16 encoding.-streamUtf16BE :: ByteString -> Stream Char-streamUtf16BE bs = Stream next 0 l+streamUtf16BE :: OnDecodeError -> ByteString -> Stream Char+streamUtf16BE onErr bs = Stream next 0 l     where       l = B.length bs       {-# INLINE next #-}@@ -111,7 +113,7 @@           | i >= l                         = Done           | i+1 < l && U16.validate1 x1    = Yield (unsafeChr x1) (i+2)           | i+3 < l && U16.validate2 x1 x2 = Yield (U16.chr2 x1 x2) (i+4)-          | otherwise = encodingError "UTF16-BE"+          | otherwise = decodeError "streamUtf16BE" "UTF-16BE" onErr Nothing (i+1)           where             x1    = (idx i `shiftL` 8)       + idx (i + 1)             x2    = (idx (i + 2) `shiftL` 8) + idx (i + 3)@@ -120,15 +122,15 @@  -- | /O(n)/ Convert a 'ByteString' into a 'Stream Char', using big -- endian UTF-32 encoding.-streamUtf32BE :: ByteString -> Stream Char-streamUtf32BE bs = Stream next 0 l+streamUtf32BE :: OnDecodeError -> ByteString -> Stream Char+streamUtf32BE onErr bs = Stream next 0 l     where       l = B.length bs       {-# INLINE next #-}       next i           | i >= l                    = Done           | i+3 < l && U32.validate x = Yield (unsafeChr32 x) (i+4)-          | otherwise                 = encodingError "UTF-32BE"+          | otherwise = decodeError "streamUtf32BE" "UTF-32BE" onErr Nothing (i+1)           where             x     = shiftL x1 24 + shiftL x2 16 + shiftL x3 8 + x4             x1    = idx i@@ -140,15 +142,15 @@  -- | /O(n)/ Convert a 'ByteString' into a 'Stream Char', using little -- endian UTF-32 encoding.-streamUtf32LE :: ByteString -> Stream Char-streamUtf32LE bs = Stream next 0 l+streamUtf32LE :: OnDecodeError -> ByteString -> Stream Char+streamUtf32LE onErr bs = Stream next 0 l     where       l = B.length bs       {-# INLINE next #-}       next i           | i >= l                    = Done           | i+3 < l && U32.validate x = Yield (unsafeChr32 x) (i+4)-          | otherwise                 = encodingError "UTF-32LE"+          | otherwise = decodeError "streamUtf32LE" "UTF-32LE" onErr Nothing (i+1)           where             x     = shiftL x4 24 + shiftL x3 16 + shiftL x2 8 + x1             x1    = idx i@@ -187,6 +189,11 @@                   memcpy dest' src' (fromIntegral srcLen)           return dest -encodingError :: String -> a-encodingError encoding =-    error $ "Data.Text.Encoding.Fusion: Bad " ++ encoding ++ " stream"+decodeError :: forall s. String -> String -> OnDecodeError -> Maybe Word8+            -> s -> Step s Char+decodeError func kind onErr mb i =+    case onErr desc mb of+      Nothing -> Skip i+      Just c  -> Yield c i+    where desc = "Data.Text.Encoding.Fusion." ++ func ++ ": Invalid " +++                 kind ++ " stream"
Data/Text/Encoding/Fusion/Common.hs view
@@ -27,10 +27,11 @@     , restreamUtf32BE     ) where -import Data.Bits (shiftR, (.&.))+import Data.Bits ((.&.)) import Data.Char (ord) import Data.Text.Fusion (Step(..), Stream(..)) import Data.Text.Fusion.Internal (M(..), S(..))+import Data.Text.UnsafeShift (shiftR) import Data.Word (Word8) import qualified Data.Text.Encoding.Utf8 as U8
Data/Text/Encoding/Utf8.hs view
@@ -32,7 +32,8 @@  import Control.Exception (assert) import Data.Char (ord)-import Data.Bits (shiftR, (.&.))+import Data.Bits ((.&.))+import Data.Text.UnsafeShift (shiftR) import GHC.Exts import GHC.Word (Word8(..))
Data/Text/Fusion.hs view
@@ -50,10 +50,11 @@ import Prelude (Bool(..), Char, Eq(..), Maybe(..), Monad(..), Int,                 Num(..), Ord(..), ($), (&&),                 fromIntegral, otherwise)-import Data.Bits ((.&.), shiftR)+import Data.Bits ((.&.)) import Data.Char (ord) import Data.Text.Internal (Text(..)) import Data.Text.UnsafeChar (unsafeChr, unsafeWrite)+import Data.Text.UnsafeShift (shiftR) import qualified Data.Text.Array as A import qualified Data.Text.Fusion.Common as S import Data.Text.Fusion.Internal
+ Data/Text/Fusion/CaseMapping.hs view
@@ -0,0 +1,456 @@+{-# LANGUAGE Rank2Types #-}+-- AUTOMATICALLY GENERATED - DO NOT EDIT+-- Generated by scripts/SpecialCasing.hs+module Data.Text.Fusion.CaseMapping where+import Data.Char+import Data.Text.Fusion.Internal++upperMapping :: forall s. Char -> s -> Step (PairS (PairS s Char) Char) Char+{-# INLINE upperMapping #-}+-- LATIN SMALL LETTER SHARP S+upperMapping '\x00df' s = Yield '\x0053' (s :!: '\x0053' :!: '\x0000')+-- LATIN SMALL LIGATURE FF+upperMapping '\xfb00' s = Yield '\x0046' (s :!: '\x0046' :!: '\x0000')+-- LATIN SMALL LIGATURE FI+upperMapping '\xfb01' s = Yield '\x0046' (s :!: '\x0049' :!: '\x0000')+-- LATIN SMALL LIGATURE FL+upperMapping '\xfb02' s = Yield '\x0046' (s :!: '\x004c' :!: '\x0000')+-- LATIN SMALL LIGATURE FFI+upperMapping '\xfb03' s = Yield '\x0046' (s :!: '\x0046' :!: '\x0049')+-- LATIN SMALL LIGATURE FFL+upperMapping '\xfb04' s = Yield '\x0046' (s :!: '\x0046' :!: '\x004c')+-- LATIN SMALL LIGATURE LONG S T+upperMapping '\xfb05' s = Yield '\x0053' (s :!: '\x0054' :!: '\x0000')+-- LATIN SMALL LIGATURE ST+upperMapping '\xfb06' s = Yield '\x0053' (s :!: '\x0054' :!: '\x0000')+-- ARMENIAN SMALL LIGATURE ECH YIWN+upperMapping '\x0587' s = Yield '\x0535' (s :!: '\x0552' :!: '\x0000')+-- ARMENIAN SMALL LIGATURE MEN NOW+upperMapping '\xfb13' s = Yield '\x0544' (s :!: '\x0546' :!: '\x0000')+-- ARMENIAN SMALL LIGATURE MEN ECH+upperMapping '\xfb14' s = Yield '\x0544' (s :!: '\x0535' :!: '\x0000')+-- ARMENIAN SMALL LIGATURE MEN INI+upperMapping '\xfb15' s = Yield '\x0544' (s :!: '\x053b' :!: '\x0000')+-- ARMENIAN SMALL LIGATURE VEW NOW+upperMapping '\xfb16' s = Yield '\x054e' (s :!: '\x0546' :!: '\x0000')+-- ARMENIAN SMALL LIGATURE MEN XEH+upperMapping '\xfb17' s = Yield '\x0544' (s :!: '\x053d' :!: '\x0000')+-- LATIN SMALL LETTER N PRECEDED BY APOSTROPHE+upperMapping '\x0149' s = Yield '\x02bc' (s :!: '\x004e' :!: '\x0000')+-- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND TONOS+upperMapping '\x0390' s = Yield '\x0399' (s :!: '\x0308' :!: '\x0301')+-- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND TONOS+upperMapping '\x03b0' s = Yield '\x03a5' (s :!: '\x0308' :!: '\x0301')+-- LATIN SMALL LETTER J WITH CARON+upperMapping '\x01f0' s = Yield '\x004a' (s :!: '\x030c' :!: '\x0000')+-- LATIN SMALL LETTER H WITH LINE BELOW+upperMapping '\x1e96' s = Yield '\x0048' (s :!: '\x0331' :!: '\x0000')+-- LATIN SMALL LETTER T WITH DIAERESIS+upperMapping '\x1e97' s = Yield '\x0054' (s :!: '\x0308' :!: '\x0000')+-- LATIN SMALL LETTER W WITH RING ABOVE+upperMapping '\x1e98' s = Yield '\x0057' (s :!: '\x030a' :!: '\x0000')+-- LATIN SMALL LETTER Y WITH RING ABOVE+upperMapping '\x1e99' s = Yield '\x0059' (s :!: '\x030a' :!: '\x0000')+-- LATIN SMALL LETTER A WITH RIGHT HALF RING+upperMapping '\x1e9a' s = Yield '\x0041' (s :!: '\x02be' :!: '\x0000')+-- GREEK SMALL LETTER UPSILON WITH PSILI+upperMapping '\x1f50' s = Yield '\x03a5' (s :!: '\x0313' :!: '\x0000')+-- GREEK SMALL LETTER UPSILON WITH PSILI AND VARIA+upperMapping '\x1f52' s = Yield '\x03a5' (s :!: '\x0313' :!: '\x0300')+-- GREEK SMALL LETTER UPSILON WITH PSILI AND OXIA+upperMapping '\x1f54' s = Yield '\x03a5' (s :!: '\x0313' :!: '\x0301')+-- GREEK SMALL LETTER UPSILON WITH PSILI AND PERISPOMENI+upperMapping '\x1f56' s = Yield '\x03a5' (s :!: '\x0313' :!: '\x0342')+-- GREEK SMALL LETTER ALPHA WITH PERISPOMENI+upperMapping '\x1fb6' s = Yield '\x0391' (s :!: '\x0342' :!: '\x0000')+-- GREEK SMALL LETTER ETA WITH PERISPOMENI+upperMapping '\x1fc6' s = Yield '\x0397' (s :!: '\x0342' :!: '\x0000')+-- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND VARIA+upperMapping '\x1fd2' s = Yield '\x0399' (s :!: '\x0308' :!: '\x0300')+-- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND OXIA+upperMapping '\x1fd3' s = Yield '\x0399' (s :!: '\x0308' :!: '\x0301')+-- GREEK SMALL LETTER IOTA WITH PERISPOMENI+upperMapping '\x1fd6' s = Yield '\x0399' (s :!: '\x0342' :!: '\x0000')+-- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND PERISPOMENI+upperMapping '\x1fd7' s = Yield '\x0399' (s :!: '\x0308' :!: '\x0342')+-- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND VARIA+upperMapping '\x1fe2' s = Yield '\x03a5' (s :!: '\x0308' :!: '\x0300')+-- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND OXIA+upperMapping '\x1fe3' s = Yield '\x03a5' (s :!: '\x0308' :!: '\x0301')+-- GREEK SMALL LETTER RHO WITH PSILI+upperMapping '\x1fe4' s = Yield '\x03a1' (s :!: '\x0313' :!: '\x0000')+-- GREEK SMALL LETTER UPSILON WITH PERISPOMENI+upperMapping '\x1fe6' s = Yield '\x03a5' (s :!: '\x0342' :!: '\x0000')+-- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND PERISPOMENI+upperMapping '\x1fe7' s = Yield '\x03a5' (s :!: '\x0308' :!: '\x0342')+-- GREEK SMALL LETTER OMEGA WITH PERISPOMENI+upperMapping '\x1ff6' s = Yield '\x03a9' (s :!: '\x0342' :!: '\x0000')+-- GREEK SMALL LETTER ALPHA WITH PSILI AND YPOGEGRAMMENI+upperMapping '\x1f80' s = Yield '\x1f08' (s :!: '\x0399' :!: '\x0000')+-- GREEK SMALL LETTER ALPHA WITH DASIA AND YPOGEGRAMMENI+upperMapping '\x1f81' s = Yield '\x1f09' (s :!: '\x0399' :!: '\x0000')+-- GREEK SMALL LETTER ALPHA WITH PSILI AND VARIA AND YPOGEGRAMMENI+upperMapping '\x1f82' s = Yield '\x1f0a' (s :!: '\x0399' :!: '\x0000')+-- GREEK SMALL LETTER ALPHA WITH DASIA AND VARIA AND YPOGEGRAMMENI+upperMapping '\x1f83' s = Yield '\x1f0b' (s :!: '\x0399' :!: '\x0000')+-- GREEK SMALL LETTER ALPHA WITH PSILI AND OXIA AND YPOGEGRAMMENI+upperMapping '\x1f84' s = Yield '\x1f0c' (s :!: '\x0399' :!: '\x0000')+-- GREEK SMALL LETTER ALPHA WITH DASIA AND OXIA AND YPOGEGRAMMENI+upperMapping '\x1f85' s = Yield '\x1f0d' (s :!: '\x0399' :!: '\x0000')+-- GREEK SMALL LETTER ALPHA WITH PSILI AND PERISPOMENI AND YPOGEGRAMMENI+upperMapping '\x1f86' s = Yield '\x1f0e' (s :!: '\x0399' :!: '\x0000')+-- GREEK SMALL LETTER ALPHA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI+upperMapping '\x1f87' s = Yield '\x1f0f' (s :!: '\x0399' :!: '\x0000')+-- GREEK CAPITAL LETTER ALPHA WITH PSILI AND PROSGEGRAMMENI+upperMapping '\x1f88' s = Yield '\x1f08' (s :!: '\x0399' :!: '\x0000')+-- GREEK CAPITAL LETTER ALPHA WITH DASIA AND PROSGEGRAMMENI+upperMapping '\x1f89' s = Yield '\x1f09' (s :!: '\x0399' :!: '\x0000')+-- GREEK CAPITAL LETTER ALPHA WITH PSILI AND VARIA AND PROSGEGRAMMENI+upperMapping '\x1f8a' s = Yield '\x1f0a' (s :!: '\x0399' :!: '\x0000')+-- GREEK CAPITAL LETTER ALPHA WITH DASIA AND VARIA AND PROSGEGRAMMENI+upperMapping '\x1f8b' s = Yield '\x1f0b' (s :!: '\x0399' :!: '\x0000')+-- GREEK CAPITAL LETTER ALPHA WITH PSILI AND OXIA AND PROSGEGRAMMENI+upperMapping '\x1f8c' s = Yield '\x1f0c' (s :!: '\x0399' :!: '\x0000')+-- GREEK CAPITAL LETTER ALPHA WITH DASIA AND OXIA AND PROSGEGRAMMENI+upperMapping '\x1f8d' s = Yield '\x1f0d' (s :!: '\x0399' :!: '\x0000')+-- GREEK CAPITAL LETTER ALPHA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI+upperMapping '\x1f8e' s = Yield '\x1f0e' (s :!: '\x0399' :!: '\x0000')+-- GREEK CAPITAL LETTER ALPHA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI+upperMapping '\x1f8f' s = Yield '\x1f0f' (s :!: '\x0399' :!: '\x0000')+-- GREEK SMALL LETTER ETA WITH PSILI AND YPOGEGRAMMENI+upperMapping '\x1f90' s = Yield '\x1f28' (s :!: '\x0399' :!: '\x0000')+-- GREEK SMALL LETTER ETA WITH DASIA AND YPOGEGRAMMENI+upperMapping '\x1f91' s = Yield '\x1f29' (s :!: '\x0399' :!: '\x0000')+-- GREEK SMALL LETTER ETA WITH PSILI AND VARIA AND YPOGEGRAMMENI+upperMapping '\x1f92' s = Yield '\x1f2a' (s :!: '\x0399' :!: '\x0000')+-- GREEK SMALL LETTER ETA WITH DASIA AND VARIA AND YPOGEGRAMMENI+upperMapping '\x1f93' s = Yield '\x1f2b' (s :!: '\x0399' :!: '\x0000')+-- GREEK SMALL LETTER ETA WITH PSILI AND OXIA AND YPOGEGRAMMENI+upperMapping '\x1f94' s = Yield '\x1f2c' (s :!: '\x0399' :!: '\x0000')+-- GREEK SMALL LETTER ETA WITH DASIA AND OXIA AND YPOGEGRAMMENI+upperMapping '\x1f95' s = Yield '\x1f2d' (s :!: '\x0399' :!: '\x0000')+-- GREEK SMALL LETTER ETA WITH PSILI AND PERISPOMENI AND YPOGEGRAMMENI+upperMapping '\x1f96' s = Yield '\x1f2e' (s :!: '\x0399' :!: '\x0000')+-- GREEK SMALL LETTER ETA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI+upperMapping '\x1f97' s = Yield '\x1f2f' (s :!: '\x0399' :!: '\x0000')+-- GREEK CAPITAL LETTER ETA WITH PSILI AND PROSGEGRAMMENI+upperMapping '\x1f98' s = Yield '\x1f28' (s :!: '\x0399' :!: '\x0000')+-- GREEK CAPITAL LETTER ETA WITH DASIA AND PROSGEGRAMMENI+upperMapping '\x1f99' s = Yield '\x1f29' (s :!: '\x0399' :!: '\x0000')+-- GREEK CAPITAL LETTER ETA WITH PSILI AND VARIA AND PROSGEGRAMMENI+upperMapping '\x1f9a' s = Yield '\x1f2a' (s :!: '\x0399' :!: '\x0000')+-- GREEK CAPITAL LETTER ETA WITH DASIA AND VARIA AND PROSGEGRAMMENI+upperMapping '\x1f9b' s = Yield '\x1f2b' (s :!: '\x0399' :!: '\x0000')+-- GREEK CAPITAL LETTER ETA WITH PSILI AND OXIA AND PROSGEGRAMMENI+upperMapping '\x1f9c' s = Yield '\x1f2c' (s :!: '\x0399' :!: '\x0000')+-- GREEK CAPITAL LETTER ETA WITH DASIA AND OXIA AND PROSGEGRAMMENI+upperMapping '\x1f9d' s = Yield '\x1f2d' (s :!: '\x0399' :!: '\x0000')+-- GREEK CAPITAL LETTER ETA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI+upperMapping '\x1f9e' s = Yield '\x1f2e' (s :!: '\x0399' :!: '\x0000')+-- GREEK CAPITAL LETTER ETA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI+upperMapping '\x1f9f' s = Yield '\x1f2f' (s :!: '\x0399' :!: '\x0000')+-- GREEK SMALL LETTER OMEGA WITH PSILI AND YPOGEGRAMMENI+upperMapping '\x1fa0' s = Yield '\x1f68' (s :!: '\x0399' :!: '\x0000')+-- GREEK SMALL LETTER OMEGA WITH DASIA AND YPOGEGRAMMENI+upperMapping '\x1fa1' s = Yield '\x1f69' (s :!: '\x0399' :!: '\x0000')+-- GREEK SMALL LETTER OMEGA WITH PSILI AND VARIA AND YPOGEGRAMMENI+upperMapping '\x1fa2' s = Yield '\x1f6a' (s :!: '\x0399' :!: '\x0000')+-- GREEK SMALL LETTER OMEGA WITH DASIA AND VARIA AND YPOGEGRAMMENI+upperMapping '\x1fa3' s = Yield '\x1f6b' (s :!: '\x0399' :!: '\x0000')+-- GREEK SMALL LETTER OMEGA WITH PSILI AND OXIA AND YPOGEGRAMMENI+upperMapping '\x1fa4' s = Yield '\x1f6c' (s :!: '\x0399' :!: '\x0000')+-- GREEK SMALL LETTER OMEGA WITH DASIA AND OXIA AND YPOGEGRAMMENI+upperMapping '\x1fa5' s = Yield '\x1f6d' (s :!: '\x0399' :!: '\x0000')+-- GREEK SMALL LETTER OMEGA WITH PSILI AND PERISPOMENI AND YPOGEGRAMMENI+upperMapping '\x1fa6' s = Yield '\x1f6e' (s :!: '\x0399' :!: '\x0000')+-- GREEK SMALL LETTER OMEGA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI+upperMapping '\x1fa7' s = Yield '\x1f6f' (s :!: '\x0399' :!: '\x0000')+-- GREEK CAPITAL LETTER OMEGA WITH PSILI AND PROSGEGRAMMENI+upperMapping '\x1fa8' s = Yield '\x1f68' (s :!: '\x0399' :!: '\x0000')+-- GREEK CAPITAL LETTER OMEGA WITH DASIA AND PROSGEGRAMMENI+upperMapping '\x1fa9' s = Yield '\x1f69' (s :!: '\x0399' :!: '\x0000')+-- GREEK CAPITAL LETTER OMEGA WITH PSILI AND VARIA AND PROSGEGRAMMENI+upperMapping '\x1faa' s = Yield '\x1f6a' (s :!: '\x0399' :!: '\x0000')+-- GREEK CAPITAL LETTER OMEGA WITH DASIA AND VARIA AND PROSGEGRAMMENI+upperMapping '\x1fab' s = Yield '\x1f6b' (s :!: '\x0399' :!: '\x0000')+-- GREEK CAPITAL LETTER OMEGA WITH PSILI AND OXIA AND PROSGEGRAMMENI+upperMapping '\x1fac' s = Yield '\x1f6c' (s :!: '\x0399' :!: '\x0000')+-- GREEK CAPITAL LETTER OMEGA WITH DASIA AND OXIA AND PROSGEGRAMMENI+upperMapping '\x1fad' s = Yield '\x1f6d' (s :!: '\x0399' :!: '\x0000')+-- GREEK CAPITAL LETTER OMEGA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI+upperMapping '\x1fae' s = Yield '\x1f6e' (s :!: '\x0399' :!: '\x0000')+-- GREEK CAPITAL LETTER OMEGA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI+upperMapping '\x1faf' s = Yield '\x1f6f' (s :!: '\x0399' :!: '\x0000')+-- GREEK SMALL LETTER ALPHA WITH YPOGEGRAMMENI+upperMapping '\x1fb3' s = Yield '\x0391' (s :!: '\x0399' :!: '\x0000')+-- GREEK CAPITAL LETTER ALPHA WITH PROSGEGRAMMENI+upperMapping '\x1fbc' s = Yield '\x0391' (s :!: '\x0399' :!: '\x0000')+-- GREEK SMALL LETTER ETA WITH YPOGEGRAMMENI+upperMapping '\x1fc3' s = Yield '\x0397' (s :!: '\x0399' :!: '\x0000')+-- GREEK CAPITAL LETTER ETA WITH PROSGEGRAMMENI+upperMapping '\x1fcc' s = Yield '\x0397' (s :!: '\x0399' :!: '\x0000')+-- GREEK SMALL LETTER OMEGA WITH YPOGEGRAMMENI+upperMapping '\x1ff3' s = Yield '\x03a9' (s :!: '\x0399' :!: '\x0000')+-- GREEK CAPITAL LETTER OMEGA WITH PROSGEGRAMMENI+upperMapping '\x1ffc' s = Yield '\x03a9' (s :!: '\x0399' :!: '\x0000')+-- GREEK SMALL LETTER ALPHA WITH VARIA AND YPOGEGRAMMENI+upperMapping '\x1fb2' s = Yield '\x1fba' (s :!: '\x0399' :!: '\x0000')+-- GREEK SMALL LETTER ALPHA WITH OXIA AND YPOGEGRAMMENI+upperMapping '\x1fb4' s = Yield '\x0386' (s :!: '\x0399' :!: '\x0000')+-- GREEK SMALL LETTER ETA WITH VARIA AND YPOGEGRAMMENI+upperMapping '\x1fc2' s = Yield '\x1fca' (s :!: '\x0399' :!: '\x0000')+-- GREEK SMALL LETTER ETA WITH OXIA AND YPOGEGRAMMENI+upperMapping '\x1fc4' s = Yield '\x0389' (s :!: '\x0399' :!: '\x0000')+-- GREEK SMALL LETTER OMEGA WITH VARIA AND YPOGEGRAMMENI+upperMapping '\x1ff2' s = Yield '\x1ffa' (s :!: '\x0399' :!: '\x0000')+-- GREEK SMALL LETTER OMEGA WITH OXIA AND YPOGEGRAMMENI+upperMapping '\x1ff4' s = Yield '\x038f' (s :!: '\x0399' :!: '\x0000')+-- GREEK SMALL LETTER ALPHA WITH PERISPOMENI AND YPOGEGRAMMENI+upperMapping '\x1fb7' s = Yield '\x0391' (s :!: '\x0342' :!: '\x0399')+-- GREEK SMALL LETTER ETA WITH PERISPOMENI AND YPOGEGRAMMENI+upperMapping '\x1fc7' s = Yield '\x0397' (s :!: '\x0342' :!: '\x0399')+-- GREEK SMALL LETTER OMEGA WITH PERISPOMENI AND YPOGEGRAMMENI+upperMapping '\x1ff7' s = Yield '\x03a9' (s :!: '\x0342' :!: '\x0399')+upperMapping c s = Yield (toUpper c) (s :!: '\0' :!: '\0')+lowerMapping :: forall s. Char -> s -> Step (PairS (PairS s Char) Char) Char+{-# INLINE lowerMapping #-}+-- LATIN CAPITAL LETTER I WITH DOT ABOVE+lowerMapping '\x0130' s = Yield '\x0069' (s :!: '\x0307' :!: '\x0000')+lowerMapping c s = Yield (toLower c) (s :!: '\0' :!: '\0')+foldMapping :: forall s. Char -> s -> Step (PairS (PairS s Char) Char) Char+{-# INLINE foldMapping #-}+-- MICRO SIGN+foldMapping '\x00b5' s = Yield '\x03bc' (s :!: '\x0000' :!: '\x0000')+-- LATIN SMALL LETTER SHARP S+foldMapping '\x00df' s = Yield '\x0073' (s :!: '\x0073' :!: '\x0000')+-- LATIN CAPITAL LETTER I WITH DOT ABOVE+foldMapping '\x0130' s = Yield '\x0069' (s :!: '\x0307' :!: '\x0000')+-- LATIN SMALL LETTER N PRECEDED BY APOSTROPHE+foldMapping '\x0149' s = Yield '\x02bc' (s :!: '\x006e' :!: '\x0000')+-- LATIN SMALL LETTER LONG S+foldMapping '\x017f' s = Yield '\x0073' (s :!: '\x0000' :!: '\x0000')+-- LATIN SMALL LETTER J WITH CARON+foldMapping '\x01f0' s = Yield '\x006a' (s :!: '\x030c' :!: '\x0000')+-- COMBINING GREEK YPOGEGRAMMENI+foldMapping '\x0345' s = Yield '\x03b9' (s :!: '\x0000' :!: '\x0000')+-- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND TONOS+foldMapping '\x0390' s = Yield '\x03b9' (s :!: '\x0308' :!: '\x0301')+-- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND TONOS+foldMapping '\x03b0' s = Yield '\x03c5' (s :!: '\x0308' :!: '\x0301')+-- GREEK SMALL LETTER FINAL SIGMA+foldMapping '\x03c2' s = Yield '\x03c3' (s :!: '\x0000' :!: '\x0000')+-- GREEK BETA SYMBOL+foldMapping '\x03d0' s = Yield '\x03b2' (s :!: '\x0000' :!: '\x0000')+-- GREEK THETA SYMBOL+foldMapping '\x03d1' s = Yield '\x03b8' (s :!: '\x0000' :!: '\x0000')+-- GREEK PHI SYMBOL+foldMapping '\x03d5' s = Yield '\x03c6' (s :!: '\x0000' :!: '\x0000')+-- GREEK PI SYMBOL+foldMapping '\x03d6' s = Yield '\x03c0' (s :!: '\x0000' :!: '\x0000')+-- GREEK KAPPA SYMBOL+foldMapping '\x03f0' s = Yield '\x03ba' (s :!: '\x0000' :!: '\x0000')+-- GREEK RHO SYMBOL+foldMapping '\x03f1' s = Yield '\x03c1' (s :!: '\x0000' :!: '\x0000')+-- GREEK LUNATE EPSILON SYMBOL+foldMapping '\x03f5' s = Yield '\x03b5' (s :!: '\x0000' :!: '\x0000')+-- ARMENIAN SMALL LIGATURE ECH YIWN+foldMapping '\x0587' s = Yield '\x0565' (s :!: '\x0582' :!: '\x0000')+-- LATIN SMALL LETTER H WITH LINE BELOW+foldMapping '\x1e96' s = Yield '\x0068' (s :!: '\x0331' :!: '\x0000')+-- LATIN SMALL LETTER T WITH DIAERESIS+foldMapping '\x1e97' s = Yield '\x0074' (s :!: '\x0308' :!: '\x0000')+-- LATIN SMALL LETTER W WITH RING ABOVE+foldMapping '\x1e98' s = Yield '\x0077' (s :!: '\x030a' :!: '\x0000')+-- LATIN SMALL LETTER Y WITH RING ABOVE+foldMapping '\x1e99' s = Yield '\x0079' (s :!: '\x030a' :!: '\x0000')+-- LATIN SMALL LETTER A WITH RIGHT HALF RING+foldMapping '\x1e9a' s = Yield '\x0061' (s :!: '\x02be' :!: '\x0000')+-- LATIN SMALL LETTER LONG S WITH DOT ABOVE+foldMapping '\x1e9b' s = Yield '\x1e61' (s :!: '\x0000' :!: '\x0000')+-- LATIN CAPITAL LETTER SHARP S+foldMapping '\x1e9e' s = Yield '\x0073' (s :!: '\x0073' :!: '\x0000')+-- GREEK SMALL LETTER UPSILON WITH PSILI+foldMapping '\x1f50' s = Yield '\x03c5' (s :!: '\x0313' :!: '\x0000')+-- GREEK SMALL LETTER UPSILON WITH PSILI AND VARIA+foldMapping '\x1f52' s = Yield '\x03c5' (s :!: '\x0313' :!: '\x0300')+-- GREEK SMALL LETTER UPSILON WITH PSILI AND OXIA+foldMapping '\x1f54' s = Yield '\x03c5' (s :!: '\x0313' :!: '\x0301')+-- GREEK SMALL LETTER UPSILON WITH PSILI AND PERISPOMENI+foldMapping '\x1f56' s = Yield '\x03c5' (s :!: '\x0313' :!: '\x0342')+-- GREEK SMALL LETTER ALPHA WITH PSILI AND YPOGEGRAMMENI+foldMapping '\x1f80' s = Yield '\x1f00' (s :!: '\x03b9' :!: '\x0000')+-- GREEK SMALL LETTER ALPHA WITH DASIA AND YPOGEGRAMMENI+foldMapping '\x1f81' s = Yield '\x1f01' (s :!: '\x03b9' :!: '\x0000')+-- GREEK SMALL LETTER ALPHA WITH PSILI AND VARIA AND YPOGEGRAMMENI+foldMapping '\x1f82' s = Yield '\x1f02' (s :!: '\x03b9' :!: '\x0000')+-- GREEK SMALL LETTER ALPHA WITH DASIA AND VARIA AND YPOGEGRAMMENI+foldMapping '\x1f83' s = Yield '\x1f03' (s :!: '\x03b9' :!: '\x0000')+-- GREEK SMALL LETTER ALPHA WITH PSILI AND OXIA AND YPOGEGRAMMENI+foldMapping '\x1f84' s = Yield '\x1f04' (s :!: '\x03b9' :!: '\x0000')+-- GREEK SMALL LETTER ALPHA WITH DASIA AND OXIA AND YPOGEGRAMMENI+foldMapping '\x1f85' s = Yield '\x1f05' (s :!: '\x03b9' :!: '\x0000')+-- GREEK SMALL LETTER ALPHA WITH PSILI AND PERISPOMENI AND YPOGEGRAMMENI+foldMapping '\x1f86' s = Yield '\x1f06' (s :!: '\x03b9' :!: '\x0000')+-- GREEK SMALL LETTER ALPHA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI+foldMapping '\x1f87' s = Yield '\x1f07' (s :!: '\x03b9' :!: '\x0000')+-- GREEK CAPITAL LETTER ALPHA WITH PSILI AND PROSGEGRAMMENI+foldMapping '\x1f88' s = Yield '\x1f00' (s :!: '\x03b9' :!: '\x0000')+-- GREEK CAPITAL LETTER ALPHA WITH DASIA AND PROSGEGRAMMENI+foldMapping '\x1f89' s = Yield '\x1f01' (s :!: '\x03b9' :!: '\x0000')+-- GREEK CAPITAL LETTER ALPHA WITH PSILI AND VARIA AND PROSGEGRAMMENI+foldMapping '\x1f8a' s = Yield '\x1f02' (s :!: '\x03b9' :!: '\x0000')+-- GREEK CAPITAL LETTER ALPHA WITH DASIA AND VARIA AND PROSGEGRAMMENI+foldMapping '\x1f8b' s = Yield '\x1f03' (s :!: '\x03b9' :!: '\x0000')+-- GREEK CAPITAL LETTER ALPHA WITH PSILI AND OXIA AND PROSGEGRAMMENI+foldMapping '\x1f8c' s = Yield '\x1f04' (s :!: '\x03b9' :!: '\x0000')+-- GREEK CAPITAL LETTER ALPHA WITH DASIA AND OXIA AND PROSGEGRAMMENI+foldMapping '\x1f8d' s = Yield '\x1f05' (s :!: '\x03b9' :!: '\x0000')+-- GREEK CAPITAL LETTER ALPHA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI+foldMapping '\x1f8e' s = Yield '\x1f06' (s :!: '\x03b9' :!: '\x0000')+-- GREEK CAPITAL LETTER ALPHA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI+foldMapping '\x1f8f' s = Yield '\x1f07' (s :!: '\x03b9' :!: '\x0000')+-- GREEK SMALL LETTER ETA WITH PSILI AND YPOGEGRAMMENI+foldMapping '\x1f90' s = Yield '\x1f20' (s :!: '\x03b9' :!: '\x0000')+-- GREEK SMALL LETTER ETA WITH DASIA AND YPOGEGRAMMENI+foldMapping '\x1f91' s = Yield '\x1f21' (s :!: '\x03b9' :!: '\x0000')+-- GREEK SMALL LETTER ETA WITH PSILI AND VARIA AND YPOGEGRAMMENI+foldMapping '\x1f92' s = Yield '\x1f22' (s :!: '\x03b9' :!: '\x0000')+-- GREEK SMALL LETTER ETA WITH DASIA AND VARIA AND YPOGEGRAMMENI+foldMapping '\x1f93' s = Yield '\x1f23' (s :!: '\x03b9' :!: '\x0000')+-- GREEK SMALL LETTER ETA WITH PSILI AND OXIA AND YPOGEGRAMMENI+foldMapping '\x1f94' s = Yield '\x1f24' (s :!: '\x03b9' :!: '\x0000')+-- GREEK SMALL LETTER ETA WITH DASIA AND OXIA AND YPOGEGRAMMENI+foldMapping '\x1f95' s = Yield '\x1f25' (s :!: '\x03b9' :!: '\x0000')+-- GREEK SMALL LETTER ETA WITH PSILI AND PERISPOMENI AND YPOGEGRAMMENI+foldMapping '\x1f96' s = Yield '\x1f26' (s :!: '\x03b9' :!: '\x0000')+-- GREEK SMALL LETTER ETA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI+foldMapping '\x1f97' s = Yield '\x1f27' (s :!: '\x03b9' :!: '\x0000')+-- GREEK CAPITAL LETTER ETA WITH PSILI AND PROSGEGRAMMENI+foldMapping '\x1f98' s = Yield '\x1f20' (s :!: '\x03b9' :!: '\x0000')+-- GREEK CAPITAL LETTER ETA WITH DASIA AND PROSGEGRAMMENI+foldMapping '\x1f99' s = Yield '\x1f21' (s :!: '\x03b9' :!: '\x0000')+-- GREEK CAPITAL LETTER ETA WITH PSILI AND VARIA AND PROSGEGRAMMENI+foldMapping '\x1f9a' s = Yield '\x1f22' (s :!: '\x03b9' :!: '\x0000')+-- GREEK CAPITAL LETTER ETA WITH DASIA AND VARIA AND PROSGEGRAMMENI+foldMapping '\x1f9b' s = Yield '\x1f23' (s :!: '\x03b9' :!: '\x0000')+-- GREEK CAPITAL LETTER ETA WITH PSILI AND OXIA AND PROSGEGRAMMENI+foldMapping '\x1f9c' s = Yield '\x1f24' (s :!: '\x03b9' :!: '\x0000')+-- GREEK CAPITAL LETTER ETA WITH DASIA AND OXIA AND PROSGEGRAMMENI+foldMapping '\x1f9d' s = Yield '\x1f25' (s :!: '\x03b9' :!: '\x0000')+-- GREEK CAPITAL LETTER ETA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI+foldMapping '\x1f9e' s = Yield '\x1f26' (s :!: '\x03b9' :!: '\x0000')+-- GREEK CAPITAL LETTER ETA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI+foldMapping '\x1f9f' s = Yield '\x1f27' (s :!: '\x03b9' :!: '\x0000')+-- GREEK SMALL LETTER OMEGA WITH PSILI AND YPOGEGRAMMENI+foldMapping '\x1fa0' s = Yield '\x1f60' (s :!: '\x03b9' :!: '\x0000')+-- GREEK SMALL LETTER OMEGA WITH DASIA AND YPOGEGRAMMENI+foldMapping '\x1fa1' s = Yield '\x1f61' (s :!: '\x03b9' :!: '\x0000')+-- GREEK SMALL LETTER OMEGA WITH PSILI AND VARIA AND YPOGEGRAMMENI+foldMapping '\x1fa2' s = Yield '\x1f62' (s :!: '\x03b9' :!: '\x0000')+-- GREEK SMALL LETTER OMEGA WITH DASIA AND VARIA AND YPOGEGRAMMENI+foldMapping '\x1fa3' s = Yield '\x1f63' (s :!: '\x03b9' :!: '\x0000')+-- GREEK SMALL LETTER OMEGA WITH PSILI AND OXIA AND YPOGEGRAMMENI+foldMapping '\x1fa4' s = Yield '\x1f64' (s :!: '\x03b9' :!: '\x0000')+-- GREEK SMALL LETTER OMEGA WITH DASIA AND OXIA AND YPOGEGRAMMENI+foldMapping '\x1fa5' s = Yield '\x1f65' (s :!: '\x03b9' :!: '\x0000')+-- GREEK SMALL LETTER OMEGA WITH PSILI AND PERISPOMENI AND YPOGEGRAMMENI+foldMapping '\x1fa6' s = Yield '\x1f66' (s :!: '\x03b9' :!: '\x0000')+-- GREEK SMALL LETTER OMEGA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI+foldMapping '\x1fa7' s = Yield '\x1f67' (s :!: '\x03b9' :!: '\x0000')+-- GREEK CAPITAL LETTER OMEGA WITH PSILI AND PROSGEGRAMMENI+foldMapping '\x1fa8' s = Yield '\x1f60' (s :!: '\x03b9' :!: '\x0000')+-- GREEK CAPITAL LETTER OMEGA WITH DASIA AND PROSGEGRAMMENI+foldMapping '\x1fa9' s = Yield '\x1f61' (s :!: '\x03b9' :!: '\x0000')+-- GREEK CAPITAL LETTER OMEGA WITH PSILI AND VARIA AND PROSGEGRAMMENI+foldMapping '\x1faa' s = Yield '\x1f62' (s :!: '\x03b9' :!: '\x0000')+-- GREEK CAPITAL LETTER OMEGA WITH DASIA AND VARIA AND PROSGEGRAMMENI+foldMapping '\x1fab' s = Yield '\x1f63' (s :!: '\x03b9' :!: '\x0000')+-- GREEK CAPITAL LETTER OMEGA WITH PSILI AND OXIA AND PROSGEGRAMMENI+foldMapping '\x1fac' s = Yield '\x1f64' (s :!: '\x03b9' :!: '\x0000')+-- GREEK CAPITAL LETTER OMEGA WITH DASIA AND OXIA AND PROSGEGRAMMENI+foldMapping '\x1fad' s = Yield '\x1f65' (s :!: '\x03b9' :!: '\x0000')+-- GREEK CAPITAL LETTER OMEGA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI+foldMapping '\x1fae' s = Yield '\x1f66' (s :!: '\x03b9' :!: '\x0000')+-- GREEK CAPITAL LETTER OMEGA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI+foldMapping '\x1faf' s = Yield '\x1f67' (s :!: '\x03b9' :!: '\x0000')+-- GREEK SMALL LETTER ALPHA WITH VARIA AND YPOGEGRAMMENI+foldMapping '\x1fb2' s = Yield '\x1f70' (s :!: '\x03b9' :!: '\x0000')+-- GREEK SMALL LETTER ALPHA WITH YPOGEGRAMMENI+foldMapping '\x1fb3' s = Yield '\x03b1' (s :!: '\x03b9' :!: '\x0000')+-- GREEK SMALL LETTER ALPHA WITH OXIA AND YPOGEGRAMMENI+foldMapping '\x1fb4' s = Yield '\x03ac' (s :!: '\x03b9' :!: '\x0000')+-- GREEK SMALL LETTER ALPHA WITH PERISPOMENI+foldMapping '\x1fb6' s = Yield '\x03b1' (s :!: '\x0342' :!: '\x0000')+-- GREEK SMALL LETTER ALPHA WITH PERISPOMENI AND YPOGEGRAMMENI+foldMapping '\x1fb7' s = Yield '\x03b1' (s :!: '\x0342' :!: '\x03b9')+-- GREEK CAPITAL LETTER ALPHA WITH PROSGEGRAMMENI+foldMapping '\x1fbc' s = Yield '\x03b1' (s :!: '\x03b9' :!: '\x0000')+-- GREEK PROSGEGRAMMENI+foldMapping '\x1fbe' s = Yield '\x03b9' (s :!: '\x0000' :!: '\x0000')+-- GREEK SMALL LETTER ETA WITH VARIA AND YPOGEGRAMMENI+foldMapping '\x1fc2' s = Yield '\x1f74' (s :!: '\x03b9' :!: '\x0000')+-- GREEK SMALL LETTER ETA WITH YPOGEGRAMMENI+foldMapping '\x1fc3' s = Yield '\x03b7' (s :!: '\x03b9' :!: '\x0000')+-- GREEK SMALL LETTER ETA WITH OXIA AND YPOGEGRAMMENI+foldMapping '\x1fc4' s = Yield '\x03ae' (s :!: '\x03b9' :!: '\x0000')+-- GREEK SMALL LETTER ETA WITH PERISPOMENI+foldMapping '\x1fc6' s = Yield '\x03b7' (s :!: '\x0342' :!: '\x0000')+-- GREEK SMALL LETTER ETA WITH PERISPOMENI AND YPOGEGRAMMENI+foldMapping '\x1fc7' s = Yield '\x03b7' (s :!: '\x0342' :!: '\x03b9')+-- GREEK CAPITAL LETTER ETA WITH PROSGEGRAMMENI+foldMapping '\x1fcc' s = Yield '\x03b7' (s :!: '\x03b9' :!: '\x0000')+-- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND VARIA+foldMapping '\x1fd2' s = Yield '\x03b9' (s :!: '\x0308' :!: '\x0300')+-- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND OXIA+foldMapping '\x1fd3' s = Yield '\x03b9' (s :!: '\x0308' :!: '\x0301')+-- GREEK SMALL LETTER IOTA WITH PERISPOMENI+foldMapping '\x1fd6' s = Yield '\x03b9' (s :!: '\x0342' :!: '\x0000')+-- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND PERISPOMENI+foldMapping '\x1fd7' s = Yield '\x03b9' (s :!: '\x0308' :!: '\x0342')+-- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND VARIA+foldMapping '\x1fe2' s = Yield '\x03c5' (s :!: '\x0308' :!: '\x0300')+-- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND OXIA+foldMapping '\x1fe3' s = Yield '\x03c5' (s :!: '\x0308' :!: '\x0301')+-- GREEK SMALL LETTER RHO WITH PSILI+foldMapping '\x1fe4' s = Yield '\x03c1' (s :!: '\x0313' :!: '\x0000')+-- GREEK SMALL LETTER UPSILON WITH PERISPOMENI+foldMapping '\x1fe6' s = Yield '\x03c5' (s :!: '\x0342' :!: '\x0000')+-- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND PERISPOMENI+foldMapping '\x1fe7' s = Yield '\x03c5' (s :!: '\x0308' :!: '\x0342')+-- GREEK SMALL LETTER OMEGA WITH VARIA AND YPOGEGRAMMENI+foldMapping '\x1ff2' s = Yield '\x1f7c' (s :!: '\x03b9' :!: '\x0000')+-- GREEK SMALL LETTER OMEGA WITH YPOGEGRAMMENI+foldMapping '\x1ff3' s = Yield '\x03c9' (s :!: '\x03b9' :!: '\x0000')+-- GREEK SMALL LETTER OMEGA WITH OXIA AND YPOGEGRAMMENI+foldMapping '\x1ff4' s = Yield '\x03ce' (s :!: '\x03b9' :!: '\x0000')+-- GREEK SMALL LETTER OMEGA WITH PERISPOMENI+foldMapping '\x1ff6' s = Yield '\x03c9' (s :!: '\x0342' :!: '\x0000')+-- GREEK SMALL LETTER OMEGA WITH PERISPOMENI AND YPOGEGRAMMENI+foldMapping '\x1ff7' s = Yield '\x03c9' (s :!: '\x0342' :!: '\x03b9')+-- GREEK CAPITAL LETTER OMEGA WITH PROSGEGRAMMENI+foldMapping '\x1ffc' s = Yield '\x03c9' (s :!: '\x03b9' :!: '\x0000')+-- LATIN SMALL LIGATURE FF+foldMapping '\xfb00' s = Yield '\x0066' (s :!: '\x0066' :!: '\x0000')+-- LATIN SMALL LIGATURE FI+foldMapping '\xfb01' s = Yield '\x0066' (s :!: '\x0069' :!: '\x0000')+-- LATIN SMALL LIGATURE FL+foldMapping '\xfb02' s = Yield '\x0066' (s :!: '\x006c' :!: '\x0000')+-- LATIN SMALL LIGATURE FFI+foldMapping '\xfb03' s = Yield '\x0066' (s :!: '\x0066' :!: '\x0069')+-- LATIN SMALL LIGATURE FFL+foldMapping '\xfb04' s = Yield '\x0066' (s :!: '\x0066' :!: '\x006c')+-- LATIN SMALL LIGATURE LONG S T+foldMapping '\xfb05' s = Yield '\x0073' (s :!: '\x0074' :!: '\x0000')+-- LATIN SMALL LIGATURE ST+foldMapping '\xfb06' s = Yield '\x0073' (s :!: '\x0074' :!: '\x0000')+-- ARMENIAN SMALL LIGATURE MEN NOW+foldMapping '\xfb13' s = Yield '\x0574' (s :!: '\x0576' :!: '\x0000')+-- ARMENIAN SMALL LIGATURE MEN ECH+foldMapping '\xfb14' s = Yield '\x0574' (s :!: '\x0565' :!: '\x0000')+-- ARMENIAN SMALL LIGATURE MEN INI+foldMapping '\xfb15' s = Yield '\x0574' (s :!: '\x056b' :!: '\x0000')+-- ARMENIAN SMALL LIGATURE VEW NOW+foldMapping '\xfb16' s = Yield '\x057e' (s :!: '\x0576' :!: '\x0000')+-- ARMENIAN SMALL LIGATURE MEN XEH+foldMapping '\xfb17' s = Yield '\x0574' (s :!: '\x056d' :!: '\x0000')+foldMapping c s = Yield (toLower c) (s :!: '\0' :!: '\0')
Data/Text/Fusion/Common.hs view
@@ -1,4 +1,4 @@-{-# LANGUAGE BangPatterns #-}+{-# LANGUAGE BangPatterns, Rank2Types #-} -- | -- Module      : Data.Text.Fusion.Common -- Copyright   : (c) Bryan O'Sullivan 2009@@ -35,6 +35,12 @@     , intercalate     , intersperse +    -- ** Case conversion+    -- $case+    , toCaseFold+    , toLower+    , toUpper+     -- * Folds     , foldl     , foldl'@@ -96,6 +102,7 @@ import qualified Data.List as L import qualified Prelude as P import Data.Text.Fusion.Internal+import Data.Text.Fusion.CaseMapping (foldMapping, lowerMapping, upperMapping)  singleton :: Char -> Stream Char singleton c = Stream next False 1 -- HINT maybe too low@@ -291,6 +298,65 @@         Yield x s' -> Yield c (s' :!: J x :!: S1)       next _ = internalError "intersperse" {-# INLINE [0] intersperse #-}++-- ----------------------------------------------------------------------------+-- ** Case conversions (folds)++-- $case+--+-- With Unicode text, it is incorrect to use combinators like @map+-- toUpper@ to case convert each character of a string individually.+-- Instead, use the whole-string case conversion functions from this+-- module.  For correctness in different writing systems, these+-- functions may map one input character to two or three output+-- characters.++caseConvert :: (forall s. Char -> s -> Step (PairS (PairS s Char) Char) Char)+            -> Stream Char -> Stream Char+caseConvert remap (Stream next0 s0 len) = Stream next (s0 :!: '\0' :!: '\0') len+  where+    {-# INLINE next #-}+    next (s :!: '\0' :!: _) =+        case next0 s of+          Done       -> Done+          Skip s'    -> Skip (s' :!: '\0' :!: '\0')+          Yield c s' -> remap c s'+    next (s :!: a :!: b) = Yield a (s :!: b :!: '\0')++-- | /O(n)/ Convert a string to folded case.  This function is mainly+-- useful for performing caseless (or case insensitive) string+-- comparisons.+--+-- A string @x@ is a caseless match for a string @y@ if and only if:+--+-- @toCaseFold x == toCaseFold y@+--+-- The result string may be longer than the input string, and may+-- differ from applying 'toLower' to the input string.  For instance,+-- the Armenian small ligature men now (U+FB13) is case folded to the+-- bigram men now (U+0574 U+0576), while the micro sign (U+00B5) is+-- case folded to the Greek small letter letter mu (U+03BC) instead of+-- itself.+toCaseFold :: Stream Char -> Stream Char+toCaseFold = caseConvert foldMapping+{-# INLINE [0] toCaseFold #-}++-- | /O(n)/ Convert a string to upper case, using simple case+-- conversion.  The result string may be longer than the input string.+-- For instance, the German eszett (U+00DF) maps to the two-letter+-- sequence SS.+toUpper :: Stream Char -> Stream Char+toUpper = caseConvert upperMapping+{-# INLINE [0] toUpper #-}++-- | /O(n)/ Convert a string to lower case, using simple case+-- conversion.  The result string may be longer than the input string.+-- For instance, the Latin capital letter I with dot above (U+0130)+-- maps to the sequence Latin small letter i (U+0069) followed by+-- combining dot above (U+0307).+toLower :: Stream Char -> Stream Char+toLower = caseConvert lowerMapping+{-# INLINE [0] toLower #-}  -- ---------------------------------------------------------------------------- -- * Reducing Streams (folds)
Data/Text/Lazy.hs view
@@ -58,6 +58,12 @@     , transpose     , reverse +    -- ** Case conversion+    -- $case+    , toCaseFold+    , toLower+    , toUpper+     -- * Folds     , foldl     , foldl'@@ -366,6 +372,53 @@ reverse = rev Empty   where rev a Empty        = a         rev a (Chunk t ts) = rev (Chunk (T.reverse t) a) ts++-- ----------------------------------------------------------------------------+-- ** Case conversions (folds)++-- $case+--+-- With Unicode text, it is incorrect to use combinators like @map+-- toUpper@ to case convert each character of a string individually.+-- Instead, use the whole-string case conversion functions from this+-- module.  For correctness in different writing systems, these+-- functions may map one input character to two or three output+-- characters.++-- | /O(n)/ Convert a string to folded case.  This function is mainly+-- useful for performing caseless (or case insensitive) string+-- comparisons.+--+-- A string @x@ is a caseless match for a string @y@ if and only if:+--+-- @toCaseFold x == toCaseFold y@+--+-- The result string may be longer than the input string, and may+-- differ from applying 'toLower' to the input string.  For instance,+-- the Armenian small ligature men now (U+FB13) is case folded to the+-- bigram men now (U+0574 U+0576), while the micro sign (U+00B5) is+-- case folded to the Greek small letter letter mu (U+03BC) instead of+-- itself.+toCaseFold :: Text -> Text+toCaseFold t = unstream (S.toCaseFold (stream t))+{-# INLINE [0] toCaseFold #-}++-- | /O(n)/ Convert a string to lower case, using simple case+-- conversion.  The result string may be longer than the input string.+-- For instance, the Latin capital letter I with dot above (U+0130)+-- maps to the sequence Latin small letter i (U+0069) followed by+-- combining dot above (U+0307).+toLower :: Text -> Text+toLower t = unstream (S.toLower (stream t))+{-# INLINE toLower #-}++-- | /O(n)/ Convert a string to upper case, using simple case+-- conversion.  The result string may be longer than the input string.+-- For instance, the German eszett (U+00DF) maps to the two-letter+-- sequence SS.+toUpper :: Text -> Text+toUpper t = unstream (S.toUpper (stream t))+{-# INLINE toUpper #-}  -- | /O(n)/ 'foldl', applied to a binary operator, a starting value -- (typically the left-identity of the operator), and a 'Text',
Data/Text/Lazy/Encoding.hs view
@@ -19,6 +19,7 @@     -- * Decoding ByteStrings to Text     --  decodeASCII       decodeUtf8+    , decodeUtf8With     --, decodeUtf16LE     --, decodeUtf16BE     --, decodeUtf32LE@@ -33,13 +34,19 @@     ) where  import Data.ByteString.Lazy (ByteString)+import Data.Text.Encoding.Error (OnDecodeError, strictDecode) import Data.Text.Lazy (Text) import qualified Data.Text.Lazy.Fusion as F import qualified Data.Text.Lazy.Encoding.Fusion as E  -- | Decode a 'ByteString' containing UTF-8 encoded text.+decodeUtf8With :: OnDecodeError -> ByteString -> Text+decodeUtf8With onErr bs = F.unstream (E.streamUtf8 onErr bs)+{-# INLINE decodeUtf8With #-}++-- | Decode a 'ByteString' containing UTF-8 encoded text. decodeUtf8 :: ByteString -> Text-decodeUtf8 bs = F.unstream (E.streamUtf8 bs)+decodeUtf8 = decodeUtf8With strictDecode {-# INLINE decodeUtf8 #-}  -- | Encode text using UTF-8 encoding.
Data/Text/Lazy/Encoding/Fusion.hs view
@@ -1,4 +1,4 @@-{-# LANGUAGE BangPatterns #-}+{-# LANGUAGE BangPatterns, Rank2Types #-}  -- | -- Module      : Data.Text.Lazy.Encoding.Fusion@@ -33,6 +33,7 @@ import qualified Data.ByteString as B import qualified Data.ByteString.Unsafe as B import Data.Text.Encoding.Fusion.Common+import Data.Text.Encoding.Error import Data.Text.Fusion (Step(..), Stream(..)) import Data.Text.Fusion.Internal (M(..), PairS(..), S(..)) import Data.Text.UnsafeChar (unsafeChr8)@@ -50,10 +51,25 @@  -- | /O(n)/ Convert a lazy 'ByteString' into a 'Stream Char', using -- UTF-8 encoding.-streamUtf8 :: ByteString -> Stream Char-streamUtf8 bs0 = Stream next (bs0 :!: S N N N N :!: 0) unknownLength+streamUtf8 :: OnDecodeError -> ByteString -> Stream Char+streamUtf8 onErr bs0 = Stream next (bs0 :!: empty :!: 0) unknownLength     where+      empty = S N N N N       {-# INLINE next #-}+      next (bs@(Chunk ps _) :!: S N _ _ _ :!: i)+          | i < len && U8.validate1 a =+              Yield (unsafeChr8 a) (bs :!: empty :!: i+1)+          | i + 1 < len && U8.validate2 a b =+              Yield (U8.chr2 a b) (bs :!: empty :!: i+2)+          | i + 2 < len && U8.validate3 a b c =+              Yield (U8.chr3 a b c) (bs :!: empty :!: i+3)+          | i + 4 < len && U8.validate4 a b c d =+              Yield (U8.chr4 a b c d) (bs :!: empty :!: i+4)+          where len = B.length ps+                a = B.unsafeIndex ps i+                b = B.unsafeIndex ps (i+1)+                c = B.unsafeIndex ps (i+2)+                d = B.unsafeIndex ps (i+3)       next st@(bs :!: s :!: i) =         case s of           S (J a) N _ _             | U8.validate1 a ->@@ -65,21 +81,21 @@           S (J a) (J b) (J c) (J d) | U8.validate4 a b c d ->             Yield (U8.chr4 a b c d) es           _ -> consume st-         where es = bs :!: S N N N N :!: i+         where es = bs :!: empty :!: i       {-# INLINE consume #-}       consume (bs@(Chunk ps rest) :!: s :!: i)-          | i >= len    = consume (rest :!: s  :!: 0)-          | otherwise   = next    (bs   :!: s' :!: i+1)-          where s' = case s of-                       S N _ _ _ -> S x N N N-                       S a N _ _ -> S a x N N-                       S a b N _ -> S a b x N-                       S a b c N -> S a b c x-                       _         -> encodingError "streamUtf8" "UTF-8"-                x   = J (B.unsafeIndex ps i)-                len = B.length ps+          | i >= B.length ps = consume (rest :!: s  :!: 0)+          | otherwise =+        case s of+          S N _ _ _ -> next (bs :!: S x N N N :!: i+1)+          S a N _ _ -> next (bs :!: S a x N N :!: i+1)+          S a b N _ -> next (bs :!: S a b x N :!: i+1)+          S a b c N -> next (bs :!: S a b c x :!: i+1)+          S (J a) b c d -> decodeError "streamUtf8" "UTF-8" onErr (Just a)+                           (bs :!: S b c d N :!: i+1)+          where x = J (B.unsafeIndex ps i)       consume (Empty :!: S N _ _ _ :!: _) = Done-      consume _ = encodingError "streamUtf8" "UTF-8"+      consume st = decodeError "streamUtf8" "UTF-8" onErr Nothing st {-# INLINE [0] streamUtf8 #-}  -- | /O(n)/ Convert a 'Stream' 'Word8' to a lazy 'ByteString'.@@ -123,7 +139,11 @@ unstream :: Stream Word8 -> ByteString unstream = unstreamChunks defaultChunkSize -encodingError :: String -> String -> a-encodingError func encoding =-    error $ "Data.Text.Lazy.Encoding.Fusion." ++ func ++ ": Bad " ++-            encoding ++ " stream"+decodeError :: forall s. String -> String -> OnDecodeError -> Maybe Word8+            -> s -> Step s Char+decodeError func kind onErr mb i =+    case onErr desc mb of+      Nothing -> Skip i+      Just c  -> Yield c i+    where desc = "Data.Text.Lazy.Encoding.Fusion." ++ func ++ ": Invalid " +++                 kind ++ " stream"
Data/Text/UnsafeChar.hs view
@@ -24,8 +24,9 @@  import Control.Exception (assert) import Control.Monad.ST (ST)-import Data.Bits ((.&.), shiftR)+import Data.Bits ((.&.)) import Data.Char (ord)+import Data.Text.UnsafeShift (shiftR) import GHC.Exts (Char(..), chr#, word2Int#) import GHC.Word (Word8(..), Word16(..), Word32(..)) import qualified Data.Text.Array as A
+ Data/Text/UnsafeShift.hs view
@@ -0,0 +1,67 @@+{-# LANGUAGE MagicHash #-}++-- |+-- Module      : Data.Text.UnsafeShift+-- Copyright   : (c) Bryan O'Sullivan 2009+--+-- License     : BSD-style+-- Maintainer  : bos@serpentine.com, rtharper@aftereternity.co.uk,+--               duncan@haskell.org+-- Stability   : experimental+-- Portability : GHC+--+-- Fast, unchecked bit shifting functions.++module Data.Text.UnsafeShift+    (+      UnsafeShift(..)+    ) where++import qualified Data.Bits as Bits+import GHC.Base+import GHC.Word++-- | This is a workaround for poor optimisation in GHC 6.8.2.  It+-- fails to notice constant-width shifts, and adds a test and branch+-- to every shift.  This imposes about a 10% performance hit.+--+-- These functions are undefined when the amount being shifted by is+-- greater than the size in bits of a machine Int#.+class UnsafeShift a where+    shiftL :: a -> Int -> a+    shiftR :: a -> Int -> a++instance UnsafeShift Word16 where+    {-# INLINE shiftL #-}+    shiftL (W16# x#) (I# i#) = W16# (x# `uncheckedShiftL#` i#)++    {-# INLINE shiftR #-}+    shiftR (W16# x#) (I# i#) = W16# (x# `uncheckedShiftRL#` i#)++instance UnsafeShift Word32 where+    {-# INLINE shiftL #-}+    shiftL (W32# x#) (I# i#) = W32# (x# `uncheckedShiftL#` i#)++    {-# INLINE shiftR #-}+    shiftR (W32# x#) (I# i#) = W32# (x# `uncheckedShiftRL#` i#)++instance UnsafeShift Word64 where+    {-# INLINE shiftL #-}+    shiftL (W64# x#) (I# i#) = W64# (x# `uncheckedShiftL64#` i#)++    {-# INLINE shiftR #-}+    shiftR (W64# x#) (I# i#) = W64# (x# `uncheckedShiftRL64#` i#)++instance UnsafeShift Int where+    {-# INLINE shiftL #-}+    shiftL (I# x#) (I# i#) = I# (x# `iShiftL#` i#)++    {-# INLINE shiftR #-}+    shiftR (I# x#) (I# i#) = I# (x# `iShiftRA#` i#)++instance UnsafeShift Integer where+    {-# INLINE shiftL #-}+    shiftL = Bits.shiftL++    {-# INLINE shiftR #-}+    shiftR = Bits.shiftR
text.cabal view
@@ -1,5 +1,5 @@ name:           text-version:        0.2+version:        0.3 synopsis:       An efficient packed Unicode text type description:    An efficient packed Unicode text type. license:        BSD3@@ -18,6 +18,7 @@   exposed-modules:     Data.Text     Data.Text.Encoding+    Data.Text.Encoding.Error     Data.Text.Encoding.Fusion     Data.Text.Foreign     Data.Text.Fusion@@ -31,9 +32,11 @@     Data.Text.Internal     Data.Text.Encoding.Fusion.Common     Data.Text.Fusion.Internal+    Data.Text.Fusion.CaseMapping     Data.Text.Lazy.Internal     Data.Text.Unsafe     Data.Text.UnsafeChar+    Data.Text.UnsafeShift     Data.Text.Encoding.Utf8     Data.Text.Encoding.Utf16     Data.Text.Encoding.Utf32