diff --git a/Data/Text.hs b/Data/Text.hs --- a/Data/Text.hs +++ b/Data/Text.hs @@ -97,7 +97,6 @@ -- ** Generation and unfolding , replicate - , replicateChar , unfoldr , unfoldrN @@ -114,8 +113,9 @@ , stripStart , stripEnd , splitAt - , span + , spanBy , break + , breakBy , group , groupBy , inits @@ -124,9 +124,7 @@ -- ** Breaking into many substrings -- $split , split - , splitTimes - , splitTimesEnd - , splitWith + , splitBy , chunksOf -- ** Breaking into lines and words @@ -142,10 +140,10 @@ , isInfixOf -- * Searching - , elem , filter , find - , partition + , findBy + , partitionBy -- , findSubstring @@ -153,9 +151,6 @@ -- $index , index , findIndex - , findIndices - , elemIndex - , elemIndices , count -- * Zipping and unzipping @@ -179,16 +174,15 @@ import Data.Monoid (Monoid(..)) import Data.Word (Word16) import Data.String (IsString(..)) - import qualified Data.Text.Fusion as S import qualified Data.Text.Fusion.Common as S import Data.Text.Fusion (stream, reverseStream, unstream) - import Data.Text.Internal (Text(..), empty, text, textP) import qualified Prelude as P import Data.Text.Unsafe (iter, iter_, reverseIter, unsafeHead, unsafeTail) import Data.Text.UnsafeChar (unsafeChr) import qualified Data.Text.Encoding.Utf16 as U16 +import Data.Text.Search (indices) -- $fusion -- @@ -368,6 +362,12 @@ S.null (stream t) = null t #-} +-- | /O(1)/ Tests whether a 'Text' contains exactly one character. +-- Subject to fusion. +isSingleton :: Text -> Bool +isSingleton = S.isSingleton . stream +{-# INLINE isSingleton #-} + -- | /O(n)/ Returns the number of characters in a 'Text'. -- Subject to fusion. length :: Text -> Int @@ -400,7 +400,7 @@ reverse t = S.reverse (stream t) {-# INLINE reverse #-} --- | /O(m)*O(n)/ Replace every occurrence of one substring with another. +-- | /O(m*n)/ Replace every occurrence of one substring with another. replace :: Text -- ^ Text to search for -> Text -- ^ Replacement text -> Text -- ^ Input text @@ -658,7 +658,9 @@ -- | /O(n*m)/ 'replicate' @n@ @t@ is a 'Text' consisting of the input -- @t@ repeated @n@ times. Subject to fusion. replicate :: Int -> Text -> Text -replicate n t = unstream (S.replicateI (fromIntegral n) (S.stream t)) +replicate n t + | isSingleton t = replicateChar n (unsafeHead t) + | otherwise = unstream (S.replicateI (fromIntegral n) (S.stream t)) {-# INLINE [1] replicate #-} {-# RULES @@ -670,6 +672,7 @@ -- value of every element. Subject to fusion. replicateChar :: Int -> Char -> Text replicateChar n c = unstream (S.replicateCharI n c) +{-# INLINE replicateChar #-} -- | /O(n)/, where @n@ is the length of the result. The 'unfoldr' -- function is analogous to the List 'L.unfoldr'. 'unfoldr' builds a @@ -838,23 +841,23 @@ where d = iter_ t i {-# INLINE splitAt #-} --- | /O(n)/ 'span', applied to a predicate @p@ and text @t@, returns a --- pair whose first element is the longest prefix (possibly empty) of --- @t@ of elements that satisfy @p@, and whose second is the remainder --- of the list. -span :: (Char -> Bool) -> Text -> (Text, Text) -span p t@(Text arr off len) = (textP arr off k, textP arr (off+k) (len-k)) +-- | /O(n)/ 'spanBy', applied to a predicate @p@ and text @t@, returns +-- a pair whose first element is the longest prefix (possibly empty) +-- of @t@ of elements that satisfy @p@, and whose second is the +-- remainder of the list. +spanBy :: (Char -> Bool) -> Text -> (Text, Text) +spanBy p t@(Text arr off len) = (textP arr off k, textP arr (off+k) (len-k)) where k = loop 0 loop !i | i >= len || not (p c) = i | otherwise = loop (i+d) where (c,d) = iter t i -{-# INLINE span #-} +{-# INLINE spanBy #-} --- | /O(n)/ 'break' is like 'span', but the prefix returned is over --- elements that fail the predicate @p@. -break :: (Char -> Bool) -> Text -> (Text, Text) -break p = span (not . p) -{-# INLINE break #-} +-- | /O(n)/ 'breakBy' is like 'spanBy', but the prefix returned is +-- over elements that fail the predicate @p@. +breakBy :: (Char -> Bool) -> Text -> (Text, Text) +breakBy p = spanBy (not . p) +{-# INLINE breakBy #-} -- | /O(n)/ Group characters in a string according to a predicate. groupBy :: (Char -> Char -> Bool) -> Text -> [Text] @@ -898,7 +901,7 @@ -- copies to create substrings; they just construct new 'Text's that -- are slices of the original. --- | /O(m)*O(n)/ Break a 'Text' into pieces separated by the first +-- | /O(m+n)/ Break a 'Text' into pieces separated by the first -- 'Text' argument, consuming the delimiter. An empty delimiter is -- invalid, and will cause an error to be raised. -- @@ -911,94 +914,39 @@ -- and -- -- > intercalate s . split s == id --- > split (singleton c) == splitWith (==c) -split :: Text -- ^ Text to split on - -> Text -- ^ Input text - -> [Text] -split pat src0 - | l == 0 = emptyError "split" - | l == 1 = splitWith (== (unsafeHead pat)) src0 - | otherwise = go src0 +-- > split (singleton c) == splitBy (==c) +-- +-- In (unlikely) bad cases, this function's time complexity degrades +-- towards /O(n*m)/. +split :: Text -> Text -> [Text] +split pat@(Text _ _ l) src@(Text arr off len) + | l <= 0 = emptyError "split" + | isSingleton pat = splitBy (== unsafeHead pat) src + | otherwise = go 0 (indices pat src) where - l = length pat - go src = search 0 src - where - search !n !s - | null s = [src] -- not found - | pat `isPrefixOf` s = take n src : go (drop l s) - | otherwise = search (n+1) (unsafeTail s) + go !s (x:xs) = textP arr (s+off) (x-s) : go (x+l) xs + go s _ = [textP arr (s+off) (len-s)] {-# INLINE [1] split #-} {-# RULES -"TEXT split/singleton -> splitWith/==" [~1] forall c t. - split (singleton c) t = splitWith (==c) t +"TEXT split/singleton -> splitBy/==" [~1] forall c t. + split (singleton c) t = splitBy (==c) t #-} --- | /O(m)*O(n)/ Break a 'Text' into pieces at most @k@ times, --- treating the first 'Text' argument as the delimiter to break on, --- and consuming the delimiter. The last element of the list contains --- the remaining text after the number of times to split has been --- reached. A value of zero or less for @k@ causes no splitting to --- occur. An empty delimiter is invalid, and will cause an error to be --- raised. --- --- Examples: --- --- > splitTimes 0 "//" "a//b//c" == ["a//b//c"] --- > splitTimes 2 ":" "a:b:c:d:e" == ["a","b","c:d:e"] --- > splitTimes 100 "???" "a????b" == ["a","?b"] --- --- and --- --- > intercalate s . splitTimes k s == id -splitTimes :: Int -- ^ Maximum number of times to split - -> Text -- ^ Text to split on - -> Text -- ^ Input text - -> [Text] -splitTimes k pat src0 - | k <= 0 = [src0] - | l == 0 = emptyError "splitTimes" - | otherwise = go k src0 - where - l = length pat - go !i src = search 0 src - where - search !n !s - | i == 0 || null s = [src] -- not found or limit reached - | pat `isPrefixOf` s = take n src : go (i-1) (drop l s) - | otherwise = search (n+1) (unsafeTail s) -{-# INLINE splitTimes #-} - --- | /O(m)*O(n)/ Break a 'Text' into pieces at most @k@ times, like --- 'splitTimes', but start from the end of the input and work towards --- the start. --- --- Examples: --- --- > splitTimes 2 "::" "a::b::c::d::e" == ["a","b","c::d::e"] --- > splitTimesEnd 2 "::" "a::b::c::d::e" == ["a::b::c","d","e"] -splitTimesEnd :: Int -- ^ Maximum number of times to split - -> Text -- ^ Text to split on - -> Text -- ^ Input text - -> [Text] -splitTimesEnd k pat src = - L.reverse . L.map reverse $ splitTimes k (reverse pat) (reverse src) -{-# INLINE splitTimesEnd #-} - -- | /O(n)/ Splits a 'Text' into components delimited by separators, -- where the predicate returns True for a separator element. The -- resulting components do not contain the separators. Two adjacent -- separators result in an empty component in the output. eg. -- --- > splitWith (=='a') "aabbaca" == ["","","bb","c",""] --- > splitWith (=='a') "" == [""] -splitWith :: (Char -> Bool) -> Text -> [Text] -splitWith _ t@(Text _off _arr 0) = [t] -splitWith p t = loop t +-- > splitBy (=='a') "aabbaca" == ["","","bb","c",""] +-- > splitBy (=='a') "" == [""] +splitBy :: (Char -> Bool) -> Text -> [Text] +splitBy _ t@(Text _off _arr 0) = [t] +splitBy p t = loop t where loop s | null s' = [l] | otherwise = l : loop (unsafeTail s') - where (l, s') = break p s -{-# INLINE splitWith #-} + where (l, s') = breakBy p s +{-# INLINE splitBy #-} -- | /O(n)/ Splits a 'Text' into components of length @k@. The last -- element may be shorter than the other chunks, depending on the @@ -1018,31 +966,23 @@ -- * Searching ------------------------------------------------------------------------------- --- ** Searching by equality - --- | /O(n)/ 'elem' is the 'Text' membership predicate. -elem :: Char -> Text -> Bool -elem c t = S.elem c (stream t) -{-# INLINE elem #-} - -------------------------------------------------------------------------------- -- ** Searching with a predicate --- | /O(n)/ The 'find' function takes a predicate and a 'Text', --- and returns the first element in matching the predicate, or 'Nothing' +-- | /O(n)/ The 'findBy' function takes a predicate and a 'Text', and +-- returns the first element in matching the predicate, or 'Nothing' -- if there is no such element. -find :: (Char -> Bool) -> Text -> Maybe Char -find p t = S.find p (stream t) -{-# INLINE find #-} +findBy :: (Char -> Bool) -> Text -> Maybe Char +findBy p t = S.findBy p (stream t) +{-# INLINE findBy #-} --- | /O(n)/ The 'partition' function takes a predicate and a 'Text', +-- | /O(n)/ The 'partitionBy' function takes a predicate and a 'Text', -- and returns the pair of 'Text's with elements which do and do not -- satisfy the predicate, respectively; i.e. -- --- > partition p t == (filter p t, filter (not . p) t) -partition :: (Char -> Bool) -> Text -> (Text, Text) -partition p t = (filter p t, filter (not . p) t) -{-# INLINE partition #-} +-- > partitionBy p t == (filter p t, filter (not . p) t) +partitionBy :: (Char -> Bool) -> Text -> (Text, Text) +partitionBy p t = (filter p t, filter (not . p) t) +{-# INLINE partitionBy #-} -- | /O(n)/ 'filter', applied to a predicate and a 'Text', -- returns a 'Text' containing those characters that satisfy the @@ -1051,7 +991,67 @@ filter p t = unstream (S.filter p (stream t)) {-# INLINE filter #-} +-- | /O(n+m)/ Find the first instance of @needle@ (which must be +-- non-'null') in @haystack@. The first element of the returned tuple +-- is the prefix of @haystack@ before @needle@ is matched. The second +-- is the remainder of @haystack@, starting with the match. +-- +-- Examples: +-- +-- > break "::" "a::b::c" ==> ("a", "::b::c") +-- > break "/" "foobar" ==> ("foobar", "") +-- +-- Laws: +-- +-- > append prefix match == haystack +-- > where (prefix, match) = break needle haystack +-- +-- If you need to break a string by a substring repeatedly (e.g. you +-- want to break on every instance of a substring), use 'find' +-- instead, as it has lower startup overhead. +-- +-- In (unlikely) bad cases, this function's time complexity degrades +-- towards /O(n*m)/. +break :: Text -> Text -> (Text, Text) +break pat src@(Text arr off len) + | null pat = emptyError "break" + | otherwise = case indices pat src of + [] -> (src, empty) + (x:_) -> (textP arr off x, textP arr (off+x) (len-x)) +{-# INLINE break #-} +-- | /O(n+m)/ Find all non-overlapping instances of @needle@ in +-- @haystack@. The first element of the returned pair is the prefix +-- of @haystack@ prior to any matches of @needle@. The second is a +-- list of pairs. +-- +-- The first element of each pair in the list is a span from the +-- beginning of a match to the beginning of the next match, while the +-- second is a span from the beginning of the match to the end of the +-- input. +-- +-- Examples: +-- +-- > find "::" "" +-- > ==> ("", []) +-- > find "/" "a/b/c/d" +-- > ==> ("a", [("/b","/b/c/d"), ("/c","/c/d"), ("/d","/d")]) +-- +-- In (unlikely) bad cases, this function's time complexity degrades +-- towards /O(n*m)/. +find :: Text -> Text -> (Text, [(Text, Text)]) +find pat src@(Text arr off len) + | null pat = emptyError "find" + | otherwise = case indices pat src of + [] -> (src, []) + (x:xs) -> (chunk 0 x, go x xs) + where + go !s (x:xs) = (chunk s (x-s), chunk s (len-s)) : go x xs + go s _ = let c = chunk s (len-s) + in [(c,c)] + chunk !n !l = textP arr (n+off) l +{-# INLINE find #-} + ------------------------------------------------------------------------------- -- ** Indexing 'Text's @@ -1069,8 +1069,7 @@ -- For example, suppose you have a string that you want to split on -- the substring @\"::\"@, such as @\"foo::bar::quux\"@. Instead of -- searching for the index of @\"::\"@ and taking the substrings --- before and after that index, you would instead use @splitTimes 1 --- "::"@. +-- before and after that index, you would instead use @find "::"@. -- | /O(n)/ 'Text' index (subscript) operator, starting from 0. index :: Text -> Int -> Char @@ -1084,42 +1083,17 @@ findIndex p t = S.findIndex p (stream t) {-# INLINE findIndex #-} --- | The 'findIndices' function extends 'findIndex', by returning the --- indices of all elements satisfying the predicate, in ascending --- order. Subject to fusion. -findIndices :: (Char -> Bool) -> Text -> [Int] -findIndices p t = S.findIndices p (stream t) -{-# INLINE findIndices #-} - --- | /O(n)/ The 'elemIndex' function returns the index of the first --- element in the given 'Text' which is equal to the query element, or --- 'Nothing' if there is no such element. Subject to fusion. -elemIndex :: Char -> Text -> Maybe Int -elemIndex c t = S.elemIndex c (stream t) -{-# INLINE elemIndex #-} - --- | /O(n)/ The 'elemIndices' function returns the index of every --- element in the given 'Text' which is equal to the query --- element. Subject to fusion. -elemIndices :: Char -> Text -> [Int] -elemIndices c t = S.elemIndices c (stream t) -{-# INLINE elemIndices #-} - --- | /O(n*m)/ The 'count' function returns the number of times the +-- | /O(n+m)/ The 'count' function returns the number of times the -- query string appears in the given 'Text'. An empty query string is -- invalid, and will cause an error to be raised. +-- +-- In (unlikely) bad cases, this function's time complexity degrades +-- towards /O(n*m)/. count :: Text -> Text -> Int -count pat src0 - | l == 0 = emptyError "count" - | l == 1 = countChar (unsafeHead pat) src0 - | otherwise = go 0 src0 - where - l = length pat - go !n src = search src - where - search s | null s = n - | pat `isPrefixOf` s = go (n+1) (drop l s) - | otherwise = search (unsafeTail s) +count pat src + | null pat = emptyError "count" + | isSingleton pat = countChar (unsafeHead pat) src + | otherwise = L.length (indices pat src) {-# INLINE [1] count #-} {-# RULES @@ -1174,7 +1148,7 @@ | otherwise = h : if null t then [] else lines (unsafeTail t) - where (h,t) = span (/= '\n') ps + where (h,t) = spanBy (/= '\n') ps {-# INLINE lines #-} -- | /O(n)/ Portably breaks a 'Text' up into a list of 'Text's at line @@ -1235,13 +1209,23 @@ | otherwise = Text barr (boff+d) alen {-# INLINE isSuffixOf #-} --- | /O(n)/ The 'isInfixOf' function takes two 'Text's and returns +-- | /O(n+m)/ The 'isInfixOf' function takes two 'Text's and returns -- 'True' iff the first is contained, wholly and intact, anywhere -- within the second. +-- +-- In (unlikely) bad cases, this function's time complexity degrades +-- towards /O(n*m)/. isInfixOf :: Text -> Text -> Bool -isInfixOf needle haystack = L.any (isPrefixOf needle) (tails haystack) -{-# INLINE isInfixOf #-} --- TODO: a better implementation +isInfixOf needle haystack + | null needle = True + | isSingleton needle = S.elem (unsafeHead needle) . S.stream $ haystack + | otherwise = not . L.null . indices needle $ haystack +{-# INLINE [1] isInfixOf #-} + +{-# RULES +"TEXT isInfixOf/singleton -> S.elem/S.stream" [~1] forall n h. + isInfixOf (singleton n) h = S.elem n (S.stream h) + #-} emptyError :: String -> a emptyError fun = P.error ("Data.Text." ++ fun ++ ": empty input") diff --git a/Data/Text/Encoding/Fusion.hs b/Data/Text/Encoding/Fusion.hs --- a/Data/Text/Encoding/Fusion.hs +++ b/Data/Text/Encoding/Fusion.hs @@ -35,10 +35,11 @@ import Data.ByteString as B import Data.ByteString.Internal (ByteString(..), mallocByteString, memcpy) import Data.Text.Fusion (Step(..), Stream(..)) +import Data.Text.Fusion.Size import Data.Text.Encoding.Error import Data.Text.Encoding.Fusion.Common import Data.Text.UnsafeChar (unsafeChr, unsafeChr8, unsafeChr32) -import Data.Text.UnsafeShift (shiftL) +import Data.Text.UnsafeShift (shiftL, shiftR) import Data.Word (Word8, Word16, Word32) import Foreign.ForeignPtr (withForeignPtr, ForeignPtr) import Foreign.Storable (pokeByteOff) @@ -50,7 +51,7 @@ import qualified Data.Text.Encoding.Utf32 as U32 streamASCII :: ByteString -> Stream Char -streamASCII bs = Stream next 0 l +streamASCII bs = Stream next 0 (maxSize l) where l = B.length bs {-# INLINE next #-} @@ -64,7 +65,7 @@ -- | /O(n)/ Convert a 'ByteString' into a 'Stream Char', using UTF-8 -- encoding. streamUtf8 :: OnDecodeError -> ByteString -> Stream Char -streamUtf8 onErr bs = Stream next 0 l +streamUtf8 onErr bs = Stream next 0 (maxSize l) where l = B.length bs {-# INLINE next #-} @@ -87,7 +88,7 @@ -- | /O(n)/ Convert a 'ByteString' into a 'Stream Char', using little -- endian UTF-16 encoding. streamUtf16LE :: OnDecodeError -> ByteString -> Stream Char -streamUtf16LE onErr bs = Stream next 0 l +streamUtf16LE onErr bs = Stream next 0 (maxSize (l `shiftR` 1)) where l = B.length bs {-# INLINE next #-} @@ -105,7 +106,7 @@ -- | /O(n)/ Convert a 'ByteString' into a 'Stream Char', using big -- endian UTF-16 encoding. streamUtf16BE :: OnDecodeError -> ByteString -> Stream Char -streamUtf16BE onErr bs = Stream next 0 l +streamUtf16BE onErr bs = Stream next 0 (maxSize (l `shiftR` 1)) where l = B.length bs {-# INLINE next #-} @@ -123,7 +124,7 @@ -- | /O(n)/ Convert a 'ByteString' into a 'Stream Char', using big -- endian UTF-32 encoding. streamUtf32BE :: OnDecodeError -> ByteString -> Stream Char -streamUtf32BE onErr bs = Stream next 0 l +streamUtf32BE onErr bs = Stream next 0 (maxSize (l `shiftR` 2)) where l = B.length bs {-# INLINE next #-} @@ -143,7 +144,7 @@ -- | /O(n)/ Convert a 'ByteString' into a 'Stream Char', using little -- endian UTF-32 encoding. streamUtf32LE :: OnDecodeError -> ByteString -> Stream Char -streamUtf32LE onErr bs = Stream next 0 l +streamUtf32LE onErr bs = Stream next 0 (maxSize (l `shiftR` 2)) where l = B.length bs {-# INLINE next #-} @@ -163,7 +164,8 @@ -- | /O(n)/ Convert a 'Stream' 'Word8' to a 'ByteString'. unstream :: Stream Word8 -> ByteString unstream (Stream next s0 len) = unsafePerformIO $ do - mallocByteString len >>= loop len 0 s0 + let mlen = upperBound 4 len + mallocByteString mlen >>= loop mlen 0 s0 where loop !n !off !s fp = case next s of Done -> trimUp fp n off diff --git a/Data/Text/Fusion.hs b/Data/Text/Fusion.hs --- a/Data/Text/Fusion.hs +++ b/Data/Text/Fusion.hs @@ -40,24 +40,21 @@ -- * Indexing , index , findIndex - , findIndices - , findIndexOrEnd - , elemIndex - , elemIndices , countChar ) where -import Prelude (Bool(..), Char, Eq(..), Maybe(..), Monad(..), Int, +import Prelude (Bool(..), Char, Maybe(..), Monad(..), Int, Num(..), Ord(..), ($), (&&), fromIntegral, otherwise) import Data.Bits ((.&.)) import Data.Char (ord) import Data.Text.Internal (Text(..)) import Data.Text.UnsafeChar (unsafeChr, unsafeWrite) -import Data.Text.UnsafeShift (shiftR) +import Data.Text.UnsafeShift (shiftL, shiftR) import qualified Data.Text.Array as A import qualified Data.Text.Fusion.Common as S import Data.Text.Fusion.Internal +import Data.Text.Fusion.Size import qualified Data.Text.Internal as I import qualified Data.Text.Encoding.Utf16 as U16 import qualified Prelude as P @@ -66,7 +63,7 @@ -- | /O(n)/ Convert a 'Text' into a 'Stream Char'. stream :: Text -> Stream Char -stream (Text arr off len) = Stream next off len +stream (Text arr off len) = Stream next off (maxSize len) where end = off+len {-# INLINE next #-} @@ -82,7 +79,7 @@ -- | /O(n)/ Convert a 'Text' into a 'Stream Char', but iterate -- backwards. reverseStream :: Text -> Stream Char -reverseStream (Text arr off len) = Stream next (off+len-1) len +reverseStream (Text arr off len) = Stream next (off+len-1) (maxSize len) where {-# INLINE next #-} next !i @@ -96,17 +93,17 @@ -- | /O(n)/ Convert a 'Stream Char' into a 'Text'. unstream :: Stream Char -> Text -unstream (Stream next0 s0 len) - | len == 0 = I.empty - | otherwise = I.textP (P.fst a) 0 (P.snd a) +unstream (Stream next0 s0 len) = I.textP (P.fst a) 0 (P.snd a) where - a = A.run2 (A.unsafeNew len >>= (\arr -> loop arr len s0 0)) + mlen = upperBound 4 len + a = A.run2 (A.unsafeNew mlen >>= (\arr -> loop arr mlen s0 0)) loop arr !top !s !i | i + 1 >= top = case next0 s of Done -> return (arr, i) _ -> do - arr' <- A.unsafeNew (top*2) - A.copy arr arr' >> loop arr' (top*2) s i + let top' = (top `shiftL` 1) + 1 + arr' <- A.unsafeNew top' + A.copy arr arr' >> loop arr' top' s i | otherwise = case next0 s of Done -> return (arr, i) Skip s' -> loop arr top s' i @@ -125,10 +122,10 @@ -- | /O(n)/ Reverse the characters of a string. reverse :: Stream Char -> Text reverse (Stream next s len0) - | len0 == 0 = I.empty - | otherwise = I.textP arr off' len' + | isEmpty len0 = I.empty + | otherwise = I.textP arr off' len' where - len0' = max len0 4 + len0' = upperBound 4 (larger len0 4) (arr, (off', len')) = A.run2 (A.unsafeNew len0' >>= loop s (len0'-1) len0') loop !s0 !i !len marr = case next s0 of @@ -160,14 +157,14 @@ -- | /O(n)/ Perform the equivalent of 'scanr' over a list, only with -- the input and result reversed. reverseScanr :: (Char -> Char -> Char) -> Char -> Stream Char -> Stream Char -reverseScanr f z0 (Stream next0 s0 len) = Stream next (S1 :!: z0 :!: s0) (len+1) -- HINT maybe too low +reverseScanr f z0 (Stream next0 s0 len) = Stream next (S1 :*: z0 :*: s0) (len+1) -- HINT maybe too low where {-# INLINE next #-} - next (S1 :!: z :!: s) = Yield z (S2 :!: z :!: s) - next (S2 :!: z :!: s) = case next0 s of + next (S1 :*: z :*: s) = Yield z (S2 :*: z :*: s) + next (S2 :*: z :*: s) = case next0 s of Yield x s' -> let !x' = f x z - in Yield x' (S2 :!: x' :!: s') - Skip s' -> Skip (S2 :!: z :!: s') + in Yield x' (S2 :*: x' :*: s') + Skip s' -> Skip (S2 :*: z :*: s') Done -> Done {-# INLINE reverseScanr #-} @@ -193,39 +190,6 @@ findIndex :: (Char -> Bool) -> Stream Char -> Maybe Int findIndex = S.findIndexI {-# INLINE [0] findIndex #-} - --- | The 'findIndices' function takes a predicate and a stream and --- returns all indices of the elements in the stream --- satisfying the predicate. -findIndices :: (Char -> Bool) -> Stream Char -> [Int] -findIndices = S.findIndicesI -{-# INLINE [0] findIndices #-} - --- | The 'findIndexOrEnd' function takes a predicate and a stream and --- returns the index of the first element in the stream --- satisfying the predicate. -findIndexOrEnd :: (Char -> Bool) -> Stream Char -> Int -findIndexOrEnd p (Stream next s0 _len) = loop_findIndex 0 s0 - where - loop_findIndex !i !s = case next s of - Done -> i - Skip s' -> loop_findIndex i s' -- hmm. not caught by QC - Yield x s' | p x -> i - | otherwise -> loop_findIndex (i+1) s' -{-# INLINE [0] findIndexOrEnd #-} - --- | /O(n)/ The 'elemIndex' function returns the index of the first --- element in the given stream which is equal to the query --- element, or 'Nothing' if there is no such element. -elemIndex :: Char -> Stream Char -> Maybe Int -elemIndex = S.elemIndexI -{-# INLINE [0] elemIndex #-} - --- | /O(n)/ The 'elemIndices' function returns the index of every --- element in the given stream which is equal to the query element. -elemIndices :: Char -> Stream Char -> [Int] -elemIndices = S.elemIndicesI -{-# INLINE [0] elemIndices #-} -- | /O(n)/ The 'count' function returns the number of times the query -- element appears in the given stream. diff --git a/Data/Text/Fusion/CaseMapping.hs b/Data/Text/Fusion/CaseMapping.hs --- a/Data/Text/Fusion/CaseMapping.hs +++ b/Data/Text/Fusion/CaseMapping.hs @@ -8,449 +8,449 @@ upperMapping :: forall s. Char -> s -> Step (PairS (PairS s Char) Char) Char {-# INLINE upperMapping #-} -- LATIN SMALL LETTER SHARP S -upperMapping '\x00df' s = Yield '\x0053' (s :!: '\x0053' :!: '\x0000') +upperMapping '\x00df' s = Yield '\x0053' (s :*: '\x0053' :*: '\x0000') -- LATIN SMALL LIGATURE FF -upperMapping '\xfb00' s = Yield '\x0046' (s :!: '\x0046' :!: '\x0000') +upperMapping '\xfb00' s = Yield '\x0046' (s :*: '\x0046' :*: '\x0000') -- LATIN SMALL LIGATURE FI -upperMapping '\xfb01' s = Yield '\x0046' (s :!: '\x0049' :!: '\x0000') +upperMapping '\xfb01' s = Yield '\x0046' (s :*: '\x0049' :*: '\x0000') -- LATIN SMALL LIGATURE FL -upperMapping '\xfb02' s = Yield '\x0046' (s :!: '\x004c' :!: '\x0000') +upperMapping '\xfb02' s = Yield '\x0046' (s :*: '\x004c' :*: '\x0000') -- LATIN SMALL LIGATURE FFI -upperMapping '\xfb03' s = Yield '\x0046' (s :!: '\x0046' :!: '\x0049') +upperMapping '\xfb03' s = Yield '\x0046' (s :*: '\x0046' :*: '\x0049') -- LATIN SMALL LIGATURE FFL -upperMapping '\xfb04' s = Yield '\x0046' (s :!: '\x0046' :!: '\x004c') +upperMapping '\xfb04' s = Yield '\x0046' (s :*: '\x0046' :*: '\x004c') -- LATIN SMALL LIGATURE LONG S T -upperMapping '\xfb05' s = Yield '\x0053' (s :!: '\x0054' :!: '\x0000') +upperMapping '\xfb05' s = Yield '\x0053' (s :*: '\x0054' :*: '\x0000') -- LATIN SMALL LIGATURE ST -upperMapping '\xfb06' s = Yield '\x0053' (s :!: '\x0054' :!: '\x0000') +upperMapping '\xfb06' s = Yield '\x0053' (s :*: '\x0054' :*: '\x0000') -- ARMENIAN SMALL LIGATURE ECH YIWN -upperMapping '\x0587' s = Yield '\x0535' (s :!: '\x0552' :!: '\x0000') +upperMapping '\x0587' s = Yield '\x0535' (s :*: '\x0552' :*: '\x0000') -- ARMENIAN SMALL LIGATURE MEN NOW -upperMapping '\xfb13' s = Yield '\x0544' (s :!: '\x0546' :!: '\x0000') +upperMapping '\xfb13' s = Yield '\x0544' (s :*: '\x0546' :*: '\x0000') -- ARMENIAN SMALL LIGATURE MEN ECH -upperMapping '\xfb14' s = Yield '\x0544' (s :!: '\x0535' :!: '\x0000') +upperMapping '\xfb14' s = Yield '\x0544' (s :*: '\x0535' :*: '\x0000') -- ARMENIAN SMALL LIGATURE MEN INI -upperMapping '\xfb15' s = Yield '\x0544' (s :!: '\x053b' :!: '\x0000') +upperMapping '\xfb15' s = Yield '\x0544' (s :*: '\x053b' :*: '\x0000') -- ARMENIAN SMALL LIGATURE VEW NOW -upperMapping '\xfb16' s = Yield '\x054e' (s :!: '\x0546' :!: '\x0000') +upperMapping '\xfb16' s = Yield '\x054e' (s :*: '\x0546' :*: '\x0000') -- ARMENIAN SMALL LIGATURE MEN XEH -upperMapping '\xfb17' s = Yield '\x0544' (s :!: '\x053d' :!: '\x0000') +upperMapping '\xfb17' s = Yield '\x0544' (s :*: '\x053d' :*: '\x0000') -- LATIN SMALL LETTER N PRECEDED BY APOSTROPHE -upperMapping '\x0149' s = Yield '\x02bc' (s :!: '\x004e' :!: '\x0000') +upperMapping '\x0149' s = Yield '\x02bc' (s :*: '\x004e' :*: '\x0000') -- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND TONOS -upperMapping '\x0390' s = Yield '\x0399' (s :!: '\x0308' :!: '\x0301') +upperMapping '\x0390' s = Yield '\x0399' (s :*: '\x0308' :*: '\x0301') -- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND TONOS -upperMapping '\x03b0' s = Yield '\x03a5' (s :!: '\x0308' :!: '\x0301') +upperMapping '\x03b0' s = Yield '\x03a5' (s :*: '\x0308' :*: '\x0301') -- LATIN SMALL LETTER J WITH CARON -upperMapping '\x01f0' s = Yield '\x004a' (s :!: '\x030c' :!: '\x0000') +upperMapping '\x01f0' s = Yield '\x004a' (s :*: '\x030c' :*: '\x0000') -- LATIN SMALL LETTER H WITH LINE BELOW -upperMapping '\x1e96' s = Yield '\x0048' (s :!: '\x0331' :!: '\x0000') +upperMapping '\x1e96' s = Yield '\x0048' (s :*: '\x0331' :*: '\x0000') -- LATIN SMALL LETTER T WITH DIAERESIS -upperMapping '\x1e97' s = Yield '\x0054' (s :!: '\x0308' :!: '\x0000') +upperMapping '\x1e97' s = Yield '\x0054' (s :*: '\x0308' :*: '\x0000') -- LATIN SMALL LETTER W WITH RING ABOVE -upperMapping '\x1e98' s = Yield '\x0057' (s :!: '\x030a' :!: '\x0000') +upperMapping '\x1e98' s = Yield '\x0057' (s :*: '\x030a' :*: '\x0000') -- LATIN SMALL LETTER Y WITH RING ABOVE -upperMapping '\x1e99' s = Yield '\x0059' (s :!: '\x030a' :!: '\x0000') +upperMapping '\x1e99' s = Yield '\x0059' (s :*: '\x030a' :*: '\x0000') -- LATIN SMALL LETTER A WITH RIGHT HALF RING -upperMapping '\x1e9a' s = Yield '\x0041' (s :!: '\x02be' :!: '\x0000') +upperMapping '\x1e9a' s = Yield '\x0041' (s :*: '\x02be' :*: '\x0000') -- GREEK SMALL LETTER UPSILON WITH PSILI -upperMapping '\x1f50' s = Yield '\x03a5' (s :!: '\x0313' :!: '\x0000') +upperMapping '\x1f50' s = Yield '\x03a5' (s :*: '\x0313' :*: '\x0000') -- GREEK SMALL LETTER UPSILON WITH PSILI AND VARIA -upperMapping '\x1f52' s = Yield '\x03a5' (s :!: '\x0313' :!: '\x0300') +upperMapping '\x1f52' s = Yield '\x03a5' (s :*: '\x0313' :*: '\x0300') -- GREEK SMALL LETTER UPSILON WITH PSILI AND OXIA -upperMapping '\x1f54' s = Yield '\x03a5' (s :!: '\x0313' :!: '\x0301') +upperMapping '\x1f54' s = Yield '\x03a5' (s :*: '\x0313' :*: '\x0301') -- GREEK SMALL LETTER UPSILON WITH PSILI AND PERISPOMENI -upperMapping '\x1f56' s = Yield '\x03a5' (s :!: '\x0313' :!: '\x0342') +upperMapping '\x1f56' s = Yield '\x03a5' (s :*: '\x0313' :*: '\x0342') -- GREEK SMALL LETTER ALPHA WITH PERISPOMENI -upperMapping '\x1fb6' s = Yield '\x0391' (s :!: '\x0342' :!: '\x0000') +upperMapping '\x1fb6' s = Yield '\x0391' (s :*: '\x0342' :*: '\x0000') -- GREEK SMALL LETTER ETA WITH PERISPOMENI -upperMapping '\x1fc6' s = Yield '\x0397' (s :!: '\x0342' :!: '\x0000') +upperMapping '\x1fc6' s = Yield '\x0397' (s :*: '\x0342' :*: '\x0000') -- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND VARIA -upperMapping '\x1fd2' s = Yield '\x0399' (s :!: '\x0308' :!: '\x0300') +upperMapping '\x1fd2' s = Yield '\x0399' (s :*: '\x0308' :*: '\x0300') -- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND OXIA -upperMapping '\x1fd3' s = Yield '\x0399' (s :!: '\x0308' :!: '\x0301') +upperMapping '\x1fd3' s = Yield '\x0399' (s :*: '\x0308' :*: '\x0301') -- GREEK SMALL LETTER IOTA WITH PERISPOMENI -upperMapping '\x1fd6' s = Yield '\x0399' (s :!: '\x0342' :!: '\x0000') +upperMapping '\x1fd6' s = Yield '\x0399' (s :*: '\x0342' :*: '\x0000') -- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND PERISPOMENI -upperMapping '\x1fd7' s = Yield '\x0399' (s :!: '\x0308' :!: '\x0342') +upperMapping '\x1fd7' s = Yield '\x0399' (s :*: '\x0308' :*: '\x0342') -- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND VARIA -upperMapping '\x1fe2' s = Yield '\x03a5' (s :!: '\x0308' :!: '\x0300') +upperMapping '\x1fe2' s = Yield '\x03a5' (s :*: '\x0308' :*: '\x0300') -- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND OXIA -upperMapping '\x1fe3' s = Yield '\x03a5' (s :!: '\x0308' :!: '\x0301') +upperMapping '\x1fe3' s = Yield '\x03a5' (s :*: '\x0308' :*: '\x0301') -- GREEK SMALL LETTER RHO WITH PSILI -upperMapping '\x1fe4' s = Yield '\x03a1' (s :!: '\x0313' :!: '\x0000') +upperMapping '\x1fe4' s = Yield '\x03a1' (s :*: '\x0313' :*: '\x0000') -- GREEK SMALL LETTER UPSILON WITH PERISPOMENI -upperMapping '\x1fe6' s = Yield '\x03a5' (s :!: '\x0342' :!: '\x0000') +upperMapping '\x1fe6' s = Yield '\x03a5' (s :*: '\x0342' :*: '\x0000') -- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND PERISPOMENI -upperMapping '\x1fe7' s = Yield '\x03a5' (s :!: '\x0308' :!: '\x0342') +upperMapping '\x1fe7' s = Yield '\x03a5' (s :*: '\x0308' :*: '\x0342') -- GREEK SMALL LETTER OMEGA WITH PERISPOMENI -upperMapping '\x1ff6' s = Yield '\x03a9' (s :!: '\x0342' :!: '\x0000') +upperMapping '\x1ff6' s = Yield '\x03a9' (s :*: '\x0342' :*: '\x0000') -- GREEK SMALL LETTER ALPHA WITH PSILI AND YPOGEGRAMMENI -upperMapping '\x1f80' s = Yield '\x1f08' (s :!: '\x0399' :!: '\x0000') +upperMapping '\x1f80' s = Yield '\x1f08' (s :*: '\x0399' :*: '\x0000') -- GREEK SMALL LETTER ALPHA WITH DASIA AND YPOGEGRAMMENI -upperMapping '\x1f81' s = Yield '\x1f09' (s :!: '\x0399' :!: '\x0000') +upperMapping '\x1f81' s = Yield '\x1f09' (s :*: '\x0399' :*: '\x0000') -- GREEK SMALL LETTER ALPHA WITH PSILI AND VARIA AND YPOGEGRAMMENI -upperMapping '\x1f82' s = Yield '\x1f0a' (s :!: '\x0399' :!: '\x0000') +upperMapping '\x1f82' s = Yield '\x1f0a' (s :*: '\x0399' :*: '\x0000') -- GREEK SMALL LETTER ALPHA WITH DASIA AND VARIA AND YPOGEGRAMMENI -upperMapping '\x1f83' s = Yield '\x1f0b' (s :!: '\x0399' :!: '\x0000') +upperMapping '\x1f83' s = Yield '\x1f0b' (s :*: '\x0399' :*: '\x0000') -- GREEK SMALL LETTER ALPHA WITH PSILI AND OXIA AND YPOGEGRAMMENI -upperMapping '\x1f84' s = Yield '\x1f0c' (s :!: '\x0399' :!: '\x0000') +upperMapping '\x1f84' s = Yield '\x1f0c' (s :*: '\x0399' :*: '\x0000') -- GREEK SMALL LETTER ALPHA WITH DASIA AND OXIA AND YPOGEGRAMMENI -upperMapping '\x1f85' s = Yield '\x1f0d' (s :!: '\x0399' :!: '\x0000') +upperMapping '\x1f85' s = Yield '\x1f0d' (s :*: '\x0399' :*: '\x0000') -- GREEK SMALL LETTER ALPHA WITH PSILI AND PERISPOMENI AND YPOGEGRAMMENI -upperMapping '\x1f86' s = Yield '\x1f0e' (s :!: '\x0399' :!: '\x0000') +upperMapping '\x1f86' s = Yield '\x1f0e' (s :*: '\x0399' :*: '\x0000') -- GREEK SMALL LETTER ALPHA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI -upperMapping '\x1f87' s = Yield '\x1f0f' (s :!: '\x0399' :!: '\x0000') +upperMapping '\x1f87' s = Yield '\x1f0f' (s :*: '\x0399' :*: '\x0000') -- GREEK CAPITAL LETTER ALPHA WITH PSILI AND PROSGEGRAMMENI -upperMapping '\x1f88' s = Yield '\x1f08' (s :!: '\x0399' :!: '\x0000') +upperMapping '\x1f88' s = Yield '\x1f08' (s :*: '\x0399' :*: '\x0000') -- GREEK CAPITAL LETTER ALPHA WITH DASIA AND PROSGEGRAMMENI -upperMapping '\x1f89' s = Yield '\x1f09' (s :!: '\x0399' :!: '\x0000') +upperMapping '\x1f89' s = Yield '\x1f09' (s :*: '\x0399' :*: '\x0000') -- GREEK CAPITAL LETTER ALPHA WITH PSILI AND VARIA AND PROSGEGRAMMENI -upperMapping '\x1f8a' s = Yield '\x1f0a' (s :!: '\x0399' :!: '\x0000') +upperMapping '\x1f8a' s = Yield '\x1f0a' (s :*: '\x0399' :*: '\x0000') -- GREEK CAPITAL LETTER ALPHA WITH DASIA AND VARIA AND PROSGEGRAMMENI -upperMapping '\x1f8b' s = Yield '\x1f0b' (s :!: '\x0399' :!: '\x0000') +upperMapping '\x1f8b' s = Yield '\x1f0b' (s :*: '\x0399' :*: '\x0000') -- GREEK CAPITAL LETTER ALPHA WITH PSILI AND OXIA AND PROSGEGRAMMENI -upperMapping '\x1f8c' s = Yield '\x1f0c' (s :!: '\x0399' :!: '\x0000') +upperMapping '\x1f8c' s = Yield '\x1f0c' (s :*: '\x0399' :*: '\x0000') -- GREEK CAPITAL LETTER ALPHA WITH DASIA AND OXIA AND PROSGEGRAMMENI -upperMapping '\x1f8d' s = Yield '\x1f0d' (s :!: '\x0399' :!: '\x0000') +upperMapping '\x1f8d' s = Yield '\x1f0d' (s :*: '\x0399' :*: '\x0000') -- GREEK CAPITAL LETTER ALPHA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI -upperMapping '\x1f8e' s = Yield '\x1f0e' (s :!: '\x0399' :!: '\x0000') +upperMapping '\x1f8e' s = Yield '\x1f0e' (s :*: '\x0399' :*: '\x0000') -- GREEK CAPITAL LETTER ALPHA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI -upperMapping '\x1f8f' s = Yield '\x1f0f' (s :!: '\x0399' :!: '\x0000') +upperMapping '\x1f8f' s = Yield '\x1f0f' (s :*: '\x0399' :*: '\x0000') -- GREEK SMALL LETTER ETA WITH PSILI AND YPOGEGRAMMENI -upperMapping '\x1f90' s = Yield '\x1f28' (s :!: '\x0399' :!: '\x0000') +upperMapping '\x1f90' s = Yield '\x1f28' (s :*: '\x0399' :*: '\x0000') -- GREEK SMALL LETTER ETA WITH DASIA AND YPOGEGRAMMENI -upperMapping '\x1f91' s = Yield '\x1f29' (s :!: '\x0399' :!: '\x0000') +upperMapping '\x1f91' s = Yield '\x1f29' (s :*: '\x0399' :*: '\x0000') -- GREEK SMALL LETTER ETA WITH PSILI AND VARIA AND YPOGEGRAMMENI -upperMapping '\x1f92' s = Yield '\x1f2a' (s :!: '\x0399' :!: '\x0000') +upperMapping '\x1f92' s = Yield '\x1f2a' (s :*: '\x0399' :*: '\x0000') -- GREEK SMALL LETTER ETA WITH DASIA AND VARIA AND YPOGEGRAMMENI -upperMapping '\x1f93' s = Yield '\x1f2b' (s :!: '\x0399' :!: '\x0000') +upperMapping '\x1f93' s = Yield '\x1f2b' (s :*: '\x0399' :*: '\x0000') -- GREEK SMALL LETTER ETA WITH PSILI AND OXIA AND YPOGEGRAMMENI -upperMapping '\x1f94' s = Yield '\x1f2c' (s :!: '\x0399' :!: '\x0000') +upperMapping '\x1f94' s = Yield '\x1f2c' (s :*: '\x0399' :*: '\x0000') -- GREEK SMALL LETTER ETA WITH DASIA AND OXIA AND YPOGEGRAMMENI -upperMapping '\x1f95' s = Yield '\x1f2d' (s :!: '\x0399' :!: '\x0000') +upperMapping '\x1f95' s = Yield '\x1f2d' (s :*: '\x0399' :*: '\x0000') -- GREEK SMALL LETTER ETA WITH PSILI AND PERISPOMENI AND YPOGEGRAMMENI -upperMapping '\x1f96' s = Yield '\x1f2e' (s :!: '\x0399' :!: '\x0000') +upperMapping '\x1f96' s = Yield '\x1f2e' (s :*: '\x0399' :*: '\x0000') -- GREEK SMALL LETTER ETA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI -upperMapping '\x1f97' s = Yield '\x1f2f' (s :!: '\x0399' :!: '\x0000') +upperMapping '\x1f97' s = Yield '\x1f2f' (s :*: '\x0399' :*: '\x0000') -- GREEK CAPITAL LETTER ETA WITH PSILI AND PROSGEGRAMMENI -upperMapping '\x1f98' s = Yield '\x1f28' (s :!: '\x0399' :!: '\x0000') +upperMapping '\x1f98' s = Yield '\x1f28' (s :*: '\x0399' :*: '\x0000') -- GREEK CAPITAL LETTER ETA WITH DASIA AND PROSGEGRAMMENI -upperMapping '\x1f99' s = Yield '\x1f29' (s :!: '\x0399' :!: '\x0000') +upperMapping '\x1f99' s = Yield '\x1f29' (s :*: '\x0399' :*: '\x0000') -- GREEK CAPITAL LETTER ETA WITH PSILI AND VARIA AND PROSGEGRAMMENI -upperMapping '\x1f9a' s = Yield '\x1f2a' (s :!: '\x0399' :!: '\x0000') +upperMapping '\x1f9a' s = Yield '\x1f2a' (s :*: '\x0399' :*: '\x0000') -- GREEK CAPITAL LETTER ETA WITH DASIA AND VARIA AND PROSGEGRAMMENI -upperMapping '\x1f9b' s = Yield '\x1f2b' (s :!: '\x0399' :!: '\x0000') +upperMapping '\x1f9b' s = Yield '\x1f2b' (s :*: '\x0399' :*: '\x0000') -- GREEK CAPITAL LETTER ETA WITH PSILI AND OXIA AND PROSGEGRAMMENI -upperMapping '\x1f9c' s = Yield '\x1f2c' (s :!: '\x0399' :!: '\x0000') +upperMapping '\x1f9c' s = Yield '\x1f2c' (s :*: '\x0399' :*: '\x0000') -- GREEK CAPITAL LETTER ETA WITH DASIA AND OXIA AND PROSGEGRAMMENI -upperMapping '\x1f9d' s = Yield '\x1f2d' (s :!: '\x0399' :!: '\x0000') +upperMapping '\x1f9d' s = Yield '\x1f2d' (s :*: '\x0399' :*: '\x0000') -- GREEK CAPITAL LETTER ETA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI -upperMapping '\x1f9e' s = Yield '\x1f2e' (s :!: '\x0399' :!: '\x0000') +upperMapping '\x1f9e' s = Yield '\x1f2e' (s :*: '\x0399' :*: '\x0000') -- GREEK CAPITAL LETTER ETA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI -upperMapping '\x1f9f' s = Yield '\x1f2f' (s :!: '\x0399' :!: '\x0000') +upperMapping '\x1f9f' s = Yield '\x1f2f' (s :*: '\x0399' :*: '\x0000') -- GREEK SMALL LETTER OMEGA WITH PSILI AND YPOGEGRAMMENI -upperMapping '\x1fa0' s = Yield '\x1f68' (s :!: '\x0399' :!: '\x0000') +upperMapping '\x1fa0' s = Yield '\x1f68' (s :*: '\x0399' :*: '\x0000') -- GREEK SMALL LETTER OMEGA WITH DASIA AND YPOGEGRAMMENI -upperMapping '\x1fa1' s = Yield '\x1f69' (s :!: '\x0399' :!: '\x0000') +upperMapping '\x1fa1' s = Yield '\x1f69' (s :*: '\x0399' :*: '\x0000') -- GREEK SMALL LETTER OMEGA WITH PSILI AND VARIA AND YPOGEGRAMMENI -upperMapping '\x1fa2' s = Yield '\x1f6a' (s :!: '\x0399' :!: '\x0000') +upperMapping '\x1fa2' s = Yield '\x1f6a' (s :*: '\x0399' :*: '\x0000') -- GREEK SMALL LETTER OMEGA WITH DASIA AND VARIA AND YPOGEGRAMMENI -upperMapping '\x1fa3' s = Yield '\x1f6b' (s :!: '\x0399' :!: '\x0000') +upperMapping '\x1fa3' s = Yield '\x1f6b' (s :*: '\x0399' :*: '\x0000') -- GREEK SMALL LETTER OMEGA WITH PSILI AND OXIA AND YPOGEGRAMMENI -upperMapping '\x1fa4' s = Yield '\x1f6c' (s :!: '\x0399' :!: '\x0000') +upperMapping '\x1fa4' s = Yield '\x1f6c' (s :*: '\x0399' :*: '\x0000') -- GREEK SMALL LETTER OMEGA WITH DASIA AND OXIA AND YPOGEGRAMMENI -upperMapping '\x1fa5' s = Yield '\x1f6d' (s :!: '\x0399' :!: '\x0000') +upperMapping '\x1fa5' s = Yield '\x1f6d' (s :*: '\x0399' :*: '\x0000') -- GREEK SMALL LETTER OMEGA WITH PSILI AND PERISPOMENI AND YPOGEGRAMMENI -upperMapping '\x1fa6' s = Yield '\x1f6e' (s :!: '\x0399' :!: '\x0000') +upperMapping '\x1fa6' s = Yield '\x1f6e' (s :*: '\x0399' :*: '\x0000') -- GREEK SMALL LETTER OMEGA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI -upperMapping '\x1fa7' s = Yield '\x1f6f' (s :!: '\x0399' :!: '\x0000') +upperMapping '\x1fa7' s = Yield '\x1f6f' (s :*: '\x0399' :*: '\x0000') -- GREEK CAPITAL LETTER OMEGA WITH PSILI AND PROSGEGRAMMENI -upperMapping '\x1fa8' s = Yield '\x1f68' (s :!: '\x0399' :!: '\x0000') +upperMapping '\x1fa8' s = Yield '\x1f68' (s :*: '\x0399' :*: '\x0000') -- GREEK CAPITAL LETTER OMEGA WITH DASIA AND PROSGEGRAMMENI -upperMapping '\x1fa9' s = Yield '\x1f69' (s :!: '\x0399' :!: '\x0000') +upperMapping '\x1fa9' s = Yield '\x1f69' (s :*: '\x0399' :*: '\x0000') -- GREEK CAPITAL LETTER OMEGA WITH PSILI AND VARIA AND PROSGEGRAMMENI -upperMapping '\x1faa' s = Yield '\x1f6a' (s :!: '\x0399' :!: '\x0000') +upperMapping '\x1faa' s = Yield '\x1f6a' (s :*: '\x0399' :*: '\x0000') -- GREEK CAPITAL LETTER OMEGA WITH DASIA AND VARIA AND PROSGEGRAMMENI -upperMapping '\x1fab' s = Yield '\x1f6b' (s :!: '\x0399' :!: '\x0000') +upperMapping '\x1fab' s = Yield '\x1f6b' (s :*: '\x0399' :*: '\x0000') -- GREEK CAPITAL LETTER OMEGA WITH PSILI AND OXIA AND PROSGEGRAMMENI -upperMapping '\x1fac' s = Yield '\x1f6c' (s :!: '\x0399' :!: '\x0000') +upperMapping '\x1fac' s = Yield '\x1f6c' (s :*: '\x0399' :*: '\x0000') -- GREEK CAPITAL LETTER OMEGA WITH DASIA AND OXIA AND PROSGEGRAMMENI -upperMapping '\x1fad' s = Yield '\x1f6d' (s :!: '\x0399' :!: '\x0000') +upperMapping '\x1fad' s = Yield '\x1f6d' (s :*: '\x0399' :*: '\x0000') -- GREEK CAPITAL LETTER OMEGA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI -upperMapping '\x1fae' s = Yield '\x1f6e' (s :!: '\x0399' :!: '\x0000') +upperMapping '\x1fae' s = Yield '\x1f6e' (s :*: '\x0399' :*: '\x0000') -- GREEK CAPITAL LETTER OMEGA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI -upperMapping '\x1faf' s = Yield '\x1f6f' (s :!: '\x0399' :!: '\x0000') +upperMapping '\x1faf' s = Yield '\x1f6f' (s :*: '\x0399' :*: '\x0000') -- GREEK SMALL LETTER ALPHA WITH YPOGEGRAMMENI -upperMapping '\x1fb3' s = Yield '\x0391' (s :!: '\x0399' :!: '\x0000') +upperMapping '\x1fb3' s = Yield '\x0391' (s :*: '\x0399' :*: '\x0000') -- GREEK CAPITAL LETTER ALPHA WITH PROSGEGRAMMENI -upperMapping '\x1fbc' s = Yield '\x0391' (s :!: '\x0399' :!: '\x0000') +upperMapping '\x1fbc' s = Yield '\x0391' (s :*: '\x0399' :*: '\x0000') -- GREEK SMALL LETTER ETA WITH YPOGEGRAMMENI -upperMapping '\x1fc3' s = Yield '\x0397' (s :!: '\x0399' :!: '\x0000') +upperMapping '\x1fc3' s = Yield '\x0397' (s :*: '\x0399' :*: '\x0000') -- GREEK CAPITAL LETTER ETA WITH PROSGEGRAMMENI -upperMapping '\x1fcc' s = Yield '\x0397' (s :!: '\x0399' :!: '\x0000') +upperMapping '\x1fcc' s = Yield '\x0397' (s :*: '\x0399' :*: '\x0000') -- GREEK SMALL LETTER OMEGA WITH YPOGEGRAMMENI -upperMapping '\x1ff3' s = Yield '\x03a9' (s :!: '\x0399' :!: '\x0000') +upperMapping '\x1ff3' s = Yield '\x03a9' (s :*: '\x0399' :*: '\x0000') -- GREEK CAPITAL LETTER OMEGA WITH PROSGEGRAMMENI -upperMapping '\x1ffc' s = Yield '\x03a9' (s :!: '\x0399' :!: '\x0000') +upperMapping '\x1ffc' s = Yield '\x03a9' (s :*: '\x0399' :*: '\x0000') -- GREEK SMALL LETTER ALPHA WITH VARIA AND YPOGEGRAMMENI -upperMapping '\x1fb2' s = Yield '\x1fba' (s :!: '\x0399' :!: '\x0000') +upperMapping '\x1fb2' s = Yield '\x1fba' (s :*: '\x0399' :*: '\x0000') -- GREEK SMALL LETTER ALPHA WITH OXIA AND YPOGEGRAMMENI -upperMapping '\x1fb4' s = Yield '\x0386' (s :!: '\x0399' :!: '\x0000') +upperMapping '\x1fb4' s = Yield '\x0386' (s :*: '\x0399' :*: '\x0000') -- GREEK SMALL LETTER ETA WITH VARIA AND YPOGEGRAMMENI -upperMapping '\x1fc2' s = Yield '\x1fca' (s :!: '\x0399' :!: '\x0000') +upperMapping '\x1fc2' s = Yield '\x1fca' (s :*: '\x0399' :*: '\x0000') -- GREEK SMALL LETTER ETA WITH OXIA AND YPOGEGRAMMENI -upperMapping '\x1fc4' s = Yield '\x0389' (s :!: '\x0399' :!: '\x0000') +upperMapping '\x1fc4' s = Yield '\x0389' (s :*: '\x0399' :*: '\x0000') -- GREEK SMALL LETTER OMEGA WITH VARIA AND YPOGEGRAMMENI -upperMapping '\x1ff2' s = Yield '\x1ffa' (s :!: '\x0399' :!: '\x0000') +upperMapping '\x1ff2' s = Yield '\x1ffa' (s :*: '\x0399' :*: '\x0000') -- GREEK SMALL LETTER OMEGA WITH OXIA AND YPOGEGRAMMENI -upperMapping '\x1ff4' s = Yield '\x038f' (s :!: '\x0399' :!: '\x0000') +upperMapping '\x1ff4' s = Yield '\x038f' (s :*: '\x0399' :*: '\x0000') -- GREEK SMALL LETTER ALPHA WITH PERISPOMENI AND YPOGEGRAMMENI -upperMapping '\x1fb7' s = Yield '\x0391' (s :!: '\x0342' :!: '\x0399') +upperMapping '\x1fb7' s = Yield '\x0391' (s :*: '\x0342' :*: '\x0399') -- GREEK SMALL LETTER ETA WITH PERISPOMENI AND YPOGEGRAMMENI -upperMapping '\x1fc7' s = Yield '\x0397' (s :!: '\x0342' :!: '\x0399') +upperMapping '\x1fc7' s = Yield '\x0397' (s :*: '\x0342' :*: '\x0399') -- GREEK SMALL LETTER OMEGA WITH PERISPOMENI AND YPOGEGRAMMENI -upperMapping '\x1ff7' s = Yield '\x03a9' (s :!: '\x0342' :!: '\x0399') -upperMapping c s = Yield (toUpper c) (s :!: '\0' :!: '\0') +upperMapping '\x1ff7' s = Yield '\x03a9' (s :*: '\x0342' :*: '\x0399') +upperMapping c s = Yield (toUpper c) (s :*: '\0' :*: '\0') lowerMapping :: forall s. Char -> s -> Step (PairS (PairS s Char) Char) Char {-# INLINE lowerMapping #-} -- LATIN CAPITAL LETTER I WITH DOT ABOVE -lowerMapping '\x0130' s = Yield '\x0069' (s :!: '\x0307' :!: '\x0000') -lowerMapping c s = Yield (toLower c) (s :!: '\0' :!: '\0') +lowerMapping '\x0130' s = Yield '\x0069' (s :*: '\x0307' :*: '\x0000') +lowerMapping c s = Yield (toLower c) (s :*: '\0' :*: '\0') foldMapping :: forall s. Char -> s -> Step (PairS (PairS s Char) Char) Char {-# INLINE foldMapping #-} -- MICRO SIGN -foldMapping '\x00b5' s = Yield '\x03bc' (s :!: '\x0000' :!: '\x0000') +foldMapping '\x00b5' s = Yield '\x03bc' (s :*: '\x0000' :*: '\x0000') -- LATIN SMALL LETTER SHARP S -foldMapping '\x00df' s = Yield '\x0073' (s :!: '\x0073' :!: '\x0000') +foldMapping '\x00df' s = Yield '\x0073' (s :*: '\x0073' :*: '\x0000') -- LATIN CAPITAL LETTER I WITH DOT ABOVE -foldMapping '\x0130' s = Yield '\x0069' (s :!: '\x0307' :!: '\x0000') +foldMapping '\x0130' s = Yield '\x0069' (s :*: '\x0307' :*: '\x0000') -- LATIN SMALL LETTER N PRECEDED BY APOSTROPHE -foldMapping '\x0149' s = Yield '\x02bc' (s :!: '\x006e' :!: '\x0000') +foldMapping '\x0149' s = Yield '\x02bc' (s :*: '\x006e' :*: '\x0000') -- LATIN SMALL LETTER LONG S -foldMapping '\x017f' s = Yield '\x0073' (s :!: '\x0000' :!: '\x0000') +foldMapping '\x017f' s = Yield '\x0073' (s :*: '\x0000' :*: '\x0000') -- LATIN SMALL LETTER J WITH CARON -foldMapping '\x01f0' s = Yield '\x006a' (s :!: '\x030c' :!: '\x0000') +foldMapping '\x01f0' s = Yield '\x006a' (s :*: '\x030c' :*: '\x0000') -- COMBINING GREEK YPOGEGRAMMENI -foldMapping '\x0345' s = Yield '\x03b9' (s :!: '\x0000' :!: '\x0000') +foldMapping '\x0345' s = Yield '\x03b9' (s :*: '\x0000' :*: '\x0000') -- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND TONOS -foldMapping '\x0390' s = Yield '\x03b9' (s :!: '\x0308' :!: '\x0301') +foldMapping '\x0390' s = Yield '\x03b9' (s :*: '\x0308' :*: '\x0301') -- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND TONOS -foldMapping '\x03b0' s = Yield '\x03c5' (s :!: '\x0308' :!: '\x0301') +foldMapping '\x03b0' s = Yield '\x03c5' (s :*: '\x0308' :*: '\x0301') -- GREEK SMALL LETTER FINAL SIGMA -foldMapping '\x03c2' s = Yield '\x03c3' (s :!: '\x0000' :!: '\x0000') +foldMapping '\x03c2' s = Yield '\x03c3' (s :*: '\x0000' :*: '\x0000') -- GREEK BETA SYMBOL -foldMapping '\x03d0' s = Yield '\x03b2' (s :!: '\x0000' :!: '\x0000') +foldMapping '\x03d0' s = Yield '\x03b2' (s :*: '\x0000' :*: '\x0000') -- GREEK THETA SYMBOL -foldMapping '\x03d1' s = Yield '\x03b8' (s :!: '\x0000' :!: '\x0000') +foldMapping '\x03d1' s = Yield '\x03b8' (s :*: '\x0000' :*: '\x0000') -- GREEK PHI SYMBOL -foldMapping '\x03d5' s = Yield '\x03c6' (s :!: '\x0000' :!: '\x0000') +foldMapping '\x03d5' s = Yield '\x03c6' (s :*: '\x0000' :*: '\x0000') -- GREEK PI SYMBOL -foldMapping '\x03d6' s = Yield '\x03c0' (s :!: '\x0000' :!: '\x0000') +foldMapping '\x03d6' s = Yield '\x03c0' (s :*: '\x0000' :*: '\x0000') -- GREEK KAPPA SYMBOL -foldMapping '\x03f0' s = Yield '\x03ba' (s :!: '\x0000' :!: '\x0000') +foldMapping '\x03f0' s = Yield '\x03ba' (s :*: '\x0000' :*: '\x0000') -- GREEK RHO SYMBOL -foldMapping '\x03f1' s = Yield '\x03c1' (s :!: '\x0000' :!: '\x0000') +foldMapping '\x03f1' s = Yield '\x03c1' (s :*: '\x0000' :*: '\x0000') -- GREEK LUNATE EPSILON SYMBOL -foldMapping '\x03f5' s = Yield '\x03b5' (s :!: '\x0000' :!: '\x0000') +foldMapping '\x03f5' s = Yield '\x03b5' (s :*: '\x0000' :*: '\x0000') -- ARMENIAN SMALL LIGATURE ECH YIWN -foldMapping '\x0587' s = Yield '\x0565' (s :!: '\x0582' :!: '\x0000') +foldMapping '\x0587' s = Yield '\x0565' (s :*: '\x0582' :*: '\x0000') -- LATIN SMALL LETTER H WITH LINE BELOW -foldMapping '\x1e96' s = Yield '\x0068' (s :!: '\x0331' :!: '\x0000') +foldMapping '\x1e96' s = Yield '\x0068' (s :*: '\x0331' :*: '\x0000') -- LATIN SMALL LETTER T WITH DIAERESIS -foldMapping '\x1e97' s = Yield '\x0074' (s :!: '\x0308' :!: '\x0000') +foldMapping '\x1e97' s = Yield '\x0074' (s :*: '\x0308' :*: '\x0000') -- LATIN SMALL LETTER W WITH RING ABOVE -foldMapping '\x1e98' s = Yield '\x0077' (s :!: '\x030a' :!: '\x0000') +foldMapping '\x1e98' s = Yield '\x0077' (s :*: '\x030a' :*: '\x0000') -- LATIN SMALL LETTER Y WITH RING ABOVE -foldMapping '\x1e99' s = Yield '\x0079' (s :!: '\x030a' :!: '\x0000') +foldMapping '\x1e99' s = Yield '\x0079' (s :*: '\x030a' :*: '\x0000') -- LATIN SMALL LETTER A WITH RIGHT HALF RING -foldMapping '\x1e9a' s = Yield '\x0061' (s :!: '\x02be' :!: '\x0000') +foldMapping '\x1e9a' s = Yield '\x0061' (s :*: '\x02be' :*: '\x0000') -- LATIN SMALL LETTER LONG S WITH DOT ABOVE -foldMapping '\x1e9b' s = Yield '\x1e61' (s :!: '\x0000' :!: '\x0000') +foldMapping '\x1e9b' s = Yield '\x1e61' (s :*: '\x0000' :*: '\x0000') -- LATIN CAPITAL LETTER SHARP S -foldMapping '\x1e9e' s = Yield '\x0073' (s :!: '\x0073' :!: '\x0000') +foldMapping '\x1e9e' s = Yield '\x0073' (s :*: '\x0073' :*: '\x0000') -- GREEK SMALL LETTER UPSILON WITH PSILI -foldMapping '\x1f50' s = Yield '\x03c5' (s :!: '\x0313' :!: '\x0000') +foldMapping '\x1f50' s = Yield '\x03c5' (s :*: '\x0313' :*: '\x0000') -- GREEK SMALL LETTER UPSILON WITH PSILI AND VARIA -foldMapping '\x1f52' s = Yield '\x03c5' (s :!: '\x0313' :!: '\x0300') +foldMapping '\x1f52' s = Yield '\x03c5' (s :*: '\x0313' :*: '\x0300') -- GREEK SMALL LETTER UPSILON WITH PSILI AND OXIA -foldMapping '\x1f54' s = Yield '\x03c5' (s :!: '\x0313' :!: '\x0301') +foldMapping '\x1f54' s = Yield '\x03c5' (s :*: '\x0313' :*: '\x0301') -- GREEK SMALL LETTER UPSILON WITH PSILI AND PERISPOMENI -foldMapping '\x1f56' s = Yield '\x03c5' (s :!: '\x0313' :!: '\x0342') +foldMapping '\x1f56' s = Yield '\x03c5' (s :*: '\x0313' :*: '\x0342') -- GREEK SMALL LETTER ALPHA WITH PSILI AND YPOGEGRAMMENI -foldMapping '\x1f80' s = Yield '\x1f00' (s :!: '\x03b9' :!: '\x0000') +foldMapping '\x1f80' s = Yield '\x1f00' (s :*: '\x03b9' :*: '\x0000') -- GREEK SMALL LETTER ALPHA WITH DASIA AND YPOGEGRAMMENI -foldMapping '\x1f81' s = Yield '\x1f01' (s :!: '\x03b9' :!: '\x0000') +foldMapping '\x1f81' s = Yield '\x1f01' (s :*: '\x03b9' :*: '\x0000') -- GREEK SMALL LETTER ALPHA WITH PSILI AND VARIA AND YPOGEGRAMMENI -foldMapping '\x1f82' s = Yield '\x1f02' (s :!: '\x03b9' :!: '\x0000') +foldMapping '\x1f82' s = Yield '\x1f02' (s :*: '\x03b9' :*: '\x0000') -- GREEK SMALL LETTER ALPHA WITH DASIA AND VARIA AND YPOGEGRAMMENI -foldMapping '\x1f83' s = Yield '\x1f03' (s :!: '\x03b9' :!: '\x0000') +foldMapping '\x1f83' s = Yield '\x1f03' (s :*: '\x03b9' :*: '\x0000') -- GREEK SMALL LETTER ALPHA WITH PSILI AND OXIA AND YPOGEGRAMMENI -foldMapping '\x1f84' s = Yield '\x1f04' (s :!: '\x03b9' :!: '\x0000') +foldMapping '\x1f84' s = Yield '\x1f04' (s :*: '\x03b9' :*: '\x0000') -- GREEK SMALL LETTER ALPHA WITH DASIA AND OXIA AND YPOGEGRAMMENI -foldMapping '\x1f85' s = Yield '\x1f05' (s :!: '\x03b9' :!: '\x0000') +foldMapping '\x1f85' s = Yield '\x1f05' (s :*: '\x03b9' :*: '\x0000') -- GREEK SMALL LETTER ALPHA WITH PSILI AND PERISPOMENI AND YPOGEGRAMMENI -foldMapping '\x1f86' s = Yield '\x1f06' (s :!: '\x03b9' :!: '\x0000') +foldMapping '\x1f86' s = Yield '\x1f06' (s :*: '\x03b9' :*: '\x0000') -- GREEK SMALL LETTER ALPHA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI -foldMapping '\x1f87' s = Yield '\x1f07' (s :!: '\x03b9' :!: '\x0000') +foldMapping '\x1f87' s = Yield '\x1f07' (s :*: '\x03b9' :*: '\x0000') -- GREEK CAPITAL LETTER ALPHA WITH PSILI AND PROSGEGRAMMENI -foldMapping '\x1f88' s = Yield '\x1f00' (s :!: '\x03b9' :!: '\x0000') +foldMapping '\x1f88' s = Yield '\x1f00' (s :*: '\x03b9' :*: '\x0000') -- GREEK CAPITAL LETTER ALPHA WITH DASIA AND PROSGEGRAMMENI -foldMapping '\x1f89' s = Yield '\x1f01' (s :!: '\x03b9' :!: '\x0000') +foldMapping '\x1f89' s = Yield '\x1f01' (s :*: '\x03b9' :*: '\x0000') -- GREEK CAPITAL LETTER ALPHA WITH PSILI AND VARIA AND PROSGEGRAMMENI -foldMapping '\x1f8a' s = Yield '\x1f02' (s :!: '\x03b9' :!: '\x0000') +foldMapping '\x1f8a' s = Yield '\x1f02' (s :*: '\x03b9' :*: '\x0000') -- GREEK CAPITAL LETTER ALPHA WITH DASIA AND VARIA AND PROSGEGRAMMENI -foldMapping '\x1f8b' s = Yield '\x1f03' (s :!: '\x03b9' :!: '\x0000') +foldMapping '\x1f8b' s = Yield '\x1f03' (s :*: '\x03b9' :*: '\x0000') -- GREEK CAPITAL LETTER ALPHA WITH PSILI AND OXIA AND PROSGEGRAMMENI -foldMapping '\x1f8c' s = Yield '\x1f04' (s :!: '\x03b9' :!: '\x0000') +foldMapping '\x1f8c' s = Yield '\x1f04' (s :*: '\x03b9' :*: '\x0000') -- GREEK CAPITAL LETTER ALPHA WITH DASIA AND OXIA AND PROSGEGRAMMENI -foldMapping '\x1f8d' s = Yield '\x1f05' (s :!: '\x03b9' :!: '\x0000') +foldMapping '\x1f8d' s = Yield '\x1f05' (s :*: '\x03b9' :*: '\x0000') -- GREEK CAPITAL LETTER ALPHA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI -foldMapping '\x1f8e' s = Yield '\x1f06' (s :!: '\x03b9' :!: '\x0000') +foldMapping '\x1f8e' s = Yield '\x1f06' (s :*: '\x03b9' :*: '\x0000') -- GREEK CAPITAL LETTER ALPHA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI -foldMapping '\x1f8f' s = Yield '\x1f07' (s :!: '\x03b9' :!: '\x0000') +foldMapping '\x1f8f' s = Yield '\x1f07' (s :*: '\x03b9' :*: '\x0000') -- GREEK SMALL LETTER ETA WITH PSILI AND YPOGEGRAMMENI -foldMapping '\x1f90' s = Yield '\x1f20' (s :!: '\x03b9' :!: '\x0000') +foldMapping '\x1f90' s = Yield '\x1f20' (s :*: '\x03b9' :*: '\x0000') -- GREEK SMALL LETTER ETA WITH DASIA AND YPOGEGRAMMENI -foldMapping '\x1f91' s = Yield '\x1f21' (s :!: '\x03b9' :!: '\x0000') +foldMapping '\x1f91' s = Yield '\x1f21' (s :*: '\x03b9' :*: '\x0000') -- GREEK SMALL LETTER ETA WITH PSILI AND VARIA AND YPOGEGRAMMENI -foldMapping '\x1f92' s = Yield '\x1f22' (s :!: '\x03b9' :!: '\x0000') +foldMapping '\x1f92' s = Yield '\x1f22' (s :*: '\x03b9' :*: '\x0000') -- GREEK SMALL LETTER ETA WITH DASIA AND VARIA AND YPOGEGRAMMENI -foldMapping '\x1f93' s = Yield '\x1f23' (s :!: '\x03b9' :!: '\x0000') +foldMapping '\x1f93' s = Yield '\x1f23' (s :*: '\x03b9' :*: '\x0000') -- GREEK SMALL LETTER ETA WITH PSILI AND OXIA AND YPOGEGRAMMENI -foldMapping '\x1f94' s = Yield '\x1f24' (s :!: '\x03b9' :!: '\x0000') +foldMapping '\x1f94' s = Yield '\x1f24' (s :*: '\x03b9' :*: '\x0000') -- GREEK SMALL LETTER ETA WITH DASIA AND OXIA AND YPOGEGRAMMENI -foldMapping '\x1f95' s = Yield '\x1f25' (s :!: '\x03b9' :!: '\x0000') +foldMapping '\x1f95' s = Yield '\x1f25' (s :*: '\x03b9' :*: '\x0000') -- GREEK SMALL LETTER ETA WITH PSILI AND PERISPOMENI AND YPOGEGRAMMENI -foldMapping '\x1f96' s = Yield '\x1f26' (s :!: '\x03b9' :!: '\x0000') +foldMapping '\x1f96' s = Yield '\x1f26' (s :*: '\x03b9' :*: '\x0000') -- GREEK SMALL LETTER ETA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI -foldMapping '\x1f97' s = Yield '\x1f27' (s :!: '\x03b9' :!: '\x0000') +foldMapping '\x1f97' s = Yield '\x1f27' (s :*: '\x03b9' :*: '\x0000') -- GREEK CAPITAL LETTER ETA WITH PSILI AND PROSGEGRAMMENI -foldMapping '\x1f98' s = Yield '\x1f20' (s :!: '\x03b9' :!: '\x0000') +foldMapping '\x1f98' s = Yield '\x1f20' (s :*: '\x03b9' :*: '\x0000') -- GREEK CAPITAL LETTER ETA WITH DASIA AND PROSGEGRAMMENI -foldMapping '\x1f99' s = Yield '\x1f21' (s :!: '\x03b9' :!: '\x0000') +foldMapping '\x1f99' s = Yield '\x1f21' (s :*: '\x03b9' :*: '\x0000') -- GREEK CAPITAL LETTER ETA WITH PSILI AND VARIA AND PROSGEGRAMMENI -foldMapping '\x1f9a' s = Yield '\x1f22' (s :!: '\x03b9' :!: '\x0000') +foldMapping '\x1f9a' s = Yield '\x1f22' (s :*: '\x03b9' :*: '\x0000') -- GREEK CAPITAL LETTER ETA WITH DASIA AND VARIA AND PROSGEGRAMMENI -foldMapping '\x1f9b' s = Yield '\x1f23' (s :!: '\x03b9' :!: '\x0000') +foldMapping '\x1f9b' s = Yield '\x1f23' (s :*: '\x03b9' :*: '\x0000') -- GREEK CAPITAL LETTER ETA WITH PSILI AND OXIA AND PROSGEGRAMMENI -foldMapping '\x1f9c' s = Yield '\x1f24' (s :!: '\x03b9' :!: '\x0000') +foldMapping '\x1f9c' s = Yield '\x1f24' (s :*: '\x03b9' :*: '\x0000') -- GREEK CAPITAL LETTER ETA WITH DASIA AND OXIA AND PROSGEGRAMMENI -foldMapping '\x1f9d' s = Yield '\x1f25' (s :!: '\x03b9' :!: '\x0000') +foldMapping '\x1f9d' s = Yield '\x1f25' (s :*: '\x03b9' :*: '\x0000') -- GREEK CAPITAL LETTER ETA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI -foldMapping '\x1f9e' s = Yield '\x1f26' (s :!: '\x03b9' :!: '\x0000') +foldMapping '\x1f9e' s = Yield '\x1f26' (s :*: '\x03b9' :*: '\x0000') -- GREEK CAPITAL LETTER ETA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI -foldMapping '\x1f9f' s = Yield '\x1f27' (s :!: '\x03b9' :!: '\x0000') +foldMapping '\x1f9f' s = Yield '\x1f27' (s :*: '\x03b9' :*: '\x0000') -- GREEK SMALL LETTER OMEGA WITH PSILI AND YPOGEGRAMMENI -foldMapping '\x1fa0' s = Yield '\x1f60' (s :!: '\x03b9' :!: '\x0000') +foldMapping '\x1fa0' s = Yield '\x1f60' (s :*: '\x03b9' :*: '\x0000') -- GREEK SMALL LETTER OMEGA WITH DASIA AND YPOGEGRAMMENI -foldMapping '\x1fa1' s = Yield '\x1f61' (s :!: '\x03b9' :!: '\x0000') +foldMapping '\x1fa1' s = Yield '\x1f61' (s :*: '\x03b9' :*: '\x0000') -- GREEK SMALL LETTER OMEGA WITH PSILI AND VARIA AND YPOGEGRAMMENI -foldMapping '\x1fa2' s = Yield '\x1f62' (s :!: '\x03b9' :!: '\x0000') +foldMapping '\x1fa2' s = Yield '\x1f62' (s :*: '\x03b9' :*: '\x0000') -- GREEK SMALL LETTER OMEGA WITH DASIA AND VARIA AND YPOGEGRAMMENI -foldMapping '\x1fa3' s = Yield '\x1f63' (s :!: '\x03b9' :!: '\x0000') +foldMapping '\x1fa3' s = Yield '\x1f63' (s :*: '\x03b9' :*: '\x0000') -- GREEK SMALL LETTER OMEGA WITH PSILI AND OXIA AND YPOGEGRAMMENI -foldMapping '\x1fa4' s = Yield '\x1f64' (s :!: '\x03b9' :!: '\x0000') +foldMapping '\x1fa4' s = Yield '\x1f64' (s :*: '\x03b9' :*: '\x0000') -- GREEK SMALL LETTER OMEGA WITH DASIA AND OXIA AND YPOGEGRAMMENI -foldMapping '\x1fa5' s = Yield '\x1f65' (s :!: '\x03b9' :!: '\x0000') +foldMapping '\x1fa5' s = Yield '\x1f65' (s :*: '\x03b9' :*: '\x0000') -- GREEK SMALL LETTER OMEGA WITH PSILI AND PERISPOMENI AND YPOGEGRAMMENI -foldMapping '\x1fa6' s = Yield '\x1f66' (s :!: '\x03b9' :!: '\x0000') +foldMapping '\x1fa6' s = Yield '\x1f66' (s :*: '\x03b9' :*: '\x0000') -- GREEK SMALL LETTER OMEGA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI -foldMapping '\x1fa7' s = Yield '\x1f67' (s :!: '\x03b9' :!: '\x0000') +foldMapping '\x1fa7' s = Yield '\x1f67' (s :*: '\x03b9' :*: '\x0000') -- GREEK CAPITAL LETTER OMEGA WITH PSILI AND PROSGEGRAMMENI -foldMapping '\x1fa8' s = Yield '\x1f60' (s :!: '\x03b9' :!: '\x0000') +foldMapping '\x1fa8' s = Yield '\x1f60' (s :*: '\x03b9' :*: '\x0000') -- GREEK CAPITAL LETTER OMEGA WITH DASIA AND PROSGEGRAMMENI -foldMapping '\x1fa9' s = Yield '\x1f61' (s :!: '\x03b9' :!: '\x0000') +foldMapping '\x1fa9' s = Yield '\x1f61' (s :*: '\x03b9' :*: '\x0000') -- GREEK CAPITAL LETTER OMEGA WITH PSILI AND VARIA AND PROSGEGRAMMENI -foldMapping '\x1faa' s = Yield '\x1f62' (s :!: '\x03b9' :!: '\x0000') +foldMapping '\x1faa' s = Yield '\x1f62' (s :*: '\x03b9' :*: '\x0000') -- GREEK CAPITAL LETTER OMEGA WITH DASIA AND VARIA AND PROSGEGRAMMENI -foldMapping '\x1fab' s = Yield '\x1f63' (s :!: '\x03b9' :!: '\x0000') +foldMapping '\x1fab' s = Yield '\x1f63' (s :*: '\x03b9' :*: '\x0000') -- GREEK CAPITAL LETTER OMEGA WITH PSILI AND OXIA AND PROSGEGRAMMENI -foldMapping '\x1fac' s = Yield '\x1f64' (s :!: '\x03b9' :!: '\x0000') +foldMapping '\x1fac' s = Yield '\x1f64' (s :*: '\x03b9' :*: '\x0000') -- GREEK CAPITAL LETTER OMEGA WITH DASIA AND OXIA AND PROSGEGRAMMENI -foldMapping '\x1fad' s = Yield '\x1f65' (s :!: '\x03b9' :!: '\x0000') +foldMapping '\x1fad' s = Yield '\x1f65' (s :*: '\x03b9' :*: '\x0000') -- GREEK CAPITAL LETTER OMEGA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI -foldMapping '\x1fae' s = Yield '\x1f66' (s :!: '\x03b9' :!: '\x0000') +foldMapping '\x1fae' s = Yield '\x1f66' (s :*: '\x03b9' :*: '\x0000') -- GREEK CAPITAL LETTER OMEGA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI -foldMapping '\x1faf' s = Yield '\x1f67' (s :!: '\x03b9' :!: '\x0000') +foldMapping '\x1faf' s = Yield '\x1f67' (s :*: '\x03b9' :*: '\x0000') -- GREEK SMALL LETTER ALPHA WITH VARIA AND YPOGEGRAMMENI -foldMapping '\x1fb2' s = Yield '\x1f70' (s :!: '\x03b9' :!: '\x0000') +foldMapping '\x1fb2' s = Yield '\x1f70' (s :*: '\x03b9' :*: '\x0000') -- GREEK SMALL LETTER ALPHA WITH YPOGEGRAMMENI -foldMapping '\x1fb3' s = Yield '\x03b1' (s :!: '\x03b9' :!: '\x0000') +foldMapping '\x1fb3' s = Yield '\x03b1' (s :*: '\x03b9' :*: '\x0000') -- GREEK SMALL LETTER ALPHA WITH OXIA AND YPOGEGRAMMENI -foldMapping '\x1fb4' s = Yield '\x03ac' (s :!: '\x03b9' :!: '\x0000') +foldMapping '\x1fb4' s = Yield '\x03ac' (s :*: '\x03b9' :*: '\x0000') -- GREEK SMALL LETTER ALPHA WITH PERISPOMENI -foldMapping '\x1fb6' s = Yield '\x03b1' (s :!: '\x0342' :!: '\x0000') +foldMapping '\x1fb6' s = Yield '\x03b1' (s :*: '\x0342' :*: '\x0000') -- GREEK SMALL LETTER ALPHA WITH PERISPOMENI AND YPOGEGRAMMENI -foldMapping '\x1fb7' s = Yield '\x03b1' (s :!: '\x0342' :!: '\x03b9') +foldMapping '\x1fb7' s = Yield '\x03b1' (s :*: '\x0342' :*: '\x03b9') -- GREEK CAPITAL LETTER ALPHA WITH PROSGEGRAMMENI -foldMapping '\x1fbc' s = Yield '\x03b1' (s :!: '\x03b9' :!: '\x0000') +foldMapping '\x1fbc' s = Yield '\x03b1' (s :*: '\x03b9' :*: '\x0000') -- GREEK PROSGEGRAMMENI -foldMapping '\x1fbe' s = Yield '\x03b9' (s :!: '\x0000' :!: '\x0000') +foldMapping '\x1fbe' s = Yield '\x03b9' (s :*: '\x0000' :*: '\x0000') -- GREEK SMALL LETTER ETA WITH VARIA AND YPOGEGRAMMENI -foldMapping '\x1fc2' s = Yield '\x1f74' (s :!: '\x03b9' :!: '\x0000') +foldMapping '\x1fc2' s = Yield '\x1f74' (s :*: '\x03b9' :*: '\x0000') -- GREEK SMALL LETTER ETA WITH YPOGEGRAMMENI -foldMapping '\x1fc3' s = Yield '\x03b7' (s :!: '\x03b9' :!: '\x0000') +foldMapping '\x1fc3' s = Yield '\x03b7' (s :*: '\x03b9' :*: '\x0000') -- GREEK SMALL LETTER ETA WITH OXIA AND YPOGEGRAMMENI -foldMapping '\x1fc4' s = Yield '\x03ae' (s :!: '\x03b9' :!: '\x0000') +foldMapping '\x1fc4' s = Yield '\x03ae' (s :*: '\x03b9' :*: '\x0000') -- GREEK SMALL LETTER ETA WITH PERISPOMENI -foldMapping '\x1fc6' s = Yield '\x03b7' (s :!: '\x0342' :!: '\x0000') +foldMapping '\x1fc6' s = Yield '\x03b7' (s :*: '\x0342' :*: '\x0000') -- GREEK SMALL LETTER ETA WITH PERISPOMENI AND YPOGEGRAMMENI -foldMapping '\x1fc7' s = Yield '\x03b7' (s :!: '\x0342' :!: '\x03b9') +foldMapping '\x1fc7' s = Yield '\x03b7' (s :*: '\x0342' :*: '\x03b9') -- GREEK CAPITAL LETTER ETA WITH PROSGEGRAMMENI -foldMapping '\x1fcc' s = Yield '\x03b7' (s :!: '\x03b9' :!: '\x0000') +foldMapping '\x1fcc' s = Yield '\x03b7' (s :*: '\x03b9' :*: '\x0000') -- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND VARIA -foldMapping '\x1fd2' s = Yield '\x03b9' (s :!: '\x0308' :!: '\x0300') +foldMapping '\x1fd2' s = Yield '\x03b9' (s :*: '\x0308' :*: '\x0300') -- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND OXIA -foldMapping '\x1fd3' s = Yield '\x03b9' (s :!: '\x0308' :!: '\x0301') +foldMapping '\x1fd3' s = Yield '\x03b9' (s :*: '\x0308' :*: '\x0301') -- GREEK SMALL LETTER IOTA WITH PERISPOMENI -foldMapping '\x1fd6' s = Yield '\x03b9' (s :!: '\x0342' :!: '\x0000') +foldMapping '\x1fd6' s = Yield '\x03b9' (s :*: '\x0342' :*: '\x0000') -- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND PERISPOMENI -foldMapping '\x1fd7' s = Yield '\x03b9' (s :!: '\x0308' :!: '\x0342') +foldMapping '\x1fd7' s = Yield '\x03b9' (s :*: '\x0308' :*: '\x0342') -- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND VARIA -foldMapping '\x1fe2' s = Yield '\x03c5' (s :!: '\x0308' :!: '\x0300') +foldMapping '\x1fe2' s = Yield '\x03c5' (s :*: '\x0308' :*: '\x0300') -- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND OXIA -foldMapping '\x1fe3' s = Yield '\x03c5' (s :!: '\x0308' :!: '\x0301') +foldMapping '\x1fe3' s = Yield '\x03c5' (s :*: '\x0308' :*: '\x0301') -- GREEK SMALL LETTER RHO WITH PSILI -foldMapping '\x1fe4' s = Yield '\x03c1' (s :!: '\x0313' :!: '\x0000') +foldMapping '\x1fe4' s = Yield '\x03c1' (s :*: '\x0313' :*: '\x0000') -- GREEK SMALL LETTER UPSILON WITH PERISPOMENI -foldMapping '\x1fe6' s = Yield '\x03c5' (s :!: '\x0342' :!: '\x0000') +foldMapping '\x1fe6' s = Yield '\x03c5' (s :*: '\x0342' :*: '\x0000') -- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND PERISPOMENI -foldMapping '\x1fe7' s = Yield '\x03c5' (s :!: '\x0308' :!: '\x0342') +foldMapping '\x1fe7' s = Yield '\x03c5' (s :*: '\x0308' :*: '\x0342') -- GREEK SMALL LETTER OMEGA WITH VARIA AND YPOGEGRAMMENI -foldMapping '\x1ff2' s = Yield '\x1f7c' (s :!: '\x03b9' :!: '\x0000') +foldMapping '\x1ff2' s = Yield '\x1f7c' (s :*: '\x03b9' :*: '\x0000') -- GREEK SMALL LETTER OMEGA WITH YPOGEGRAMMENI -foldMapping '\x1ff3' s = Yield '\x03c9' (s :!: '\x03b9' :!: '\x0000') +foldMapping '\x1ff3' s = Yield '\x03c9' (s :*: '\x03b9' :*: '\x0000') -- GREEK SMALL LETTER OMEGA WITH OXIA AND YPOGEGRAMMENI -foldMapping '\x1ff4' s = Yield '\x03ce' (s :!: '\x03b9' :!: '\x0000') +foldMapping '\x1ff4' s = Yield '\x03ce' (s :*: '\x03b9' :*: '\x0000') -- GREEK SMALL LETTER OMEGA WITH PERISPOMENI -foldMapping '\x1ff6' s = Yield '\x03c9' (s :!: '\x0342' :!: '\x0000') +foldMapping '\x1ff6' s = Yield '\x03c9' (s :*: '\x0342' :*: '\x0000') -- GREEK SMALL LETTER OMEGA WITH PERISPOMENI AND YPOGEGRAMMENI -foldMapping '\x1ff7' s = Yield '\x03c9' (s :!: '\x0342' :!: '\x03b9') +foldMapping '\x1ff7' s = Yield '\x03c9' (s :*: '\x0342' :*: '\x03b9') -- GREEK CAPITAL LETTER OMEGA WITH PROSGEGRAMMENI -foldMapping '\x1ffc' s = Yield '\x03c9' (s :!: '\x03b9' :!: '\x0000') +foldMapping '\x1ffc' s = Yield '\x03c9' (s :*: '\x03b9' :*: '\x0000') -- LATIN SMALL LIGATURE FF -foldMapping '\xfb00' s = Yield '\x0066' (s :!: '\x0066' :!: '\x0000') +foldMapping '\xfb00' s = Yield '\x0066' (s :*: '\x0066' :*: '\x0000') -- LATIN SMALL LIGATURE FI -foldMapping '\xfb01' s = Yield '\x0066' (s :!: '\x0069' :!: '\x0000') +foldMapping '\xfb01' s = Yield '\x0066' (s :*: '\x0069' :*: '\x0000') -- LATIN SMALL LIGATURE FL -foldMapping '\xfb02' s = Yield '\x0066' (s :!: '\x006c' :!: '\x0000') +foldMapping '\xfb02' s = Yield '\x0066' (s :*: '\x006c' :*: '\x0000') -- LATIN SMALL LIGATURE FFI -foldMapping '\xfb03' s = Yield '\x0066' (s :!: '\x0066' :!: '\x0069') +foldMapping '\xfb03' s = Yield '\x0066' (s :*: '\x0066' :*: '\x0069') -- LATIN SMALL LIGATURE FFL -foldMapping '\xfb04' s = Yield '\x0066' (s :!: '\x0066' :!: '\x006c') +foldMapping '\xfb04' s = Yield '\x0066' (s :*: '\x0066' :*: '\x006c') -- LATIN SMALL LIGATURE LONG S T -foldMapping '\xfb05' s = Yield '\x0073' (s :!: '\x0074' :!: '\x0000') +foldMapping '\xfb05' s = Yield '\x0073' (s :*: '\x0074' :*: '\x0000') -- LATIN SMALL LIGATURE ST -foldMapping '\xfb06' s = Yield '\x0073' (s :!: '\x0074' :!: '\x0000') +foldMapping '\xfb06' s = Yield '\x0073' (s :*: '\x0074' :*: '\x0000') -- ARMENIAN SMALL LIGATURE MEN NOW -foldMapping '\xfb13' s = Yield '\x0574' (s :!: '\x0576' :!: '\x0000') +foldMapping '\xfb13' s = Yield '\x0574' (s :*: '\x0576' :*: '\x0000') -- ARMENIAN SMALL LIGATURE MEN ECH -foldMapping '\xfb14' s = Yield '\x0574' (s :!: '\x0565' :!: '\x0000') +foldMapping '\xfb14' s = Yield '\x0574' (s :*: '\x0565' :*: '\x0000') -- ARMENIAN SMALL LIGATURE MEN INI -foldMapping '\xfb15' s = Yield '\x0574' (s :!: '\x056b' :!: '\x0000') +foldMapping '\xfb15' s = Yield '\x0574' (s :*: '\x056b' :*: '\x0000') -- ARMENIAN SMALL LIGATURE VEW NOW -foldMapping '\xfb16' s = Yield '\x057e' (s :!: '\x0576' :!: '\x0000') +foldMapping '\xfb16' s = Yield '\x057e' (s :*: '\x0576' :*: '\x0000') -- ARMENIAN SMALL LIGATURE MEN XEH -foldMapping '\xfb17' s = Yield '\x0574' (s :!: '\x056d' :!: '\x0000') -foldMapping c s = Yield (toLower c) (s :!: '\0' :!: '\0') +foldMapping '\xfb17' s = Yield '\x0574' (s :*: '\x056d' :*: '\x0000') +foldMapping c s = Yield (toLower c) (s :*: '\0' :*: '\0') diff --git a/Data/Text/Fusion/Common.hs b/Data/Text/Fusion/Common.hs --- a/Data/Text/Fusion/Common.hs +++ b/Data/Text/Fusion/Common.hs @@ -29,6 +29,7 @@ , init , null , lengthI + , isSingleton -- * Transformations , map @@ -88,12 +89,9 @@ , filter -- * Indexing - , find + , findBy , indexI , findIndexI - , findIndicesI - , elemIndexI - , elemIndicesI , countCharI -- * Zipping and unzipping @@ -108,20 +106,19 @@ import Data.Int (Int64) import Data.Text.Fusion.Internal import Data.Text.Fusion.CaseMapping (foldMapping, lowerMapping, upperMapping) +import Data.Text.Fusion.Size singleton :: Char -> Stream Char -singleton c = Stream next False 1 -- HINT maybe too low +singleton c = Stream next False 1 where next False = Yield c True next True = Done {-# INLINE singleton #-} streamList :: [a] -> Stream a {-# INLINE [0] streamList #-} -streamList [] = empty -streamList s = Stream next s unknownLength +streamList s = Stream next s unknownSize where next [] = Done next (x:xs) = Yield x xs - unknownLength = 8 -- random HINT unstreamList :: Stream a -> [a] {-# INLINE [0] unstreamList #-} @@ -138,19 +135,19 @@ -- | /O(n)/ Adds a character to the front of a Stream Char. cons :: Char -> Stream Char -> Stream Char -cons w (Stream next0 s0 len) = Stream next (S2 :!: s0) (len+2) -- HINT maybe too high +cons w (Stream next0 s0 len) = Stream next (S2 :*: s0) (len+1) where {-# INLINE next #-} - next (S2 :!: s) = Yield w (S1 :!: s) - next (S1 :!: s) = case next0 s of + next (S2 :*: s) = Yield w (S1 :*: s) + next (S1 :*: s) = case next0 s of Done -> Done - Skip s' -> Skip (S1 :!: s') - Yield x s' -> Yield x (S1 :!: s') + Skip s' -> Skip (S1 :*: s') + Yield x s' -> Yield x (S1 :*: s') {-# INLINE [0] cons #-} -- | /O(n)/ Adds a character to the end of a stream. snoc :: Stream Char -> Char -> Stream Char -snoc (Stream next0 xs0 len) w = Stream next (J xs0) (len+2) -- HINT maybe too high +snoc (Stream next0 xs0 len) w = Stream next (J xs0) (len+1) where {-# INLINE next #-} next (J xs) = case next0 xs of @@ -193,7 +190,7 @@ uncons (Stream next s0 len) = loop_uncons s0 where loop_uncons !s = case next s of - Yield x s1 -> Just (x, Stream next s1 (len-1)) -- HINT maybe too high + Yield x s1 -> Just (x, Stream next s1 (len-1)) Skip s' -> loop_uncons s' Done -> Nothing {-# INLINE [0] uncons #-} @@ -216,34 +213,34 @@ -- | /O(1)/ Returns all characters after the head of a Stream Char, which must -- be non-empty. tail :: Stream Char -> Stream Char -tail (Stream next0 s0 len) = Stream next (False :!: s0) (len-1) -- HINT maybe too high +tail (Stream next0 s0 len) = Stream next (False :*: s0) (len-1) where {-# INLINE next #-} - next (False :!: s) = case next0 s of - Done -> emptyError "tail" - Skip s' -> Skip (False :!: s') - Yield _ s' -> Skip (True :!: s') - next (True :!: s) = case next0 s of - Done -> Done - Skip s' -> Skip (True :!: s') - Yield x s' -> Yield x (True :!: s') + next (False :*: s) = case next0 s of + Done -> emptyError "tail" + Skip s' -> Skip (False :*: s') + Yield _ s' -> Skip (True :*: s') + next (True :*: s) = case next0 s of + Done -> Done + Skip s' -> Skip (True :*: s') + Yield x s' -> Yield x (True :*: s') {-# INLINE [0] tail #-} -- | /O(1)/ Returns all but the last character of a Stream Char, which -- must be non-empty. init :: Stream Char -> Stream Char -init (Stream next0 s0 len) = Stream next (N :!: s0) (len-1) -- HINT maybe too high +init (Stream next0 s0 len) = Stream next (N :*: s0) (len-1) where {-# INLINE next #-} - next (N :!: s) = case next0 s of + next (N :*: s) = case next0 s of Done -> emptyError "init" - Skip s' -> Skip (N :!: s') - Yield x s' -> Skip (J x :!: s') - next (J x :!: s) = case next0 s of + Skip s' -> Skip (N :*: s') + Yield x s' -> Skip (J x :*: s') + next (J x :*: s) = case next0 s of Done -> Done - Skip s' -> Skip (J x :!: s') - Yield x' s' -> Yield x (J x' :!: s') + Skip s' -> Skip (J x :*: s') + Yield x' s' -> Yield x (J x' :*: s') {-# INLINE [0] init #-} -- | /O(1)/ Tests whether a Stream Char is empty or not. @@ -256,7 +253,7 @@ Skip s' -> loop_null s' {-# INLINE[0] null #-} --- | /O(n)/ Returns the number of characters in a text. +-- | /O(n)/ Returns the number of characters in a string. lengthI :: Integral a => Stream Char -> a lengthI (Stream next s0 _len) = loop_length 0 s0 where @@ -266,13 +263,25 @@ Yield _ s' -> loop_length (z + 1) s' {-# INLINE[0] lengthI #-} +-- | /O(n)/ Indicate whether a string contains exactly one element. +isSingleton :: Stream Char -> Bool +isSingleton (Stream next s0 _len) = loop 0 s0 + where + loop !z s = case next s of + Done -> z == (1::Int) + Skip s' -> loop z s' + Yield _ s' + | z >= 1 -> False + | otherwise -> loop (z+1) s' +{-# INLINE[0] isSingleton #-} + -- ---------------------------------------------------------------------------- -- * Stream transformations --- | /O(n)/ 'map' @f @xs is the Stream Char obtained by applying @f@ to each element of --- @xs@. +-- | /O(n)/ 'map' @f @xs is the Stream Char obtained by applying @f@ +-- to each element of @xs@. map :: (Char -> Char) -> Stream Char -> Stream Char -map f (Stream next0 s0 len) = Stream next s0 len -- HINT depends on f +map f (Stream next0 s0 len) = Stream next s0 len where {-# INLINE next #-} next !s = case next0 s of @@ -289,18 +298,18 @@ -- | /O(n)/ Take a character and place it between each of the -- characters of a 'Stream Char'. intersperse :: Char -> Stream Char -> Stream Char -intersperse c (Stream next0 s0 len) = Stream next (s0 :!: N :!: S1) len -- HINT maybe too low +intersperse c (Stream next0 s0 len) = Stream next (s0 :*: N :*: S1) len where {-# INLINE next #-} - next (s :!: N :!: S1) = case next0 s of + next (s :*: N :*: S1) = case next0 s of Done -> Done - Skip s' -> Skip (s' :!: N :!: S1) - Yield x s' -> Skip (s' :!: J x :!: S1) - next (s :!: J x :!: S1) = Yield x (s :!: N :!: S2) - next (s :!: N :!: S2) = case next0 s of + Skip s' -> Skip (s' :*: N :*: S1) + Yield x s' -> Skip (s' :*: J x :*: S1) + next (s :*: J x :*: S1) = Yield x (s :*: N :*: S2) + next (s :*: N :*: S2) = case next0 s of Done -> Done - Skip s' -> Skip (s' :!: N :!: S2) - Yield x s' -> Yield c (s' :!: J x :!: S1) + Skip s' -> Skip (s' :*: N :*: S2) + Yield x s' -> Yield c (s' :*: J x :*: S1) next _ = internalError "intersperse" {-# INLINE [0] intersperse #-} @@ -318,15 +327,15 @@ caseConvert :: (forall s. Char -> s -> Step (PairS (PairS s Char) Char) Char) -> Stream Char -> Stream Char -caseConvert remap (Stream next0 s0 len) = Stream next (s0 :!: '\0' :!: '\0') len +caseConvert remap (Stream next0 s0 len) = Stream next (s0 :*: '\0' :*: '\0') len where {-# INLINE next #-} - next (s :!: '\0' :!: _) = + next (s :*: '\0' :*: _) = case next0 s of Done -> Done - Skip s' -> Skip (s' :!: '\0' :!: '\0') + Skip s' -> Skip (s' :*: '\0' :*: '\0') Yield c s' -> remap c s' - next (s :!: a :!: b) = Yield a (s :!: b :!: '\0') + next (s :*: a :*: b) = Yield a (s :*: b :*: '\0') -- | /O(n)/ Convert a string to folded case. This function is mainly -- useful for performing caseless (or case insensitive) string @@ -364,18 +373,16 @@ {-# INLINE [0] toLower #-} justifyLeftI :: Integral a => a -> Char -> Stream Char -> Stream Char -justifyLeftI k c (Stream next0 s0 len) = Stream next (s0 :!: S1 :!: 0) newLen +justifyLeftI k c (Stream next0 s0 len) = + Stream next (s0 :*: S1 :*: 0) (larger (fromIntegral k) len) where - j = fromIntegral k - newLen | j > len = j - | otherwise = len - next (s :!: S1 :!: n) = + next (s :*: S1 :*: n) = case next0 s of - Done -> next (s :!: S2 :!: n) - Skip s' -> Skip (s' :!: S1 :!: n) - Yield x s' -> Yield x (s' :!: S1 :!: n+1) - next (s :!: S2 :!: n) - | n < k = Yield c (s :!: S2 :!: n+1) + Done -> next (s :*: S2 :*: n) + Skip s' -> Skip (s' :*: S1 :*: n) + Yield x s' -> Yield x (s' :*: S1 :*: n+1) + next (s :*: S2 :*: n) + | n < k = Yield c (s :*: S2 :*: n+1) | otherwise = Done {-# INLINE next #-} {-# INLINE [0] justifyLeftI #-} @@ -541,14 +548,14 @@ -- * Building streams scanl :: (Char -> Char -> Char) -> Char -> Stream Char -> Stream Char -scanl f z0 (Stream next0 s0 len) = Stream next (S1 :!: z0 :!: s0) (len+1) -- HINT maybe too low +scanl f z0 (Stream next0 s0 len) = Stream next (S1 :*: z0 :*: s0) (len+1) -- HINT maybe too low where {-# INLINE next #-} - next (S1 :!: z :!: s) = Yield z (S2 :!: z :!: s) - next (S2 :!: z :!: s) = case next0 s of + next (S1 :*: z :*: s) = Yield z (S2 :*: z :*: s) + next (S2 :*: z :*: s) = case next0 s of Yield x s' -> let !x' = f z x - in Yield x' (S2 :!: x' :!: s') - Skip s' -> Skip (S2 :!: z :!: s') + in Yield x' (S2 :*: x' :*: s') + Skip s' -> Skip (S2 :*: z :*: s') Done -> Done {-# INLINE [0] scanl #-} @@ -564,13 +571,13 @@ -- return a final value for the accumulator, because the nature of -- streams precludes it. mapAccumL :: (a -> b -> (a,b)) -> a -> Stream b -> Stream b -mapAccumL f z0 (Stream next0 s0 len) = Stream next (s0 :!: z0) len -- HINT depends on f +mapAccumL f z0 (Stream next0 s0 len) = Stream next (s0 :*: z0) len -- HINT depends on f where {-# INLINE next #-} - next (s :!: z) = case next0 s of + next (s :*: z) = case next0 s of Yield x s' -> let (z',y) = f z x - in Yield y (s' :!: z') - Skip s' -> Skip (s' :!: z) + in Yield y (s' :*: z') + Skip s' -> Skip (s' :*: z) Done -> Done {-# INLINE [0] mapAccumL #-} -} @@ -590,14 +597,14 @@ replicateI :: Int64 -> Stream Char -> Stream Char replicateI n (Stream next0 s0 len) = - Stream next (0 :!: s0) (max 0 (fromIntegral n * len)) + Stream next (0 :*: s0) (fromIntegral (max 0 n) * len) where - next (k :!: s) + next (k :*: s) | k >= n = Done | otherwise = case next0 s of - Done -> Skip (k+1 :!: s0) - Skip s' -> Skip (k :!: s') - Yield x s' -> Yield x (k :!: s') + Done -> Skip (k+1 :*: s0) + Skip s' -> Skip (k :*: s') + Yield x s' -> Yield x (k :*: s') {-# INLINE [0] replicateI #-} -- | /O(n)/, where @n@ is the length of the result. The unfoldr function @@ -621,13 +628,13 @@ -- 'unfoldr' when the length of the result is known. unfoldrNI :: Integral a => a -> (b -> Maybe (Char,b)) -> b -> Stream Char unfoldrNI n f s0 | n < 0 = empty - | otherwise = Stream next (0 :!: s0) (fromIntegral (n*2)) -- HINT maybe too high + | otherwise = Stream next (0 :*: s0) (fromIntegral (n*2)) -- HINT maybe too high where {-# INLINE next #-} - next (z :!: s) = case f s of + next (z :*: s) = case f s of Nothing -> Done Just (w, s') | z >= n -> Done - | otherwise -> Yield w ((z + 1) :!: s') + | otherwise -> Yield w ((z + 1) :*: s') {-# INLINE unfoldrNI #-} ------------------------------------------------------------------------------- @@ -637,14 +644,15 @@ -- stream of length @n@, or the stream itself if @n@ is greater than the -- length of the stream. take :: Integral a => a -> Stream Char -> Stream Char -take n0 (Stream next0 s0 len) = Stream next (n0 :!: s0) (min 0 (len - fromIntegral n0)) -- HINT maybe too high +take n0 (Stream next0 s0 len) = + Stream next (n0 :*: s0) (smaller len (fromIntegral (max 0 n0))) where {-# INLINE next #-} - next (n :!: s) | n <= 0 = Done + next (n :*: s) | n <= 0 = Done | otherwise = case next0 s of Done -> Done - Skip s' -> Skip (n :!: s') - Yield x s' -> Yield x ((n-1) :!: s') + Skip s' -> Skip (n :*: s') + Yield x s' -> Yield x ((n-1) :*: s') {-# INLINE [0] take #-} -- | /O(n)/ drop n, applied to a stream, returns the suffix of the @@ -652,19 +660,19 @@ -- length of the stream. drop :: Integral a => a -> Stream Char -> Stream Char drop n0 (Stream next0 s0 len) = - Stream next (J (max 0 n0) :!: s0) (len - fromIntegral n0) -- HINT maybe too high + Stream next (J n0 :*: s0) (len - fromIntegral (max 0 n0)) where {-# INLINE next #-} - next (J n :!: s) - | n == 0 = Skip (N :!: s) + next (J n :*: s) + | n <= 0 = Skip (N :*: s) | otherwise = case next0 s of Done -> Done - Skip s' -> Skip (J n :!: s') - Yield _ s' -> Skip (J (n-1) :!: s') - next (N :!: s) = case next0 s of + Skip s' -> Skip (J n :*: s') + Yield _ s' -> Skip (J (n-1) :*: s') + next (N :*: s) = case next0 s of Done -> Done - Skip s' -> Skip (N :!: s') - Yield x s' -> Yield x (N :!: s') + Skip s' -> Skip (N :*: s') + Yield x s' -> Yield x (N :*: s') {-# INLINE [0] drop #-} -- | takeWhile, applied to a predicate @p@ and a stream, returns the @@ -682,18 +690,18 @@ -- | dropWhile @p @xs returns the suffix remaining after takeWhile @p @xs. dropWhile :: (Char -> Bool) -> Stream Char -> Stream Char -dropWhile p (Stream next0 s0 len) = Stream next (S1 :!: s0) len -- HINT maybe too high +dropWhile p (Stream next0 s0 len) = Stream next (S1 :*: s0) len -- HINT maybe too high where {-# INLINE next #-} - next (S1 :!: s) = case next0 s of + next (S1 :*: s) = case next0 s of Done -> Done - Skip s' -> Skip (S1 :!: s') - Yield x s' | p x -> Skip (S1 :!: s') - | otherwise -> Yield x (S2 :!: s') - next (S2 :!: s) = case next0 s of + Skip s' -> Skip (S1 :*: s') + Yield x s' | p x -> Skip (S1 :*: s') + | otherwise -> Yield x (S2 :*: s') + next (S2 :*: s) = case next0 s of Done -> Done - Skip s' -> Skip (S2 :!: s') - Yield x s' -> Yield x (S2 :!: s') + Skip s' -> Skip (S2 :*: s') + Yield x s' -> Yield x (S2 :*: s') {-# INLINE [0] dropWhile #-} -- | /O(n)/ The 'isPrefixOf' function takes two 'Stream's and returns @@ -731,19 +739,19 @@ ------------------------------------------------------------------------------- -- ** Searching with a predicate --- | /O(n)/ The 'find' function takes a predicate and a stream, +-- | /O(n)/ The 'findBy' function takes a predicate and a stream, -- and returns the first element in matching the predicate, or 'Nothing' -- if there is no such element. -find :: (Char -> Bool) -> Stream Char -> Maybe Char -find p (Stream next s0 _len) = loop_find s0 +findBy :: (Char -> Bool) -> Stream Char -> Maybe Char +findBy p (Stream next s0 _len) = loop_find s0 where loop_find !s = case next s of Done -> Nothing Skip s' -> loop_find s' Yield x s' | p x -> Just x | otherwise -> loop_find s' -{-# INLINE [0] find #-} +{-# INLINE [0] findBy #-} -- | /O(n)/ Stream index (subscript) operator, starting from 0. indexI :: Integral a => Stream Char -> a -> Char @@ -805,40 +813,20 @@ -- | zipWith generalises 'zip' by zipping with the function given as -- the first argument, instead of a tupling function. zipWith :: (a -> a -> b) -> Stream a -> Stream a -> Stream b -zipWith f (Stream next0 sa0 len1) (Stream next1 sb0 len2) = Stream next (sa0 :!: sb0 :!: N) (min len1 len2) +zipWith f (Stream next0 sa0 len1) (Stream next1 sb0 len2) = + Stream next (sa0 :*: sb0 :*: N) (smaller len1 len2) where {-# INLINE next #-} - next (sa :!: sb :!: N) = case next0 sa of + next (sa :*: sb :*: N) = case next0 sa of Done -> Done - Skip sa' -> Skip (sa' :!: sb :!: N) - Yield a sa' -> Skip (sa' :!: sb :!: J a) + Skip sa' -> Skip (sa' :*: sb :*: N) + Yield a sa' -> Skip (sa' :*: sb :*: J a) - next (sa' :!: sb :!: J a) = case next1 sb of + next (sa' :*: sb :*: J a) = case next1 sb of Done -> Done - Skip sb' -> Skip (sa' :!: sb' :!: J a) - Yield b sb' -> Yield (f a b) (sa' :!: sb' :!: N) + Skip sb' -> Skip (sa' :*: sb' :*: J a) + Yield b sb' -> Yield (f a b) (sa' :*: sb' :*: N) {-# INLINE [0] zipWith #-} - --- | /O(n)/ The 'elemIndexI' function returns the index of the first --- element in the given stream which is equal to the query --- element, or 'Nothing' if there is no such element. -elemIndexI :: Integral a => Char -> Stream Char -> Maybe a -elemIndexI a s = case elemIndicesI a s of - (i:_) -> Just i - _ -> Nothing -{-# INLINE [0] elemIndexI #-} - --- | /O(n)/ The 'elemIndicesI' function returns the index of every --- element in the given stream which is equal to the query element. -elemIndicesI :: Integral a => Char -> Stream Char -> [a] -elemIndicesI a (Stream next s0 _len) = loop 0 s0 - where - loop !i !s = case next s of - Done -> [] - Skip s' -> loop i s' - Yield x s' | a == x -> i : loop (i+1) s' - | otherwise -> loop (i+1) s' -{-# INLINE [0] elemIndicesI #-} -- | /O(n)/ The 'countCharI' function returns the number of times the -- query element appears in the given stream. diff --git a/Data/Text/Fusion/Internal.hs b/Data/Text/Fusion/Internal.hs --- a/Data/Text/Fusion/Internal.hs +++ b/Data/Text/Fusion/Internal.hs @@ -25,6 +25,7 @@ , empty ) where +import Data.Text.Fusion.Size import Data.Word (Word8) -- | Specialised, strict Maybe-like type. @@ -37,8 +38,9 @@ data S s = S {-# UNPACK #-} !s {-# UNPACK #-} !M8 {-# UNPACK #-} !M8 {-# UNPACK #-} !M8 -infixl 2 :!: -data PairS a b = !a :!: !b +infixl 2 :*: +data PairS a b = {-# UNPACK #-} !a :*: {-# UNPACK #-} !b + deriving (Eq, Ord, Show) -- | Allow a function over a stream to switch between two states. data Switch = S1 | S2 @@ -47,12 +49,10 @@ | Skip !s | Yield !a !s -{- -instance Show a => Show (Step s a) +instance (Show a) => Show (Step s a) where show Done = "Done" show (Skip _) = "Skip" show (Yield x _) = "Yield " ++ show x --} instance (Eq a) => Eq (Stream a) where (==) = eq @@ -76,7 +76,7 @@ forall s. Stream (s -> Step s a) -- stepper function !s -- current state - {-# UNPACK #-}!Int -- length hint + {-# UNPACK #-} !Size -- size hint -- | /O(n)/ Determines if two streams are equal. eq :: (Eq a) => Stream a -> Stream a -> Bool diff --git a/Data/Text/Fusion/Size.hs b/Data/Text/Fusion/Size.hs new file mode 100644 --- /dev/null +++ b/Data/Text/Fusion/Size.hs @@ -0,0 +1,129 @@ +{-# OPTIONS_GHC -fno-warn-missing-methods #-} +-- | +-- Module : Data.Text.Fusion.Internal +-- Copyright : (c) Roman Leshchinskiy 2008, +-- (c) Bryan O'Sullivan 2009 +-- +-- License : BSD-style +-- Maintainer : bos@serpentine.com, rtharper@aftereternity.co.uk, +-- duncan@haskell.org +-- Stability : experimental +-- Portability : portable +-- +-- Size hints. + +module Data.Text.Fusion.Size + ( + Size + , exactSize + , maxSize + , unknownSize + , smaller + , larger + , toMax + , upperBound + , lowerBound + , isEmpty + ) where + +import Control.Exception (assert) + +data Size = Exact {-# UNPACK #-} !Int -- ^ Exact size. + | Max {-# UNPACK #-} !Int -- ^ Upper bound on size. + | Unknown -- ^ Unknown size. + deriving (Eq, Show) + +exactSize :: Int -> Size +exactSize n = assert (n >= 0) Exact n +{-# INLINE exactSize #-} + +maxSize :: Int -> Size +maxSize n = assert (n >= 0) Max n +{-# INLINE maxSize #-} + +unknownSize :: Size +unknownSize = Unknown +{-# INLINE unknownSize #-} + +instance Num Size where + (+) = addSize + (-) = subtractSize + (*) = mulSize + + fromInteger = f where f = Exact . fromInteger + {-# INLINE f #-} + +addSize :: Size -> Size -> Size +addSize (Exact m) (Exact n) = Exact (m+n) +addSize (Exact m) (Max n) = Max (m+n) +addSize (Max m) (Exact n) = Max (m+n) +addSize (Max m) (Max n) = Max (m+n) +addSize _ _ = Unknown +{-# INLINE addSize #-} + +subtractSize :: Size -> Size -> Size +subtractSize (Exact m) (Exact n) = Exact (max (m-n) 0) +subtractSize (Exact m) (Max _) = Max m +subtractSize (Max m) (Exact n) = Max (max (m-n) 0) +subtractSize a@(Max _) (Max _) = a +subtractSize a@(Max _) Unknown = a +subtractSize _ _ = Unknown +{-# INLINE subtractSize #-} + +mulSize :: Size -> Size -> Size +mulSize (Exact m) (Exact n) = Exact (m*n) +mulSize (Exact m) (Max n) = Max (m*n) +mulSize (Max m) (Exact n) = Max (m*n) +mulSize (Max m) (Max n) = Max (m*n) +mulSize _ _ = Unknown +{-# INLINE mulSize #-} + +-- | Minimum of two size hints. +smaller :: Size -> Size -> Size +smaller (Exact m) (Exact n) = Exact (m `min` n) +smaller (Exact m) (Max n) = Max (m `min` n) +smaller (Exact m) Unknown = Max m +smaller (Max m) (Exact n) = Max (m `min` n) +smaller (Max m) (Max n) = Max (m `min` n) +smaller a@(Max _) Unknown = a +smaller Unknown (Exact n) = Max n +smaller Unknown (Max n) = Max n +smaller Unknown Unknown = Unknown +{-# INLINE smaller #-} + +-- | Maximum of two size hints. +larger :: Size -> Size -> Size +larger (Exact m) (Exact n) = Exact (m `max` n) +larger a@(Exact m) b@(Max n) | m >= n = a + | otherwise = b +larger a@(Max m) b@(Exact n) | n >= m = b + | otherwise = a +larger (Max m) (Max n) = Max (m `max` n) +larger _ _ = Unknown +{-# INLINE larger #-} + +-- | Convert a size hint to an upper bound. +toMax :: Size -> Size +toMax (Exact n) = Max n +toMax a@(Max _) = a +toMax Unknown = Unknown +{-# INLINE toMax #-} + +-- | Compute the minimum size from a size hint. +lowerBound :: Size -> Int +lowerBound (Exact n) = n +lowerBound _ = 0 +{-# INLINE lowerBound #-} + +-- | Compute the maximum size from a size hint, if possible. +upperBound :: Int -> Size -> Int +upperBound _ (Exact n) = n +upperBound _ (Max n) = n +upperBound k _ = k +{-# INLINE upperBound #-} + +isEmpty :: Size -> Bool +isEmpty (Exact n) = n <= 0 +isEmpty (Max n) = n <= 0 +isEmpty _ = False +{-# INLINE isEmpty #-} diff --git a/Data/Text/Lazy.hs b/Data/Text/Lazy.hs --- a/Data/Text/Lazy.hs +++ b/Data/Text/Lazy.hs @@ -101,7 +101,6 @@ -- ** Generation and unfolding , replicate - , replicateChar , unfoldr , unfoldrN @@ -118,8 +117,9 @@ , stripStart , stripEnd , splitAt - , span + , spanBy , break + , breakBy , group , groupBy , inits @@ -128,9 +128,7 @@ -- ** Breaking into many substrings -- $split , split - , splitTimes - , splitTimesEnd - , splitWith + , splitBy , chunksOf -- , breakSubstring @@ -146,19 +144,15 @@ , isInfixOf -- * Searching - , elem , filter , find - , partition + , findBy + , partitionBy -- , findSubstring -- * Indexing , index - , findIndex - , findIndices - , elemIndex - , elemIndices , count -- * Zipping and unzipping @@ -171,7 +165,7 @@ import Prelude (Char, Bool(..), Int, Maybe(..), String, Eq(..), Ord(..), Read(..), Show(..), - (&&), (||), (+), (-), (.), ($), (++), + (&&), (+), (-), (.), ($), (++), div, flip, fromIntegral, not, otherwise) import qualified Prelude as P import Data.Int (Int64) @@ -180,11 +174,15 @@ import Data.Monoid (Monoid(..)) import Data.String (IsString(..)) import qualified Data.Text as T +import qualified Data.Text.Internal as T import qualified Data.Text.Fusion.Common as S import qualified Data.Text.Unsafe as T import qualified Data.Text.Lazy.Fusion as S +import Data.Text.Fusion.Internal (PairS(..)) import Data.Text.Lazy.Fusion (stream, unstream) -import Data.Text.Lazy.Internal +import Data.Text.Lazy.Internal (Text(..), chunk, empty, foldlChunks, foldrChunks) +import Data.Text.Internal (textP) +import Data.Text.Lazy.Search (indices) instance Eq Text where t1 == t2 = stream t1 == stream t2 @@ -334,6 +332,12 @@ S.null (stream t) = null t #-} +-- | /O(1)/ Tests whether a 'Text' contains exactly one character. +-- Subject to fusion. +isSingleton :: Text -> Bool +isSingleton = S.isSingleton . stream +{-# INLINE isSingleton #-} + -- | /O(1)/ Returns the last character of a 'Text', which must be -- non-empty. Subject to array fusion. last :: Text -> Char @@ -635,13 +639,16 @@ -- | /O(n*m)/ 'replicate' @n@ @t@ is a 'Text' consisting of the input -- @t@ repeated @n@ times. Subject to fusion. replicate :: Int64 -> Text -> Text -replicate n t = unstream (S.replicateI n (S.stream t)) +replicate n t + | isSingleton t = replicateChar n (head t) + | otherwise = unstream (S.replicateI n (S.stream t)) {-# INLINE replicate #-} -- | /O(n)/ 'replicateChar' @n@ @c@ is a 'Text' of length @n@ with @c@ the -- value of every element. Subject to fusion. replicateChar :: Int64 -> Char -> Text replicateChar n c = unstream (S.replicateCharI n c) +{-# INLINE replicateChar #-} {-# RULES "LAZY TEXT replicate/singleton -> replicateChar" [~1] forall n c. @@ -693,14 +700,14 @@ -- length of the 'Text'. Subject to fusion. drop :: Int64 -> Text -> Text drop i t0 - | i <= 0 = t0 + | i <= 0 = t0 | otherwise = drop' i t0 where drop' 0 ts = ts drop' _ Empty = Empty drop' n (Chunk t ts) - | n < len = Chunk (T.drop (fromIntegral n) t) ts + | n < len = Chunk (T.drop (fromIntegral n) t) ts | otherwise = drop' (n - len) ts - where len = fromIntegral (T.length t) + where len = fromIntegral (T.length t) {-# INLINE [1] drop #-} {-# RULES @@ -710,6 +717,21 @@ unstream (S.drop n (stream t)) = drop n t #-} +-- | /O(n)/ 'dropWords' @n@ returns the suffix with @n@ 'Word16' +-- values dropped, or the empty 'Text' if @n@ is greater than the +-- number of 'Word16' values present. +dropWords :: Int64 -> Text -> Text +dropWords i t0 + | i <= 0 = t0 + | otherwise = drop' i t0 + where drop' 0 ts = ts + drop' _ Empty = Empty + drop' n (Chunk (T.Text arr off len) ts) + | n < len' = chunk (textP arr (off+n') (len-n')) ts + | otherwise = drop' (n - len') ts + where len' = fromIntegral len + n' = fromIntegral n + -- | /O(n)/ 'takeWhile', applied to a predicate @p@ and a 'Text', returns -- the longest prefix (possibly empty) of elements that satisfy @p@. -- This function is subject to array fusion. @@ -808,10 +830,88 @@ in (Chunk t ts', ts'') where len = fromIntegral (T.length t) --- | /O(n)/ 'break' is like 'span', but the prefix returned is over +-- | /O(n)/ 'splitAtWord' @n t@ returns a strict pair whose first +-- element is a prefix of @t@ whose chunks contain @n@ 'Word16' +-- values, and whose second is the remainder of the string. +splitAtWord :: Int64 -> Text -> PairS Text Text +splitAtWord _ Empty = empty :*: empty +splitAtWord x (Chunk c@(T.Text arr off len) cs) + | y >= len = let h :*: t = splitAtWord (x-fromIntegral len) cs + in Chunk c h :*: t + | otherwise = chunk (textP arr off y) empty :*: + chunk (textP arr (off+y) (len-y)) cs + where y = fromIntegral x + +-- | /O(n+m)/ Find the first instance of @needle@ (which must be +-- non-'null') in @haystack@. The first element of the returned tuple +-- is the prefix of @haystack@ before @needle@ is matched. The second +-- is the remainder of @haystack@, starting with the match. +-- +-- Examples: +-- +-- > break "::" "a::b::c" ==> ("a", "::b::c") +-- > break "/" "foobar" ==> ("foobar", "") +-- +-- Laws: +-- +-- > append prefix match == haystack +-- > where (prefix, match) = break needle haystack +-- +-- If you need to break a string by a substring repeatedly (e.g. you +-- want to break on every instance of a substring), use 'find' +-- instead, as it has lower startup overhead. +-- +-- This function is strict in its first argument, and lazy in its +-- second. +-- +-- In (unlikely) bad cases, this function's time complexity degrades +-- towards /O(n*m)/. +break :: Text -> Text -> (Text, Text) +break pat src + | null pat = emptyError "break" + | otherwise = case indices pat src of + [] -> (src, empty) + (x:_) -> let h :*: t = splitAtWord x src + in (h, t) + +-- | /O(n+m)/ Find all non-overlapping instances of @needle@ in +-- @haystack@. The first element of the returned pair is the prefix +-- of @haystack@ prior to any matches of @needle@. The second is a +-- list of pairs. +-- +-- The first element of each pair in the list is a span from the +-- beginning of a match to the beginning of the next match, while the +-- second is a span from the beginning of the match to the end of the +-- input. +-- +-- Examples: +-- +-- > find "::" "" +-- > ==> ("", []) +-- > find "/" "a/b/c/d" +-- > ==> ("a", [("/b","/b/c/d"), ("/c","/c/d"), ("/d","/d")]) +-- +-- This function is strict in its first argument, and lazy in its +-- second. +-- +-- In (unlikely) bad cases, this function's time complexity degrades +-- towards /O(n*m)/. +find :: Text -> Text -> (Text, [(Text, Text)]) +find pat src + | null pat = emptyError "find" + | otherwise = case indices pat src of + [] -> (src, []) + (x:xs) -> let h :*: t = splitAtWord x src + in (h, go x xs t) + where + go !i (x:xs) cs = let h :*: t = splitAtWord (x-i) cs + in (h, cs) : go x xs t + go _ _ cs = [(cs,cs)] + +-- | /O(n)/ 'breakBy' is like 'spanBy', but the prefix returned is over -- elements that fail the predicate @p@. -break :: (Char -> Bool) -> Text -> (Text, Text) -break p t0 = break' t0 +breakBy :: (Char -> Bool) -> Text -> (Text, Text) +breakBy p t0 = break' t0 where break' Empty = (empty, empty) break' c@(Chunk t ts) = case T.findIndex p t of @@ -821,13 +921,13 @@ | otherwise -> let (a,b) = T.splitAt n t in (Chunk a Empty, Chunk b ts) --- | /O(n)/ 'span', applied to a predicate @p@ and text @t@, returns a --- pair whose first element is the longest prefix (possibly empty) of --- @t@ of elements that satisfy @p@, and whose second is the remainder --- of the list. -span :: (Char -> Bool) -> Text -> (Text, Text) -span p = break (not . p) -{-# INLINE span #-} +-- | /O(n)/ 'spanBy', applied to a predicate @p@ and text @t@, returns +-- a pair whose first element is the longest prefix (possibly empty) +-- of @t@ of elements that satisfy @p@, and whose second is the +-- remainder of the list. +spanBy :: (Char -> Bool) -> Text -> (Text, Text) +spanBy p = breakBy (not . p) +{-# INLINE spanBy #-} -- | The 'group' function takes a 'Text' and returns a list of 'Text's -- such that the concatenation of the result is equal to the argument. @@ -846,7 +946,7 @@ groupBy :: (Char -> Char -> Bool) -> Text -> [Text] groupBy _ Empty = [] groupBy eq (Chunk t ts) = cons x ys : groupBy eq zs - where (ys,zs) = span (eq x) xs + where (ys,zs) = spanBy (eq x) xs x = T.unsafeHead t xs = chunk (T.unsafeTail t) ts @@ -872,8 +972,8 @@ -- copies to create substrings; they just construct new 'Text's that -- are slices of the original. --- | /O(m)*O(n)/ Break a 'Text' into pieces separated by the first --- 'Text' argument, consuming the delimiter. An empty delimiter is +-- | /O(m+n)/ Break a 'Text' into pieces separated by the first +-- 'Text' argument, consuming the delimiter. An empty delimiter is -- invalid, and will cause an error to be raised. -- -- Examples: @@ -885,95 +985,47 @@ -- and -- -- > intercalate s . split s == id --- > split (singleton c) == splitWith (==c) +-- > split (singleton c) == splitBy (==c) +-- +-- This function is strict in its first argument, and lazy in its +-- second. +-- +-- In (unlikely) bad cases, this function's time complexity degrades +-- towards /O(n*m)/. split :: Text -- ^ Text to split on -> Text -- ^ Input text -> [Text] -split pat src0 - | l == 0 = emptyError "split" - | l == 1 = splitWith (== (head pat)) src0 - | otherwise = go src0 +split pat src + | null pat = emptyError "split" + | isSingleton pat = splitBy (== head pat) src + | otherwise = go 0 (indices pat src) src where - l = length pat - go src = search 0 src - where - search !n !s - | null s = [src] -- not found - | pat `isPrefixOf` s = take n src : go (drop l s) - | otherwise = search (n+1) (tail s) + go _ [] cs = [cs] + go !i (x:xs) cs = let h :*: t = splitAtWord (x-i) cs + in h : go (x+l) xs (dropWords l t) + l = foldlChunks (\a (T.Text _ _ b) -> a + fromIntegral b) 0 pat {-# INLINE [1] split #-} {-# RULES -"LAZY TEXT split/singleton -> splitWith/==" [~1] forall c t. - split (singleton c) t = splitWith (==c) t +"LAZY TEXT split/singleton -> splitBy/==" [~1] forall c t. + split (singleton c) t = splitBy (==c) t #-} --- | /O(m)*O(n)/ Break a 'Text' into pieces at most @k@ times, --- treating the first 'Text' argument as the delimiter to break on, --- and consuming the delimiter. The last element of the list contains --- the remaining text after the number of times to split has been --- reached. A value of zero or less for @k@ causes no splitting to --- occur. An empty delimiter is invalid, and will cause an error to --- be raised. --- --- Examples: --- --- > splitTimes 0 "//" "a//b//c" == ["a//b//c"] --- > splitTimes 2 ":" "a:b:c:d:e" == ["a","b","c:d:e"] --- > splitTimes 100 "???" "a????b" == ["a","?b"] --- --- and --- --- > intercalate s . splitTimes k s == id -splitTimes :: Int64 -- ^ Maximum number of times to split - -> Text -- ^ Text to split on - -> Text -- ^ Input text - -> [Text] -splitTimes k pat src0 - | k <= 0 = [src0] - | l == 0 = emptyError "splitTimes" - | otherwise = go k src0 - where - l = length pat - go !i src = search 0 src - where - search !n !s - | i == 0 || null s = [src] -- not found or limit reached - | pat `isPrefixOf` s = take n src : go (i-1) (drop l s) - | otherwise = search (n+1) (tail s) -{-# INLINE splitTimes #-} - --- | /O(m)*O(n)/ Break a 'Text' into pieces at most @k@ times, like --- 'splitTimes', but start from the end of the input and work towards --- the start. --- --- Examples: --- --- > splitTimes 2 "::" "a::b::c::d::e" == ["a","b","c::d::e"] --- > splitTimesEnd 2 "::" "a::b::c::d::e" == ["a::b::c","d","e"] -splitTimesEnd :: Int64 -- ^ Maximum number of times to split - -> Text -- ^ Text to split on - -> Text -- ^ Input text - -> [Text] -splitTimesEnd k pat src = - L.reverse . L.map reverse $ splitTimes k (reverse pat) (reverse src) -{-# INLINE splitTimesEnd #-} - -- | /O(n)/ Splits a 'Text' into components delimited by separators, -- where the predicate returns True for a separator element. The -- resulting components do not contain the separators. Two adjacent -- separators result in an empty component in the output. eg. -- --- > splitWith (=='a') "aabbaca" == ["","","bb","c",""] --- > splitWith (=='a') [] == [""] -splitWith :: (Char -> Bool) -> Text -> [Text] -splitWith _ Empty = [Empty] -splitWith p (Chunk t0 ts0) = comb [] (T.splitWith p t0) ts0 +-- > splitBy (=='a') "aabbaca" == ["","","bb","c",""] +-- > splitBy (=='a') [] == [""] +splitBy :: (Char -> Bool) -> Text -> [Text] +splitBy _ Empty = [Empty] +splitBy p (Chunk t0 ts0) = comb [] (T.splitBy p t0) ts0 where comb acc (s:[]) Empty = revChunks (s:acc) : [] - comb acc (s:[]) (Chunk t ts) = comb (s:acc) (T.splitWith p t) ts + comb acc (s:[]) (Chunk t ts) = comb (s:acc) (T.splitBy p t) ts comb acc (s:ss) ts = revChunks (s:acc) : comb [] ss ts - comb _ [] _ = impossibleError "splitWith" -{-# INLINE splitWith #-} + comb _ [] _ = impossibleError "splitBy" +{-# INLINE splitBy #-} -- | /O(n)/ Splits a 'Text' into components of length @k@. The last -- element may be shorter than the other chunks, depending on the @@ -993,14 +1045,14 @@ -- newline 'Char's. The resulting strings do not contain newlines. lines :: Text -> [Text] lines Empty = [] -lines t = let (l,t') = break ((==) '\n') t +lines t = let (l,t') = breakBy ((==) '\n') t in l : if null t' then [] else lines (tail t') -- | /O(n)/ Breaks a 'Text' up into a list of words, delimited by 'Char's -- representing white space. words :: Text -> [Text] -words = L.filter (not . null) . splitWith isSpace +words = L.filter (not . null) . splitBy isSpace {-# INLINE words #-} -- | /O(n)/ Joins lines, after appending a terminating newline to @@ -1044,18 +1096,26 @@ {-# INLINE isSuffixOf #-} -- TODO: a better implementation --- | /O(n)/ The 'isInfixOf' function takes two 'Text's and returns +-- | /O(n+m)/ The 'isInfixOf' function takes two 'Text's and returns -- 'True' iff the first is contained, wholly and intact, anywhere -- within the second. +-- +-- This function is strict in its first argument, and lazy in its +-- second. +-- +-- In (unlikely) bad cases, this function's time complexity degrades +-- towards /O(n*m)/. isInfixOf :: Text -> Text -> Bool -isInfixOf needle haystack = L.any (isPrefixOf needle) (tails haystack) -{-# INLINE isInfixOf #-} --- TODO: a better implementation +isInfixOf needle haystack + | null needle = True + | isSingleton needle = S.elem (head needle) . S.stream $ haystack + | otherwise = not . L.null . indices needle $ haystack +{-# INLINE [1] isInfixOf #-} --- | /O(n)/ 'elem' is the 'Text' membership predicate. -elem :: Char -> Text -> Bool -elem c t = S.elem c (stream t) -{-# INLINE elem #-} +{-# RULES +"LAZY TEXT isInfixOf/singleton -> S.elem/S.stream" [~1] forall n h. + isInfixOf (singleton n) h = S.elem n (S.stream h) + #-} -- | /O(n)/ 'filter', applied to a predicate and a 'Text', -- returns a 'Text' containing those characters that satisfy the @@ -1064,71 +1124,39 @@ filter p t = unstream (S.filter p (stream t)) {-# INLINE filter #-} --- | /O(n)/ The 'find' function takes a predicate and a 'Text', --- and returns the first element in matching the predicate, or 'Nothing' +-- | /O(n)/ The 'findBy' function takes a predicate and a 'Text', and +-- returns the first element in matching the predicate, or 'Nothing' -- if there is no such element. -find :: (Char -> Bool) -> Text -> Maybe Char -find p t = S.find p (stream t) -{-# INLINE find #-} +findBy :: (Char -> Bool) -> Text -> Maybe Char +findBy p t = S.findBy p (stream t) +{-# INLINE findBy #-} --- | /O(n)/ The 'partition' function takes a predicate and a 'Text', +-- | /O(n)/ The 'partitionBy' function takes a predicate and a 'Text', -- and returns the pair of 'Text's with elements which do and do not -- satisfy the predicate, respectively; i.e. -- --- > partition p t == (filter p t, filter (not . p) t) -partition :: (Char -> Bool) -> Text -> (Text, Text) -partition p t = (filter p t, filter (not . p) t) -{-# INLINE partition #-} +-- > partitionBy p t == (filter p t, filter (not . p) t) +partitionBy :: (Char -> Bool) -> Text -> (Text, Text) +partitionBy p t = (filter p t, filter (not . p) t) +{-# INLINE partitionBy #-} -- | /O(n)/ 'Text' index (subscript) operator, starting from 0. index :: Text -> Int64 -> Char index t n = S.index (stream t) n {-# INLINE index #-} --- | /O(n)/ The 'findIndex' function takes a predicate and a 'Text' --- and returns the index of the first element in the 'Text' satisfying --- the predicate. This function is subject to fusion. -findIndex :: (Char -> Bool) -> Text -> Maybe Int64 -findIndex p t = S.findIndex p (stream t) -{-# INLINE findIndex #-} - --- | The 'findIndices' function extends 'findIndex', by returning the --- indices of all elements satisfying the predicate, in ascending --- order. This function is subject to fusion. -findIndices :: (Char -> Bool) -> Text -> [Int64] -findIndices p t = S.findIndices p (stream t) -{-# INLINE findIndices #-} - --- | /O(n)/ The 'elemIndex' function returns the index of the first --- element in the given 'Text' which is equal to the query element, or --- 'Nothing' if there is no such element. This function is subject to --- fusion. -elemIndex :: Char -> Text -> Maybe Int64 -elemIndex c t = S.elemIndex c (stream t) -{-# INLINE elemIndex #-} - --- | /O(n)/ The 'elemIndices' function returns the index of every --- element in the given 'Text' which is equal to the query --- element. This function is subject to fusion. -elemIndices :: Char -> Text -> [Int64] -elemIndices c t = S.elemIndices c (stream t) -{-# INLINE elemIndices #-} - --- | /O(n*m)/ The 'count' function returns the number of times the +-- | /O(n+m)/ The 'count' function returns the number of times the -- query string appears in the given 'Text'. An empty query string is -- invalid, and will cause an error to be raised. +-- +-- In (unlikely) bad cases, this function's time complexity degrades +-- towards /O(n*m)/. count :: Text -> Text -> Int64 -count pat src0 - | l == 0 = emptyError "count" - | l == 1 = countChar (head pat) src0 - | otherwise = go 0 src0 - where - l = length pat - go !n src = search src - where - search s | null s = n - | pat `isPrefixOf` s = go (n+1) (drop l s) - | otherwise = search (tail s) +count pat src + | null pat = emptyError "count" + | otherwise = len (indices pat src) + where len [] = 0 + len (_:xs) = 1 + len xs {-# INLINE [1] count #-} {-# RULES diff --git a/Data/Text/Lazy/Encoding/Fusion.hs b/Data/Text/Lazy/Encoding/Fusion.hs --- a/Data/Text/Lazy/Encoding/Fusion.hs +++ b/Data/Text/Lazy/Encoding/Fusion.hs @@ -36,6 +36,7 @@ import Data.Text.Encoding.Error import Data.Text.Fusion (Step(..), Stream(..)) import Data.Text.Fusion.Internal (M(..), PairS(..), S(..)) +import Data.Text.Fusion.Size import Data.Text.UnsafeChar (unsafeChr8) import Data.Word (Word8) import qualified Data.Text.Encoding.Utf8 as U8 @@ -46,31 +47,28 @@ import Control.Exception (assert) import qualified Data.ByteString.Internal as B -unknownLength :: Int -unknownLength = 4 - -- | /O(n)/ Convert a lazy 'ByteString' into a 'Stream Char', using -- UTF-8 encoding. streamUtf8 :: OnDecodeError -> ByteString -> Stream Char -streamUtf8 onErr bs0 = Stream next (bs0 :!: empty :!: 0) unknownLength +streamUtf8 onErr bs0 = Stream next (bs0 :*: empty :*: 0) unknownSize where empty = S N N N N {-# INLINE next #-} - next (bs@(Chunk ps _) :!: S N _ _ _ :!: i) + next (bs@(Chunk ps _) :*: S N _ _ _ :*: i) | i < len && U8.validate1 a = - Yield (unsafeChr8 a) (bs :!: empty :!: i+1) + Yield (unsafeChr8 a) (bs :*: empty :*: i+1) | i + 1 < len && U8.validate2 a b = - Yield (U8.chr2 a b) (bs :!: empty :!: i+2) + Yield (U8.chr2 a b) (bs :*: empty :*: i+2) | i + 2 < len && U8.validate3 a b c = - Yield (U8.chr3 a b c) (bs :!: empty :!: i+3) + Yield (U8.chr3 a b c) (bs :*: empty :*: i+3) | i + 4 < len && U8.validate4 a b c d = - Yield (U8.chr4 a b c d) (bs :!: empty :!: i+4) + Yield (U8.chr4 a b c d) (bs :*: empty :*: i+4) where len = B.length ps a = B.unsafeIndex ps i b = B.unsafeIndex ps (i+1) c = B.unsafeIndex ps (i+2) d = B.unsafeIndex ps (i+3) - next st@(bs :!: s :!: i) = + next st@(bs :*: s :*: i) = case s of S (J a) N _ _ | U8.validate1 a -> Yield (unsafeChr8 a) es @@ -81,28 +79,28 @@ S (J a) (J b) (J c) (J d) | U8.validate4 a b c d -> Yield (U8.chr4 a b c d) es _ -> consume st - where es = bs :!: empty :!: i + where es = bs :*: empty :*: i {-# INLINE consume #-} - consume (bs@(Chunk ps rest) :!: s :!: i) - | i >= B.length ps = consume (rest :!: s :!: 0) + consume (bs@(Chunk ps rest) :*: s :*: i) + | i >= B.length ps = consume (rest :*: s :*: 0) | otherwise = case s of - S N _ _ _ -> next (bs :!: S x N N N :!: i+1) - S a N _ _ -> next (bs :!: S a x N N :!: i+1) - S a b N _ -> next (bs :!: S a b x N :!: i+1) - S a b c N -> next (bs :!: S a b c x :!: i+1) + S N _ _ _ -> next (bs :*: S x N N N :*: i+1) + S a N _ _ -> next (bs :*: S a x N N :*: i+1) + S a b N _ -> next (bs :*: S a b x N :*: i+1) + S a b c N -> next (bs :*: S a b c x :*: i+1) S (J a) b c d -> decodeError "streamUtf8" "UTF-8" onErr (Just a) - (bs :!: S b c d N :!: i+1) + (bs :*: S b c d N :*: i+1) where x = J (B.unsafeIndex ps i) - consume (Empty :!: S N _ _ _ :!: _) = Done + consume (Empty :*: S N _ _ _ :*: _) = Done consume st = decodeError "streamUtf8" "UTF-8" onErr Nothing st {-# INLINE [0] streamUtf8 #-} -- | /O(n)/ Convert a 'Stream' 'Word8' to a lazy 'ByteString'. unstreamChunks :: Int -> Stream Word8 -> ByteString -unstreamChunks chunkSize (Stream next s0 len0) = chunk s0 len0 +unstreamChunks chunkSize (Stream next s0 len0) = chunk s0 (upperBound 4 len0) where chunk s1 len1 = unsafePerformIO $ do - let len = min (max len1 unknownLength) chunkSize + let len = min len1 chunkSize mallocByteString len >>= loop len 0 s1 where loop !n !off !s fp = case next s of diff --git a/Data/Text/Lazy/Fusion.hs b/Data/Text/Lazy/Fusion.hs --- a/Data/Text/Lazy/Fusion.hs +++ b/Data/Text/Lazy/Fusion.hs @@ -18,16 +18,13 @@ , length , unfoldrN , index - , findIndex - , findIndices - , elemIndex - , elemIndices , countChar ) where import Prelude hiding (length) import qualified Data.Text.Fusion.Common as S import Data.Text.Fusion.Internal +import Data.Text.Fusion.Size (isEmpty) import Data.Text.Lazy.Internal import qualified Data.Text.Internal as I import qualified Data.Text.Array as A @@ -39,12 +36,12 @@ -- | /O(n)/ Convert a 'Text' into a 'Stream Char'. stream :: Text -> Stream Char -stream text = Stream next (text :!: 0) 4 -- random HINT +stream text = Stream next (text :*: 0) 4 -- random HINT where - next (Empty :!: _) = Done - next (txt@(Chunk t@(I.Text _ _ len) ts) :!: i) - | i >= len = next (ts :!: 0) - | otherwise = Yield c (txt :!: i+d) + next (Empty :*: _) = Done + next (txt@(Chunk t@(I.Text _ _ len) ts) :*: i) + | i >= len = next (ts :*: 0) + | otherwise = Yield c (txt :*: i+d) where (c,d) = iter t i {-# INLINE [0] stream #-} @@ -52,8 +49,8 @@ -- chunk size. unstreamChunks :: Int -> Stream Char -> Text unstreamChunks chunkSize (Stream next s0 len0) - | len0 == 0 = Empty - | otherwise = outer s0 + | isEmpty len0 = Empty + | otherwise = outer s0 where outer s = case next s of Done -> Empty @@ -104,33 +101,6 @@ index :: Stream Char -> Int64 -> Char index = S.indexI {-# INLINE [0] index #-} - --- | The 'findIndex' function takes a predicate and a stream and --- returns the index of the first element in the stream --- satisfying the predicate. -findIndex :: (Char -> Bool) -> Stream Char -> Maybe Int64 -findIndex = S.findIndexI -{-# INLINE [0] findIndex #-} - --- | The 'findIndices' function takes a predicate and a stream and --- returns all indices of the elements in the stream --- satisfying the predicate. -findIndices :: (Char -> Bool) -> Stream Char -> [Int64] -findIndices = S.findIndicesI -{-# INLINE [0] findIndices #-} - --- | /O(n)/ The 'elemIndex' function returns the index of the first --- element in the given stream which is equal to the query --- element, or 'Nothing' if there is no such element. -elemIndex :: Char -> Stream Char -> Maybe Int64 -elemIndex = S.elemIndexI -{-# INLINE [0] elemIndex #-} - --- | /O(n)/ The 'elemIndices' function returns the index of every --- element in the given stream which is equal to the query element. -elemIndices :: Char -> Stream Char -> [Int64] -elemIndices = S.elemIndicesI -{-# INLINE [0] elemIndices #-} -- | /O(n)/ The 'count' function returns the number of times the query -- element appears in the given stream. diff --git a/Data/Text/Lazy/Internal.hs b/Data/Text/Lazy/Internal.hs --- a/Data/Text/Lazy/Internal.hs +++ b/Data/Text/Lazy/Internal.hs @@ -91,6 +91,7 @@ defaultChunkSize :: Int defaultChunkSize = 32 * k - chunkOverhead where k = 1024 `div` sizeOf (undefined :: Word16) +{-# INLINE defaultChunkSize #-} -- | Currently set to 4k, less the memory management overhead. smallChunkSize :: Int diff --git a/Data/Text/Lazy/Search.hs b/Data/Text/Lazy/Search.hs new file mode 100644 --- /dev/null +++ b/Data/Text/Lazy/Search.hs @@ -0,0 +1,124 @@ +{-# LANGUAGE BangPatterns, ScopedTypeVariables #-} + +-- | +-- Module : Data.Text.Lazy.Search +-- Copyright : (c) Bryan O'Sullivan 2009 +-- +-- License : BSD-style +-- Maintainer : bos@serpentine.com, rtharper@aftereternity.co.uk, +-- duncan@haskell.org +-- Stability : experimental +-- Portability : GHC +-- +-- Fast substring search for lazy 'Text', based on work by Boyer, +-- Moore, Horspool, Sunday, and Lundh. Adapted from the strict +-- implementation. + +module Data.Text.Lazy.Search + ( + indices + ) where + +import qualified Data.Text.Array as A +import Data.Int (Int64) +import Data.Word (Word16, Word64) +import qualified Data.Text.Internal as T +import qualified Data.Text as T +import Data.Text.Fusion.Internal (PairS(..)) +import Data.Text.Lazy.Internal (Text(..), foldlChunks) +import Data.Bits ((.|.), (.&.)) +import Data.Text.UnsafeShift (shiftL) + +-- | /O(n+m)/ Find the offsets of all non-overlapping indices of +-- @needle@ within @haystack@. +-- +-- This function is strict in @needle@, and lazy (as far as possible) +-- in the chunks of @haystack@. +-- +-- In (unlikely) bad cases, this algorithm's complexity degrades +-- towards /O(n*m)/. +indices :: Text -- ^ Substring to search for (@needle@) + -> Text -- ^ Text to search in (@haystack@) + -> [Int64] +indices needle@(Chunk n ns) _haystack@(Chunk k ks) + | nlen <= 0 = [] + | nlen == 1 = scanOne (nindex 0) 0 k ks + | otherwise = scan 0 0 k ks + where + scan !g !i x@(T.Text _ _ l) xs + | i >= m = case xs of + Empty -> [] + Chunk y ys -> scan g (i-m) y ys + | lackingHay (i + nlen) x xs = [] + | c == z && candidateMatch 0 = g : scan (g+nlen) (i+nlen) x xs + | otherwise = scan (g+delta) (i+delta) x xs + where + m = fromIntegral l + c = hindex (i + nlast) + delta | nextInPattern = nlen + 1 + | c == z = skip + 1 + | otherwise = 1 + nextInPattern = mask .&. swizzle (hindex (i+nlen)) == 0 + candidateMatch !j + | j >= nlast = True + | hindex (i+j) /= nindex j = False + | otherwise = candidateMatch (j+1) + hindex = index x xs + nlen = wordLength needle + nlast = nlen - 1 + nindex = index n ns + z = foldlChunks fin 0 needle + where fin _ (T.Text farr foff flen) = A.unsafeIndex farr (foff+flen-1) + (mask :: Word64) :*: skip = buildTable n ns 0 0 0 (nlen-2) + swizzle w = 1 `shiftL` (fromIntegral w .&. 0x3f) + buildTable (T.Text xarr xoff xlen) xs = go + where + go !(g::Int64) !i !msk !skp + | i >= xlast = case xs of + Empty -> (msk .|. swizzle z) :*: skp + Chunk y ys -> buildTable y ys g 0 msk skp + | otherwise = go (g+1) (i+1) (msk .|. swizzle c) skp' + where c = A.unsafeIndex xarr (xoff+i) + skp' | c == z = nlen - fromIntegral g - 2 + | otherwise = skp + xlast = xlen - 1 + scanOne c i (T.Text oarr ooff olen) os = go 0 + where + go h | h >= olen = case os of + Empty -> [] + Chunk y ys -> scanOne c (i+fromIntegral olen) y ys + | on == c = i + fromIntegral h : go (h+1) + | otherwise = go (h+1) + where on = A.unsafeIndex oarr (ooff+h) + -- | Check whether an attempt to index into the haystack at the + -- given offset would fail. + lackingHay q = go 0 + where + go p (T.Text _ _ l) ps = p' < q && case ps of + Empty -> True + Chunk r rs -> go p' r rs + where p' = p + fromIntegral l +indices _ _ = [] + +-- | Fast index into a partly unpacked 'Text'. We take into account +-- the possibility that the caller might try to access one element +-- past the end. +index :: T.Text -> Text -> Int64 -> Word16 +index (T.Text arr off len) xs i + | j < len = A.unsafeIndex arr (off+j) + | otherwise = case xs of + Empty + -- out of bounds, but legal + | j == len -> 0 + -- should never happen, due to lackingHay above + | otherwise -> emptyError "index" + Chunk c cs -> index c cs (i-fromIntegral len) + where j = fromIntegral i + +-- | The number of 'Word16' values in a 'Text'. +wordLength :: Text -> Int64 +wordLength = foldlChunks sumLength 0 + where sumLength i (T.Text _ _ l) = i + fromIntegral l + +emptyError :: String -> a +emptyError fun = error ("Data.Text.Lazy.Search." ++ fun ++ ": empty input") diff --git a/Data/Text/Search.hs b/Data/Text/Search.hs new file mode 100644 --- /dev/null +++ b/Data/Text/Search.hs @@ -0,0 +1,85 @@ +{-# LANGUAGE BangPatterns, ScopedTypeVariables #-} + +-- | +-- Module : Data.Text.Search +-- Copyright : (c) Bryan O'Sullivan 2009 +-- +-- License : BSD-style +-- Maintainer : bos@serpentine.com, rtharper@aftereternity.co.uk, +-- duncan@haskell.org +-- Stability : experimental +-- Portability : GHC +-- +-- Fast substring search for 'Text', based on work by Boyer, Moore, +-- Horspool, Sunday, and Lundh. +-- +-- References: +-- +-- * R. S. Boyer, J. S. Moore: A Fast String Searching Algorithm. +-- Communications of the ACM, 20, 10, 762-772 (1977) +-- +-- * R. N. Horspool: Practical Fast Searching in Strings. Software - +-- Practice and Experience 10, 501-506 (1980) +-- +-- * D. M. Sunday: A Very Fast Substring Search Algorithm. +-- Communications of the ACM, 33, 8, 132-142 (1990) +-- +-- * F. Lundh: The Fast Search Algorithm. +-- (2006) + +module Data.Text.Search + ( + indices + ) where + +import qualified Data.Text.Array as A +import Data.Word (Word64) +import Data.Text.Internal (Text(..)) +import Data.Text.Fusion.Internal (PairS(..)) +import Data.Bits ((.|.), (.&.)) +import Data.Text.UnsafeShift (shiftL) + +-- | /O(n+m)/ Find the offsets of all non-overlapping indices of +-- @needle@ within @haystack@. +-- +-- In (unlikely) bad cases, this algorithm's complexity degrades +-- towards /O(n*m)/. +indices :: Text -- ^ Substring to search for (@needle@) + -> Text -- ^ Text to search in (@haystack@) + -> [Int] +indices _needle@(Text narr noff nlen) _haystack@(Text harr hoff hlen) + | nlen == 1 = scanOne (nindex 0) + | nlen <= 0 || ldiff < 0 = [] + | otherwise = scan 0 + where + ldiff = hlen - nlen + nlast = nlen - 1 + z = nindex nlast + nindex k = A.unsafeIndex narr (noff+k) + hindex k = A.unsafeIndex harr (hoff+k) + (mask :: Word64) :*: skip = buildTable 0 0 (nlen-2) + buildTable !i !msk !skp + | i >= nlast = (msk .|. swizzle z) :*: skp + | otherwise = buildTable (i+1) (msk .|. swizzle c) skp' + where c = nindex i + skp' | c == z = nlen - i - 2 + | otherwise = skp + swizzle k = 1 `shiftL` (fromIntegral k .&. 0x3f) + scan !i + | i > ldiff = [] + | c == z && candidateMatch 0 = i : scan (i + nlen) + | otherwise = scan (i + delta) + where c = hindex (i + nlast) + candidateMatch !j + | j >= nlast = True + | hindex (i+j) /= nindex j = False + | otherwise = candidateMatch (j+1) + delta | nextInPattern = nlen + 1 + | c == z = skip + 1 + | otherwise = 1 + nextInPattern = mask .&. swizzle (hindex (i+nlen)) == 0 + scanOne c = loop 0 + where loop !i | i >= hlen = [] + | hindex i == c = i : loop (i+1) + | otherwise = loop (i+1) +{-# INLINE indices #-} diff --git a/Data/Text/UnsafeShift.hs b/Data/Text/UnsafeShift.hs --- a/Data/Text/UnsafeShift.hs +++ b/Data/Text/UnsafeShift.hs @@ -45,14 +45,12 @@ {-# INLINE shiftR #-} shiftR (W32# x#) (I# i#) = W32# (x# `uncheckedShiftRL#` i#) -{- instance UnsafeShift Word64 where {-# INLINE shiftL #-} shiftL (W64# x#) (I# i#) = W64# (x# `uncheckedShiftL64#` i#) {-# INLINE shiftR #-} shiftR (W64# x#) (I# i#) = W64# (x# `uncheckedShiftRL64#` i#) --} instance UnsafeShift Int where {-# INLINE shiftL #-} diff --git a/README b/README --- a/README +++ b/README @@ -26,6 +26,12 @@ darcs get http://code.haskell.org/text +Reporting bugs, asking for enhancements +--------------------------------------- + +http://trac.haskell.org/text/ + + Authors ------- diff --git a/TODO b/TODO new file mode 100644 --- /dev/null +++ b/TODO @@ -0,0 +1,6 @@ +Normalization. + +Collation. + +Case conversion lacks locale sensitivity, so cannot handle a few +Turkic and Azeri code points completely correctly. diff --git a/scripts/ApiCompare.hs b/scripts/ApiCompare.hs new file mode 100644 --- /dev/null +++ b/scripts/ApiCompare.hs @@ -0,0 +1,24 @@ +-- This script compares the strict and lazy Text APIs to ensure that +-- they're reasonably in sync. + +{-# LANGUAGE OverloadedStrings #-} + +import qualified Data.Set as S +import qualified Data.Text as T +import System.Process + +main = do + let tidy pkg = (S.fromList . filter (T.isInfixOf "::") . T.lines . + T.replace "GHC.Int.Int64" "Int" . + T.replace "\n " "" . + T.replace (T.append (T.pack pkg) ".") "" . T.pack) `fmap` + readProcess "ghci" [] (":browse " ++ pkg) + let diff a b = mapM_ (putStrLn . (" "++) . T.unpack) . S.toList $ + S.difference a b + text <- tidy "Data.Text" + lazy <- tidy "Data.Text.Lazy" + putStrLn "In Data.Text:" + diff text lazy + putStrLn "" + putStrLn "In Data.Text.Lazy:" + diff lazy text diff --git a/scripts/Arsec.hs b/scripts/Arsec.hs new file mode 100644 --- /dev/null +++ b/scripts/Arsec.hs @@ -0,0 +1,49 @@ +module Arsec + ( + comment + , semi + , showC + , unichar + , unichars + , module Control.Applicative + , module Control.Monad + , module Data.Char + , module Text.ParserCombinators.Parsec.Char + , module Text.ParserCombinators.Parsec.Combinator + , module Text.ParserCombinators.Parsec.Error + , module Text.ParserCombinators.Parsec.Prim + ) where + +import Control.Monad +import Control.Applicative +import Data.Char +import Numeric +import Text.ParserCombinators.Parsec.Char hiding (lower, upper) +import Text.ParserCombinators.Parsec.Combinator hiding (optional) +import Text.ParserCombinators.Parsec.Error +import Text.ParserCombinators.Parsec.Prim hiding ((<|>), many) + +instance Applicative (GenParser s a) where + pure = return + (<*>) = ap + +instance Alternative (GenParser s a) where + empty = mzero + (<|>) = mplus + +unichar :: Parser Char +unichar = chr . fst . head . readHex <$> many1 hexDigit + +unichars :: Parser [Char] +unichars = manyTill (unichar <* spaces) semi + +semi :: Parser () +semi = char ';' *> spaces *> pure () + +comment :: Parser String +comment = (char '#' *> manyTill anyToken (char '\n')) <|> string "\n" + +showC :: Char -> String +showC c = "'\\x" ++ d ++ "'" + where h = showHex (ord c) "" + d = replicate (4 - length h) '0' ++ h diff --git a/scripts/CaseFolding.hs b/scripts/CaseFolding.hs new file mode 100644 --- /dev/null +++ b/scripts/CaseFolding.hs @@ -0,0 +1,42 @@ +-- This script processes the following source file: +-- +-- http://unicode.org/Public/UNIDATA/CaseFolding.txt + +module CaseFolding + ( + Fold(..) + , parseCF + , mapCF + ) where + +import Arsec + +data Fold = Fold { + code :: Char + , status :: Char + , mapping :: [Char] + , name :: String + } deriving (Eq, Ord, Show) + +entries :: Parser [Fold] +entries = many comment *> many (entry <* many comment) + where + entry = Fold <$> unichar <* semi + <*> oneOf "CFST" <* semi + <*> unichars + <*> (string "# " *> manyTill anyToken (char '\n')) + +parseCF :: FilePath -> IO (Either ParseError [Fold]) +parseCF name = parse entries name <$> readFile name + +mapCF :: [Fold] -> [String] +mapCF ms = typ ++ (map nice . filter p $ ms) ++ [last] + where + typ = ["foldMapping :: forall s. Char -> s -> Step (PairS (PairS s Char) Char) Char" + ,"{-# INLINE foldMapping #-}"] + last = "foldMapping c s = Yield (toLower c) (s :!: '\\0' :!: '\\0')" + nice c = "-- " ++ name c ++ "\n" ++ + "foldMapping " ++ showC (code c) ++ " s = Yield " ++ x ++ " (s :!: " ++ y ++ " :!: " ++ z ++ ")" + where [x,y,z] = (map showC . take 3) (mapping c ++ repeat '\0') + p f = status f `elem` "CF" && + mapping f /= [toLower (code f)] diff --git a/scripts/CaseFolding.txt b/scripts/CaseFolding.txt new file mode 100644 --- /dev/null +++ b/scripts/CaseFolding.txt @@ -0,0 +1,1196 @@ +# CaseFolding-5.1.0.txt +# Date: 2008-03-03, 21:57:14 GMT [MD] +# +# Unicode Character Database +# Copyright (c) 1991-2008 Unicode, Inc. +# For terms of use, see http://www.unicode.org/terms_of_use.html +# For documentation, see UCD.html +# +# Case Folding Properties +# +# This file is a supplement to the UnicodeData file. +# It provides a case folding mapping generated from the Unicode Character Database. +# If all characters are mapped according to the full mapping below, then +# case differences (according to UnicodeData.txt and SpecialCasing.txt) +# are eliminated. +# +# The data supports both implementations that require simple case foldings +# (where string lengths don't change), and implementations that allow full case folding +# (where string lengths may grow). Note that where they can be supported, the +# full case foldings are superior: for example, they allow "MASSE" and "Maße" to match. +# +# All code points not listed in this file map to themselves. +# +# NOTE: case folding does not preserve normalization formats! +# +# For information on case folding, including how to have case folding +# preserve normalization formats, see Section 3.13 Default Case Algorithms in +# The Unicode Standard, Version 5.0. +# +# ================================================================================ +# Format +# ================================================================================ +# The entries in this file are in the following machine-readable format: +# +# ; ; ; # +# +# The status field is: +# C: common case folding, common mappings shared by both simple and full mappings. +# F: full case folding, mappings that cause strings to grow in length. Multiple characters are separated by spaces. +# S: simple case folding, mappings to single characters where different from F. +# T: special case for uppercase I and dotted uppercase I +# - For non-Turkic languages, this mapping is normally not used. +# - For Turkic languages (tr, az), this mapping can be used instead of the normal mapping for these characters. +# Note that the Turkic mappings do not maintain canonical equivalence without additional processing. +# See the discussions of case mapping in the Unicode Standard for more information. +# +# Usage: +# A. To do a simple case folding, use the mappings with status C + S. +# B. To do a full case folding, use the mappings with status C + F. +# +# The mappings with status T can be used or omitted depending on the desired case-folding +# behavior. (The default option is to exclude them.) +# +# ================================================================= +# @missing 0000..10FFFF; +0041; C; 0061; # LATIN CAPITAL LETTER A +0042; C; 0062; # LATIN CAPITAL LETTER B +0043; C; 0063; # LATIN CAPITAL LETTER C +0044; C; 0064; # LATIN CAPITAL LETTER D +0045; C; 0065; # LATIN CAPITAL LETTER E +0046; C; 0066; # LATIN CAPITAL LETTER F +0047; C; 0067; # LATIN CAPITAL LETTER G +0048; C; 0068; # LATIN CAPITAL LETTER H +0049; C; 0069; # LATIN CAPITAL LETTER I +0049; T; 0131; # LATIN CAPITAL LETTER I +004A; C; 006A; # LATIN CAPITAL LETTER J +004B; C; 006B; # LATIN CAPITAL LETTER K +004C; C; 006C; # LATIN CAPITAL LETTER L +004D; C; 006D; # LATIN CAPITAL LETTER M +004E; C; 006E; # LATIN CAPITAL LETTER N +004F; C; 006F; # LATIN CAPITAL LETTER O +0050; C; 0070; # LATIN CAPITAL LETTER P +0051; C; 0071; # LATIN CAPITAL LETTER Q +0052; C; 0072; # LATIN CAPITAL LETTER R +0053; C; 0073; # LATIN CAPITAL LETTER S +0054; C; 0074; # LATIN CAPITAL LETTER T +0055; C; 0075; # LATIN CAPITAL LETTER U +0056; C; 0076; # LATIN CAPITAL LETTER V +0057; C; 0077; # LATIN CAPITAL LETTER W +0058; C; 0078; # LATIN CAPITAL LETTER X +0059; C; 0079; # LATIN CAPITAL LETTER Y +005A; C; 007A; # LATIN CAPITAL LETTER Z +00B5; C; 03BC; # MICRO SIGN +00C0; C; 00E0; # LATIN CAPITAL LETTER A WITH GRAVE +00C1; C; 00E1; # LATIN CAPITAL LETTER A WITH ACUTE +00C2; C; 00E2; # LATIN CAPITAL LETTER A WITH CIRCUMFLEX +00C3; C; 00E3; # LATIN CAPITAL LETTER A WITH TILDE +00C4; C; 00E4; # LATIN CAPITAL LETTER A WITH DIAERESIS +00C5; C; 00E5; # LATIN CAPITAL LETTER A WITH RING ABOVE +00C6; C; 00E6; # LATIN CAPITAL LETTER AE +00C7; C; 00E7; # LATIN CAPITAL LETTER C WITH CEDILLA +00C8; C; 00E8; # LATIN CAPITAL LETTER E WITH GRAVE +00C9; C; 00E9; # LATIN CAPITAL LETTER E WITH ACUTE +00CA; C; 00EA; # LATIN CAPITAL LETTER E WITH CIRCUMFLEX +00CB; C; 00EB; # LATIN CAPITAL LETTER E WITH DIAERESIS +00CC; C; 00EC; # LATIN CAPITAL LETTER I WITH GRAVE +00CD; C; 00ED; # LATIN CAPITAL LETTER I WITH ACUTE +00CE; C; 00EE; # LATIN CAPITAL LETTER I WITH CIRCUMFLEX +00CF; C; 00EF; # LATIN CAPITAL LETTER I WITH DIAERESIS +00D0; C; 00F0; # LATIN CAPITAL LETTER ETH +00D1; C; 00F1; # LATIN CAPITAL LETTER N WITH TILDE +00D2; C; 00F2; # LATIN CAPITAL LETTER O WITH GRAVE +00D3; C; 00F3; # LATIN CAPITAL LETTER O WITH ACUTE +00D4; C; 00F4; # LATIN CAPITAL LETTER O WITH CIRCUMFLEX +00D5; C; 00F5; # LATIN CAPITAL LETTER O WITH TILDE +00D6; C; 00F6; # LATIN CAPITAL LETTER O WITH DIAERESIS +00D8; C; 00F8; # LATIN CAPITAL LETTER O WITH STROKE +00D9; C; 00F9; # LATIN CAPITAL LETTER U WITH GRAVE +00DA; C; 00FA; # LATIN CAPITAL LETTER U WITH ACUTE +00DB; C; 00FB; # LATIN CAPITAL LETTER U WITH CIRCUMFLEX +00DC; C; 00FC; # LATIN CAPITAL LETTER U WITH DIAERESIS +00DD; C; 00FD; # LATIN CAPITAL LETTER Y WITH ACUTE +00DE; C; 00FE; # LATIN CAPITAL LETTER THORN +00DF; F; 0073 0073; # LATIN SMALL LETTER SHARP S +0100; C; 0101; # LATIN CAPITAL LETTER A WITH MACRON +0102; C; 0103; # LATIN CAPITAL LETTER A WITH BREVE +0104; C; 0105; # LATIN CAPITAL LETTER A WITH OGONEK +0106; C; 0107; # LATIN CAPITAL LETTER C WITH ACUTE +0108; C; 0109; # LATIN CAPITAL LETTER C WITH CIRCUMFLEX +010A; C; 010B; # LATIN CAPITAL LETTER C WITH DOT ABOVE +010C; C; 010D; # LATIN CAPITAL LETTER C WITH CARON +010E; C; 010F; # LATIN CAPITAL LETTER D WITH CARON +0110; C; 0111; # LATIN CAPITAL LETTER D WITH STROKE +0112; C; 0113; # LATIN CAPITAL LETTER E WITH MACRON +0114; C; 0115; # LATIN CAPITAL LETTER E WITH BREVE +0116; C; 0117; # LATIN CAPITAL LETTER E WITH DOT ABOVE +0118; C; 0119; # LATIN CAPITAL LETTER E WITH OGONEK +011A; C; 011B; # LATIN CAPITAL LETTER E WITH CARON +011C; C; 011D; # LATIN CAPITAL LETTER G WITH CIRCUMFLEX +011E; C; 011F; # LATIN CAPITAL LETTER G WITH BREVE +0120; C; 0121; # LATIN CAPITAL LETTER G WITH DOT ABOVE +0122; C; 0123; # LATIN CAPITAL LETTER G WITH CEDILLA +0124; C; 0125; # LATIN CAPITAL LETTER H WITH CIRCUMFLEX +0126; C; 0127; # LATIN CAPITAL LETTER H WITH STROKE +0128; C; 0129; # LATIN CAPITAL LETTER I WITH TILDE +012A; C; 012B; # LATIN CAPITAL LETTER I WITH MACRON +012C; C; 012D; # LATIN CAPITAL LETTER I WITH BREVE +012E; C; 012F; # LATIN CAPITAL LETTER I WITH OGONEK +0130; F; 0069 0307; # LATIN CAPITAL LETTER I WITH DOT ABOVE +0130; T; 0069; # LATIN CAPITAL LETTER I WITH DOT ABOVE +0132; C; 0133; # LATIN CAPITAL LIGATURE IJ +0134; C; 0135; # LATIN CAPITAL LETTER J WITH CIRCUMFLEX +0136; C; 0137; # LATIN CAPITAL LETTER K WITH CEDILLA +0139; C; 013A; # LATIN CAPITAL LETTER L WITH ACUTE +013B; C; 013C; # LATIN CAPITAL LETTER L WITH CEDILLA +013D; C; 013E; # LATIN CAPITAL LETTER L WITH CARON +013F; C; 0140; # LATIN CAPITAL LETTER L WITH MIDDLE DOT +0141; C; 0142; # LATIN CAPITAL LETTER L WITH STROKE +0143; C; 0144; # LATIN CAPITAL LETTER N WITH ACUTE +0145; C; 0146; # LATIN CAPITAL LETTER N WITH CEDILLA +0147; C; 0148; # LATIN CAPITAL LETTER N WITH CARON +0149; F; 02BC 006E; # LATIN SMALL LETTER N PRECEDED BY APOSTROPHE +014A; C; 014B; # LATIN CAPITAL LETTER ENG +014C; C; 014D; # LATIN CAPITAL LETTER O WITH MACRON +014E; C; 014F; # LATIN CAPITAL LETTER O WITH BREVE +0150; C; 0151; # LATIN CAPITAL LETTER O WITH DOUBLE ACUTE +0152; C; 0153; # LATIN CAPITAL LIGATURE OE +0154; C; 0155; # LATIN CAPITAL LETTER R WITH ACUTE +0156; C; 0157; # LATIN CAPITAL LETTER R WITH CEDILLA +0158; C; 0159; # LATIN CAPITAL LETTER R WITH CARON +015A; C; 015B; # LATIN CAPITAL LETTER S WITH ACUTE +015C; C; 015D; # LATIN CAPITAL LETTER S WITH CIRCUMFLEX +015E; C; 015F; # LATIN CAPITAL LETTER S WITH CEDILLA +0160; C; 0161; # LATIN CAPITAL LETTER S WITH CARON +0162; C; 0163; # LATIN CAPITAL LETTER T WITH CEDILLA +0164; C; 0165; # LATIN CAPITAL LETTER T WITH CARON +0166; C; 0167; # LATIN CAPITAL LETTER T WITH STROKE +0168; C; 0169; # LATIN CAPITAL LETTER U WITH TILDE +016A; C; 016B; # LATIN CAPITAL LETTER U WITH MACRON +016C; C; 016D; # LATIN CAPITAL LETTER U WITH BREVE +016E; C; 016F; # LATIN CAPITAL LETTER U WITH RING ABOVE +0170; C; 0171; # LATIN CAPITAL LETTER U WITH DOUBLE ACUTE +0172; C; 0173; # LATIN CAPITAL LETTER U WITH OGONEK +0174; C; 0175; # LATIN CAPITAL LETTER W WITH CIRCUMFLEX +0176; C; 0177; # LATIN CAPITAL LETTER Y WITH CIRCUMFLEX +0178; C; 00FF; # LATIN CAPITAL LETTER Y WITH DIAERESIS +0179; C; 017A; # LATIN CAPITAL LETTER Z WITH ACUTE +017B; C; 017C; # LATIN CAPITAL LETTER Z WITH DOT ABOVE +017D; C; 017E; # LATIN CAPITAL LETTER Z WITH CARON +017F; C; 0073; # LATIN SMALL LETTER LONG S +0181; C; 0253; # LATIN CAPITAL LETTER B WITH HOOK +0182; C; 0183; # LATIN CAPITAL LETTER B WITH TOPBAR +0184; C; 0185; # LATIN CAPITAL LETTER TONE SIX +0186; C; 0254; # LATIN CAPITAL LETTER OPEN O +0187; C; 0188; # LATIN CAPITAL LETTER C WITH HOOK +0189; C; 0256; # LATIN CAPITAL LETTER AFRICAN D +018A; C; 0257; # LATIN CAPITAL LETTER D WITH HOOK +018B; C; 018C; # LATIN CAPITAL LETTER D WITH TOPBAR +018E; C; 01DD; # LATIN CAPITAL LETTER REVERSED E +018F; C; 0259; # LATIN CAPITAL LETTER SCHWA +0190; C; 025B; # LATIN CAPITAL LETTER OPEN E +0191; C; 0192; # LATIN CAPITAL LETTER F WITH HOOK +0193; C; 0260; # LATIN CAPITAL LETTER G WITH HOOK +0194; C; 0263; # LATIN CAPITAL LETTER GAMMA +0196; C; 0269; # LATIN CAPITAL LETTER IOTA +0197; C; 0268; # LATIN CAPITAL LETTER I WITH STROKE +0198; C; 0199; # LATIN CAPITAL LETTER K WITH HOOK +019C; C; 026F; # LATIN CAPITAL LETTER TURNED M +019D; C; 0272; # LATIN CAPITAL LETTER N WITH LEFT HOOK +019F; C; 0275; # LATIN CAPITAL LETTER O WITH MIDDLE TILDE +01A0; C; 01A1; # LATIN CAPITAL LETTER O WITH HORN +01A2; C; 01A3; # LATIN CAPITAL LETTER OI +01A4; C; 01A5; # LATIN CAPITAL LETTER P WITH HOOK +01A6; C; 0280; # LATIN LETTER YR +01A7; C; 01A8; # LATIN CAPITAL LETTER TONE TWO +01A9; C; 0283; # LATIN CAPITAL LETTER ESH +01AC; C; 01AD; # LATIN CAPITAL LETTER T WITH HOOK +01AE; C; 0288; # LATIN CAPITAL LETTER T WITH RETROFLEX HOOK +01AF; C; 01B0; # LATIN CAPITAL LETTER U WITH HORN +01B1; C; 028A; # LATIN CAPITAL LETTER UPSILON +01B2; C; 028B; # LATIN CAPITAL LETTER V WITH HOOK +01B3; C; 01B4; # LATIN CAPITAL LETTER Y WITH HOOK +01B5; C; 01B6; # LATIN CAPITAL LETTER Z WITH STROKE +01B7; C; 0292; # LATIN CAPITAL LETTER EZH +01B8; C; 01B9; # LATIN CAPITAL LETTER EZH REVERSED +01BC; C; 01BD; # LATIN CAPITAL LETTER TONE FIVE +01C4; C; 01C6; # LATIN CAPITAL LETTER DZ WITH CARON +01C5; C; 01C6; # LATIN CAPITAL LETTER D WITH SMALL LETTER Z WITH CARON +01C7; C; 01C9; # LATIN CAPITAL LETTER LJ +01C8; C; 01C9; # LATIN CAPITAL LETTER L WITH SMALL LETTER J +01CA; C; 01CC; # LATIN CAPITAL LETTER NJ +01CB; C; 01CC; # LATIN CAPITAL LETTER N WITH SMALL LETTER J +01CD; C; 01CE; # LATIN CAPITAL LETTER A WITH CARON +01CF; C; 01D0; # LATIN CAPITAL LETTER I WITH CARON +01D1; C; 01D2; # LATIN CAPITAL LETTER O WITH CARON +01D3; C; 01D4; # LATIN CAPITAL LETTER U WITH CARON +01D5; C; 01D6; # LATIN CAPITAL LETTER U WITH DIAERESIS AND MACRON +01D7; C; 01D8; # LATIN CAPITAL LETTER U WITH DIAERESIS AND ACUTE +01D9; C; 01DA; # LATIN CAPITAL LETTER U WITH DIAERESIS AND CARON +01DB; C; 01DC; # LATIN CAPITAL LETTER U WITH DIAERESIS AND GRAVE +01DE; C; 01DF; # LATIN CAPITAL LETTER A WITH DIAERESIS AND MACRON +01E0; C; 01E1; # LATIN CAPITAL LETTER A WITH DOT ABOVE AND MACRON +01E2; C; 01E3; # LATIN CAPITAL LETTER AE WITH MACRON +01E4; C; 01E5; # LATIN CAPITAL LETTER G WITH STROKE +01E6; C; 01E7; # LATIN CAPITAL LETTER G WITH CARON +01E8; C; 01E9; # LATIN CAPITAL LETTER K WITH CARON +01EA; C; 01EB; # LATIN CAPITAL LETTER O WITH OGONEK +01EC; C; 01ED; # LATIN CAPITAL LETTER O WITH OGONEK AND MACRON +01EE; C; 01EF; # LATIN CAPITAL LETTER EZH WITH CARON +01F0; F; 006A 030C; # LATIN SMALL LETTER J WITH CARON +01F1; C; 01F3; # LATIN CAPITAL LETTER DZ +01F2; C; 01F3; # LATIN CAPITAL LETTER D WITH SMALL LETTER Z +01F4; C; 01F5; # LATIN CAPITAL LETTER G WITH ACUTE +01F6; C; 0195; # LATIN CAPITAL LETTER HWAIR +01F7; C; 01BF; # LATIN CAPITAL LETTER WYNN +01F8; C; 01F9; # LATIN CAPITAL LETTER N WITH GRAVE +01FA; C; 01FB; # LATIN CAPITAL LETTER A WITH RING ABOVE AND ACUTE +01FC; C; 01FD; # LATIN CAPITAL LETTER AE WITH ACUTE +01FE; C; 01FF; # LATIN CAPITAL LETTER O WITH STROKE AND ACUTE +0200; C; 0201; # LATIN CAPITAL LETTER A WITH DOUBLE GRAVE +0202; C; 0203; # LATIN CAPITAL LETTER A WITH INVERTED BREVE +0204; C; 0205; # LATIN CAPITAL LETTER E WITH DOUBLE GRAVE +0206; C; 0207; # LATIN CAPITAL LETTER E WITH INVERTED BREVE +0208; C; 0209; # LATIN CAPITAL LETTER I WITH DOUBLE GRAVE +020A; C; 020B; # LATIN CAPITAL LETTER I WITH INVERTED BREVE +020C; C; 020D; # LATIN CAPITAL LETTER O WITH DOUBLE GRAVE +020E; C; 020F; # LATIN CAPITAL LETTER O WITH INVERTED BREVE +0210; C; 0211; # LATIN CAPITAL LETTER R WITH DOUBLE GRAVE +0212; C; 0213; # LATIN CAPITAL LETTER R WITH INVERTED BREVE +0214; C; 0215; # LATIN CAPITAL LETTER U WITH DOUBLE GRAVE +0216; C; 0217; # LATIN CAPITAL LETTER U WITH INVERTED BREVE +0218; C; 0219; # LATIN CAPITAL LETTER S WITH COMMA BELOW +021A; C; 021B; # LATIN CAPITAL LETTER T WITH COMMA BELOW +021C; C; 021D; # LATIN CAPITAL LETTER YOGH +021E; C; 021F; # LATIN CAPITAL LETTER H WITH CARON +0220; C; 019E; # LATIN CAPITAL LETTER N WITH LONG RIGHT LEG +0222; C; 0223; # LATIN CAPITAL LETTER OU +0224; C; 0225; # LATIN CAPITAL LETTER Z WITH HOOK +0226; C; 0227; # LATIN CAPITAL LETTER A WITH DOT ABOVE +0228; C; 0229; # LATIN CAPITAL LETTER E WITH CEDILLA +022A; C; 022B; # LATIN CAPITAL LETTER O WITH DIAERESIS AND MACRON +022C; C; 022D; # LATIN CAPITAL LETTER O WITH TILDE AND MACRON +022E; C; 022F; # LATIN CAPITAL LETTER O WITH DOT ABOVE +0230; C; 0231; # LATIN CAPITAL LETTER O WITH DOT ABOVE AND MACRON +0232; C; 0233; # LATIN CAPITAL LETTER Y WITH MACRON +023A; C; 2C65; # LATIN CAPITAL LETTER A WITH STROKE +023B; C; 023C; # LATIN CAPITAL LETTER C WITH STROKE +023D; C; 019A; # LATIN CAPITAL LETTER L WITH BAR +023E; C; 2C66; # LATIN CAPITAL LETTER T WITH DIAGONAL STROKE +0241; C; 0242; # LATIN CAPITAL LETTER GLOTTAL STOP +0243; C; 0180; # LATIN CAPITAL LETTER B WITH STROKE +0244; C; 0289; # LATIN CAPITAL LETTER U BAR +0245; C; 028C; # LATIN CAPITAL LETTER TURNED V +0246; C; 0247; # LATIN CAPITAL LETTER E WITH STROKE +0248; C; 0249; # LATIN CAPITAL LETTER J WITH STROKE +024A; C; 024B; # LATIN CAPITAL LETTER SMALL Q WITH HOOK TAIL +024C; C; 024D; # LATIN CAPITAL LETTER R WITH STROKE +024E; C; 024F; # LATIN CAPITAL LETTER Y WITH STROKE +0345; C; 03B9; # COMBINING GREEK YPOGEGRAMMENI +0370; C; 0371; # GREEK CAPITAL LETTER HETA +0372; C; 0373; # GREEK CAPITAL LETTER ARCHAIC SAMPI +0376; C; 0377; # GREEK CAPITAL LETTER PAMPHYLIAN DIGAMMA +0386; C; 03AC; # GREEK CAPITAL LETTER ALPHA WITH TONOS +0388; C; 03AD; # GREEK CAPITAL LETTER EPSILON WITH TONOS +0389; C; 03AE; # GREEK CAPITAL LETTER ETA WITH TONOS +038A; C; 03AF; # GREEK CAPITAL LETTER IOTA WITH TONOS +038C; C; 03CC; # GREEK CAPITAL LETTER OMICRON WITH TONOS +038E; C; 03CD; # GREEK CAPITAL LETTER UPSILON WITH TONOS +038F; C; 03CE; # GREEK CAPITAL LETTER OMEGA WITH TONOS +0390; F; 03B9 0308 0301; # GREEK SMALL LETTER IOTA WITH DIALYTIKA AND TONOS +0391; C; 03B1; # GREEK CAPITAL LETTER ALPHA +0392; C; 03B2; # GREEK CAPITAL LETTER BETA +0393; C; 03B3; # GREEK CAPITAL LETTER GAMMA +0394; C; 03B4; # GREEK CAPITAL LETTER DELTA +0395; C; 03B5; # GREEK CAPITAL LETTER EPSILON +0396; C; 03B6; # GREEK CAPITAL LETTER ZETA +0397; C; 03B7; # GREEK CAPITAL LETTER ETA +0398; C; 03B8; # GREEK CAPITAL LETTER THETA +0399; C; 03B9; # GREEK CAPITAL LETTER IOTA +039A; C; 03BA; # GREEK CAPITAL LETTER KAPPA +039B; C; 03BB; # GREEK CAPITAL LETTER LAMDA +039C; C; 03BC; # GREEK CAPITAL LETTER MU +039D; C; 03BD; # GREEK CAPITAL LETTER NU +039E; C; 03BE; # GREEK CAPITAL LETTER XI +039F; C; 03BF; # GREEK CAPITAL LETTER OMICRON +03A0; C; 03C0; # GREEK CAPITAL LETTER PI +03A1; C; 03C1; # GREEK CAPITAL LETTER RHO +03A3; C; 03C3; # GREEK CAPITAL LETTER SIGMA +03A4; C; 03C4; # GREEK CAPITAL LETTER TAU +03A5; C; 03C5; # GREEK CAPITAL LETTER UPSILON +03A6; C; 03C6; # GREEK CAPITAL LETTER PHI +03A7; C; 03C7; # GREEK CAPITAL LETTER CHI +03A8; C; 03C8; # GREEK CAPITAL LETTER PSI +03A9; C; 03C9; # GREEK CAPITAL LETTER OMEGA +03AA; C; 03CA; # GREEK CAPITAL LETTER IOTA WITH DIALYTIKA +03AB; C; 03CB; # GREEK CAPITAL LETTER UPSILON WITH DIALYTIKA +03B0; F; 03C5 0308 0301; # GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND TONOS +03C2; C; 03C3; # GREEK SMALL LETTER FINAL SIGMA +03CF; C; 03D7; # GREEK CAPITAL KAI SYMBOL +03D0; C; 03B2; # GREEK BETA SYMBOL +03D1; C; 03B8; # GREEK THETA SYMBOL +03D5; C; 03C6; # GREEK PHI SYMBOL +03D6; C; 03C0; # GREEK PI SYMBOL +03D8; C; 03D9; # GREEK LETTER ARCHAIC KOPPA +03DA; C; 03DB; # GREEK LETTER STIGMA +03DC; C; 03DD; # GREEK LETTER DIGAMMA +03DE; C; 03DF; # GREEK LETTER KOPPA +03E0; C; 03E1; # GREEK LETTER SAMPI +03E2; C; 03E3; # COPTIC CAPITAL LETTER SHEI +03E4; C; 03E5; # COPTIC CAPITAL LETTER FEI +03E6; C; 03E7; # COPTIC CAPITAL LETTER KHEI +03E8; C; 03E9; # COPTIC CAPITAL LETTER HORI +03EA; C; 03EB; # COPTIC CAPITAL LETTER GANGIA +03EC; C; 03ED; # COPTIC CAPITAL LETTER SHIMA +03EE; C; 03EF; # COPTIC CAPITAL LETTER DEI +03F0; C; 03BA; # GREEK KAPPA SYMBOL +03F1; C; 03C1; # GREEK RHO SYMBOL +03F4; C; 03B8; # GREEK CAPITAL THETA SYMBOL +03F5; C; 03B5; # GREEK LUNATE EPSILON SYMBOL +03F7; C; 03F8; # GREEK CAPITAL LETTER SHO +03F9; C; 03F2; # GREEK CAPITAL LUNATE SIGMA SYMBOL +03FA; C; 03FB; # GREEK CAPITAL LETTER SAN +03FD; C; 037B; # GREEK CAPITAL REVERSED LUNATE SIGMA SYMBOL +03FE; C; 037C; # GREEK CAPITAL DOTTED LUNATE SIGMA SYMBOL +03FF; C; 037D; # GREEK CAPITAL REVERSED DOTTED LUNATE SIGMA SYMBOL +0400; C; 0450; # CYRILLIC CAPITAL LETTER IE WITH GRAVE +0401; C; 0451; # CYRILLIC CAPITAL LETTER IO +0402; C; 0452; # CYRILLIC CAPITAL LETTER DJE +0403; C; 0453; # CYRILLIC CAPITAL LETTER GJE +0404; C; 0454; # CYRILLIC CAPITAL LETTER UKRAINIAN IE +0405; C; 0455; # CYRILLIC CAPITAL LETTER DZE +0406; C; 0456; # CYRILLIC CAPITAL LETTER BYELORUSSIAN-UKRAINIAN I +0407; C; 0457; # CYRILLIC CAPITAL LETTER YI +0408; C; 0458; # CYRILLIC CAPITAL LETTER JE +0409; C; 0459; # CYRILLIC CAPITAL LETTER LJE +040A; C; 045A; # CYRILLIC CAPITAL LETTER NJE +040B; C; 045B; # CYRILLIC CAPITAL LETTER TSHE +040C; C; 045C; # CYRILLIC CAPITAL LETTER KJE +040D; C; 045D; # CYRILLIC CAPITAL LETTER I WITH GRAVE +040E; C; 045E; # CYRILLIC CAPITAL LETTER SHORT U +040F; C; 045F; # CYRILLIC CAPITAL LETTER DZHE +0410; C; 0430; # CYRILLIC CAPITAL LETTER A +0411; C; 0431; # CYRILLIC CAPITAL LETTER BE +0412; C; 0432; # CYRILLIC CAPITAL LETTER VE +0413; C; 0433; # CYRILLIC CAPITAL LETTER GHE +0414; C; 0434; # CYRILLIC CAPITAL LETTER DE +0415; C; 0435; # CYRILLIC CAPITAL LETTER IE +0416; C; 0436; # CYRILLIC CAPITAL LETTER ZHE +0417; C; 0437; # CYRILLIC CAPITAL LETTER ZE +0418; C; 0438; # CYRILLIC CAPITAL LETTER I +0419; C; 0439; # CYRILLIC CAPITAL LETTER SHORT I +041A; C; 043A; # CYRILLIC CAPITAL LETTER KA +041B; C; 043B; # CYRILLIC CAPITAL LETTER EL +041C; C; 043C; # CYRILLIC CAPITAL LETTER EM +041D; C; 043D; # CYRILLIC CAPITAL LETTER EN +041E; C; 043E; # CYRILLIC CAPITAL LETTER O +041F; C; 043F; # CYRILLIC CAPITAL LETTER PE +0420; C; 0440; # CYRILLIC CAPITAL LETTER ER +0421; C; 0441; # CYRILLIC CAPITAL LETTER ES +0422; C; 0442; # CYRILLIC CAPITAL LETTER TE +0423; C; 0443; # CYRILLIC CAPITAL LETTER U +0424; C; 0444; # CYRILLIC CAPITAL LETTER EF +0425; C; 0445; # CYRILLIC CAPITAL LETTER HA +0426; C; 0446; # CYRILLIC CAPITAL LETTER TSE +0427; C; 0447; # CYRILLIC CAPITAL LETTER CHE +0428; C; 0448; # CYRILLIC CAPITAL LETTER SHA +0429; C; 0449; # CYRILLIC CAPITAL LETTER SHCHA +042A; C; 044A; # CYRILLIC CAPITAL LETTER HARD SIGN +042B; C; 044B; # CYRILLIC CAPITAL LETTER YERU +042C; C; 044C; # CYRILLIC CAPITAL LETTER SOFT SIGN +042D; C; 044D; # CYRILLIC CAPITAL LETTER E +042E; C; 044E; # CYRILLIC CAPITAL LETTER YU +042F; C; 044F; # CYRILLIC CAPITAL LETTER YA +0460; C; 0461; # CYRILLIC CAPITAL LETTER OMEGA +0462; C; 0463; # CYRILLIC CAPITAL LETTER YAT +0464; C; 0465; # CYRILLIC CAPITAL LETTER IOTIFIED E +0466; C; 0467; # CYRILLIC CAPITAL LETTER LITTLE YUS +0468; C; 0469; # CYRILLIC CAPITAL LETTER IOTIFIED LITTLE YUS +046A; C; 046B; # CYRILLIC CAPITAL LETTER BIG YUS +046C; C; 046D; # CYRILLIC CAPITAL LETTER IOTIFIED BIG YUS +046E; C; 046F; # CYRILLIC CAPITAL LETTER KSI +0470; C; 0471; # CYRILLIC CAPITAL LETTER PSI +0472; C; 0473; # CYRILLIC CAPITAL LETTER FITA +0474; C; 0475; # CYRILLIC CAPITAL LETTER IZHITSA +0476; C; 0477; # CYRILLIC CAPITAL LETTER IZHITSA WITH DOUBLE GRAVE ACCENT +0478; C; 0479; # CYRILLIC CAPITAL LETTER UK +047A; C; 047B; # CYRILLIC CAPITAL LETTER ROUND OMEGA +047C; C; 047D; # CYRILLIC CAPITAL LETTER OMEGA WITH TITLO +047E; C; 047F; # CYRILLIC CAPITAL LETTER OT +0480; C; 0481; # CYRILLIC CAPITAL LETTER KOPPA +048A; C; 048B; # CYRILLIC CAPITAL LETTER SHORT I WITH TAIL +048C; C; 048D; # CYRILLIC CAPITAL LETTER SEMISOFT SIGN +048E; C; 048F; # CYRILLIC CAPITAL LETTER ER WITH TICK +0490; C; 0491; # CYRILLIC CAPITAL LETTER GHE WITH UPTURN +0492; C; 0493; # CYRILLIC CAPITAL LETTER GHE WITH STROKE +0494; C; 0495; # CYRILLIC CAPITAL LETTER GHE WITH MIDDLE HOOK +0496; C; 0497; # CYRILLIC CAPITAL LETTER ZHE WITH DESCENDER +0498; C; 0499; # CYRILLIC CAPITAL LETTER ZE WITH DESCENDER +049A; C; 049B; # CYRILLIC CAPITAL LETTER KA WITH DESCENDER +049C; C; 049D; # CYRILLIC CAPITAL LETTER KA WITH VERTICAL STROKE +049E; C; 049F; # CYRILLIC CAPITAL LETTER KA WITH STROKE +04A0; C; 04A1; # CYRILLIC CAPITAL LETTER BASHKIR KA +04A2; C; 04A3; # CYRILLIC CAPITAL LETTER EN WITH DESCENDER +04A4; C; 04A5; # CYRILLIC CAPITAL LIGATURE EN GHE +04A6; C; 04A7; # CYRILLIC CAPITAL LETTER PE WITH MIDDLE HOOK +04A8; C; 04A9; # CYRILLIC CAPITAL LETTER ABKHASIAN HA +04AA; C; 04AB; # CYRILLIC CAPITAL LETTER ES WITH DESCENDER +04AC; C; 04AD; # CYRILLIC CAPITAL LETTER TE WITH DESCENDER +04AE; C; 04AF; # CYRILLIC CAPITAL LETTER STRAIGHT U +04B0; C; 04B1; # CYRILLIC CAPITAL LETTER STRAIGHT U WITH STROKE +04B2; C; 04B3; # CYRILLIC CAPITAL LETTER HA WITH DESCENDER +04B4; C; 04B5; # CYRILLIC CAPITAL LIGATURE TE TSE +04B6; C; 04B7; # CYRILLIC CAPITAL LETTER CHE WITH DESCENDER +04B8; C; 04B9; # CYRILLIC CAPITAL LETTER CHE WITH VERTICAL STROKE +04BA; C; 04BB; # CYRILLIC CAPITAL LETTER SHHA +04BC; C; 04BD; # CYRILLIC CAPITAL LETTER ABKHASIAN CHE +04BE; C; 04BF; # CYRILLIC CAPITAL LETTER ABKHASIAN CHE WITH DESCENDER +04C0; C; 04CF; # CYRILLIC LETTER PALOCHKA +04C1; C; 04C2; # CYRILLIC CAPITAL LETTER ZHE WITH BREVE +04C3; C; 04C4; # CYRILLIC CAPITAL LETTER KA WITH HOOK +04C5; C; 04C6; # CYRILLIC CAPITAL LETTER EL WITH TAIL +04C7; C; 04C8; # CYRILLIC CAPITAL LETTER EN WITH HOOK +04C9; C; 04CA; # CYRILLIC CAPITAL LETTER EN WITH TAIL +04CB; C; 04CC; # CYRILLIC CAPITAL LETTER KHAKASSIAN CHE +04CD; C; 04CE; # CYRILLIC CAPITAL LETTER EM WITH TAIL +04D0; C; 04D1; # CYRILLIC CAPITAL LETTER A WITH BREVE +04D2; C; 04D3; # CYRILLIC CAPITAL LETTER A WITH DIAERESIS +04D4; C; 04D5; # CYRILLIC CAPITAL LIGATURE A IE +04D6; C; 04D7; # CYRILLIC CAPITAL LETTER IE WITH BREVE +04D8; C; 04D9; # CYRILLIC CAPITAL LETTER SCHWA +04DA; C; 04DB; # CYRILLIC CAPITAL LETTER SCHWA WITH DIAERESIS +04DC; C; 04DD; # CYRILLIC CAPITAL LETTER ZHE WITH DIAERESIS +04DE; C; 04DF; # CYRILLIC CAPITAL LETTER ZE WITH DIAERESIS +04E0; C; 04E1; # CYRILLIC CAPITAL LETTER ABKHASIAN DZE +04E2; C; 04E3; # CYRILLIC CAPITAL LETTER I WITH MACRON +04E4; C; 04E5; # CYRILLIC CAPITAL LETTER I WITH DIAERESIS +04E6; C; 04E7; # CYRILLIC CAPITAL LETTER O WITH DIAERESIS +04E8; C; 04E9; # CYRILLIC CAPITAL LETTER BARRED O +04EA; C; 04EB; # CYRILLIC CAPITAL LETTER BARRED O WITH DIAERESIS +04EC; C; 04ED; # CYRILLIC CAPITAL LETTER E WITH DIAERESIS +04EE; C; 04EF; # CYRILLIC CAPITAL LETTER U WITH MACRON +04F0; C; 04F1; # CYRILLIC CAPITAL LETTER U WITH DIAERESIS +04F2; C; 04F3; # CYRILLIC CAPITAL LETTER U WITH DOUBLE ACUTE +04F4; C; 04F5; # CYRILLIC CAPITAL LETTER CHE WITH DIAERESIS +04F6; C; 04F7; # CYRILLIC CAPITAL LETTER GHE WITH DESCENDER +04F8; C; 04F9; # CYRILLIC CAPITAL LETTER YERU WITH DIAERESIS +04FA; C; 04FB; # CYRILLIC CAPITAL LETTER GHE WITH STROKE AND HOOK +04FC; C; 04FD; # CYRILLIC CAPITAL LETTER HA WITH HOOK +04FE; C; 04FF; # CYRILLIC CAPITAL LETTER HA WITH STROKE +0500; C; 0501; # CYRILLIC CAPITAL LETTER KOMI DE +0502; C; 0503; # CYRILLIC CAPITAL LETTER KOMI DJE +0504; C; 0505; # CYRILLIC CAPITAL LETTER KOMI ZJE +0506; C; 0507; # CYRILLIC CAPITAL LETTER KOMI DZJE +0508; C; 0509; # CYRILLIC CAPITAL LETTER KOMI LJE +050A; C; 050B; # CYRILLIC CAPITAL LETTER KOMI NJE +050C; C; 050D; # CYRILLIC CAPITAL LETTER KOMI SJE +050E; C; 050F; # CYRILLIC CAPITAL LETTER KOMI TJE +0510; C; 0511; # CYRILLIC CAPITAL LETTER REVERSED ZE +0512; C; 0513; # CYRILLIC CAPITAL LETTER EL WITH HOOK +0514; C; 0515; # CYRILLIC CAPITAL LETTER LHA +0516; C; 0517; # CYRILLIC CAPITAL LETTER RHA +0518; C; 0519; # CYRILLIC CAPITAL LETTER YAE +051A; C; 051B; # CYRILLIC CAPITAL LETTER QA +051C; C; 051D; # CYRILLIC CAPITAL LETTER WE +051E; C; 051F; # CYRILLIC CAPITAL LETTER ALEUT KA +0520; C; 0521; # CYRILLIC CAPITAL LETTER EL WITH MIDDLE HOOK +0522; C; 0523; # CYRILLIC CAPITAL LETTER EN WITH MIDDLE HOOK +0531; C; 0561; # ARMENIAN CAPITAL LETTER AYB +0532; C; 0562; # ARMENIAN CAPITAL LETTER BEN +0533; C; 0563; # ARMENIAN CAPITAL LETTER GIM +0534; C; 0564; # ARMENIAN CAPITAL LETTER DA +0535; C; 0565; # ARMENIAN CAPITAL LETTER ECH +0536; C; 0566; # ARMENIAN CAPITAL LETTER ZA +0537; C; 0567; # ARMENIAN CAPITAL LETTER EH +0538; C; 0568; # ARMENIAN CAPITAL LETTER ET +0539; C; 0569; # ARMENIAN CAPITAL LETTER TO +053A; C; 056A; # ARMENIAN CAPITAL LETTER ZHE +053B; C; 056B; # ARMENIAN CAPITAL LETTER INI +053C; C; 056C; # ARMENIAN CAPITAL LETTER LIWN +053D; C; 056D; # ARMENIAN CAPITAL LETTER XEH +053E; C; 056E; # ARMENIAN CAPITAL LETTER CA +053F; C; 056F; # ARMENIAN CAPITAL LETTER KEN +0540; C; 0570; # ARMENIAN CAPITAL LETTER HO +0541; C; 0571; # ARMENIAN CAPITAL LETTER JA +0542; C; 0572; # ARMENIAN CAPITAL LETTER GHAD +0543; C; 0573; # ARMENIAN CAPITAL LETTER CHEH +0544; C; 0574; # ARMENIAN CAPITAL LETTER MEN +0545; C; 0575; # ARMENIAN CAPITAL LETTER YI +0546; C; 0576; # ARMENIAN CAPITAL LETTER NOW +0547; C; 0577; # ARMENIAN CAPITAL LETTER SHA +0548; C; 0578; # ARMENIAN CAPITAL LETTER VO +0549; C; 0579; # ARMENIAN CAPITAL LETTER CHA +054A; C; 057A; # ARMENIAN CAPITAL LETTER PEH +054B; C; 057B; # ARMENIAN CAPITAL LETTER JHEH +054C; C; 057C; # ARMENIAN CAPITAL LETTER RA +054D; C; 057D; # ARMENIAN CAPITAL LETTER SEH +054E; C; 057E; # ARMENIAN CAPITAL LETTER VEW +054F; C; 057F; # ARMENIAN CAPITAL LETTER TIWN +0550; C; 0580; # ARMENIAN CAPITAL LETTER REH +0551; C; 0581; # ARMENIAN CAPITAL LETTER CO +0552; C; 0582; # ARMENIAN CAPITAL LETTER YIWN +0553; C; 0583; # ARMENIAN CAPITAL LETTER PIWR +0554; C; 0584; # ARMENIAN CAPITAL LETTER KEH +0555; C; 0585; # ARMENIAN CAPITAL LETTER OH +0556; C; 0586; # ARMENIAN CAPITAL LETTER FEH +0587; F; 0565 0582; # ARMENIAN SMALL LIGATURE ECH YIWN +10A0; C; 2D00; # GEORGIAN CAPITAL LETTER AN +10A1; C; 2D01; # GEORGIAN CAPITAL LETTER BAN +10A2; C; 2D02; # GEORGIAN CAPITAL LETTER GAN +10A3; C; 2D03; # GEORGIAN CAPITAL LETTER DON +10A4; C; 2D04; # GEORGIAN CAPITAL LETTER EN +10A5; C; 2D05; # GEORGIAN CAPITAL LETTER VIN +10A6; C; 2D06; # GEORGIAN CAPITAL LETTER ZEN +10A7; C; 2D07; # GEORGIAN CAPITAL LETTER TAN +10A8; C; 2D08; # GEORGIAN CAPITAL LETTER IN +10A9; C; 2D09; # GEORGIAN CAPITAL LETTER KAN +10AA; C; 2D0A; # GEORGIAN CAPITAL LETTER LAS +10AB; C; 2D0B; # GEORGIAN CAPITAL LETTER MAN +10AC; C; 2D0C; # GEORGIAN CAPITAL LETTER NAR +10AD; C; 2D0D; # GEORGIAN CAPITAL LETTER ON +10AE; C; 2D0E; # GEORGIAN CAPITAL LETTER PAR +10AF; C; 2D0F; # GEORGIAN CAPITAL LETTER ZHAR +10B0; C; 2D10; # GEORGIAN CAPITAL LETTER RAE +10B1; C; 2D11; # GEORGIAN CAPITAL LETTER SAN +10B2; C; 2D12; # GEORGIAN CAPITAL LETTER TAR +10B3; C; 2D13; # GEORGIAN CAPITAL LETTER UN +10B4; C; 2D14; # GEORGIAN CAPITAL LETTER PHAR +10B5; C; 2D15; # GEORGIAN CAPITAL LETTER KHAR +10B6; C; 2D16; # GEORGIAN CAPITAL LETTER GHAN +10B7; C; 2D17; # GEORGIAN CAPITAL LETTER QAR +10B8; C; 2D18; # GEORGIAN CAPITAL LETTER SHIN +10B9; C; 2D19; # GEORGIAN CAPITAL LETTER CHIN +10BA; C; 2D1A; # GEORGIAN CAPITAL LETTER CAN +10BB; C; 2D1B; # GEORGIAN CAPITAL LETTER JIL +10BC; C; 2D1C; # GEORGIAN CAPITAL LETTER CIL +10BD; C; 2D1D; # GEORGIAN CAPITAL LETTER CHAR +10BE; C; 2D1E; # GEORGIAN CAPITAL LETTER XAN +10BF; C; 2D1F; # GEORGIAN CAPITAL LETTER JHAN +10C0; C; 2D20; # GEORGIAN CAPITAL LETTER HAE +10C1; C; 2D21; # GEORGIAN CAPITAL LETTER HE +10C2; C; 2D22; # GEORGIAN CAPITAL LETTER HIE +10C3; C; 2D23; # GEORGIAN CAPITAL LETTER WE +10C4; C; 2D24; # GEORGIAN CAPITAL LETTER HAR +10C5; C; 2D25; # GEORGIAN CAPITAL LETTER HOE +1E00; C; 1E01; # LATIN CAPITAL LETTER A WITH RING BELOW +1E02; C; 1E03; # LATIN CAPITAL LETTER B WITH DOT ABOVE +1E04; C; 1E05; # LATIN CAPITAL LETTER B WITH DOT BELOW +1E06; C; 1E07; # LATIN CAPITAL LETTER B WITH LINE BELOW +1E08; C; 1E09; # LATIN CAPITAL LETTER C WITH CEDILLA AND ACUTE +1E0A; C; 1E0B; # LATIN CAPITAL LETTER D WITH DOT ABOVE +1E0C; C; 1E0D; # LATIN CAPITAL LETTER D WITH DOT BELOW +1E0E; C; 1E0F; # LATIN CAPITAL LETTER D WITH LINE BELOW +1E10; C; 1E11; # LATIN CAPITAL LETTER D WITH CEDILLA +1E12; C; 1E13; # LATIN CAPITAL LETTER D WITH CIRCUMFLEX BELOW +1E14; C; 1E15; # LATIN CAPITAL LETTER E WITH MACRON AND GRAVE +1E16; C; 1E17; # LATIN CAPITAL LETTER E WITH MACRON AND ACUTE +1E18; C; 1E19; # LATIN CAPITAL LETTER E WITH CIRCUMFLEX BELOW +1E1A; C; 1E1B; # LATIN CAPITAL LETTER E WITH TILDE BELOW +1E1C; C; 1E1D; # LATIN CAPITAL LETTER E WITH CEDILLA AND BREVE +1E1E; C; 1E1F; # LATIN CAPITAL LETTER F WITH DOT ABOVE +1E20; C; 1E21; # LATIN CAPITAL LETTER G WITH MACRON +1E22; C; 1E23; # LATIN CAPITAL LETTER H WITH DOT ABOVE +1E24; C; 1E25; # LATIN CAPITAL LETTER H WITH DOT BELOW +1E26; C; 1E27; # LATIN CAPITAL LETTER H WITH DIAERESIS +1E28; C; 1E29; # LATIN CAPITAL LETTER H WITH CEDILLA +1E2A; C; 1E2B; # LATIN CAPITAL LETTER H WITH BREVE BELOW +1E2C; C; 1E2D; # LATIN CAPITAL LETTER I WITH TILDE BELOW +1E2E; C; 1E2F; # LATIN CAPITAL LETTER I WITH DIAERESIS AND ACUTE +1E30; C; 1E31; # LATIN CAPITAL LETTER K WITH ACUTE +1E32; C; 1E33; # LATIN CAPITAL LETTER K WITH DOT BELOW +1E34; C; 1E35; # LATIN CAPITAL LETTER K WITH LINE BELOW +1E36; C; 1E37; # LATIN CAPITAL LETTER L WITH DOT BELOW +1E38; C; 1E39; # LATIN CAPITAL LETTER L WITH DOT BELOW AND MACRON +1E3A; C; 1E3B; # LATIN CAPITAL LETTER L WITH LINE BELOW +1E3C; C; 1E3D; # LATIN CAPITAL LETTER L WITH CIRCUMFLEX BELOW +1E3E; C; 1E3F; # LATIN CAPITAL LETTER M WITH ACUTE +1E40; C; 1E41; # LATIN CAPITAL LETTER M WITH DOT ABOVE +1E42; C; 1E43; # LATIN CAPITAL LETTER M WITH DOT BELOW +1E44; C; 1E45; # LATIN CAPITAL LETTER N WITH DOT ABOVE +1E46; C; 1E47; # LATIN CAPITAL LETTER N WITH DOT BELOW +1E48; C; 1E49; # LATIN CAPITAL LETTER N WITH LINE BELOW +1E4A; C; 1E4B; # LATIN CAPITAL LETTER N WITH CIRCUMFLEX BELOW +1E4C; C; 1E4D; # LATIN CAPITAL LETTER O WITH TILDE AND ACUTE +1E4E; C; 1E4F; # LATIN CAPITAL LETTER O WITH TILDE AND DIAERESIS +1E50; C; 1E51; # LATIN CAPITAL LETTER O WITH MACRON AND GRAVE +1E52; C; 1E53; # LATIN CAPITAL LETTER O WITH MACRON AND ACUTE +1E54; C; 1E55; # LATIN CAPITAL LETTER P WITH ACUTE +1E56; C; 1E57; # LATIN CAPITAL LETTER P WITH DOT ABOVE +1E58; C; 1E59; # LATIN CAPITAL LETTER R WITH DOT ABOVE +1E5A; C; 1E5B; # LATIN CAPITAL LETTER R WITH DOT BELOW +1E5C; C; 1E5D; # LATIN CAPITAL LETTER R WITH DOT BELOW AND MACRON +1E5E; C; 1E5F; # LATIN CAPITAL LETTER R WITH LINE BELOW +1E60; C; 1E61; # LATIN CAPITAL LETTER S WITH DOT ABOVE +1E62; C; 1E63; # LATIN CAPITAL LETTER S WITH DOT BELOW +1E64; C; 1E65; # LATIN CAPITAL LETTER S WITH ACUTE AND DOT ABOVE +1E66; C; 1E67; # LATIN CAPITAL LETTER S WITH CARON AND DOT ABOVE +1E68; C; 1E69; # LATIN CAPITAL LETTER S WITH DOT BELOW AND DOT ABOVE +1E6A; C; 1E6B; # LATIN CAPITAL LETTER T WITH DOT ABOVE +1E6C; C; 1E6D; # LATIN CAPITAL LETTER T WITH DOT BELOW +1E6E; C; 1E6F; # LATIN CAPITAL LETTER T WITH LINE BELOW +1E70; C; 1E71; # LATIN CAPITAL LETTER T WITH CIRCUMFLEX BELOW +1E72; C; 1E73; # LATIN CAPITAL LETTER U WITH DIAERESIS BELOW +1E74; C; 1E75; # LATIN CAPITAL LETTER U WITH TILDE BELOW +1E76; C; 1E77; # LATIN CAPITAL LETTER U WITH CIRCUMFLEX BELOW +1E78; C; 1E79; # LATIN CAPITAL LETTER U WITH TILDE AND ACUTE +1E7A; C; 1E7B; # LATIN CAPITAL LETTER U WITH MACRON AND DIAERESIS +1E7C; C; 1E7D; # LATIN CAPITAL LETTER V WITH TILDE +1E7E; C; 1E7F; # LATIN CAPITAL LETTER V WITH DOT BELOW +1E80; C; 1E81; # LATIN CAPITAL LETTER W WITH GRAVE +1E82; C; 1E83; # LATIN CAPITAL LETTER W WITH ACUTE +1E84; C; 1E85; # LATIN CAPITAL LETTER W WITH DIAERESIS +1E86; C; 1E87; # LATIN CAPITAL LETTER W WITH DOT ABOVE +1E88; C; 1E89; # LATIN CAPITAL LETTER W WITH DOT BELOW +1E8A; C; 1E8B; # LATIN CAPITAL LETTER X WITH DOT ABOVE +1E8C; C; 1E8D; # LATIN CAPITAL LETTER X WITH DIAERESIS +1E8E; C; 1E8F; # LATIN CAPITAL LETTER Y WITH DOT ABOVE +1E90; C; 1E91; # LATIN CAPITAL LETTER Z WITH CIRCUMFLEX +1E92; C; 1E93; # LATIN CAPITAL LETTER Z WITH DOT BELOW +1E94; C; 1E95; # LATIN CAPITAL LETTER Z WITH LINE BELOW +1E96; F; 0068 0331; # LATIN SMALL LETTER H WITH LINE BELOW +1E97; F; 0074 0308; # LATIN SMALL LETTER T WITH DIAERESIS +1E98; F; 0077 030A; # LATIN SMALL LETTER W WITH RING ABOVE +1E99; F; 0079 030A; # LATIN SMALL LETTER Y WITH RING ABOVE +1E9A; F; 0061 02BE; # LATIN SMALL LETTER A WITH RIGHT HALF RING +1E9B; C; 1E61; # LATIN SMALL LETTER LONG S WITH DOT ABOVE +1E9E; F; 0073 0073; # LATIN CAPITAL LETTER SHARP S +1E9E; S; 00DF; # LATIN CAPITAL LETTER SHARP S +1EA0; C; 1EA1; # LATIN CAPITAL LETTER A WITH DOT BELOW +1EA2; C; 1EA3; # LATIN CAPITAL LETTER A WITH HOOK ABOVE +1EA4; C; 1EA5; # LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND ACUTE +1EA6; C; 1EA7; # LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND GRAVE +1EA8; C; 1EA9; # LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND HOOK ABOVE +1EAA; C; 1EAB; # LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND TILDE +1EAC; C; 1EAD; # LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND DOT BELOW +1EAE; C; 1EAF; # LATIN CAPITAL LETTER A WITH BREVE AND ACUTE +1EB0; C; 1EB1; # LATIN CAPITAL LETTER A WITH BREVE AND GRAVE +1EB2; C; 1EB3; # LATIN CAPITAL LETTER A WITH BREVE AND HOOK ABOVE +1EB4; C; 1EB5; # LATIN CAPITAL LETTER A WITH BREVE AND TILDE +1EB6; C; 1EB7; # LATIN CAPITAL LETTER A WITH BREVE AND DOT BELOW +1EB8; C; 1EB9; # LATIN CAPITAL LETTER E WITH DOT BELOW +1EBA; C; 1EBB; # LATIN CAPITAL LETTER E WITH HOOK ABOVE +1EBC; C; 1EBD; # LATIN CAPITAL LETTER E WITH TILDE +1EBE; C; 1EBF; # LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND ACUTE +1EC0; C; 1EC1; # LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND GRAVE +1EC2; C; 1EC3; # LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND HOOK ABOVE +1EC4; C; 1EC5; # LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND TILDE +1EC6; C; 1EC7; # LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND DOT BELOW +1EC8; C; 1EC9; # LATIN CAPITAL LETTER I WITH HOOK ABOVE +1ECA; C; 1ECB; # LATIN CAPITAL LETTER I WITH DOT BELOW +1ECC; C; 1ECD; # LATIN CAPITAL LETTER O WITH DOT BELOW +1ECE; C; 1ECF; # LATIN CAPITAL LETTER O WITH HOOK ABOVE +1ED0; C; 1ED1; # LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND ACUTE +1ED2; C; 1ED3; # LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND GRAVE +1ED4; C; 1ED5; # LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND HOOK ABOVE +1ED6; C; 1ED7; # LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND TILDE +1ED8; C; 1ED9; # LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND DOT BELOW +1EDA; C; 1EDB; # LATIN CAPITAL LETTER O WITH HORN AND ACUTE +1EDC; C; 1EDD; # LATIN CAPITAL LETTER O WITH HORN AND GRAVE +1EDE; C; 1EDF; # LATIN CAPITAL LETTER O WITH HORN AND HOOK ABOVE +1EE0; C; 1EE1; # LATIN CAPITAL LETTER O WITH HORN AND TILDE +1EE2; C; 1EE3; # LATIN CAPITAL LETTER O WITH HORN AND DOT BELOW +1EE4; C; 1EE5; # LATIN CAPITAL LETTER U WITH DOT BELOW +1EE6; C; 1EE7; # LATIN CAPITAL LETTER U WITH HOOK ABOVE +1EE8; C; 1EE9; # LATIN CAPITAL LETTER U WITH HORN AND ACUTE +1EEA; C; 1EEB; # LATIN CAPITAL LETTER U WITH HORN AND GRAVE +1EEC; C; 1EED; # LATIN CAPITAL LETTER U WITH HORN AND HOOK ABOVE +1EEE; C; 1EEF; # LATIN CAPITAL LETTER U WITH HORN AND TILDE +1EF0; C; 1EF1; # LATIN CAPITAL LETTER U WITH HORN AND DOT BELOW +1EF2; C; 1EF3; # LATIN CAPITAL LETTER Y WITH GRAVE +1EF4; C; 1EF5; # LATIN CAPITAL LETTER Y WITH DOT BELOW +1EF6; C; 1EF7; # LATIN CAPITAL LETTER Y WITH HOOK ABOVE +1EF8; C; 1EF9; # LATIN CAPITAL LETTER Y WITH TILDE +1EFA; C; 1EFB; # LATIN CAPITAL LETTER MIDDLE-WELSH LL +1EFC; C; 1EFD; # LATIN CAPITAL LETTER MIDDLE-WELSH V +1EFE; C; 1EFF; # LATIN CAPITAL LETTER Y WITH LOOP +1F08; C; 1F00; # GREEK CAPITAL LETTER ALPHA WITH PSILI +1F09; C; 1F01; # GREEK CAPITAL LETTER ALPHA WITH DASIA +1F0A; C; 1F02; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND VARIA +1F0B; C; 1F03; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND VARIA +1F0C; C; 1F04; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND OXIA +1F0D; C; 1F05; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND OXIA +1F0E; C; 1F06; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND PERISPOMENI +1F0F; C; 1F07; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND PERISPOMENI +1F18; C; 1F10; # GREEK CAPITAL LETTER EPSILON WITH PSILI +1F19; C; 1F11; # GREEK CAPITAL LETTER EPSILON WITH DASIA +1F1A; C; 1F12; # GREEK CAPITAL LETTER EPSILON WITH PSILI AND VARIA +1F1B; C; 1F13; # GREEK CAPITAL LETTER EPSILON WITH DASIA AND VARIA +1F1C; C; 1F14; # GREEK CAPITAL LETTER EPSILON WITH PSILI AND OXIA +1F1D; C; 1F15; # GREEK CAPITAL LETTER EPSILON WITH DASIA AND OXIA +1F28; C; 1F20; # GREEK CAPITAL LETTER ETA WITH PSILI +1F29; C; 1F21; # GREEK CAPITAL LETTER ETA WITH DASIA +1F2A; C; 1F22; # GREEK CAPITAL LETTER ETA WITH PSILI AND VARIA +1F2B; C; 1F23; # GREEK CAPITAL LETTER ETA WITH DASIA AND VARIA +1F2C; C; 1F24; # GREEK CAPITAL LETTER ETA WITH PSILI AND OXIA +1F2D; C; 1F25; # GREEK CAPITAL LETTER ETA WITH DASIA AND OXIA +1F2E; C; 1F26; # GREEK CAPITAL LETTER ETA WITH PSILI AND PERISPOMENI +1F2F; C; 1F27; # GREEK CAPITAL LETTER ETA WITH DASIA AND PERISPOMENI +1F38; C; 1F30; # GREEK CAPITAL LETTER IOTA WITH PSILI +1F39; C; 1F31; # GREEK CAPITAL LETTER IOTA WITH DASIA +1F3A; C; 1F32; # GREEK CAPITAL LETTER IOTA WITH PSILI AND VARIA +1F3B; C; 1F33; # GREEK CAPITAL LETTER IOTA WITH DASIA AND VARIA +1F3C; C; 1F34; # GREEK CAPITAL LETTER IOTA WITH PSILI AND OXIA +1F3D; C; 1F35; # GREEK CAPITAL LETTER IOTA WITH DASIA AND OXIA +1F3E; C; 1F36; # GREEK CAPITAL LETTER IOTA WITH PSILI AND PERISPOMENI +1F3F; C; 1F37; # GREEK CAPITAL LETTER IOTA WITH DASIA AND PERISPOMENI +1F48; C; 1F40; # GREEK CAPITAL LETTER OMICRON WITH PSILI +1F49; C; 1F41; # GREEK CAPITAL LETTER OMICRON WITH DASIA +1F4A; C; 1F42; # GREEK CAPITAL LETTER OMICRON WITH PSILI AND VARIA +1F4B; C; 1F43; # GREEK CAPITAL LETTER OMICRON WITH DASIA AND VARIA +1F4C; C; 1F44; # GREEK CAPITAL LETTER OMICRON WITH PSILI AND OXIA +1F4D; C; 1F45; # GREEK CAPITAL LETTER OMICRON WITH DASIA AND OXIA +1F50; F; 03C5 0313; # GREEK SMALL LETTER UPSILON WITH PSILI +1F52; F; 03C5 0313 0300; # GREEK SMALL LETTER UPSILON WITH PSILI AND VARIA +1F54; F; 03C5 0313 0301; # GREEK SMALL LETTER UPSILON WITH PSILI AND OXIA +1F56; F; 03C5 0313 0342; # GREEK SMALL LETTER UPSILON WITH PSILI AND PERISPOMENI +1F59; C; 1F51; # GREEK CAPITAL LETTER UPSILON WITH DASIA +1F5B; C; 1F53; # GREEK CAPITAL LETTER UPSILON WITH DASIA AND VARIA +1F5D; C; 1F55; # GREEK CAPITAL LETTER UPSILON WITH DASIA AND OXIA +1F5F; C; 1F57; # GREEK CAPITAL LETTER UPSILON WITH DASIA AND PERISPOMENI +1F68; C; 1F60; # GREEK CAPITAL LETTER OMEGA WITH PSILI +1F69; C; 1F61; # GREEK CAPITAL LETTER OMEGA WITH DASIA +1F6A; C; 1F62; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND VARIA +1F6B; C; 1F63; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND VARIA +1F6C; C; 1F64; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND OXIA +1F6D; C; 1F65; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND OXIA +1F6E; C; 1F66; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND PERISPOMENI +1F6F; C; 1F67; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND PERISPOMENI +1F80; F; 1F00 03B9; # GREEK SMALL LETTER ALPHA WITH PSILI AND YPOGEGRAMMENI +1F81; F; 1F01 03B9; # GREEK SMALL LETTER ALPHA WITH DASIA AND YPOGEGRAMMENI +1F82; F; 1F02 03B9; # GREEK SMALL LETTER ALPHA WITH PSILI AND VARIA AND YPOGEGRAMMENI +1F83; F; 1F03 03B9; # GREEK SMALL LETTER ALPHA WITH DASIA AND VARIA AND YPOGEGRAMMENI +1F84; F; 1F04 03B9; # GREEK SMALL LETTER ALPHA WITH PSILI AND OXIA AND YPOGEGRAMMENI +1F85; F; 1F05 03B9; # GREEK SMALL LETTER ALPHA WITH DASIA AND OXIA AND YPOGEGRAMMENI +1F86; F; 1F06 03B9; # GREEK SMALL LETTER ALPHA WITH PSILI AND PERISPOMENI AND YPOGEGRAMMENI +1F87; F; 1F07 03B9; # GREEK SMALL LETTER ALPHA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI +1F88; F; 1F00 03B9; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND PROSGEGRAMMENI +1F88; S; 1F80; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND PROSGEGRAMMENI +1F89; F; 1F01 03B9; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND PROSGEGRAMMENI +1F89; S; 1F81; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND PROSGEGRAMMENI +1F8A; F; 1F02 03B9; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND VARIA AND PROSGEGRAMMENI +1F8A; S; 1F82; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND VARIA AND PROSGEGRAMMENI +1F8B; F; 1F03 03B9; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND VARIA AND PROSGEGRAMMENI +1F8B; S; 1F83; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND VARIA AND PROSGEGRAMMENI +1F8C; F; 1F04 03B9; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND OXIA AND PROSGEGRAMMENI +1F8C; S; 1F84; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND OXIA AND PROSGEGRAMMENI +1F8D; F; 1F05 03B9; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND OXIA AND PROSGEGRAMMENI +1F8D; S; 1F85; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND OXIA AND PROSGEGRAMMENI +1F8E; F; 1F06 03B9; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI +1F8E; S; 1F86; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI +1F8F; F; 1F07 03B9; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI +1F8F; S; 1F87; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI +1F90; F; 1F20 03B9; # GREEK SMALL LETTER ETA WITH PSILI AND YPOGEGRAMMENI +1F91; F; 1F21 03B9; # GREEK SMALL LETTER ETA WITH DASIA AND YPOGEGRAMMENI +1F92; F; 1F22 03B9; # GREEK SMALL LETTER ETA WITH PSILI AND VARIA AND YPOGEGRAMMENI +1F93; F; 1F23 03B9; # GREEK SMALL LETTER ETA WITH DASIA AND VARIA AND YPOGEGRAMMENI +1F94; F; 1F24 03B9; # GREEK SMALL LETTER ETA WITH PSILI AND OXIA AND YPOGEGRAMMENI +1F95; F; 1F25 03B9; # GREEK SMALL LETTER ETA WITH DASIA AND OXIA AND YPOGEGRAMMENI +1F96; F; 1F26 03B9; # GREEK SMALL LETTER ETA WITH PSILI AND PERISPOMENI AND YPOGEGRAMMENI +1F97; F; 1F27 03B9; # GREEK SMALL LETTER ETA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI +1F98; F; 1F20 03B9; # GREEK CAPITAL LETTER ETA WITH PSILI AND PROSGEGRAMMENI +1F98; S; 1F90; # GREEK CAPITAL LETTER ETA WITH PSILI AND PROSGEGRAMMENI +1F99; F; 1F21 03B9; # GREEK CAPITAL LETTER ETA WITH DASIA AND PROSGEGRAMMENI +1F99; S; 1F91; # GREEK CAPITAL LETTER ETA WITH DASIA AND PROSGEGRAMMENI +1F9A; F; 1F22 03B9; # GREEK CAPITAL LETTER ETA WITH PSILI AND VARIA AND PROSGEGRAMMENI +1F9A; S; 1F92; # GREEK CAPITAL LETTER ETA WITH PSILI AND VARIA AND PROSGEGRAMMENI +1F9B; F; 1F23 03B9; # GREEK CAPITAL LETTER ETA WITH DASIA AND VARIA AND PROSGEGRAMMENI +1F9B; S; 1F93; # GREEK CAPITAL LETTER ETA WITH DASIA AND VARIA AND PROSGEGRAMMENI +1F9C; F; 1F24 03B9; # GREEK CAPITAL LETTER ETA WITH PSILI AND OXIA AND PROSGEGRAMMENI +1F9C; S; 1F94; # GREEK CAPITAL LETTER ETA WITH PSILI AND OXIA AND PROSGEGRAMMENI +1F9D; F; 1F25 03B9; # GREEK CAPITAL LETTER ETA WITH DASIA AND OXIA AND PROSGEGRAMMENI +1F9D; S; 1F95; # GREEK CAPITAL LETTER ETA WITH DASIA AND OXIA AND PROSGEGRAMMENI +1F9E; F; 1F26 03B9; # GREEK CAPITAL LETTER ETA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI +1F9E; S; 1F96; # GREEK CAPITAL LETTER ETA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI +1F9F; F; 1F27 03B9; # GREEK CAPITAL LETTER ETA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI +1F9F; S; 1F97; # GREEK CAPITAL LETTER ETA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI +1FA0; F; 1F60 03B9; # GREEK SMALL LETTER OMEGA WITH PSILI AND YPOGEGRAMMENI +1FA1; F; 1F61 03B9; # GREEK SMALL LETTER OMEGA WITH DASIA AND YPOGEGRAMMENI +1FA2; F; 1F62 03B9; # GREEK SMALL LETTER OMEGA WITH PSILI AND VARIA AND YPOGEGRAMMENI +1FA3; F; 1F63 03B9; # GREEK SMALL LETTER OMEGA WITH DASIA AND VARIA AND YPOGEGRAMMENI +1FA4; F; 1F64 03B9; # GREEK SMALL LETTER OMEGA WITH PSILI AND OXIA AND YPOGEGRAMMENI +1FA5; F; 1F65 03B9; # GREEK SMALL LETTER OMEGA WITH DASIA AND OXIA AND YPOGEGRAMMENI +1FA6; F; 1F66 03B9; # GREEK SMALL LETTER OMEGA WITH PSILI AND PERISPOMENI AND YPOGEGRAMMENI +1FA7; F; 1F67 03B9; # GREEK SMALL LETTER OMEGA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI +1FA8; F; 1F60 03B9; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND PROSGEGRAMMENI +1FA8; S; 1FA0; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND PROSGEGRAMMENI +1FA9; F; 1F61 03B9; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND PROSGEGRAMMENI +1FA9; S; 1FA1; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND PROSGEGRAMMENI +1FAA; F; 1F62 03B9; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND VARIA AND PROSGEGRAMMENI +1FAA; S; 1FA2; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND VARIA AND PROSGEGRAMMENI +1FAB; F; 1F63 03B9; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND VARIA AND PROSGEGRAMMENI +1FAB; S; 1FA3; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND VARIA AND PROSGEGRAMMENI +1FAC; F; 1F64 03B9; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND OXIA AND PROSGEGRAMMENI +1FAC; S; 1FA4; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND OXIA AND PROSGEGRAMMENI +1FAD; F; 1F65 03B9; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND OXIA AND PROSGEGRAMMENI +1FAD; S; 1FA5; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND OXIA AND PROSGEGRAMMENI +1FAE; F; 1F66 03B9; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI +1FAE; S; 1FA6; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI +1FAF; F; 1F67 03B9; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI +1FAF; S; 1FA7; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI +1FB2; F; 1F70 03B9; # GREEK SMALL LETTER ALPHA WITH VARIA AND YPOGEGRAMMENI +1FB3; F; 03B1 03B9; # GREEK SMALL LETTER ALPHA WITH YPOGEGRAMMENI +1FB4; F; 03AC 03B9; # GREEK SMALL LETTER ALPHA WITH OXIA AND YPOGEGRAMMENI +1FB6; F; 03B1 0342; # GREEK SMALL LETTER ALPHA WITH PERISPOMENI +1FB7; F; 03B1 0342 03B9; # GREEK SMALL LETTER ALPHA WITH PERISPOMENI AND YPOGEGRAMMENI +1FB8; C; 1FB0; # GREEK CAPITAL LETTER ALPHA WITH VRACHY +1FB9; C; 1FB1; # GREEK CAPITAL LETTER ALPHA WITH MACRON +1FBA; C; 1F70; # GREEK CAPITAL LETTER ALPHA WITH VARIA +1FBB; C; 1F71; # GREEK CAPITAL LETTER ALPHA WITH OXIA +1FBC; F; 03B1 03B9; # GREEK CAPITAL LETTER ALPHA WITH PROSGEGRAMMENI +1FBC; S; 1FB3; # GREEK CAPITAL LETTER ALPHA WITH PROSGEGRAMMENI +1FBE; C; 03B9; # GREEK PROSGEGRAMMENI +1FC2; F; 1F74 03B9; # GREEK SMALL LETTER ETA WITH VARIA AND YPOGEGRAMMENI +1FC3; F; 03B7 03B9; # GREEK SMALL LETTER ETA WITH YPOGEGRAMMENI +1FC4; F; 03AE 03B9; # GREEK SMALL LETTER ETA WITH OXIA AND YPOGEGRAMMENI +1FC6; F; 03B7 0342; # GREEK SMALL LETTER ETA WITH PERISPOMENI +1FC7; F; 03B7 0342 03B9; # GREEK SMALL LETTER ETA WITH PERISPOMENI AND YPOGEGRAMMENI +1FC8; C; 1F72; # GREEK CAPITAL LETTER EPSILON WITH VARIA +1FC9; C; 1F73; # GREEK CAPITAL LETTER EPSILON WITH OXIA +1FCA; C; 1F74; # GREEK CAPITAL LETTER ETA WITH VARIA +1FCB; C; 1F75; # GREEK CAPITAL LETTER ETA WITH OXIA +1FCC; F; 03B7 03B9; # GREEK CAPITAL LETTER ETA WITH PROSGEGRAMMENI +1FCC; S; 1FC3; # GREEK CAPITAL LETTER ETA WITH PROSGEGRAMMENI +1FD2; F; 03B9 0308 0300; # GREEK SMALL LETTER IOTA WITH DIALYTIKA AND VARIA +1FD3; F; 03B9 0308 0301; # GREEK SMALL LETTER IOTA WITH DIALYTIKA AND OXIA +1FD6; F; 03B9 0342; # GREEK SMALL LETTER IOTA WITH PERISPOMENI +1FD7; F; 03B9 0308 0342; # GREEK SMALL LETTER IOTA WITH DIALYTIKA AND PERISPOMENI +1FD8; C; 1FD0; # GREEK CAPITAL LETTER IOTA WITH VRACHY +1FD9; C; 1FD1; # GREEK CAPITAL LETTER IOTA WITH MACRON +1FDA; C; 1F76; # GREEK CAPITAL LETTER IOTA WITH VARIA +1FDB; C; 1F77; # GREEK CAPITAL LETTER IOTA WITH OXIA +1FE2; F; 03C5 0308 0300; # GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND VARIA +1FE3; F; 03C5 0308 0301; # GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND OXIA +1FE4; F; 03C1 0313; # GREEK SMALL LETTER RHO WITH PSILI +1FE6; F; 03C5 0342; # GREEK SMALL LETTER UPSILON WITH PERISPOMENI +1FE7; F; 03C5 0308 0342; # GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND PERISPOMENI +1FE8; C; 1FE0; # GREEK CAPITAL LETTER UPSILON WITH VRACHY +1FE9; C; 1FE1; # GREEK CAPITAL LETTER UPSILON WITH MACRON +1FEA; C; 1F7A; # GREEK CAPITAL LETTER UPSILON WITH VARIA +1FEB; C; 1F7B; # GREEK CAPITAL LETTER UPSILON WITH OXIA +1FEC; C; 1FE5; # GREEK CAPITAL LETTER RHO WITH DASIA +1FF2; F; 1F7C 03B9; # GREEK SMALL LETTER OMEGA WITH VARIA AND YPOGEGRAMMENI +1FF3; F; 03C9 03B9; # GREEK SMALL LETTER OMEGA WITH YPOGEGRAMMENI +1FF4; F; 03CE 03B9; # GREEK SMALL LETTER OMEGA WITH OXIA AND YPOGEGRAMMENI +1FF6; F; 03C9 0342; # GREEK SMALL LETTER OMEGA WITH PERISPOMENI +1FF7; F; 03C9 0342 03B9; # GREEK SMALL LETTER OMEGA WITH PERISPOMENI AND YPOGEGRAMMENI +1FF8; C; 1F78; # GREEK CAPITAL LETTER OMICRON WITH VARIA +1FF9; C; 1F79; # GREEK CAPITAL LETTER OMICRON WITH OXIA +1FFA; C; 1F7C; # GREEK CAPITAL LETTER OMEGA WITH VARIA +1FFB; C; 1F7D; # GREEK CAPITAL LETTER OMEGA WITH OXIA +1FFC; F; 03C9 03B9; # GREEK CAPITAL LETTER OMEGA WITH PROSGEGRAMMENI +1FFC; S; 1FF3; # GREEK CAPITAL LETTER OMEGA WITH PROSGEGRAMMENI +2126; C; 03C9; # OHM SIGN +212A; C; 006B; # KELVIN SIGN +212B; C; 00E5; # ANGSTROM SIGN +2132; C; 214E; # TURNED CAPITAL F +2160; C; 2170; # ROMAN NUMERAL ONE +2161; C; 2171; # ROMAN NUMERAL TWO +2162; C; 2172; # ROMAN NUMERAL THREE +2163; C; 2173; # ROMAN NUMERAL FOUR +2164; C; 2174; # ROMAN NUMERAL FIVE +2165; C; 2175; # ROMAN NUMERAL SIX +2166; C; 2176; # ROMAN NUMERAL SEVEN +2167; C; 2177; # ROMAN NUMERAL EIGHT +2168; C; 2178; # ROMAN NUMERAL NINE +2169; C; 2179; # ROMAN NUMERAL TEN +216A; C; 217A; # ROMAN NUMERAL ELEVEN +216B; C; 217B; # ROMAN NUMERAL TWELVE +216C; C; 217C; # ROMAN NUMERAL FIFTY +216D; C; 217D; # ROMAN NUMERAL ONE HUNDRED +216E; C; 217E; # ROMAN NUMERAL FIVE HUNDRED +216F; C; 217F; # ROMAN NUMERAL ONE THOUSAND +2183; C; 2184; # ROMAN NUMERAL REVERSED ONE HUNDRED +24B6; C; 24D0; # CIRCLED LATIN CAPITAL LETTER A +24B7; C; 24D1; # CIRCLED LATIN CAPITAL LETTER B +24B8; C; 24D2; # CIRCLED LATIN CAPITAL LETTER C +24B9; C; 24D3; # CIRCLED LATIN CAPITAL LETTER D +24BA; C; 24D4; # CIRCLED LATIN CAPITAL LETTER E +24BB; C; 24D5; # CIRCLED LATIN CAPITAL LETTER F +24BC; C; 24D6; # CIRCLED LATIN CAPITAL LETTER G +24BD; C; 24D7; # CIRCLED LATIN CAPITAL LETTER H +24BE; C; 24D8; # CIRCLED LATIN CAPITAL LETTER I +24BF; C; 24D9; # CIRCLED LATIN CAPITAL LETTER J +24C0; C; 24DA; # CIRCLED LATIN CAPITAL LETTER K +24C1; C; 24DB; # CIRCLED LATIN CAPITAL LETTER L +24C2; C; 24DC; # CIRCLED LATIN CAPITAL LETTER M +24C3; C; 24DD; # CIRCLED LATIN CAPITAL LETTER N +24C4; C; 24DE; # CIRCLED LATIN CAPITAL LETTER O +24C5; C; 24DF; # CIRCLED LATIN CAPITAL LETTER P +24C6; C; 24E0; # CIRCLED LATIN CAPITAL LETTER Q +24C7; C; 24E1; # CIRCLED LATIN CAPITAL LETTER R +24C8; C; 24E2; # CIRCLED LATIN CAPITAL LETTER S +24C9; C; 24E3; # CIRCLED LATIN CAPITAL LETTER T +24CA; C; 24E4; # CIRCLED LATIN CAPITAL LETTER U +24CB; C; 24E5; # CIRCLED LATIN CAPITAL LETTER V +24CC; C; 24E6; # CIRCLED LATIN CAPITAL LETTER W +24CD; C; 24E7; # CIRCLED LATIN CAPITAL LETTER X +24CE; C; 24E8; # CIRCLED LATIN CAPITAL LETTER Y +24CF; C; 24E9; # CIRCLED LATIN CAPITAL LETTER Z +2C00; C; 2C30; # GLAGOLITIC CAPITAL LETTER AZU +2C01; C; 2C31; # GLAGOLITIC CAPITAL LETTER BUKY +2C02; C; 2C32; # GLAGOLITIC CAPITAL LETTER VEDE +2C03; C; 2C33; # GLAGOLITIC CAPITAL LETTER GLAGOLI +2C04; C; 2C34; # GLAGOLITIC CAPITAL LETTER DOBRO +2C05; C; 2C35; # GLAGOLITIC CAPITAL LETTER YESTU +2C06; C; 2C36; # GLAGOLITIC CAPITAL LETTER ZHIVETE +2C07; C; 2C37; # GLAGOLITIC CAPITAL LETTER DZELO +2C08; C; 2C38; # GLAGOLITIC CAPITAL LETTER ZEMLJA +2C09; C; 2C39; # GLAGOLITIC CAPITAL LETTER IZHE +2C0A; C; 2C3A; # GLAGOLITIC CAPITAL LETTER INITIAL IZHE +2C0B; C; 2C3B; # GLAGOLITIC CAPITAL LETTER I +2C0C; C; 2C3C; # GLAGOLITIC CAPITAL LETTER DJERVI +2C0D; C; 2C3D; # GLAGOLITIC CAPITAL LETTER KAKO +2C0E; C; 2C3E; # GLAGOLITIC CAPITAL LETTER LJUDIJE +2C0F; C; 2C3F; # GLAGOLITIC CAPITAL LETTER MYSLITE +2C10; C; 2C40; # GLAGOLITIC CAPITAL LETTER NASHI +2C11; C; 2C41; # GLAGOLITIC CAPITAL LETTER ONU +2C12; C; 2C42; # GLAGOLITIC CAPITAL LETTER POKOJI +2C13; C; 2C43; # GLAGOLITIC CAPITAL LETTER RITSI +2C14; C; 2C44; # GLAGOLITIC CAPITAL LETTER SLOVO +2C15; C; 2C45; # GLAGOLITIC CAPITAL LETTER TVRIDO +2C16; C; 2C46; # GLAGOLITIC CAPITAL LETTER UKU +2C17; C; 2C47; # GLAGOLITIC CAPITAL LETTER FRITU +2C18; C; 2C48; # GLAGOLITIC CAPITAL LETTER HERU +2C19; C; 2C49; # GLAGOLITIC CAPITAL LETTER OTU +2C1A; C; 2C4A; # GLAGOLITIC CAPITAL LETTER PE +2C1B; C; 2C4B; # GLAGOLITIC CAPITAL LETTER SHTA +2C1C; C; 2C4C; # GLAGOLITIC CAPITAL LETTER TSI +2C1D; C; 2C4D; # GLAGOLITIC CAPITAL LETTER CHRIVI +2C1E; C; 2C4E; # GLAGOLITIC CAPITAL LETTER SHA +2C1F; C; 2C4F; # GLAGOLITIC CAPITAL LETTER YERU +2C20; C; 2C50; # GLAGOLITIC CAPITAL LETTER YERI +2C21; C; 2C51; # GLAGOLITIC CAPITAL LETTER YATI +2C22; C; 2C52; # GLAGOLITIC CAPITAL LETTER SPIDERY HA +2C23; C; 2C53; # GLAGOLITIC CAPITAL LETTER YU +2C24; C; 2C54; # GLAGOLITIC CAPITAL LETTER SMALL YUS +2C25; C; 2C55; # GLAGOLITIC CAPITAL LETTER SMALL YUS WITH TAIL +2C26; C; 2C56; # GLAGOLITIC CAPITAL LETTER YO +2C27; C; 2C57; # GLAGOLITIC CAPITAL LETTER IOTATED SMALL YUS +2C28; C; 2C58; # GLAGOLITIC CAPITAL LETTER BIG YUS +2C29; C; 2C59; # GLAGOLITIC CAPITAL LETTER IOTATED BIG YUS +2C2A; C; 2C5A; # GLAGOLITIC CAPITAL LETTER FITA +2C2B; C; 2C5B; # GLAGOLITIC CAPITAL LETTER IZHITSA +2C2C; C; 2C5C; # GLAGOLITIC CAPITAL LETTER SHTAPIC +2C2D; C; 2C5D; # GLAGOLITIC CAPITAL LETTER TROKUTASTI A +2C2E; C; 2C5E; # GLAGOLITIC CAPITAL LETTER LATINATE MYSLITE +2C60; C; 2C61; # LATIN CAPITAL LETTER L WITH DOUBLE BAR +2C62; C; 026B; # LATIN CAPITAL LETTER L WITH MIDDLE TILDE +2C63; C; 1D7D; # LATIN CAPITAL LETTER P WITH STROKE +2C64; C; 027D; # LATIN CAPITAL LETTER R WITH TAIL +2C67; C; 2C68; # LATIN CAPITAL LETTER H WITH DESCENDER +2C69; C; 2C6A; # LATIN CAPITAL LETTER K WITH DESCENDER +2C6B; C; 2C6C; # LATIN CAPITAL LETTER Z WITH DESCENDER +2C6D; C; 0251; # LATIN CAPITAL LETTER ALPHA +2C6E; C; 0271; # LATIN CAPITAL LETTER M WITH HOOK +2C6F; C; 0250; # LATIN CAPITAL LETTER TURNED A +2C72; C; 2C73; # LATIN CAPITAL LETTER W WITH HOOK +2C75; C; 2C76; # LATIN CAPITAL LETTER HALF H +2C80; C; 2C81; # COPTIC CAPITAL LETTER ALFA +2C82; C; 2C83; # COPTIC CAPITAL LETTER VIDA +2C84; C; 2C85; # COPTIC CAPITAL LETTER GAMMA +2C86; C; 2C87; # COPTIC CAPITAL LETTER DALDA +2C88; C; 2C89; # COPTIC CAPITAL LETTER EIE +2C8A; C; 2C8B; # COPTIC CAPITAL LETTER SOU +2C8C; C; 2C8D; # COPTIC CAPITAL LETTER ZATA +2C8E; C; 2C8F; # COPTIC CAPITAL LETTER HATE +2C90; C; 2C91; # COPTIC CAPITAL LETTER THETHE +2C92; C; 2C93; # COPTIC CAPITAL LETTER IAUDA +2C94; C; 2C95; # COPTIC CAPITAL LETTER KAPA +2C96; C; 2C97; # COPTIC CAPITAL LETTER LAULA +2C98; C; 2C99; # COPTIC CAPITAL LETTER MI +2C9A; C; 2C9B; # COPTIC CAPITAL LETTER NI +2C9C; C; 2C9D; # COPTIC CAPITAL LETTER KSI +2C9E; C; 2C9F; # COPTIC CAPITAL LETTER O +2CA0; C; 2CA1; # COPTIC CAPITAL LETTER PI +2CA2; C; 2CA3; # COPTIC CAPITAL LETTER RO +2CA4; C; 2CA5; # COPTIC CAPITAL LETTER SIMA +2CA6; C; 2CA7; # COPTIC CAPITAL LETTER TAU +2CA8; C; 2CA9; # COPTIC CAPITAL LETTER UA +2CAA; C; 2CAB; # COPTIC CAPITAL LETTER FI +2CAC; C; 2CAD; # COPTIC CAPITAL LETTER KHI +2CAE; C; 2CAF; # COPTIC CAPITAL LETTER PSI +2CB0; C; 2CB1; # COPTIC CAPITAL LETTER OOU +2CB2; C; 2CB3; # COPTIC CAPITAL LETTER DIALECT-P ALEF +2CB4; C; 2CB5; # COPTIC CAPITAL LETTER OLD COPTIC AIN +2CB6; C; 2CB7; # COPTIC CAPITAL LETTER CRYPTOGRAMMIC EIE +2CB8; C; 2CB9; # COPTIC CAPITAL LETTER DIALECT-P KAPA +2CBA; C; 2CBB; # COPTIC CAPITAL LETTER DIALECT-P NI +2CBC; C; 2CBD; # COPTIC CAPITAL LETTER CRYPTOGRAMMIC NI +2CBE; C; 2CBF; # COPTIC CAPITAL LETTER OLD COPTIC OOU +2CC0; C; 2CC1; # COPTIC CAPITAL LETTER SAMPI +2CC2; C; 2CC3; # COPTIC CAPITAL LETTER CROSSED SHEI +2CC4; C; 2CC5; # COPTIC CAPITAL LETTER OLD COPTIC SHEI +2CC6; C; 2CC7; # COPTIC CAPITAL LETTER OLD COPTIC ESH +2CC8; C; 2CC9; # COPTIC CAPITAL LETTER AKHMIMIC KHEI +2CCA; C; 2CCB; # COPTIC CAPITAL LETTER DIALECT-P HORI +2CCC; C; 2CCD; # COPTIC CAPITAL LETTER OLD COPTIC HORI +2CCE; C; 2CCF; # COPTIC CAPITAL LETTER OLD COPTIC HA +2CD0; C; 2CD1; # COPTIC CAPITAL LETTER L-SHAPED HA +2CD2; C; 2CD3; # COPTIC CAPITAL LETTER OLD COPTIC HEI +2CD4; C; 2CD5; # COPTIC CAPITAL LETTER OLD COPTIC HAT +2CD6; C; 2CD7; # COPTIC CAPITAL LETTER OLD COPTIC GANGIA +2CD8; C; 2CD9; # COPTIC CAPITAL LETTER OLD COPTIC DJA +2CDA; C; 2CDB; # COPTIC CAPITAL LETTER OLD COPTIC SHIMA +2CDC; C; 2CDD; # COPTIC CAPITAL LETTER OLD NUBIAN SHIMA +2CDE; C; 2CDF; # COPTIC CAPITAL LETTER OLD NUBIAN NGI +2CE0; C; 2CE1; # COPTIC CAPITAL LETTER OLD NUBIAN NYI +2CE2; C; 2CE3; # COPTIC CAPITAL LETTER OLD NUBIAN WAU +A640; C; A641; # CYRILLIC CAPITAL LETTER ZEMLYA +A642; C; A643; # CYRILLIC CAPITAL LETTER DZELO +A644; C; A645; # CYRILLIC CAPITAL LETTER REVERSED DZE +A646; C; A647; # CYRILLIC CAPITAL LETTER IOTA +A648; C; A649; # CYRILLIC CAPITAL LETTER DJERV +A64A; C; A64B; # CYRILLIC CAPITAL LETTER MONOGRAPH UK +A64C; C; A64D; # CYRILLIC CAPITAL LETTER BROAD OMEGA +A64E; C; A64F; # CYRILLIC CAPITAL LETTER NEUTRAL YER +A650; C; A651; # CYRILLIC CAPITAL LETTER YERU WITH BACK YER +A652; C; A653; # CYRILLIC CAPITAL LETTER IOTIFIED YAT +A654; C; A655; # CYRILLIC CAPITAL LETTER REVERSED YU +A656; C; A657; # CYRILLIC CAPITAL LETTER IOTIFIED A +A658; C; A659; # CYRILLIC CAPITAL LETTER CLOSED LITTLE YUS +A65A; C; A65B; # CYRILLIC CAPITAL LETTER BLENDED YUS +A65C; C; A65D; # CYRILLIC CAPITAL LETTER IOTIFIED CLOSED LITTLE YUS +A65E; C; A65F; # CYRILLIC CAPITAL LETTER YN +A662; C; A663; # CYRILLIC CAPITAL LETTER SOFT DE +A664; C; A665; # CYRILLIC CAPITAL LETTER SOFT EL +A666; C; A667; # CYRILLIC CAPITAL LETTER SOFT EM +A668; C; A669; # CYRILLIC CAPITAL LETTER MONOCULAR O +A66A; C; A66B; # CYRILLIC CAPITAL LETTER BINOCULAR O +A66C; C; A66D; # CYRILLIC CAPITAL LETTER DOUBLE MONOCULAR O +A680; C; A681; # CYRILLIC CAPITAL LETTER DWE +A682; C; A683; # CYRILLIC CAPITAL LETTER DZWE +A684; C; A685; # CYRILLIC CAPITAL LETTER ZHWE +A686; C; A687; # CYRILLIC CAPITAL LETTER CCHE +A688; C; A689; # CYRILLIC CAPITAL LETTER DZZE +A68A; C; A68B; # CYRILLIC CAPITAL LETTER TE WITH MIDDLE HOOK +A68C; C; A68D; # CYRILLIC CAPITAL LETTER TWE +A68E; C; A68F; # CYRILLIC CAPITAL LETTER TSWE +A690; C; A691; # CYRILLIC CAPITAL LETTER TSSE +A692; C; A693; # CYRILLIC CAPITAL LETTER TCHE +A694; C; A695; # CYRILLIC CAPITAL LETTER HWE +A696; C; A697; # CYRILLIC CAPITAL LETTER SHWE +A722; C; A723; # LATIN CAPITAL LETTER EGYPTOLOGICAL ALEF +A724; C; A725; # LATIN CAPITAL LETTER EGYPTOLOGICAL AIN +A726; C; A727; # LATIN CAPITAL LETTER HENG +A728; C; A729; # LATIN CAPITAL LETTER TZ +A72A; C; A72B; # LATIN CAPITAL LETTER TRESILLO +A72C; C; A72D; # LATIN CAPITAL LETTER CUATRILLO +A72E; C; A72F; # LATIN CAPITAL LETTER CUATRILLO WITH COMMA +A732; C; A733; # LATIN CAPITAL LETTER AA +A734; C; A735; # LATIN CAPITAL LETTER AO +A736; C; A737; # LATIN CAPITAL LETTER AU +A738; C; A739; # LATIN CAPITAL LETTER AV +A73A; C; A73B; # LATIN CAPITAL LETTER AV WITH HORIZONTAL BAR +A73C; C; A73D; # LATIN CAPITAL LETTER AY +A73E; C; A73F; # LATIN CAPITAL LETTER REVERSED C WITH DOT +A740; C; A741; # LATIN CAPITAL LETTER K WITH STROKE +A742; C; A743; # LATIN CAPITAL LETTER K WITH DIAGONAL STROKE +A744; C; A745; # LATIN CAPITAL LETTER K WITH STROKE AND DIAGONAL STROKE +A746; C; A747; # LATIN CAPITAL LETTER BROKEN L +A748; C; A749; # LATIN CAPITAL LETTER L WITH HIGH STROKE +A74A; C; A74B; # LATIN CAPITAL LETTER O WITH LONG STROKE OVERLAY +A74C; C; A74D; # LATIN CAPITAL LETTER O WITH LOOP +A74E; C; A74F; # LATIN CAPITAL LETTER OO +A750; C; A751; # LATIN CAPITAL LETTER P WITH STROKE THROUGH DESCENDER +A752; C; A753; # LATIN CAPITAL LETTER P WITH FLOURISH +A754; C; A755; # LATIN CAPITAL LETTER P WITH SQUIRREL TAIL +A756; C; A757; # LATIN CAPITAL LETTER Q WITH STROKE THROUGH DESCENDER +A758; C; A759; # LATIN CAPITAL LETTER Q WITH DIAGONAL STROKE +A75A; C; A75B; # LATIN CAPITAL LETTER R ROTUNDA +A75C; C; A75D; # LATIN CAPITAL LETTER RUM ROTUNDA +A75E; C; A75F; # LATIN CAPITAL LETTER V WITH DIAGONAL STROKE +A760; C; A761; # LATIN CAPITAL LETTER VY +A762; C; A763; # LATIN CAPITAL LETTER VISIGOTHIC Z +A764; C; A765; # LATIN CAPITAL LETTER THORN WITH STROKE +A766; C; A767; # LATIN CAPITAL LETTER THORN WITH STROKE THROUGH DESCENDER +A768; C; A769; # LATIN CAPITAL LETTER VEND +A76A; C; A76B; # LATIN CAPITAL LETTER ET +A76C; C; A76D; # LATIN CAPITAL LETTER IS +A76E; C; A76F; # LATIN CAPITAL LETTER CON +A779; C; A77A; # LATIN CAPITAL LETTER INSULAR D +A77B; C; A77C; # LATIN CAPITAL LETTER INSULAR F +A77D; C; 1D79; # LATIN CAPITAL LETTER INSULAR G +A77E; C; A77F; # LATIN CAPITAL LETTER TURNED INSULAR G +A780; C; A781; # LATIN CAPITAL LETTER TURNED L +A782; C; A783; # LATIN CAPITAL LETTER INSULAR R +A784; C; A785; # LATIN CAPITAL LETTER INSULAR S +A786; C; A787; # LATIN CAPITAL LETTER INSULAR T +A78B; C; A78C; # LATIN CAPITAL LETTER SALTILLO +FB00; F; 0066 0066; # LATIN SMALL LIGATURE FF +FB01; F; 0066 0069; # LATIN SMALL LIGATURE FI +FB02; F; 0066 006C; # LATIN SMALL LIGATURE FL +FB03; F; 0066 0066 0069; # LATIN SMALL LIGATURE FFI +FB04; F; 0066 0066 006C; # LATIN SMALL LIGATURE FFL +FB05; F; 0073 0074; # LATIN SMALL LIGATURE LONG S T +FB06; F; 0073 0074; # LATIN SMALL LIGATURE ST +FB13; F; 0574 0576; # ARMENIAN SMALL LIGATURE MEN NOW +FB14; F; 0574 0565; # ARMENIAN SMALL LIGATURE MEN ECH +FB15; F; 0574 056B; # ARMENIAN SMALL LIGATURE MEN INI +FB16; F; 057E 0576; # ARMENIAN SMALL LIGATURE VEW NOW +FB17; F; 0574 056D; # ARMENIAN SMALL LIGATURE MEN XEH +FF21; C; FF41; # FULLWIDTH LATIN CAPITAL LETTER A +FF22; C; FF42; # FULLWIDTH LATIN CAPITAL LETTER B +FF23; C; FF43; # FULLWIDTH LATIN CAPITAL LETTER C +FF24; C; FF44; # FULLWIDTH LATIN CAPITAL LETTER D +FF25; C; FF45; # FULLWIDTH LATIN CAPITAL LETTER E +FF26; C; FF46; # FULLWIDTH LATIN CAPITAL LETTER F +FF27; C; FF47; # FULLWIDTH LATIN CAPITAL LETTER G +FF28; C; FF48; # FULLWIDTH LATIN CAPITAL LETTER H +FF29; C; FF49; # FULLWIDTH LATIN CAPITAL LETTER I +FF2A; C; FF4A; # FULLWIDTH LATIN CAPITAL LETTER J +FF2B; C; FF4B; # FULLWIDTH LATIN CAPITAL LETTER K +FF2C; C; FF4C; # FULLWIDTH LATIN CAPITAL LETTER L +FF2D; C; FF4D; # FULLWIDTH LATIN CAPITAL LETTER M +FF2E; C; FF4E; # FULLWIDTH LATIN CAPITAL LETTER N +FF2F; C; FF4F; # FULLWIDTH LATIN CAPITAL LETTER O +FF30; C; FF50; # FULLWIDTH LATIN CAPITAL LETTER P +FF31; C; FF51; # FULLWIDTH LATIN CAPITAL LETTER Q +FF32; C; FF52; # FULLWIDTH LATIN CAPITAL LETTER R +FF33; C; FF53; # FULLWIDTH LATIN CAPITAL LETTER S +FF34; C; FF54; # FULLWIDTH LATIN CAPITAL LETTER T +FF35; C; FF55; # FULLWIDTH LATIN CAPITAL LETTER U +FF36; C; FF56; # FULLWIDTH LATIN CAPITAL LETTER V +FF37; C; FF57; # FULLWIDTH LATIN CAPITAL LETTER W +FF38; C; FF58; # FULLWIDTH LATIN CAPITAL LETTER X +FF39; C; FF59; # FULLWIDTH LATIN CAPITAL LETTER Y +FF3A; C; FF5A; # FULLWIDTH LATIN CAPITAL LETTER Z +10400; C; 10428; # DESERET CAPITAL LETTER LONG I +10401; C; 10429; # DESERET CAPITAL LETTER LONG E +10402; C; 1042A; # DESERET CAPITAL LETTER LONG A +10403; C; 1042B; # DESERET CAPITAL LETTER LONG AH +10404; C; 1042C; # DESERET CAPITAL LETTER LONG O +10405; C; 1042D; # DESERET CAPITAL LETTER LONG OO +10406; C; 1042E; # DESERET CAPITAL LETTER SHORT I +10407; C; 1042F; # DESERET CAPITAL LETTER SHORT E +10408; C; 10430; # DESERET CAPITAL LETTER SHORT A +10409; C; 10431; # DESERET CAPITAL LETTER SHORT AH +1040A; C; 10432; # DESERET CAPITAL LETTER SHORT O +1040B; C; 10433; # DESERET CAPITAL LETTER SHORT OO +1040C; C; 10434; # DESERET CAPITAL LETTER AY +1040D; C; 10435; # DESERET CAPITAL LETTER OW +1040E; C; 10436; # DESERET CAPITAL LETTER WU +1040F; C; 10437; # DESERET CAPITAL LETTER YEE +10410; C; 10438; # DESERET CAPITAL LETTER H +10411; C; 10439; # DESERET CAPITAL LETTER PEE +10412; C; 1043A; # DESERET CAPITAL LETTER BEE +10413; C; 1043B; # DESERET CAPITAL LETTER TEE +10414; C; 1043C; # DESERET CAPITAL LETTER DEE +10415; C; 1043D; # DESERET CAPITAL LETTER CHEE +10416; C; 1043E; # DESERET CAPITAL LETTER JEE +10417; C; 1043F; # DESERET CAPITAL LETTER KAY +10418; C; 10440; # DESERET CAPITAL LETTER GAY +10419; C; 10441; # DESERET CAPITAL LETTER EF +1041A; C; 10442; # DESERET CAPITAL LETTER VEE +1041B; C; 10443; # DESERET CAPITAL LETTER ETH +1041C; C; 10444; # DESERET CAPITAL LETTER THEE +1041D; C; 10445; # DESERET CAPITAL LETTER ES +1041E; C; 10446; # DESERET CAPITAL LETTER ZEE +1041F; C; 10447; # DESERET CAPITAL LETTER ESH +10420; C; 10448; # DESERET CAPITAL LETTER ZHEE +10421; C; 10449; # DESERET CAPITAL LETTER ER +10422; C; 1044A; # DESERET CAPITAL LETTER EL +10423; C; 1044B; # DESERET CAPITAL LETTER EM +10424; C; 1044C; # DESERET CAPITAL LETTER EN +10425; C; 1044D; # DESERET CAPITAL LETTER ENG +10426; C; 1044E; # DESERET CAPITAL LETTER OI +10427; C; 1044F; # DESERET CAPITAL LETTER EW diff --git a/scripts/CaseMapping.hs b/scripts/CaseMapping.hs new file mode 100644 --- /dev/null +++ b/scripts/CaseMapping.hs @@ -0,0 +1,32 @@ +import System.Environment +import System.IO + +import Arsec +import CaseFolding +import SpecialCasing + +main = do + args <- getArgs + let oname = case args of + [] -> "../Data/Text/Fusion/CaseMapping.hs" + [o] -> o + psc <- parseSC "SpecialCasing.txt" + pcf <- parseCF "CaseFolding.txt" + scs <- case psc of + Left err -> print err >> return undefined + Right ms -> return ms + cfs <- case pcf of + Left err -> print err >> return undefined + Right ms -> return ms + h <- openFile oname WriteMode + mapM_ (hPutStrLn h) ["{-# LANGUAGE Rank2Types #-}" + ,"-- AUTOMATICALLY GENERATED - DO NOT EDIT" + ,"-- Generated by scripts/SpecialCasing.hs" + ,"module Data.Text.Fusion.CaseMapping where" + ,"import Data.Char" + ,"import Data.Text.Fusion.Internal" + ,""] + mapM_ (hPutStrLn h) (mapSC "upper" upper toUpper scs) + mapM_ (hPutStrLn h) (mapSC "lower" lower toLower scs) + mapM_ (hPutStrLn h) (mapCF cfs) + hClose h diff --git a/scripts/SpecialCasing.hs b/scripts/SpecialCasing.hs new file mode 100644 --- /dev/null +++ b/scripts/SpecialCasing.hs @@ -0,0 +1,50 @@ +-- This script processes the following source file: +-- +-- http://unicode.org/Public/UNIDATA/SpecialCasing.txt + +module SpecialCasing + ( + Case(..) + , parseSC + , mapSC + ) where + +import Arsec + +data Case = Case { + code :: Char + , lower :: [Char] + , title :: [Char] + , upper :: [Char] + , conditions :: String + , name :: String + } deriving (Eq, Ord, Show) + +entries :: Parser [Case] +entries = many comment *> many (entry <* many comment) + where + entry = Case <$> unichar <* semi + <*> unichars + <*> unichars + <*> unichars + <*> manyTill anyToken (string "# ") + <*> manyTill anyToken (char '\n') + +parseSC :: FilePath -> IO (Either ParseError [Case]) +parseSC name = parse entries name <$> readFile name + +mapSC :: String -> (Case -> String) -> (Char -> Char) -> [Case] -> [String] +mapSC which access twiddle ms = typ ++ (map nice . filter p $ ms) ++ [last] + where + typ = [which ++ "Mapping :: forall s. Char -> s -> Step (PairS (PairS s Char) Char) Char" + ,"{-# INLINE " ++ which ++ "Mapping #-}"] + last = which ++ "Mapping c s = Yield (to" ++ ucFirst which ++ " c) (s :!: '\\0' :!: '\\0')" + nice c = "-- " ++ name c ++ "\n" ++ + which ++ "Mapping " ++ showC (code c) ++ " s = Yield " ++ x ++ " (s :!: " ++ y ++ " :!: " ++ z ++ ")" + where [x,y,z] = (map showC . take 3) (access c ++ repeat '\0') + p c = [k] /= a && a /= [twiddle k] && null (conditions c) + where a = access c + k = code c + +ucFirst (c:cs) = toUpper c : cs +ucFirst [] = [] diff --git a/scripts/SpecialCasing.txt b/scripts/SpecialCasing.txt new file mode 100644 --- /dev/null +++ b/scripts/SpecialCasing.txt @@ -0,0 +1,274 @@ +# SpecialCasing-5.1.0.txt +# Date: 2008-03-03, 21:58:10 GMT [MD] +# +# Unicode Character Database +# Copyright (c) 1991-2008 Unicode, Inc. +# For terms of use, see http://www.unicode.org/terms_of_use.html +# For documentation, see UCD.html +# +# Special Casing Properties +# +# This file is a supplement to the UnicodeData file. +# It contains additional information about the casing of Unicode characters. +# (For compatibility, the UnicodeData.txt file only contains case mappings for +# characters where they are 1-1, and independent of context and language. +# For more information, see the discussion of Case Mappings in the Unicode Standard. +# +# All code points not listed in this file that do not have a simple case mappings +# in UnicodeData.txt map to themselves. +# ================================================================================ +# Format +# ================================================================================ +# The entries in this file are in the following machine-readable format: +# +# ; ; ; <upper> ; (<condition_list> ;)? # <comment> +# +# <code>, <lower>, <title>, and <upper> provide character values in hex. If there is more +# than one character, they are separated by spaces. Other than as used to separate +# elements, spaces are to be ignored. +# +# The <condition_list> is optional. Where present, it consists of one or more language IDs +# or contexts, separated by spaces. In these conditions: +# - A condition list overrides the normal behavior if all of the listed conditions are true. +# - The context is always the context of the characters in the original string, +# NOT in the resulting string. +# - Case distinctions in the condition list are not significant. +# - Conditions preceded by "Not_" represent the negation of the condition. +# The condition list is not represented in the UCD as a formal property. +# +# A language ID is defined by BCP 47, with '-' and '_' treated equivalently. +# +# A context for a character C is defined by Section 3.13 Default Case +# Operations, of The Unicode Standard, Version 5.0. +# (This is identical to the context defined by Unicode 4.1.0, +# as specified in http://www.unicode.org/versions/Unicode4.1.0/) +# +# Parsers of this file must be prepared to deal with future additions to this format: +# * Additional contexts +# * Additional fields +# ================================================================================ +# @missing 0000..10FFFF; <slc>; <stc>; <suc> +# ================================================================================ +# Unconditional mappings +# ================================================================================ + +# The German es-zed is special--the normal mapping is to SS. +# Note: the titlecase should never occur in practice. It is equal to titlecase(uppercase(<es-zed>)) + +00DF; 00DF; 0053 0073; 0053 0053; # LATIN SMALL LETTER SHARP S + +# Preserve canonical equivalence for I with dot. Turkic is handled below. + +0130; 0069 0307; 0130; 0130; # LATIN CAPITAL LETTER I WITH DOT ABOVE + +# Ligatures + +FB00; FB00; 0046 0066; 0046 0046; # LATIN SMALL LIGATURE FF +FB01; FB01; 0046 0069; 0046 0049; # LATIN SMALL LIGATURE FI +FB02; FB02; 0046 006C; 0046 004C; # LATIN SMALL LIGATURE FL +FB03; FB03; 0046 0066 0069; 0046 0046 0049; # LATIN SMALL LIGATURE FFI +FB04; FB04; 0046 0066 006C; 0046 0046 004C; # LATIN SMALL LIGATURE FFL +FB05; FB05; 0053 0074; 0053 0054; # LATIN SMALL LIGATURE LONG S T +FB06; FB06; 0053 0074; 0053 0054; # LATIN SMALL LIGATURE ST + +0587; 0587; 0535 0582; 0535 0552; # ARMENIAN SMALL LIGATURE ECH YIWN +FB13; FB13; 0544 0576; 0544 0546; # ARMENIAN SMALL LIGATURE MEN NOW +FB14; FB14; 0544 0565; 0544 0535; # ARMENIAN SMALL LIGATURE MEN ECH +FB15; FB15; 0544 056B; 0544 053B; # ARMENIAN SMALL LIGATURE MEN INI +FB16; FB16; 054E 0576; 054E 0546; # ARMENIAN SMALL LIGATURE VEW NOW +FB17; FB17; 0544 056D; 0544 053D; # ARMENIAN SMALL LIGATURE MEN XEH + +# No corresponding uppercase precomposed character + +0149; 0149; 02BC 004E; 02BC 004E; # LATIN SMALL LETTER N PRECEDED BY APOSTROPHE +0390; 0390; 0399 0308 0301; 0399 0308 0301; # GREEK SMALL LETTER IOTA WITH DIALYTIKA AND TONOS +03B0; 03B0; 03A5 0308 0301; 03A5 0308 0301; # GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND TONOS +01F0; 01F0; 004A 030C; 004A 030C; # LATIN SMALL LETTER J WITH CARON +1E96; 1E96; 0048 0331; 0048 0331; # LATIN SMALL LETTER H WITH LINE BELOW +1E97; 1E97; 0054 0308; 0054 0308; # LATIN SMALL LETTER T WITH DIAERESIS +1E98; 1E98; 0057 030A; 0057 030A; # LATIN SMALL LETTER W WITH RING ABOVE +1E99; 1E99; 0059 030A; 0059 030A; # LATIN SMALL LETTER Y WITH RING ABOVE +1E9A; 1E9A; 0041 02BE; 0041 02BE; # LATIN SMALL LETTER A WITH RIGHT HALF RING +1F50; 1F50; 03A5 0313; 03A5 0313; # GREEK SMALL LETTER UPSILON WITH PSILI +1F52; 1F52; 03A5 0313 0300; 03A5 0313 0300; # GREEK SMALL LETTER UPSILON WITH PSILI AND VARIA +1F54; 1F54; 03A5 0313 0301; 03A5 0313 0301; # GREEK SMALL LETTER UPSILON WITH PSILI AND OXIA +1F56; 1F56; 03A5 0313 0342; 03A5 0313 0342; # GREEK SMALL LETTER UPSILON WITH PSILI AND PERISPOMENI +1FB6; 1FB6; 0391 0342; 0391 0342; # GREEK SMALL LETTER ALPHA WITH PERISPOMENI +1FC6; 1FC6; 0397 0342; 0397 0342; # GREEK SMALL LETTER ETA WITH PERISPOMENI +1FD2; 1FD2; 0399 0308 0300; 0399 0308 0300; # GREEK SMALL LETTER IOTA WITH DIALYTIKA AND VARIA +1FD3; 1FD3; 0399 0308 0301; 0399 0308 0301; # GREEK SMALL LETTER IOTA WITH DIALYTIKA AND OXIA +1FD6; 1FD6; 0399 0342; 0399 0342; # GREEK SMALL LETTER IOTA WITH PERISPOMENI +1FD7; 1FD7; 0399 0308 0342; 0399 0308 0342; # GREEK SMALL LETTER IOTA WITH DIALYTIKA AND PERISPOMENI +1FE2; 1FE2; 03A5 0308 0300; 03A5 0308 0300; # GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND VARIA +1FE3; 1FE3; 03A5 0308 0301; 03A5 0308 0301; # GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND OXIA +1FE4; 1FE4; 03A1 0313; 03A1 0313; # GREEK SMALL LETTER RHO WITH PSILI +1FE6; 1FE6; 03A5 0342; 03A5 0342; # GREEK SMALL LETTER UPSILON WITH PERISPOMENI +1FE7; 1FE7; 03A5 0308 0342; 03A5 0308 0342; # GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND PERISPOMENI +1FF6; 1FF6; 03A9 0342; 03A9 0342; # GREEK SMALL LETTER OMEGA WITH PERISPOMENI + +# IMPORTANT-when capitalizing iota-subscript (0345) +# It MUST be in normalized form--moved to the end of any sequence of combining marks. +# This is because logically it represents a following base character! +# E.g. <iota_subscript> (<Mn> | <Mc> | <Me>)+ => (<Mn> | <Mc> | <Me>)+ <iota_subscript> +# It should never be the first character in a word, so in titlecasing it can be left as is. + +# The following cases are already in the UnicodeData file, so are only commented here. + +# 0345; 0345; 0345; 0399; # COMBINING GREEK YPOGEGRAMMENI + +# All letters with YPOGEGRAMMENI (iota-subscript) or PROSGEGRAMMENI (iota adscript) +# have special uppercases. +# Note: characters with PROSGEGRAMMENI are actually titlecase, not uppercase! + +1F80; 1F80; 1F88; 1F08 0399; # GREEK SMALL LETTER ALPHA WITH PSILI AND YPOGEGRAMMENI +1F81; 1F81; 1F89; 1F09 0399; # GREEK SMALL LETTER ALPHA WITH DASIA AND YPOGEGRAMMENI +1F82; 1F82; 1F8A; 1F0A 0399; # GREEK SMALL LETTER ALPHA WITH PSILI AND VARIA AND YPOGEGRAMMENI +1F83; 1F83; 1F8B; 1F0B 0399; # GREEK SMALL LETTER ALPHA WITH DASIA AND VARIA AND YPOGEGRAMMENI +1F84; 1F84; 1F8C; 1F0C 0399; # GREEK SMALL LETTER ALPHA WITH PSILI AND OXIA AND YPOGEGRAMMENI +1F85; 1F85; 1F8D; 1F0D 0399; # GREEK SMALL LETTER ALPHA WITH DASIA AND OXIA AND YPOGEGRAMMENI +1F86; 1F86; 1F8E; 1F0E 0399; # GREEK SMALL LETTER ALPHA WITH PSILI AND PERISPOMENI AND YPOGEGRAMMENI +1F87; 1F87; 1F8F; 1F0F 0399; # GREEK SMALL LETTER ALPHA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI +1F88; 1F80; 1F88; 1F08 0399; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND PROSGEGRAMMENI +1F89; 1F81; 1F89; 1F09 0399; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND PROSGEGRAMMENI +1F8A; 1F82; 1F8A; 1F0A 0399; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND VARIA AND PROSGEGRAMMENI +1F8B; 1F83; 1F8B; 1F0B 0399; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND VARIA AND PROSGEGRAMMENI +1F8C; 1F84; 1F8C; 1F0C 0399; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND OXIA AND PROSGEGRAMMENI +1F8D; 1F85; 1F8D; 1F0D 0399; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND OXIA AND PROSGEGRAMMENI +1F8E; 1F86; 1F8E; 1F0E 0399; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI +1F8F; 1F87; 1F8F; 1F0F 0399; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI +1F90; 1F90; 1F98; 1F28 0399; # GREEK SMALL LETTER ETA WITH PSILI AND YPOGEGRAMMENI +1F91; 1F91; 1F99; 1F29 0399; # GREEK SMALL LETTER ETA WITH DASIA AND YPOGEGRAMMENI +1F92; 1F92; 1F9A; 1F2A 0399; # GREEK SMALL LETTER ETA WITH PSILI AND VARIA AND YPOGEGRAMMENI +1F93; 1F93; 1F9B; 1F2B 0399; # GREEK SMALL LETTER ETA WITH DASIA AND VARIA AND YPOGEGRAMMENI +1F94; 1F94; 1F9C; 1F2C 0399; # GREEK SMALL LETTER ETA WITH PSILI AND OXIA AND YPOGEGRAMMENI +1F95; 1F95; 1F9D; 1F2D 0399; # GREEK SMALL LETTER ETA WITH DASIA AND OXIA AND YPOGEGRAMMENI +1F96; 1F96; 1F9E; 1F2E 0399; # GREEK SMALL LETTER ETA WITH PSILI AND PERISPOMENI AND YPOGEGRAMMENI +1F97; 1F97; 1F9F; 1F2F 0399; # GREEK SMALL LETTER ETA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI +1F98; 1F90; 1F98; 1F28 0399; # GREEK CAPITAL LETTER ETA WITH PSILI AND PROSGEGRAMMENI +1F99; 1F91; 1F99; 1F29 0399; # GREEK CAPITAL LETTER ETA WITH DASIA AND PROSGEGRAMMENI +1F9A; 1F92; 1F9A; 1F2A 0399; # GREEK CAPITAL LETTER ETA WITH PSILI AND VARIA AND PROSGEGRAMMENI +1F9B; 1F93; 1F9B; 1F2B 0399; # GREEK CAPITAL LETTER ETA WITH DASIA AND VARIA AND PROSGEGRAMMENI +1F9C; 1F94; 1F9C; 1F2C 0399; # GREEK CAPITAL LETTER ETA WITH PSILI AND OXIA AND PROSGEGRAMMENI +1F9D; 1F95; 1F9D; 1F2D 0399; # GREEK CAPITAL LETTER ETA WITH DASIA AND OXIA AND PROSGEGRAMMENI +1F9E; 1F96; 1F9E; 1F2E 0399; # GREEK CAPITAL LETTER ETA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI +1F9F; 1F97; 1F9F; 1F2F 0399; # GREEK CAPITAL LETTER ETA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI +1FA0; 1FA0; 1FA8; 1F68 0399; # GREEK SMALL LETTER OMEGA WITH PSILI AND YPOGEGRAMMENI +1FA1; 1FA1; 1FA9; 1F69 0399; # GREEK SMALL LETTER OMEGA WITH DASIA AND YPOGEGRAMMENI +1FA2; 1FA2; 1FAA; 1F6A 0399; # GREEK SMALL LETTER OMEGA WITH PSILI AND VARIA AND YPOGEGRAMMENI +1FA3; 1FA3; 1FAB; 1F6B 0399; # GREEK SMALL LETTER OMEGA WITH DASIA AND VARIA AND YPOGEGRAMMENI +1FA4; 1FA4; 1FAC; 1F6C 0399; # GREEK SMALL LETTER OMEGA WITH PSILI AND OXIA AND YPOGEGRAMMENI +1FA5; 1FA5; 1FAD; 1F6D 0399; # GREEK SMALL LETTER OMEGA WITH DASIA AND OXIA AND YPOGEGRAMMENI +1FA6; 1FA6; 1FAE; 1F6E 0399; # GREEK SMALL LETTER OMEGA WITH PSILI AND PERISPOMENI AND YPOGEGRAMMENI +1FA7; 1FA7; 1FAF; 1F6F 0399; # GREEK SMALL LETTER OMEGA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI +1FA8; 1FA0; 1FA8; 1F68 0399; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND PROSGEGRAMMENI +1FA9; 1FA1; 1FA9; 1F69 0399; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND PROSGEGRAMMENI +1FAA; 1FA2; 1FAA; 1F6A 0399; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND VARIA AND PROSGEGRAMMENI +1FAB; 1FA3; 1FAB; 1F6B 0399; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND VARIA AND PROSGEGRAMMENI +1FAC; 1FA4; 1FAC; 1F6C 0399; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND OXIA AND PROSGEGRAMMENI +1FAD; 1FA5; 1FAD; 1F6D 0399; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND OXIA AND PROSGEGRAMMENI +1FAE; 1FA6; 1FAE; 1F6E 0399; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI +1FAF; 1FA7; 1FAF; 1F6F 0399; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI +1FB3; 1FB3; 1FBC; 0391 0399; # GREEK SMALL LETTER ALPHA WITH YPOGEGRAMMENI +1FBC; 1FB3; 1FBC; 0391 0399; # GREEK CAPITAL LETTER ALPHA WITH PROSGEGRAMMENI +1FC3; 1FC3; 1FCC; 0397 0399; # GREEK SMALL LETTER ETA WITH YPOGEGRAMMENI +1FCC; 1FC3; 1FCC; 0397 0399; # GREEK CAPITAL LETTER ETA WITH PROSGEGRAMMENI +1FF3; 1FF3; 1FFC; 03A9 0399; # GREEK SMALL LETTER OMEGA WITH YPOGEGRAMMENI +1FFC; 1FF3; 1FFC; 03A9 0399; # GREEK CAPITAL LETTER OMEGA WITH PROSGEGRAMMENI + +# Some characters with YPOGEGRAMMENI also have no corresponding titlecases + +1FB2; 1FB2; 1FBA 0345; 1FBA 0399; # GREEK SMALL LETTER ALPHA WITH VARIA AND YPOGEGRAMMENI +1FB4; 1FB4; 0386 0345; 0386 0399; # GREEK SMALL LETTER ALPHA WITH OXIA AND YPOGEGRAMMENI +1FC2; 1FC2; 1FCA 0345; 1FCA 0399; # GREEK SMALL LETTER ETA WITH VARIA AND YPOGEGRAMMENI +1FC4; 1FC4; 0389 0345; 0389 0399; # GREEK SMALL LETTER ETA WITH OXIA AND YPOGEGRAMMENI +1FF2; 1FF2; 1FFA 0345; 1FFA 0399; # GREEK SMALL LETTER OMEGA WITH VARIA AND YPOGEGRAMMENI +1FF4; 1FF4; 038F 0345; 038F 0399; # GREEK SMALL LETTER OMEGA WITH OXIA AND YPOGEGRAMMENI + +1FB7; 1FB7; 0391 0342 0345; 0391 0342 0399; # GREEK SMALL LETTER ALPHA WITH PERISPOMENI AND YPOGEGRAMMENI +1FC7; 1FC7; 0397 0342 0345; 0397 0342 0399; # GREEK SMALL LETTER ETA WITH PERISPOMENI AND YPOGEGRAMMENI +1FF7; 1FF7; 03A9 0342 0345; 03A9 0342 0399; # GREEK SMALL LETTER OMEGA WITH PERISPOMENI AND YPOGEGRAMMENI + +# ================================================================================ +# Conditional Mappings +# The remainder of this file provides conditional casing data used to produce +# full case mappings. +# ================================================================================ +# Language-Insensitive Mappings +# These are characters whose full case mappings do not depend on language, but do +# depend on context (which characters come before or after). For more information +# see the header of this file and the Unicode Standard. +# ================================================================================ + +# Special case for final form of sigma + +03A3; 03C2; 03A3; 03A3; Final_Sigma; # GREEK CAPITAL LETTER SIGMA + +# Note: the following cases for non-final are already in the UnicodeData file. + +# 03A3; 03C3; 03A3; 03A3; # GREEK CAPITAL LETTER SIGMA +# 03C3; 03C3; 03A3; 03A3; # GREEK SMALL LETTER SIGMA +# 03C2; 03C2; 03A3; 03A3; # GREEK SMALL LETTER FINAL SIGMA + +# Note: the following cases are not included, since they would case-fold in lowercasing + +# 03C3; 03C2; 03A3; 03A3; Final_Sigma; # GREEK SMALL LETTER SIGMA +# 03C2; 03C3; 03A3; 03A3; Not_Final_Sigma; # GREEK SMALL LETTER FINAL SIGMA + +# ================================================================================ +# Language-Sensitive Mappings +# These are characters whose full case mappings depend on language and perhaps also +# context (which characters come before or after). For more information +# see the header of this file and the Unicode Standard. +# ================================================================================ + +# Lithuanian + +# Lithuanian retains the dot in a lowercase i when followed by accents. + +# Remove DOT ABOVE after "i" with upper or titlecase + +0307; 0307; ; ; lt After_Soft_Dotted; # COMBINING DOT ABOVE + +# Introduce an explicit dot above when lowercasing capital I's and J's +# whenever there are more accents above. +# (of the accents used in Lithuanian: grave, acute, tilde above, and ogonek) + +0049; 0069 0307; 0049; 0049; lt More_Above; # LATIN CAPITAL LETTER I +004A; 006A 0307; 004A; 004A; lt More_Above; # LATIN CAPITAL LETTER J +012E; 012F 0307; 012E; 012E; lt More_Above; # LATIN CAPITAL LETTER I WITH OGONEK +00CC; 0069 0307 0300; 00CC; 00CC; lt; # LATIN CAPITAL LETTER I WITH GRAVE +00CD; 0069 0307 0301; 00CD; 00CD; lt; # LATIN CAPITAL LETTER I WITH ACUTE +0128; 0069 0307 0303; 0128; 0128; lt; # LATIN CAPITAL LETTER I WITH TILDE + +# ================================================================================ + +# Turkish and Azeri + +# I and i-dotless; I-dot and i are case pairs in Turkish and Azeri +# The following rules handle those cases. + +0130; 0069; 0130; 0130; tr; # LATIN CAPITAL LETTER I WITH DOT ABOVE +0130; 0069; 0130; 0130; az; # LATIN CAPITAL LETTER I WITH DOT ABOVE + +# When lowercasing, remove dot_above in the sequence I + dot_above, which will turn into i. +# This matches the behavior of the canonically equivalent I-dot_above + +0307; ; 0307; 0307; tr After_I; # COMBINING DOT ABOVE +0307; ; 0307; 0307; az After_I; # COMBINING DOT ABOVE + +# When lowercasing, unless an I is before a dot_above, it turns into a dotless i. + +0049; 0131; 0049; 0049; tr Not_Before_Dot; # LATIN CAPITAL LETTER I +0049; 0131; 0049; 0049; az Not_Before_Dot; # LATIN CAPITAL LETTER I + +# When uppercasing, i turns into a dotted capital I + +0069; 0069; 0130; 0130; tr; # LATIN SMALL LETTER I +0069; 0069; 0130; 0130; az; # LATIN SMALL LETTER I + +# Note: the following case is already in the UnicodeData file. + +# 0131; 0131; 0049; 0049; tr; # LATIN SMALL LETTER DOTLESS I + +# EOF + diff --git a/tests/Makefile b/tests/Makefile new file mode 100644 --- /dev/null +++ b/tests/Makefile @@ -0,0 +1,69 @@ +version := $(shell awk '/^version:/{print $$2}' ../text.cabal) +ghc := ghc +ghc-opt-flags = -O0 +ghc-base-flags := -funbox-strict-fields -package criterion \ + -package bytestring -package QuickCheck -package test-framework \ + -package test-framework-quickcheck -ignore-package text \ + -fno-ignore-asserts +ghc-base-flags += -Wall -fno-warn-orphans -fno-warn-missing-signatures +ghc-flags := $(ghc-base-flags) -i../dist/build -package-name text-$(version) +ghc-hpc-flags := $(ghc-base-flags) -fhpc -fno-ignore-asserts -odir hpcdir \ + -hidir hpcdir -i.. +lib := ../dist/build/libHStext-$(version).a +lib-srcs := $(shell grep '^ *Data' ../text.cabal | \ + sed -e 's,\.,/,g' -e 's,$$,.hs,') + +cabal := $(shell which cabal 2>/dev/null) + +all: qc coverage + +lib: $(lib) + +$(lib): $(lib-srcs:%=../%) +ifeq ($(cabal),) + cd .. && runghc Setup configure --user --prefix=$(HOME) + cd .. && runghc Setup build +else + cd .. && cabal configure + cd .. && cabal build +endif + +Properties.o: QuickCheckUtils.o SlowFunctions.o + +QuickCheckUtils.o: $(lib) + +qc: Properties.o QuickCheckUtils.o SlowFunctions.o + $(ghc) $(ghc-flags) -threaded -o $@ $^ $(lib) + +sb: SearchBench.o + $(ghc) $(ghc-flags) -threaded -o $@ $^ $(lib) + +qc-hpc: Properties.hs QuickCheckUtils.hs $(lib-srcs:%=../%) + -mkdir -p hpcdir + @rm -f $@.tix + $(ghc) $(ghc-hpc-flags) $(ghc-opt-flags) -ihpcdir \ + --make -threaded -o $@ $< + +coverage: qc-hpc-html/hpc_index.html + +qc-hpc-html/hpc_index.html: qc-hpc + ./qc-hpc -a 100 +RTS -N2 + hpc markup qc-hpc --exclude=Main --exclude=Properties \ + --exclude=Data.Text.Fusion.CaseMapping \ + --exclude=QuickCheckUtils --srcdir=.. --srcdir=. --destdir=$(dir $@) + +SearchBench.o: ghc-opt-flags = -O +%.o: %.hs + $(ghc) $(ghc-flags) $(ghc-opt-flags) -c -o $@ $< + +hpcdir/%.o: %.hs + $(ghc) $(ghc-hpc-flags) $(ghc-opt-flags) --make -c -o $@ $< + +.PHONY: testdata +testdata: text-testdata.tar.bz2 + +text-testdata.tar.bz2: + curl -O http://projects.haskell.org/text/text-testdata.tar.bz2 + +clean: + -rm -rf *.o *.hi *.tix qc qc-hpc hpcdir .hpc qc-hpc-html diff --git a/tests/Properties.hs b/tests/Properties.hs new file mode 100644 --- /dev/null +++ b/tests/Properties.hs @@ -0,0 +1,888 @@ +{-# LANGUAGE BangPatterns, FlexibleInstances, OverloadedStrings, + ScopedTypeVariables, TypeSynonymInstances #-} +{-# OPTIONS_GHC -fno-enable-rewrite-rules #-} + +import Test.QuickCheck hiding (evaluate) +import Text.Show.Functions () + +import qualified Data.Bits as Bits (shiftL, shiftR) +import Data.Char (chr, isLower, isSpace, isUpper, ord) +import Data.Monoid (Monoid(..)) +import Data.String (fromString) +import Debug.Trace (trace) +import Control.Arrow ((***), second) +import Data.Word (Word8, Word16, Word32) +import qualified Data.Text as T +import qualified Data.Text.Lazy as TL +import qualified Data.Text.Encoding as E +import Control.Exception (SomeException, evaluate, try) +import qualified Data.Text.Fusion as S +import qualified Data.Text.Fusion.Common as S +import Data.Text.Fusion.Size +import qualified Data.Text.Lazy.Encoding as EL +import qualified Data.Text.Lazy.Fusion as SL +import qualified Data.Text.UnsafeShift as U +import qualified Data.List as L +import Prelude hiding (replicate) +import System.IO.Unsafe (unsafePerformIO) +import Test.Framework (defaultMain, testGroup) +import Test.Framework.Providers.QuickCheck (testProperty) +import Data.Text.Search (indices) +import qualified Data.Text.Lazy.Search as S (indices) +import qualified SlowFunctions as Slow + +import QuickCheckUtils (NotEmpty(..), small) + +-- Ensure that two potentially bottom values (in the sense of crashing +-- for some inputs, not looping infinitely) either both crash, or both +-- give comparable results for some input. +(=^=) :: (Eq a, Show a) => a -> a -> Bool +{-# NOINLINE (=^=) #-} +i =^= j = unsafePerformIO $ do + x <- try (evaluate i) + y <- try (evaluate j) + case (x,y) of + (Left (_ :: SomeException), Left (_ :: SomeException)) + -> return True + (Right a, Right b) -> return (a == b) + e -> trace ("*** Divergence: " ++ show e) return False +infix 4 =^= + +t_pack_unpack = (T.unpack . T.pack) `eq` id +tl_pack_unpack = (TL.unpack . TL.pack) `eq` id +t_stream_unstream = (S.unstream . S.stream) `eq` id +tl_stream_unstream = (SL.unstream . SL.stream) `eq` id +t_reverse_stream t = (S.reverse . S.reverseStream) t == t +t_singleton c = [c] == (T.unpack . T.singleton) c +tl_singleton c = [c] == (TL.unpack . TL.singleton) c +tl_unstreamChunks x = f 11 x == f 1000 x + where f n = SL.unstreamChunks n . S.streamList +tl_chunk_unchunk = (TL.fromChunks . TL.toChunks) `eq` id + +t_ascii t = E.decodeASCII (E.encodeUtf8 a) == a + where a = T.map (\c -> chr (ord c `mod` 128)) t +t_utf8 = (E.decodeUtf8 . E.encodeUtf8) `eq` id +tl_utf8 = (EL.decodeUtf8 . EL.encodeUtf8) `eq` id +t_utf16LE = (E.decodeUtf16LE . E.encodeUtf16LE) `eq` id +t_utf16BE = (E.decodeUtf16BE . E.encodeUtf16BE) `eq` id +t_utf32LE = (E.decodeUtf32LE . E.encodeUtf32LE) `eq` id +t_utf32BE = (E.decodeUtf32BE . E.encodeUtf32BE) `eq` id + +class Stringy s where + packS :: String -> s + unpackS :: s -> String + splitAtS :: Int -> s -> (s,s) + packSChunkSize :: Int -> String -> s + packSChunkSize _ = packS + +instance Stringy String where + packS = id + unpackS = id + splitAtS = splitAt + +instance Stringy (S.Stream Char) where + packS = S.streamList + unpackS = S.unstreamList + splitAtS n s = (S.take n s, S.drop n s) + +instance Stringy T.Text where + packS = T.pack + unpackS = T.unpack + splitAtS = T.splitAt + +instance Stringy TL.Text where + packSChunkSize k = SL.unstreamChunks k . S.streamList + packS = TL.pack + unpackS = TL.unpack + splitAtS = TL.splitAt . fromIntegral + +-- Do two functions give the same answer? +eq :: (Eq a, Show a) => (t -> a) -> (t -> a) -> t -> Bool +eq a b s = a s =^= b s + +-- What about with the RHS packed? +eqP :: (Eq a, Show a, Stringy s) => + (String -> a) -> (s -> a) -> String -> Word8 -> Bool +eqP f g s w = eql "orig" (f s) (g t) && + eql "mini" (f s) (g mini) && + eql "head" (f sa) (g ta) && + eql "tail" (f sb) (g tb) + where t = packS s + mini = packSChunkSize 10 s + (sa,sb) = splitAt m s + (ta,tb) = splitAtS m t + l = length s + m | l == 0 = n + | otherwise = n `mod` l + n = fromIntegral w + eql d a b | a =^= b = True + | otherwise = trace (d ++ ": " ++ show a ++ " /= " ++ show b) False + +-- For tests that have O(n^2) running times or input sizes, resize +-- their inputs to the square root of the originals. +unsquare :: (Arbitrary a, Show a, Testable b) => (a -> b) -> Property +unsquare = forAll . sized $ \n -> resize (smallish n) arbitrary + where smallish = round . (sqrt :: Double -> Double) . fromIntegral + +s_Eq s = (s==) `eq` ((S.streamList s==) . S.streamList) + where _types = s :: String +sf_Eq p s = + ((L.filter p s==) . L.filter p) `eq` + (((S.filter p $ S.streamList s)==) . S.filter p . S.streamList) +t_Eq s = (s==) `eq` ((T.pack s==) . T.pack) +tl_Eq s = (s==) `eq` ((TL.pack s==) . TL.pack) +s_Ord s = (compare s) `eq` (compare (S.streamList s) . S.streamList) + where _types = s :: String +sf_Ord p s = + ((compare $ L.filter p s) . L.filter p) `eq` + (compare (S.filter p $ S.streamList s) . S.filter p . S.streamList) +t_Ord s = (compare s) `eq` (compare (T.pack s) . T.pack) +tl_Ord s = (compare s) `eq` (compare (TL.pack s) . TL.pack) +t_Read = id `eq` (T.unpack . read . show) +tl_Read = id `eq` (TL.unpack . read . show) +t_Show = show `eq` (show . T.pack) +tl_Show = show `eq` (show . TL.pack) +t_mappend s = mappend s`eqP` (unpackS . mappend (T.pack s)) +tl_mappend s = mappend s`eqP` (unpackS . mappend (TL.pack s)) +t_mconcat = unsquare (mconcat `eq` (unpackS . mconcat . L.map T.pack)) +tl_mconcat = unsquare (mconcat `eq` (unpackS . mconcat . L.map TL.pack)) +t_IsString = fromString `eqP` (T.unpack . fromString) +tl_IsString = fromString `eqP` (TL.unpack . fromString) + +s_cons x = (x:) `eqP` (unpackS . S.cons x) +s_cons_s x = (x:) `eqP` (unpackS . S.unstream . S.cons x) +sf_cons p x = ((x:) . L.filter p) `eqP` (unpackS . S.cons x . S.filter p) +t_cons x = (x:) `eqP` (unpackS . T.cons x) +tl_cons x = (x:) `eqP` (unpackS . TL.cons x) +s_snoc x = (++ [x]) `eqP` (unpackS . (flip S.snoc) x) +t_snoc x = (++ [x]) `eqP` (unpackS . (flip T.snoc) x) +tl_snoc x = (++ [x]) `eqP` (unpackS . (flip TL.snoc) x) +s_append s = (s++) `eqP` (unpackS . S.append (S.streamList s)) +s_append_s s = (s++) `eqP` (unpackS . S.unstream . S.append (S.streamList s)) +sf_append p s = (L.filter p s++) `eqP` (unpackS . S.append (S.filter p $ S.streamList s)) +t_append s = (s++) `eqP` (unpackS . T.append (packS s)) + +uncons (x:xs) = Just (x,xs) +uncons _ = Nothing + +s_uncons = uncons `eqP` (fmap (second unpackS) . S.uncons) +sf_uncons p = (uncons . L.filter p) `eqP` (fmap (second unpackS) . S.uncons . S.filter p) +t_uncons = uncons `eqP` (fmap (second unpackS) . T.uncons) +tl_uncons = uncons `eqP` (fmap (second unpackS) . TL.uncons) +s_head = head `eqP` S.head +sf_head p = (head . L.filter p) `eqP` (S.head . S.filter p) +t_head = head `eqP` T.head +tl_head = head `eqP` TL.head +s_last = last `eqP` S.last +sf_last p = (last . L.filter p) `eqP` (S.last . S.filter p) +t_last = last `eqP` T.last +tl_last = last `eqP` TL.last +s_tail = tail `eqP` (unpackS . S.tail) +s_tail_s = tail `eqP` (unpackS . S.unstream . S.tail) +sf_tail p = (tail . L.filter p) `eqP` (unpackS . S.tail . S.filter p) +t_tail = tail `eqP` (unpackS . T.tail) +tl_tail = tail `eqP` (unpackS . TL.tail) +s_init = init `eqP` (unpackS . S.init) +s_init_s = init `eqP` (unpackS . S.unstream . S.init) +sf_init p = (init . L.filter p) `eqP` (unpackS . S.init . S.filter p) +t_init = init `eqP` (unpackS . T.init) +tl_init = init `eqP` (unpackS . TL.init) +s_null = null `eqP` S.null +sf_null p = (null . L.filter p) `eqP` (S.null . S.filter p) +t_null = null `eqP` T.null +tl_null = null `eqP` TL.null +s_length = length `eqP` S.length +sf_length p = (length . L.filter p) `eqP` (S.length . S.filter p) +t_length = length `eqP` T.length +tl_length = length `eqP` (fromIntegral . TL.length) + +s_map f = map f `eqP` (unpackS . S.map f) +s_map_s f = map f `eqP` (unpackS . S.unstream . S.map f) +sf_map p f = (map f . L.filter p) `eqP` (unpackS . S.map f . S.filter p) +t_map f = map f `eqP` (unpackS . T.map f) +tl_map f = map f `eqP` (unpackS . TL.map f) +t_intercalate c = unsquare (L.intercalate c `eq` (unpackS . T.intercalate (packS c) . map packS)) +tl_intercalate c = unsquare (L.intercalate c `eq` (unpackS . TL.intercalate (TL.pack c) . map TL.pack)) +s_intersperse c = L.intersperse c `eqP` (unpackS . S.intersperse c) +s_intersperse_s c = L.intersperse c `eqP` (unpackS . S.unstream . S.intersperse c) +sf_intersperse p c= (L.intersperse c . L.filter p) `eqP` (unpackS . S.intersperse c . S.filter p) +t_intersperse c = L.intersperse c `eqP` (unpackS . T.intersperse c) +tl_intersperse c = L.intersperse c `eqP` (unpackS . TL.intersperse c) +t_transpose = unsquare (L.transpose `eq` (map unpackS . T.transpose . map packS)) +tl_transpose = unsquare (L.transpose `eq` (map unpackS . TL.transpose . map TL.pack)) +t_reverse = L.reverse `eqP` (unpackS . T.reverse) +tl_reverse = L.reverse `eqP` (unpackS . TL.reverse) +t_reverse_short n = L.reverse `eqP` (unpackS . S.reverse . shorten n . S.stream) + +t_replace s d = (L.intercalate d . split s) `eqP` (unpackS . T.replace (T.pack s) (T.pack d)) +tl_replace s d = (L.intercalate d . split s) `eqP` (unpackS . TL.replace (TL.pack s) (TL.pack d)) + +split :: (Eq a) => [a] -> [a] -> [[a]] +split pat src0 + | l == 0 = error "empty" + | otherwise = go src0 + where + l = length pat + go src = search 0 src + where + search _ [] = [src] + search !n s@(_:s') + | pat `L.isPrefixOf` s = take n src : go (drop l s) + | otherwise = search (n+1) s' + +s_toCaseFold_length xs = S.length (S.toCaseFold s) >= length xs + where s = S.streamList xs +sf_toCaseFold_length p xs = + (S.length . S.toCaseFold . S.filter p $ s) >= (length . L.filter p $ xs) + where s = S.streamList xs +t_toCaseFold_length t = T.length (T.toCaseFold t) >= T.length t +tl_toCaseFold_length t = TL.length (TL.toCaseFold t) >= TL.length t +t_toLower_length t = T.length (T.toLower t) >= T.length t +t_toLower_lower t = p (T.toLower t) >= p t + where p = T.length . T.filter isLower +tl_toLower_lower t = p (TL.toLower t) >= p t + where p = TL.length . TL.filter isLower +t_toUpper_length t = T.length (T.toUpper t) >= T.length t +t_toUpper_upper t = p (T.toUpper t) >= p t + where p = T.length . T.filter isUpper +tl_toUpper_upper t = p (TL.toUpper t) >= p t + where p = TL.length . TL.filter isUpper + +justifyLeft k c s = s ++ L.replicate (k - length s) c +justifyRight m n s = L.replicate (m - length s) n ++ s + +s_justifyLeft k c = justifyLeft k c `eqP` (unpackS . S.justifyLeftI k c) +s_justifyLeft_s k c = justifyLeft k c `eqP` (unpackS . S.unstream . S.justifyLeftI k c) +sf_justifyLeft p k c = + (justifyLeft k c . L.filter p) `eqP` (unpackS . S.justifyLeftI k c . S.filter p) +t_justifyLeft k c = justifyLeft k c `eqP` (unpackS . T.justifyLeft k c) +tl_justifyLeft k c = justifyLeft k c `eqP` (unpackS . TL.justifyLeft (fromIntegral k) c) +t_justifyRight k c = justifyRight k c `eqP` (unpackS . T.justifyRight k c) +tl_justifyRight k c = justifyRight k c `eqP` (unpackS . TL.justifyRight (fromIntegral k) c) + +sf_foldl p f z = (L.foldl f z . L.filter p) `eqP` (S.foldl f z . S.filter p) + where _types = f :: Char -> Char -> Char +t_foldl f z = L.foldl f z `eqP` (T.foldl f z) + where _types = f :: Char -> Char -> Char +tl_foldl f z = L.foldl f z `eqP` (TL.foldl f z) + where _types = f :: Char -> Char -> Char +sf_foldl' p f z = (L.foldl' f z . L.filter p) `eqP` (S.foldl' f z . S.filter p) + where _types = f :: Char -> Char -> Char +t_foldl' f z = L.foldl' f z `eqP` T.foldl' f z + where _types = f :: Char -> Char -> Char +tl_foldl' f z = L.foldl' f z `eqP` TL.foldl' f z + where _types = f :: Char -> Char -> Char +sf_foldl1 p f = (L.foldl1 f . L.filter p) `eqP` (S.foldl1 f . S.filter p) +t_foldl1 f = L.foldl1 f `eqP` T.foldl1 f +tl_foldl1 f = L.foldl1 f `eqP` TL.foldl1 f +sf_foldl1' p f = (L.foldl1' f . L.filter p) `eqP` (S.foldl1' f . S.filter p) +t_foldl1' f = L.foldl1' f `eqP` T.foldl1' f +tl_foldl1' f = L.foldl1' f `eqP` TL.foldl1' f +sf_foldr p f z = (L.foldr f z . L.filter p) `eqP` (S.foldr f z . S.filter p) + where _types = f :: Char -> Char -> Char +t_foldr f z = L.foldr f z `eqP` T.foldr f z + where _types = f :: Char -> Char -> Char +tl_foldr f z = L.foldr f z `eqP` TL.foldr f z + where _types = f :: Char -> Char -> Char +sf_foldr1 p f = (L.foldr1 f . L.filter p) `eqP` (S.foldr1 f . S.filter p) +t_foldr1 f = L.foldr1 f `eqP` T.foldr1 f +tl_foldr1 f = L.foldr1 f `eqP` TL.foldr1 f + +s_concat_s = unsquare (L.concat `eq` (unpackS . S.unstream . S.concat . map packS)) +sf_concat p = unsquare ((L.concat . map (L.filter p)) `eq` (unpackS . S.concat . map (S.filter p . packS))) +t_concat = unsquare (L.concat `eq` (unpackS . T.concat . map packS)) +tl_concat = unsquare (L.concat `eq` (unpackS . TL.concat . map TL.pack)) +sf_concatMap p f = unsquare ((L.concatMap f . L.filter p) `eqP` (unpackS . S.concatMap (packS . f) . S.filter p)) +t_concatMap f = unsquare (L.concatMap f `eqP` (unpackS . T.concatMap (packS . f))) +tl_concatMap f = unsquare (L.concatMap f `eqP` (unpackS . TL.concatMap (TL.pack . f))) +sf_any q p = (L.any p . L.filter q) `eqP` (S.any p . S.filter q) +t_any p = L.any p `eqP` T.any p +tl_any p = L.any p `eqP` TL.any p +sf_all q p = (L.all p . L.filter q) `eqP` (S.all p . S.filter q) +t_all p = L.all p `eqP` T.all p +tl_all p = L.all p `eqP` TL.all p +sf_maximum p = (L.maximum . L.filter p) `eqP` (S.maximum . S.filter p) +t_maximum = L.maximum `eqP` T.maximum +tl_maximum = L.maximum `eqP` TL.maximum +sf_minimum p = (L.minimum . L.filter p) `eqP` (S.minimum . S.filter p) +t_minimum = L.minimum `eqP` T.minimum +tl_minimum = L.minimum `eqP` TL.minimum + +sf_scanl p f z = (L.scanl f z . L.filter p) `eqP` (unpackS . S.scanl f z . S.filter p) +t_scanl f z = L.scanl f z `eqP` (unpackS . T.scanl f z) +tl_scanl f z = L.scanl f z `eqP` (unpackS . TL.scanl f z) +t_scanl1 f = L.scanl1 f `eqP` (unpackS . T.scanl1 f) +tl_scanl1 f = L.scanl1 f `eqP` (unpackS . TL.scanl1 f) +t_scanr f z = L.scanr f z `eqP` (unpackS . T.scanr f z) +tl_scanr f z = L.scanr f z `eqP` (unpackS . TL.scanr f z) +t_scanr1 f = L.scanr1 f `eqP` (unpackS . T.scanr1 f) +tl_scanr1 f = L.scanr1 f `eqP` (unpackS . TL.scanr1 f) + +t_mapAccumL f z = unsquare (L.mapAccumL f z `eqP` (second unpackS . T.mapAccumL f z)) + where _types = f :: Int -> Char -> (Int,Char) +tl_mapAccumL f z = unsquare (L.mapAccumL f z `eqP` (second unpackS . TL.mapAccumL f z)) + where _types = f :: Int -> Char -> (Int,Char) +t_mapAccumR f z = unsquare (L.mapAccumR f z `eqP` (second unpackS . T.mapAccumR f z)) + where _types = f :: Int -> Char -> (Int,Char) +tl_mapAccumR f z = unsquare (L.mapAccumR f z `eqP` (second unpackS . TL.mapAccumR f z)) + where _types = f :: Int -> Char -> (Int,Char) + +replicate n l = concat (L.replicate n l) + +t_replicate n = replicate n `eq` (unpackS . T.replicate n . packS) +tl_replicate n = replicate n `eq` (unpackS . TL.replicate (fromIntegral n) . packS) + +unf :: Int -> Char -> Maybe (Char, Char) +unf n c | fromEnum c * 100 > n = Nothing + | otherwise = Just (c, succ c) + +t_unfoldr n = L.unfoldr (unf n) `eq` (unpackS . T.unfoldr (unf n)) +tl_unfoldr n = L.unfoldr (unf n) `eq` (unpackS . TL.unfoldr (unf n)) +t_unfoldrN n m = (L.take n . L.unfoldr (unf m)) `eq` + (unpackS . T.unfoldrN n (unf m)) +tl_unfoldrN n m = (L.take n . L.unfoldr (unf m)) `eq` + (unpackS . TL.unfoldrN (fromIntegral n) (unf m)) + +unpack2 :: (Stringy s) => (s,s) -> (String,String) +unpack2 = unpackS *** unpackS + +s_take n = L.take n `eqP` (unpackS . S.take n) +s_take_s m = L.take n `eqP` (unpackS . S.unstream . S.take n) + where n = small m +sf_take p n = (L.take n . L.filter p) `eqP` (unpackS . S.take n . S.filter p) +t_take n = L.take n `eqP` (unpackS . T.take n) +tl_take n = L.take n `eqP` (unpackS . TL.take (fromIntegral n)) +s_drop n = L.drop n `eqP` (unpackS . S.drop n) +s_drop_s m = L.drop n `eqP` (unpackS . S.unstream . S.drop n) + where n = small m +sf_drop p n = (L.drop n . L.filter p) `eqP` (unpackS . S.drop n . S.filter p) +t_drop n = L.drop n `eqP` (unpackS . T.drop n) +tl_drop n = L.drop n `eqP` (unpackS . TL.drop (fromIntegral n)) +s_take_drop m = (L.take n . L.drop n) `eqP` (unpackS . S.take n . S.drop n) + where n = small m +s_take_drop_s m = (L.take n . L.drop n) `eqP` (unpackS . S.unstream . S.take n . S.drop n) + where n = small m +s_takeWhile p = L.takeWhile p `eqP` (unpackS . S.takeWhile p) +s_takeWhile_s p = L.takeWhile p `eqP` (unpackS . S.unstream . S.takeWhile p) +sf_takeWhile q p = (L.takeWhile p . L.filter q) `eqP` (unpackS . S.takeWhile p . S.filter q) +t_takeWhile p = L.takeWhile p `eqP` (unpackS . T.takeWhile p) +tl_takeWhile p = L.takeWhile p `eqP` (unpackS . TL.takeWhile p) +s_dropWhile p = L.dropWhile p `eqP` (unpackS . S.dropWhile p) +s_dropWhile_s p = L.dropWhile p `eqP` (unpackS . S.unstream . S.dropWhile p) +sf_dropWhile q p = (L.dropWhile p . L.filter q) `eqP` (unpackS . S.dropWhile p . S.filter q) +t_dropWhile p = L.dropWhile p `eqP` (unpackS . T.dropWhile p) +tl_dropWhile p = L.dropWhile p `eqP` (unpackS . S.dropWhile p) +t_dropWhileEnd p = (L.reverse . L.dropWhile p . L.reverse) `eqP` (unpackS . T.dropWhileEnd p) +tl_dropWhileEnd p = (L.reverse . L.dropWhile p . L.reverse) `eqP` (unpackS . TL.dropWhileEnd p) +t_dropAround p = (L.dropWhile p . L.reverse . L.dropWhile p . L.reverse) `eqP` (unpackS . T.dropAround p) +tl_dropAround p = (L.dropWhile p . L.reverse . L.dropWhile p . L.reverse) `eqP` (unpackS . TL.dropAround p) +t_stripStart = T.dropWhile isSpace `eq` T.stripStart +tl_stripStart = TL.dropWhile isSpace `eq` TL.stripStart +t_stripEnd = T.dropWhileEnd isSpace `eq` T.stripEnd +tl_stripEnd = TL.dropWhileEnd isSpace `eq` TL.stripEnd +t_strip = T.dropAround isSpace `eq` T.strip +tl_strip = TL.dropAround isSpace `eq` TL.strip +t_splitAt n = L.splitAt n `eqP` (unpack2 . T.splitAt n) +tl_splitAt n = L.splitAt n `eqP` (unpack2 . TL.splitAt (fromIntegral n)) +t_spanBy p = L.span p `eqP` (unpack2 . T.spanBy p) +tl_spanBy p = L.span p `eqP` (unpack2 . TL.spanBy p) + +t_break_id s = squid `eq` (uncurry T.append . T.break s) + where squid t | T.null s = error "empty" + | otherwise = t +tl_break_id s = squid `eq` (uncurry TL.append . TL.break s) + where squid t | TL.null s = error "empty" + | otherwise = t +t_break_start (NotEmpty s) t = let (_,m) = T.break s t + in T.null m || s `T.isPrefixOf` m +tl_break_start (NotEmpty s) t = let (_,m) = TL.break s t + in TL.null m || s `TL.isPrefixOf` m +t_breakBy p = L.break p `eqP` (unpack2 . T.breakBy p) +tl_breakBy p = L.break p `eqP` (unpack2 . TL.breakBy p) +t_group = L.group `eqP` (map unpackS . T.group) +tl_group = L.group `eqP` (map unpackS . TL.group) +t_groupBy p = L.groupBy p `eqP` (map unpackS . T.groupBy p) +tl_groupBy p = L.groupBy p `eqP` (map unpackS . TL.groupBy p) +t_inits = L.inits `eqP` (map unpackS . T.inits) +tl_inits = L.inits `eqP` (map unpackS . TL.inits) +t_tails = L.tails `eqP` (map unpackS . T.tails) +tl_tails = L.tails `eqP` (map unpackS . TL.tails) + +t_findSplit s t = (T.split s `eq` splitty) u + where splitty v = case T.find s v of + (x,xs) -> x : L.map (T.drop (T.length s) . fst) xs + u = T.concat [t,s,t] +tl_findSplit s t = (TL.split s `eq` splitty) u + where splitty v = case TL.find s v of + (x,xs) -> x : L.map (TL.drop (TL.length s) . fst) xs + u = TL.concat [t,s,t] +t_split_split s = unsquare ((T.split s `eq` Slow.split s) . + T.intercalate s) +tl_split_split s = unsquare (((TL.split (chunkify s) . chunkify) `eq` + (map chunkify . T.split s)) . + T.intercalate s) +t_split_i (NotEmpty t) = id `eq` (T.intercalate t . T.split t) +tl_split_i (NotEmpty t) = id `eq` (TL.intercalate t . TL.split t) + +t_splitBy p = splitBy p `eqP` (map unpackS . T.splitBy p) +t_splitBy_count c = (L.length . T.splitBy (==c)) `eq` ((1+) . T.count (T.singleton c)) +t_splitBy_split c = T.splitBy (==c) `eq` T.split (T.singleton c) +tl_splitBy p = splitBy p `eqP` (map unpackS . TL.splitBy p) + +splitBy :: (a -> Bool) -> [a] -> [[a]] +splitBy _ [] = [[]] +splitBy p xs = loop xs + where loop s | null s' = [l] + | otherwise = l : loop (tail s') + where (l, s') = break p s + +t_chunksOf_same_lengths k = all ((==k) . T.length) . ini . T.chunksOf k + where ini [] = [] + ini xs = init xs + +t_chunksOf_length k t = len == T.length t || (k <= 0 && len == 0) + where len = L.sum . L.map T.length $ T.chunksOf k t + +chunkify = TL.fromChunks . (:[]) + +tl_chunksOf k = T.chunksOf k `eq` (map (T.concat . TL.toChunks) . TL.chunksOf (fromIntegral k) . chunkify) + +t_lines = L.lines `eqP` (map unpackS . T.lines) +tl_lines = L.lines `eqP` (map unpackS . TL.lines) +{- +t_lines' = lines' `eqP` (map unpackS . T.lines') + where lines' "" = [] + lines' s = let (l, s') = break eol s + in l : case s' of + [] -> [] + ('\r':'\n':s'') -> lines' s'' + (_:s'') -> lines' s'' + eol c = c == '\r' || c == '\n' +-} +t_words = L.words `eqP` (map unpackS . T.words) + +tl_words = L.words `eqP` (map unpackS . TL.words) +t_unlines = unsquare (L.unlines `eq` (unpackS . T.unlines . map packS)) +tl_unlines = unsquare (L.unlines `eq` (unpackS . TL.unlines . map packS)) +t_unwords = unsquare (L.unwords `eq` (unpackS . T.unwords . map packS)) +tl_unwords = unsquare (L.unwords `eq` (unpackS . TL.unwords . map packS)) + +s_isPrefixOf s = L.isPrefixOf s `eqP` (S.isPrefixOf (S.stream $ packS s) . S.stream) +sf_isPrefixOf p s = (L.isPrefixOf s . L.filter p) `eqP` (S.isPrefixOf (S.stream $ packS s) . S.filter p . S.stream) +t_isPrefixOf s = L.isPrefixOf s`eqP` T.isPrefixOf (packS s) +tl_isPrefixOf s = L.isPrefixOf s`eqP` TL.isPrefixOf (packS s) +t_isSuffixOf s = L.isSuffixOf s`eqP` T.isSuffixOf (packS s) +tl_isSuffixOf s = L.isSuffixOf s`eqP` TL.isSuffixOf (packS s) +t_isInfixOf s = L.isInfixOf s `eqP` T.isInfixOf (packS s) +tl_isInfixOf s = L.isInfixOf s `eqP` TL.isInfixOf (packS s) + +sf_elem p c = (L.elem c . L.filter p) `eqP` (S.elem c . S.filter p) +sf_filter q p = (L.filter p . L.filter q) `eqP` (unpackS . S.filter p . S.filter q) +t_filter p = L.filter p `eqP` (unpackS . T.filter p) +tl_filter p = L.filter p `eqP` (unpackS . TL.filter p) +sf_findBy q p = (L.find p . L.filter q) `eqP` (S.findBy p . S.filter q) +t_findBy p = L.find p `eqP` T.findBy p +tl_findBy p = L.find p `eqP` TL.findBy p +t_partition p = L.partition p `eqP` (unpack2 . T.partitionBy p) +tl_partition p = L.partition p `eqP` (unpack2 . TL.partitionBy p) + +sf_index p s = forAll (choose (-l,l*2)) ((L.filter p s L.!!) `eq` S.index (S.filter p $ packS s)) + where l = L.length s +t_index s = forAll (choose (-l,l*2)) ((s L.!!) `eq` T.index (packS s)) + where l = L.length s + +tl_index s = forAll (choose (-l,l*2)) ((s L.!!) `eq` (TL.index (packS s) . fromIntegral)) + where l = L.length s + +t_findIndex p = L.findIndex p `eqP` T.findIndex p +t_count t = (subtract 1 . L.length . T.split t) `eq` T.count t +tl_count t = (subtract 1 . L.genericLength . TL.split t) `eq` TL.count t +t_zip s = L.zip s `eqP` T.zip (packS s) +tl_zip s = L.zip s `eqP` TL.zip (packS s) +sf_zipWith p c s = (L.zipWith c (L.filter p s) . L.filter p) `eqP` (unpackS . S.zipWith c (S.filter p $ packS s) . S.filter p) +t_zipWith c s = L.zipWith c s `eqP` (unpackS . T.zipWith c (packS s)) +tl_zipWith c s = L.zipWith c s `eqP` (unpackS . TL.zipWith c (packS s)) + +t_indices = unsquare (\s -> Slow.indices s `eq` indices s) +tl_indices = unsquare (\s -> lazyIndices s `eq` S.indices s) + where lazyIndices s t = map fromIntegral $ Slow.indices (conc s) (conc t) + conc = T.concat . TL.toChunks +t_indices_occurs t = unsquare (\ts -> let s = T.intercalate t ts + in Slow.indices t s == indices t s) + +-- Bit shifts. +shiftL w = forAll (choose (0,width-1)) $ \k -> Bits.shiftL w k == U.shiftL w k + where width = round (log (fromIntegral m) / log 2 :: Double) + (m,_) = (maxBound, m == w) +shiftR w = forAll (choose (0,width-1)) $ \k -> Bits.shiftR w k == U.shiftR w k + where width = round (log (fromIntegral m) / log 2 :: Double) + (m,_) = (maxBound, m == w) + +shiftL_Int = shiftL :: Int -> Property +shiftL_Word16 = shiftL :: Word16 -> Property +shiftL_Word32 = shiftL :: Word32 -> Property +shiftR_Int = shiftR :: Int -> Property +shiftR_Word16 = shiftR :: Word16 -> Property +shiftR_Word32 = shiftR :: Word32 -> Property + +-- Regression tests. +s_filter_eq s = S.filter p t == S.streamList (filter p s) + where p = (/= S.last t) + t = S.streamList s + +-- Make a stream appear shorter than it really is, to ensure that +-- functions that consume inaccurately sized streams behave +-- themselves. +shorten :: Int -> S.Stream a -> S.Stream a +shorten n t@(S.Stream arr off len) + | n > 0 = S.Stream arr off (smaller (exactSize n) len) + | otherwise = t + +main = defaultMain tests + +tests = [ + testGroup "creation/elimination" [ + testProperty "t_pack_unpack" t_pack_unpack, + testProperty "tl_pack_unpack" tl_pack_unpack, + testProperty "t_stream_unstream" t_stream_unstream, + testProperty "tl_stream_unstream" tl_stream_unstream, + testProperty "t_reverse_stream" t_reverse_stream, + testProperty "t_singleton" t_singleton, + testProperty "tl_singleton" tl_singleton, + testProperty "tl_unstreamChunks" tl_unstreamChunks, + testProperty "tl_chunk_unchunk" tl_chunk_unchunk + ], + + testGroup "transcoding" [ + testProperty "t_ascii" t_ascii, + testProperty "t_utf8" t_utf8, + testProperty "tl_utf8" tl_utf8, + testProperty "t_utf16LE" t_utf16LE, + testProperty "t_utf16BE" t_utf16BE, + testProperty "t_utf32LE" t_utf32LE, + testProperty "t_utf32BE" t_utf32BE + ], + + testGroup "instances" [ + testProperty "s_Eq" s_Eq, + testProperty "sf_Eq" sf_Eq, + testProperty "t_Eq" t_Eq, + testProperty "tl_Eq" tl_Eq, + testProperty "s_Ord" s_Ord, + testProperty "sf_Ord" sf_Ord, + testProperty "t_Ord" t_Ord, + testProperty "tl_Ord" tl_Ord, + testProperty "t_Read" t_Read, + testProperty "tl_Read" tl_Read, + testProperty "t_Show" t_Show, + testProperty "tl_Show" tl_Show, + testProperty "t_mappend" t_mappend, + testProperty "tl_mappend" tl_mappend, + testProperty "t_mconcat" t_mconcat, + testProperty "tl_mconcat" tl_mconcat, + testProperty "t_IsString" t_IsString, + testProperty "tl_IsString" tl_IsString + ], + + testGroup "basics" [ + testProperty "s_cons" s_cons, + testProperty "s_cons_s" s_cons_s, + testProperty "sf_cons" sf_cons, + testProperty "t_cons" t_cons, + testProperty "tl_cons" tl_cons, + testProperty "s_snoc" s_snoc, + testProperty "t_snoc" t_snoc, + testProperty "tl_snoc" tl_snoc, + testProperty "s_append" s_append, + testProperty "s_append_s" s_append_s, + testProperty "sf_append" sf_append, + testProperty "t_append" t_append, + testProperty "s_uncons" s_uncons, + testProperty "sf_uncons" sf_uncons, + testProperty "t_uncons" t_uncons, + testProperty "tl_uncons" tl_uncons, + testProperty "s_head" s_head, + testProperty "sf_head" sf_head, + testProperty "t_head" t_head, + testProperty "tl_head" tl_head, + testProperty "s_last" s_last, + testProperty "sf_last" sf_last, + testProperty "t_last" t_last, + testProperty "tl_last" tl_last, + testProperty "s_tail" s_tail, + testProperty "s_tail_s" s_tail_s, + testProperty "sf_tail" sf_tail, + testProperty "t_tail" t_tail, + testProperty "tl_tail" tl_tail, + testProperty "s_init" s_init, + testProperty "s_init_s" s_init_s, + testProperty "sf_init" sf_init, + testProperty "t_init" t_init, + testProperty "tl_init" tl_init, + testProperty "s_null" s_null, + testProperty "sf_null" sf_null, + testProperty "t_null" t_null, + testProperty "tl_null" tl_null, + testProperty "s_length" s_length, + testProperty "sf_length" sf_length, + testProperty "t_length" t_length, + testProperty "tl_length" tl_length + ], + + testGroup "transformations" [ + testProperty "s_map" s_map, + testProperty "s_map_s" s_map_s, + testProperty "sf_map" sf_map, + testProperty "t_map" t_map, + testProperty "tl_map" tl_map, + testProperty "t_intercalate" t_intercalate, + testProperty "tl_intercalate" tl_intercalate, + testProperty "s_intersperse" s_intersperse, + testProperty "s_intersperse_s" s_intersperse_s, + testProperty "sf_intersperse" sf_intersperse, + testProperty "t_intersperse" t_intersperse, + testProperty "tl_intersperse" tl_intersperse, + testProperty "t_transpose" t_transpose, + testProperty "tl_transpose" tl_transpose, + testProperty "t_reverse" t_reverse, + testProperty "tl_reverse" tl_reverse, + testProperty "t_reverse_short" t_reverse_short, + testProperty "t_replace" t_replace, + testProperty "tl_replace" tl_replace, + + testGroup "case conversion" [ + testProperty "s_toCaseFold_length" s_toCaseFold_length, + testProperty "sf_toCaseFold_length" sf_toCaseFold_length, + testProperty "t_toCaseFold_length" t_toCaseFold_length, + testProperty "tl_toCaseFold_length" tl_toCaseFold_length, + testProperty "t_toLower_length" t_toLower_length, + testProperty "t_toLower_lower" t_toLower_lower, + testProperty "tl_toLower_lower" tl_toLower_lower, + testProperty "t_toUpper_length" t_toUpper_length, + testProperty "t_toUpper_upper" t_toUpper_upper, + testProperty "tl_toUpper_upper" tl_toUpper_upper + ], + + testGroup "justification" [ + testProperty "s_justifyLeft" s_justifyLeft, + testProperty "s_justifyLeft_s" s_justifyLeft_s, + testProperty "sf_justifyLeft" sf_justifyLeft, + testProperty "t_justifyLeft" t_justifyLeft, + testProperty "tl_justifyLeft" tl_justifyLeft, + testProperty "t_justifyRight" t_justifyRight, + testProperty "tl_justifyRight" tl_justifyRight + ] + ], + + testGroup "folds" [ + testProperty "sf_foldl" sf_foldl, + testProperty "t_foldl" t_foldl, + testProperty "tl_foldl" tl_foldl, + testProperty "sf_foldl'" sf_foldl', + testProperty "t_foldl'" t_foldl', + testProperty "tl_foldl'" tl_foldl', + testProperty "sf_foldl1" sf_foldl1, + testProperty "t_foldl1" t_foldl1, + testProperty "tl_foldl1" tl_foldl1, + testProperty "t_foldl1'" t_foldl1', + testProperty "sf_foldl1'" sf_foldl1', + testProperty "tl_foldl1'" tl_foldl1', + testProperty "sf_foldr" sf_foldr, + testProperty "t_foldr" t_foldr, + testProperty "tl_foldr" tl_foldr, + testProperty "sf_foldr1" sf_foldr1, + testProperty "t_foldr1" t_foldr1, + testProperty "tl_foldr1" tl_foldr1, + + testGroup "special" [ + testProperty "s_concat_s" s_concat_s, + testProperty "sf_concat" sf_concat, + testProperty "t_concat" t_concat, + testProperty "tl_concat" tl_concat, + testProperty "sf_concatMap" sf_concatMap, + testProperty "t_concatMap" t_concatMap, + testProperty "tl_concatMap" tl_concatMap, + testProperty "sf_any" sf_any, + testProperty "t_any" t_any, + testProperty "tl_any" tl_any, + testProperty "sf_all" sf_all, + testProperty "t_all" t_all, + testProperty "tl_all" tl_all, + testProperty "sf_maximum" sf_maximum, + testProperty "t_maximum" t_maximum, + testProperty "tl_maximum" tl_maximum, + testProperty "sf_minimum" sf_minimum, + testProperty "t_minimum" t_minimum, + testProperty "tl_minimum" tl_minimum + ] + ], + + testGroup "construction" [ + testGroup "scans" [ + testProperty "sf_scanl" sf_scanl, + testProperty "t_scanl" t_scanl, + testProperty "tl_scanl" tl_scanl, + testProperty "t_scanl1" t_scanl1, + testProperty "tl_scanl1" tl_scanl1, + testProperty "t_scanr" t_scanr, + testProperty "tl_scanr" tl_scanr, + testProperty "t_scanr1" t_scanr1, + testProperty "tl_scanr1" tl_scanr1 + ], + + testGroup "mapAccum" [ + testProperty "t_mapAccumL" t_mapAccumL, + testProperty "tl_mapAccumL" tl_mapAccumL, + testProperty "t_mapAccumR" t_mapAccumR, + testProperty "tl_mapAccumR" tl_mapAccumR + ], + + testGroup "unfolds" [ + testProperty "t_replicate" t_replicate, + testProperty "tl_replicate" tl_replicate, + testProperty "t_unfoldr" t_unfoldr, + testProperty "tl_unfoldr" tl_unfoldr, + testProperty "t_unfoldrN" t_unfoldrN, + testProperty "tl_unfoldrN" tl_unfoldrN + ] + ], + + testGroup "substrings" [ + testGroup "breaking" [ + testProperty "s_take" s_take, + testProperty "s_take_s" s_take_s, + testProperty "sf_take" sf_take, + testProperty "t_take" t_take, + testProperty "tl_take" tl_take, + testProperty "s_drop" s_drop, + testProperty "s_drop_s" s_drop_s, + testProperty "sf_drop" sf_drop, + testProperty "t_drop" t_drop, + testProperty "tl_drop" tl_drop, + testProperty "s_take_drop" s_take_drop, + testProperty "s_take_drop_s" s_take_drop_s, + testProperty "s_takeWhile" s_takeWhile, + testProperty "s_takeWhile_s" s_takeWhile_s, + testProperty "sf_takeWhile" sf_takeWhile, + testProperty "t_takeWhile" t_takeWhile, + testProperty "tl_takeWhile" tl_takeWhile, + testProperty "sf_dropWhile" sf_dropWhile, + testProperty "s_dropWhile" s_dropWhile, + testProperty "s_dropWhile_s" s_dropWhile_s, + testProperty "t_dropWhile" t_dropWhile, + testProperty "tl_dropWhile" tl_dropWhile, + testProperty "t_dropWhileEnd" t_dropWhileEnd, + testProperty "tl_dropWhileEnd" tl_dropWhileEnd, + testProperty "t_dropAround" t_dropAround, + testProperty "tl_dropAround" tl_dropAround, + testProperty "t_stripStart" t_stripStart, + testProperty "tl_stripStart" tl_stripStart, + testProperty "t_stripEnd" t_stripEnd, + testProperty "tl_stripEnd" tl_stripEnd, + testProperty "t_strip" t_strip, + testProperty "tl_strip" tl_strip, + testProperty "t_splitAt" t_splitAt, + testProperty "tl_splitAt" tl_splitAt, + testProperty "t_spanBy" t_spanBy, + testProperty "tl_spanBy" tl_spanBy, + testProperty "t_break_id" t_break_id, + testProperty "tl_break_id" tl_break_id, + testProperty "t_break_start" t_break_start, + testProperty "tl_break_start" tl_break_start, + testProperty "t_breakBy" t_breakBy, + testProperty "tl_breakBy" tl_breakBy, + testProperty "t_group" t_group, + testProperty "tl_group" tl_group, + testProperty "t_groupBy" t_groupBy, + testProperty "tl_groupBy" tl_groupBy, + testProperty "t_inits" t_inits, + testProperty "tl_inits" tl_inits, + testProperty "t_tails" t_tails, + testProperty "tl_tails" tl_tails + ], + + testGroup "breaking many" [ + testProperty "t_findSplit" t_findSplit, + testProperty "tl_findSplit" tl_findSplit, + testProperty "t_split_split" t_split_split, + testProperty "tl_split_split" tl_split_split, + testProperty "t_split_i" t_split_i, + testProperty "tl_split_i" tl_split_i, + testProperty "t_splitBy" t_splitBy, + testProperty "t_splitBy_count" t_splitBy_count, + testProperty "t_splitBy_split" t_splitBy_split, + testProperty "tl_splitBy" tl_splitBy, + testProperty "t_chunksOf_same_lengths" t_chunksOf_same_lengths, + testProperty "t_chunksOf_length" t_chunksOf_length, + testProperty "tl_chunksOf" tl_chunksOf + ], + + testGroup "lines and words" [ + testProperty "t_lines" t_lines, + testProperty "tl_lines" tl_lines, + --testProperty "t_lines'" t_lines', + testProperty "t_words" t_words, + testProperty "tl_words" tl_words, + testProperty "t_unlines" t_unlines, + testProperty "tl_unlines" tl_unlines, + testProperty "t_unwords" t_unwords, + testProperty "tl_unwords" tl_unwords + ] + ], + + testGroup "predicates" [ + testProperty "s_isPrefixOf" s_isPrefixOf, + testProperty "sf_isPrefixOf" sf_isPrefixOf, + testProperty "t_isPrefixOf" t_isPrefixOf, + testProperty "tl_isPrefixOf" tl_isPrefixOf, + testProperty "t_isSuffixOf" t_isSuffixOf, + testProperty "tl_isSuffixOf" tl_isSuffixOf, + testProperty "t_isInfixOf" t_isInfixOf, + testProperty "tl_isInfixOf" tl_isInfixOf + ], + + testGroup "searching" [ + testProperty "sf_elem" sf_elem, + testProperty "sf_filter" sf_filter, + testProperty "t_filter" t_filter, + testProperty "tl_filter" tl_filter, + testProperty "sf_findBy" sf_findBy, + testProperty "t_findBy" t_findBy, + testProperty "tl_findBy" tl_findBy, + testProperty "t_partition" t_partition, + testProperty "tl_partition" tl_partition + ], + + testGroup "indexing" [ + testProperty "sf_index" sf_index, + testProperty "t_index" t_index, + testProperty "tl_index" tl_index, + testProperty "t_findIndex" t_findIndex, + testProperty "t_count" t_count, + testProperty "tl_count" tl_count, + testProperty "t_indices" t_indices, + testProperty "tl_indices" tl_indices, + testProperty "t_indices_occurs" t_indices_occurs + ], + + testGroup "zips" [ + testProperty "t_zip" t_zip, + testProperty "tl_zip" tl_zip, + testProperty "sf_zipWith" sf_zipWith, + testProperty "t_zipWith" t_zipWith, + testProperty "tl_zipWith" tl_zipWith + ], + + testGroup "regressions" [ + testProperty "s_filter_eq" s_filter_eq + ], + + testGroup "shifts" [ + testProperty "shiftL_Int" shiftL_Int, + testProperty "shiftL_Word16" shiftL_Word16, + testProperty "shiftL_Word32" shiftL_Word32, + testProperty "shiftR_Int" shiftR_Int, + testProperty "shiftR_Word16" shiftR_Word16, + testProperty "shiftR_Word32" shiftR_Word32 + ] + ] diff --git a/tests/QuickCheckUtils.hs b/tests/QuickCheckUtils.hs new file mode 100644 --- /dev/null +++ b/tests/QuickCheckUtils.hs @@ -0,0 +1,128 @@ +{-# LANGUAGE FlexibleInstances #-} + +module QuickCheckUtils where + +import Control.Arrow (first) +import Data.Int (Int64) +import Data.Word (Word8, Word16, Word32) +import qualified Data.Text as T +import qualified Data.Text.Lazy as TL +import System.Random (Random(..), RandomGen) +import Test.QuickCheck (Arbitrary(..), choose, oneof, sized, variant, vector) +import qualified Data.ByteString as B + +instance Random Int64 where + randomR = integralRandomR + random = randomR (minBound,maxBound) + +instance Arbitrary Int64 where + arbitrary = choose (minBound,maxBound) + coarbitrary c = variant (fromEnum c `rem` 4) + +instance Random Word8 where + randomR = integralRandomR + random = randomR (minBound,maxBound) + +instance Arbitrary Word8 where + arbitrary = choose (minBound,maxBound) + coarbitrary c = variant (fromEnum c `rem` 4) + +instance Arbitrary B.ByteString where + arbitrary = B.pack `fmap` arbitrary + coarbitrary s = coarbitrary (B.unpack s) + +instance Random Word16 where + randomR = integralRandomR + random = randomR (minBound,maxBound) + +instance Arbitrary Word16 where + arbitrary = choose (minBound,maxBound) + coarbitrary c = variant (fromEnum c `rem` 4) + +instance Random Word32 where + randomR = integralRandomR + random = randomR (minBound,maxBound) + +instance Arbitrary Word32 where + arbitrary = choose (minBound,maxBound) + coarbitrary c = variant (fromEnum c `rem` 4) + +instance Arbitrary Char where + arbitrary = oneof [choose ('\0','\55295'), choose ('\57344','\1114111')] + coarbitrary c = variant (fromEnum c `rem` 4) + +instance Arbitrary T.Text where + arbitrary = T.pack `fmap` arbitrary + coarbitrary s = coarbitrary (T.unpack s) + +instance Arbitrary TL.Text where + arbitrary = TL.pack `fmap` arbitrary + coarbitrary s = coarbitrary (TL.unpack s) + +newtype NotEmpty a = NotEmpty { notEmpty :: a } + deriving (Eq, Ord, Show) + +instance Functor NotEmpty where + fmap f (NotEmpty a) = NotEmpty (f a) + +instance Arbitrary a => Arbitrary (NotEmpty [a]) where + arbitrary = sized (\n -> NotEmpty `fmap` (choose (1,n+1) >>= vector)) + coarbitrary = coarbitrary . notEmpty + +instance Arbitrary (NotEmpty T.Text) where + arbitrary = (fmap T.pack) `fmap` arbitrary + coarbitrary = coarbitrary . notEmpty + +instance Arbitrary (NotEmpty TL.Text) where + arbitrary = (fmap TL.pack) `fmap` arbitrary + coarbitrary = coarbitrary . notEmpty + +instance Arbitrary (NotEmpty B.ByteString) where + arbitrary = (fmap B.pack) `fmap` arbitrary + coarbitrary = coarbitrary . notEmpty + +data Small = S0 | S1 | S2 | S3 | S4 | S5 | S6 | S7 + | S8 | S9 | S10 | S11 | S12 | S13 | S14 | S15 + | S16 | S17 | S18 | S19 | S20 | S21 | S22 | S23 + | S24 | S25 | S26 | S27 | S28 | S29 | S30 | S31 + deriving (Eq, Ord, Enum, Bounded) + +small :: Small -> Int +small = fromEnum + +intf f a b = toEnum ((fromEnum a `f` fromEnum b) `mod` 32) + +instance Show Small where + show = show . fromEnum + +instance Read Small where + readsPrec n = map (first toEnum) . readsPrec n + +instance Num Small where + fromInteger = toEnum . fromIntegral + signum _ = 1 + abs = id + (+) = intf (+) + (-) = intf (-) + (*) = intf (*) + +instance Real Small where + toRational = toRational . fromEnum + +instance Integral Small where + toInteger = toInteger . fromEnum + quotRem a b = (toEnum x, toEnum y) + where (x, y) = fromEnum a `quotRem` fromEnum b + +instance Random Small where + randomR = integralRandomR + random = randomR (minBound,maxBound) + +instance Arbitrary Small where + arbitrary = choose (minBound,maxBound) + coarbitrary c = variant (fromEnum c `rem` 4) + +integralRandomR :: (Integral a, RandomGen g) => (a,a) -> g -> (a,g) +integralRandomR (a,b) g = case randomR (fromIntegral a :: Integer, + fromIntegral b :: Integer) g of + (x,h) -> (fromIntegral x, h) diff --git a/tests/SlowFunctions.hs b/tests/SlowFunctions.hs new file mode 100644 --- /dev/null +++ b/tests/SlowFunctions.hs @@ -0,0 +1,40 @@ +{-# LANGUAGE BangPatterns #-} + +module SlowFunctions + ( + indices + , split + ) where + +import qualified Data.Text as T +import Data.Text.Internal (Text(..)) +import Data.Text.Unsafe (iter_, unsafeHead, unsafeTail) + +indices :: T.Text -- ^ Substring to search for (@needle@) + -> T.Text -- ^ Text to search in (@haystack@) + -> [Int] +indices needle@(Text _narr _noff nlen) haystack@(Text harr hoff hlen) + | T.null needle = [] + | otherwise = scan 0 + where + scan i | i >= hlen = [] + | needle `T.isPrefixOf` t = i : scan (i+nlen) + | otherwise = scan (i+d) + where t = Text harr (hoff+i) (hlen-i) + d = iter_ haystack i + +split :: T.Text -- ^ Text to split on + -> T.Text -- ^ Input text + -> [T.Text] +split pat src0 + | T.null pat = error "split: empty" + | l == 1 = T.splitBy (== (unsafeHead pat)) src0 + | otherwise = go src0 + where + l = T.length pat + go src = search 0 src + where + search !n !s + | T.null s = [src] -- not found + | pat `T.isPrefixOf` s = T.take n src : go (T.drop l s) + | otherwise = search (n+1) (unsafeTail s) diff --git a/text.cabal b/text.cabal --- a/text.cabal +++ b/text.cabal @@ -1,5 +1,5 @@ name: text -version: 0.4 +version: 0.5 synopsis: An efficient packed Unicode text type description: An efficient packed Unicode text type. license: BSD3 @@ -12,8 +12,25 @@ category: Data, Text build-type: Simple cabal-version: >= 1.2 -extra-source-files: README +extra-source-files: + README + TODO + scripts/ApiCompare.hs + scripts/Arsec.hs + scripts/CaseFolding.hs + scripts/CaseFolding.txt + scripts/CaseMapping.hs + scripts/SpecialCasing.hs + scripts/SpecialCasing.txt + tests/Makefile + tests/Properties.hs + tests/QuickCheckUtils.hs + tests/SlowFunctions.hs +flag developer + description: operate in developer mode + default: False + library exposed-modules: Data.Text @@ -33,10 +50,13 @@ Data.Text.Fusion.CaseMapping Data.Text.Fusion.Common Data.Text.Fusion.Internal + Data.Text.Fusion.Size Data.Text.Internal Data.Text.Lazy.Encoding.Fusion Data.Text.Lazy.Fusion Data.Text.Lazy.Internal + Data.Text.Lazy.Search + Data.Text.Search Data.Text.Unsafe Data.Text.UnsafeChar Data.Text.UnsafeShift @@ -54,3 +74,5 @@ ghc-options: -Wall -funbox-strict-fields -O2 if impl(ghc >= 6.8) ghc-options: -fwarn-tabs + if flag(developer) + ghc-options: -Werror