diff --git a/Text/Trifecta/CharSet.hs b/Text/Trifecta/CharSet.hs deleted file mode 100644 --- a/Text/Trifecta/CharSet.hs +++ /dev/null @@ -1,359 +0,0 @@ -{-# LANGUAGE CPP #-} -{-# OPTIONS_GHC -fspec-constr #-} ------------------------------------------------------------------------------ --- | --- Module : Text.Trifecta.CharSet --- Copyright : (c) Edward Kmett 2010-2011 --- License : BSD3 --- Maintainer : ekmett@gmail.com --- Stability : experimental --- Portability : non-portable (Data, BangPatterns, MagicHash) --- --- Fast set membership tests for 'Char' values --- --- Stored as a (possibly negated) IntMap and a fast set used for the head byte. --- --- The set of valid (possibly negated) head bytes is stored unboxed as a 32-byte --- bytestring-based lookup table. --- --- Designed to be imported qualified: --- --- > import Text.Trifecta.CharSet (CharSet) --- > import qualified Text.Trifecta.CharSet as CharSet --- - ------------------------------------------------------------------------------- --- -------------------------------------------------------------------------------- - -module Text.Trifecta.CharSet - ( - -- * Set type - CharSet(..) - -- * Operators - , (\\) - -- * Query - , null - , size - , member - , notMember - , overlaps, isSubsetOf - , isComplemented - -- * Construction - , build - , empty - , singleton - , full - , insert - , delete - , complement - , range - -- * Combine - , union - , intersection - , difference - -- * Filter - , filter - , partition - -- * Map - , map - -- * Fold - , fold - -- * Conversion - -- ** List - , toList - , fromList - -- ** Ordered list - , toAscList - , fromAscList - , fromDistinctAscList - -- ** IntMaps - , fromCharSet - , toCharSet - -- ** Array - , toArray - ) where - -import Data.Array.Unboxed hiding (range) -import Data.Data -import Data.Function (on) -import Data.IntSet (IntSet) -import Text.Trifecta.Util.ByteSet (ByteSet) -import qualified Text.Trifecta.Util.ByteSet as ByteSet -import Data.Bits hiding (complement) -import Data.Word -import Data.ByteString.Internal (c2w) -import Data.Semigroup -import qualified Data.IntSet as I -import qualified Data.List as L -import Prelude hiding (filter, map, null) -import qualified Prelude as P -import Text.Read - -data CharSet = CharSet !Bool {-# UNPACK #-} !ByteSet !IntSet - -charSet :: Bool -> IntSet -> CharSet -charSet b s = CharSet b (ByteSet.fromList (fmap headByte (I.toAscList s))) s - -headByte :: Int -> Word8 -headByte i - | i <= 0x7f = toEnum i - | i <= 0x7ff = toEnum $ 0x80 + (i `shiftR` 6) - | i <= 0xffff = toEnum $ 0xe0 + (i `shiftR` 12) - | otherwise = toEnum $ 0xf0 + (i `shiftR` 18) - -pos :: IntSet -> CharSet -pos = charSet True - -neg :: IntSet -> CharSet -neg = charSet False - -(\\) :: CharSet -> CharSet -> CharSet -(\\) = difference - -build :: (Char -> Bool) -> CharSet -build p = fromDistinctAscList $ P.filter p [minBound .. maxBound] -{-# INLINE build #-} - -map :: (Char -> Char) -> CharSet -> CharSet -map f (CharSet True _ i) = pos (I.map (fromEnum . f . toEnum) i) -map f (CharSet False _ i) = fromList $ P.map f $ P.filter (\x -> fromEnum x `I.notMember` i) [ul..uh] -{-# INLINE map #-} - -isComplemented :: CharSet -> Bool -isComplemented (CharSet True _ _) = False -isComplemented (CharSet False _ _) = True -{-# INLINE isComplemented #-} - -toList :: CharSet -> String -toList (CharSet True _ i) = P.map toEnum (I.toList i) -toList (CharSet False _ i) = P.filter (\x -> fromEnum x `I.notMember` i) [ul..uh] -{-# INLINE toList #-} - -toAscList :: CharSet -> String -toAscList (CharSet True _ i) = P.map toEnum (I.toAscList i) -toAscList (CharSet False _ i) = P.filter (\x -> fromEnum x `I.notMember` i) [ul..uh] -{-# INLINE toAscList #-} - -empty :: CharSet -empty = pos I.empty - -singleton :: Char -> CharSet -singleton = pos . I.singleton . fromEnum -{-# INLINE singleton #-} - -full :: CharSet -full = neg I.empty - --- | /O(n)/ worst case -null :: CharSet -> Bool -null (CharSet True _ i) = I.null i -null (CharSet False _ i) = I.size i == numChars -{-# INLINE null #-} - --- | /O(n)/ -size :: CharSet -> Int -size (CharSet True _ i) = I.size i -size (CharSet False _ i) = numChars - I.size i -{-# INLINE size #-} - -insert :: Char -> CharSet -> CharSet -insert c (CharSet True _ i) = pos (I.insert (fromEnum c) i) -insert c (CharSet False _ i) = neg (I.delete (fromEnum c) i) -{-# INLINE insert #-} - -range :: Char -> Char -> CharSet -range a b - | a <= b = fromDistinctAscList [a..b] - | otherwise = empty - -delete :: Char -> CharSet -> CharSet -delete c (CharSet True _ i) = pos (I.delete (fromEnum c) i) -delete c (CharSet False _ i) = neg (I.insert (fromEnum c) i) -{-# INLINE delete #-} - -complement :: CharSet -> CharSet -complement (CharSet True s i) = CharSet False s i -complement (CharSet False s i) = CharSet True s i -{-# INLINE complement #-} - -union :: CharSet -> CharSet -> CharSet -union (CharSet True _ i) (CharSet True _ j) = pos (I.union i j) -union (CharSet True _ i) (CharSet False _ j) = neg (I.difference j i) -union (CharSet False _ i) (CharSet True _ j) = neg (I.difference i j) -union (CharSet False _ i) (CharSet False _ j) = neg (I.intersection i j) -{-# INLINE union #-} - -intersection :: CharSet -> CharSet -> CharSet -intersection (CharSet True _ i) (CharSet True _ j) = pos (I.intersection i j) -intersection (CharSet True _ i) (CharSet False _ j) = pos (I.difference i j) -intersection (CharSet False _ i) (CharSet True _ j) = pos (I.difference j i) -intersection (CharSet False _ i) (CharSet False _ j) = neg (I.union i j) -{-# INLINE intersection #-} - -difference :: CharSet -> CharSet -> CharSet -difference (CharSet True _ i) (CharSet True _ j) = pos (I.difference i j) -difference (CharSet True _ i) (CharSet False _ j) = pos (I.intersection i j) -difference (CharSet False _ i) (CharSet True _ j) = neg (I.union i j) -difference (CharSet False _ i) (CharSet False _ j) = pos (I.difference j i) -{-# INLINE difference #-} - -member :: Char -> CharSet -> Bool -member c (CharSet True b i) - | c <= toEnum 0x7f = ByteSet.member (c2w c) b - | otherwise = I.member (fromEnum c) i -member c (CharSet False b i) - | c <= toEnum 0x7f = not (ByteSet.member (c2w c) b) - | otherwise = I.notMember (fromEnum c) i -{-# INLINE member #-} - -notMember :: Char -> CharSet -> Bool -notMember c s = not (member c s) -{-# INLINE notMember #-} - -fold :: (Char -> b -> b) -> b -> CharSet -> b -fold f z (CharSet True _ i) = I.fold (f . toEnum) z i -fold f z (CharSet False _ i) = foldr f z $ P.filter (\x -> fromEnum x `I.notMember` i) [ul..uh] -{-# INLINE fold #-} - -filter :: (Char -> Bool) -> CharSet -> CharSet -filter p (CharSet True _ i) = pos (I.filter (p . toEnum) i) -filter p (CharSet False _ i) = neg $ foldr (I.insert) i $ P.filter (\x -> (x `I.notMember` i) && not (p (toEnum x))) [ol..oh] -{-# INLINE filter #-} - -partition :: (Char -> Bool) -> CharSet -> (CharSet, CharSet) -partition p (CharSet True _ i) = (pos l, pos r) - where (l,r) = I.partition (p . toEnum) i -partition p (CharSet False _ i) = (neg (foldr I.insert i l), neg (foldr I.insert i r)) - where (l,r) = L.partition (p . toEnum) $ P.filter (\x -> x `I.notMember` i) [ol..oh] -{-# INLINE partition #-} - -overlaps :: CharSet -> CharSet -> Bool -overlaps (CharSet True _ i) (CharSet True _ j) = not (I.null (I.intersection i j)) -overlaps (CharSet True _ i) (CharSet False _ j) = not (I.isSubsetOf j i) -overlaps (CharSet False _ i) (CharSet True _ j) = not (I.isSubsetOf i j) -overlaps (CharSet False _ i) (CharSet False _ j) = any (\x -> I.notMember x i && I.notMember x j) [ol..oh] -- not likely -{-# INLINE overlaps #-} - -isSubsetOf :: CharSet -> CharSet -> Bool -isSubsetOf (CharSet True _ i) (CharSet True _ j) = I.isSubsetOf i j -isSubsetOf (CharSet True _ i) (CharSet False _ j) = I.null (I.intersection i j) -isSubsetOf (CharSet False _ i) (CharSet True _ j) = all (\x -> I.member x i && I.member x j) [ol..oh] -- not bloody likely -isSubsetOf (CharSet False _ i) (CharSet False _ j) = I.isSubsetOf j i -{-# INLINE isSubsetOf #-} - -fromList :: String -> CharSet -fromList = pos . I.fromList . P.map fromEnum -{-# INLINE fromList #-} - -fromAscList :: String -> CharSet -fromAscList = pos . I.fromAscList . P.map fromEnum -{-# INLINE fromAscList #-} - -fromDistinctAscList :: String -> CharSet -fromDistinctAscList = pos . I.fromDistinctAscList . P.map fromEnum -{-# INLINE fromDistinctAscList #-} - --- isProperSubsetOf :: CharSet -> CharSet -> Bool --- isProperSubsetOf (P i) (P j) = I.isProperSubsetOf i j --- isProperSubsetOf (P i) (N j) = null (I.intersection i j) && ... --- isProperSubsetOf (N i) (N j) = I.isProperSubsetOf j i - -ul, uh :: Char -ul = minBound -uh = maxBound -{-# INLINE ul #-} -{-# INLINE uh #-} - -ol, oh :: Int -ol = fromEnum ul -oh = fromEnum uh -{-# INLINE ol #-} -{-# INLINE oh #-} - -numChars :: Int -numChars = oh - ol + 1 -{-# INLINE numChars #-} - -instance Typeable CharSet where - typeOf _ = mkTyConApp charSetTyCon [] - -charSetTyCon :: TyCon -#if __GLASGOW_HASKELL__ < 704 -charSetTyCon = mkTyCon "Text.Trifecta.CharSet.CharSet" -#else -charSetTyCon = mkTyCon3 "trifecta" "Text.Trifecta.CharSet" "CharSet" -#endif -{-# NOINLINE charSetTyCon #-} - -instance Data CharSet where - gfoldl k z set - | isComplemented set = z complement `k` complement set - | otherwise = z fromList `k` toList set - - toConstr set - | isComplemented set = complementConstr - | otherwise = fromListConstr - - dataTypeOf _ = charSetDataType - - gunfold k z c = case constrIndex c of - 1 -> k (z fromList) - 2 -> k (z complement) - _ -> error "gunfold" - -fromListConstr :: Constr -fromListConstr = mkConstr charSetDataType "fromList" [] Prefix -{-# NOINLINE fromListConstr #-} - -complementConstr :: Constr -complementConstr = mkConstr charSetDataType "complement" [] Prefix -{-# NOINLINE complementConstr #-} - -charSetDataType :: DataType -charSetDataType = mkDataType "Text.Trifecta.CharSet.CharSet" [fromListConstr, complementConstr] -{-# NOINLINE charSetDataType #-} - --- returns an intset and if the charSet is positive -fromCharSet :: CharSet -> (Bool, IntSet) -fromCharSet (CharSet b _ i) = (b, i) -{-# INLINE fromCharSet #-} - -toCharSet :: IntSet -> CharSet -toCharSet = pos -{-# INLINE toCharSet #-} - -instance Eq CharSet where - (==) = (==) `on` toAscList - -instance Ord CharSet where - compare = compare `on` toAscList - -instance Bounded CharSet where - minBound = empty - maxBound = full - --- TODO return a tighter bounded array perhaps starting from the least element present to the last element present? -toArray :: CharSet -> UArray Char Bool -toArray set = array (minBound, maxBound) $ fmap (\x -> (x, x `member` set)) [minBound .. maxBound] - -instance Show CharSet where - showsPrec d i - | isComplemented i = showParen (d > 10) $ showString "complement " . showsPrec 11 (complement i) - | otherwise = showParen (d > 10) $ showString "fromDistinctAscList " . showsPrec 11 (toAscList i) - -instance Read CharSet where - readPrec = parens $ complemented +++ normal - where - complemented = prec 10 $ do - Ident "complement" <- lexP - complement `fmap` step readPrec - normal = prec 10 $ do - Ident "fromDistinctAscList" <- lexP - fromDistinctAscList `fmap` step readPrec - -instance Semigroup CharSet where - (<>) = union - -instance Monoid CharSet where - mempty = empty - mappend = union diff --git a/Text/Trifecta/CharSet/Common.hs b/Text/Trifecta/CharSet/Common.hs deleted file mode 100644 --- a/Text/Trifecta/CharSet/Common.hs +++ /dev/null @@ -1,65 +0,0 @@ ------------------------------------------------------------------------------ --- | --- Module : Text.Trifecta.CharSet.Common --- Copyright : (c) Edward Kmett 2010-2011 --- License : BSD3 --- Maintainer : ekmett@gmail.com --- Stability : experimental --- Portability : portable --- --- The various character classifications from "Data.Char" as 'CharSet's -------------------------------------------------------------------------------- - -module Text.Trifecta.CharSet.Common - ( - -- ** Data.Char classes - control - , space - , lower - , upper - , alpha - , alphaNum - , print - , digit - , octDigit - , letter - , mark - , number - , punctuation - , symbol - , separator - , ascii - , latin1 - , asciiUpper - , asciiLower - ) where - -import Prelude () -import Data.Char -import Text.Trifecta.CharSet - --- Haskell character classes from Data.Char -control, space, lower, upper, alpha, alphaNum, - print, digit, octDigit, letter, mark, number, - punctuation, symbol, separator, ascii, latin1 - , asciiUpper, asciiLower :: CharSet - -control = build isControl -space = build isSpace -lower = build isLower -upper = build isUpper -alpha = build isAlpha -alphaNum = build isAlphaNum -print = build isPrint -digit = build isDigit -octDigit = build isOctDigit -letter = build isLetter -mark = build isMark -number = build isNumber -punctuation = build isPunctuation -symbol = build isSymbol -separator = build isSeparator -ascii = build isAscii -latin1 = build isLatin1 -asciiUpper = build isAsciiUpper -asciiLower = build isAsciiLower diff --git a/Text/Trifecta/CharSet/Posix.hs b/Text/Trifecta/CharSet/Posix.hs deleted file mode 100644 --- a/Text/Trifecta/CharSet/Posix.hs +++ /dev/null @@ -1,21 +0,0 @@ ------------------------------------------------------------------------------ --- | --- Module : Text.Trifecta.CharSet.Posix --- Copyright : (c) Edward Kmett 2011 --- License : BSD3 --- Maintainer : ekmett@gmail.com --- Stability : experimental --- Portability : portable --- -------------------------------------------------------------------------------- - -module Text.Trifecta.CharSet.Posix - ( posixAscii - , lookupPosixAsciiCharSet - , posixUnicode - , lookupPosixUnicodeCharSet - ) where - -import Text.Trifecta.CharSet.Posix.Ascii -import Text.Trifecta.CharSet.Posix.Unicode -import Prelude () diff --git a/Text/Trifecta/CharSet/Posix/Ascii.hs b/Text/Trifecta/CharSet/Posix/Ascii.hs deleted file mode 100644 --- a/Text/Trifecta/CharSet/Posix/Ascii.hs +++ /dev/null @@ -1,61 +0,0 @@ ------------------------------------------------------------------------------ --- | --- Module : Text.Trifecta.CharSet.Posix.Ascii --- Copyright : (c) Edward Kmett 2010 --- License : BSD3 --- Maintainer : ekmett@gmail.com --- Stability : experimental --- Portability : portable --- -------------------------------------------------------------------------------- - -module Text.Trifecta.CharSet.Posix.Ascii - ( posixAscii - , lookupPosixAsciiCharSet - -- * Traditional POSIX ASCII \"classes\" - , alnum, alpha, ascii, blank, cntrl, digit, graph, print, word, punct, space, upper, lower, xdigit - ) where - -import Prelude hiding (print) -import Data.Char -import Text.Trifecta.CharSet -import Data.HashMap.Lazy (HashMap) -import qualified Data.HashMap.Lazy as HashMap - -alnum, alpha, ascii, blank, cntrl, digit, graph, print, word, punct, space, upper, lower, xdigit :: CharSet -alnum = alpha `union` digit -alpha = lower `union` upper -ascii = range '\x00' '\x7f' -blank = fromList " \t" -cntrl = insert '\x7f' $ range '\x00' '\x1f' -digit = range '0' '9' -lower = range 'a' 'z' -upper = range 'A' 'Z' -graph = range '\x21' '\x7e' -print = insert '\x20' graph -word = insert '_' alnum -punct = fromList "-!\"#$%&'()*+,./:;<=>?@[\\]^_`{|}~" -space = fromList " \t\r\n\v\f" -xdigit = digit `union` range 'a' 'f' `union` range 'A' 'F' - --- :digit:, etc. -posixAscii :: HashMap String CharSet -posixAscii = HashMap.fromList - [ ("alnum", alnum) - , ("alpha", alpha) - , ("ascii", ascii) - , ("blank", blank) - , ("cntrl", cntrl) - , ("digit", digit) - , ("graph", graph) - , ("print", print) - , ("word", word) - , ("punct", punct) - , ("space", space) - , ("upper", upper) - , ("lower", lower) - , ("xdigit", xdigit) - ] - -lookupPosixAsciiCharSet :: String -> Maybe CharSet -lookupPosixAsciiCharSet s = HashMap.lookup (Prelude.map toLower s) posixAscii diff --git a/Text/Trifecta/CharSet/Posix/Unicode.hs b/Text/Trifecta/CharSet/Posix/Unicode.hs deleted file mode 100644 --- a/Text/Trifecta/CharSet/Posix/Unicode.hs +++ /dev/null @@ -1,64 +0,0 @@ ------------------------------------------------------------------------------ --- | --- Module : Text.Trifecta.CharSet.Posix.Unicode --- Copyright : (c) Edward Kmett 2010 --- License : BSD3 --- Maintainer : ekmett@gmail.com --- Stability : experimental --- Portability : portable --- -------------------------------------------------------------------------------- - -module Text.Trifecta.CharSet.Posix.Unicode - ( posixUnicode - , lookupPosixUnicodeCharSet - -- * POSIX ASCII \"classes\" - , alnum, alpha, ascii, blank, cntrl, digit, graph, print, word, punct, space, upper, lower, xdigit - ) where - -import Prelude hiding (print) -import Data.Char -import Text.Trifecta.CharSet -import qualified Text.Trifecta.CharSet.Unicode.Category as Category -import qualified Text.Trifecta.CharSet.Unicode.Block as Block -import Data.HashMap.Lazy (HashMap) -import qualified Data.HashMap.Lazy as HashMap - -alnum, alpha, ascii, blank, cntrl, digit, graph, print, word, punct, space, upper, lower, xdigit :: CharSet -alnum = alpha `union` digit -ascii = Block.basicLatin -alpha = Category.letterAnd -blank = insert '\t' Category.space -cntrl = Category.control -digit = Category.decimalNumber -lower = Category.lowercaseLetter -upper = Category.uppercaseLetter -graph = complement (Category.separator `union` Category.other) -print = complement (Category.other) -word = Category.letter `union` Category.number `union` Category.connectorPunctuation -punct = Category.punctuation `union` Category.symbol -space = fromList " \t\r\n\v\f" `union` Category.separator -xdigit = digit `union` range 'a' 'f' `union` range 'A' 'F' - --- :digit:, etc. -posixUnicode :: HashMap String CharSet -posixUnicode = HashMap.fromList - [ ("alnum", alnum) - , ("alpha", alpha) - , ("ascii", ascii) - , ("blank", blank) - , ("cntrl", cntrl) - , ("digit", digit) - , ("graph", graph) - , ("print", print) - , ("word", word) - , ("punct", punct) - , ("space", space) - , ("upper", upper) - , ("lower", lower) - , ("xdigit", xdigit) - ] - -lookupPosixUnicodeCharSet :: String -> Maybe CharSet -lookupPosixUnicodeCharSet s = HashMap.lookup (Prelude.map toLower s) posixUnicode - diff --git a/Text/Trifecta/CharSet/Unicode.hs b/Text/Trifecta/CharSet/Unicode.hs deleted file mode 100644 --- a/Text/Trifecta/CharSet/Unicode.hs +++ /dev/null @@ -1,177 +0,0 @@ -{-# LANGUAGE DeriveDataTypeable #-} ------------------------------------------------------------------------------ --- | --- Module : Text.Trifecta.CharSet.Unicode --- Copyright : (c) Edward Kmett 2010 --- License : BSD3 --- Maintainer : ekmett@gmail.com --- Stability : experimental --- Portability : portable --- --- Provides unicode general categories, which are typically connoted by --- @\p{Ll}@ or @\p{Modifier_Letter}@. Lookups can be constructed using 'categories' --- or individual character sets can be used directly. -------------------------------------------------------------------------------- - -module Text.Trifecta.CharSet.Unicode - ( - -- * Unicode General Category - UnicodeCategory(..) - -- * Lookup - , unicodeCategories - -- * CharSets by UnicodeCategory - -- ** Letter - , modifierLetter, otherLetter, letter - -- *** Letter\& - , lowercaseLetter, uppercaseLetter, titlecaseLetter, letterAnd - -- ** Mark - , nonSpacingMark, spacingCombiningMark, enclosingMark, mark - -- ** Separator - , space, lineSeparator, paragraphSeparator, separator - -- ** Symbol - , mathSymbol, currencySymbol, modifierSymbol, otherSymbol, symbol - -- ** Number - , decimalNumber, letterNumber, otherNumber, number - -- ** Punctuation - , dashPunctuation, openPunctuation, closePunctuation, initialQuote - , finalQuote, connectorPunctuation, otherPunctuation, punctuation - -- ** Other - , control, format, privateUse, surrogate, notAssigned, other - ) where - -import Data.Char -import Data.Data -import Text.Trifecta.CharSet - -data UnicodeCategory = UnicodeCategory String String CharSet String - deriving (Show, Data, Typeable) - --- \p{Letter} or \p{Mc} -unicodeCategories :: [UnicodeCategory] -unicodeCategories = - [ UnicodeCategory "Letter" "L" letter "any kind of letter from any language." - , UnicodeCategory "Lowercase_Letter" "Ll" lowercaseLetter "a lowercase letter that has an uppercase variant" - , UnicodeCategory "Uppercase_Letter" "Lu" uppercaseLetter "an uppercase letter that has a lowercase variant" - , UnicodeCategory "Titlecase_Letter" "Lt" titlecaseLetter "a letter that appears at the start of a word when only the first letter of the word is capitalized" - , UnicodeCategory "Letter&" "L&" letterAnd "a letter that exists in lowercase and uppercase variants (combination of Ll, Lu and Lt)" - , UnicodeCategory "Modifier_Letter" "Lm" modifierLetter "a special character that is used like a letter" - , UnicodeCategory "Other_Letter" "Lo" otherLetter "a letter or ideograph that does not have lowercase and uppercase variants" - , UnicodeCategory "Mark" "M" mark "a character intended to be combined with another character (e.g. accents, umlauts, enclosing boxes, etc.)" - , UnicodeCategory "Non_Spacing_Mark" "Mn" nonSpacingMark "a character intended to be combined with another character without taking up extra space (e.g. accents, umlauts, etc.)" - , UnicodeCategory "Spacing_Combining_Mark" "Mc" spacingCombiningMark "a character intended to be combined with another character that takes up extra space (vowel signs in many Eastern languages)" - , UnicodeCategory "Enclosing_Mark" "Me" enclosingMark "a character that encloses the character is is combined with (circle, square, keycap, etc.)" - , UnicodeCategory "Separator" "Z" separator "any kind of whitespace or invisible separator" - , UnicodeCategory "Space_Separator" "Zs" space "a whitespace character that is invisible, but does take up space" - , UnicodeCategory "Line_Separator" "Zl" lineSeparator "line separator character U+2028" - , UnicodeCategory "Paragraph_Separator" "Zp" paragraphSeparator "paragraph separator character U+2029" - , UnicodeCategory "Symbol" "S" symbol "math symbols, currency signs, dingbats, box-drawing characters, etc." - , UnicodeCategory "Math_Symbol" "Sm" mathSymbol "any mathematical symbol" - , UnicodeCategory "Currency_Symbol" "Sc" currencySymbol "any currency sign" - , UnicodeCategory "Modifier_Symbol" "Sk" modifierSymbol "a combining character (mark) as a full character on its own" - , UnicodeCategory "Other_Symbol" "So" otherSymbol "various symbols that are not math symbols, currency signs, or combining characters" - , UnicodeCategory "Number" "N" number "any kind of numeric character in any script" - , UnicodeCategory "Decimal_Digit_Number" "Nd" decimalNumber "a digit zero through nine in any script except ideographic scripts" - , UnicodeCategory "Letter_Number" "Nl" letterNumber "a number that looks like a letter, such as a Roman numeral" - , UnicodeCategory "Other_Number" "No" otherNumber "a superscript or subscript digit, or a number that is not a digit 0..9 (excluding numbers from ideographic scripts)" - , UnicodeCategory "Punctuation" "P" punctuation "any kind of punctuation character" - , UnicodeCategory "Dash_Punctuation" "Pd" dashPunctuation "any kind of hyphen or dash" - , UnicodeCategory "Open_Punctuation" "Ps" openPunctuation "any kind of opening bracket" - , UnicodeCategory "Close_Punctuation" "Pe" closePunctuation "any kind of closing bracket" - , UnicodeCategory "Initial_Punctuation" "Pi" initialQuote "any kind of opening quote" - , UnicodeCategory "Final_Punctuation" "Pf" finalQuote "any kind of closing quote" - , UnicodeCategory "Connector_Punctuation" "Pc" connectorPunctuation "a punctuation character such as an underscore that connects words" - , UnicodeCategory "Other_Punctuation" "Po" otherPunctuation "any kind of punctuation character that is not a dash, bracket, quote or connector" - , UnicodeCategory "Other" "C" other "invisible control characters and unused code points" - , UnicodeCategory "Control" "Cc" control "an ASCII 0x00..0x1F or Latin-1 0x80..0x9F control character" - , UnicodeCategory "Format" "Cf" format "invisible formatting indicator" - , UnicodeCategory "Private_Use" "Co" privateUse "any code point reserved for private use" - , UnicodeCategory "Surrogate" "Cs" surrogate "one half of a surrogate pair in UTF-16 encoding" - , UnicodeCategory "Unassigned" "Cn" notAssigned "any code point to which no character has been assigned.properties" ] - -cat :: GeneralCategory -> CharSet -cat category = build ((category ==) . generalCategory) - --- Letter -lowercaseLetter, uppercaseLetter, titlecaseLetter, letterAnd, modifierLetter, otherLetter, letter :: CharSet -lowercaseLetter = cat LowercaseLetter -uppercaseLetter = cat UppercaseLetter -titlecaseLetter = cat TitlecaseLetter -letterAnd = lowercaseLetter - `union` uppercaseLetter - `union` titlecaseLetter -modifierLetter = cat ModifierLetter -otherLetter = cat OtherLetter -letter - = letterAnd - `union` modifierLetter - `union` otherLetter - --- Marks -nonSpacingMark, spacingCombiningMark, enclosingMark, mark :: CharSet -nonSpacingMark = cat NonSpacingMark -spacingCombiningMark = cat SpacingCombiningMark -enclosingMark = cat EnclosingMark -mark - = nonSpacingMark - `union` spacingCombiningMark - `union` enclosingMark - -space, lineSeparator, paragraphSeparator, separator :: CharSet -space = cat Space -lineSeparator = cat LineSeparator -paragraphSeparator = cat ParagraphSeparator -separator - = space - `union` lineSeparator - `union` paragraphSeparator - -mathSymbol, currencySymbol, modifierSymbol, otherSymbol, symbol :: CharSet -mathSymbol = cat MathSymbol -currencySymbol = cat CurrencySymbol -modifierSymbol = cat ModifierSymbol -otherSymbol = cat OtherSymbol -symbol - = mathSymbol - `union` currencySymbol - `union` modifierSymbol - `union` otherSymbol - -decimalNumber, letterNumber, otherNumber, number :: CharSet -decimalNumber = cat DecimalNumber -letterNumber = cat LetterNumber -otherNumber = cat OtherNumber -number - = decimalNumber - `union` letterNumber - `union` otherNumber - -dashPunctuation, openPunctuation, closePunctuation, initialQuote, - finalQuote, connectorPunctuation, otherPunctuation, punctuation :: CharSet - -dashPunctuation = cat DashPunctuation -openPunctuation = cat OpenPunctuation -closePunctuation = cat ClosePunctuation -initialQuote = cat InitialQuote -finalQuote = cat FinalQuote -connectorPunctuation = cat ConnectorPunctuation -otherPunctuation = cat OtherPunctuation -punctuation - = dashPunctuation - `union` openPunctuation - `union` closePunctuation - `union` initialQuote - `union` finalQuote - `union` connectorPunctuation - `union` otherPunctuation - -control, format, privateUse, surrogate, notAssigned, other :: CharSet -control = cat Control -format = cat Format -privateUse = cat PrivateUse -surrogate = cat Surrogate -notAssigned = cat NotAssigned -other = control - `union` format - `union` privateUse - `union` surrogate - `union` notAssigned diff --git a/Text/Trifecta/CharSet/Unicode/Block.hs b/Text/Trifecta/CharSet/Unicode/Block.hs deleted file mode 100644 --- a/Text/Trifecta/CharSet/Unicode/Block.hs +++ /dev/null @@ -1,382 +0,0 @@ -{-# LANGUAGE DeriveDataTypeable #-} -{-# OPTIONS_GHC -fno-warn-missing-signatures #-} ------------------------------------------------------------------------------ --- | --- Module : Text.Trifecta.CharSet.Unicode.Block --- Copyright : (c) Edward Kmett 2010-2011 --- License : BSD3 --- Maintainer : ekmett@gmail.com --- Stability : experimental --- Portability : portable --- --- Provides unicode general categories, which are typically connoted by --- @\p{InBasicLatin}@ or @\p{InIPA_Extensions}@. Lookups can be constructed using 'categories' --- or individual character sets can be used directly. -------------------------------------------------------------------------------- - -module Text.Trifecta.CharSet.Unicode.Block - ( - -- * Unicode General Category - Block(..) - -- * Lookup - , blocks - , lookupBlock - , lookupBlockCharSet - -- * CharSets by Block - , basicLatin - , latin1Supplement - , latinExtendedA - , latinExtendedB - , ipaExtensions - , spacingModifierLetters - , combiningDiacriticalMarks - , greekAndCoptic - , cyrillic - , cyrillicSupplementary - , armenian - , hebrew - , arabic - , syriac - , thaana - , devanagari - , bengali - , gurmukhi - , gujarati - , oriya - , tamil - , telugu - , kannada - , malayalam - , sinhala - , thai - , lao - , tibetan - , myanmar - , georgian - , hangulJamo - , ethiopic - , cherokee - , unifiedCanadianAboriginalSyllabics - , ogham - , runic - , tagalog - , hanunoo - , buhid - , tagbanwa - , khmer - , mongolian - , limbu - , taiLe - , khmerSymbols - , phoneticExtensions - , latinExtendedAdditional - , greekExtended - , generalPunctuation - , superscriptsAndSubscripts - , currencySymbols - , combiningDiacriticalMarksForSymbols - , letterlikeSymbols - , numberForms - , arrows - , mathematicalOperators - , miscellaneousTechnical - , controlPictures - , opticalCharacterRecognition - , enclosedAlphanumerics - , boxDrawing - , blockElements - , geometricShapes - , miscellaneousSymbols - , dingbats - , miscellaneousMathematicalSymbolsA - , supplementalArrowsA - , braillePatterns - , supplementalArrowsB - , miscellaneousMathematicalSymbolsB - , supplementalMathematicalOperators - , miscellaneousSymbolsAndArrows - , cjkRadicalsSupplement - , kangxiRadicals - , ideographicDescriptionCharacters - , cjkSymbolsAndPunctuation - , hiragana - , katakana - , bopomofo - , hangulCompatibilityJamo - , kanbun - , bopomofoExtended - , katakanaPhoneticExtensions - , enclosedCjkLettersAndMonths - , cjkCompatibility - , cjkUnifiedIdeographsExtensionA - , yijingHexagramSymbols - , cjkUnifiedIdeographs - , yiSyllables - , yiRadicals - , hangulSyllables - , highSurrogates - , highPrivateUseSurrogates - , lowSurrogates - , privateUseArea - , cjkCompatibilityIdeographs - , alphabeticPresentationForms - , arabicPresentationFormsA - , variationSelectors - , combiningHalfMarks - , cjkCompatibilityForms - , smallFormVariants - , arabicPresentationFormsB - , halfwidthAndFullwidthForms - , specials - ) where - -import Data.Char -import Text.Trifecta.CharSet -import Data.Data -import Data.HashMap.Lazy (HashMap) -import qualified Data.HashMap.Lazy as HashMap - -data Block = Block - { blockName :: String - , blockCharSet :: CharSet - } deriving (Show, Data, Typeable) - -blocks :: [Block] -blocks = - [ Block "Basic_Latin" basicLatin - , Block "Latin-1_Supplement" latin1Supplement - , Block "Latin_Extended-A" latinExtendedA - , Block "IPA_Extensions" ipaExtensions - , Block "Spacing_Modifier_Letters" spacingModifierLetters - - , Block "Latin_Extended-A" latinExtendedA - , Block "Latin_Extended-B" latinExtendedB - , Block "IPA_Extensions" ipaExtensions - , Block "Spacing_Modifier_Letters" spacingModifierLetters - , Block "Combining_Diacritical_Marks" combiningDiacriticalMarks - , Block "Greek_and_Coptic" greekAndCoptic - , Block "Cyrillic" cyrillic - , Block "Cyrillic_Supplementary" cyrillicSupplementary - , Block "Armenian" armenian - , Block "Hebrew" hebrew - , Block "Arabic" arabic - , Block "Syriac" syriac - , Block "Thaana" thaana - , Block "Devanagari" devanagari - , Block "Bengali" bengali - , Block "Gurmukhi" gurmukhi - , Block "Gujarati" gujarati - , Block "Oriya" oriya - , Block "Tamil" tamil - , Block "Telugu" telugu - , Block "Kannada" kannada - , Block "Malayalam" malayalam - , Block "Sinhala" sinhala - , Block "Thai" thai - , Block "Lao" lao - , Block "Tibetan" tibetan - , Block "Myanmar" myanmar - , Block "Georgian" georgian - , Block "Hangul_Jamo" hangulJamo - , Block "Ethiopic" ethiopic - , Block "Cherokee" cherokee - , Block "Unified_Canadian_Aboriginal_Syllabics" unifiedCanadianAboriginalSyllabics - , Block "Ogham" ogham - , Block "Runic" runic - , Block "Tagalog" tagalog - , Block "Hanunoo" hanunoo - , Block "Buhid" buhid - , Block "Tagbanwa" tagbanwa - , Block "Khmer" khmer - , Block "Mongolian" mongolian - , Block "Limbu" limbu - , Block "Tai_Le" taiLe - , Block "Khmer_Symbols" khmerSymbols - , Block "Phonetic_Extensions" phoneticExtensions - , Block "Latin_Extended_Additional" latinExtendedAdditional - , Block "Greek_Extended" greekExtended - , Block "General_Punctuation" generalPunctuation - , Block "Superscripts_and_Subscripts" superscriptsAndSubscripts - , Block "Currency_Symbols" currencySymbols - , Block "Combining_Diacritical_Marks_for_Symbols" combiningDiacriticalMarksForSymbols - , Block "Letterlike_Symbols" letterlikeSymbols - , Block "Number_Forms" numberForms - , Block "Arrows" arrows - , Block "Mathematical_Operators" mathematicalOperators - , Block "Miscellaneous_Technical" miscellaneousTechnical - , Block "Control_Pictures" controlPictures - , Block "Optical_Character_Recognition" opticalCharacterRecognition - , Block "Enclosed_Alphanumerics" enclosedAlphanumerics - , Block "Box_Drawing" boxDrawing - , Block "Block_Elements" blockElements - , Block "Geometric_Shapes" geometricShapes - , Block "Miscellaneous_Symbols" miscellaneousSymbols - , Block "Dingbats" dingbats - , Block "Miscellaneous_Mathematical_Symbols-A" miscellaneousMathematicalSymbolsA - , Block "Supplemental_Arrows-A" supplementalArrowsA - , Block "Braille_Patterns" braillePatterns - , Block "Supplemental_Arrows-B" supplementalArrowsB - , Block "Miscellaneous_Mathematical_Symbols-B" miscellaneousMathematicalSymbolsB - , Block "Supplemental_Mathematical_Operators" supplementalMathematicalOperators - , Block "Miscellaneous_Symbols_and_Arrows" miscellaneousSymbolsAndArrows - , Block "CJK_Radicals_Supplement" cjkRadicalsSupplement - , Block "Kangxi_Radicals" kangxiRadicals - , Block "Ideographic_Description_Characters" ideographicDescriptionCharacters - , Block "CJK_Symbols_and_Punctuation" cjkSymbolsAndPunctuation - , Block "Hiragana" hiragana - , Block "Katakana" katakana - , Block "Bopomofo" bopomofo - , Block "Hangul_Compatibility_Jamo" hangulCompatibilityJamo - , Block "Kanbun" kanbun - , Block "Bopomofo_Extended" bopomofoExtended - , Block "Katakana_Phonetic_Extensions" katakanaPhoneticExtensions - , Block "Enclosed_CJK_Letters_and_Months" enclosedCjkLettersAndMonths - , Block "CJK_Compatibility" cjkCompatibility - , Block "CJK_Unified_Ideographs_Extension_A" cjkUnifiedIdeographsExtensionA - , Block "Yijing_Hexagram_Symbols" yijingHexagramSymbols - , Block "CJK_Unified_Ideographs" cjkUnifiedIdeographs - , Block "Yi_Syllables" yiSyllables - , Block "Yi_Radicals" yiRadicals - , Block "Hangul_Syllables" hangulSyllables - , Block "High_Surrogates" highSurrogates - , Block "High_Private_Use_Surrogates" highPrivateUseSurrogates - , Block "Low_Surrogates" lowSurrogates - , Block "Private_Use_Area" privateUseArea - , Block "CJK_Compatibility_Ideographs" cjkCompatibilityIdeographs - , Block "Alphabetic_Presentation_Forms" alphabeticPresentationForms - , Block "Arabic_Presentation_Forms-A" arabicPresentationFormsA - , Block "Variation_Selectors" variationSelectors - , Block "Combining_Half_Marks" combiningHalfMarks - , Block "CJK_Compatibility_Forms" cjkCompatibilityForms - , Block "Small_Form_Variants" smallFormVariants - , Block "Arabic_Presentation_Forms-B" arabicPresentationFormsB - , Block "Halfwidth_and_Fullwidth_Forms" halfwidthAndFullwidthForms - , Block "Specials" specials ] - -lookupTable :: HashMap String Block -lookupTable = HashMap.fromList $ - Prelude.map (\y@(Block x _) -> (canonicalize x, y)) - blocks - -canonicalize :: String -> String -canonicalize s = case Prelude.map toLower s of - 'i': 'n' : xs -> go xs - xs -> go xs - where - go ('-':xs) = go xs - go ('_':xs) = go xs - go (' ':xs) = go xs - go (x:xs) = x : go xs - go [] = [] - -lookupBlock :: String -> Maybe Block -lookupBlock s = HashMap.lookup (canonicalize s) lookupTable - -lookupBlockCharSet :: String -> Maybe CharSet -lookupBlockCharSet = fmap blockCharSet . lookupBlock - -basicLatin = range '\x0000' '\x007f' -latin1Supplement = range '\x0080' '\x00ff' -latinExtendedA = range '\x0100' '\x017F' -latinExtendedB = range '\x0180' '\x024F' -ipaExtensions = range '\x0250' '\x02AF' -spacingModifierLetters = range '\x02B0' '\x02FF' -combiningDiacriticalMarks = range '\x0300' '\x036F' -greekAndCoptic = range '\x0370' '\x03FF' -cyrillic = range '\x0400' '\x04FF' -cyrillicSupplementary = range '\x0500' '\x052F' -armenian = range '\x0530' '\x058F' -hebrew = range '\x0590' '\x05FF' -arabic = range '\x0600' '\x06FF' -syriac = range '\x0700' '\x074F' -thaana = range '\x0780' '\x07BF' -devanagari = range '\x0900' '\x097F' -bengali = range '\x0980' '\x09FF' -gurmukhi = range '\x0A00' '\x0A7F' -gujarati = range '\x0A80' '\x0AFF' -oriya = range '\x0B00' '\x0B7F' -tamil = range '\x0B80' '\x0BFF' -telugu = range '\x0C00' '\x0C7F' -kannada = range '\x0C80' '\x0CFF' -malayalam = range '\x0D00' '\x0D7F' -sinhala = range '\x0D80' '\x0DFF' -thai = range '\x0E00' '\x0E7F' -lao = range '\x0E80' '\x0EFF' -tibetan = range '\x0F00' '\x0FFF' -myanmar = range '\x1000' '\x109F' -georgian = range '\x10A0' '\x10FF' -hangulJamo = range '\x1100' '\x11FF' -ethiopic = range '\x1200' '\x137F' -cherokee = range '\x13A0' '\x13FF' -unifiedCanadianAboriginalSyllabics = range '\x1400' '\x167F' -ogham = range '\x1680' '\x169F' -runic = range '\x16A0' '\x16FF' -tagalog = range '\x1700' '\x171F' -hanunoo = range '\x1720' '\x173F' -buhid = range '\x1740' '\x175F' -tagbanwa = range '\x1760' '\x177F' -khmer = range '\x1780' '\x17FF' -mongolian = range '\x1800' '\x18AF' -limbu = range '\x1900' '\x194F' -taiLe = range '\x1950' '\x197F' -khmerSymbols = range '\x19E0' '\x19FF' -phoneticExtensions = range '\x1D00' '\x1D7F' -latinExtendedAdditional = range '\x1E00' '\x1EFF' -greekExtended = range '\x1F00' '\x1FFF' -generalPunctuation = range '\x2000' '\x206F' -superscriptsAndSubscripts = range '\x2070' '\x209F' -currencySymbols = range '\x20A0' '\x20CF' -combiningDiacriticalMarksForSymbols = range '\x20D0' '\x20FF' -letterlikeSymbols = range '\x2100' '\x214F' -numberForms = range '\x2150' '\x218F' -arrows = range '\x2190' '\x21FF' -mathematicalOperators = range '\x2200' '\x22FF' -miscellaneousTechnical = range '\x2300' '\x23FF' -controlPictures = range '\x2400' '\x243F' -opticalCharacterRecognition = range '\x2440' '\x245F' -enclosedAlphanumerics = range '\x2460' '\x24FF' -boxDrawing = range '\x2500' '\x257F' -blockElements = range '\x2580' '\x259F' -geometricShapes = range '\x25A0' '\x25FF' -miscellaneousSymbols = range '\x2600' '\x26FF' -dingbats = range '\x2700' '\x27BF' -miscellaneousMathematicalSymbolsA = range '\x27C0' '\x27EF' -supplementalArrowsA = range '\x27F0' '\x27FF' -braillePatterns = range '\x2800' '\x28FF' -supplementalArrowsB = range '\x2900' '\x297F' -miscellaneousMathematicalSymbolsB = range '\x2980' '\x29FF' -supplementalMathematicalOperators = range '\x2A00' '\x2AFF' -miscellaneousSymbolsAndArrows = range '\x2B00' '\x2BFF' -cjkRadicalsSupplement = range '\x2E80' '\x2EFF' -kangxiRadicals = range '\x2F00' '\x2FDF' -ideographicDescriptionCharacters = range '\x2FF0' '\x2FFF' -cjkSymbolsAndPunctuation = range '\x3000' '\x303F' -hiragana = range '\x3040' '\x309F' -katakana = range '\x30A0' '\x30FF' -bopomofo = range '\x3100' '\x312F' -hangulCompatibilityJamo = range '\x3130' '\x318F' -kanbun = range '\x3190' '\x319F' -bopomofoExtended = range '\x31A0' '\x31BF' -katakanaPhoneticExtensions = range '\x31F0' '\x31FF' -enclosedCjkLettersAndMonths = range '\x3200' '\x32FF' -cjkCompatibility = range '\x3300' '\x33FF' -cjkUnifiedIdeographsExtensionA = range '\x3400' '\x4DBF' -yijingHexagramSymbols = range '\x4DC0' '\x4DFF' -cjkUnifiedIdeographs = range '\x4E00' '\x9FFF' -yiSyllables = range '\xA000' '\xA48F' -yiRadicals = range '\xA490' '\xA4CF' -hangulSyllables = range '\xAC00' '\xD7AF' -highSurrogates = range '\xD800' '\xDB7F' -highPrivateUseSurrogates = range '\xDB80' '\xDBFF' -lowSurrogates = range '\xDC00' '\xDFFF' -privateUseArea = range '\xE000' '\xF8FF' -cjkCompatibilityIdeographs = range '\xF900' '\xFAFF' -alphabeticPresentationForms = range '\xFB00' '\xFB4F' -arabicPresentationFormsA = range '\xFB50' '\xFDFF' -variationSelectors = range '\xFE00' '\xFE0F' -combiningHalfMarks = range '\xFE20' '\xFE2F' -cjkCompatibilityForms = range '\xFE30' '\xFE4F' -smallFormVariants = range '\xFE50' '\xFE6F' -arabicPresentationFormsB = range '\xFE70' '\xFEFF' -halfwidthAndFullwidthForms = range '\xFF00' '\xFFEF' -specials = range '\xFFF0' '\xFFFF' diff --git a/Text/Trifecta/CharSet/Unicode/Category.hs b/Text/Trifecta/CharSet/Unicode/Category.hs deleted file mode 100644 --- a/Text/Trifecta/CharSet/Unicode/Category.hs +++ /dev/null @@ -1,212 +0,0 @@ -{-# LANGUAGE DeriveDataTypeable #-} ------------------------------------------------------------------------------ --- | --- Module : Text.Trifecta.CharSet.Unicode.Category --- Copyright : (c) Edward Kmett 2010-2011 --- License : BSD3 --- Maintainer : ekmett@gmail.com --- Stability : experimental --- Portability : portable --- --- Provides unicode general categories, which are typically connoted by --- @\p{Ll}@ or @\p{Modifier_Letter}@. Lookups can be constructed using 'categories' --- or individual character sets can be used directly. --- --- A case, @_@ and @-@ insensitive lookup is provided by 'lookupCategory' --- and can be used to provide behavior similar to that of Perl or PCRE. -------------------------------------------------------------------------------- - -module Text.Trifecta.CharSet.Unicode.Category - ( - -- * Unicode General Category - Category(..) - -- * Lookup - , categories - , lookupCategory - , lookupCategoryCharSet - -- * CharSets by Category - -- ** Letter - , modifierLetter, otherLetter, letter - -- *** Letter\& - , lowercaseLetter, uppercaseLetter, titlecaseLetter, letterAnd - -- ** Mark - , nonSpacingMark, spacingCombiningMark, enclosingMark, mark - -- ** Separator - , space, lineSeparator, paragraphSeparator, separator - -- ** Symbol - , mathSymbol, currencySymbol, modifierSymbol, otherSymbol, symbol - -- ** Number - , decimalNumber, letterNumber, otherNumber, number - -- ** Punctuation - , dashPunctuation, openPunctuation, closePunctuation, initialQuote - , finalQuote, connectorPunctuation, otherPunctuation, punctuation - -- ** Other - , control, format, privateUse, surrogate, notAssigned, other - ) where - -import Data.Char -import Text.Trifecta.CharSet -import Data.Data -import Data.HashMap.Lazy (HashMap) -import qualified Data.HashMap.Lazy as HashMap - -data Category = Category - { categoryName :: String - , categoryAbbreviation :: String - , categoryCharSet :: CharSet - , categoryDescription :: String - } deriving (Show, Data, Typeable) - --- \p{Letter} or \p{Mc} -categories :: [Category] -categories = - [ Category "Letter" "L" letter "any kind of letter from any language." - , Category "Lowercase_Letter" "Ll" lowercaseLetter "a lowercase letter that has an uppercase variant" - , Category "Uppercase_Letter" "Lu" uppercaseLetter "an uppercase letter that has a lowercase variant" - , Category "Titlecase_Letter" "Lt" titlecaseLetter "a letter that appears at the start of a word when only the first letter of the word is capitalized" - , Category "Letter&" "L&" letterAnd "a letter that exists in lowercase and uppercase variants (combination of Ll, Lu and Lt)" - , Category "Modifier_Letter" "Lm" modifierLetter "a special character that is used like a letter" - , Category "Other_Letter" "Lo" otherLetter "a letter or ideograph that does not have lowercase and uppercase variants" - , Category "Mark" "M" mark "a character intended to be combined with another character (e.g. accents, umlauts, enclosing boxes, etc.)" - , Category "Non_Spacing_Mark" "Mn" nonSpacingMark "a character intended to be combined with another character without taking up extra space (e.g. accents, umlauts, etc.)" - , Category "Spacing_Combining_Mark" "Mc" spacingCombiningMark "a character intended to be combined with another character that takes up extra space (vowel signs in many Eastern languages)" - , Category "Enclosing_Mark" "Me" enclosingMark "a character that encloses the character is is combined with (circle, square, keycap, etc.)" - , Category "Separator" "Z" separator "any kind of whitespace or invisible separator" - , Category "Space_Separator" "Zs" space "a whitespace character that is invisible, but does take up space" - , Category "Line_Separator" "Zl" lineSeparator "line separator character U+2028" - , Category "Paragraph_Separator" "Zp" paragraphSeparator "paragraph separator character U+2029" - , Category "Symbol" "S" symbol "math symbols, currency signs, dingbats, box-drawing characters, etc." - , Category "Math_Symbol" "Sm" mathSymbol "any mathematical symbol" - , Category "Currency_Symbol" "Sc" currencySymbol "any currency sign" - , Category "Modifier_Symbol" "Sk" modifierSymbol "a combining character (mark) as a full character on its own" - , Category "Other_Symbol" "So" otherSymbol "various symbols that are not math symbols, currency signs, or combining characters" - , Category "Number" "N" number "any kind of numeric character in any script" - , Category "Decimal_Digit_Number" "Nd" decimalNumber "a digit zero through nine in any script except ideographic scripts" - , Category "Letter_Number" "Nl" letterNumber "a number that looks like a letter, such as a Roman numeral" - , Category "Other_Number" "No" otherNumber "a superscript or subscript digit, or a number that is not a digit 0..9 (excluding numbers from ideographic scripts)" - , Category "Punctuation" "P" punctuation "any kind of punctuation character" - , Category "Dash_Punctuation" "Pd" dashPunctuation "any kind of hyphen or dash" - , Category "Open_Punctuation" "Ps" openPunctuation "any kind of opening bracket" - , Category "Close_Punctuation" "Pe" closePunctuation "any kind of closing bracket" - , Category "Initial_Punctuation" "Pi" initialQuote "any kind of opening quote" - , Category "Final_Punctuation" "Pf" finalQuote "any kind of closing quote" - , Category "Connector_Punctuation" "Pc" connectorPunctuation "a punctuation character such as an underscore that connects words" - , Category "Other_Punctuation" "Po" otherPunctuation "any kind of punctuation character that is not a dash, bracket, quote or connector" - , Category "Other" "C" other "invisible control characters and unused code points" - , Category "Control" "Cc" control "an ASCII 0x00..0x1F or Latin-1 0x80..0x9F control character" - , Category "Format" "Cf" format "invisible formatting indicator" - , Category "Private_Use" "Co" privateUse "any code point reserved for private use" - , Category "Surrogate" "Cs" surrogate "one half of a surrogate pair in UTF-16 encoding" - , Category "Unassigned" "Cn" notAssigned "any code point to which no character has been assigned.properties" ] - -lookupTable :: HashMap String Category -lookupTable = HashMap.fromList - [ (canonicalize x, category) - | category@(Category l s _ _) <- categories - , x <- [l,s] - ] - -lookupCategory :: String -> Maybe Category -lookupCategory s = HashMap.lookup (canonicalize s) lookupTable - -lookupCategoryCharSet :: String -> Maybe CharSet -lookupCategoryCharSet = fmap categoryCharSet . lookupCategory - -canonicalize :: String -> String -canonicalize s = case Prelude.map toLower s of - 'i' : 's' : xs -> go xs - xs -> go xs - where - go ('-':xs) = go xs - go ('_':xs) = go xs - go (' ':xs) = go xs - go (x:xs) = x : go xs - go [] = [] - -cat :: GeneralCategory -> CharSet -cat category = build ((category ==) . generalCategory) - --- Letter -lowercaseLetter, uppercaseLetter, titlecaseLetter, letterAnd, modifierLetter, otherLetter, letter :: CharSet -lowercaseLetter = cat LowercaseLetter -uppercaseLetter = cat UppercaseLetter -titlecaseLetter = cat TitlecaseLetter -letterAnd = lowercaseLetter - `union` uppercaseLetter - `union` titlecaseLetter -modifierLetter = cat ModifierLetter -otherLetter = cat OtherLetter -letter - = letterAnd - `union` modifierLetter - `union` otherLetter - --- Marks -nonSpacingMark, spacingCombiningMark, enclosingMark, mark :: CharSet -nonSpacingMark = cat NonSpacingMark -spacingCombiningMark = cat SpacingCombiningMark -enclosingMark = cat EnclosingMark -mark - = nonSpacingMark - `union` spacingCombiningMark - `union` enclosingMark - -space, lineSeparator, paragraphSeparator, separator :: CharSet -space = cat Space -lineSeparator = cat LineSeparator -paragraphSeparator = cat ParagraphSeparator -separator - = space - `union` lineSeparator - `union` paragraphSeparator - -mathSymbol, currencySymbol, modifierSymbol, otherSymbol, symbol :: CharSet -mathSymbol = cat MathSymbol -currencySymbol = cat CurrencySymbol -modifierSymbol = cat ModifierSymbol -otherSymbol = cat OtherSymbol -symbol - = mathSymbol - `union` currencySymbol - `union` modifierSymbol - `union` otherSymbol - -decimalNumber, letterNumber, otherNumber, number :: CharSet -decimalNumber = cat DecimalNumber -letterNumber = cat LetterNumber -otherNumber = cat OtherNumber -number - = decimalNumber - `union` letterNumber - `union` otherNumber - -dashPunctuation, openPunctuation, closePunctuation, initialQuote, - finalQuote, connectorPunctuation, otherPunctuation, punctuation :: CharSet - -dashPunctuation = cat DashPunctuation -openPunctuation = cat OpenPunctuation -closePunctuation = cat ClosePunctuation -initialQuote = cat InitialQuote -finalQuote = cat FinalQuote -connectorPunctuation = cat ConnectorPunctuation -otherPunctuation = cat OtherPunctuation -punctuation - = dashPunctuation - `union` openPunctuation - `union` closePunctuation - `union` initialQuote - `union` finalQuote - `union` connectorPunctuation - `union` otherPunctuation - -control, format, privateUse, surrogate, notAssigned, other :: CharSet -control = cat Control -format = cat Format -privateUse = cat PrivateUse -surrogate = cat Surrogate -notAssigned = cat NotAssigned -other = control - `union` format - `union` privateUse - `union` surrogate - `union` notAssigned diff --git a/Text/Trifecta/Parser/ByteString.hs b/Text/Trifecta/Parser/ByteString.hs --- a/Text/Trifecta/Parser/ByteString.hs +++ b/Text/Trifecta/Parser/ByteString.hs @@ -18,6 +18,8 @@ module Text.Trifecta.Parser.ByteString ( parseFromFile , parseFromFileEx + , parseByteString + , parseTest ) where import Control.Applicative @@ -33,8 +35,6 @@ import Text.Trifecta.Parser.Result import Data.Sequence as Seq import qualified Data.ByteString.UTF8 as UTF8 -import Text.Trifecta.Rope.Prim -import qualified Data.FingerTree as F -- | @parseFromFile p filePath@ runs a parser @p@ on the @@ -68,13 +68,26 @@ -- > parseFromFileEx :: Show a => (forall r. Parser r String a) -> String -> IO (Result TermDoc a) -parseFromFileEx p fn = k <$> B.readFile fn where - k i = starve - $ feed (rope (F.fromList [LineDirective (UTF8.fromString fn) 0, strand i])) +parseFromFileEx p fn = parseByteString p (Directed (UTF8.fromString fn) 0 0 0 0) <$> B.readFile fn + +-- | @parseByteString p delta i@ runs a parser @p@ on @i@. + +parseByteString :: Show a => (forall r. Parser r String a) -> Delta -> UTF8.ByteString -> Result TermDoc a +parseByteString p d inp = starve + $ feed inp $ stepParser (fmap prettyTerm) (why prettyTerm) - (release (Directed (UTF8.fromString fn) 0 0 0 0) *> p) + (release d *> p) mempty True mempty mempty + + +parseTest :: Show a => (forall r. Parser r String a) -> String -> IO () +parseTest p s = case parseByteString p mempty (UTF8.fromString s) of + Failure xs -> displayLn $ toList xs + Success xs a -> do + unless (Seq.null xs) $ displayLn $ toList xs + print a + diff --git a/Text/Trifecta/Parser/Char.hs b/Text/Trifecta/Parser/Char.hs --- a/Text/Trifecta/Parser/Char.hs +++ b/Text/Trifecta/Parser/Char.hs @@ -40,9 +40,9 @@ import Text.Trifecta.Parser.Class import Text.Trifecta.Rope.Delta import qualified Data.IntSet as IntSet -import Text.Trifecta.CharSet (CharSet(..)) -import qualified Text.Trifecta.CharSet as CharSet -import qualified Text.Trifecta.Util.ByteSet as ByteSet +import Data.CharSet (CharSet(..)) +import qualified Data.CharSet as CharSet +import qualified Data.CharSet.ByteSet as ByteSet import qualified Data.ByteString as Strict import Data.ByteString.Internal (w2c,c2w) import Data.ByteString.UTF8 as UTF8 diff --git a/Text/Trifecta/Parser/Char8.hs b/Text/Trifecta/Parser/Char8.hs --- a/Text/Trifecta/Parser/Char8.hs +++ b/Text/Trifecta/Parser/Char8.hs @@ -49,8 +49,8 @@ import Control.Monad (guard) import Text.Trifecta.Parser.Class hiding (satisfy) import Text.Trifecta.Rope.Delta -import Text.Trifecta.Util.ByteSet (ByteSet(..)) -import qualified Text.Trifecta.Util.ByteSet as ByteSet +import Data.CharSet.ByteSet (ByteSet(..)) +import qualified Data.CharSet.ByteSet as ByteSet import qualified Data.ByteString as Strict import Data.ByteString.Internal (w2c,c2w) import qualified Data.ByteString.Char8 as Char8 diff --git a/Text/Trifecta/Parser/Class.hs b/Text/Trifecta/Parser/Class.hs --- a/Text/Trifecta/Parser/Class.hs +++ b/Text/Trifecta/Parser/Class.hs @@ -32,7 +32,6 @@ import Control.Monad.Trans.RWS.Strict as Strict import Control.Monad.Trans.Reader import Control.Monad.Trans.Identity -import Data.Functor.Yoneda import Data.Word import Data.ByteString as Strict import Data.Char (isSpace) @@ -230,22 +229,6 @@ position = lift position slicedWith f (IdentityT m) = IdentityT $ slicedWith f m lookAhead (IdentityT m) = IdentityT $ lookAhead m - -instance MonadParser m => MonadParser (Yoneda m) where - try = lift . try . lowerYoneda - labels m ss = lift $ labels (lowerYoneda m) ss - line = lift line - unexpected = lift . unexpected - satisfy = lift . satisfy - satisfy8 = lift . satisfy8 - someSpace = lift someSpace - semi = lift semi - highlightInterval h s e = lift $ highlightInterval h s e - nesting (Yoneda m) = Yoneda $ \f -> nesting (m f) - skipping = lift . skipping - position = lift position - slicedWith f = lift . slicedWith f . lowerYoneda - lookAhead = lift . lookAhead . lowerYoneda -- | Skip zero or more bytes worth of white space. More complex parsers are -- free to consider comments as white space. diff --git a/Text/Trifecta/Parser/Mark.hs b/Text/Trifecta/Parser/Mark.hs --- a/Text/Trifecta/Parser/Mark.hs +++ b/Text/Trifecta/Parser/Mark.hs @@ -23,7 +23,6 @@ import Control.Monad.Trans.RWS.Strict as Strict import Control.Monad.Trans.Reader import Control.Monad.Trans.Identity -import Data.Functor.Yoneda import Data.Monoid import Text.Trifecta.Rope.Delta import Text.Trifecta.Parser.Class @@ -66,6 +65,3 @@ mark = lift mark release = lift . release -instance MonadMark d m => MonadMark d (Yoneda m) where - mark = lift mark - release = lift . release diff --git a/Text/Trifecta/Parser/Prim.hs b/Text/Trifecta/Parser/Prim.hs --- a/Text/Trifecta/Parser/Prim.hs +++ b/Text/Trifecta/Parser/Prim.hs @@ -14,7 +14,6 @@ ( Parser(..) , why , stepParser - , parseTest , manyAccum ) where @@ -55,7 +54,6 @@ import Text.Trifecta.Rope.Delta as Delta import Text.Trifecta.Rope.Prim import Text.Trifecta.Rope.Bytes -import System.Console.Terminfo.PrettyPrint data Parser r e a = Parser { unparser :: @@ -322,12 +320,3 @@ errLoc (PanicErr r _) = Just $ delta r errLoc (Err (Diagnostic (Left _) _ _ _)) = Nothing errLoc (Err (Diagnostic (Right r) _ _ _)) = Just $ delta r - -parseTest :: Show a => (forall r. Parser r String a) -> String -> IO () -parseTest p s = case starve - $ feed (UTF8.fromString s) - $ stepParser (fmap prettyTerm) (why prettyTerm) (release mempty *> p) mempty True mempty mempty of - Failure xs -> displayLn $ toList xs - Success xs a -> do - unless (Seq.null xs) $ displayLn $ toList xs - print a diff --git a/Text/Trifecta/Util/ByteSet.hs b/Text/Trifecta/Util/ByteSet.hs deleted file mode 100644 --- a/Text/Trifecta/Util/ByteSet.hs +++ /dev/null @@ -1,64 +0,0 @@ -{-# LANGUAGE BangPatterns, MagicHash #-} ------------------------------------------------------------------------------ --- | --- Module : Text.Trifecta.Util.ByteSet --- Copyright : Edward Kmett 2011 --- Bryan O'Sullivan 2008 --- License : BSD3 --- --- Maintainer : ekmett@gmail.com --- Stability : experimental --- Portability : unknown --- --- Fast set membership tests for byte values, The set representation is --- unboxed for efficiency and uses a lookup table. This is a fairly minimal --- API. You probably want to use CharSet. ------------------------------------------------------------------------------ -module Text.Trifecta.Util.ByteSet - ( - -- * Data type - ByteSet(..) - -- * Construction - , fromList - -- * Lookup - , member - ) where - -import Data.Bits ((.&.), (.|.)) -import Foreign.Storable (peekByteOff, pokeByteOff) -import GHC.Base (Int(I#), iShiftRA#, narrow8Word#, shiftL#) -import GHC.Word (Word8(W8#)) -import qualified Data.ByteString as B -import qualified Data.ByteString.Internal as I -import qualified Data.ByteString.Unsafe as U - -newtype ByteSet = ByteSet B.ByteString deriving (Eq, Ord, Show) - -data I = I {-# UNPACK #-} !Int {-# UNPACK #-} !Word8 - -shiftR :: Int -> Int -> Int -shiftR (I# x#) (I# i#) = I# (x# `iShiftRA#` i#) - -shiftL :: Word8 -> Int -> Word8 -shiftL (W8# x#) (I# i#) = W8# (narrow8Word# (x# `shiftL#` i#)) - -index :: Int -> I -index i = I (i `shiftR` 3) (1 `shiftL` (i .&. 7)) -{-# INLINE index #-} - -fromList :: [Word8] -> ByteSet -fromList s0 = ByteSet $ I.unsafeCreate 32 $ \t -> do - _ <- I.memset t 0 32 - let go [] = return () - go (c:cs) = do - prev <- peekByteOff t byte :: IO Word8 - pokeByteOff t byte (prev .|. bit) - go cs - where I byte bit = index (fromIntegral c) - go s0 - --- | Check the set for membership. -member :: Word8 -> ByteSet -> Bool -member w (ByteSet t) = U.unsafeIndex t byte .&. bit /= 0 - where - I byte bit = index (fromIntegral w) diff --git a/trifecta.cabal b/trifecta.cabal --- a/trifecta.cabal +++ b/trifecta.cabal @@ -1,6 +1,6 @@ name: trifecta category: Text, Parsing, Diagnostics, Pretty Printer, Logging -version: 0.51.0.1 +version: 0.52 license: BSD3 cabal-version: >= 1.6 license-file: LICENSE @@ -25,14 +25,6 @@ library exposed-modules: Text.Trifecta - Text.Trifecta.CharSet - Text.Trifecta.CharSet.Common - Text.Trifecta.CharSet.Posix - Text.Trifecta.CharSet.Posix.Ascii - Text.Trifecta.CharSet.Posix.Unicode - Text.Trifecta.CharSet.Unicode - Text.Trifecta.CharSet.Unicode.Block - Text.Trifecta.CharSet.Unicode.Category Text.Trifecta.IntervalMap Text.Trifecta.Rope Text.Trifecta.Rope.Bytes @@ -94,7 +86,6 @@ Text.Trifecta.Parser.Identifier Text.Trifecta.Parser.Identifier.Style Text.Trifecta.Util.Array - Text.Trifecta.Util.ByteSet other-modules: Text.Trifecta.Util.Combinators @@ -104,6 +95,7 @@ build-depends: base >= 4 && < 5, array >= 0.3.0.2 && < 0.5, + charset >= 0.3.2 && < 0.4, containers >= 0.3 && < 0.6, unordered-containers >= 0.2.1 && < 0.3, blaze-builder >= 0.3.0.1 && < 0.4, @@ -123,7 +115,6 @@ semigroupoids >= 1.3.1.2 && < 1.4, pointed >= 2.1.0.1 && < 2.2, transformers >= 0.2 && < 0.4, - kan-extensions >= 2.4.0.1 && < 2.5, comonad >= 1.1.1.3 && < 1.2, terminfo >= 0.3.2 && < 0.4, keys >= 2.1.3.1 && < 2.2,