charset 0.2.3 → 0.3

raw patch · 11 files changed

+495/−181 lines, 11 filesdep +bytestringdep +semigroupsdep +unordered-containersdep ~basesetup-changedPVP ok

version bump matches the API change (PVP)

Dependencies added: bytestring, semigroups, unordered-containers

Dependency ranges changed: base

API changes (from Hackage documentation)

Files

• Data/CharSet.hs +146/−118
• Data/CharSet/ByteSet.hs +64/−0
• Data/CharSet/Common.hs +1/−1
• Data/CharSet/Posix.hs +21/−0
• Data/CharSet/Posix/Ascii.hs +6/−6
• Data/CharSet/Posix/Unicode.hs +7/−7
• Data/CharSet/Unicode.hs +177/−0
• Data/CharSet/Unicode/Block.hs +9/−10
• Data/CharSet/Unicode/Category.hs +20/−19
• Setup.lhs +1/−1
• charset.cabal +43/−19

@@ -1,25 +1,31 @@-{-# LANGUAGE CPP #-}+{-# OPTIONS_GHC -fspec-constr #-} ----------------------------------------------------------------------------- -- | -- Module      :  Data.CharSet--- Copyright   :  (c) Edward Kmett 2010+-- Copyright   :  (c) Edward Kmett 2010-2011 -- License     :  BSD3 -- Maintainer  :  ekmett@gmail.com -- Stability   :  experimental--- Portability :  portable (CPP)+-- Portability :  portable ----- Fast complementable character sets+-- Fast set membership tests for 'Char' values --+-- Stored as a (possibly negated) IntMap and a fast set used for the head byte.+--+-- The set of valid (possibly negated) head bytes is stored unboxed as a 32-byte+-- bytestring-based lookup table.+-- -- Designed to be imported qualified:--- +-- -- > import Data.CharSet (CharSet) -- > import qualified Data.CharSet as CharSet+-- ------------------------------------------------------------------------------- -module Data.CharSet -    ( +module Data.CharSet+    (     -- * Set type-      CharSet+      CharSet(..)     -- * Operators     , (\\)     -- * Query@@ -28,7 +34,7 @@     , member     , notMember     , overlaps, isSubsetOf-    , isComplemented +    , isComplemented     -- * Construction     , build     , empty@@ -68,15 +74,36 @@ import Data.Data import Data.Function (on) import Data.IntSet (IntSet)-import Data.Monoid (Monoid(..))+import Data.CharSet.ByteSet (ByteSet)+import qualified Data.CharSet.ByteSet as ByteSet+import Data.Bits hiding (complement)+import Data.Word+import Data.ByteString.Internal (c2w)+import Data.Semigroup import qualified Data.IntSet as I import qualified Data.List as L import Prelude hiding (filter, map, null) import qualified Prelude as P import Text.Read -data CharSet = P IntSet | N IntSet+data CharSet = CharSet !Bool {-# UNPACK #-} !ByteSet !IntSet +charSet :: Bool -> IntSet -> CharSet+charSet b s = CharSet b (ByteSet.fromList (fmap headByte (I.toAscList s))) s++headByte :: Int -> Word8+headByte i+  | i <= 0x7f   = toEnum i+  | i <= 0x7ff  = toEnum $ 0x80 + (i `shiftR` 6)+  | i <= 0xffff = toEnum $ 0xe0 + (i `shiftR` 12)+  | otherwise   = toEnum $ 0xf0 + (i `shiftR` 18)++pos :: IntSet -> CharSet+pos = charSet True++neg :: IntSet -> CharSet+neg = charSet False+ (\\) :: CharSet -> CharSet -> CharSet (\\) = difference @@ -85,139 +112,142 @@ {-# INLINE build #-}  map :: (Char -> Char) -> CharSet -> CharSet-map f (P i) = P (I.map (fromEnum . f . toEnum) i)-map f (N i) = fromList $ P.map f $ P.filter (\x -> fromEnum x `I.notMember` i) [ul..uh] +map f (CharSet True _ i) = pos (I.map (fromEnum . f . toEnum) i)+map f (CharSet False _ i) = fromList $ P.map f $ P.filter (\x -> fromEnum x `I.notMember` i) [ul..uh] {-# INLINE map #-}  isComplemented :: CharSet -> Bool-isComplemented (P _) = False-isComplemented (N _) = True+isComplemented (CharSet True _ _) = False+isComplemented (CharSet False _ _) = True {-# INLINE isComplemented #-}  toList :: CharSet -> String-toList (P i) = P.map toEnum (I.toList i)-toList (N i) = P.filter (\x -> fromEnum x `I.notMember` i) [ul..uh]+toList (CharSet True _ i) = P.map toEnum (I.toList i)+toList (CharSet False _ i) = P.filter (\x -> fromEnum x `I.notMember` i) [ul..uh] {-# INLINE toList #-}  toAscList :: CharSet -> String-toAscList (P i) = P.map toEnum (I.toAscList i)-toAscList (N i) = P.filter (\x -> fromEnum x `I.notMember` i) [ul..uh]+toAscList (CharSet True _ i) = P.map toEnum (I.toAscList i)+toAscList (CharSet False _ i) = P.filter (\x -> fromEnum x `I.notMember` i) [ul..uh] {-# INLINE toAscList #-}-    + empty :: CharSet-empty = P I.empty-{-# INLINE empty #-}+empty = pos I.empty  singleton :: Char -> CharSet-singleton = P . I.singleton . fromEnum+singleton = pos . I.singleton . fromEnum {-# INLINE singleton #-}  full :: CharSet-full = N I.empty-{-# INLINE full #-}+full = neg I.empty +-- | /O(n)/ worst case null :: CharSet -> Bool-null (P i) = I.null i-null (N i) = I.size i == numChars -- badly normalized!+null (CharSet True _ i) = I.null i+null (CharSet False _ i) = I.size i == numChars {-# INLINE null #-} +-- | /O(n)/ size :: CharSet -> Int-size (P i) = I.size i-size (N i) = numChars - I.size i+size (CharSet True _ i) = I.size i+size (CharSet False _ i) = numChars - I.size i {-# INLINE size #-}  insert :: Char -> CharSet -> CharSet-insert c (P i) = P (I.insert (fromEnum c) i)-insert c (N i) = P (I.delete (fromEnum c) i)+insert c (CharSet True _ i) = pos (I.insert (fromEnum c) i)+insert c (CharSet False _ i) = neg (I.delete (fromEnum c) i) {-# INLINE insert #-}  range :: Char -> Char -> CharSet-range a b -    | a <= b = fromDistinctAscList [a..b]-    | otherwise = empty+range a b+  | a <= b = fromDistinctAscList [a..b]+  | otherwise = empty  delete :: Char -> CharSet -> CharSet-delete c (P i) = P (I.delete (fromEnum c) i)-delete c (N i) = N (I.insert (fromEnum c) i)+delete c (CharSet True _ i) = pos (I.delete (fromEnum c) i)+delete c (CharSet False _ i) = neg (I.insert (fromEnum c) i) {-# INLINE delete #-}  complement :: CharSet -> CharSet-complement (P i) = N i-complement (N i) = P i+complement (CharSet True s i) = CharSet False s i+complement (CharSet False s i) = CharSet True s i {-# INLINE complement #-}  union :: CharSet -> CharSet -> CharSet-union (P i) (P j) = P (I.union i j)-union (P i) (N j) = N (I.difference j i)-union (N i) (P j) = N (I.difference i j)-union (N i) (N j) = N (I.intersection i j)+union (CharSet True _ i) (CharSet True _ j) = pos (I.union i j)+union (CharSet True _ i) (CharSet False _ j) = neg (I.difference j i)+union (CharSet False _ i) (CharSet True _ j) = neg (I.difference i j)+union (CharSet False _ i) (CharSet False _ j) = neg (I.intersection i j) {-# INLINE union #-}  intersection :: CharSet -> CharSet -> CharSet-intersection (P i) (P j) = P (I.intersection i j)-intersection (P i) (N j) = P (I.difference i j)-intersection (N i) (P j) = P (I.difference j i)-intersection (N i) (N j) = N (I.union i j)+intersection (CharSet True _ i) (CharSet True _ j) = pos (I.intersection i j)+intersection (CharSet True _ i) (CharSet False _ j) = pos (I.difference i j)+intersection (CharSet False _ i) (CharSet True _ j) = pos (I.difference j i)+intersection (CharSet False _ i) (CharSet False _ j) = neg (I.union i j) {-# INLINE intersection #-} -difference :: CharSet -> CharSet -> CharSet -difference (P i) (P j) = P (I.difference i j)-difference (P i) (N j) = P (I.intersection i j)-difference (N i) (P j) = N (I.union i j)-difference (N i) (N j) = P (I.difference j i)+difference :: CharSet -> CharSet -> CharSet+difference (CharSet True _ i) (CharSet True _ j) = pos (I.difference i j)+difference (CharSet True _ i) (CharSet False _ j) = pos (I.intersection i j)+difference (CharSet False _ i) (CharSet True _ j) = neg (I.union i j)+difference (CharSet False _ i) (CharSet False _ j) = pos (I.difference j i) {-# INLINE difference #-}  member :: Char -> CharSet -> Bool-member c (P i) = I.member (fromEnum c) i-member c (N i) = I.notMember (fromEnum c) i+member c (CharSet True b i)+  | c <= toEnum 0x7f = ByteSet.member (c2w c) b+  | otherwise        = I.member (fromEnum c) i+member c (CharSet False b i)+  | c <= toEnum 0x7f = not (ByteSet.member (c2w c) b)+  | otherwise        = I.notMember (fromEnum c) i {-# INLINE member #-}  notMember :: Char -> CharSet -> Bool-notMember c (P i) = I.notMember (fromEnum c) i-notMember c (N i) = I.member (fromEnum c) i+notMember c s = not (member c s) {-# INLINE notMember #-}  fold :: (Char -> b -> b) -> b -> CharSet -> b-fold f z (P i) = I.fold (f . toEnum) z i-fold f z (N i) = foldr f z $ P.filter (\x -> fromEnum x `I.notMember` i) [ul..uh]+fold f z (CharSet True _ i) = I.fold (f . toEnum) z i+fold f z (CharSet False _ i) = foldr f z $ P.filter (\x -> fromEnum x `I.notMember` i) [ul..uh] {-# INLINE fold #-} -filter :: (Char -> Bool) -> CharSet -> CharSet -filter p (P i) = P (I.filter (p . toEnum) i)-filter p (N i) = N $ foldr (I.insert) i $ P.filter (\x -> (x `I.notMember` i) && not (p (toEnum x))) [ol..oh]+filter :: (Char -> Bool) -> CharSet -> CharSet+filter p (CharSet True _ i) = pos (I.filter (p . toEnum) i)+filter p (CharSet False _ i) = neg $ foldr (I.insert) i $ P.filter (\x -> (x `I.notMember` i) && not (p (toEnum x))) [ol..oh] {-# INLINE filter #-}  partition :: (Char -> Bool) -> CharSet -> (CharSet, CharSet)-partition p (P i) = (P l, P r)+partition p (CharSet True _ i) = (pos l, pos r)     where (l,r) = I.partition (p . toEnum) i-partition p (N i) = (N (foldr I.insert i l), N (foldr I.insert i r))+partition p (CharSet False _ i) = (neg (foldr I.insert i l), neg (foldr I.insert i r))     where (l,r) = L.partition (p . toEnum) $ P.filter (\x -> x `I.notMember` i) [ol..oh] {-# INLINE partition #-}  overlaps :: CharSet -> CharSet -> Bool-overlaps (P i) (P j) = not (I.null (I.intersection i j))-overlaps (P i) (N j) = not (I.isSubsetOf j i)-overlaps (N i) (P j) = not (I.isSubsetOf i j)-overlaps (N i) (N j) = any (\x -> I.notMember x i && I.notMember x j) [ol..oh] -- not likely+overlaps (CharSet True _ i) (CharSet True _ j) = not (I.null (I.intersection i j))+overlaps (CharSet True _ i) (CharSet False _ j) = not (I.isSubsetOf j i)+overlaps (CharSet False _ i) (CharSet True _ j) = not (I.isSubsetOf i j)+overlaps (CharSet False _ i) (CharSet False _ j) = any (\x -> I.notMember x i && I.notMember x j) [ol..oh] -- not likely {-# INLINE overlaps #-}  isSubsetOf :: CharSet -> CharSet -> Bool-isSubsetOf (P i) (P j) = I.isSubsetOf i j-isSubsetOf (P i) (N j) = I.null (I.intersection i j)-isSubsetOf (N i) (P j) = all (\x -> I.member x i && I.member x j) [ol..oh]-- not bloody likely-isSubsetOf (N i) (N j) = I.isSubsetOf j i+isSubsetOf (CharSet True _ i) (CharSet True _ j) = I.isSubsetOf i j+isSubsetOf (CharSet True _ i) (CharSet False _ j) = I.null (I.intersection i j)+isSubsetOf (CharSet False _ i) (CharSet True _ j) = all (\x -> I.member x i && I.member x j) [ol..oh] -- not bloody likely+isSubsetOf (CharSet False _ i) (CharSet False _ j) = I.isSubsetOf j i {-# INLINE isSubsetOf #-} -fromList :: String -> CharSet -fromList = P . I.fromList . P.map fromEnum+fromList :: String -> CharSet+fromList = pos . I.fromList . P.map fromEnum {-# INLINE fromList #-}  fromAscList :: String -> CharSet-fromAscList = P . I.fromAscList . P.map fromEnum+fromAscList = pos . I.fromAscList . P.map fromEnum {-# INLINE fromAscList #-}  fromDistinctAscList :: String -> CharSet-fromDistinctAscList = P . I.fromDistinctAscList . P.map fromEnum+fromDistinctAscList = pos . I.fromDistinctAscList . P.map fromEnum {-# INLINE fromDistinctAscList #-}  -- isProperSubsetOf :: CharSet -> CharSet -> Bool@@ -242,26 +272,31 @@ {-# INLINE numChars #-}  instance Typeable CharSet where-    typeOf _ = mkTyConApp charSetTyCon []+  typeOf _ = mkTyConApp charSetTyCon []  charSetTyCon :: TyCon-charSetTyCon = mkTyCon "Data.CharSet.CharSet"+#ifdef OLD_TYPEABLE+charSetTyCon = mkTyCon "Data.CharSet.harSet"+#else+charSetTyCon = mkTyCon3 "charset" "Data.CharSet" "CharSet"+#endif {-# NOINLINE charSetTyCon #-}  instance Data CharSet where-    gfoldl k z set | isComplemented set = z complement `k` complement set-                   | otherwise          = z fromList `k` toList set+  gfoldl k z set+    | isComplemented set = z complement `k` complement set+    | otherwise          = z fromList `k` toList set -    toConstr set -        | isComplemented set = complementConstr-        | otherwise = fromListConstr+  toConstr set+    | isComplemented set = complementConstr+    | otherwise = fromListConstr -    dataTypeOf _ = charSetDataType+  dataTypeOf _ = charSetDataType -    gunfold k z c = case constrIndex c of-        1 -> k (z fromList)-        2 -> k (z complement)-        _ -> error "gunfold"+  gunfold k z c = case constrIndex c of+    1 -> k (z fromList)+    2 -> k (z complement)+    _ -> error "gunfold"  fromListConstr :: Constr fromListConstr   = mkConstr charSetDataType "fromList" [] Prefix@@ -275,54 +310,47 @@ charSetDataType  = mkDataType "Data.CharSet.CharSet" [fromListConstr, complementConstr] {-# NOINLINE charSetDataType #-} --- returns an intset and if the intset should be complemented to obtain the contents of the CharSet+-- returns an intset and if the charSet is positive fromCharSet :: CharSet -> (Bool, IntSet)-fromCharSet (P i) = (False, i)-fromCharSet (N i) = (True, i) +fromCharSet (CharSet b _ i) = (b, i) {-# INLINE fromCharSet #-}  toCharSet :: IntSet -> CharSet-toCharSet = P+toCharSet = pos {-# INLINE toCharSet #-}  instance Eq CharSet where-    (==) = (==) `on` toAscList+  (==) = (==) `on` toAscList  instance Ord CharSet where-    compare = compare `on` toAscList+  compare = compare `on` toAscList  instance Bounded CharSet where-    minBound = empty-    maxBound = full+  minBound = empty+  maxBound = full +-- TODO return a tighter bounded array perhaps starting from the least element present to the last element present? toArray :: CharSet -> UArray Char Bool toArray set = array (minBound, maxBound) $ fmap (\x -> (x, x `member` set)) [minBound .. maxBound]- + instance Show CharSet where-   showsPrec d i-        | isComplemented i = showParen (d > 10) $ showString "complement " . showsPrec 11 (complement i)-        | otherwise        = showParen (d > 10) $ showString "fromDistinctAscList " . showsPrec 11 (toAscList i)+  showsPrec d i+    | isComplemented i = showParen (d > 10) $ showString "complement " . showsPrec 11 (complement i)+    | otherwise        = showParen (d > 10) $ showString "fromDistinctAscList " . showsPrec 11 (toAscList i)  instance Read CharSet where-#ifdef __GLASGOW_HASKELL__ -    readPrec = parens $ complemented +++ normal -      where-        complemented = prec 10 $ do -                Ident "complement" <- lexP-                complement `fmap` step readPrec-        normal = prec 10 $ do-                Ident "fromDistinctAscList" <- lexP-                fromDistinctAscList `fmap` step readPrec-#else-    readsPrec d r = -        readParen (d > 10) (\r -> [ (complement m, t) -                                  | ("complement", s) <- lex r-                                  , (m, t) <- readsPrec 11 s]) r-     ++ readParen (d > 10) (\r -> [ (fromDistinctAscList m, t) -                                  | ("fromDistinctAscList", s) <- lex r-                                  , (m, t) <- readsPrec 11 s]) r-#endif+  readPrec = parens $ complemented +++ normal+    where+      complemented = prec 10 $ do+        Ident "complement" <- lexP+        complement `fmap` step readPrec+      normal = prec 10 $ do+        Ident "fromDistinctAscList" <- lexP+        fromDistinctAscList `fmap` step readPrec +instance Semigroup CharSet where+  (<>) = union+ instance Monoid CharSet where-    mempty = empty-    mappend = union+  mempty = empty+  mappend = union

@@ -0,0 +1,64 @@+{-# LANGUAGE BangPatterns, MagicHash #-}+-----------------------------------------------------------------------------+-- |+-- Module      :  Data.CharSet.ByteSet+-- Copyright   :  Edward Kmett 2011+--                Bryan O'Sullivan 2008+-- License     :  BSD3+--+-- Maintainer  :  ekmett@gmail.com+-- Stability   :  experimental+-- Portability :  non-portable (BangPatterns, MagicHash)+--+-- Fast set membership tests for byte values, The set representation is+-- unboxed for efficiency and uses a lookup table. This is a fairly minimal+-- API. You probably want to use CharSet.+-----------------------------------------------------------------------------+module Data.CharSet.ByteSet+    (+    -- * Data type+      ByteSet(..)+    -- * Construction+    , fromList+    -- * Lookup+    , member+    ) where++import Data.Bits ((.&.), (.|.))+import Foreign.Storable (peekByteOff, pokeByteOff)+import GHC.Base (Int(I#), iShiftRA#, narrow8Word#, shiftL#)+import GHC.Word (Word8(W8#))+import qualified Data.ByteString as B+import qualified Data.ByteString.Internal as I+import qualified Data.ByteString.Unsafe as U++newtype ByteSet = ByteSet B.ByteString deriving (Eq, Ord, Show)++data I = I {-# UNPACK #-} !Int {-# UNPACK #-} !Word8++shiftR :: Int -> Int -> Int+shiftR (I# x#) (I# i#) = I# (x# `iShiftRA#` i#)++shiftL :: Word8 -> Int -> Word8+shiftL (W8# x#) (I# i#) = W8# (narrow8Word# (x# `shiftL#` i#))++index :: Int -> I+index i = I (i `shiftR` 3) (1 `shiftL` (i .&. 7))+{-# INLINE index #-}++fromList :: [Word8] -> ByteSet+fromList s0 = ByteSet $ I.unsafeCreate 32 $ \t -> do+  _ <- I.memset t 0 32+  let go [] = return ()+      go (c:cs) = do+        prev <- peekByteOff t byte :: IO Word8+        pokeByteOff t byte (prev .|. bit)+        go cs+        where I byte bit = index (fromIntegral c)+  go s0++-- | Check the set for membership.+member :: Word8 -> ByteSet -> Bool+member w (ByteSet t) = U.unsafeIndex t byte .&. bit /= 0+  where+    I byte bit = index (fromIntegral w)

@@ -1,7 +1,7 @@ ----------------------------------------------------------------------------- -- | -- Module      :  Data.CharSet.Common--- Copyright   :  (c) Edward Kmett 2010+-- Copyright   :  (c) Edward Kmett 2010-2012 -- License     :  BSD3 -- Maintainer  :  ekmett@gmail.com -- Stability   :  experimental

@@ -0,0 +1,21 @@+-----------------------------------------------------------------------------+-- |+-- Module      :  Data.CharSet.Posix+-- Copyright   :  (c) Edward Kmett 2011-2012+-- License     :  BSD3+-- Maintainer  :  ekmett@gmail.com+-- Stability   :  experimental+-- Portability :  portable+--+-------------------------------------------------------------------------------++module Data.CharSet.Posix+    ( posixAscii+    , lookupPosixAsciiCharSet+    , posixUnicode+    , lookupPosixUnicodeCharSet+    ) where++import Data.CharSet.Posix.Ascii+import Data.CharSet.Posix.Unicode+import Prelude ()

@@ -19,8 +19,8 @@ import Prelude hiding (print) import Data.Char import Data.CharSet-import Data.Map (Map)-import qualified Data.Map as Map+import Data.HashMap.Lazy (HashMap)+import qualified Data.HashMap.Lazy as HashMap  alnum, alpha, ascii, blank, cntrl, digit, graph, print, word, punct, space, upper, lower, xdigit :: CharSet alnum = alpha `union` digit@@ -39,15 +39,15 @@ xdigit = digit `union` range 'a' 'f' `union` range 'A' 'F'  -- :digit:, etc.-posixAscii :: Map String CharSet-posixAscii = Map.fromList+posixAscii :: HashMap String CharSet+posixAscii = HashMap.fromList     [ ("alnum", alnum)     , ("alpha", alpha)     , ("ascii", ascii)     , ("blank", blank)     , ("cntrl", cntrl)     , ("digit", digit)-    , ("graph", graph) +    , ("graph", graph)     , ("print", print)     , ("word",  word)     , ("punct", punct)@@ -58,4 +58,4 @@     ]  lookupPosixAsciiCharSet :: String -> Maybe CharSet-lookupPosixAsciiCharSet s = Map.lookup (Prelude.map toLower s) posixAscii+lookupPosixAsciiCharSet s = HashMap.lookup (Prelude.map toLower s) posixAscii

@@ -21,14 +21,14 @@ import Data.CharSet import qualified Data.CharSet.Unicode.Category as Category import qualified Data.CharSet.Unicode.Block as Block-import Data.Map (Map)-import qualified Data.Map as Map+import Data.HashMap.Lazy (HashMap)+import qualified Data.HashMap.Lazy as HashMap  alnum, alpha, ascii, blank, cntrl, digit, graph, print, word, punct, space, upper, lower, xdigit :: CharSet alnum = alpha `union` digit ascii = Block.basicLatin alpha = Category.letterAnd-blank = insert '\t' Category.space +blank = insert '\t' Category.space cntrl = Category.control digit = Category.decimalNumber lower = Category.lowercaseLetter@@ -41,15 +41,15 @@ xdigit = digit `union` range 'a' 'f' `union` range 'A' 'F'  -- :digit:, etc.-posixUnicode :: Map String CharSet-posixUnicode = Map.fromList+posixUnicode :: HashMap String CharSet+posixUnicode = HashMap.fromList     [ ("alnum", alnum)     , ("alpha", alpha)     , ("ascii", ascii)     , ("blank", blank)     , ("cntrl", cntrl)     , ("digit", digit)-    , ("graph", graph) +    , ("graph", graph)     , ("print", print)     , ("word",  word)     , ("punct", punct)@@ -60,5 +60,5 @@     ]  lookupPosixUnicodeCharSet :: String -> Maybe CharSet-lookupPosixUnicodeCharSet s = Map.lookup (Prelude.map toLower s) posixUnicode+lookupPosixUnicodeCharSet s = HashMap.lookup (Prelude.map toLower s) posixUnicode 

@@ -0,0 +1,177 @@+{-# LANGUAGE DeriveDataTypeable #-}+-----------------------------------------------------------------------------+-- |+-- Module      :  Data.CharSet.Unicode+-- Copyright   :  (c) Edward Kmett 2010+-- License     :  BSD3+-- Maintainer  :  ekmett@gmail.com+-- Stability   :  experimental+-- Portability :  portable+--+-- Provides unicode general categories, which are typically connoted by+-- @\p{Ll}@ or @\p{Modifier_Letter}@. Lookups can be constructed using 'categories'+-- or individual character sets can be used directly.+-------------------------------------------------------------------------------++module Data.CharSet.Unicode+    (+    -- * Unicode General Category+      UnicodeCategory(..)+    -- * Lookup+    , unicodeCategories+    -- * CharSets by UnicodeCategory+    -- ** Letter+    , modifierLetter, otherLetter, letter+    -- *** Letter\&+    , lowercaseLetter, uppercaseLetter, titlecaseLetter, letterAnd+    -- ** Mark+    , nonSpacingMark, spacingCombiningMark, enclosingMark, mark+    -- ** Separator+    , space, lineSeparator, paragraphSeparator, separator+    -- ** Symbol+    , mathSymbol, currencySymbol, modifierSymbol, otherSymbol, symbol+    -- ** Number+    , decimalNumber, letterNumber, otherNumber, number+    -- ** Punctuation+    , dashPunctuation, openPunctuation, closePunctuation, initialQuote+    , finalQuote, connectorPunctuation, otherPunctuation, punctuation+    -- ** Other+    , control, format, privateUse, surrogate, notAssigned, other+    ) where++import Data.Char+import Data.Data+import Data.CharSet++data UnicodeCategory = UnicodeCategory String String CharSet String+    deriving (Show, Data, Typeable)++-- \p{Letter} or \p{Mc}+unicodeCategories :: [UnicodeCategory]+unicodeCategories =+    [ UnicodeCategory "Letter" "L" letter "any kind of letter from any language."+    ,     UnicodeCategory "Lowercase_Letter" "Ll" lowercaseLetter "a lowercase letter that has an uppercase variant"+    ,     UnicodeCategory "Uppercase_Letter" "Lu" uppercaseLetter "an uppercase letter that has a lowercase variant"+    ,     UnicodeCategory "Titlecase_Letter" "Lt" titlecaseLetter "a letter that appears at the start of a word when only the first letter of the word is capitalized"+    ,     UnicodeCategory "Letter&" "L&" letterAnd "a letter that exists in lowercase and uppercase variants (combination of Ll, Lu and Lt)"+    ,     UnicodeCategory "Modifier_Letter" "Lm" modifierLetter "a special character that is used like a letter"+    ,     UnicodeCategory "Other_Letter" "Lo" otherLetter "a letter or ideograph that does not have lowercase and uppercase variants"+    , UnicodeCategory "Mark" "M" mark "a character intended to be combined with another character (e.g. accents, umlauts, enclosing boxes, etc.)"+    ,     UnicodeCategory "Non_Spacing_Mark" "Mn" nonSpacingMark "a character intended to be combined with another character without taking up extra space (e.g. accents, umlauts, etc.)"+    ,     UnicodeCategory "Spacing_Combining_Mark" "Mc" spacingCombiningMark "a character intended to be combined with another character that takes up extra space (vowel signs in many Eastern languages)"+    ,     UnicodeCategory "Enclosing_Mark" "Me" enclosingMark "a character that encloses the character is is combined with (circle, square, keycap, etc.)"+    , UnicodeCategory "Separator" "Z" separator "any kind of whitespace or invisible separator"+    ,     UnicodeCategory "Space_Separator" "Zs" space "a whitespace character that is invisible, but does take up space"+    ,     UnicodeCategory "Line_Separator" "Zl" lineSeparator "line separator character U+2028"+    ,     UnicodeCategory "Paragraph_Separator" "Zp" paragraphSeparator "paragraph separator character U+2029"+    , UnicodeCategory "Symbol" "S" symbol "math symbols, currency signs, dingbats, box-drawing characters, etc."+    ,     UnicodeCategory "Math_Symbol" "Sm" mathSymbol "any mathematical symbol"+    ,     UnicodeCategory "Currency_Symbol" "Sc" currencySymbol "any currency sign"+    ,     UnicodeCategory "Modifier_Symbol" "Sk" modifierSymbol "a combining character (mark) as a full character on its own"+    ,     UnicodeCategory "Other_Symbol" "So" otherSymbol "various symbols that are not math symbols, currency signs, or combining characters"+    , UnicodeCategory "Number" "N" number "any kind of numeric character in any script"+    ,     UnicodeCategory "Decimal_Digit_Number" "Nd" decimalNumber "a digit zero through nine in any script except ideographic scripts"+    ,     UnicodeCategory "Letter_Number" "Nl" letterNumber "a number that looks like a letter, such as a Roman numeral"+    ,     UnicodeCategory "Other_Number" "No" otherNumber "a superscript or subscript digit, or a number that is not a digit 0..9 (excluding numbers from ideographic scripts)"+    , UnicodeCategory "Punctuation" "P" punctuation "any kind of punctuation character"+    ,     UnicodeCategory "Dash_Punctuation" "Pd" dashPunctuation "any kind of hyphen or dash"+    ,     UnicodeCategory "Open_Punctuation" "Ps" openPunctuation "any kind of opening bracket"+    ,     UnicodeCategory "Close_Punctuation" "Pe" closePunctuation "any kind of closing bracket"+    ,     UnicodeCategory "Initial_Punctuation" "Pi" initialQuote "any kind of opening quote"+    ,     UnicodeCategory "Final_Punctuation" "Pf" finalQuote "any kind of closing quote"+    ,     UnicodeCategory "Connector_Punctuation" "Pc" connectorPunctuation "a punctuation character such as an underscore that connects words"+    ,     UnicodeCategory "Other_Punctuation" "Po" otherPunctuation "any kind of punctuation character that is not a dash, bracket, quote or connector"+    , UnicodeCategory "Other" "C" other "invisible control characters and unused code points"+    ,     UnicodeCategory "Control" "Cc" control "an ASCII 0x00..0x1F or Latin-1 0x80..0x9F control character"+    ,     UnicodeCategory "Format" "Cf" format "invisible formatting indicator"+    ,     UnicodeCategory "Private_Use" "Co" privateUse "any code point reserved for private use"+    ,     UnicodeCategory "Surrogate" "Cs" surrogate "one half of a surrogate pair in UTF-16 encoding"+    ,     UnicodeCategory "Unassigned" "Cn" notAssigned "any code point to which no character has been assigned.properties" ]++cat :: GeneralCategory -> CharSet+cat category = build ((category ==) . generalCategory)++-- Letter+lowercaseLetter, uppercaseLetter, titlecaseLetter, letterAnd, modifierLetter, otherLetter, letter :: CharSet+lowercaseLetter = cat LowercaseLetter+uppercaseLetter = cat UppercaseLetter+titlecaseLetter = cat TitlecaseLetter+letterAnd = lowercaseLetter+    `union` uppercaseLetter+    `union` titlecaseLetter+modifierLetter  = cat ModifierLetter+otherLetter = cat OtherLetter+letter+          = letterAnd+    `union` modifierLetter+    `union` otherLetter++-- Marks+nonSpacingMark, spacingCombiningMark, enclosingMark, mark :: CharSet+nonSpacingMark = cat NonSpacingMark+spacingCombiningMark = cat SpacingCombiningMark+enclosingMark = cat EnclosingMark+mark+          = nonSpacingMark+    `union` spacingCombiningMark+    `union` enclosingMark++space, lineSeparator, paragraphSeparator, separator :: CharSet+space = cat Space+lineSeparator = cat LineSeparator+paragraphSeparator = cat ParagraphSeparator+separator+          = space+    `union` lineSeparator+    `union` paragraphSeparator++mathSymbol, currencySymbol, modifierSymbol, otherSymbol, symbol :: CharSet+mathSymbol = cat MathSymbol+currencySymbol = cat CurrencySymbol+modifierSymbol = cat ModifierSymbol+otherSymbol = cat OtherSymbol+symbol+          = mathSymbol+    `union` currencySymbol+    `union` modifierSymbol+    `union` otherSymbol++decimalNumber, letterNumber, otherNumber, number :: CharSet+decimalNumber = cat DecimalNumber+letterNumber = cat LetterNumber+otherNumber = cat OtherNumber+number+          = decimalNumber+    `union` letterNumber+    `union` otherNumber++dashPunctuation, openPunctuation, closePunctuation, initialQuote,+  finalQuote, connectorPunctuation, otherPunctuation, punctuation :: CharSet++dashPunctuation = cat DashPunctuation+openPunctuation = cat OpenPunctuation+closePunctuation = cat ClosePunctuation+initialQuote = cat InitialQuote+finalQuote = cat FinalQuote+connectorPunctuation  = cat ConnectorPunctuation+otherPunctuation = cat OtherPunctuation+punctuation+          = dashPunctuation+    `union` openPunctuation+    `union` closePunctuation+    `union` initialQuote+    `union` finalQuote+    `union` connectorPunctuation+    `union` otherPunctuation++control, format, privateUse, surrogate, notAssigned, other :: CharSet+control = cat Control+format = cat Format+privateUse = cat PrivateUse+surrogate = cat Surrogate+notAssigned = cat NotAssigned+other = control+    `union` format+    `union` privateUse+    `union` surrogate+    `union` notAssigned

@@ -3,19 +3,19 @@ ----------------------------------------------------------------------------- -- | -- Module      :  Data.CharSet.Unicode.Block--- Copyright   :  (c) Edward Kmett 2010+-- Copyright   :  (c) Edward Kmett 2010-2011 -- License     :  BSD3 -- Maintainer  :  ekmett@gmail.com -- Stability   :  experimental -- Portability :  portable ----- Provides unicode general categories, which are typically connoted by +-- Provides unicode general categories, which are typically connoted by -- @\p{InBasicLatin}@ or @\p{InIPA_Extensions}@. Lookups can be constructed using 'categories' -- or individual character sets can be used directly. -------------------------------------------------------------------------------  module Data.CharSet.Unicode.Block-    ( +    (     -- * Unicode General Category       Block(..)     -- * Lookup@@ -133,10 +133,10 @@ import Data.Char import Data.CharSet import Data.Data-import Data.Map (Map)-import qualified Data.Map as Map+import Data.HashMap.Lazy (HashMap)+import qualified Data.HashMap.Lazy as HashMap -data Block = Block +data Block = Block     { blockName :: String     , blockCharSet :: CharSet     } deriving (Show, Data, Typeable)@@ -253,8 +253,8 @@     , Block "Halfwidth_and_Fullwidth_Forms" halfwidthAndFullwidthForms     , Block "Specials" specials ] -lookupTable :: Map String Block-lookupTable = Map.fromList $ +lookupTable :: HashMap String Block+lookupTable = HashMap.fromList $               Prelude.map (\y@(Block x _) -> (canonicalize x, y))               blocks @@ -270,7 +270,7 @@         go [] = []  lookupBlock :: String -> Maybe Block-lookupBlock s = Map.lookup (canonicalize s) lookupTable+lookupBlock s = HashMap.lookup (canonicalize s) lookupTable  lookupBlockCharSet :: String -> Maybe CharSet lookupBlockCharSet = fmap blockCharSet . lookupBlock@@ -380,4 +380,3 @@ arabicPresentationFormsB = range '\xFE70' '\xFEFF' halfwidthAndFullwidthForms = range '\xFF00' '\xFFEF' specials = range '\xFFF0' '\xFFFF'-

@@ -2,11 +2,11 @@ ----------------------------------------------------------------------------- -- | -- Module      :  Data.CharSet.Unicode.Category--- Copyright   :  (c) Edward Kmett 2010+-- Copyright   :  (c) Edward Kmett 2010-2012 -- License     :  BSD3 -- Maintainer  :  ekmett@gmail.com -- Stability   :  experimental--- Portability :  portable+-- Portability :  DeriveDataTypeable -- -- Provides unicode general categories, which are typically connoted by  -- @\p{Ll}@ or @\p{Modifier_Letter}@. Lookups can be constructed using 'categories'@@ -47,10 +47,10 @@ import Data.Char import Data.CharSet import Data.Data-import Data.Map (Map)-import qualified Data.Map as Map+import Data.HashMap.Lazy (HashMap)+import qualified Data.HashMap.Lazy as HashMap -data Category = Category +data Category = Category     { categoryName :: String     , categoryAbbreviation :: String     , categoryCharSet :: CharSet@@ -99,28 +99,29 @@     ,     Category "Surrogate" "Cs" surrogate "one half of a surrogate pair in UTF-16 encoding"     ,     Category "Unassigned" "Cn" notAssigned "any code point to which no character has been assigned.properties" ] -lookupTable :: Map String Category-lookupTable = -    Map.fromList [ (canonicalize x, category) -                 | category@(Category l s _ _) <- categories-                 , x <- [l,s] ]+lookupTable :: HashMap String Category+lookupTable = HashMap.fromList +  [ (canonicalize x, category) +  | category@(Category l s _ _) <- categories+  , x <- [l,s] +  ]  lookupCategory :: String -> Maybe Category-lookupCategory s = Map.lookup (canonicalize s) lookupTable+lookupCategory s = HashMap.lookup (canonicalize s) lookupTable  lookupCategoryCharSet :: String -> Maybe CharSet lookupCategoryCharSet = fmap categoryCharSet . lookupCategory  canonicalize :: String -> String canonicalize s = case Prelude.map toLower s of-    'i' : 's' : xs -> go xs-    xs -> go xs-    where-        go ('-':xs) = go xs-        go ('_':xs) = go xs-        go (' ':xs) = go xs-        go (x:xs) = x : go xs-        go [] = []+  'i' : 's' : xs -> go xs+  xs -> go xs+  where+    go ('-':xs) = go xs+    go ('_':xs) = go xs+    go (' ':xs) = go xs+    go (x:xs) = x : go xs+    go [] = []  cat :: GeneralCategory -> CharSet cat category = build ((category ==) . generalCategory)

@@ -3,6 +3,6 @@ \begin{code}  import Distribution.Simple-main = defaultMainWithHooks defaultUserHooks+main = defaultMainWithHooks simpleUserHooks  \end{code}

@@ -1,28 +1,52 @@-name:         charset-version:      0.2.3-license:      BSD3-license-File: LICENSE-copyright:    (c) Edward Kmett 2010-author:       Edward Kmett-maintainer:   ekmett@gmail.com-stability:    Experimental-category:     Data-homepage:     http://github.com/ekmett/charset-synopsis:     Fast unicode character sets based on complemented PATRICIA tries-description:  Fast unicode character sets based on complemented PATRICIA tries+name:          charset+version:       0.3+license:       BSD3+license-File:  LICENSE+copyright:     (c) Edward Kmett 2010-2012+author:        Edward Kmett+maintainer:    ekmett@gmail.com+cabal-version: >= 1.6+stability:     Experimental+category:      Data+homepage:      http://github.com/ekmett/charset+synopsis:      Fast unicode character sets based on complemented PATRICIA tries+description:   Fast unicode character sets based on complemented PATRICIA tries+build-type:    Simple -build-type:   Simple-build-depends:-    base >= 4 && < 5,-    containers >= 0.2 && < 0.5,-    array >= 0.2 && < 0.5+source-repository head+  type: git+  location: git://github.com/ekmett/charset.git -exposed-modules:+flag OldTypeable+  manual: False+  default: False++library+  extensions: CPP+  other-extensions: MagicHash, BangPatterns++  if flag(OldTypeable)+    build-depends: base >= 4.0 && < 4.4+    cpp-options: -DOLD_TYPEABLE+  else +    build-depends: base >= 4.4 && < 5++  build-depends:+    array                >= 0.2     && < 0.5,+    bytestring           >= 0.9.2.1 && < 0.10,+    containers           >= 0.2     && < 0.5,+    semigroups           >= 0.8     && < 0.9,+    unordered-containers >= 0.1.4.6 && < 0.2++  exposed-modules:     Data.CharSet     Data.CharSet.Common+    Data.CharSet.Posix     Data.CharSet.Posix.Ascii     Data.CharSet.Posix.Unicode+    Data.CharSet.Unicode     Data.CharSet.Unicode.Block     Data.CharSet.Unicode.Category+    Data.CharSet.ByteSet -GHC-Options: -Wall -fspec-constr -fdicts-cheap -O2+  ghc-options: -Wall -fspec-constr -fdicts-cheap -O2