charset 0.1 → 0.2.0

raw patch · 4 files changed

+595/−179 lines, 4 filesPVP ok

version bump matches the API change (PVP)

API changes (from Hackage documentation)

Files

• Data/CharSet/Unicode.hs +0/−177
• Data/CharSet/Unicode/Block.hs +382/−0
• Data/CharSet/Unicode/Category.hs +210/−0
• charset.cabal +3/−2

@@ -1,177 +0,0 @@-{-# LANGUAGE DeriveDataTypeable #-}--------------------------------------------------------------------------------- |--- Module      :  Data.CharSet.Unicode--- Copyright   :  (c) Edward Kmett 2010--- License     :  BSD3--- Maintainer  :  ekmett@gmail.com--- Stability   :  experimental--- Portability :  portable------ Provides unicode general categories, which are typically connoted by --- @\p{Ll}@ or @\p{Modifier_Letter}@. Lookups can be constructed using 'categories'--- or individual character sets can be used directly.----------------------------------------------------------------------------------module Data.CharSet.Unicode-    ( -    -- * Unicode General Category-      UnicodeCategory(..)-    -- * Lookup-    , unicodeCategories-    -- * CharSets by UnicodeCategory-    -- ** Letter-    , modifierLetter, otherLetter, letter-    -- *** Letter\&-    , lowercaseLetter, uppercaseLetter, titlecaseLetter, letterAnd-    -- ** Mark-    , nonSpacingMark, spacingCombiningMark, enclosingMark, mark-    -- ** Separator-    , space, lineSeparator, paragraphSeparator, separator-    -- ** Symbol-    , mathSymbol, currencySymbol, modifierSymbol, otherSymbol, symbol-    -- ** Number-    , decimalNumber, letterNumber, otherNumber, number-    -- ** Punctuation-    , dashPunctuation, openPunctuation, closePunctuation, initialQuote-    , finalQuote, connectorPunctuation, otherPunctuation, punctuation -    -- ** Other-    , control, format, privateUse, surrogate, notAssigned, other-    ) where--import Data.Char-import Data.CharSet-import Data.Data--data UnicodeCategory = UnicodeCategory String String CharSet String-    deriving (Show, Data, Typeable)---- \p{Letter} or \p{Mc}-unicodeCategories :: [UnicodeCategory]-unicodeCategories =-    [ UnicodeCategory "Letter" "L" letter "any kind of letter from any language."-    ,     UnicodeCategory "Lowercase_Letter" "Ll" lowercaseLetter "a lowercase letter that has an uppercase variant"-    ,     UnicodeCategory "Uppercase_Letter" "Lu" uppercaseLetter "an uppercase letter that has a lowercase variant"-    ,     UnicodeCategory "Titlecase_Letter" "Lt" titlecaseLetter "a letter that appears at the start of a word when only the first letter of the word is capitalized"-    ,     UnicodeCategory "Letter&" "L&" letterAnd "a letter that exists in lowercase and uppercase variants (combination of Ll, Lu and Lt)"-    ,     UnicodeCategory "Modifier_Letter" "Lm" modifierLetter "a special character that is used like a letter"-    ,     UnicodeCategory "Other_Letter" "Lo" otherLetter "a letter or ideograph that does not have lowercase and uppercase variants"-    , UnicodeCategory "Mark" "M" mark "a character intended to be combined with another character (e.g. accents, umlauts, enclosing boxes, etc.)"-    ,     UnicodeCategory "Non_Spacing_Mark" "Mn" nonSpacingMark "a character intended to be combined with another character without taking up extra space (e.g. accents, umlauts, etc.)"-    ,     UnicodeCategory "Spacing_Combining_Mark" "Mc" spacingCombiningMark "a character intended to be combined with another character that takes up extra space (vowel signs in many Eastern languages)"-    ,     UnicodeCategory "Enclosing_Mark" "Me" enclosingMark "a character that encloses the character is is combined with (circle, square, keycap, etc.)"-    , UnicodeCategory "Separator" "Z" separator "any kind of whitespace or invisible separator"-    ,     UnicodeCategory "Space_Separator" "Zs" space "a whitespace character that is invisible, but does take up space"-    ,     UnicodeCategory "Line_Separator" "Zl" lineSeparator "line separator character U+2028"-    ,     UnicodeCategory "Paragraph_Separator" "Zp" paragraphSeparator "paragraph separator character U+2029"-    , UnicodeCategory "Symbol" "S" symbol "math symbols, currency signs, dingbats, box-drawing characters, etc."-    ,     UnicodeCategory "Math_Symbol" "Sm" mathSymbol "any mathematical symbol"-    ,     UnicodeCategory "Currency_Symbol" "Sc" currencySymbol "any currency sign"-    ,     UnicodeCategory "Modifier_Symbol" "Sk" modifierSymbol "a combining character (mark) as a full character on its own"-    ,     UnicodeCategory "Other_Symbol" "So" otherSymbol "various symbols that are not math symbols, currency signs, or combining characters"-    , UnicodeCategory "Number" "N" number "any kind of numeric character in any script"-    ,     UnicodeCategory "Decimal_Digit_Number" "Nd" decimalNumber "a digit zero through nine in any script except ideographic scripts"-    ,     UnicodeCategory "Letter_Number" "Nl" letterNumber "a number that looks like a letter, such as a Roman numeral"-    ,     UnicodeCategory "Other_Number" "No" otherNumber "a superscript or subscript digit, or a number that is not a digit 0..9 (excluding numbers from ideographic scripts)"-    , UnicodeCategory "Punctuation" "P" punctuation "any kind of punctuation character"-    ,     UnicodeCategory "Dash_Punctuation" "Pd" dashPunctuation "any kind of hyphen or dash"-    ,     UnicodeCategory "Open_Punctuation" "Ps" openPunctuation "any kind of opening bracket"-    ,     UnicodeCategory "Close_Punctuation" "Pe" closePunctuation "any kind of closing bracket"-    ,     UnicodeCategory "Initial_Punctuation" "Pi" initialQuote "any kind of opening quote"-    ,     UnicodeCategory "Final_Punctuation" "Pf" finalQuote "any kind of closing quote"-    ,     UnicodeCategory "Connector_Punctuation" "Pc" connectorPunctuation "a punctuation character such as an underscore that connects words"-    ,     UnicodeCategory "Other_Punctuation" "Po" otherPunctuation "any kind of punctuation character that is not a dash, bracket, quote or connector"-    , UnicodeCategory "Other" "C" other "invisible control characters and unused code points"-    ,     UnicodeCategory "Control" "Cc" control "an ASCII 0x00..0x1F or Latin-1 0x80..0x9F control character"-    ,     UnicodeCategory "Format" "Cf" format "invisible formatting indicator"-    ,     UnicodeCategory "Private_Use" "Co" privateUse "any code point reserved for private use"-    ,     UnicodeCategory "Surrogate" "Cs" surrogate "one half of a surrogate pair in UTF-16 encoding"-    ,     UnicodeCategory "Unassigned" "Cn" notAssigned "any code point to which no character has been assigned.properties" ]--cat :: GeneralCategory -> CharSet-cat category = build ((category ==) . generalCategory)---- Letter-lowercaseLetter, uppercaseLetter, titlecaseLetter, letterAnd, modifierLetter, otherLetter, letter :: CharSet-lowercaseLetter = cat LowercaseLetter-uppercaseLetter = cat UppercaseLetter-titlecaseLetter = cat TitlecaseLetter-letterAnd = lowercaseLetter -    `union` uppercaseLetter -    `union` titlecaseLetter-modifierLetter  = cat ModifierLetter-otherLetter = cat OtherLetter-letter -          = letterAnd -    `union` modifierLetter -    `union` otherLetter---- Marks-nonSpacingMark, spacingCombiningMark, enclosingMark, mark :: CharSet-nonSpacingMark = cat NonSpacingMark-spacingCombiningMark = cat SpacingCombiningMark-enclosingMark = cat EnclosingMark-mark -          = nonSpacingMark -    `union` spacingCombiningMark -    `union` enclosingMark--space, lineSeparator, paragraphSeparator, separator :: CharSet-space = cat Space-lineSeparator = cat LineSeparator-paragraphSeparator = cat ParagraphSeparator-separator -          = space -    `union` lineSeparator -    `union` paragraphSeparator--mathSymbol, currencySymbol, modifierSymbol, otherSymbol, symbol :: CharSet-mathSymbol = cat MathSymbol-currencySymbol = cat CurrencySymbol-modifierSymbol = cat ModifierSymbol-otherSymbol = cat OtherSymbol-symbol -          = mathSymbol -    `union` currencySymbol -    `union` modifierSymbol -    `union` otherSymbol--decimalNumber, letterNumber, otherNumber, number :: CharSet-decimalNumber = cat DecimalNumber-letterNumber = cat LetterNumber-otherNumber = cat OtherNumber-number -          = decimalNumber -    `union` letterNumber -    `union` otherNumber--dashPunctuation, openPunctuation, closePunctuation, initialQuote, -  finalQuote, connectorPunctuation, otherPunctuation, punctuation :: CharSet--dashPunctuation = cat DashPunctuation-openPunctuation = cat OpenPunctuation-closePunctuation = cat ClosePunctuation-initialQuote = cat InitialQuote-finalQuote = cat FinalQuote-connectorPunctuation  = cat ConnectorPunctuation-otherPunctuation = cat OtherPunctuation-punctuation -          = dashPunctuation -    `union` openPunctuation -    `union` closePunctuation -    `union` initialQuote -    `union` finalQuote -    `union` connectorPunctuation -    `union` otherPunctuation--control, format, privateUse, surrogate, notAssigned, other :: CharSet-control = cat Control-format = cat Format-privateUse = cat PrivateUse-surrogate = cat Surrogate-notAssigned = cat NotAssigned-other = control -    `union` format -    `union` privateUse -    `union` surrogate -    `union` notAssigned

@@ -0,0 +1,382 @@+{-# LANGUAGE DeriveDataTypeable #-}+{-# OPTIONS_GHC -fno-warn-missing-signatures #-}+-----------------------------------------------------------------------------+-- |+-- Module      :  Data.CharSet.Unicode.Block+-- Copyright   :  (c) Edward Kmett 2010+-- License     :  BSD3+-- Maintainer  :  ekmett@gmail.com+-- Stability   :  experimental+-- Portability :  portable+--+-- Provides unicode general categories, which are typically connoted by +-- @\p{InBasicLatin}@ or @\p{InIPA_Extensions}@. Lookups can be constructed using 'categories'+-- or individual character sets can be used directly.+-------------------------------------------------------------------------------++module Data.CharSet.Unicode.Block+    ( +    -- * Unicode General Category+      Block(..)+    -- * Lookup+    , blocks+    , lookupBlock+    , lookupBlockCharSet+    -- * CharSets by Block+    , basicLatin+    , latin1Supplement+    , latinExtendedA+    , latinExtendedB+    , ipaExtensions+    , spacingModifierLetters+    , combiningDiacriticalMarks+    , greekAndCoptic+    , cyrillic+    , cyrillicSupplementary+    , armenian+    , hebrew+    , arabic+    , syriac+    , thaana+    , devanagari+    , bengali+    , gurmukhi+    , gujarati+    , oriya+    , tamil+    , telugu+    , kannada+    , malayalam+    , sinhala+    , thai+    , lao+    , tibetan+    , myanmar+    , georgian+    , hangulJamo+    , ethiopic+    , cherokee+    , unifiedCanadianAboriginalSyllabics+    , ogham+    , runic+    , tagalog+    , hanunoo+    , buhid+    , tagbanwa+    , khmer+    , mongolian+    , limbu+    , taiLe+    , khmerSymbols+    , phoneticExtensions+    , latinExtendedAdditional+    , greekExtended+    , generalPunctuation+    , superscriptsAndSubscripts+    , currencySymbols+    , combiningDiacriticalMarksForSymbols+    , letterlikeSymbols+    , numberForms+    , arrows+    , mathematicalOperators+    , miscellaneousTechnical+    , controlPictures+    , opticalCharacterRecognition+    , enclosedAlphanumerics+    , boxDrawing+    , blockElements+    , geometricShapes+    , miscellaneousSymbols+    , dingbats+    , miscellaneousMathematicalSymbolsA+    , supplementalArrowsA+    , braillePatterns+    , supplementalArrowsB+    , miscellaneousMathematicalSymbolsB+    , supplementalMathematicalOperators+    , miscellaneousSymbolsAndArrows+    , cjkRadicalsSupplement+    , kangxiRadicals+    , ideographicDescriptionCharacters+    , cjkSymbolsAndPunctuation+    , hiragana+    , katakana+    , bopomofo+    , hangulCompatibilityJamo+    , kanbun+    , bopomofoExtended+    , katakanaPhoneticExtensions+    , enclosedCjkLettersAndMonths+    , cjkCompatibility+    , cjkUnifiedIdeographsExtensionA+    , yijingHexagramSymbols+    , cjkUnifiedIdeographs+    , yiSyllables+    , yiRadicals+    , hangulSyllables+    , highSurrogates+    , highPrivateUseSurrogates+    , lowSurrogates+    , privateUseArea+    , cjkCompatibilityIdeographs+    , alphabeticPresentationForms+    , arabicPresentationFormsA+    , variationSelectors+    , combiningHalfMarks+    , cjkCompatibilityForms+    , smallFormVariants+    , arabicPresentationFormsB+    , halfwidthAndFullwidthForms+    , specials+    ) where++import Data.Char+import Data.CharSet+import Data.Data+import Data.Map (Map)+import qualified Data.Map as Map++data Block = Block +    { blockName :: String+    , blockCharSet :: CharSet+    } deriving (Show, Data, Typeable)++blocks :: [Block]+blocks =+    [ Block "Basic_Latin" basicLatin+    , Block "Latin-1_Supplement" latin1Supplement+    , Block "Latin_Extended-A" latinExtendedA+    , Block "IPA_Extensions" ipaExtensions+    , Block "Spacing_Modifier_Letters" spacingModifierLetters++    , Block "Latin_Extended-A" latinExtendedA+    , Block "Latin_Extended-B" latinExtendedB+    , Block "IPA_Extensions" ipaExtensions+    , Block "Spacing_Modifier_Letters" spacingModifierLetters+    , Block "Combining_Diacritical_Marks" combiningDiacriticalMarks+    , Block "Greek_and_Coptic" greekAndCoptic+    , Block "Cyrillic" cyrillic+    , Block "Cyrillic_Supplementary" cyrillicSupplementary+    , Block "Armenian" armenian+    , Block "Hebrew" hebrew+    , Block "Arabic" arabic+    , Block "Syriac" syriac+    , Block "Thaana" thaana+    , Block "Devanagari" devanagari+    , Block "Bengali" bengali+    , Block "Gurmukhi" gurmukhi+    , Block "Gujarati" gujarati+    , Block "Oriya" oriya+    , Block "Tamil" tamil+    , Block "Telugu" telugu+    , Block "Kannada" kannada+    , Block "Malayalam" malayalam+    , Block "Sinhala" sinhala+    , Block "Thai" thai+    , Block "Lao" lao+    , Block "Tibetan" tibetan+    , Block "Myanmar" myanmar+    , Block "Georgian" georgian+    , Block "Hangul_Jamo" hangulJamo+    , Block "Ethiopic" ethiopic+    , Block "Cherokee" cherokee+    , Block "Unified_Canadian_Aboriginal_Syllabics" unifiedCanadianAboriginalSyllabics+    , Block "Ogham" ogham+    , Block "Runic" runic+    , Block "Tagalog" tagalog+    , Block "Hanunoo" hanunoo+    , Block "Buhid" buhid+    , Block "Tagbanwa" tagbanwa+    , Block "Khmer" khmer+    , Block "Mongolian" mongolian+    , Block "Limbu" limbu+    , Block "Tai_Le" taiLe+    , Block "Khmer_Symbols" khmerSymbols+    , Block "Phonetic_Extensions" phoneticExtensions+    , Block "Latin_Extended_Additional" latinExtendedAdditional+    , Block "Greek_Extended" greekExtended+    , Block "General_Punctuation" generalPunctuation+    , Block "Superscripts_and_Subscripts" superscriptsAndSubscripts+    , Block "Currency_Symbols" currencySymbols+    , Block "Combining_Diacritical_Marks_for_Symbols" combiningDiacriticalMarksForSymbols+    , Block "Letterlike_Symbols" letterlikeSymbols+    , Block "Number_Forms" numberForms+    , Block "Arrows" arrows+    , Block "Mathematical_Operators" mathematicalOperators+    , Block "Miscellaneous_Technical" miscellaneousTechnical+    , Block "Control_Pictures" controlPictures+    , Block "Optical_Character_Recognition" opticalCharacterRecognition+    , Block "Enclosed_Alphanumerics" enclosedAlphanumerics+    , Block "Box_Drawing" boxDrawing+    , Block "Block_Elements" blockElements+    , Block "Geometric_Shapes" geometricShapes+    , Block "Miscellaneous_Symbols" miscellaneousSymbols+    , Block "Dingbats" dingbats+    , Block "Miscellaneous_Mathematical_Symbols-A" miscellaneousMathematicalSymbolsA+    , Block "Supplemental_Arrows-A" supplementalArrowsA+    , Block "Braille_Patterns" braillePatterns+    , Block "Supplemental_Arrows-B" supplementalArrowsB+    , Block "Miscellaneous_Mathematical_Symbols-B" miscellaneousMathematicalSymbolsB+    , Block "Supplemental_Mathematical_Operators" supplementalMathematicalOperators+    , Block "Miscellaneous_Symbols_and_Arrows" miscellaneousSymbolsAndArrows+    , Block "CJK_Radicals_Supplement" cjkRadicalsSupplement+    , Block "Kangxi_Radicals" kangxiRadicals+    , Block "Ideographic_Description_Characters" ideographicDescriptionCharacters+    , Block "CJK_Symbols_and_Punctuation" cjkSymbolsAndPunctuation+    , Block "Hiragana" hiragana+    , Block "Katakana" katakana+    , Block "Bopomofo" bopomofo+    , Block "Hangul_Compatibility_Jamo" hangulCompatibilityJamo+    , Block "Kanbun" kanbun+    , Block "Bopomofo_Extended" bopomofoExtended+    , Block "Katakana_Phonetic_Extensions" katakanaPhoneticExtensions+    , Block "Enclosed_CJK_Letters_and_Months" enclosedCjkLettersAndMonths+    , Block "CJK_Compatibility" cjkCompatibility+    , Block "CJK_Unified_Ideographs_Extension_A" cjkUnifiedIdeographsExtensionA+    , Block "Yijing_Hexagram_Symbols" yijingHexagramSymbols+    , Block "CJK_Unified_Ideographs" cjkUnifiedIdeographs+    , Block "Yi_Syllables" yiSyllables+    , Block "Yi_Radicals" yiRadicals+    , Block "Hangul_Syllables" hangulSyllables+    , Block "High_Surrogates" highSurrogates+    , Block "High_Private_Use_Surrogates" highPrivateUseSurrogates+    , Block "Low_Surrogates" lowSurrogates+    , Block "Private_Use_Area" privateUseArea+    , Block "CJK_Compatibility_Ideographs" cjkCompatibilityIdeographs+    , Block "Alphabetic_Presentation_Forms" alphabeticPresentationForms+    , Block "Arabic_Presentation_Forms-A" arabicPresentationFormsA+    , Block "Variation_Selectors" variationSelectors+    , Block "Combining_Half_Marks" combiningHalfMarks+    , Block "CJK_Compatibility_Forms" cjkCompatibilityForms+    , Block "Small_Form_Variants" smallFormVariants+    , Block "Arabic_Presentation_Forms-B" arabicPresentationFormsB+    , Block "Halfwidth_and_Fullwidth_Forms" halfwidthAndFullwidthForms+    , Block "Specials" specials ]++lookupTable :: Map String Block+lookupTable = Map.fromList $ +              Prelude.map (\y@(Block x _) -> (canonicalize x, y))+              blocks++canonicalize :: String -> String+canonicalize s = case Prelude.map toLower s of+    'i': 'n' : xs -> go xs+    xs -> go xs+    where+        go ('-':xs) = go xs+        go ('_':xs) = go xs+        go (x:xs) = x : go xs+        go [] = []++lookupBlock :: String -> Maybe Block+lookupBlock s = Map.lookup (canonicalize s) lookupTable++lookupBlockCharSet :: String -> Maybe CharSet+lookupBlockCharSet = fmap blockCharSet . lookupBlock++basicLatin = range '\x0000' '\x007f'+latin1Supplement = range '\x0080' '\x00ff'+latinExtendedA = range '\x0100' '\x017F'+latinExtendedB = range '\x0180' '\x024F'+ipaExtensions = range '\x0250' '\x02AF'+spacingModifierLetters = range '\x02B0' '\x02FF'+combiningDiacriticalMarks = range '\x0300' '\x036F'+greekAndCoptic = range '\x0370' '\x03FF'+cyrillic = range '\x0400' '\x04FF'+cyrillicSupplementary = range '\x0500' '\x052F'+armenian = range '\x0530' '\x058F'+hebrew = range '\x0590' '\x05FF'+arabic = range '\x0600' '\x06FF'+syriac = range '\x0700' '\x074F'+thaana = range '\x0780' '\x07BF'+devanagari = range '\x0900' '\x097F'+bengali = range '\x0980' '\x09FF'+gurmukhi = range '\x0A00' '\x0A7F'+gujarati = range '\x0A80' '\x0AFF'+oriya = range '\x0B00' '\x0B7F'+tamil = range '\x0B80' '\x0BFF'+telugu = range '\x0C00' '\x0C7F'+kannada = range '\x0C80' '\x0CFF'+malayalam = range '\x0D00' '\x0D7F'+sinhala = range '\x0D80' '\x0DFF'+thai = range '\x0E00' '\x0E7F'+lao = range '\x0E80' '\x0EFF'+tibetan = range '\x0F00' '\x0FFF'+myanmar = range '\x1000' '\x109F'+georgian = range '\x10A0' '\x10FF'+hangulJamo = range '\x1100' '\x11FF'+ethiopic = range '\x1200' '\x137F'+cherokee = range '\x13A0' '\x13FF'+unifiedCanadianAboriginalSyllabics = range '\x1400' '\x167F'+ogham = range '\x1680' '\x169F'+runic = range '\x16A0' '\x16FF'+tagalog = range '\x1700' '\x171F'+hanunoo = range '\x1720' '\x173F'+buhid = range '\x1740' '\x175F'+tagbanwa = range '\x1760' '\x177F'+khmer = range '\x1780' '\x17FF'+mongolian = range '\x1800' '\x18AF'+limbu = range '\x1900' '\x194F'+taiLe = range '\x1950' '\x197F'+khmerSymbols = range '\x19E0' '\x19FF'+phoneticExtensions = range '\x1D00' '\x1D7F'+latinExtendedAdditional = range '\x1E00' '\x1EFF'+greekExtended = range '\x1F00' '\x1FFF'+generalPunctuation = range '\x2000' '\x206F'+superscriptsAndSubscripts = range '\x2070' '\x209F'+currencySymbols = range '\x20A0' '\x20CF'+combiningDiacriticalMarksForSymbols = range '\x20D0' '\x20FF'+letterlikeSymbols = range '\x2100' '\x214F'+numberForms = range '\x2150' '\x218F'+arrows = range '\x2190' '\x21FF'+mathematicalOperators = range '\x2200' '\x22FF'+miscellaneousTechnical = range '\x2300' '\x23FF'+controlPictures = range '\x2400' '\x243F'+opticalCharacterRecognition = range '\x2440' '\x245F'+enclosedAlphanumerics = range '\x2460' '\x24FF'+boxDrawing = range '\x2500' '\x257F'+blockElements = range '\x2580' '\x259F'+geometricShapes = range '\x25A0' '\x25FF'+miscellaneousSymbols = range '\x2600' '\x26FF'+dingbats = range '\x2700' '\x27BF'+miscellaneousMathematicalSymbolsA = range '\x27C0' '\x27EF'+supplementalArrowsA = range '\x27F0' '\x27FF'+braillePatterns = range '\x2800' '\x28FF'+supplementalArrowsB = range '\x2900' '\x297F'+miscellaneousMathematicalSymbolsB = range '\x2980' '\x29FF'+supplementalMathematicalOperators = range '\x2A00' '\x2AFF'+miscellaneousSymbolsAndArrows = range '\x2B00' '\x2BFF'+cjkRadicalsSupplement = range '\x2E80' '\x2EFF'+kangxiRadicals = range '\x2F00' '\x2FDF'+ideographicDescriptionCharacters = range '\x2FF0' '\x2FFF'+cjkSymbolsAndPunctuation = range '\x3000' '\x303F'+hiragana = range '\x3040' '\x309F'+katakana = range '\x30A0' '\x30FF'+bopomofo = range '\x3100' '\x312F'+hangulCompatibilityJamo = range '\x3130' '\x318F'+kanbun = range '\x3190' '\x319F'+bopomofoExtended = range '\x31A0' '\x31BF'+katakanaPhoneticExtensions = range '\x31F0' '\x31FF'+enclosedCjkLettersAndMonths = range '\x3200' '\x32FF'+cjkCompatibility = range '\x3300' '\x33FF'+cjkUnifiedIdeographsExtensionA = range '\x3400' '\x4DBF'+yijingHexagramSymbols = range '\x4DC0' '\x4DFF'+cjkUnifiedIdeographs = range '\x4E00' '\x9FFF'+yiSyllables = range '\xA000' '\xA48F'+yiRadicals = range '\xA490' '\xA4CF'+hangulSyllables = range '\xAC00' '\xD7AF'+highSurrogates = range '\xD800' '\xDB7F'+highPrivateUseSurrogates = range '\xDB80' '\xDBFF'+lowSurrogates = range '\xDC00' '\xDFFF'+privateUseArea = range '\xE000' '\xF8FF'+cjkCompatibilityIdeographs = range '\xF900' '\xFAFF'+alphabeticPresentationForms = range '\xFB00' '\xFB4F'+arabicPresentationFormsA = range '\xFB50' '\xFDFF'+variationSelectors = range '\xFE00' '\xFE0F'+combiningHalfMarks = range '\xFE20' '\xFE2F'+cjkCompatibilityForms = range '\xFE30' '\xFE4F'+smallFormVariants = range '\xFE50' '\xFE6F'+arabicPresentationFormsB = range '\xFE70' '\xFEFF'+halfwidthAndFullwidthForms = range '\xFF00' '\xFFEF'+specials = range '\xFFF0' '\xFFFF'+

@@ -0,0 +1,210 @@+{-# LANGUAGE DeriveDataTypeable #-}+-----------------------------------------------------------------------------+-- |+-- Module      :  Data.CharSet.Unicode.Category+-- Copyright   :  (c) Edward Kmett 2010+-- License     :  BSD3+-- Maintainer  :  ekmett@gmail.com+-- Stability   :  experimental+-- Portability :  portable+--+-- Provides unicode general categories, which are typically connoted by +-- @\p{Ll}@ or @\p{Modifier_Letter}@. Lookups can be constructed using 'categories'+-- or individual character sets can be used directly.+-- +-- A case, @_@ and @-@ insensitive lookup is provided by 'lookupCategory'+-- and can be used to provide behavior similar to that of Perl or PCRE.+-------------------------------------------------------------------------------++module Data.CharSet.Unicode.Category+    ( +    -- * Unicode General Category+      Category(..)+    -- * Lookup+    , categories+    , lookupCategory+    , lookupCategoryCharSet+    -- * CharSets by Category+    -- ** Letter+    , modifierLetter, otherLetter, letter+    -- *** Letter\&+    , lowercaseLetter, uppercaseLetter, titlecaseLetter, letterAnd+    -- ** Mark+    , nonSpacingMark, spacingCombiningMark, enclosingMark, mark+    -- ** Separator+    , space, lineSeparator, paragraphSeparator, separator+    -- ** Symbol+    , mathSymbol, currencySymbol, modifierSymbol, otherSymbol, symbol+    -- ** Number+    , decimalNumber, letterNumber, otherNumber, number+    -- ** Punctuation+    , dashPunctuation, openPunctuation, closePunctuation, initialQuote+    , finalQuote, connectorPunctuation, otherPunctuation, punctuation +    -- ** Other+    , control, format, privateUse, surrogate, notAssigned, other+    ) where++import Data.Char+import Data.CharSet+import Data.Data+import Data.Map (Map)+import qualified Data.Map as Map++data Category = Category +    { categoryName :: String+    , categoryAbbreviation :: String+    , categoryCharSet :: CharSet+    , categoryDescription :: String+    } deriving (Show, Data, Typeable)++-- \p{Letter} or \p{Mc}+categories :: [Category]+categories =+    [ Category "Letter" "L" letter "any kind of letter from any language."+    ,     Category "Lowercase_Letter" "Ll" lowercaseLetter "a lowercase letter that has an uppercase variant"+    ,     Category "Uppercase_Letter" "Lu" uppercaseLetter "an uppercase letter that has a lowercase variant"+    ,     Category "Titlecase_Letter" "Lt" titlecaseLetter "a letter that appears at the start of a word when only the first letter of the word is capitalized"+    ,     Category "Letter&" "L&" letterAnd "a letter that exists in lowercase and uppercase variants (combination of Ll, Lu and Lt)"+    ,     Category "Modifier_Letter" "Lm" modifierLetter "a special character that is used like a letter"+    ,     Category "Other_Letter" "Lo" otherLetter "a letter or ideograph that does not have lowercase and uppercase variants"+    , Category "Mark" "M" mark "a character intended to be combined with another character (e.g. accents, umlauts, enclosing boxes, etc.)"+    ,     Category "Non_Spacing_Mark" "Mn" nonSpacingMark "a character intended to be combined with another character without taking up extra space (e.g. accents, umlauts, etc.)"+    ,     Category "Spacing_Combining_Mark" "Mc" spacingCombiningMark "a character intended to be combined with another character that takes up extra space (vowel signs in many Eastern languages)"+    ,     Category "Enclosing_Mark" "Me" enclosingMark "a character that encloses the character is is combined with (circle, square, keycap, etc.)"+    , Category "Separator" "Z" separator "any kind of whitespace or invisible separator"+    ,     Category "Space_Separator" "Zs" space "a whitespace character that is invisible, but does take up space"+    ,     Category "Line_Separator" "Zl" lineSeparator "line separator character U+2028"+    ,     Category "Paragraph_Separator" "Zp" paragraphSeparator "paragraph separator character U+2029"+    , Category "Symbol" "S" symbol "math symbols, currency signs, dingbats, box-drawing characters, etc."+    ,     Category "Math_Symbol" "Sm" mathSymbol "any mathematical symbol"+    ,     Category "Currency_Symbol" "Sc" currencySymbol "any currency sign"+    ,     Category "Modifier_Symbol" "Sk" modifierSymbol "a combining character (mark) as a full character on its own"+    ,     Category "Other_Symbol" "So" otherSymbol "various symbols that are not math symbols, currency signs, or combining characters"+    , Category "Number" "N" number "any kind of numeric character in any script"+    ,     Category "Decimal_Digit_Number" "Nd" decimalNumber "a digit zero through nine in any script except ideographic scripts"+    ,     Category "Letter_Number" "Nl" letterNumber "a number that looks like a letter, such as a Roman numeral"+    ,     Category "Other_Number" "No" otherNumber "a superscript or subscript digit, or a number that is not a digit 0..9 (excluding numbers from ideographic scripts)"+    , Category "Punctuation" "P" punctuation "any kind of punctuation character"+    ,     Category "Dash_Punctuation" "Pd" dashPunctuation "any kind of hyphen or dash"+    ,     Category "Open_Punctuation" "Ps" openPunctuation "any kind of opening bracket"+    ,     Category "Close_Punctuation" "Pe" closePunctuation "any kind of closing bracket"+    ,     Category "Initial_Punctuation" "Pi" initialQuote "any kind of opening quote"+    ,     Category "Final_Punctuation" "Pf" finalQuote "any kind of closing quote"+    ,     Category "Connector_Punctuation" "Pc" connectorPunctuation "a punctuation character such as an underscore that connects words"+    ,     Category "Other_Punctuation" "Po" otherPunctuation "any kind of punctuation character that is not a dash, bracket, quote or connector"+    , Category "Other" "C" other "invisible control characters and unused code points"+    ,     Category "Control" "Cc" control "an ASCII 0x00..0x1F or Latin-1 0x80..0x9F control character"+    ,     Category "Format" "Cf" format "invisible formatting indicator"+    ,     Category "Private_Use" "Co" privateUse "any code point reserved for private use"+    ,     Category "Surrogate" "Cs" surrogate "one half of a surrogate pair in UTF-16 encoding"+    ,     Category "Unassigned" "Cn" notAssigned "any code point to which no character has been assigned.properties" ]++lookupTable :: Map String Category+lookupTable = +    Map.fromList [ (canonicalize x, category) +                 | category@(Category l s _ _) <- categories+                 , x <- [l,s] ]++lookupCategory :: String -> Maybe Category+lookupCategory s = Map.lookup (canonicalize s) lookupTable++lookupCategoryCharSet :: String -> Maybe CharSet+lookupCategoryCharSet = fmap categoryCharSet . lookupCategory++canonicalize :: String -> String+canonicalize s = case Prelude.map toLower s of+    'i' : 's' : xs -> go xs+    xs -> go xs+    where+        go ('-':xs) = go xs+        go ('_':xs) = go xs+        go (x:xs) = x : go xs+        go [] = []++cat :: GeneralCategory -> CharSet+cat category = build ((category ==) . generalCategory)++-- Letter+lowercaseLetter, uppercaseLetter, titlecaseLetter, letterAnd, modifierLetter, otherLetter, letter :: CharSet+lowercaseLetter = cat LowercaseLetter+uppercaseLetter = cat UppercaseLetter+titlecaseLetter = cat TitlecaseLetter+letterAnd = lowercaseLetter +    `union` uppercaseLetter +    `union` titlecaseLetter+modifierLetter  = cat ModifierLetter+otherLetter = cat OtherLetter+letter +          = letterAnd +    `union` modifierLetter +    `union` otherLetter++-- Marks+nonSpacingMark, spacingCombiningMark, enclosingMark, mark :: CharSet+nonSpacingMark = cat NonSpacingMark+spacingCombiningMark = cat SpacingCombiningMark+enclosingMark = cat EnclosingMark+mark +          = nonSpacingMark +    `union` spacingCombiningMark +    `union` enclosingMark++space, lineSeparator, paragraphSeparator, separator :: CharSet+space = cat Space+lineSeparator = cat LineSeparator+paragraphSeparator = cat ParagraphSeparator+separator +          = space +    `union` lineSeparator +    `union` paragraphSeparator++mathSymbol, currencySymbol, modifierSymbol, otherSymbol, symbol :: CharSet+mathSymbol = cat MathSymbol+currencySymbol = cat CurrencySymbol+modifierSymbol = cat ModifierSymbol+otherSymbol = cat OtherSymbol+symbol +          = mathSymbol +    `union` currencySymbol +    `union` modifierSymbol +    `union` otherSymbol++decimalNumber, letterNumber, otherNumber, number :: CharSet+decimalNumber = cat DecimalNumber+letterNumber = cat LetterNumber+otherNumber = cat OtherNumber+number +          = decimalNumber +    `union` letterNumber +    `union` otherNumber++dashPunctuation, openPunctuation, closePunctuation, initialQuote, +  finalQuote, connectorPunctuation, otherPunctuation, punctuation :: CharSet++dashPunctuation = cat DashPunctuation+openPunctuation = cat OpenPunctuation+closePunctuation = cat ClosePunctuation+initialQuote = cat InitialQuote+finalQuote = cat FinalQuote+connectorPunctuation  = cat ConnectorPunctuation+otherPunctuation = cat OtherPunctuation+punctuation +          = dashPunctuation +    `union` openPunctuation +    `union` closePunctuation +    `union` initialQuote +    `union` finalQuote +    `union` connectorPunctuation +    `union` otherPunctuation++control, format, privateUse, surrogate, notAssigned, other :: CharSet+control = cat Control+format = cat Format+privateUse = cat PrivateUse+surrogate = cat Surrogate+notAssigned = cat NotAssigned+other = control +    `union` format +    `union` privateUse +    `union` surrogate +    `union` notAssigned

@@ -1,5 +1,5 @@ name:         charset-version:      0.1+version:      0.2.0 license:      BSD3 license-File: LICENSE copyright:    (c) Edward Kmett 2010@@ -21,6 +21,7 @@     Data.CharSet     Data.CharSet.Common     Data.CharSet.Posix.Ascii-    Data.CharSet.Unicode+    Data.CharSet.Unicode.Block+    Data.CharSet.Unicode.Category  GHC-Options: -Wall -fspec-constr -fdicts-cheap -O2