diff --git a/Data/CharSet/Unicode.hs b/Data/CharSet/Unicode.hs deleted file mode 100644 --- a/Data/CharSet/Unicode.hs +++ /dev/null @@ -1,177 +0,0 @@ -{-# LANGUAGE DeriveDataTypeable #-} ------------------------------------------------------------------------------ --- | --- Module : Data.CharSet.Unicode --- Copyright : (c) Edward Kmett 2010 --- License : BSD3 --- Maintainer : ekmett@gmail.com --- Stability : experimental --- Portability : portable --- --- Provides unicode general categories, which are typically connoted by --- @\p{Ll}@ or @\p{Modifier_Letter}@. Lookups can be constructed using 'categories' --- or individual character sets can be used directly. -------------------------------------------------------------------------------- - -module Data.CharSet.Unicode - ( - -- * Unicode General Category - UnicodeCategory(..) - -- * Lookup - , unicodeCategories - -- * CharSets by UnicodeCategory - -- ** Letter - , modifierLetter, otherLetter, letter - -- *** Letter\& - , lowercaseLetter, uppercaseLetter, titlecaseLetter, letterAnd - -- ** Mark - , nonSpacingMark, spacingCombiningMark, enclosingMark, mark - -- ** Separator - , space, lineSeparator, paragraphSeparator, separator - -- ** Symbol - , mathSymbol, currencySymbol, modifierSymbol, otherSymbol, symbol - -- ** Number - , decimalNumber, letterNumber, otherNumber, number - -- ** Punctuation - , dashPunctuation, openPunctuation, closePunctuation, initialQuote - , finalQuote, connectorPunctuation, otherPunctuation, punctuation - -- ** Other - , control, format, privateUse, surrogate, notAssigned, other - ) where - -import Data.Char -import Data.CharSet -import Data.Data - -data UnicodeCategory = UnicodeCategory String String CharSet String - deriving (Show, Data, Typeable) - --- \p{Letter} or \p{Mc} -unicodeCategories :: [UnicodeCategory] -unicodeCategories = - [ UnicodeCategory "Letter" "L" letter "any kind of letter from any language." - , UnicodeCategory "Lowercase_Letter" "Ll" lowercaseLetter "a lowercase letter that has an uppercase variant" - , UnicodeCategory "Uppercase_Letter" "Lu" uppercaseLetter "an uppercase letter that has a lowercase variant" - , UnicodeCategory "Titlecase_Letter" "Lt" titlecaseLetter "a letter that appears at the start of a word when only the first letter of the word is capitalized" - , UnicodeCategory "Letter&" "L&" letterAnd "a letter that exists in lowercase and uppercase variants (combination of Ll, Lu and Lt)" - , UnicodeCategory "Modifier_Letter" "Lm" modifierLetter "a special character that is used like a letter" - , UnicodeCategory "Other_Letter" "Lo" otherLetter "a letter or ideograph that does not have lowercase and uppercase variants" - , UnicodeCategory "Mark" "M" mark "a character intended to be combined with another character (e.g. accents, umlauts, enclosing boxes, etc.)" - , UnicodeCategory "Non_Spacing_Mark" "Mn" nonSpacingMark "a character intended to be combined with another character without taking up extra space (e.g. accents, umlauts, etc.)" - , UnicodeCategory "Spacing_Combining_Mark" "Mc" spacingCombiningMark "a character intended to be combined with another character that takes up extra space (vowel signs in many Eastern languages)" - , UnicodeCategory "Enclosing_Mark" "Me" enclosingMark "a character that encloses the character is is combined with (circle, square, keycap, etc.)" - , UnicodeCategory "Separator" "Z" separator "any kind of whitespace or invisible separator" - , UnicodeCategory "Space_Separator" "Zs" space "a whitespace character that is invisible, but does take up space" - , UnicodeCategory "Line_Separator" "Zl" lineSeparator "line separator character U+2028" - , UnicodeCategory "Paragraph_Separator" "Zp" paragraphSeparator "paragraph separator character U+2029" - , UnicodeCategory "Symbol" "S" symbol "math symbols, currency signs, dingbats, box-drawing characters, etc." - , UnicodeCategory "Math_Symbol" "Sm" mathSymbol "any mathematical symbol" - , UnicodeCategory "Currency_Symbol" "Sc" currencySymbol "any currency sign" - , UnicodeCategory "Modifier_Symbol" "Sk" modifierSymbol "a combining character (mark) as a full character on its own" - , UnicodeCategory "Other_Symbol" "So" otherSymbol "various symbols that are not math symbols, currency signs, or combining characters" - , UnicodeCategory "Number" "N" number "any kind of numeric character in any script" - , UnicodeCategory "Decimal_Digit_Number" "Nd" decimalNumber "a digit zero through nine in any script except ideographic scripts" - , UnicodeCategory "Letter_Number" "Nl" letterNumber "a number that looks like a letter, such as a Roman numeral" - , UnicodeCategory "Other_Number" "No" otherNumber "a superscript or subscript digit, or a number that is not a digit 0..9 (excluding numbers from ideographic scripts)" - , UnicodeCategory "Punctuation" "P" punctuation "any kind of punctuation character" - , UnicodeCategory "Dash_Punctuation" "Pd" dashPunctuation "any kind of hyphen or dash" - , UnicodeCategory "Open_Punctuation" "Ps" openPunctuation "any kind of opening bracket" - , UnicodeCategory "Close_Punctuation" "Pe" closePunctuation "any kind of closing bracket" - , UnicodeCategory "Initial_Punctuation" "Pi" initialQuote "any kind of opening quote" - , UnicodeCategory "Final_Punctuation" "Pf" finalQuote "any kind of closing quote" - , UnicodeCategory "Connector_Punctuation" "Pc" connectorPunctuation "a punctuation character such as an underscore that connects words" - , UnicodeCategory "Other_Punctuation" "Po" otherPunctuation "any kind of punctuation character that is not a dash, bracket, quote or connector" - , UnicodeCategory "Other" "C" other "invisible control characters and unused code points" - , UnicodeCategory "Control" "Cc" control "an ASCII 0x00..0x1F or Latin-1 0x80..0x9F control character" - , UnicodeCategory "Format" "Cf" format "invisible formatting indicator" - , UnicodeCategory "Private_Use" "Co" privateUse "any code point reserved for private use" - , UnicodeCategory "Surrogate" "Cs" surrogate "one half of a surrogate pair in UTF-16 encoding" - , UnicodeCategory "Unassigned" "Cn" notAssigned "any code point to which no character has been assigned.properties" ] - -cat :: GeneralCategory -> CharSet -cat category = build ((category ==) . generalCategory) - --- Letter -lowercaseLetter, uppercaseLetter, titlecaseLetter, letterAnd, modifierLetter, otherLetter, letter :: CharSet -lowercaseLetter = cat LowercaseLetter -uppercaseLetter = cat UppercaseLetter -titlecaseLetter = cat TitlecaseLetter -letterAnd = lowercaseLetter - `union` uppercaseLetter - `union` titlecaseLetter -modifierLetter = cat ModifierLetter -otherLetter = cat OtherLetter -letter - = letterAnd - `union` modifierLetter - `union` otherLetter - --- Marks -nonSpacingMark, spacingCombiningMark, enclosingMark, mark :: CharSet -nonSpacingMark = cat NonSpacingMark -spacingCombiningMark = cat SpacingCombiningMark -enclosingMark = cat EnclosingMark -mark - = nonSpacingMark - `union` spacingCombiningMark - `union` enclosingMark - -space, lineSeparator, paragraphSeparator, separator :: CharSet -space = cat Space -lineSeparator = cat LineSeparator -paragraphSeparator = cat ParagraphSeparator -separator - = space - `union` lineSeparator - `union` paragraphSeparator - -mathSymbol, currencySymbol, modifierSymbol, otherSymbol, symbol :: CharSet -mathSymbol = cat MathSymbol -currencySymbol = cat CurrencySymbol -modifierSymbol = cat ModifierSymbol -otherSymbol = cat OtherSymbol -symbol - = mathSymbol - `union` currencySymbol - `union` modifierSymbol - `union` otherSymbol - -decimalNumber, letterNumber, otherNumber, number :: CharSet -decimalNumber = cat DecimalNumber -letterNumber = cat LetterNumber -otherNumber = cat OtherNumber -number - = decimalNumber - `union` letterNumber - `union` otherNumber - -dashPunctuation, openPunctuation, closePunctuation, initialQuote, - finalQuote, connectorPunctuation, otherPunctuation, punctuation :: CharSet - -dashPunctuation = cat DashPunctuation -openPunctuation = cat OpenPunctuation -closePunctuation = cat ClosePunctuation -initialQuote = cat InitialQuote -finalQuote = cat FinalQuote -connectorPunctuation = cat ConnectorPunctuation -otherPunctuation = cat OtherPunctuation -punctuation - = dashPunctuation - `union` openPunctuation - `union` closePunctuation - `union` initialQuote - `union` finalQuote - `union` connectorPunctuation - `union` otherPunctuation - -control, format, privateUse, surrogate, notAssigned, other :: CharSet -control = cat Control -format = cat Format -privateUse = cat PrivateUse -surrogate = cat Surrogate -notAssigned = cat NotAssigned -other = control - `union` format - `union` privateUse - `union` surrogate - `union` notAssigned diff --git a/Data/CharSet/Unicode/Block.hs b/Data/CharSet/Unicode/Block.hs new file mode 100644 --- /dev/null +++ b/Data/CharSet/Unicode/Block.hs @@ -0,0 +1,382 @@ +{-# LANGUAGE DeriveDataTypeable #-} +{-# OPTIONS_GHC -fno-warn-missing-signatures #-} +----------------------------------------------------------------------------- +-- | +-- Module : Data.CharSet.Unicode.Block +-- Copyright : (c) Edward Kmett 2010 +-- License : BSD3 +-- Maintainer : ekmett@gmail.com +-- Stability : experimental +-- Portability : portable +-- +-- Provides unicode general categories, which are typically connoted by +-- @\p{InBasicLatin}@ or @\p{InIPA_Extensions}@. Lookups can be constructed using 'categories' +-- or individual character sets can be used directly. +------------------------------------------------------------------------------- + +module Data.CharSet.Unicode.Block + ( + -- * Unicode General Category + Block(..) + -- * Lookup + , blocks + , lookupBlock + , lookupBlockCharSet + -- * CharSets by Block + , basicLatin + , latin1Supplement + , latinExtendedA + , latinExtendedB + , ipaExtensions + , spacingModifierLetters + , combiningDiacriticalMarks + , greekAndCoptic + , cyrillic + , cyrillicSupplementary + , armenian + , hebrew + , arabic + , syriac + , thaana + , devanagari + , bengali + , gurmukhi + , gujarati + , oriya + , tamil + , telugu + , kannada + , malayalam + , sinhala + , thai + , lao + , tibetan + , myanmar + , georgian + , hangulJamo + , ethiopic + , cherokee + , unifiedCanadianAboriginalSyllabics + , ogham + , runic + , tagalog + , hanunoo + , buhid + , tagbanwa + , khmer + , mongolian + , limbu + , taiLe + , khmerSymbols + , phoneticExtensions + , latinExtendedAdditional + , greekExtended + , generalPunctuation + , superscriptsAndSubscripts + , currencySymbols + , combiningDiacriticalMarksForSymbols + , letterlikeSymbols + , numberForms + , arrows + , mathematicalOperators + , miscellaneousTechnical + , controlPictures + , opticalCharacterRecognition + , enclosedAlphanumerics + , boxDrawing + , blockElements + , geometricShapes + , miscellaneousSymbols + , dingbats + , miscellaneousMathematicalSymbolsA + , supplementalArrowsA + , braillePatterns + , supplementalArrowsB + , miscellaneousMathematicalSymbolsB + , supplementalMathematicalOperators + , miscellaneousSymbolsAndArrows + , cjkRadicalsSupplement + , kangxiRadicals + , ideographicDescriptionCharacters + , cjkSymbolsAndPunctuation + , hiragana + , katakana + , bopomofo + , hangulCompatibilityJamo + , kanbun + , bopomofoExtended + , katakanaPhoneticExtensions + , enclosedCjkLettersAndMonths + , cjkCompatibility + , cjkUnifiedIdeographsExtensionA + , yijingHexagramSymbols + , cjkUnifiedIdeographs + , yiSyllables + , yiRadicals + , hangulSyllables + , highSurrogates + , highPrivateUseSurrogates + , lowSurrogates + , privateUseArea + , cjkCompatibilityIdeographs + , alphabeticPresentationForms + , arabicPresentationFormsA + , variationSelectors + , combiningHalfMarks + , cjkCompatibilityForms + , smallFormVariants + , arabicPresentationFormsB + , halfwidthAndFullwidthForms + , specials + ) where + +import Data.Char +import Data.CharSet +import Data.Data +import Data.Map (Map) +import qualified Data.Map as Map + +data Block = Block + { blockName :: String + , blockCharSet :: CharSet + } deriving (Show, Data, Typeable) + +blocks :: [Block] +blocks = + [ Block "Basic_Latin" basicLatin + , Block "Latin-1_Supplement" latin1Supplement + , Block "Latin_Extended-A" latinExtendedA + , Block "IPA_Extensions" ipaExtensions + , Block "Spacing_Modifier_Letters" spacingModifierLetters + + , Block "Latin_Extended-A" latinExtendedA + , Block "Latin_Extended-B" latinExtendedB + , Block "IPA_Extensions" ipaExtensions + , Block "Spacing_Modifier_Letters" spacingModifierLetters + , Block "Combining_Diacritical_Marks" combiningDiacriticalMarks + , Block "Greek_and_Coptic" greekAndCoptic + , Block "Cyrillic" cyrillic + , Block "Cyrillic_Supplementary" cyrillicSupplementary + , Block "Armenian" armenian + , Block "Hebrew" hebrew + , Block "Arabic" arabic + , Block "Syriac" syriac + , Block "Thaana" thaana + , Block "Devanagari" devanagari + , Block "Bengali" bengali + , Block "Gurmukhi" gurmukhi + , Block "Gujarati" gujarati + , Block "Oriya" oriya + , Block "Tamil" tamil + , Block "Telugu" telugu + , Block "Kannada" kannada + , Block "Malayalam" malayalam + , Block "Sinhala" sinhala + , Block "Thai" thai + , Block "Lao" lao + , Block "Tibetan" tibetan + , Block "Myanmar" myanmar + , Block "Georgian" georgian + , Block "Hangul_Jamo" hangulJamo + , Block "Ethiopic" ethiopic + , Block "Cherokee" cherokee + , Block "Unified_Canadian_Aboriginal_Syllabics" unifiedCanadianAboriginalSyllabics + , Block "Ogham" ogham + , Block "Runic" runic + , Block "Tagalog" tagalog + , Block "Hanunoo" hanunoo + , Block "Buhid" buhid + , Block "Tagbanwa" tagbanwa + , Block "Khmer" khmer + , Block "Mongolian" mongolian + , Block "Limbu" limbu + , Block "Tai_Le" taiLe + , Block "Khmer_Symbols" khmerSymbols + , Block "Phonetic_Extensions" phoneticExtensions + , Block "Latin_Extended_Additional" latinExtendedAdditional + , Block "Greek_Extended" greekExtended + , Block "General_Punctuation" generalPunctuation + , Block "Superscripts_and_Subscripts" superscriptsAndSubscripts + , Block "Currency_Symbols" currencySymbols + , Block "Combining_Diacritical_Marks_for_Symbols" combiningDiacriticalMarksForSymbols + , Block "Letterlike_Symbols" letterlikeSymbols + , Block "Number_Forms" numberForms + , Block "Arrows" arrows + , Block "Mathematical_Operators" mathematicalOperators + , Block "Miscellaneous_Technical" miscellaneousTechnical + , Block "Control_Pictures" controlPictures + , Block "Optical_Character_Recognition" opticalCharacterRecognition + , Block "Enclosed_Alphanumerics" enclosedAlphanumerics + , Block "Box_Drawing" boxDrawing + , Block "Block_Elements" blockElements + , Block "Geometric_Shapes" geometricShapes + , Block "Miscellaneous_Symbols" miscellaneousSymbols + , Block "Dingbats" dingbats + , Block "Miscellaneous_Mathematical_Symbols-A" miscellaneousMathematicalSymbolsA + , Block "Supplemental_Arrows-A" supplementalArrowsA + , Block "Braille_Patterns" braillePatterns + , Block "Supplemental_Arrows-B" supplementalArrowsB + , Block "Miscellaneous_Mathematical_Symbols-B" miscellaneousMathematicalSymbolsB + , Block "Supplemental_Mathematical_Operators" supplementalMathematicalOperators + , Block "Miscellaneous_Symbols_and_Arrows" miscellaneousSymbolsAndArrows + , Block "CJK_Radicals_Supplement" cjkRadicalsSupplement + , Block "Kangxi_Radicals" kangxiRadicals + , Block "Ideographic_Description_Characters" ideographicDescriptionCharacters + , Block "CJK_Symbols_and_Punctuation" cjkSymbolsAndPunctuation + , Block "Hiragana" hiragana + , Block "Katakana" katakana + , Block "Bopomofo" bopomofo + , Block "Hangul_Compatibility_Jamo" hangulCompatibilityJamo + , Block "Kanbun" kanbun + , Block "Bopomofo_Extended" bopomofoExtended + , Block "Katakana_Phonetic_Extensions" katakanaPhoneticExtensions + , Block "Enclosed_CJK_Letters_and_Months" enclosedCjkLettersAndMonths + , Block "CJK_Compatibility" cjkCompatibility + , Block "CJK_Unified_Ideographs_Extension_A" cjkUnifiedIdeographsExtensionA + , Block "Yijing_Hexagram_Symbols" yijingHexagramSymbols + , Block "CJK_Unified_Ideographs" cjkUnifiedIdeographs + , Block "Yi_Syllables" yiSyllables + , Block "Yi_Radicals" yiRadicals + , Block "Hangul_Syllables" hangulSyllables + , Block "High_Surrogates" highSurrogates + , Block "High_Private_Use_Surrogates" highPrivateUseSurrogates + , Block "Low_Surrogates" lowSurrogates + , Block "Private_Use_Area" privateUseArea + , Block "CJK_Compatibility_Ideographs" cjkCompatibilityIdeographs + , Block "Alphabetic_Presentation_Forms" alphabeticPresentationForms + , Block "Arabic_Presentation_Forms-A" arabicPresentationFormsA + , Block "Variation_Selectors" variationSelectors + , Block "Combining_Half_Marks" combiningHalfMarks + , Block "CJK_Compatibility_Forms" cjkCompatibilityForms + , Block "Small_Form_Variants" smallFormVariants + , Block "Arabic_Presentation_Forms-B" arabicPresentationFormsB + , Block "Halfwidth_and_Fullwidth_Forms" halfwidthAndFullwidthForms + , Block "Specials" specials ] + +lookupTable :: Map String Block +lookupTable = Map.fromList $ + Prelude.map (\y@(Block x _) -> (canonicalize x, y)) + blocks + +canonicalize :: String -> String +canonicalize s = case Prelude.map toLower s of + 'i': 'n' : xs -> go xs + xs -> go xs + where + go ('-':xs) = go xs + go ('_':xs) = go xs + go (x:xs) = x : go xs + go [] = [] + +lookupBlock :: String -> Maybe Block +lookupBlock s = Map.lookup (canonicalize s) lookupTable + +lookupBlockCharSet :: String -> Maybe CharSet +lookupBlockCharSet = fmap blockCharSet . lookupBlock + +basicLatin = range '\x0000' '\x007f' +latin1Supplement = range '\x0080' '\x00ff' +latinExtendedA = range '\x0100' '\x017F' +latinExtendedB = range '\x0180' '\x024F' +ipaExtensions = range '\x0250' '\x02AF' +spacingModifierLetters = range '\x02B0' '\x02FF' +combiningDiacriticalMarks = range '\x0300' '\x036F' +greekAndCoptic = range '\x0370' '\x03FF' +cyrillic = range '\x0400' '\x04FF' +cyrillicSupplementary = range '\x0500' '\x052F' +armenian = range '\x0530' '\x058F' +hebrew = range '\x0590' '\x05FF' +arabic = range '\x0600' '\x06FF' +syriac = range '\x0700' '\x074F' +thaana = range '\x0780' '\x07BF' +devanagari = range '\x0900' '\x097F' +bengali = range '\x0980' '\x09FF' +gurmukhi = range '\x0A00' '\x0A7F' +gujarati = range '\x0A80' '\x0AFF' +oriya = range '\x0B00' '\x0B7F' +tamil = range '\x0B80' '\x0BFF' +telugu = range '\x0C00' '\x0C7F' +kannada = range '\x0C80' '\x0CFF' +malayalam = range '\x0D00' '\x0D7F' +sinhala = range '\x0D80' '\x0DFF' +thai = range '\x0E00' '\x0E7F' +lao = range '\x0E80' '\x0EFF' +tibetan = range '\x0F00' '\x0FFF' +myanmar = range '\x1000' '\x109F' +georgian = range '\x10A0' '\x10FF' +hangulJamo = range '\x1100' '\x11FF' +ethiopic = range '\x1200' '\x137F' +cherokee = range '\x13A0' '\x13FF' +unifiedCanadianAboriginalSyllabics = range '\x1400' '\x167F' +ogham = range '\x1680' '\x169F' +runic = range '\x16A0' '\x16FF' +tagalog = range '\x1700' '\x171F' +hanunoo = range '\x1720' '\x173F' +buhid = range '\x1740' '\x175F' +tagbanwa = range '\x1760' '\x177F' +khmer = range '\x1780' '\x17FF' +mongolian = range '\x1800' '\x18AF' +limbu = range '\x1900' '\x194F' +taiLe = range '\x1950' '\x197F' +khmerSymbols = range '\x19E0' '\x19FF' +phoneticExtensions = range '\x1D00' '\x1D7F' +latinExtendedAdditional = range '\x1E00' '\x1EFF' +greekExtended = range '\x1F00' '\x1FFF' +generalPunctuation = range '\x2000' '\x206F' +superscriptsAndSubscripts = range '\x2070' '\x209F' +currencySymbols = range '\x20A0' '\x20CF' +combiningDiacriticalMarksForSymbols = range '\x20D0' '\x20FF' +letterlikeSymbols = range '\x2100' '\x214F' +numberForms = range '\x2150' '\x218F' +arrows = range '\x2190' '\x21FF' +mathematicalOperators = range '\x2200' '\x22FF' +miscellaneousTechnical = range '\x2300' '\x23FF' +controlPictures = range '\x2400' '\x243F' +opticalCharacterRecognition = range '\x2440' '\x245F' +enclosedAlphanumerics = range '\x2460' '\x24FF' +boxDrawing = range '\x2500' '\x257F' +blockElements = range '\x2580' '\x259F' +geometricShapes = range '\x25A0' '\x25FF' +miscellaneousSymbols = range '\x2600' '\x26FF' +dingbats = range '\x2700' '\x27BF' +miscellaneousMathematicalSymbolsA = range '\x27C0' '\x27EF' +supplementalArrowsA = range '\x27F0' '\x27FF' +braillePatterns = range '\x2800' '\x28FF' +supplementalArrowsB = range '\x2900' '\x297F' +miscellaneousMathematicalSymbolsB = range '\x2980' '\x29FF' +supplementalMathematicalOperators = range '\x2A00' '\x2AFF' +miscellaneousSymbolsAndArrows = range '\x2B00' '\x2BFF' +cjkRadicalsSupplement = range '\x2E80' '\x2EFF' +kangxiRadicals = range '\x2F00' '\x2FDF' +ideographicDescriptionCharacters = range '\x2FF0' '\x2FFF' +cjkSymbolsAndPunctuation = range '\x3000' '\x303F' +hiragana = range '\x3040' '\x309F' +katakana = range '\x30A0' '\x30FF' +bopomofo = range '\x3100' '\x312F' +hangulCompatibilityJamo = range '\x3130' '\x318F' +kanbun = range '\x3190' '\x319F' +bopomofoExtended = range '\x31A0' '\x31BF' +katakanaPhoneticExtensions = range '\x31F0' '\x31FF' +enclosedCjkLettersAndMonths = range '\x3200' '\x32FF' +cjkCompatibility = range '\x3300' '\x33FF' +cjkUnifiedIdeographsExtensionA = range '\x3400' '\x4DBF' +yijingHexagramSymbols = range '\x4DC0' '\x4DFF' +cjkUnifiedIdeographs = range '\x4E00' '\x9FFF' +yiSyllables = range '\xA000' '\xA48F' +yiRadicals = range '\xA490' '\xA4CF' +hangulSyllables = range '\xAC00' '\xD7AF' +highSurrogates = range '\xD800' '\xDB7F' +highPrivateUseSurrogates = range '\xDB80' '\xDBFF' +lowSurrogates = range '\xDC00' '\xDFFF' +privateUseArea = range '\xE000' '\xF8FF' +cjkCompatibilityIdeographs = range '\xF900' '\xFAFF' +alphabeticPresentationForms = range '\xFB00' '\xFB4F' +arabicPresentationFormsA = range '\xFB50' '\xFDFF' +variationSelectors = range '\xFE00' '\xFE0F' +combiningHalfMarks = range '\xFE20' '\xFE2F' +cjkCompatibilityForms = range '\xFE30' '\xFE4F' +smallFormVariants = range '\xFE50' '\xFE6F' +arabicPresentationFormsB = range '\xFE70' '\xFEFF' +halfwidthAndFullwidthForms = range '\xFF00' '\xFFEF' +specials = range '\xFFF0' '\xFFFF' + diff --git a/Data/CharSet/Unicode/Category.hs b/Data/CharSet/Unicode/Category.hs new file mode 100644 --- /dev/null +++ b/Data/CharSet/Unicode/Category.hs @@ -0,0 +1,210 @@ +{-# LANGUAGE DeriveDataTypeable #-} +----------------------------------------------------------------------------- +-- | +-- Module : Data.CharSet.Unicode.Category +-- Copyright : (c) Edward Kmett 2010 +-- License : BSD3 +-- Maintainer : ekmett@gmail.com +-- Stability : experimental +-- Portability : portable +-- +-- Provides unicode general categories, which are typically connoted by +-- @\p{Ll}@ or @\p{Modifier_Letter}@. Lookups can be constructed using 'categories' +-- or individual character sets can be used directly. +-- +-- A case, @_@ and @-@ insensitive lookup is provided by 'lookupCategory' +-- and can be used to provide behavior similar to that of Perl or PCRE. +------------------------------------------------------------------------------- + +module Data.CharSet.Unicode.Category + ( + -- * Unicode General Category + Category(..) + -- * Lookup + , categories + , lookupCategory + , lookupCategoryCharSet + -- * CharSets by Category + -- ** Letter + , modifierLetter, otherLetter, letter + -- *** Letter\& + , lowercaseLetter, uppercaseLetter, titlecaseLetter, letterAnd + -- ** Mark + , nonSpacingMark, spacingCombiningMark, enclosingMark, mark + -- ** Separator + , space, lineSeparator, paragraphSeparator, separator + -- ** Symbol + , mathSymbol, currencySymbol, modifierSymbol, otherSymbol, symbol + -- ** Number + , decimalNumber, letterNumber, otherNumber, number + -- ** Punctuation + , dashPunctuation, openPunctuation, closePunctuation, initialQuote + , finalQuote, connectorPunctuation, otherPunctuation, punctuation + -- ** Other + , control, format, privateUse, surrogate, notAssigned, other + ) where + +import Data.Char +import Data.CharSet +import Data.Data +import Data.Map (Map) +import qualified Data.Map as Map + +data Category = Category + { categoryName :: String + , categoryAbbreviation :: String + , categoryCharSet :: CharSet + , categoryDescription :: String + } deriving (Show, Data, Typeable) + +-- \p{Letter} or \p{Mc} +categories :: [Category] +categories = + [ Category "Letter" "L" letter "any kind of letter from any language." + , Category "Lowercase_Letter" "Ll" lowercaseLetter "a lowercase letter that has an uppercase variant" + , Category "Uppercase_Letter" "Lu" uppercaseLetter "an uppercase letter that has a lowercase variant" + , Category "Titlecase_Letter" "Lt" titlecaseLetter "a letter that appears at the start of a word when only the first letter of the word is capitalized" + , Category "Letter&" "L&" letterAnd "a letter that exists in lowercase and uppercase variants (combination of Ll, Lu and Lt)" + , Category "Modifier_Letter" "Lm" modifierLetter "a special character that is used like a letter" + , Category "Other_Letter" "Lo" otherLetter "a letter or ideograph that does not have lowercase and uppercase variants" + , Category "Mark" "M" mark "a character intended to be combined with another character (e.g. accents, umlauts, enclosing boxes, etc.)" + , Category "Non_Spacing_Mark" "Mn" nonSpacingMark "a character intended to be combined with another character without taking up extra space (e.g. accents, umlauts, etc.)" + , Category "Spacing_Combining_Mark" "Mc" spacingCombiningMark "a character intended to be combined with another character that takes up extra space (vowel signs in many Eastern languages)" + , Category "Enclosing_Mark" "Me" enclosingMark "a character that encloses the character is is combined with (circle, square, keycap, etc.)" + , Category "Separator" "Z" separator "any kind of whitespace or invisible separator" + , Category "Space_Separator" "Zs" space "a whitespace character that is invisible, but does take up space" + , Category "Line_Separator" "Zl" lineSeparator "line separator character U+2028" + , Category "Paragraph_Separator" "Zp" paragraphSeparator "paragraph separator character U+2029" + , Category "Symbol" "S" symbol "math symbols, currency signs, dingbats, box-drawing characters, etc." + , Category "Math_Symbol" "Sm" mathSymbol "any mathematical symbol" + , Category "Currency_Symbol" "Sc" currencySymbol "any currency sign" + , Category "Modifier_Symbol" "Sk" modifierSymbol "a combining character (mark) as a full character on its own" + , Category "Other_Symbol" "So" otherSymbol "various symbols that are not math symbols, currency signs, or combining characters" + , Category "Number" "N" number "any kind of numeric character in any script" + , Category "Decimal_Digit_Number" "Nd" decimalNumber "a digit zero through nine in any script except ideographic scripts" + , Category "Letter_Number" "Nl" letterNumber "a number that looks like a letter, such as a Roman numeral" + , Category "Other_Number" "No" otherNumber "a superscript or subscript digit, or a number that is not a digit 0..9 (excluding numbers from ideographic scripts)" + , Category "Punctuation" "P" punctuation "any kind of punctuation character" + , Category "Dash_Punctuation" "Pd" dashPunctuation "any kind of hyphen or dash" + , Category "Open_Punctuation" "Ps" openPunctuation "any kind of opening bracket" + , Category "Close_Punctuation" "Pe" closePunctuation "any kind of closing bracket" + , Category "Initial_Punctuation" "Pi" initialQuote "any kind of opening quote" + , Category "Final_Punctuation" "Pf" finalQuote "any kind of closing quote" + , Category "Connector_Punctuation" "Pc" connectorPunctuation "a punctuation character such as an underscore that connects words" + , Category "Other_Punctuation" "Po" otherPunctuation "any kind of punctuation character that is not a dash, bracket, quote or connector" + , Category "Other" "C" other "invisible control characters and unused code points" + , Category "Control" "Cc" control "an ASCII 0x00..0x1F or Latin-1 0x80..0x9F control character" + , Category "Format" "Cf" format "invisible formatting indicator" + , Category "Private_Use" "Co" privateUse "any code point reserved for private use" + , Category "Surrogate" "Cs" surrogate "one half of a surrogate pair in UTF-16 encoding" + , Category "Unassigned" "Cn" notAssigned "any code point to which no character has been assigned.properties" ] + +lookupTable :: Map String Category +lookupTable = + Map.fromList [ (canonicalize x, category) + | category@(Category l s _ _) <- categories + , x <- [l,s] ] + +lookupCategory :: String -> Maybe Category +lookupCategory s = Map.lookup (canonicalize s) lookupTable + +lookupCategoryCharSet :: String -> Maybe CharSet +lookupCategoryCharSet = fmap categoryCharSet . lookupCategory + +canonicalize :: String -> String +canonicalize s = case Prelude.map toLower s of + 'i' : 's' : xs -> go xs + xs -> go xs + where + go ('-':xs) = go xs + go ('_':xs) = go xs + go (x:xs) = x : go xs + go [] = [] + +cat :: GeneralCategory -> CharSet +cat category = build ((category ==) . generalCategory) + +-- Letter +lowercaseLetter, uppercaseLetter, titlecaseLetter, letterAnd, modifierLetter, otherLetter, letter :: CharSet +lowercaseLetter = cat LowercaseLetter +uppercaseLetter = cat UppercaseLetter +titlecaseLetter = cat TitlecaseLetter +letterAnd = lowercaseLetter + `union` uppercaseLetter + `union` titlecaseLetter +modifierLetter = cat ModifierLetter +otherLetter = cat OtherLetter +letter + = letterAnd + `union` modifierLetter + `union` otherLetter + +-- Marks +nonSpacingMark, spacingCombiningMark, enclosingMark, mark :: CharSet +nonSpacingMark = cat NonSpacingMark +spacingCombiningMark = cat SpacingCombiningMark +enclosingMark = cat EnclosingMark +mark + = nonSpacingMark + `union` spacingCombiningMark + `union` enclosingMark + +space, lineSeparator, paragraphSeparator, separator :: CharSet +space = cat Space +lineSeparator = cat LineSeparator +paragraphSeparator = cat ParagraphSeparator +separator + = space + `union` lineSeparator + `union` paragraphSeparator + +mathSymbol, currencySymbol, modifierSymbol, otherSymbol, symbol :: CharSet +mathSymbol = cat MathSymbol +currencySymbol = cat CurrencySymbol +modifierSymbol = cat ModifierSymbol +otherSymbol = cat OtherSymbol +symbol + = mathSymbol + `union` currencySymbol + `union` modifierSymbol + `union` otherSymbol + +decimalNumber, letterNumber, otherNumber, number :: CharSet +decimalNumber = cat DecimalNumber +letterNumber = cat LetterNumber +otherNumber = cat OtherNumber +number + = decimalNumber + `union` letterNumber + `union` otherNumber + +dashPunctuation, openPunctuation, closePunctuation, initialQuote, + finalQuote, connectorPunctuation, otherPunctuation, punctuation :: CharSet + +dashPunctuation = cat DashPunctuation +openPunctuation = cat OpenPunctuation +closePunctuation = cat ClosePunctuation +initialQuote = cat InitialQuote +finalQuote = cat FinalQuote +connectorPunctuation = cat ConnectorPunctuation +otherPunctuation = cat OtherPunctuation +punctuation + = dashPunctuation + `union` openPunctuation + `union` closePunctuation + `union` initialQuote + `union` finalQuote + `union` connectorPunctuation + `union` otherPunctuation + +control, format, privateUse, surrogate, notAssigned, other :: CharSet +control = cat Control +format = cat Format +privateUse = cat PrivateUse +surrogate = cat Surrogate +notAssigned = cat NotAssigned +other = control + `union` format + `union` privateUse + `union` surrogate + `union` notAssigned diff --git a/charset.cabal b/charset.cabal --- a/charset.cabal +++ b/charset.cabal @@ -1,5 +1,5 @@ name: charset -version: 0.1 +version: 0.2.0 license: BSD3 license-File: LICENSE copyright: (c) Edward Kmett 2010 @@ -21,6 +21,7 @@ Data.CharSet Data.CharSet.Common Data.CharSet.Posix.Ascii - Data.CharSet.Unicode + Data.CharSet.Unicode.Block + Data.CharSet.Unicode.Category GHC-Options: -Wall -fspec-constr -fdicts-cheap -O2