text-icu 0.8.0.1 → 0.8.0.2

raw patch · 18 files changed

+152/−36 lines, 18 filesPVP: minor bump suggested

API additions: PVP suggests at least a minor version bump

API changes (from Hackage documentation)

Files

• Data/Text/ICU.hs +3/−1
• Data/Text/ICU/BiDi.hsc +2/−2
• Data/Text/ICU/Break.hsc +2/−2
• Data/Text/ICU/Calendar.hsc +1/−1
• Data/Text/ICU/Char.hsc +9/−9
• Data/Text/ICU/Collate.hsc +45/−3
• Data/Text/ICU/Collate/Pure.hs +25/−1
• Data/Text/ICU/Convert.hs +2/−2
• Data/Text/ICU/Normalize.hsc +1/−1
• Data/Text/ICU/Normalize2.hsc +2/−2
• Data/Text/ICU/Number.hsc +2/−2
• Data/Text/ICU/Regex/Internal.hsc +1/−1
• Data/Text/ICU/Shape.hsc +2/−2
• cbits/text_icu.c +12/−0
• changelog.md +4/−0
• include/hs_text_icu.h +4/−0
• tests/Properties.hs +31/−3
• text-icu.cabal +4/−4

@@ -62,6 +62,8 @@     , Collator     , collator     , collatorWith+    , collatorFromRules+    , collatorFromRulesWith     , collate     , collateIter     , sortKey@@ -216,7 +218,7 @@ -- -- You create a 'NumberFormat' with 'numberFormatter' according to a locale -- and a choice of pre-defined formats. A 'NumberFormat' provides a formatting--- faclity that 'format's numbers+-- facility that 'format's numbers -- according to the chosen locale. Alternatively create and apply a 'NumberFormat' -- in a single step with 'formatNumber'' (it may be faster to re-use a NumberFormat though). -- See the section \"Patterns\" at <https://unicode-org.github.io/icu-docs/apidoc/released/icu4c/classDecimalFormat.html#Patterns>

@@ -8,8 +8,8 @@ -- Stability   : experimental -- Portability : GHC ----- Implementation of Unicode Bidirection algorithm. See documentation of the libicu--- library for additional details.+-- Implementation of the Unicode Bidirectional Algorithm. See the documentation+-- of the libicu library for additional details. -- -- -- /Note/: this module is not thread safe. /Do not/ call the -- functions on one BiDi object from more than one thread!

@@ -70,7 +70,7 @@ -- -- /Important note/: All of the indices accepted and returned by -- functions in this module are offsets into the raw UTF-16 text--- array, /not/ a count of code points.+-- array, /not/ a count of codepoints.  -- | Line break status. data Line = Soft                -- ^ A soft line break is a position at@@ -228,7 +228,7 @@       _ <- handleError $ ubrk_getRuleStatusVec brk ptr n       map brStatus `fmap` peekArray (fromIntegral n) ptr --- | Determine whether the specfied position is a boundary position.+-- | Determine whether the specified position is a boundary position. -- As a side effect, leaves the iterator pointing to the first -- boundary position at or after the given offset. isBoundary :: BreakIterator a -> Int -> IO Bool

@@ -58,7 +58,7 @@  type UCalendar = CInt --- A 'Calendar' is an absttract data type that contains a foreign pointer to the ICU internal data structure.+-- A 'Calendar' is an abstract data type that contains a foreign pointer to the ICU internal data structure. data Calendar = Calendar {calendarForeignPtr :: ForeignPtr UCalendar}  -- | All the fields that comprise a 'Calendar'.

@@ -13,7 +13,7 @@ -- Access to the Unicode Character Database, implemented as bindings -- to the International Components for Unicode (ICU) libraries. ----- Unicode assigns each code point (not just assigned character) values for+-- Unicode assigns each codepoint (not just assigned character) values for -- many properties.  Most are simple boolean flags, or constants from a -- small enumerated list.  For some, values are relatively more complex -- types.@@ -1147,14 +1147,14 @@ blockCode = toEnum . fromIntegral . ublock_getCode . fromIntegral . ord {-# INLINE blockCode #-} --- | Return the bidirectional category value for the code point,+-- | Return the bidirectional category value for the codepoint, -- which is used in the Unicode bidirectional algorithm (UAX #9 -- <http://www.unicode.org/reports/tr9/>). direction :: Char -> Direction direction = toEnum . fromIntegral . u_charDirection . fromIntegral . ord {-# INLINE direction #-} --- | Determine whether the code point has the 'BidiMirrored' property.  This+-- | Determine whether the codepoint has the 'BidiMirrored' property.  This -- property is set for characters that are commonly used in Right-To-Left -- contexts and need to be displayed with a "mirrored" glyph. isMirrored :: Char -> Bool@@ -1164,13 +1164,13 @@ -- Map the specified character to a "mirror-image" character. -- -- For characters with the 'BidiMirrored' property, implementations--- sometimes need a "poor man's" mapping to another Unicode (code point)+-- sometimes need a "poor man's" mapping to another Unicode (codepoint) -- such that the default glyph may serve as the mirror image of the default -- glyph of the specified character. This is useful for text conversion to--- and from codepages with visual order, and for displays without glyph+-- and from code pages with visual order, and for displays without glyph -- selection capabilities. ----- The return value is another Unicode code point that may serve as a+-- The return value is another Unicode codepoint that may serve as a -- mirror-image substitute, or the original character itself if there -- is no such mapping or the character lacks the 'BidiMirrored' -- property.@@ -1198,7 +1198,7 @@     | otherwise = Just $! fromIntegral i   where i = u_charDigitValue . fromIntegral . ord $ c --- | Return the numeric value for a Unicode code point as defined in the+-- | Return the numeric value for a Unicode codepoint as defined in the -- Unicode Character Database. -- -- A 'Double' return type is necessary because some numeric values are@@ -1220,7 +1220,7 @@  -- | Return the full name of a Unicode character. ----- Compared to 'charName', this function gives each Unicode code point+-- Compared to 'charName', this function gives each Unicode codepoint -- a unique extended name. Extended names are lowercase followed by an -- uppercase hexadecimal number, within angle brackets. charFullName :: Char -> String@@ -1237,7 +1237,7 @@ charFromName = charFromName' (#const U_UNICODE_CHAR_NAME)  -- | Find a Unicode character by its full or extended name, and return--- its code point value.+-- its codepoint value. -- -- The name is matched exactly and completely. --

@@ -23,9 +23,11 @@     , Strength(..)     -- * Functions     , open+    , openRules     , collate     , collateIter     -- ** Utility functions+    , getRules     , getAttribute     , setAttribute     , sortKey@@ -45,17 +47,20 @@ import Data.Text.Foreign (useAsPtr) import Data.Text.ICU.Collate.Internal (Collator(..), MCollator, UCollator,                                        withCollator, wrap)-import Data.Text.ICU.Error.Internal (UErrorCode, handleError)+import Data.Text.ICU.Error (u_INVALID_FORMAT_ERROR)+import Data.Text.ICU.Error.Internal (UErrorCode, UParseError, handleError, handleParseError) import Data.Text.ICU.Internal     (LocaleName, UChar, CharIterator, UCharIterator,-     asOrdering, withCharIterator, withLocaleName, useAsUCharPtr)+     asOrdering, fromUCharPtr, withCharIterator, withLocaleName, useAsUCharPtr) import Data.Typeable (Typeable) import Data.Word (Word8) import Foreign.C.String (CString) import Foreign.C.Types (CInt(..)) import Foreign.ForeignPtr (withForeignPtr)+import Foreign.Marshal.Alloc (alloca) import Foreign.Marshal.Utils (with) import Foreign.Ptr (Ptr, nullPtr)+import Foreign.Storable (peek)  -- $api --@@ -90,7 +95,7 @@ -- collation, when it is used to distinguish between Katakana and Hiragana -- (this is achieved by setting 'HiraganaQuaternaryMode' mode to -- 'True'). Otherwise, quaternary level is affected only by the number of--- non ignorable code points in the string. Identical strength is rarely+-- non ignorable codepoints in the string. Identical strength is rarely -- useful, as it amounts to codepoints of the 'NFD' form of the string. data Strength = Primary               | Secondary@@ -156,6 +161,7 @@  type UColAttribute = CInt type UColAttributeValue = CInt+type UCollationStrength = UColAttributeValue  toUAttribute :: Attribute -> (UColAttribute, UColAttributeValue) toUAttribute (French v)@@ -179,6 +185,11 @@ toOO False = #const UCOL_OFF toOO True  = #const UCOL_ON +toDefaultOO :: (Maybe Bool) -> UColAttributeValue+toDefaultOO (Just False) = #const UCOL_OFF+toDefaultOO (Just True)  = #const UCOL_ON+toDefaultOO Nothing  = #const UCOL_DEFAULT+ toAH :: AlternateHandling -> UColAttributeValue toAH NonIgnorable = #const UCOL_NON_IGNORABLE toAH Shifted      = #const UCOL_SHIFTED@@ -195,6 +206,10 @@ toS Quaternary = #const UCOL_QUATERNARY toS Identical  = #const UCOL_IDENTICAL +toDefaultS :: Maybe Strength -> UColAttributeValue+toDefaultS (Just s)    = toS s+toDefaultS Nothing  = #const UCOL_DEFAULT_STRENGTH+ fromOO :: UColAttributeValue -> Bool fromOO (#const UCOL_OFF) = False fromOO (#const UCOL_ON)  = True@@ -244,6 +259,27 @@      -> IO MCollator open loc = wrap $ withLocaleName loc (handleError . ucol_open) +-- | Produce a 'Collator' instance according to the rules supplied.+openRules :: Text+          -- ^ A string describing the collation rules.+          -> Maybe Bool+          -- ^ The normalization mode: One of 'Just False' (expect the text to not need normalization)+          -- 'Just True' (normalize), or 'Nothing' (set the mode according to the rules)+          -> Maybe Strength+          -- ^ The default collation strength; one of 'Just Primary', 'Just Secondary', 'Just Tertiary', 'Just Identical', 'Nothing' (default strength) - can be also set in the rules.+          -> IO MCollator+openRules r n s = wrap $ useAsUCharPtr r $ \rPtr rLen -> do+  let len = fromIntegral rLen+  handleParseError (== u_INVALID_FORMAT_ERROR) $ ucol_openRules rPtr len (toDefaultOO n) (toDefaultS s)++-- | Get the rules of an 'MCollator' attribute.+getRules :: MCollator -> IO Text+getRules c =+  withCollator c $ \cPtr ->+    alloca $ \lenPtr -> do+      textPtr <- ucol_getRules cPtr lenPtr+      (fromUCharPtr textPtr . fromIntegral) =<< peek lenPtr+ -- | Set the value of an 'MCollator' attribute. setAttribute :: MCollator -> Attribute -> IO () setAttribute c a =@@ -328,8 +364,14 @@ foreign import ccall unsafe "hs_text_icu.h __hs_ucol_open" ucol_open     :: CString -> Ptr UErrorCode -> IO (Ptr UCollator) +foreign import ccall unsafe "hs_text_icu.h __hs_ucol_openRules" ucol_openRules+    :: Ptr UChar -> Int32 -> UColAttributeValue -> UCollationStrength -> Ptr UParseError -> Ptr UErrorCode -> IO (Ptr UCollator)+ foreign import ccall unsafe "hs_text_icu.h __hs_ucol_getAttribute" ucol_getAttribute     :: Ptr UCollator -> UColAttribute -> Ptr UErrorCode -> IO UColAttributeValue++foreign import ccall unsafe "hs_text_icu.h __hs_ucol_getRules" ucol_getRules+    :: Ptr UCollator -> Ptr Int32 -> IO (Ptr UChar)  foreign import ccall unsafe "hs_text_icu.h __hs_ucol_setAttribute" ucol_setAttribute     :: Ptr UCollator -> UColAttribute -> UColAttributeValue -> Ptr UErrorCode -> IO ()

@@ -1,4 +1,4 @@-{-# LANGUAGE DeriveDataTypeable, ForeignFunctionInterface #-}+{-# LANGUAGE DeriveDataTypeable, ForeignFunctionInterface, ScopedTypeVariables  #-} -- | -- Module      : Data.Text.ICU.Collate.Pure -- Copyright   : (c) 2010 Bryan O'Sullivan@@ -22,15 +22,20 @@       Collator     , collator     , collatorWith+    , collatorFromRules+    , collatorFromRulesWith     , collate     , collateIter+    , rules     , sortKey     , uca     ) where +import qualified Control.Exception as E import Control.Monad (forM_) import Data.ByteString (ByteString) import Data.Text (Text)+import Data.Text.ICU.Error.Internal (ParseError(..)) import Data.Text.ICU.Collate.Internal (Collator(..)) import Data.Text.ICU.Internal (CharIterator, LocaleName(..)) import System.IO.Unsafe (unsafePerformIO)@@ -52,6 +57,25 @@   mc <- IO.open loc   forM_ atts $ IO.setAttribute mc   return (C mc)++-- | Create an immutable 'Collator' from the given collation rules.+collatorFromRules :: Text -> Either ParseError Collator+collatorFromRules rul = collatorFromRulesWith rul []++-- | Create an immutable 'Collator' from the given collation rules with the given 'Attribute's.+collatorFromRulesWith :: Text -> [IO.Attribute] -> Either ParseError Collator+collatorFromRulesWith rul atts = unsafePerformIO $+  (Right `fmap` openAndSetAtts)+  `E.catch` \(err::ParseError) -> return (Left err)+  where+    openAndSetAtts = do+      mc <- IO.openRules rul Nothing Nothing+      forM_ atts $ IO.setAttribute mc+      return (C mc)++-- | Get rules for the given 'Collator'.+rules :: Collator -> Text+rules (C c) = unsafePerformIO $ IO.getRules c  -- | Compare two strings. collate :: Collator -> Text -> Text -> Ordering

@@ -104,7 +104,7 @@     _ -> return ()   return c --- | Encode a Unicode string into a codepage string using the given converter.+-- | Encode a Unicode string into a code page string using the given converter. fromUnicode :: Converter -> Text -> ByteString fromUnicode cnv t =   unsafePerformIO . useAsPtr t $ \tptr tlen ->@@ -138,7 +138,7 @@  -- | Determines whether the converter uses fallback mappings or not. -- This flag has restrictions.  Regardless of this flag, the converter--- will always use fallbacks from Unicode Private Use code points, as+-- will always use fallbacks from Unicode Private Use codepoints, as -- well as reverse fallbacks (to Unicode).  For details see \".ucm -- File Format\" in the Conversion Data chapter of the ICU User Guide: -- <http://www.icu-project.org/userguide/conversion-data.html#ucmformat>

@@ -243,7 +243,7 @@                                 (toNM mode)  -- | Compare two strings for canonical equivalence.  Further options--- include case-insensitive comparison and code point order (as+-- include case-insensitive comparison and codepoint order (as -- opposed to code unit order). -- -- Canonical equivalence between two strings is defined as their

@@ -223,7 +223,7 @@ -- | Create an NFC normalizer and apply this to the given text. -- -- Let's have a look at a concrete example that contains the letter a with an acute accent twice.--- First as a comination of two codepoints and second as a canonical composite or precomposed+-- First as a combination of two codepoints and second as a canonical composite or precomposed -- character. Both look exactly the same but one character consists of two and one of only one -- codepoint. A bytewise comparison does not give equality of these. --@@ -370,7 +370,7 @@     where a `orO` b = a .|. fromCompareOption b  -- | Compare two strings for canonical equivalence. Further options--- include case-insensitive comparison and code point order (as+-- include case-insensitive comparison and codepoint order (as -- opposed to code unit order). -- -- Canonical equivalence between two strings is defined as their

@@ -51,7 +51,7 @@ -- can be completely independent of the locale conventions for decimal points, -- thousands-separators, or even the particular decimal digits used, or whether -- the number format is even decimal. There are different number format styles--- like decimal, currency, percent and spellout.+-- like decimal, currency, percent and spelled-out. -- -- Use 'formatter' to create a formatter and 'format' to format numbers. @@ -103,7 +103,7 @@ toNFS NUM_DEFAULT = #const UNUM_DEFAULT toNFS NUM_IGNORE = #const UNUM_IGNORE --- | Create and return a new MumberFormat for formatting and parsing numbers.+-- | Create and return a new NumberFormat for formatting and parsing numbers. -- -- A NumberFormat may be used to format numbers by calling unum_format, and -- to parse numbers by calling unum_parse. The caller must call unum_close when

@@ -107,7 +107,7 @@     --     -- By default, the matching time is not limited.     | StackLimit Int-    -- ^ Set the amount of heap storage avaliable for use by the match+    -- ^ Set the amount of heap storage available for use by the match     -- backtracking stack.     --     -- ICU uses a backtracking regular expression engine, with the

@@ -56,7 +56,7 @@   | LettersUnshape   -- ^ Letter shaping option: replace "shaped" letter characters by abstract ones.   | LettersShapeTashkeelIsolated-  -- ^ The only difference with LettersShape is that Tashkeel letters are always "shaped" into the isolated form instead of the medial form (selecting code points from the Arabic Presentation Forms-B block).+  -- ^ The only difference with LettersShape is that Tashkeel letters are always "shaped" into the isolated form instead of the medial form (selecting codepoints from the Arabic Presentation Forms-B block).   | PreservePresentation   -- ^ Presentation form option: Don't replace Arabic Presentation Forms-A and Arabic Presentation Forms-B characters with 0+06xx characters, before shaping.   | TextDirectionVisualLTR@@ -85,7 +85,7 @@  -- | Shape Arabic text on a character basis. ----- Text-based shaping means that some character code points in the text are replaced by+-- Text-based shaping means that some character codepoints in the text are replaced by -- others depending on the context. It transforms one kind of text into another. -- In comparison, modern displays for Arabic text select appropriate, context-dependent font -- glyphs for each text element, which means that they transform text into a glyph vector.

@@ -238,9 +238,21 @@     return ucol_open(loc, status); } +UCollator* __hs_ucol_openRules(const UChar *rules, int32_t rulesLength,+                               UColAttributeValue normalizationMode, UCollationStrength strength,+                               UParseError *parseError, UErrorCode *status)+{+  return ucol_openRules(rules, rulesLength, normalizationMode, strength, parseError, status);+}+ void __hs_ucol_close(UCollator *coll) {     ucol_close(coll);+}++const UChar *__hs_ucol_getRules(const UCollator *coll, int32_t *length)+{+  return ucol_getRules(coll, length); }  void __hs_ucol_setAttribute(UCollator *coll, UColAttribute attr,

@@ -1,3 +1,7 @@+0.8.0.2++* Support for creating a collator from custom rules (#76)+ 0.8.0.1  * Restore build with GHC 7.10 - 8.8 (#61)

@@ -114,7 +114,11 @@ /* ucol.h */  UCollator *__hs_ucol_open(const char *loc, UErrorCode *status);+UCollator* __hs_ucol_openRules(const UChar *rules, int32_t rulesLength,+                               UColAttributeValue normalizationMode, UCollationStrength strength,+                               UParseError *parseError, UErrorCode *status); void __hs_ucol_close(UCollator *coll);+const UChar *__hs_ucol_getRules(const UCollator *coll, int32_t *length); void __hs_ucol_setAttribute(UCollator *coll, UColAttribute attr,                             UColAttributeValue value, UErrorCode *status); UColAttributeValue __hs_ucol_getAttribute(const UCollator *coll,

@@ -13,7 +13,7 @@ import Data.Function (on) import Data.Maybe (fromMaybe) import Data.Text (Text)-import Data.Text.ICU (LocaleName(..))+import Data.Text.ICU (LocaleName(..), ParseError(..)) import QuickCheckUtils (NonEmptyText(..), LatinSpoofableText(..),                         NonSpoofableText(..), Utf8Text(..)) import Data.Text.ICU.Normalize2 (NormalizationMode(..))@@ -21,8 +21,8 @@ import Test.Framework (Test, testGroup) import Test.Framework.Providers.QuickCheck2 (testProperty) import Test.Framework.Providers.HUnit (hUnitTestToTests)-import Test.HUnit ((~?=), (@?=))-import qualified Test.HUnit (Test(..))+import Test.HUnit ((~?=), (@?=), (~:))+import qualified Test.HUnit (Test(..), assertFailure) import Test.QuickCheck.Monadic (monadicIO, run, assert) import qualified Data.Text as T import qualified Data.Text.Encoding as T@@ -32,6 +32,7 @@ import qualified Data.Text.ICU.Convert as I import qualified Data.Text.ICU.Char as I import qualified Data.Text.ICU.CharsetDetection as CD+import qualified Data.Text.ICU.Error as Err import qualified Data.Text.ICU.Number as N import qualified Data.Text.ICU.Shape as S import System.IO.Unsafe (unsafePerformIO)@@ -80,6 +81,12 @@ t_collate a b = c a b == flipOrdering (c b a)     where c = I.collate I.uca +t_collate_emptyRule a b = I.collate cUca a b == I.collate cEmpty a b+  where+    cUca = I.uca+    cEmpty = either (error "Can’t create empty collator") id+      $ I.collatorFromRules ""+ flipOrdering :: Ordering -> Ordering flipOrdering = \ case     GT -> LT@@ -135,6 +142,7 @@   , testProperty "t_charIterator_Utf8" t_charIterator_Utf8   , testProperty "t_quickCheck_isNormalized" t_quickCheck_isNormalized   , testProperty "t_collate" t_collate+  , testProperty "t_collate_emptyRule" t_collate_emptyRule   , testProperty "t_convert" t_convert   , testProperty "t_blockCode" t_blockCode   , testProperty "t_charFullName" t_charFullName@@ -209,6 +217,8 @@     <$> I.numberFormatter "precision-currency-cash currency/EUR" (Locale "it"))    `ioEq` "12.345,68\160€" +  , Test.HUnit.TestLabel "collate" testCases_collate+   ]   <>   concat@@ -225,3 +235,21 @@         ioEq io a = Test.HUnit.TestCase $ do             x <- io             x @?= a+++testCases_collate :: Test.HUnit.Test+testCases_collate = Test.HUnit.TestList $+  [ Test.HUnit.TestLabel "invalid format" $+    assertParseError (I.collatorFromRules "& a < <") Err.u_INVALID_FORMAT_ERROR (Just 0) (Just 4)+  , Test.HUnit.TestLabel "custom collator" $ Test.HUnit.TestCase $ do+      let c = either (error "Can’t create b<a collator") id+              $ I.collatorFromRules "& b < a"+      I.collate c "a" "b" @?= GT+  ]+  where+    assertParseError (Left e) err line offset = Test.HUnit.TestList+      [ "errError" ~: errError e ~?= err+      , "errLine" ~: errLine e ~?= line+      , "errOffset" ~: errOffset e ~?= offset+      ]+    assertParseError (Right _) _ _ _ = Test.HUnit.TestCase $ Test.HUnit.assertFailure "Expects a Left"

@@ -1,5 +1,5 @@ name:           text-icu-version:        0.8.0.1+version:        0.8.0.2 synopsis:       Bindings to the ICU library homepage:       https://github.com/haskell/text-icu bug-reports:    https://github.com/haskell/text-icu/issues@@ -66,7 +66,7 @@   build-depends:     base >= 4.8 && < 5,     bytestring,-    deepseq >= 1.4.3.0,+    deepseq >= 1.4.2.0,     text >=0.9.1.0 && <1.3 || >=2.0 && <2.1,     time >=1.5 && <1.13   pkgconfig-depends: icu-i18n >= 62.1@@ -145,9 +145,9 @@     HUnit >= 1.2,     QuickCheck >= 2.4,     array,-    base >= 4 && < 5,+    base,     bytestring,-    deepseq >= 1.4.3.0,+    deepseq,     directory,     ghc-prim,     random,