text-icu-normalized 0.3.0 → 0.4.1
raw patch · 3 files changed
+58/−45 lines, 3 filesdep ~lens
Dependency ranges changed: lens
Files
- Source/Library/Data/Text/ICU/Normalized/NFC.hs +31/−30
- Tests/TestNormalization.hs +7/−5
- text-icu-normalized.cabal +20/−10
Source/Library/Data/Text/ICU/Normalized/NFC.hs view
@@ -27,27 +27,36 @@ import qualified Data.Text.ICU.Normalize as ICU import Data.Text (Text)-import qualified Data.Text.Lazy as LT import GHC.Generics (Generic)-import Data.Text.Encoding (encodeUtf8, decodeUtf8)-import qualified Data.ByteString.Char8 as C8-import Data.String (IsString(..))-import Text.ParserCombinators.ReadP-import Text.Read hiding (get) import Prelude.Unicode import Data.Data-import Data.Typeable import Control.Lens hiding (strict, lazy)-import Data.ByteString (ByteString)+import Data.Text.Lazy (toStrict)+import Data.Text.Lazy.Builder (toLazyText) import qualified Data.Text.Strict.Lens as DL import Data.Text.Lens (IsText(..))+import Control.Applicative -- | An NFC-normalized Data.Text newtype NFCText = NFCText {unNFC ∷ Text} deriving (Show, Generic, Data, Typeable) -normalized ∷ Iso' NFCText Text-normalized = iso (unNFC) (nnfc)+-- | Since we're NFC, we satisfy FCD, and we can just compare+-- directly.+instance Ord NFCText where+ (NFCText a) `compare` (NFCText b) =+ let+ cmp = ICU.compare [ICU.InputIsFCD]+ in+ a `cmp` b +-- | If the Ord of the two texts is EQ, then they're equal.+instance Eq NFCText where+ a == b = a `compare` b == EQ++-- | 'Iso' between an 'IsText' and an 'NFCText'.+normalized ∷ (IsText t) ⇒ Iso' NFCText t+normalized = iso (review builder ∘ view builder ∘ unNFC) nnfc+ instance IsText NFCText where -- | 'Iso' for packing and unpacking 'NFCText' from 'String'. packed = iso (nnfc ∘ view DL.packed) (review DL.packed ∘ unNFC)@@ -55,36 +64,28 @@ -- | 'Iso' between a 'Builder' and an 'NFCText' builder = iso (view DL.builder ∘ unNFC) (nnfc ∘ review DL.builder) --- | Normalizes Text to 'NFCText'+-- | Normalizes an 'IsText' to 'NFCText'+-- -- 1. Do a 'quickCheck'. If this returns definitively, we normalize -- based on its response. -- 2. If not, we call 'isNormalized', and normalize based on its -- response.-nnfc t =+nnfc ∷ (IsText t) ⇒ t → NFCText+nnfc txt = let nt = ICU.normalize ICU.NFC + -- Make sure we start with a strict 'Text'.+ t = toStrict $ toLazyText $ txt ^. builder+ -- | If 'qc' fails to provide an answer we use 'isNormalized'.- isn = if ICU.isNormalized ICU.NFC t then t else nt t+ isn = Just $ if ICU.isNormalized ICU.NFC t then t else nt t -- | If we get a definitive answer from qc, act on it.- quick t = ICU.quickCheck ICU.NFC t >>=+ quick = ICU.quickCheck ICU.NFC t >>= \q → if q then return t else return $ nt t- in- NFCText $ case quick t of- Nothing → isn- Just q → q --- | Since we're NFC, we satisfy FCD, and we can just compare--- directly.-instance Ord NFCText where- (NFCText a) `compare` (NFCText b) =- let- cmp = ICU.compare [ICU.InputIsFCD]- in- a `cmp` b---- | If the Ord of the two texts is EQ, then they're equal.-instance Eq NFCText where- a == b = a `compare` b == EQ+ norm = (quick <|> isn) ^?! _Just+ in+ NFCText norm
Tests/TestNormalization.hs view
@@ -5,15 +5,15 @@ Description: Test normalizing various cases of non-normalized UTF8. Copyright: © 2016 All rights reserved. License: GPL-3-Maintainer: Evan Cofsky <>+Maintainer: Evan Cofsky <evan@theunixman.com> Stability: experimental Portability: POSIX -} module TestNormalization where -import Test.HUnit-import qualified Data.Map as M+import Test.Framework (Test)+import Test.HUnit hiding (Test) import Control.Lens hiding (strict) import ParseNormalizationTest import Data.Text.ICU.Normalized.NFC@@ -21,13 +21,15 @@ import qualified Data.Text as T import Test.Framework.Providers.HUnit +normalizationTests ∷ IO [Test] normalizationTests = do (Right tcs) ← loadTestCodePoints return $ concatMapOf traversed (\p → over traversed (testNormalization p) (p ^. pTestCodePoints)) tcs -testNormalization part tcp =- testCase ((T.unwords [(part ^. pPartComment),+testNormalization ∷ Part → TestCodePoint → Test+testNormalization p tcp =+ testCase ((T.unwords [(p ^. pPartComment), (tcp ^. tcComment)]) ^. unpacked) ((tcp ^. tcNFC) @?= (tcp ^. tcSource ^.re normalized ^. normalized))
text-icu-normalized.cabal view
@@ -1,8 +1,5 @@--- Initial text-icu-normalized.cabal generated by cabal init. For further--- documentation, see http://haskell.org/cabal/users-guide/- name: text-icu-normalized-version: 0.3.0+version: 0.4.1 synopsis: Dealing with Strict Text in NFC normalization. description: Handle Text in NFC normalization. homepage: https://www.lambdanow.us/browser/text-icu-normalized@@ -13,18 +10,22 @@ copyright: ©2016 Evan Cofsky category: Data, Text build-type: Simple--- extra-source-files:-cabal-version: >=1.10+cabal-version: >=1.24 data-files: NormalizationTest.txt source-repository head type: git- location: git+ssh://lambdanow.us/projects/haskellnow/text-icu-normalized.git+ location: git@gitlab.com:theunixman/text-icu-normalized.git +source-repository this+ type: git+ location: git@gitlab.com:theunixman/text-icu-normalized.git+ tag: v0.4.1+ library+ ghc-options: -Wall -Wno-redundant-constraints -Wno-type-defaults -O2 exposed-modules: Data.Text.ICU.Normalized.NFC- -- other-modules: default-extensions: DeriveDataTypeable DeriveGeneric@@ -33,13 +34,22 @@ base >=4.9 && <5.0, base-unicode-symbols >= 0.2.2 && < 0.3, bytestring >= 0.10.8 && < 0.11,- lens >= 4.14 && < 4.15,+ lens >= 4.14 && < 4.16, text >= 1.2.2 && < 1.3, text-icu >= 0.7.0 && < 0.8 hs-source-dirs: Source/Library default-language: Haskell2010 test-suite test-normalizations+ ghc-options:+ -Wall+ -Wno-redundant-constraints+ -Wno-type-defaults+ -threaded+ -feager-blackholing+ -rtsopts+ -dynamic+ -with-rtsopts=-N default-language: Haskell2010 type: exitcode-stdio-1.0 hs-source-dirs: Tests@@ -56,7 +66,7 @@ containers >= 0.5.7.1 && < 0.6, exceptions >= 0.8.3 && < 0.9, filepath >= 1.4.0.0 && < 1.5,- lens >= 4.14 && < 4.15,+ lens >= 4.14 && < 4.16, parsec, test-framework, test-framework-hunit,