text-icu-normalized 0.1.6 → 0.3.0
raw patch · 5 files changed
+140/−55 lines, 5 filesdep +HUnitdep +QuickCheckdep +containersdep ~basedep ~base-unicode-symbolsdep ~bytestring
Dependencies added: HUnit, QuickCheck, containers, exceptions, filepath, parsec, test-framework, test-framework-hunit, test-framework-quickcheck2, test-framework-th, text-icu-normalized
Dependency ranges changed: base, base-unicode-symbols, bytestring, lens, text, text-icu
Files
- NormalizationTest.txt too large to diff
- Source/Library/Data/Text/ICU/Normalized/NFC.hs +37/−44
- Tests/Main.hs +11/−0
- Tests/TestNormalization.hs +33/−0
- text-icu-normalized.cabal +59/−11
+ NormalizationTest.txt view
file too large to diff
Source/Library/Data/Text/ICU/Normalized/NFC.hs view
@@ -2,7 +2,7 @@ {-| Module : Data.Text.ICU.Normalized.NFC-Description: : Convert all Text to Unicode NFC normalized Strict Text.+Description: : NFC Text implementing the 'IsString' typeclass. Copyright : ©2016 License : GPL-3 Maintainer : Evan Cofsky <evan@theunixman.com>@@ -19,15 +19,13 @@ module Data.Text.ICU.Normalized.NFC ( NFCText,- strict,- lazy,- lower,- upper,- utf8,- utf8ByteString+ packed,+ builder,+ text,+ normalized ) where -import qualified Data.Text.ICU as ICU+import qualified Data.Text.ICU.Normalize as ICU import Data.Text (Text) import qualified Data.Text.Lazy as LT import GHC.Generics (Generic)@@ -41,57 +39,52 @@ import Data.Typeable import Control.Lens hiding (strict, lazy) import Data.ByteString (ByteString)+import qualified Data.Text.Strict.Lens as DL+import Data.Text.Lens (IsText(..)) -- | An NFC-normalized Data.Text-newtype NFCText = NFCText Text deriving (Show, Generic, Data, Typeable)-makePrisms ''NFCText---- | Prism between 'NFCText' and strict 'Text'-strict ∷ Prism' NFCText Text-strict = prism' (NFCText . ICU.normalize ICU.NFC) (preview _NFCText)---- | Prism between 'NFCText' and lazy 'LT.Text'-lazy ∷ Prism' NFCText LT.Text-lazy = prism' (review strict ∘ LT.toStrict) (\t → LT.fromStrict <$> preview _NFCText t)---- | Prism between 'NFCText' and a utf-8 encoded 'ByteString-utf8ByteString ∷ Prism' NFCText ByteString-utf8ByteString = prism' (NFCText ∘ decodeUtf8) (Just ∘ encodeUtf8 ∘ view _NFCText)+newtype NFCText = NFCText {unNFC ∷ Text} deriving (Show, Generic, Data, Typeable) --- | Prism between 'NFCText' and a utf-8 encoded 'String'-utf8 :: Prism' NFCText String-utf8 = prism' (fromString) (Just ∘ C8.unpack ∘ view utf8ByteString)+normalized ∷ Iso' NFCText Text+normalized = iso (unNFC) (nnfc) --- | Convert 'NFCText' to lower case.-lower :: Getter NFCText NFCText-lower = to (NFCText ∘ ICU.toLower nfcLocale ∘ view _NFCText)+instance IsText NFCText where+ -- | 'Iso' for packing and unpacking 'NFCText' from 'String'.+ packed = iso (nnfc ∘ view DL.packed) (review DL.packed ∘ unNFC) --- | Convert 'NFCText' to upper case.-upper :: Getter NFCText NFCText-upper = to (NFCText ∘ ICU.toUpper nfcLocale ∘ view _NFCText)+ -- | 'Iso' between a 'Builder' and an 'NFCText'+ builder = iso (view DL.builder ∘ unNFC) (nnfc ∘ review DL.builder) -instance IsString NFCText where- fromString = NFCText . decodeUtf8 . C8.pack+-- | Normalizes Text to 'NFCText'+-- 1. Do a 'quickCheck'. If this returns definitively, we normalize+-- based on its response.+-- 2. If not, we call 'isNormalized', and normalize based on its+-- response.+nnfc t =+ let+ nt = ICU.normalize ICU.NFC -nfcParser ∷ ReadP NFCText-nfcParser =- NFCText . fromString <$> many get+ -- | If 'qc' fails to provide an answer we use 'isNormalized'.+ isn = if ICU.isNormalized ICU.NFC t then t else nt t -instance Read NFCText where- readPrec = lift nfcParser+ -- | If we get a definitive answer from qc, act on it.+ quick t = ICU.quickCheck ICU.NFC t >>=+ \q → if q then return t+ else return $ nt t+ in+ NFCText $ case quick t of+ Nothing → isn+ Just q → q --- | Convert to FCD form, then compare the texts.+-- | Since we're NFC, we satisfy FCD, and we can just compare+-- directly. instance Ord NFCText where (NFCText a) `compare` (NFCText b) = let- cmp = ICU.collate ICU.uca+ cmp = ICU.compare [ICU.InputIsFCD] in a `cmp` b -- | If the Ord of the two texts is EQ, then they're equal. instance Eq NFCText where a == b = a `compare` b == EQ---- | For now this is where we are.-nfcLocale :: ICU.LocaleName-nfcLocale = ICU.Locale "en_US.UTF8"
+ Tests/Main.hs view
@@ -0,0 +1,11 @@+module Main where++import Test.Framework (defaultMain)++import qualified TestNormalization as N++main :: IO ()+main = do+ nts ← N.normalizationTests++ defaultMain nts
+ Tests/TestNormalization.hs view
@@ -0,0 +1,33 @@+{-# OPTIONS_GHC -Wno-orphans #-}++{-|+Module: TestUTF8+Description: Test normalizing various cases of non-normalized UTF8.+Copyright: © 2016 All rights reserved.+License: GPL-3+Maintainer: Evan Cofsky <>+Stability: experimental+Portability: POSIX+-}++module TestNormalization where++import Test.HUnit+import qualified Data.Map as M+import Control.Lens hiding (strict)+import ParseNormalizationTest+import Data.Text.ICU.Normalized.NFC+import Data.Text.Lens (unpacked)+import qualified Data.Text as T+import Test.Framework.Providers.HUnit++normalizationTests = do+ (Right tcs) ← loadTestCodePoints+ return+ $ concatMapOf traversed (\p → over traversed (testNormalization p) (p ^. pTestCodePoints)) tcs+++testNormalization part tcp =+ testCase ((T.unwords [(part ^. pPartComment),+ (tcp ^. tcComment)]) ^. unpacked)+ ((tcp ^. tcNFC) @?= (tcp ^. tcSource ^.re normalized ^. normalized))
text-icu-normalized.cabal view
@@ -2,10 +2,10 @@ -- documentation, see http://haskell.org/cabal/users-guide/ name: text-icu-normalized-version: 0.1.6+version: 0.3.0 synopsis: Dealing with Strict Text in NFC normalization. description: Handle Text in NFC normalization.-homepage: https://gitlab.com/theunixman/text-icu-normalized+homepage: https://www.lambdanow.us/browser/text-icu-normalized license: GPL-3 license-file: LICENSE author: Evan Cofsky@@ -15,7 +15,12 @@ build-type: Simple -- extra-source-files: cabal-version: >=1.10+data-files: NormalizationTest.txt +source-repository head+ type: git+ location: git+ssh://lambdanow.us/projects/haskellnow/text-icu-normalized.git+ library exposed-modules: Data.Text.ICU.Normalized.NFC@@ -25,15 +30,58 @@ DeriveGeneric UnicodeSyntax build-depends:- base >=4.8 && <5.0,- base-unicode-symbols,- bytestring,- lens,- text,- text-icu+ base >=4.9 && <5.0,+ base-unicode-symbols >= 0.2.2 && < 0.3,+ bytestring >= 0.10.8 && < 0.11,+ lens >= 4.14 && < 4.15,+ text >= 1.2.2 && < 1.3,+ text-icu >= 0.7.0 && < 0.8 hs-source-dirs: Source/Library default-language: Haskell2010 -source-repository head- type: git- location: git@gitlab.com:theunixman/text-icu-normalized.git+test-suite test-normalizations+ default-language: Haskell2010+ type: exitcode-stdio-1.0+ hs-source-dirs: Tests+ main-is: Main.hs+ other-modules:+ Paths_text_icu_normalized+ TestNormalization+ build-depends:+ base >=4.9 && <5.0,+ HUnit,+ QuickCheck,+ base-unicode-symbols >= 0.2.2 && < 0.3,+ bytestring >= 0.10.8 && < 0.11,+ containers >= 0.5.7.1 && < 0.6,+ exceptions >= 0.8.3 && < 0.9,+ filepath >= 1.4.0.0 && < 1.5,+ lens >= 4.14 && < 4.15,+ parsec,+ test-framework,+ test-framework-hunit,+ test-framework-quickcheck2,+ test-framework-th,+ text >= 1.2.2 && < 1.3,+ text-icu >= 0.7.0 && < 0.8,+ text-icu-normalized+ default-extensions:+ UnicodeSyntax+ LambdaCase+ ConstraintKinds+ DefaultSignatures+ FlexibleContexts+ FlexibleInstances+ FunctionalDependencies+ GADTs+ DeriveGeneric+ DeriveDataTypeable+ GeneralizedNewtypeDeriving+ KindSignatures+ MultiParamTypeClasses+ OverloadedStrings+ PartialTypeSignatures+ RankNTypes+ ScopedTypeVariables+ TypeFamilies+ TypeSynonymInstances