unicode-transforms 0.3.8 → 0.4.0.1
raw patch · 8 files changed
Files
- Changelog.md +12/−0
- Data/Unicode/Internal/NormalizeStream.hs +14/−4
- README.md +1/−1
- download-ucd-files.sh +2/−2
- test/NormalizationTest.hs +3/−1
- test/Properties.hs +6/−1
- test/data/NormalizationTest.txt too large to diff
- unicode-transforms.cabal +29/−29
Changelog.md view
@@ -1,3 +1,15 @@+## 0.4.0.1 (March 2022)++* Support text-icu == 0.8.* in tests+* Support unicode-data == 0.3.*++## 0.4.0 (November 2021)++* Bump `unicode-data` to `0.2`+* Allow `text 2.0`+* Update to Unicode version 14.0.0+* Drop support for GHC 7.10+ ## 0.3.8 * Allow ghc-prim 0.7
Data/Unicode/Internal/NormalizeStream.hs view
@@ -1,5 +1,6 @@ {-# OPTIONS_GHC -funbox-strict-fields #-} {-# LANGUAGE BangPatterns #-}+{-# LANGUAGE CPP #-} {-# LANGUAGE LambdaCase #-} {-# LANGUAGE TupleSections #-} -- |@@ -22,7 +23,6 @@ ) where -import Data.Bits (shiftR) import Data.Char (chr, ord) import GHC.ST (ST(..)) import GHC.Types (SPEC(..))@@ -30,13 +30,21 @@ import qualified Data.Text.Array as A import qualified Unicode.Char as UC +#if MIN_VERSION_text(2,0,0)+import Data.Text.Internal.Fusion (stream)+#else+import Data.Bits (shiftR)+import Data.Text.Internal.Unsafe.Char (unsafeChr)+import Data.Text.Internal.Fusion.Size (betweenSize)+import Data.Text.Internal.Encoding.Utf16 (chr2)+#endif+ -- Internal modules import Data.Text.Internal (Text(..))-import Data.Text.Internal.Fusion.Size (betweenSize, upperBound)+import Data.Text.Internal.Fusion.Size (upperBound) import Data.Text.Internal.Fusion.Types (Step(..), Stream(..)) import Data.Text.Internal.Private (runText)-import Data.Text.Internal.Unsafe.Char (unsafeChr, unsafeWrite)-import Data.Text.Internal.Encoding.Utf16 (chr2)+import Data.Text.Internal.Unsafe.Char (unsafeWrite) ------------------------------------------------------------------------------- -- Reorder buffer to hold characters till the next starter boundary@@ -142,6 +150,7 @@ n <- unsafeWrite arr j c return (j + n, Empty) +#if !MIN_VERSION_text(2,0,0) -- | /O(n)/ Convert a 'Text' into a 'Stream Char'. stream :: Text -> Stream Char stream (Text arr off len) = Stream next off (betweenSize (len `shiftR` 1) len)@@ -158,6 +167,7 @@ n = A.unsafeIndex arr i n2 = A.unsafeIndex arr (i + 1) {-# INLINE [0] stream #-}+#endif -- | /O(n)/ Convert a 'Stream Char' into a decompose-normalized 'Text'. unstream :: UC.DecomposeMode -> Stream Char -> Text
README.md view
@@ -5,7 +5,7 @@ [](https://ci.appveyor.com/project/harendra-kumar/unicode-transforms) [](https://coveralls.io/github/composewell/unicode-transforms?branch=master) -Fast Unicode 13.0.0 normalization in Haskell (NFC, NFKC, NFD, NFKD).+Fast Unicode 14.0.0 normalization in Haskell (NFC, NFKC, NFD, NFKD). ## What is normalization?
download-ucd-files.sh view
@@ -4,11 +4,11 @@ # we used to generate them earlier are exactly the same as the ones we are # downloading. To ensure that verfication of the checksum is necessary. -VERSION=13.0.0+VERSION=14.0.0 SRC_FILE="https://www.unicode.org/Public/$VERSION/ucd/NormalizationTest.txt" DST_FILE="test/data/NormalizationTest.txt"-SRC_FILE_CKSUM="d60ee55dd9169444652e48d337109cc814ecc59a9d3122eedddf7de388f2e01d"+SRC_FILE_CKSUM="7cb30cc2abe6c29c292b99095865d379ce1213045c78c4ff59c7e9391bbe2331" # Download the file
test/NormalizationTest.hs view
@@ -12,6 +12,7 @@ -- import Control.Monad (when)+import qualified Data.ByteString as B import Data.Char (chr, isSpace, ord, toUpper) #if MIN_VERSION_base(4,8,0) import Data.Function ((&))@@ -20,6 +21,7 @@ import Data.List.Split (splitOn) import Data.Text (Text) import qualified Data.Text as T+import qualified Data.Text.Encoding as T import Data.Text.Normalize (NormalizationMode(NFD, NFKD, NFC, NFKC), normalize) import Text.Printf (printf) @@ -109,7 +111,7 @@ testNormalize :: FilePath -> IO () testNormalize file = do- contents <- readFile file+ contents <- T.unpack . T.decodeUtf8 <$> B.readFile file let ls = lines contents -- split into lines & map (dropWhile isSpace) -- trim leading spaces & zip [1..] -- add numbering
test/Properties.hs view
@@ -15,7 +15,12 @@ #ifdef HAS_ICU import Data.Text (pack)-import qualified Data.Text.ICU as ICU++#if MIN_VERSION_text_icu(0,8,0)+import qualified Data.Text.ICU.Normalize2 as ICU+#else+import qualified Data.Text.ICU.Normalize as ICU+#endif toICUMode :: NormalizationMode -> ICU.NormalizationMode toICUMode mode =
test/data/NormalizationTest.txt view
file too large to diff
unicode-transforms.cabal view
@@ -1,9 +1,9 @@ cabal-version: 2.2 name: unicode-transforms-version: 0.3.8+version: 0.4.0.1 synopsis: Unicode normalization-description: Fast Unicode 13.0.0 normalization in Haskell (NFC, NFKC, NFD, NFKD).+description: Fast Unicode 14.0.0 normalization in Haskell (NFC, NFKC, NFD, NFKD). category: Data,Text,Unicode stability: Experimental homepage: http://github.com/composewell/unicode-transforms@@ -14,14 +14,14 @@ 2014–2015 Antonio Nikishaev license: BSD-3-Clause license-file: LICENSE-tested-with: GHC==7.10.3- , GHC==8.0.2+tested-with: GHC==8.0.2 , GHC==8.2.2 , GHC==8.4.4 , GHC==8.6.5 , GHC==8.8.4- , GHC==8.10.3+ , GHC==8.10.7 , GHC==9.0.1+ , GHC==9.2.1 build-type: Simple extra-source-files: Changelog.md@@ -82,14 +82,14 @@ hs-source-dirs: . ghc-options: -Wall -fwarn-identities -fwarn-incomplete-record-updates -fwarn-incomplete-uni-patterns -fwarn-tabs build-depends:- base >=4.8 && <5- , unicode-data >= 0.1 && <0.2- , bytestring >=0.9 && <0.12- , ghc-prim >= 0.2 && < 0.8+ base >= 4.8 && < 4.17+ , unicode-data >= 0.2 && < 0.4+ , bytestring >= 0.9 && < 0.12+ , ghc-prim >= 0.2 && < 0.9 -- We depend on a lot of internal modules in text. We keep the upper bound -- inclusive of the latest stable version.- , text >= 1.1.1 && <= 1.2.4.1+ , text >=1.1.1 && <=1.2.5.0 || >=2.0 && <2.1 if flag(dev) ghc-options: -O0 else@@ -108,9 +108,9 @@ ghc-options: -Wall -fwarn-identities -fwarn-incomplete-record-updates -fwarn-incomplete-uni-patterns -fwarn-tabs build-depends: QuickCheck >=2.1 && <2.15- , base >=4.7 && <5+ , base , deepseq >=1.1 && <1.5- , text >=1.1.1 && <1.3+ , text , unicode-transforms if flag(dev) ghc-options: -O0@@ -129,12 +129,12 @@ test ghc-options: -Wall -fwarn-identities -fwarn-incomplete-record-updates -fwarn-incomplete-uni-patterns -fwarn-tabs build-depends:- QuickCheck >=2.1 && <2.15- , unicode-data >= 0.1 && <0.2- , base >=4.7 && <5- , deepseq >=1.1 && <1.5- , hspec >= 2.0 && < 3- , text >=1.1.1 && <1.3+ QuickCheck+ , unicode-data+ , base+ , deepseq+ , hspec >= 2.0 && < 3+ , text , unicode-transforms if flag(dev) ghc-options: -O0@@ -145,7 +145,7 @@ if flag(has-icu) cpp-options: -DHAS_ICU build-depends:- text-icu >=0.6.2.1+ text-icu >=0.6.2.1 && <0.9 default-language: Haskell2010 test-suite ucd@@ -155,10 +155,10 @@ test ghc-options: -Wall -fwarn-identities -fwarn-incomplete-record-updates -fwarn-incomplete-uni-patterns -fwarn-tabs build-depends:- base >=4.7 && <5- , getopt-generics >=0.11 && <0.14- , split >=0.1 && <0.3- , text >=1.1.1 && <1.3+ base+ , bytestring+ , split >=0.2 && <0.3+ , text , unicode-transforms if flag(dev) ghc-options: -O0@@ -175,12 +175,12 @@ benchmark ghc-options: -Wall -fwarn-identities -fwarn-incomplete-record-updates -fwarn-incomplete-uni-patterns -fwarn-tabs build-depends:- base >=4.7 && <5- , deepseq >=1.1.0 && <1.5+ base+ , deepseq , filepath >=1.0 && <2 , path >=0.0.0 && <0.9 , path-io >=0.1.0 && <1.7- , text >=1.1.1 && <1.3+ , text , unicode-transforms if flag(use-gauge) build-depends: gauge >=0.2.0 && <0.3@@ -196,7 +196,7 @@ if flag(has-icu) cpp-options: -DBENCH_ICU build-depends:- text-icu >=0.6.2.1 && <0.8+ text-icu if impl(ghc < 7.10) build-depends: path (<0.5.12 || >0.5.12)@@ -209,9 +209,9 @@ if flag(bench-show) buildable: True build-Depends:- base >= 4.8 && < 5+ base , bench-show >= 0.3 && < 0.4- , split >= 0.2 && < 0.3+ , split , transformers >= 0.4 && < 0.6 else buildable: False