Z-Data 0.1.4.0 → 0.1.4.1

raw patch · 14 files changed

+347/−240 lines, 14 filesdep +timesetup-changedPVP: major bump suggested

API removals or changes: PVP suggests a major version bump

Dependencies added: time

API changes (from Hackage documentation)

Files

• ChangeLog.md +6/−0
• LICENSE +48/−1
• README.md +1/−3
• Setup.hs +0/−2
• Z-Data.cabal +2/−1
• Z/Data/Builder/Base.hs +1/−1
• Z/Data/Parser.hs +13/−2
• Z/Data/Parser/Base.hs +1/−11
• Z/Data/PrimRef.hs +2/−0
• Z/Data/Text.hs +11/−63
• Z/Data/Text/Base.hs +127/−91
• Z/Data/Text/UTF8Rewind.hsc +110/−60
• Z/Foreign.hs +14/−1
• test/Z/Data/CBytesSpec.hs +11/−4

@@ -1,5 +1,11 @@ # Revision history for Z-Data +## 0.1.4.1  -- 2020-09-29++* Add `fromNullTerminated` to `Z.Foreign`(and `Z.Data.Vector.Base`).+* Change `Category` and `Locale`(`Z.Data.Text`) to use pattern synonyms, add `envLocale` to `Z.Data.Text`.+* Add `validateASCII` and `validateASCIIMaybe` to `Z.Data.Text`.+ ## 0.1.4.0  -- 2020-09-24  * Make library works with GHC 8.6 and 8.8 again.

@@ -1,5 +1,9 @@-Copyright (c) 2017-2020, Haskell Z Team+--------------------------------------------------------------------------------+Z-Data LICENSE +Copyright (c) Dong Han, 2017-2020+Copyright (c) Tao He, 2017-2019+ All rights reserved.  Redistribution and use in source and binary forms, with or without@@ -28,3 +32,46 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.++--------------------------------------------------------------------------------+utf8rewind LICENSE++Copyright (C) 2014-2016 Quinten Lansu+Copyright (C) 2019-2020 Dong Han++Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:++The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.++THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.++--------------------------------------------------------------------------------+fastvalidate-utf-8 LICENSE++Daniel Lemire+Kendall Willets+Zach Bjornson++Permission is hereby granted, free of charge, to any+person obtaining a copy of this software and associated+documentation files (the "Software"), to deal in the+Software without restriction, including without+limitation the rights to use, copy, modify, merge,+publish, distribute, sublicense, and/or sell copies of+the Software, and to permit persons to whom the Software+is furnished to do so, subject to the following+conditions:++The above copyright notice and this permission notice+shall be included in all copies or substantial portions+of the Software.++THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF+ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED+TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A+PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT+SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY+CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR+IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER+DEALINGS IN THE SOFTWARE.

@@ -4,9 +4,7 @@  ## Z-Data -[![Linux Build Status](https://github.com/haskell-Z/z-data/workflows/ubuntu-ci/badge.svg)](https://github.com/haskell-Z/z-data/actions)--[![MacOS Build Status](https://github.com/haskell-Z/z-data/workflows/osx-ci/badge.svg)](https://github.com/haskell-Z/z-data/actions)+[![Linux Build Status](https://github.com/haskell-Z/z-data/workflows/ubuntu-ci/badge.svg)](https://github.com/haskell-Z/z-data/actions) [![MacOS Build Status](https://github.com/haskell-Z/z-data/workflows/osx-ci/badge.svg)](https://github.com/haskell-Z/z-data/actions) [![Windows Build Status](https://github.com/haskell-Z/z-data/workflows/win-ci/badge.svg)](https://github.com/haskell-Z/z-data/actions)  This package provides basic data structures and functions: 

@@ -1,2 +0,0 @@-import Distribution.Simple-main = defaultMain

@@ -1,6 +1,6 @@ cabal-version:              2.4 name:                       Z-Data-version:                    0.1.4.0+version:                    0.1.4.1 synopsis:                   Array, vector and text description:                This package provides array, slice and text operations license:                    BSD-3-Clause@@ -121,6 +121,7 @@                           , template-haskell        >= 2.14.0                           , unordered-containers    == 0.2.*                           , tagged                  == 0.8.*+                          , time                    >= 1.8      if flag(integer-simple)         cpp-options:        -DINTEGER_SIMPLE

@@ -20,7 +20,7 @@ Portability : non-portable  A 'Builder' records a buffer writing function, which can be 'mappend' in O(1) via composition.-In stdio a 'Builder' are designed to deal with different 'AllocateStrategy', it affects how+In Z-Data a 'Builder' are designed to deal with different 'AllocateStrategy', it affects how 'Builder' react when writing across buffer boundaries:    * When building a short strict 'Bytes' with 'buildBytes/buildByteswith',

@@ -14,9 +14,20 @@ Stability   : experimental Portability : non-portable -This module provide a simple resumable 'Parser', which is suitable for binary protocol and simple textual protocol parsing.+This module provide a simple resumable 'Parser', which is suitable for binary protocol and simple textual protocol parsing. Both binary parsers ('decodePrim' ,etc) and textual parsers are provided, and they all work on 'V.Bytes'. -You can use 'Alternative' instance to do backtracking, each branch will either succeed and may consume some input, or fail without consume anything. It's recommend to use 'peek' to avoid backtracking if possible to get high performance.+You can use 'Alternative' instance to do backtracking, each branch will either succeed and may consume some input, or fail without consume anything. It's recommend to use 'peek' or 'peekMaybe' to avoid backtracking if possible to get high performance.++Error message can be attached using '<?>', which have very small overhead, so it's recommended to attach a message in front of a composed parser like @xPacket = "Foo.Bar.xPacket" <?> do ...@, following is an example message when parsing an integer failed:++@+    >parse int "foo"+    ([102,111,111],Left ["Z.Data.Parser.Numeric.int","Std.Data.Parser.Base.takeWhile1: no satisfied byte"])+    -- It's easy to see we're trying to match a leading sign or digit here+@++Use <https://hackage.haskell.org/package/parser-combinators parser-combinators> to get combinators based on+'Applicative' or 'Monad' instance, such as @manyTill@, @sepBy@, etc.  -} module Z.Data.Parser

@@ -15,17 +15,7 @@ Stability   : experimental Portability : non-portable -This module provide a simple resumable 'Parser', which is suitable for binary protocol and simple textual protocol parsing. Both binary parsers ('decodePrim' ,etc) and textual parsers are provided, and they all work on 'V.Bytes'.--You can use 'Alternative' instance to do backtracking, each branch will either succeed and may consume some input, or fail without consume anything. It's recommend to use 'peek' or 'peekMaybe' to avoid backtracking if possible to get high performance.--Error message can be attached using '<?>', which have very small overhead, so it's recommended to attach a message in front of a composed parser like @xPacket = "Foo.Bar.xPacket" <?> do ...@, following is an example message when parsing an integer failed:--@-    >parse int "foo"-    ([102,111,111],Left ["Z.Data.Parser.Numeric.int","Std.Data.Parser.Base.takeWhile1: no satisfied byte"])-    -- It's easy to see we're trying to match a leading sign or digit here-@+This module provide internal data types for a simple resumable 'Parser', which is suitable for binary protocol and simple textual protocol parsing. 'Parser' extensively works on on 'V.Bytes', which is same to 'T.Text' representation.  -} 

@@ -5,7 +5,9 @@ Maintainer  :  winterland1989@gmail.com Stability   :  experimental Portability :  portable+ This module provide fast unboxed references for ST and IO monad, and atomic operations for 'Counter' type. Unboxed reference is implemented using single cell MutableByteArray s to eliminate indirection overhead which MutVar# s a carry, on the otherhand unboxed reference only support limited type(instances of Prim class).+ -}  module Z.Data.PrimRef

@@ -7,22 +7,25 @@ {-| Module      : Z.Data.Text Description : Unicode text processing-Copyright   : (c) Dong Han, 2017-2018+Copyright   : (c) Dong Han, 2017-2020 License     : BSD Maintainer  : winterland1989@gmail.com Stability   : experimental Portability : non-portable -A 'Text' simply wraps a 'Bytes' that are UTF-8 encoded codepoints, you can use 'validate' \/ 'validateMaybe' to construct a 'Text'.+A 'Text' wrap a 'Bytes' which will be interpreted using UTF-8 encoding. User should always use 'validate' \/ 'validateMaybe' to construt a 'Text' (instead of using construtor directly or coercing), otherwise illegal UTF-8 encoded codepoints will cause undefined behaviours. +This library also provide simple unicode processing based on <https://github.com/haskell-Z/utf8rewind/ utf8rewind>,+see 'normalize', 'caseFold' (current using unicode 13 databases).+ -}  module Z.Data.Text (   -- * Text type     Text(..)-  , validate-  , InvalidUTF8Exception(..)-  , validateMaybe+  , validate, validateASCII+  , validateMaybe, validateASCIIMaybe+  , TextException(..)   , index, indexMaybe, indexR, indexMaybeR   -- * Basic creating   , empty, singleton, copy@@ -80,67 +83,12 @@   , isNormalized, isNormalizedTo, normalize, normalizeTo     -- ** Case conversion     -- $case-  , Locale, localeDefault, localeLithuanian, localeTurkishAndAzeriLatin+  , Locale(..)+  , envLocale   , caseFold, caseFoldWith, toLower, toLowerWith, toUpper, toUpperWith, toTitle, toTitleWith     -- ** Unicode category   , isCategory, spanCategory-  , Category-  , categoryLetterUppercase-  , categoryLetterLowercase-  , categoryLetterTitlecase-  , categoryLetterOther-  , categoryLetter-  , categoryCaseMapped--  , categoryMarkNonSpacing-  , categoryMarkSpacing-  , categoryMarkEnclosing-  , categoryMark--  , categoryNumberDecimal-  , categoryNumberLetter-  , categoryNumberOther-  , categoryNumber--  , categoryPunctuationConnector-  , categoryPunctuationDash-  , categoryPunctuationOpen-  , categoryPunctuationClose-  , categoryPunctuationInitial-  , categoryPunctuationFinal-  , categoryPunctuationOther-  , categoryPunctuation--  , categorySymbolMath-  , categorySymbolCurrency-  , categorySymbolModifier-  , categorySymbolOther-  , categorySymbol--  , categorySeparatorSpace-  , categorySeparatorLine-  , categorySeparatorParagraph-  , categorySeparator-  , categoryControl-  , categoryFormat-  , categorySurrogate-  , categoryPrivateUse-  , categoryUnassigned-  , categoryCompatibility-  , categoryIgnoreGraphemeCluste-  , categoryIscntrl--  , categoryIsprint-  , categoryIsspace-  , categoryIsblank-  , categoryIsgraph-  , categoryIspunct-  , categoryIsalnum-  , categoryIsalpha-  , categoryIsupper-  , categoryIslower-  , categoryIsdigit-  , categoryIsxdigit+  , Category(..)  ) where  import           Z.Data.Text.Base

@@ -24,9 +24,9 @@   -- * Text type     Text(..)   -- * Building text-  , validate-  , InvalidUTF8Exception(..)-  , validateMaybe+  , validate, validateASCII+  , validateMaybe, validateASCIIMaybe+  , TextException(..)   , index, indexMaybe, charByteIndex, indexR, indexMaybeR, charByteIndexR   -- * Basic creating   , empty, singleton, copy@@ -52,70 +52,73 @@   , isNormalized, isNormalizedTo, normalize, normalizeTo     -- ** Case conversion     -- $case-  , Locale, localeDefault, localeLithuanian, localeTurkishAndAzeriLatin+  , Locale(LocaleDefault, LocaleLithuanian, LocaleTurkishAndAzeriLatin)+  , envLocale   , caseFold, caseFoldWith, toLower, toLowerWith, toUpper, toUpperWith, toTitle, toTitleWith     -- ** Unicode category   , isCategory, spanCategory-  , Category-  , categoryLetterUppercase-  , categoryLetterLowercase-  , categoryLetterTitlecase-  , categoryLetterOther-  , categoryLetter-  , categoryCaseMapped+  , Category(+        CategoryLetterUppercase+      , CategoryLetterLowercase+      , CategoryLetterTitlecase+      , CategoryLetterOther+      , CategoryLetter+      , CategoryCaseMapped -  , categoryMarkNonSpacing-  , categoryMarkSpacing-  , categoryMarkEnclosing-  , categoryMark+      , CategoryMarkNonSpacing+      , CategoryMarkSpacing+      , CategoryMarkEnclosing+      , CategoryMark -  , categoryNumberDecimal-  , categoryNumberLetter-  , categoryNumberOther-  , categoryNumber+      , CategoryNumberDecimal+      , CategoryNumberLetter+      , CategoryNumberOther+      , CategoryNumber -  , categoryPunctuationConnector-  , categoryPunctuationDash-  , categoryPunctuationOpen-  , categoryPunctuationClose-  , categoryPunctuationInitial-  , categoryPunctuationFinal-  , categoryPunctuationOther-  , categoryPunctuation+      , CategoryPunctuationConnector+      , CategoryPunctuationDash+      , CategoryPunctuationOpen+      , CategoryPunctuationClose+      , CategoryPunctuationInitial+      , CategoryPunctuationFinal+      , CategoryPunctuationOther+      , CategoryPunctuation -  , categorySymbolMath-  , categorySymbolCurrency-  , categorySymbolModifier-  , categorySymbolOther-  , categorySymbol+      , CategorySymbolMath+      , CategorySymbolCurrency+      , CategorySymbolModifier+      , CategorySymbolOther+      , CategorySymbol -  , categorySeparatorSpace-  , categorySeparatorLine-  , categorySeparatorParagraph-  , categorySeparator-  , categoryControl-  , categoryFormat-  , categorySurrogate-  , categoryPrivateUse-  , categoryUnassigned-  , categoryCompatibility-  , categoryIgnoreGraphemeCluste-  , categoryIscntrl+      , CategorySeparatorSpace+      , CategorySeparatorLine+      , CategorySeparatorParagraph+      , CategorySeparator+      , CategoryControl+      , CategoryFormat+      , CategorySurrogate+      , CategoryPrivateUse+      , CategoryUnassigned+      , CategoryCompatibility+      , CategoryIgnoreGraphemeCluster+      , CategoryIscntrl -  , categoryIsprint-  , categoryIsspace-  , categoryIsblank-  , categoryIsgraph-  , categoryIspunct-  , categoryIsalnum-  , categoryIsalpha-  , categoryIsupper-  , categoryIslower-  , categoryIsdigit-  , categoryIsxdigit+      , CategoryIsprint+      , CategoryIsspace+      , CategoryIsblank+      , CategoryIsgraph+      , CategoryIspunct+      , CategoryIsalnum+      , CategoryIsalpha+      , CategoryIsupper+      , CategoryIslower+      , CategoryIsdigit+      , CategoryIsxdigit)   -- * Misc   , c_utf8_validate_ba   , c_utf8_validate_addr+  , c_ascii_validate_ba+  , c_ascii_validate_addr  ) where  import           Control.DeepSeq@@ -189,36 +192,11 @@     {-# INLINE fromString #-}     fromString = pack -packASCIIAddr :: Addr# -> Text-packASCIIAddr addr0# = go addr0#-  where-    len = fromIntegral . unsafeDupablePerformIO $ c_strlen addr0#-    go addr# = runST $ do-        marr <- newPrimArray len-        copyPtrToMutablePrimArray marr 0 (Ptr addr#) len-        arr <- unsafeFreezePrimArray marr-        return $ Text (PrimVector arr 0 len)--packUTF8Addr :: Addr# -> Text-packUTF8Addr addr0# = validateAndCopy addr0#-  where-    len = fromIntegral . unsafeDupablePerformIO $ c_strlen addr0#-    valid = unsafeDupablePerformIO $ c_utf8_validate_addr addr0# len-    validateAndCopy addr#-        | valid == 0 = packN len (unpackCStringUtf8# addr#) -- three bytes surrogate -> three bytes replacement-                                                        -- two bytes NUL -> \NUL-                                                        -- the result's length will either smaller or equal-        | otherwise  = runST $ do-            marr <- newPrimArray len-            copyPtrToMutablePrimArray marr 0 (Ptr addr#) len-            arr <- unsafeFreezePrimArray marr-            return $ Text (PrimVector arr 0 len)---- | /O(n)/ Get the nth codepoint from 'Text', throw @IndexOutOfVectorRange n callStack@+-- | /O(n)/ Get the nth codepoint from 'Text', throw 'IndexOutOfTextRange' -- when out of bound. index :: HasCallStack => Text -> Int -> Char {-# INLINABLE index #-}-index t n = case t `indexMaybe` n of Nothing -> throw (V.IndexOutOfVectorRange n callStack)+index t n = case t `indexMaybe` n of Nothing -> throw (IndexOutOfTextRange n callStack)                                      Just x  -> x  @@ -299,6 +277,10 @@     | c_utf8_validate_ba ba# s# l# > 0 = Text bs     | otherwise = throw (InvalidUTF8Exception callStack) +-- | /O(n)/ Validate a sequence of bytes is UTF-8 encoded.+--+-- Return 'Nothing' in case of invalid codepoint.+-- validateMaybe :: Bytes -> Maybe Text {-# INLINE validateMaybe #-} validateMaybe bs@(V.PrimVector (PrimArray ba#) (I# s#) l@(I# l#))@@ -306,14 +288,43 @@     | c_utf8_validate_ba ba# s# l# > 0 = Just (Text bs)     | otherwise = Nothing +-- | /O(n)/ Validate a sequence of bytes is all ascii char byte(<128).+--+-- Throw 'InvalidASCIIException' in case of invalid byte, It's not faster+-- than 'validate', use it only if you want to validate if a ASCII char sequence.+--+validateASCII :: HasCallStack => Bytes -> Text+{-# INLINE validateASCII #-}+validateASCII bs@(V.PrimVector (PrimArray ba#) (I# s#) l@(I# l#))+    | l == 0 = Text bs+    | c_ascii_validate_ba ba# s# l# > 0 = Text bs+    | otherwise = throw (InvalidASCIIException callStack)++-- | /O(n)/ Validate a sequence of bytes is all ascii char byte(<128).+--+-- Return 'Nothing' in case of invalid byte.+--+validateASCIIMaybe :: Bytes -> Maybe Text+{-# INLINE validateASCIIMaybe #-}+validateASCIIMaybe bs@(V.PrimVector (PrimArray ba#) (I# s#) l@(I# l#))+    | l == 0 = Just (Text bs)+    | c_ascii_validate_ba ba# s# l# > 0 = Just (Text bs)+    | otherwise = Nothing+ foreign import ccall unsafe "text.h utf8_validate"     c_utf8_validate_ba :: ByteArray# -> Int# -> Int# -> Int foreign import ccall unsafe "text.h utf8_validate_addr"     c_utf8_validate_addr :: Addr# -> Int -> IO Int+foreign import ccall unsafe "text.h ascii_validate"+    c_ascii_validate_ba :: ByteArray# -> Int# -> Int# -> Int+foreign import ccall unsafe "text.h ascii_validate_addr"+    c_ascii_validate_addr :: Addr# -> Int -> IO Int -data InvalidUTF8Exception = InvalidUTF8Exception CallStack-                    deriving (Show, Typeable)-instance Exception InvalidUTF8Exception+data TextException = InvalidUTF8Exception CallStack+                   | InvalidASCIIException CallStack+                   | IndexOutOfTextRange Int CallStack   -- ^ first payload is invalid char index+                  deriving (Show, Typeable)+instance Exception TextException  -------------------------------------------------------------------------------- @@ -326,6 +337,31 @@ {-# RULES "pack/packASCIIAddr" forall addr . pack (unpackCString# addr) = packASCIIAddr addr #-} {-# RULES "pack/packUTF8Addr" forall addr . pack (unpackCStringUtf8# addr) = packUTF8Addr addr #-} +packASCIIAddr :: Addr# -> Text+packASCIIAddr addr0# = go addr0#+  where+    len = fromIntegral . unsafeDupablePerformIO $ c_strlen addr0#+    go addr# = runST $ do+        marr <- newPrimArray len+        copyPtrToMutablePrimArray marr 0 (Ptr addr#) len+        arr <- unsafeFreezePrimArray marr+        return $ Text (PrimVector arr 0 len)++packUTF8Addr :: Addr# -> Text+packUTF8Addr addr0# = validateAndCopy addr0#+  where+    len = fromIntegral . unsafeDupablePerformIO $ c_strlen addr0#+    valid = unsafeDupablePerformIO $ c_utf8_validate_addr addr0# len+    validateAndCopy addr#+        | valid == 0 = packN len (unpackCStringUtf8# addr#) -- three bytes surrogate -> three bytes replacement+                                                            -- two bytes NUL -> \NUL+                                                            -- the result's length will either smaller or equal+        | otherwise  = runST $ do+            marr <- newPrimArray len+            copyPtrToMutablePrimArray marr 0 (Ptr addr#) len+            arr <- unsafeFreezePrimArray marr+            return $ Text (PrimVector arr 0 len)+ -- | /O(n)/ Convert a list into a text with an approximate size(in bytes, not codepoints). -- -- If the encoded bytes length is larger than the size given, we simply double the buffer size@@ -768,7 +804,7 @@  -- | Remove case distinction from UTF-8 encoded text with default locale. caseFold :: Text -> Text-caseFold = caseFoldWith localeDefault+caseFold = caseFoldWith LocaleDefault  {-| Remove case distinction from UTF-8 encoded text.@@ -813,7 +849,7 @@  -- | Convert UTF-8 encoded text to lowercase with default locale. toLower :: Text -> Text-toLower = toLowerWith localeDefault+toLower = toLowerWith LocaleDefault  {-| Convert UTF-8 encoded text to lowercase.@@ -853,7 +889,7 @@  -- | Convert UTF-8 encoded text to uppercase with default locale. toUpper :: Text -> Text-toUpper = toUpperWith localeDefault+toUpper = toUpperWith LocaleDefault  {-| Convert UTF-8 encoded text to uppercase.@@ -891,7 +927,7 @@  -- | Convert UTF-8 encoded text to titlecase with default locale. toTitle :: Text -> Text-toTitle = toTitleWith localeDefault+toTitle = toTitleWith LocaleDefault  {-| Convert UTF-8 encoded text to titlecase.@@ -980,12 +1016,12 @@ U+0300     | 230                       | Mn (Non-spacing mark) | COMBINING GRAVE ACCENT @ -Will match with 'categoryLetterUppercase' in its entirety, because+Will match with 'CategoryLetterUppercase' in its entirety, because the COMBINING GRAVE ACCENT is treated as part of the grapheme cluster. This is useful when e.g. creating a text parser, because you do not have to normalize the text first. -If this is undesired behavior, specify the 'UTF8_CATEGORY_IGNORE_GRAPHEME_CLUSTER' flag.+If this is undesired behavior, specify the 'CategoryIgnoreGraphemeCluster' flag.  In order to maintain backwards compatibility with POSIX functions like `isdigit` and `isspace`, compatibility flags have been provided. Note,

@@ -1,5 +1,6 @@ {-# LANGUAGE DeriveGeneric #-} {-# LANGUAGE GeneralizedNewtypeDeriving #-}+{-# LANGUAGE PatternSynonyms #-}  {-| Module      : Z.Data.Text.UTF8Rewind@@ -25,14 +26,15 @@ -- | Locale for case mapping. newtype Locale = Locale CSize deriving (Show, Eq, Ord, Generic) -#{enum Locale, Locale, localeDefault    = UTF8_LOCALE_DEFAULT }-#{enum Locale, Locale, localeLithuanian = UTF8_LOCALE_LITHUANIAN }-#{enum Locale, Locale, localeTurkishAndAzeriLatin = UTF8_LOCALE_TURKISH_AND_AZERI_LATIN }+pattern LocaleDefault              :: Locale +pattern LocaleDefault               = Locale (#const UTF8_LOCALE_DEFAULT)+pattern LocaleLithuanian           :: Locale+pattern LocaleLithuanian            = Locale (#const UTF8_LOCALE_LITHUANIAN)+pattern LocaleTurkishAndAzeriLatin :: Locale+pattern LocaleTurkishAndAzeriLatin  = Locale (#const UTF8_LOCALE_TURKISH_AND_AZERI_LATIN) --- | see 'NormalizeMode' in Z.Data.Text.Base-#{enum CSize, CSize, normalizeCompose       = UTF8_NORMALIZE_COMPOSE }-#{enum CSize, CSize, normalizeDecompose     = UTF8_NORMALIZE_DECOMPOSE }-#{enum CSize, CSize, normalizeCompatibility = UTF8_NORMALIZE_COMPATIBILITY }+-- | Get environment locale+foreign import ccall unsafe "utf8envlocale" envLocale :: IO Locale  {-| These are the Unicode Normalization Forms:@@ -63,64 +65,112 @@   -- | Unicode categories.+-- -- See 'Z.Data.Text.Base.isCategory', you can combine categories with bitwise or. newtype Category = Category CSize deriving (Show, Eq, Ord, Bits, FiniteBits, Generic) -#{enum Category, Category, categoryLetterUppercase        = UTF8_CATEGORY_LETTER_UPPERCASE }-#{enum Category, Category, categoryLetterLowercase        = UTF8_CATEGORY_LETTER_LOWERCASE }-#{enum Category, Category, categoryLetterTitlecase        = UTF8_CATEGORY_LETTER_TITLECASE }-#{enum Category, Category, categoryLetterOther            = UTF8_CATEGORY_LETTER_OTHER }-#{enum Category, Category, categoryLetter                 = UTF8_CATEGORY_LETTER }-#{enum Category, Category, categoryCaseMapped             = UTF8_CATEGORY_CASE_MAPPED }--#{enum Category, Category, categoryMarkNonSpacing         = UTF8_CATEGORY_MARK_NON_SPACING }-#{enum Category, Category, categoryMarkSpacing            = UTF8_CATEGORY_MARK_SPACING }-#{enum Category, Category, categoryMarkEnclosing          = UTF8_CATEGORY_MARK_ENCLOSING }-#{enum Category, Category, categoryMark                   = UTF8_CATEGORY_MARK }--#{enum Category, Category, categoryNumberDecimal          = UTF8_CATEGORY_NUMBER_DECIMAL }-#{enum Category, Category, categoryNumberLetter           = UTF8_CATEGORY_NUMBER_LETTER }-#{enum Category, Category, categoryNumberOther            = UTF8_CATEGORY_NUMBER_OTHER }-#{enum Category, Category, categoryNumber                 = UTF8_CATEGORY_NUMBER }+pattern CategoryLetterUppercase       :: Category+pattern CategoryLetterLowercase       :: Category+pattern CategoryLetterTitlecase       :: Category+pattern CategoryLetterOther           :: Category+pattern CategoryLetter                :: Category+pattern CategoryCaseMapped            :: Category+pattern CategoryLetterUppercase        = Category (#const UTF8_CATEGORY_LETTER_UPPERCASE)+pattern CategoryLetterLowercase        = Category (#const UTF8_CATEGORY_LETTER_LOWERCASE)+pattern CategoryLetterTitlecase        = Category (#const UTF8_CATEGORY_LETTER_TITLECASE)+pattern CategoryLetterOther            = Category (#const UTF8_CATEGORY_LETTER_OTHER)+pattern CategoryLetter                 = Category (#const UTF8_CATEGORY_LETTER)+pattern CategoryCaseMapped             = Category (#const UTF8_CATEGORY_CASE_MAPPED) -#{enum Category, Category, categoryPunctuationConnector   = UTF8_CATEGORY_PUNCTUATION_CONNECTOR }-#{enum Category, Category, categoryPunctuationDash        = UTF8_CATEGORY_PUNCTUATION_DASH }-#{enum Category, Category, categoryPunctuationOpen        = UTF8_CATEGORY_PUNCTUATION_OPEN }-#{enum Category, Category, categoryPunctuationClose       = UTF8_CATEGORY_PUNCTUATION_CLOSE }-#{enum Category, Category, categoryPunctuationInitial     = UTF8_CATEGORY_PUNCTUATION_INITIAL }-#{enum Category, Category, categoryPunctuationFinal       = UTF8_CATEGORY_PUNCTUATION_FINAL }-#{enum Category, Category, categoryPunctuationOther       = UTF8_CATEGORY_PUNCTUATION_OTHER }-#{enum Category, Category, categoryPunctuation            = UTF8_CATEGORY_PUNCTUATION }+pattern CategoryMarkNonSpacing        :: Category  +pattern CategoryMarkSpacing           :: Category  +pattern CategoryMarkEnclosing         :: Category  +pattern CategoryMark                  :: Category  +pattern CategoryMarkNonSpacing         = Category (#const UTF8_CATEGORY_MARK_NON_SPACING)+pattern CategoryMarkSpacing            = Category (#const UTF8_CATEGORY_MARK_SPACING)+pattern CategoryMarkEnclosing          = Category (#const UTF8_CATEGORY_MARK_ENCLOSING)+pattern CategoryMark                   = Category (#const UTF8_CATEGORY_MARK) -#{enum Category, Category, categorySymbolMath             = UTF8_CATEGORY_SYMBOL_MATH }-#{enum Category, Category, categorySymbolCurrency         = UTF8_CATEGORY_SYMBOL_CURRENCY }-#{enum Category, Category, categorySymbolModifier         = UTF8_CATEGORY_SYMBOL_MODIFIER }-#{enum Category, Category, categorySymbolOther            = UTF8_CATEGORY_SYMBOL_OTHER }-#{enum Category, Category, categorySymbol                 = UTF8_CATEGORY_SYMBOL }+pattern CategoryNumberDecimal         :: Category +pattern CategoryNumberLetter          :: Category +pattern CategoryNumberOther           :: Category +pattern CategoryNumber                :: Category +pattern CategoryNumberDecimal          = Category (#const UTF8_CATEGORY_NUMBER_DECIMAL)+pattern CategoryNumberLetter           = Category (#const UTF8_CATEGORY_NUMBER_LETTER)+pattern CategoryNumberOther            = Category (#const UTF8_CATEGORY_NUMBER_OTHER)+pattern CategoryNumber                 = Category (#const UTF8_CATEGORY_NUMBER) -#{enum Category, Category, categorySeparatorSpace         = UTF8_CATEGORY_SEPARATOR_SPACE }-#{enum Category, Category, categorySeparatorLine          = UTF8_CATEGORY_SEPARATOR_LINE }-#{enum Category, Category, categorySeparatorParagraph     = UTF8_CATEGORY_SEPARATOR_PARAGRAPH }-#{enum Category, Category, categorySeparator              = UTF8_CATEGORY_SEPARATOR }-#{enum Category, Category, categoryControl                = UTF8_CATEGORY_CONTROL }-#{enum Category, Category, categoryFormat                 = UTF8_CATEGORY_FORMAT }-#{enum Category, Category, categorySurrogate              = UTF8_CATEGORY_SURROGATE }-#{enum Category, Category, categoryPrivateUse             = UTF8_CATEGORY_PRIVATE_USE }-#{enum Category, Category, categoryUnassigned             = UTF8_CATEGORY_UNASSIGNED }-#{enum Category, Category, categoryCompatibility          = UTF8_CATEGORY_COMPATIBILITY }-#{enum Category, Category, categoryIgnoreGraphemeCluste   = UTF8_CATEGORY_IGNORE_GRAPHEME_CLUSTER }-#{enum Category, Category, categoryIscntrl                = UTF8_CATEGORY_ISCNTRL }+pattern CategoryPunctuationConnector  :: Category+pattern CategoryPunctuationDash       :: Category+pattern CategoryPunctuationOpen       :: Category+pattern CategoryPunctuationClose      :: Category+pattern CategoryPunctuationInitial    :: Category+pattern CategoryPunctuationFinal      :: Category+pattern CategoryPunctuationOther      :: Category+pattern CategoryPunctuation           :: Category+pattern CategoryPunctuationConnector   = Category (#const UTF8_CATEGORY_PUNCTUATION_CONNECTOR)+pattern CategoryPunctuationDash        = Category (#const UTF8_CATEGORY_PUNCTUATION_DASH)+pattern CategoryPunctuationOpen        = Category (#const UTF8_CATEGORY_PUNCTUATION_OPEN)+pattern CategoryPunctuationClose       = Category (#const UTF8_CATEGORY_PUNCTUATION_CLOSE)+pattern CategoryPunctuationInitial     = Category (#const UTF8_CATEGORY_PUNCTUATION_INITIAL)+pattern CategoryPunctuationFinal       = Category (#const UTF8_CATEGORY_PUNCTUATION_FINAL)+pattern CategoryPunctuationOther       = Category (#const UTF8_CATEGORY_PUNCTUATION_OTHER)+pattern CategoryPunctuation            = Category (#const UTF8_CATEGORY_PUNCTUATION)+pattern CategorySymbolMath            :: Category +pattern CategorySymbolCurrency        :: Category +pattern CategorySymbolModifier        :: Category +pattern CategorySymbolOther           :: Category +pattern CategorySymbol                :: Category +pattern CategorySymbolMath             = Category (#const UTF8_CATEGORY_SYMBOL_MATH)+pattern CategorySymbolCurrency         = Category (#const UTF8_CATEGORY_SYMBOL_CURRENCY)+pattern CategorySymbolModifier         = Category (#const UTF8_CATEGORY_SYMBOL_MODIFIER)+pattern CategorySymbolOther            = Category (#const UTF8_CATEGORY_SYMBOL_OTHER)+pattern CategorySymbol                 = Category (#const UTF8_CATEGORY_SYMBOL) -#{enum Category, Category, categoryIsprint                = UTF8_CATEGORY_ISPRINT }-#{enum Category, Category, categoryIsspace                = UTF8_CATEGORY_ISSPACE }-#{enum Category, Category, categoryIsblank                = UTF8_CATEGORY_ISBLANK }-#{enum Category, Category, categoryIsgraph                = UTF8_CATEGORY_ISGRAPH }-#{enum Category, Category, categoryIspunct                = UTF8_CATEGORY_ISPUNCT }-#{enum Category, Category, categoryIsalnum                = UTF8_CATEGORY_ISALNUM }-#{enum Category, Category, categoryIsalpha                = UTF8_CATEGORY_ISALPHA }-#{enum Category, Category, categoryIsupper                = UTF8_CATEGORY_ISUPPER }-#{enum Category, Category, categoryIslower                = UTF8_CATEGORY_ISLOWER }-#{enum Category, Category, categoryIsdigit                = UTF8_CATEGORY_ISDIGIT }-#{enum Category, Category, categoryIsxdigit               = UTF8_CATEGORY_ISXDIGIT }+pattern CategorySeparatorSpace        :: Category +pattern CategorySeparatorLine         :: Category +pattern CategorySeparatorParagraph    :: Category +pattern CategorySeparator             :: Category +pattern CategoryControl               :: Category +pattern CategoryFormat                :: Category +pattern CategorySurrogate             :: Category +pattern CategoryPrivateUse            :: Category +pattern CategoryUnassigned            :: Category +pattern CategoryCompatibility         :: Category +pattern CategoryIgnoreGraphemeCluster :: Category +pattern CategoryIscntrl               :: Category +pattern CategorySeparatorSpace         = Category (#const UTF8_CATEGORY_SEPARATOR_SPACE)+pattern CategorySeparatorLine          = Category (#const UTF8_CATEGORY_SEPARATOR_LINE)+pattern CategorySeparatorParagraph     = Category (#const UTF8_CATEGORY_SEPARATOR_PARAGRAPH)+pattern CategorySeparator              = Category (#const UTF8_CATEGORY_SEPARATOR)+pattern CategoryControl                = Category (#const UTF8_CATEGORY_CONTROL)+pattern CategoryFormat                 = Category (#const UTF8_CATEGORY_FORMAT)+pattern CategorySurrogate              = Category (#const UTF8_CATEGORY_SURROGATE)+pattern CategoryPrivateUse             = Category (#const UTF8_CATEGORY_PRIVATE_USE)+pattern CategoryUnassigned             = Category (#const UTF8_CATEGORY_UNASSIGNED)+pattern CategoryCompatibility          = Category (#const UTF8_CATEGORY_COMPATIBILITY)+pattern CategoryIgnoreGraphemeCluster  = Category (#const UTF8_CATEGORY_IGNORE_GRAPHEME_CLUSTER)+pattern CategoryIscntrl                = Category (#const UTF8_CATEGORY_ISCNTRL) -foreign import ccall unsafe utf8envlocale :: IO Category+pattern CategoryIsprint               :: Category+pattern CategoryIsspace               :: Category+pattern CategoryIsblank               :: Category+pattern CategoryIsgraph               :: Category+pattern CategoryIspunct               :: Category+pattern CategoryIsalnum               :: Category+pattern CategoryIsalpha               :: Category+pattern CategoryIsupper               :: Category+pattern CategoryIslower               :: Category+pattern CategoryIsdigit               :: Category+pattern CategoryIsxdigit              :: Category+pattern CategoryIsprint                = Category (#const UTF8_CATEGORY_ISPRINT)+pattern CategoryIsspace                = Category (#const UTF8_CATEGORY_ISSPACE)+pattern CategoryIsblank                = Category (#const UTF8_CATEGORY_ISBLANK)+pattern CategoryIsgraph                = Category (#const UTF8_CATEGORY_ISGRAPH)+pattern CategoryIspunct                = Category (#const UTF8_CATEGORY_ISPUNCT)+pattern CategoryIsalnum                = Category (#const UTF8_CATEGORY_ISALNUM)+pattern CategoryIsalpha                = Category (#const UTF8_CATEGORY_ISALPHA)+pattern CategoryIsupper                = Category (#const UTF8_CATEGORY_ISUPPER)+pattern CategoryIslower                = Category (#const UTF8_CATEGORY_ISLOWER)+pattern CategoryIsdigit                = Category (#const UTF8_CATEGORY_ISDIGIT)+pattern CategoryIsxdigit               = Category (#const UTF8_CATEGORY_ISXDIGIT)                   

@@ -79,6 +79,7 @@   , clearMBA   , clearPtr   , castPtr+  , fromNullTerminated   -- ** re-export   , module Data.Primitive.ByteArray   , module Foreign.C.Types@@ -350,9 +351,21 @@  -- | Zero a structure. ----- There's no 'Storable' or 'Prim' constraint on 'a' type, thus the length+-- There's no 'Storable' or 'Prim' constraint on 'a' type, the length -- should be given in bytes. -- clearPtr :: Ptr a -> Int -> IO () {-# INLINE clearPtr #-} clearPtr dest nbytes = memset dest 0 (fromIntegral nbytes)++-- | Copy some bytes from a null terminated pointer(without copy the null terminator).+--+-- There's no encoding guarantee, result could be any bytes sequence.+fromNullTerminated :: Ptr a -> IO Bytes+{-# INLINE fromNullTerminated #-}+fromNullTerminated (Ptr addr#) = do+    len <- fromIntegral <$> c_strlen addr#+    marr <- newPrimArray len+    copyPtrToMutablePrimArray marr 0 (Ptr addr#) len+    arr <- unsafeFreezePrimArray marr+    return (PrimVector arr 0 len)

@@ -4,11 +4,13 @@  module Z.Data.CBytesSpec where -import qualified Data.List                as List+import qualified Data.List                  as List import           Data.Word-import           Data.Hashable            (hashWithSalt, hash)-import qualified Z.Data.CBytes          as CB-import qualified Z.Data.Vector.Base     as V+import           Data.Hashable              (hashWithSalt, hash)+import qualified Z.Data.CBytes              as CB+import           Z.Foreign                  (fromNullTerminated)+import qualified Z.Data.Vector.Base         as V+import           System.IO.Unsafe import           Test.QuickCheck import           Test.QuickCheck.Function import           Test.QuickCheck.Property@@ -47,3 +49,8 @@     describe "CBytes append == List.(++)" $ do         prop "CBytes eq === List.eq" $ \ xs ys ->             (CB.pack xs `CB.append` CB.pack ys) === CB.pack (xs ++ ys)++    describe "withCBytes fromNullTerminated  == toBytes" $ do+        prop "CBytes eq === List.eq" $ \ xs ->+            CB.toBytes (CB.pack xs) ===+                (unsafeDupablePerformIO $ CB.withCBytes (CB.pack xs) fromNullTerminated)