Z-Data 0.1.6.1 → 0.1.7.0

raw patch · 18 files changed

+324/−137 lines, 18 filesdep −timePVP: major bump suggested

API removals or changes: PVP suggests a major version bump

Dependencies removed: time

API changes (from Hackage documentation)

Files

• ChangeLog.md +7/−0
• LICENSE +0/−1
• README.md +2/−6
• Z-Data.cabal +1/−2
• Z/Data/Array.hs +2/−1
• Z/Data/Array/Checked.hs +15/−10
• Z/Data/Builder/Base.hs +25/−7
• Z/Data/Builder/Numeric.hs +28/−0
• Z/Data/CBytes.hs +128/−89
• Z/Data/Generics/Utils.hs +13/−0
• Z/Data/JSON.hs +3/−3
• Z/Data/JSON/Base.hs +84/−7
• Z/Data/Text/UTF8Codec.hs +1/−1
• Z/Data/Text/UTF8Rewind.hsc +0/−1
• Z/Data/Vector.hs +5/−5
• Z/Data/Vector/Base.hs +2/−2
• Z/Foreign.hs +2/−2
• test/Z/Data/CBytesSpec.hs +6/−0

@@ -1,5 +1,12 @@ # Revision history for Z-Data +## 0.1.7.0  -- 2020-10-15++* Change `defaultChunkSize` from 32K to 16K to reduce memory overhead.+* Change `CBytes` to use null terminated byte array, add JSON instances.+* Add JSON instances to various arrays, Fix `Char`'s JSON instance to reject string length > 1.+* Add `decodeText` and `decodeText'` to `Z.Data.JSON`.+ ## 0.1.6.1  -- 2020-10-09  * Remove `Str` newtype.

@@ -1,4 +1,3 @@--------------------------------------------------------------------------------- Z-Data LICENSE  Copyright (c) Dong Han, 2017-2020

@@ -1,12 +1,8 @@-<p align=center>-  <img src="https://github.com/haskell-Z/Z/raw/master/projectZ.svg">-</p>- ## Z-Data -[![Linux Build Status](https://github.com/haskell-Z/z-data/workflows/ubuntu-ci/badge.svg)](https://github.com/haskell-Z/z-data/actions) [![MacOS Build Status](https://github.com/haskell-Z/z-data/workflows/osx-ci/badge.svg)](https://github.com/haskell-Z/z-data/actions) [![Windows Build Status](https://github.com/haskell-Z/z-data/workflows/win-ci/badge.svg)](https://github.com/haskell-Z/z-data/actions)+[![Hackage](https://img.shields.io/hackage/v/Z-Data.svg?style=flat)](https://hackage.haskell.org/package/Z-Data) [![Linux Build Status](https://github.com/haskell-Z/z-data/workflows/ubuntu-ci/badge.svg)](https://github.com/haskell-Z/z-data/actions) [![MacOS Build Status](https://github.com/haskell-Z/z-data/workflows/osx-ci/badge.svg)](https://github.com/haskell-Z/z-data/actions) [![Windows Build Status](https://github.com/haskell-Z/z-data/workflows/win-ci/badge.svg)](https://github.com/haskell-Z/z-data/actions) -This package provides basic data structures and functions:+This package is part of [Z](https://github.com/haskell-Z/Z) project, provides basic data structures and functions:  * Array, vector(array slice) * Text based UTF-8, basic unicode manipulating

@@ -1,6 +1,6 @@ cabal-version:              2.4 name:                       Z-Data-version:                    0.1.6.1+version:                    0.1.7.0 synopsis:                   Array, vector and text description:                This package provides array, slice and text operations license:                    BSD-3-Clause@@ -121,7 +121,6 @@                           , template-haskell        >= 2.14.0                           , unordered-containers    == 0.2.*                           , tagged                  == 0.8.*-                          , time                    >= 1.8      if flag(integer-simple)         cpp-options:        -DINTEGER_SIMPLE

@@ -7,7 +7,7 @@ Stability   : experimental Portability : non-portable -Unified unboxed and boxed array operations using functional dependencies.+Unified unboxed and boxed array operations using type family.  All operations are NOT bound checked, if you need checked operations please use "Z.Data.Array.Checked". It exports exactly same APIs so that you can switch between without pain.@@ -37,6 +37,7 @@   , PrimArray(..)   , MutablePrimArray(..)   , Prim(..)+  -- * Array operations   , newPinnedPrimArray, newAlignedPinnedPrimArray   , copyPrimArrayToPtr, copyMutablePrimArrayToPtr, copyPtrToMutablePrimArray   , primArrayContents, mutablePrimArrayContents, withPrimArrayContents, withMutablePrimArrayContents

@@ -17,7 +17,17 @@   ( -- * Arr typeclass re-export     A.Arr   , RealWorld-    -- * Bound checked array operations+  -- * Boxed array type+  , A.Array(..)+  , A.MutableArray(..)+  , A.SmallArray(..)+  , A.SmallMutableArray(..)+  , A.uninitialized+  -- * Primitive array type+  , A.PrimArray(..)+  , A.MutablePrimArray(..)+  , Prim(..)+  -- * Bound checked array operations   , newArr   , newArrWith   , readArr@@ -42,15 +52,6 @@   , A.sizeofArr   , A.sizeofMutableArr   , A.sameArr-  -- * Boxed array type-  , A.Array(..)-  , A.MutableArray(..)-  , A.SmallArray(..)-  , A.SmallMutableArray(..)-  , A.uninitialized-  -- * Primitive array type-  , A.PrimArray(..)-  , A.MutablePrimArray(..)   -- * Bound checked primitive array operations   , newPinnedPrimArray, newAlignedPinnedPrimArray   , copyPrimArrayToPtr, copyMutablePrimArrayToPtr, copyPtrToMutablePrimArray@@ -63,6 +64,10 @@   , A.PrimUnlifted(..)   -- * The 'ArrayException' type   , ArrayException(..)+  -- * Cast between primitive arrays+  , A.Cast+  , A.castArray+  , A.castMutableArray   ) where  import           Control.Exception       (ArrayException (..), throw)

@@ -103,8 +103,8 @@ -- Notes on 'IsString' instance: @Builder ()@'s 'IsString' instance use 'stringModifiedUTF8', -- which is different from 'stringUTF8' in that it DOES NOT PROVIDE UTF8 GUARANTEES! : ----- * @\NUL@ will be written as @\xC0 \x80@.--- * @\xD800@ ~ @\xDFFF@ will be encoded in three bytes as normal UTF-8 codepoints.+-- * @\\NUL@ will be written as @\xC0 \x80@.+-- * @\\xD800@ ~ @\\xDFFF@ will be encoded in three bytes as normal UTF-8 codepoints. -- newtype Builder a = Builder     { runBuilder :: forall s. AllocateStrategy s -> (a -> BuildStep s) -> BuildStep s}@@ -170,7 +170,7 @@  -- | Turn 'Char' into 'Builder' with Modified UTF8 encoding ----- '\NUL' is encoded as two bytes @C0 80@ , '\xD800' ~ '\xDFFF' is encoded as a three bytes normal UTF-8 codepoint.+-- @\\NUL@ is encoded as two bytes @C0 80@ , @\\xD800@ ~ @\\xDFFF@ is encoded as a three bytes normal UTF-8 codepoint. charModifiedUTF8 :: Char -> Builder () {-# INLINE charModifiedUTF8 #-} charModifiedUTF8 chr = do@@ -537,7 +537,7 @@ #define OPEN_SQUARE 91 #define SINGLE_QUOTE 39 --- | add @{...}@ to original builder.+-- | add @(...)@ to original builder. paren :: Builder () -> Builder () {-# INLINE paren #-} paren b = encodePrim @Word8 OPEN_PAREN >> b >> encodePrim @Word8 CLOSE_PAREN@@ -552,17 +552,17 @@ {-# INLINE square #-} square b = encodePrim @Word8 OPEN_SQUARE >> b >> encodePrim @Word8 CLOSE_SQUARE --- | add @<...>@ to original builder.+-- | add @/<.../>@ to original builder. angle :: Builder () -> Builder () {-# INLINE angle #-} angle b = encodePrim @Word8 OPEN_ANGLE >> b >> encodePrim @Word8 CLOSE_ANGLE --- | add @"..."@ to original builder.+-- | add @/".../"@ to original builder. quotes :: Builder () -> Builder () {-# INLINE quotes #-} quotes b = encodePrim @Word8 DOUBLE_QUOTE >> b >> encodePrim @Word8 DOUBLE_QUOTE --- | add @'...'@ to original builder.+-- | add @/'.../'@ to original builder. squotes :: Builder () -> Builder () {-# INLINE squotes #-} squotes b = encodePrim @Word8 SINGLE_QUOTE >> b >> encodePrim @Word8 SINGLE_QUOTE@@ -578,6 +578,15 @@ comma = encodePrim @Word8 COMMA  -- | Use separator to connect a vector of builders.+--+-- @+-- import Z.Data.Builder as B+-- import Z.Data.Text    as T+-- import Z.Data.Vector  as V+--+-- > T.validate . B.buildBytes $ B.intercalateVec "," B.int (V.pack [1,2,3,4] :: V.PrimVector Int)+-- "1,2,3,4"+-- @ intercalateVec :: (V.Vec v a)             => Builder ()           -- ^ the seperator             -> (a -> Builder ())    -- ^ value formatter@@ -589,6 +598,15 @@     forM_ (V.lastMaybe v) f  -- | Use separator to connect list of builders.+--+-- @+-- import Z.Data.Builder as B+-- import Z.Data.Text    as T+-- import Z.Data.Vector  as V+--+-- T.validate . B.buildBytes $ B.intercalateList "," B.int ([1,2,3,4] :: [Int])+-- "1,2,3,4"+-- @ intercalateList :: Builder ()           -- ^ the seperator                 -> (a -> Builder ())    -- ^ value formatter                 -> [a]                  -- ^ value list

@@ -98,6 +98,19 @@ int = intWith defaultIFormat  -- | Format a 'Bounded' 'Integral' type like @Int@ or @Word16@ into decimal ASCII digits.+--+-- @+-- import Z.Data.Builder as B+-- import Z.Data.Text    as T+--+-- > T.validate . B.buildBytes $ B.intWith defaultIFormat  (12345 :: Int)+-- "12345"+-- > T.validate . B.buildBytes $ B.intWith defaultIFormat{width=10, padding=RightSpacePadding} (12345 :: Int)+-- "12345     "+-- > T.validate . B.buildBytes $ B.intWith defaultIFormat{width=10, padding=ZeroPadding} (12345 :: Int)+-- "0000012345"+-- @+-- intWith :: (Integral a, Bounded a) => IFormat -> a -> Builder () intWith = hs_intWith {-# INLINE[0] intWith #-}@@ -484,6 +497,21 @@ --------------------------------------------------------------------------------  -- | Format a 'FiniteBits' 'Integral' type into hex nibbles.+--+-- @+-- import Z.Data.Builder as B+-- import Z.Data.Text    as T+-- import Data.Word+-- import Data.Int+--+-- > T.validate . B.buildBytes $ B.hex (125 :: Int8)+-- "7d"+-- > T.validate . B.buildBytes $ B.hex (-1 :: Int8)+-- "ff"+-- > T.validate . B.buildBytes $ B.hex (125 :: Word16)+-- "007d"+-- @+-- hex :: forall a. (FiniteBits a, Integral a) => a -> Builder () {-# INLINE hex #-} hex w = writeN hexSiz (go w (hexSiz-2))

@@ -7,22 +7,23 @@ Stability   : experimental Portability : non-portable --- This module provide 'CBytes' with some useful instances \/ tools for retrieving, storing or processing--- short byte sequences, such as file path, environment variables, etc.+This module provide 'CBytes' with some useful instances \/ tools for retrieving, storing or processing+short byte sequences, such as file path, environment variables, etc.  -}  module Z.Data.CBytes-  ( CBytes(CB)-  , toPrimArray+  (  -- * The CBytes type+    CBytes(CB)+  , rawPrimArray, fromPrimArray+  , toBytes, fromBytes, toText, toTextMaybe, fromText, toBuilder, buildCBytes   , pack   , unpack   , null , length   , empty, append, concat, intercalate, intercalateElem-  , toBytes, fromBytes, toText, toTextMaybe, fromText, toBuilder, buildCBytes   , fromCString, fromCStringN   , withCBytesUnsafe, withCBytes, allocCBytesUnsafe, allocCBytes-  -- re-export+  -- * re-export   , CString   , V.c2w, V.w2c   ) where@@ -58,6 +59,8 @@ import qualified Z.Data.Builder        as B import qualified Z.Data.Text           as T import qualified Z.Data.Text.ShowT     as T+import qualified Z.Data.JSON.Base      as JSON+import           Z.Data.JSON.Base      ((<?>)) import           Z.Data.Text.UTF8Codec (encodeCharModifiedUTF8, decodeChar) import qualified Z.Data.Vector.Base    as V import           Z.Foreign@@ -73,42 +76,45 @@ -- interface, e.g. libuv do this when deal with file paths. -- -- 'CBytes' don't support O(1) slicing, it's not suitable to use it to store large byte--- chunk, If you need advance editing, convert 'CBytes' to \/ from 'PrimArray' with 'CB',--- or 'V.Bytes' with 'toBytes\/fromBytes' if you need O(1) slicing, then use vector combinators.+-- chunk, If you need advance editing, convert 'CBytes' to 'V.Bytes' with 'CB' pattern or+-- 'toBytes'\/'fromBytes', then use vector combinators. ----- When textual represatation is needed(conver to 'String', 'T.Text', 'Show' instance, etc.),+-- When textual represatation is needed e.g. converting to 'String', 'T.Text', 'Show' instance, etc., -- we assume 'CBytes' using UTF-8 encodings, 'CBytes' can be used with @OverloadedString@,--- literal encoding is UTF-8 with some modifications: @\NUL@ is encoded to 'C0 80',--- and '\xD800' ~ '\xDFFF' is encoded as a three bytes normal utf-8 codepoint.+-- literal encoding is UTF-8 with some modifications: @\\NUL@ is encoded to 'C0 80',+-- and @\\xD800@ ~ @\\xDFFF@ is encoded as a three bytes normal utf-8 codepoint. -- -- Note most of the unix API is not unicode awared though, you may find a `scandir` call -- return a filename which is not proper encoded in any unicode encoding at all. -- But still, UTF-8 is recommanded to be used when text represatation is needed.--- --+-- newtype CBytes = CBytes     {-        -- | Convert to 'PrimArray',+        -- | Convert to a @\\NUL@ terminated 'PrimArray',         ---        -- there's an invariance that this array never contains @\NUL@-        toPrimArray :: PrimArray Word8+        -- there's an invariance that this array never contains extra @\\NUL@ except terminator.+        rawPrimArray :: PrimArray Word8     } --- | Use this pattern to match or construct 'CBytes', result will be trimmed down to first byte before @\NUL@ byte if there's any.-pattern CB :: PrimArray Word8 -> CBytes-pattern CB arr <- CBytes arr where-    CB arr = fromPrimArray arr-+-- | Constuctor a 'CBytes' from arbitrary array, result will be trimmed down to first @\\NUL@ byte if there's any. fromPrimArray :: PrimArray Word8 -> CBytes {-# INLINE fromPrimArray #-}-fromPrimArray arr = runST (-    case V.elemIndex 0 arr of-        Just i -> do-            mpa <- newPrimArray i-            copyPrimArray mpa 0 arr 0 i-            pa <- unsafeFreezePrimArray mpa-            return (CBytes pa)-        _ -> return (CBytes arr))+fromPrimArray arr = runST (do+    let l = case V.elemIndex 0 arr of+            Just i -> i+            _ -> sizeofPrimArray arr+    mpa <- newPrimArray (l+1)+    copyPrimArray mpa 0 arr 0 l+    -- write \\NUL terminator+    writePrimArray mpa l 0+    pa <- unsafeFreezePrimArray mpa+    return (CBytes pa)) +-- | Use this pattern to match or construct 'CBytes', result will be trimmed down to first @\\NUL@ byte if there's any.+pattern CB :: V.Bytes -> CBytes+pattern CB bs <- (toBytes -> bs) where+    CB bs = fromBytes bs+ instance Show CBytes where     showsPrec p t = showsPrec p (unpack t) @@ -121,10 +127,12 @@  instance Eq CBytes where     {-# INLINE (==) #-}+    -- \\NUL does not affect equality     CBytes ba == CBytes bb = ba == bb  instance Ord CBytes where     {-# INLINE compare #-}+    -- \\NUL does not affect ordering     CBytes ba `compare` CBytes bb = ba `compare` bb  instance Semigroup CBytes where@@ -140,7 +148,7 @@  instance Hashable CBytes where     hashWithSalt salt (CBytes pa@(PrimArray ba#)) = unsafeDupablePerformIO $ do-        V.c_fnv_hash_ba ba# 0 (sizeofPrimArray pa) salt+        V.c_fnv_hash_ba ba# 0 (sizeofPrimArray pa - 1) salt  instance Arbitrary CBytes where     arbitrary = pack <$> arbitrary@@ -149,18 +157,20 @@ instance CoArbitrary CBytes where     coarbitrary = coarbitrary . unpack --- | This instance peek bytes until @\NUL@(or input chunk ends), poke bytes with an extra \NUL terminator.+-- | This instance peek bytes until @\\NUL@(or input chunk ends), poke bytes with an extra \\NUL terminator. instance Unaligned CBytes where     {-# INLINE unalignedSize #-}-    unalignedSize (CBytes arr) = sizeofPrimArray arr + 1+    unalignedSize (CBytes arr) = sizeofPrimArray arr     {-# INLINE peekMBA #-}     peekMBA mba# i = do         b <- getSizeofMutableByteArray (MutableByteArray mba#)         let rest = b-i         l <- c_memchr mba# i 0 rest         let l' = if l == -1 then rest else l-        mpa <- newPrimArray l'+        mpa <- newPrimArray (l'+1)         copyMutablePrimArray mpa 0 (MutablePrimArray mba#) i l'+        -- write \\NUL terminator+        writePrimArray mpa l' 0         pa <- unsafeFreezePrimArray mpa         return (CBytes pa) @@ -168,7 +178,6 @@     pokeMBA mba# i (CBytes pa) = do         let l = sizeofPrimArray pa         copyPrimArray (MutablePrimArray mba# :: MutablePrimArray RealWorld Word8) i pa 0 l-        writePrimArray (MutablePrimArray mba# :: MutablePrimArray RealWorld Word8) (i+l) 0      {-# INLINE indexBA #-}     indexBA ba# i = runST (do@@ -176,8 +185,9 @@             rest = b-i             l = V.c_memchr ba# i 0 rest             l' = if l == -1 then rest else l-        mpa <- newPrimArray l'+        mpa <- newPrimArray (l'+1)         copyPrimArray mpa 0 (PrimArray ba#) i l'+        writePrimArray mpa l' 0         pa <- unsafeFreezePrimArray mpa         return (CBytes pa)) @@ -186,15 +196,48 @@     {-# INLINE toTextBuilder #-}     toTextBuilder _ = T.stringUTF8 . show . unpack +-- | JSON instances check if 'CBytes' is proper UTF8 encoded,+-- if it is, decode/encode it as 'T.Text', otherwise as 'V.Bytes'.+--+-- @+-- > encodeText ("hello" :: CBytes)+-- "\"hello\""+-- > encodeText ("hello\\NUL" :: CBytes)     -- @\\NUL@ is encoded as C0 80+-- "[104,101,108,108,111,192,128]"+-- @+instance JSON.FromValue CBytes where+    {-# INLINE fromValue #-}+    fromValue value =+        case value of+            JSON.String t ->+                return (fromText t)+            JSON.Array arr ->+                fromBytes <$> V.traverseWithIndex+                    (\ k v -> JSON.fromValue v <?> JSON.Index k) arr+            _ -> JSON.fail'+                    "converting Z.Data.CBytes.CBytes failed, expected array or string"++instance JSON.ToValue CBytes where+    {-# INLINE toValue #-}+    toValue cbytes = case toTextMaybe cbytes of+        Just t -> JSON.toValue t+        Nothing -> JSON.toValue (toBytes cbytes)+instance JSON.EncodeJSON CBytes where+    {-# INLINE encodeJSON #-}+    encodeJSON cbytes = case toTextMaybe cbytes of+        Just t -> JSON.encodeJSON t+        Nothing -> B.square . JSON.commaVec' . toBytes $ cbytes+ append :: CBytes -> CBytes -> CBytes {-# INLINABLE append #-} append strA@(CBytes pa) strB@(CBytes pb)     | lenA == 0 = strB     | lenB == 0 = strA     | otherwise = unsafeDupablePerformIO $ do-        mpa <- newPrimArray (lenA+lenB)+        mpa <- newPrimArray (lenA+lenB+1)         copyPrimArray mpa 0    pa 0 lenA         copyPrimArray mpa lenA pb 0 lenB+        writePrimArray mpa (lenA + lenB) 0     -- the \\NUL terminator         pa' <- unsafeFreezePrimArray mpa         return (CBytes pa')   where@@ -204,7 +247,7 @@ -- | An empty 'CBytes' empty :: CBytes {-# NOINLINE empty #-}-empty = CBytes (V.empty)+empty = CBytes (V.singleton 0)  concat :: [CBytes] -> CBytes {-# INLINABLE concat #-}@@ -212,8 +255,9 @@     (0, _) -> empty     (1, _) -> let Just b = List.find (not . null) bss in b -- there must be a not empty CBytes     (_, l) -> runST $ do-        buf <- newPrimArray l+        buf <- newPrimArray (l+1)         copy bss 0 buf+        writePrimArray buf l 0 -- the \\NUL terminator         CBytes <$> unsafeFreezePrimArray buf   where     -- pre scan to decide if we really need to copy and calculate total length@@ -244,7 +288,7 @@  -- | /O(n)/ An efficient way to join 'CByte' s with a byte. ----- Intercalate bytes list with @\NUL@ will effectively leave the first bytes in the list.+-- Intercalate bytes list with @\\NUL@ will effectively leave the first bytes in the list. intercalateElem :: Word8 -> [CBytes] -> CBytes {-# INLINABLE intercalateElem #-} intercalateElem 0 [] = empty@@ -252,8 +296,9 @@ intercalateElem w8 bss = case len bss 0 of     0 -> empty     l -> runST $ do-        buf <- newPrimArray l+        buf <- newPrimArray (l+1)         copy bss 0 buf+        writePrimArray buf l 0 -- the \\NUL terminator         CBytes <$> unsafeFreezePrimArray buf   where     len []     !acc = acc@@ -287,7 +332,7 @@ packAddr :: Addr# -> CBytes packAddr addr0# = go addr0#   where-    len = (fromIntegral . unsafeDupablePerformIO $ V.c_strlen addr0#)+    len = (fromIntegral . unsafeDupablePerformIO $ V.c_strlen addr0#) + 1     go addr# = runST $ do         marr <- newPrimArray len         copyPtrToMutablePrimArray marr 0 (Ptr addr#) len@@ -296,13 +341,14 @@  -- | Pack a 'String' into 'CBytes'. ----- @\NUL@ is encoded as two bytes @C0 80@ , '\xD800' ~ '\xDFFF' is encoded as a three bytes normal UTF-8 codepoint.+-- @\\NUL@ is encoded as two bytes @C0 80@ , @\\xD800@ ~ @\\xDFFF@ is encoded as a three bytes normal UTF-8 codepoint. pack :: String -> CBytes {-# INLINE CONLIKE [1] pack #-} pack s = runST $ do     mba <- newPrimArray V.defaultInitSize     (SP2 i mba') <- foldlM go (SP2 0 mba) s-    shrinkMutablePrimArray mba' i+    writePrimArray mba' i 0     -- the \\NUL terminator+    shrinkMutablePrimArray mba' (i+1)     ba <- unsafeFreezePrimArray mba'     return (CBytes ba)   where@@ -311,7 +357,7 @@     go :: SP2 s -> Char -> ST s (SP2 s)     go (SP2 i mba) !c     = do         siz <- getSizeofMutablePrimArray mba-        if i < siz - 3  -- we need at least 4 bytes for safety+        if i < siz - 4  -- we need at least 4 bytes for safety due to extra '\0' byte         then do             i' <- encodeCharModifiedUTF8 mba i c             return (SP2 i' mba)@@ -336,7 +382,7 @@ {-# INLINE [1] unpack #-} unpack (CBytes arr) = go 0   where-    !end = sizeofPrimArray arr+    !end = sizeofPrimArray arr - 1     go !idx         | idx >= end = []         | otherwise = let (# c, i #) = decodeChar arr idx in c : go (idx + i)@@ -345,7 +391,7 @@ {-# INLINE [0] unpackFB #-} unpackFB (CBytes arr) k z = go 0   where-    !end = sizeofPrimArray arr+    !end = sizeofPrimArray arr - 1     go !idx         | idx >= end = z         | otherwise = let (# c, i #) = decodeChar arr idx in c `k` go (idx + i)@@ -361,37 +407,37 @@ -- null :: CBytes -> Bool {-# INLINE null #-}-null (CBytes pa) = sizeofPrimArray pa == 0+null (CBytes pa) = indexPrimArray pa 0 == 0  -- | Return the BTYE length of 'CBytes'. -- length :: CBytes -> Int {-# INLINE length #-}-length (CBytes pa) = sizeofPrimArray pa+length (CBytes pa) = sizeofPrimArray pa - 1  -- | /O(1)/, convert to 'V.Bytes', which can be processed by vector combinators. toBytes :: CBytes -> V.Bytes {-# INLINABLE toBytes #-}-toBytes (CBytes arr) = V.PrimVector arr 0 (sizeofPrimArray arr)+toBytes (CBytes arr) = V.PrimVector arr 0 (sizeofPrimArray arr - 1)  -- | /O(n)/, convert from 'V.Bytes' ----- Result will be trimmed down to first byte before @\NUL@ byte if there's any.+-- Result will be trimmed down to first @\\NUL@ byte if there's any. fromBytes :: V.Bytes -> CBytes {-# INLINABLE fromBytes #-}-fromBytes v@(V.PrimVector arr s l) = runST (do-    case V.elemIndex 0 v of-        Just i -> do-            mpa <- newPrimArray i-            copyPrimArray mpa 0 arr s i-            pa <- unsafeFreezePrimArray mpa-            return (CBytes pa)-        _ | s == 0 && sizeofPrimArray arr == l -> return (CBytes arr)-          | otherwise -> do-                mpa <- newPrimArray l-                copyPrimArray mpa 0 arr s l-                pa <- unsafeFreezePrimArray mpa-                return (CBytes pa))+fromBytes v@(V.PrimVector arr s l)+        -- already a \\NUL terminated bytes+    | s == 0 && sizeofPrimArray arr == (l+1) && indexPrimArray arr l == 0 =+        CBytes arr+    | otherwise = runST (do+        let l' = case V.elemIndex 0 v of+                Just i -> i+                _ -> l+        mpa <- newPrimArray (l'+1)+        copyPrimArray mpa 0 arr s l'+        writePrimArray mpa l' 0      -- the \\NUL terminator+        pa <- unsafeFreezePrimArray mpa+        return (CBytes pa))  -- | /O(n)/, convert to 'T.Text' using UTF8 encoding assumption. --@@ -409,12 +455,11 @@  -- | /O(n)/, convert from 'T.Text', ----- Result will be trimmed down to first byte before @\NUL@ byte if there's any.+-- Result will be trimmed down to first @\\NUL@ byte if there's any. fromText :: T.Text -> CBytes {-# INLINABLE fromText #-} fromText = fromBytes . T.getUTF8Bytes - -- | Write 'CBytes' \'s byte sequence to buffer. -- -- This function is different from 'ShowT' instance in that it directly write byte sequence without@@ -422,7 +467,7 @@ toBuilder :: CBytes -> B.Builder () toBuilder = B.bytes . toBytes --- | Build a 'CBytes' with builder, result will be trimmed down to first byte before @\NUL@ byte if there's any.+-- | Build a 'CBytes' with builder, result will be trimmed down to first @\\NUL@ byte if there's any. buildCBytes :: B.Builder a -> CBytes buildCBytes = fromBytes . B.buildBytes @@ -438,14 +483,15 @@     then return empty     else do         len <- fromIntegral <$> c_strlen_ptr cstring-        mpa <- newPrimArray len-        copyPtrToMutablePrimArray mpa 0 (castPtr cstring) len+        let len' = len + 1+        mpa <- newPrimArray len'+        copyPtrToMutablePrimArray mpa 0 (castPtr cstring) len'         pa <- unsafeFreezePrimArray mpa         return (CBytes pa)  -- | Same with 'fromCString', but only take at most N bytes. ----- Result will be trimmed down to first byte before @\NUL@ byte if there's any.+-- Result will be trimmed down to first @\\NUL@ byte if there's any. fromCStringN :: CString -> Int -> IO CBytes {-# INLINABLE fromCStringN #-} fromCStringN cstring len0 = do@@ -454,8 +500,9 @@     else do         len1 <- fromIntegral <$> c_strlen_ptr cstring         let len = min len0 len1-        mpa <- newPrimArray len+        mpa <- newPrimArray (len+1)         copyPtrToMutablePrimArray mpa 0 (castPtr cstring) len+        writePrimArray mpa len 0     -- the \\NUL terminator         pa <- unsafeFreezePrimArray mpa         return (CBytes pa) @@ -464,36 +511,24 @@ -- USE THIS FUNCTION WITH UNSAFE FFI CALL ONLY. withCBytesUnsafe :: CBytes -> (BA# Word8 -> IO a) -> IO a {-# INLINABLE withCBytesUnsafe #-}-withCBytesUnsafe (CBytes pa) f = do-    let l = sizeofPrimArray pa-    mpa <- newPrimArray (l+1)-    copyPrimArray mpa 0 pa 0 l-    writePrimArray mpa l 0-    pa' <- unsafeFreezePrimArray mpa-    withPrimArrayUnsafe pa' (\ p _ -> f p)+withCBytesUnsafe (CBytes pa) f = withPrimArrayUnsafe pa (\ p _ -> f p)  -- | Pass 'CBytes' to foreign function as a @const char*@. -- -- Don't pass a forever loop to this function, see <https://ghc.haskell.org/trac/ghc/ticket/14346 #14346>. withCBytes :: CBytes -> (Ptr Word8 -> IO a) -> IO a {-# INLINABLE withCBytes #-}-withCBytes (CBytes pa) f = do-    let l = sizeofPrimArray pa-    mpa <- newPinnedPrimArray (l+1)-    copyPrimArray mpa 0 pa 0 l-    writePrimArray mpa l 0-    pa' <- unsafeFreezePrimArray mpa-    withPrimArraySafe pa' (\ p _ -> f p)+withCBytes (CBytes pa) f = withPrimArraySafe pa (\ p _ -> f p)  -- | Create a 'CBytes' with IO action. ----- If (<=0) capacity is provided, a pointer pointing to @\NUL@ is passed to initialize function+-- If (<=0) capacity is provided, a pointer pointing to @\\NUL@ is passed to initialize function -- and 'empty' will be returned. This behavior is different from 'allocCBytes', which may cause -- trouble for some FFI functions. -- -- USE THIS FUNCTION WITH UNSAFE FFI CALL ONLY. allocCBytesUnsafe :: HasCallStack-                  => Int                   -- ^ capacity n(include the @\NUL@ terminator)+                  => Int                   -- ^ capacity n(including the @\\NUL@ terminator)                   -> (MBA# Word8 -> IO a)  -- ^ initialization function,                   -> IO (CBytes, a) {-# INLINABLE allocCBytesUnsafe #-}@@ -503,7 +538,9 @@     mba@(MutablePrimArray mba#) <- newPrimArray n :: IO (MutablePrimArray RealWorld Word8)     a <- fill mba#     l <- fromIntegral <$> (c_memchr mba# 0 0 n)-    shrinkMutablePrimArray mba (if l == -1 then n else l)+    let l' = if l == -1 then (n-1) else l+    shrinkMutablePrimArray mba (l'+1)+    writePrimArray mba l' 0     bs <- unsafeFreezePrimArray mba     return (CBytes bs, a) @@ -511,10 +548,10 @@ -- | Create a 'CBytes' with IO action. -- -- If (<=0) capacity is provided, a 'nullPtr' is passed to initialize function and--- 'empty' will be returned. Other than that, User have to make sure a @\NUL@ ternimated+-- 'empty' will be returned. Other than that, User have to make sure a @\\NUL@ ternimated -- string will be written. allocCBytes :: HasCallStack-            => Int                -- ^ capacity n(include the @\NUL@ terminator)+            => Int                -- ^ capacity n(including the @\\NUL@ terminator)             -> (CString -> IO a)  -- ^ initialization function,             -> IO (CBytes, a) {-# INLINABLE allocCBytes #-}@@ -523,7 +560,9 @@     mba@(MutablePrimArray mba#) <- newPinnedPrimArray n :: IO (MutablePrimArray RealWorld Word8)     a <- withMutablePrimArrayContents mba (fill . castPtr)     l <- fromIntegral <$> (c_memchr mba# 0 0 n)-    shrinkMutablePrimArray mba (if l == -1 then n else l)+    let l' = if l == -1 then (n-1) else l+    shrinkMutablePrimArray mba (l'+1)+    writePrimArray mba l' 0     bs <- unsafeFreezePrimArray mba     return (CBytes bs, a) 

@@ -1,3 +1,16 @@+{-|+Module      : Z.Data.Generics.Utils+Description : Compute Generic product size during compile time+Copyright   : (c) Dong Han, 2017-2020+License     : BSD+Maintainer  : winterland1989@gmail.com+Stability   : experimental+Portability : non-portable++This module provides a helper class to compute product size via 'Generic' instance, e.g. This class is+useful during JSON deserializing, to decide the array length used to store record KVs.+-}+ {-# LANGUAGE UndecidableInstances #-}  module Z.Data.Generics.Utils

@@ -46,7 +46,7 @@ These rules apply to user defined ADTs, but some built-in instances have different behaviour, namely:    * @Maybe a@ are encoded as JSON @null@ in 'Nothing' case, or directly encoded to its payload in 'Just' case.-  * @[a]@ are encoded to JSON array, including @[Char]@, i.e. there's no special treatment to 'String'. To get JSON string, use 'T.Text' or 'Z.Data.TextBuilder.Str'.+  * @[a]@ are encoded to JSON array, including @[Char]@, i.e. there's no special treatment to 'String'. To get JSON string, use 'T.Text'.   * 'NonEmpty', 'Vector', 'PrimVector', 'HashSet', 'FlatSet', 'FlatIntSet' are also encoded to JSON array.   * 'HashMap', 'FlatMap', 'FlatIntMap' are encoded to JSON object. @@ -98,7 +98,7 @@  The 'Value' type is different from aeson's one in that we use @Vector (Text, Value)@ to represent JSON objects, thus we can choose different strategies on key duplication, the lookup map type, etc. so instead of a single 'withObject',-we provide 'withHashMap', 'withHashMapR', 'withHashMap' and 'withHashMapR' which use different lookup map type, and different+we provide 'withHashMap', 'withHashMapR', 'withFlatMap' and 'withFlatMapR' which use different lookup map type, and different key order piority. Most of time 'FlatMap' is faster than 'HashMap' since we only use the lookup map once, the cost of constructing a 'HashMap' is higher. If you want to directly working on key-values, 'withKeyValues' provide key-values vector access.@@ -111,7 +111,7 @@ module Z.Data.JSON   ( -- * Encode & Decode     DecodeError-  , decode, decode', decodeChunks, decodeChunks', encodeBytes, encodeText, encodeTextBuilder+  , decode, decode', decodeText, decodeText', decodeChunks, decodeChunks', encodeBytes, encodeText, encodeTextBuilder     -- * Value type   , Value(..)     -- * parse into JSON Value

@@ -15,7 +15,7 @@ module Z.Data.JSON.Base   ( -- * Encode & Decode     DecodeError-  , decode, decode', decodeChunks, decodeChunks', encodeBytes, encodeText, encodeTextBuilder+  , decode, decode', decodeText, decodeText', decodeChunks, decodeChunks', encodeBytes, encodeText, encodeTextBuilder   -- * Re-export 'Value' type   , Value(..)     -- * parse into JSON Value@@ -36,6 +36,9 @@   , Field, GWriteFields(..), GMergeFields(..), GConstrToValue(..)   , LookupTable, GFromFields(..), GBuildLookup(..), GConstrFromValue(..)   , GAddPunctuation(..), GConstrEncodeJSON(..)+  -- * Internal Helper+  , commaList'+  , commaVec'   ) where  import           Control.Applicative@@ -59,12 +62,13 @@ import qualified Data.List.NonEmpty           as NonEmpty import qualified Data.Monoid                  as Monoid import           Data.Primitive.Types         (Prim)-import qualified Data.Primitive.SmallArray    as A import           Data.Proxy                   (Proxy(..)) import           Data.Ratio                   (Ratio, (%), numerator, denominator) import           Data.Scientific              (Scientific, base10Exponent, toBoundedInteger) import qualified Data.Scientific              as Scientific import qualified Data.Semigroup               as Semigroup+import qualified Data.Primitive.ByteArray     as A+import qualified Data.Primitive.SmallArray    as A import           Data.Tagged                  (Tagged (..)) import           Data.Version                 (Version, parseVersion) import           Data.Word@@ -74,6 +78,7 @@ import           GHC.Natural import           System.Exit import           Text.ParserCombinators.ReadP (readP_to_S)+import qualified Z.Data.Array               as A import qualified Z.Data.Builder             as B import           Z.Data.Generics.Utils import           Z.Data.JSON.Value          (Value(..))@@ -81,6 +86,7 @@ import qualified Z.Data.JSON.Builder        as JB import qualified Z.Data.Parser              as P import qualified Z.Data.Parser.Numeric      as P+import qualified Z.Data.Text.Base           as T import qualified Z.Data.Text                as T import qualified Z.Data.Text.ShowT          as T import qualified Z.Data.Vector.Base         as V@@ -99,7 +105,18 @@ type DecodeError = Either P.ParseError ConvertError  -- | Decode a JSON doc, only trailing JSON whitespace are allowed.---+decodeText' :: FromValue a => T.Text -> Either DecodeError a+{-# INLINE decodeText' #-}+decodeText' = decode' . T.getUTF8Bytes++-- | Decode a JSON text, return any trailing text.+decodeText :: FromValue a => T.Text -> (T.Text, Either DecodeError a)+{-# INLINE decodeText #-}+decodeText t =+    let (rest, r) = decode (T.getUTF8Bytes t)+    in (T.Text rest, r) -- JSON parser consume bytes in unit of UTF8 codepoint++-- | Decode a JSON doc, only trailing JSON whitespace are allowed. decode' :: FromValue a => V.Bytes -> Either DecodeError a {-# INLINE decode' #-} decode' bs = case P.parse_ (JV.value <* JV.skipSpaces <* P.endOfInput) bs of@@ -474,7 +491,7 @@  -- | Generic encode/decode Settings ----- There should be no control charactors in formatted texts since we don't escaping those+-- There should be no control characters in formatted texts since we don't escaping those -- field names or constructor names ('defaultSettings' relys on Haskell's lexical property). -- Otherwise 'encodeJSON' will output illegal JSON string. data Settings = Settings@@ -960,6 +977,66 @@     {-# INLINE encodeJSON #-}     encodeJSON = encodeJSON . FIS.sortedValues +instance FromValue a => FromValue (A.Array a) where+    {-# INLINE fromValue #-}+    fromValue = withArray "Z.Data.Array.Array"+        (V.traverseWithIndex $ \ k v -> fromValue v <?> Index k)+instance ToValue a => ToValue (A.Array a) where+    {-# INLINE toValue #-}+    toValue = Array . V.map toValue+instance EncodeJSON a => EncodeJSON (A.Array a) where+    {-# INLINE encodeJSON #-}+    encodeJSON = B.square . commaVec'++instance FromValue a => FromValue (A.SmallArray a) where+    {-# INLINE fromValue #-}+    fromValue = withArray "Z.Data.Array.SmallArray"+        (V.traverseWithIndex $ \ k v -> fromValue v <?> Index k)+instance ToValue a => ToValue (A.SmallArray a) where+    {-# INLINE toValue #-}+    toValue = Array . V.map toValue+instance EncodeJSON a => EncodeJSON (A.SmallArray a) where+    {-# INLINE encodeJSON #-}+    encodeJSON = B.square . commaVec'++instance (Prim a, FromValue a) => FromValue (A.PrimArray a) where+    {-# INLINE fromValue #-}+    fromValue = withArray "Z.Data.Array.PrimArray"+        (V.traverseWithIndex $ \ k v -> fromValue v <?> Index k)+instance (Prim a, ToValue a) => ToValue (A.PrimArray a) where+    {-# INLINE toValue #-}+    toValue = Array . V.map toValue+instance (Prim a, EncodeJSON a) => EncodeJSON (A.PrimArray a) where+    {-# INLINE encodeJSON #-}+    encodeJSON = B.square . commaVec'++instance FromValue A.ByteArray where+    {-# INLINE fromValue #-}+    fromValue value = do+        (A.PrimArray ba# :: A.PrimArray Word8) <-+                withArray "Data.Primitive.ByteArray"+                    (V.traverseWithIndex $ \ k v -> fromValue v <?> Index k) value+        return (A.ByteArray ba#)+instance ToValue A.ByteArray where+    {-# INLINE toValue #-}+    toValue (A.ByteArray ba#) =+        Array (V.map toValue (A.PrimArray ba# :: A.PrimArray Word8))+instance EncodeJSON A.ByteArray where+    {-# INLINE encodeJSON #-}+    encodeJSON (A.ByteArray ba#) =+        B.square (commaVec' (A.PrimArray ba# :: A.PrimArray Word8))++instance (A.PrimUnlifted a, FromValue a) => FromValue (A.UnliftedArray a) where+    {-# INLINE fromValue #-}+    fromValue = withArray "Z.Data.Array.UnliftedArray"+        (V.traverseWithIndex $ \ k v -> fromValue v <?> Index k)+instance (A.PrimUnlifted a, ToValue a) => ToValue (A.UnliftedArray a) where+    {-# INLINE toValue #-}+    toValue = Array . V.map toValue+instance (A.PrimUnlifted a, EncodeJSON a) => EncodeJSON (A.UnliftedArray a) where+    {-# INLINE encodeJSON #-}+    encodeJSON = B.square . commaVec'+ instance FromValue a => FromValue (V.Vector a) where     {-# INLINE fromValue #-}     fromValue = withArray "Z.Data.Vector.Vector"@@ -1025,9 +1102,9 @@ instance FromValue Char where     {-# INLINE fromValue #-}     fromValue = withText "Char" $ \ t ->-        case T.headMaybe t of-            Just c -> pure c-            _      -> fail' (T.concat ["converting Char failed, expected a string of length 1"])+        if (T.length t == 1)+        then pure (T.head t)+        else fail' (T.concat ["converting Char failed, expected a string of length 1"]) instance ToValue Char where     {-# INLINE toValue #-}     toValue = String . T.singleton

@@ -86,7 +86,7 @@  -- | Encode a 'Char' into bytes with non-standard UTF-8 encoding(Used in "Data.CBytes"). ----- '\NUL' is encoded as two bytes @C0 80@ , '\xD800' ~ '\xDFFF' is encoded as a three bytes normal UTF-8 codepoint.+-- @\\NUL@ is encoded as two bytes @C0 80@ , @\\xD800@ ~ @\\xDFFF@ is encoded as a three bytes normal UTF-8 codepoint. -- This function assumed there're enough space for encoded bytes, and return the advanced index. encodeCharModifiedUTF8 :: (PrimMonad m) => MutablePrimArray (PrimState m) Word8 -> Int -> Char -> m Int {-# INLINE encodeCharModifiedUTF8 #-}

@@ -13,7 +13,6 @@  module Z.Data.Text.UTF8Rewind where -import Data.Bits import Foreign.C.Types import GHC.Generics 

@@ -7,7 +7,7 @@ Stability   : experimental Portability : non-portable -This module provide fast boxed and unboxed vector with unified interface.+This module provides fast boxed and unboxed vector with unified interface. The API is similar to bytestring and vector. If you find missing functions, please report!  Performance consideration:@@ -26,11 +26,11 @@   * The 'Functor' instance for 'Vector' are lazy in order to abid 'Functor' law.     namely @fmap id vectorConatinBottom == vectorContainBottom@, if you need strict mapping     for lifted 'Vector', use 'map'' ('PrimVector' will never contain bottom thus it's not-    a problem). THIS MAY COME AS A SURPRISE SO MAKE SURE YOU USE THE CORRECT 'map' s.+    a problem). THIS MAY COME AS A SURPRISE SO MAKE SURE YOU USE THE CORRECT 'map'. -  * The 'Foldable' instance for 'Vector' is fine, use 'Prelude' functions such as-    'null', 'length', etc. should not incur performance overhead, though there're-    partial functions you should avoid, i.e. foldl1, foldr1, maximum, minimum. Use+  * The 'Foldable' instance for 'Vector' is fine, you can use 'Prelude' functions such as+    'null', 'length', etc. without incurring performance overhead, though there're+    partial functions you should avoid, i.e. 'foldl1', 'foldr1', 'maximum', 'minimum'. Use     'foldl1Maybe'', 'foldr1Maybe'', 'maximumMaybe', 'minmumMaybe' instead.    * The 'Traversable' instance have specialized implementations for 'ST' and 'IO',

@@ -1268,10 +1268,10 @@ {-# INLINE iPairFromTuple #-} iPairFromTuple (i, v) = IPair i v --- | The chunk size used for I\/O. Currently set to @32k-chunkOverhead@+-- | The chunk size used for I\/O. Currently set to @16k-chunkOverhead@ defaultChunkSize :: Int {-# INLINE defaultChunkSize #-}-defaultChunkSize = 32 * 1024 - chunkOverhead+defaultChunkSize = 16 * 1024 - chunkOverhead  -- | The recommended chunk size. Currently set to @4k - chunkOverhead@. smallChunkSize :: Int

@@ -133,7 +133,7 @@  -- | Pass primitive array to unsafe FFI as pointer. ----- Enable 'UnliftedFFITypes' extension in your haskell code, use proper pointer type and @CSize/CSsize@+-- Enable 'UnliftedFFITypes' extension in your haskell code, use proper pointer type and @HsInt@ -- to marshall @ByteArray#@ and @Int@ arguments on C side. -- -- The second 'Int' arguement is the element size not the bytes size.@@ -222,7 +222,7 @@  -- | Pass primitive array to safe FFI as pointer. ----- Use proper pointer type and @CSize/CSsize@ to marshall @Ptr a@ and @Int@ arguments on C side.+-- Use proper pointer type and @HsInt@ to marshall @Ptr a@ and @Int@ arguments on C side. -- The memory pointed by 'Ptr a' will not moved during call. After call returned, pointer is no longer valid. -- -- The second 'Int' arguement is the element size not the bytes size.

@@ -9,6 +9,7 @@ import           Data.Word import           Data.Hashable              (hashWithSalt, hash) import qualified Z.Data.CBytes              as CB+import qualified Z.Data.JSON                as JSON import           Z.Foreign import qualified Z.Data.Vector.Base         as V import           System.IO.Unsafe@@ -35,6 +36,11 @@             in hash (CB.pack ys) === hash (V.packASCII ys)         prop "CBytes a's hash should be equal to literal's hash" $             hash ("hello world!" :: CB.CBytes) === hash (CB.fromBytes "hello world!")++    describe "CBytes JSON instance property" $ do+        prop "CBytes decodeJSON . encodeJSON === id" $ \ xs ->+            let bs = CB.fromBytes (V.pack xs)+            in Right bs === JSON.decode' (JSON.encodeBytes bs)      describe "CBytes IsString instance property" $ do         prop "ASCII string" $