packages feed

streamly-text 0.1.0 → 0.1.1

raw patch · 5 files changed

+133/−54 lines, 5 filesdep ~streamly-coredep ~textnew-uploader

Dependency ranges changed: streamly-core, text

Files

CHANGELOG.md view
@@ -1,3 +1,8 @@-# Revision history for streamly-text+## 0.1.1 (Apr 2026) -* First version. Released on an unsuspecting world.+* Update documentation+* Allow streamly-core < 0.4 and text < 2.2++## 0.1.0 (Sep 2025)++* Initial release
README.md view
@@ -1,12 +1,25 @@ # streamly-text -Library for streamly and text interoperation.+Efficient interoperability between+[streamly](https://hackage.haskell.org/package/streamly) arrays and+[text](https://hackage.haskell.org/package/text). -This library is to enable interoperation of streamly with existing code that-uses `Text`.+This library provides **zero-overhead** conversions between the `Text`+type and `streamly` Array types, making it easier to use one in place of+the other. While `text` uses unpinned memory, streamly arrays can use+pinned or unpinned memory and can be intercoverted if required. -The package provides APIs to interconvert between strict `Text` and streamly-`Array Word8` and between lazy `Text` and stream of `Array Word8`.+The strict `Text` type is equivalent to UTF-8 encoded `Array Word8` in+Streamly, the underlying types are compatible so we just need to rewrap+it to interconvert. -The interconversion in the case of strict `Text` and streamly `Array Word8` has-no overhead.+In streamly we work with explicit streams instead of lazy types.+The lazy 'Text' type is equivalent to a stream of UTF-8 encoded 'Array+Word8' in streamly.++When working with streams we can work directly with Char streams or with+streams of encoded chunks of char sequences i.e. `Array Word8`. A `Char`+stream can be converted to UTF-8 encoded `Word8` stream using `encodeUtf8`+from `Streamly.Unicode.Stream` which in turn can be written as `Array+Word8`, and a stream of UTF-8 encoded `Word8` or `Array Word8` can be+decoded using `decodeUtf8` or `decodeUtf8Chunks`.
src/Streamly/Compat/Text.hs view
@@ -3,33 +3,47 @@ {-# LANGUAGE MagicHash #-} {-# LANGUAGE BangPatterns #-} +-- | Efficient interoperability between+-- <https://hackage.haskell.org/package/streamly streamly> arrays and+-- <https://hackage.haskell.org/package/text text>.+--+-- This module provides zero-overhead conversion between strict 'Text'+-- and streamly’s 'Word8' streams or 'Array' 'Word8' type.+--+-- In streamy we either work directly with 'Word8' streams or 'Array' of+-- 'Word8'. The strict 'Text' type is equivalent to a UTF-8 encoded 'Array+-- Word8' in Streamly, the underlying types are compatible so we just need to+-- rewrap it to interconvert.+ module Streamly.Compat.Text-  ( toArray-  , unsafeFromArray+  (+  -- * Construction+    unsafeFromArray+  , unsafeCreate +  -- * Elimination+  , toArray   , reader--  -- , unsafeCreateOf-  , unsafeCreate   ) where  import Control.Monad.IO.Class (MonadIO)+import Data.Text (Text) import Data.Word (Word8) import GHC.Exts (unsafeCoerce#)+import Streamly.Data.Array (Array) import Streamly.Data.Fold (Fold) import Streamly.Data.Unfold (Unfold, lmap)  import qualified Data.Text as T import qualified Data.Text.Array as TArr+import qualified Streamly.Data.Array as Array  -- Internal imports-import Data.Text.Internal (Text(..))-import Streamly.Internal.Data.Array (Array(..))-import Streamly.Internal.Data.MutByteArray (MutByteArray(..))--import qualified Streamly.Data.Array as Array-import qualified Streamly.Internal.Data.MutByteArray as MBArr+-- Tracks all the places where we are using internals of Text+import qualified Data.Text.Internal as TextInternal+import qualified Streamly.Internal.Data.Array as ArrayInternal+import qualified Streamly.Internal.Data.MutByteArray as MBArrInternal  import Prelude hiding (read) @@ -37,9 +51,9 @@ -- For text < 2 we need to consider utf16 encoding.  #if MIN_VERSION_streamly_core(0,2,2)-#define EMPTY MBArr.empty+#define EMPTY MBArrInternal.empty #else-#define EMPTY MBArr.nil+#define EMPTY MBArrInternal.nil #endif  #if MIN_VERSION_streamly_core(0,3,0)@@ -53,29 +67,29 @@ -- | Convert a 'Text' to an array of 'Word8'. It can be done in constant time. {-# INLINE toArray #-} toArray :: Text -> Array Word8-toArray (Text (TArr.ByteArray _) _ len)-    | len == 0 = Array EMPTY 0 0-toArray (Text (TArr.ByteArray barr#) off8 len8) =-    Array (MutByteArray (unsafeCoerce# barr#)) off8 (off8 + len8)+toArray (TextInternal.Text (TArr.ByteArray _) _ len)+    | len == 0 = ArrayInternal.Array EMPTY 0 0+toArray (TextInternal.Text (TArr.ByteArray barr#) off8 len8) =+    ArrayInternal.Array (MBArrInternal.MutByteArray (unsafeCoerce# barr#)) off8 (off8 + len8) --- | Treat an an array of 'Word8' as 'Text'.+-- | Treat an array of 'Word8' as 'Text'. ----- This function is unsafe: the caller must ensure that the 'Array' 'Word8' is a+-- This function is unsafe: the caller must ensure that the 'Array' 'Word8' has a -- valid UTF-8 encoding. -- -- This function unwraps the 'Array' and wraps it with 'Text' constructors and -- hence the operation is performed in constant time. {-# INLINE unsafeFromArray #-} unsafeFromArray :: Array Word8 -> Text-unsafeFromArray Array {..}+unsafeFromArray ArrayInternal.Array {..}     | len8 == 0 = T.empty-    | otherwise = Text (TArr.ByteArray (unsafeCoerce# marr#)) off8 len8+    | otherwise = TextInternal.Text (TArr.ByteArray (unsafeCoerce# marr#)) off8 len8      where      len8 = arrEnd - arrStart     off8 = arrStart-    !(MutByteArray marr#) = arrContents+    !(MBArrInternal.MutByteArray marr#) = arrContents  -- | Unfold a 'Text' to a stream of Word8. {-# INLINE reader #-}
src/Streamly/Compat/Text/Lazy.hs view
@@ -1,12 +1,37 @@ {-# LANGUAGE CPP #-} +-- | Efficient interoperability between+-- <https://hackage.haskell.org/package/streamly streamly> arrays and+-- <https://hackage.haskell.org/package/text text>.+--+-- This module provides zero-overhead conversion between lazy 'Text' and+-- streamly’s 'Array Word8' or 'Word8' streams.+--+-- In streamly we work with explicit streams instead of lazy types.+-- The lazy 'Text' type is equivalent to a stream of UTF-8 encoded 'Array+-- Word8' in streamly.+--+-- The 'Array Word8' types in a stream can be rewrapped as Text types and+-- converted into lazy Text. Similarly, the Text chunks in a lazy Text type can+-- be rewrapped as 'Array Word8' and yielded as a stream.+--+-- When working with streams we can work directly with Char streams or with+-- streams of encoded chunks of char sequences i.e. 'Array Word8'. A 'Char'+-- stream can be converted to UTF-8 encoded 'Word8' stream using 'encodeUtf8'+-- from 'Streamly.Unicode.Stream' which in turn can be written as 'Array+-- Word8', and a stream of UTF-8 encoded 'Word8' or 'Array Word8' can be+-- decoded using 'decodeUtf8' or 'decodeUtf8Chunks'.+ module Streamly.Compat.Text.Lazy-  ( chunkReader-  , reader+  (+  -- * Construction+    unsafeFromChunksIO+  , unsafeFromChunks +  -- * Elimination+  , reader   , toChunks-  , unsafeFromChunks-  , unsafeFromChunksIO+  , chunkReader   ) where @@ -33,7 +58,7 @@ #define UNFOLD_EACH Unfold.many #endif --- | Unfold a lazy 'Text' to a stream of 'Array' 'Words'.+-- | Unfold a lazy 'Text' to a stream of 'Array Word8'. {-# INLINE  chunkReader #-} chunkReader :: Monad m => Unfold m Text (Array Word8) chunkReader = Unfold step seed@@ -42,26 +67,25 @@     step (Chunk bs bl) = return $ Yield (Strict.toArray bs) bl     step Empty = return Stop --- | Unfold a lazy 'Text' to a stream of Word8+-- | Unfold a lazy 'Text' to a stream of 'Word8'. {-# INLINE reader #-} reader :: Monad m => Unfold m Text Word8 reader = UNFOLD_EACH Array.reader chunkReader  -- XXX Should this be called readChunks?--- | Convert a lazy 'Text' to a serial stream of 'Array' 'Word8'.+-- | Convert a lazy 'Text' to a stream of 'Array Word8'. {-# INLINE toChunks #-} toChunks :: Monad m => Text -> Stream m (Array Word8) toChunks = Stream.unfold chunkReader --- | Convert a serial stream of 'Array' 'Word8' to a lazy 'Text'.+-- | IMPORTANT NOTE: This function is lazy only for lazy monads (e.g.+-- Identity). For strict monads (e.g. /IO/) it consumes the entire input before+-- generating the output. For /IO/ monad use 'unsafeFromChunksIO' instead. ----- This function is unsafe: the caller must ensure that each 'Array' 'Word8'--- element in the stream is a valid UTF-8 encoding.+-- Convert a stream of 'Array' 'Word8' to a lazy 'Text'. ----- IMPORTANT NOTE: This function is lazy only for lazy monads--- (e.g. Identity). For strict monads (e.g. /IO/) it consumes the entire input--- before generating the output. For /IO/ monad please use unsafeFromChunksIO--- instead.+-- Unsafe because the caller must ensure that each 'Array Word8'+-- in the stream is UTF-8 encoded and terminates at Char boundary. -- -- For strict monads like /IO/ you could create a newtype wrapper to make the -- monad bind operation lazy and lift the stream to that type using hoist, then@@ -80,6 +104,7 @@ -- @ -- -- /unsafeFromChunks/ can then be used as,+-- -- @ -- {-# INLINE unsafeFromChunksIO #-} -- unsafeFromChunksIO :: Stream IO (Array Word8) -> IO Text@@ -89,8 +114,11 @@ unsafeFromChunks :: Monad m => Stream m (Array Word8) -> m Text unsafeFromChunks = Stream.foldr chunk Empty . fmap Strict.unsafeFromArray --- | Convert a serial stream of 'Array' 'Word8' to a lazy 'Text' in the+-- | Convert a stream of 'Array Word8' to a lazy 'Text' in the -- /IO/ monad.+--+-- Unsafe because the caller must ensure that each 'Array Word8'+-- in the stream is UTF-8 encoded and terminates at Char boundary. {-# INLINE unsafeFromChunksIO #-} unsafeFromChunksIO :: Stream IO (Array Word8) -> IO Text unsafeFromChunksIO =
streamly-text.cabal view
@@ -1,15 +1,30 @@ cabal-version:       2.4 name:                streamly-text-version:             0.1.0-synopsis:            Library for streamly and text interoperation.-description:         Please see the README on GitHub at <https://github.com/composewell/streamly-text>+version:             0.1.1+synopsis: Efficient conversion between Streamly Arrays and Text+description:+  Zero-overhead interoperability between the @Array@ type from the+  @streamly-core@ package and the @Text@ type from the @text@ package.+  .+  Provides efficient conversion in both directions without copying.+  .+  Please see the README at <https://github.com/composewell/streamly-text>+  for more details. category:            Streamly, Stream, Array, Text license:             BSD-3-Clause license-file:        LICENSE author:              Composewell Technologies maintainer:          streamly@composewell.com-extra-source-files:  CHANGELOG.md-extra-doc-files:     README.md+extra-doc-files:     README.md, CHANGELOG.md+tested-with:+    GHC==9.14.1+  , GHC==9.12.4+  , GHC==9.10.3+  , GHC==9.8.4+  , GHC==9.6.3+  , GHC==9.4.8+  , GHC==9.2.8+  , GHC==8.10.7  common compile-options     default-language: Haskell2010@@ -31,9 +46,13 @@   import:              compile-options   exposed-modules:     Streamly.Compat.Text                      , Streamly.Compat.Text.Lazy-  build-depends:       base          >=4.7   && <5-                     , streamly-core >=0.2.0 && <0.3.1-                     , text          >=2.0   && <2.1.2+  build-depends:       base          >=4.7   && < 5+                     -- Should use minor versions as we depend on text+                     -- internals, but in practice it is more hassle and+                     -- little benefit. We can revise it on hackage if+                     -- it ever becomes a problem.+                     , streamly-core >=0.2.0 && < 0.4+                     , text          >=2.0   && < 2.2   hs-source-dirs:      src   default-language:    Haskell2010