diff --git a/CHANGELOG.md b/CHANGELOG.md --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,8 @@ -# Revision history for streamly-text +## 0.1.1 (Apr 2026) -* First version. Released on an unsuspecting world. +* Update documentation +* Allow streamly-core < 0.4 and text < 2.2 + +## 0.1.0 (Sep 2025) + +* Initial release diff --git a/README.md b/README.md --- a/README.md +++ b/README.md @@ -1,12 +1,25 @@ # streamly-text -Library for streamly and text interoperation. +Efficient interoperability between +[streamly](https://hackage.haskell.org/package/streamly) arrays and +[text](https://hackage.haskell.org/package/text). -This library is to enable interoperation of streamly with existing code that -uses `Text`. +This library provides **zero-overhead** conversions between the `Text` +type and `streamly` Array types, making it easier to use one in place of +the other. While `text` uses unpinned memory, streamly arrays can use +pinned or unpinned memory and can be intercoverted if required. -The package provides APIs to interconvert between strict `Text` and streamly -`Array Word8` and between lazy `Text` and stream of `Array Word8`. +The strict `Text` type is equivalent to UTF-8 encoded `Array Word8` in +Streamly, the underlying types are compatible so we just need to rewrap +it to interconvert. -The interconversion in the case of strict `Text` and streamly `Array Word8` has -no overhead. +In streamly we work with explicit streams instead of lazy types. +The lazy 'Text' type is equivalent to a stream of UTF-8 encoded 'Array +Word8' in streamly. + +When working with streams we can work directly with Char streams or with +streams of encoded chunks of char sequences i.e. `Array Word8`. A `Char` +stream can be converted to UTF-8 encoded `Word8` stream using `encodeUtf8` +from `Streamly.Unicode.Stream` which in turn can be written as `Array +Word8`, and a stream of UTF-8 encoded `Word8` or `Array Word8` can be +decoded using `decodeUtf8` or `decodeUtf8Chunks`. diff --git a/src/Streamly/Compat/Text.hs b/src/Streamly/Compat/Text.hs --- a/src/Streamly/Compat/Text.hs +++ b/src/Streamly/Compat/Text.hs @@ -3,33 +3,47 @@ {-# LANGUAGE MagicHash #-} {-# LANGUAGE BangPatterns #-} +-- | Efficient interoperability between +-- arrays and +-- . +-- +-- This module provides zero-overhead conversion between strict 'Text' +-- and streamly’s 'Word8' streams or 'Array' 'Word8' type. +-- +-- In streamy we either work directly with 'Word8' streams or 'Array' of +-- 'Word8'. The strict 'Text' type is equivalent to a UTF-8 encoded 'Array +-- Word8' in Streamly, the underlying types are compatible so we just need to +-- rewrap it to interconvert. + module Streamly.Compat.Text - ( toArray - , unsafeFromArray + ( + -- * Construction + unsafeFromArray + , unsafeCreate + -- * Elimination + , toArray , reader - - -- , unsafeCreateOf - , unsafeCreate ) where import Control.Monad.IO.Class (MonadIO) +import Data.Text (Text) import Data.Word (Word8) import GHC.Exts (unsafeCoerce#) +import Streamly.Data.Array (Array) import Streamly.Data.Fold (Fold) import Streamly.Data.Unfold (Unfold, lmap) import qualified Data.Text as T import qualified Data.Text.Array as TArr +import qualified Streamly.Data.Array as Array -- Internal imports -import Data.Text.Internal (Text(..)) -import Streamly.Internal.Data.Array (Array(..)) -import Streamly.Internal.Data.MutByteArray (MutByteArray(..)) - -import qualified Streamly.Data.Array as Array -import qualified Streamly.Internal.Data.MutByteArray as MBArr +-- Tracks all the places where we are using internals of Text +import qualified Data.Text.Internal as TextInternal +import qualified Streamly.Internal.Data.Array as ArrayInternal +import qualified Streamly.Internal.Data.MutByteArray as MBArrInternal import Prelude hiding (read) @@ -37,9 +51,9 @@ -- For text < 2 we need to consider utf16 encoding. #if MIN_VERSION_streamly_core(0,2,2) -#define EMPTY MBArr.empty +#define EMPTY MBArrInternal.empty #else -#define EMPTY MBArr.nil +#define EMPTY MBArrInternal.nil #endif #if MIN_VERSION_streamly_core(0,3,0) @@ -53,29 +67,29 @@ -- | Convert a 'Text' to an array of 'Word8'. It can be done in constant time. {-# INLINE toArray #-} toArray :: Text -> Array Word8 -toArray (Text (TArr.ByteArray _) _ len) - | len == 0 = Array EMPTY 0 0 -toArray (Text (TArr.ByteArray barr#) off8 len8) = - Array (MutByteArray (unsafeCoerce# barr#)) off8 (off8 + len8) +toArray (TextInternal.Text (TArr.ByteArray _) _ len) + | len == 0 = ArrayInternal.Array EMPTY 0 0 +toArray (TextInternal.Text (TArr.ByteArray barr#) off8 len8) = + ArrayInternal.Array (MBArrInternal.MutByteArray (unsafeCoerce# barr#)) off8 (off8 + len8) --- | Treat an an array of 'Word8' as 'Text'. +-- | Treat an array of 'Word8' as 'Text'. -- --- This function is unsafe: the caller must ensure that the 'Array' 'Word8' is a +-- This function is unsafe: the caller must ensure that the 'Array' 'Word8' has a -- valid UTF-8 encoding. -- -- This function unwraps the 'Array' and wraps it with 'Text' constructors and -- hence the operation is performed in constant time. {-# INLINE unsafeFromArray #-} unsafeFromArray :: Array Word8 -> Text -unsafeFromArray Array {..} +unsafeFromArray ArrayInternal.Array {..} | len8 == 0 = T.empty - | otherwise = Text (TArr.ByteArray (unsafeCoerce# marr#)) off8 len8 + | otherwise = TextInternal.Text (TArr.ByteArray (unsafeCoerce# marr#)) off8 len8 where len8 = arrEnd - arrStart off8 = arrStart - !(MutByteArray marr#) = arrContents + !(MBArrInternal.MutByteArray marr#) = arrContents -- | Unfold a 'Text' to a stream of Word8. {-# INLINE reader #-} diff --git a/src/Streamly/Compat/Text/Lazy.hs b/src/Streamly/Compat/Text/Lazy.hs --- a/src/Streamly/Compat/Text/Lazy.hs +++ b/src/Streamly/Compat/Text/Lazy.hs @@ -1,12 +1,37 @@ {-# LANGUAGE CPP #-} +-- | Efficient interoperability between +-- arrays and +-- . +-- +-- This module provides zero-overhead conversion between lazy 'Text' and +-- streamly’s 'Array Word8' or 'Word8' streams. +-- +-- In streamly we work with explicit streams instead of lazy types. +-- The lazy 'Text' type is equivalent to a stream of UTF-8 encoded 'Array +-- Word8' in streamly. +-- +-- The 'Array Word8' types in a stream can be rewrapped as Text types and +-- converted into lazy Text. Similarly, the Text chunks in a lazy Text type can +-- be rewrapped as 'Array Word8' and yielded as a stream. +-- +-- When working with streams we can work directly with Char streams or with +-- streams of encoded chunks of char sequences i.e. 'Array Word8'. A 'Char' +-- stream can be converted to UTF-8 encoded 'Word8' stream using 'encodeUtf8' +-- from 'Streamly.Unicode.Stream' which in turn can be written as 'Array +-- Word8', and a stream of UTF-8 encoded 'Word8' or 'Array Word8' can be +-- decoded using 'decodeUtf8' or 'decodeUtf8Chunks'. + module Streamly.Compat.Text.Lazy - ( chunkReader - , reader + ( + -- * Construction + unsafeFromChunksIO + , unsafeFromChunks + -- * Elimination + , reader , toChunks - , unsafeFromChunks - , unsafeFromChunksIO + , chunkReader ) where @@ -33,7 +58,7 @@ #define UNFOLD_EACH Unfold.many #endif --- | Unfold a lazy 'Text' to a stream of 'Array' 'Words'. +-- | Unfold a lazy 'Text' to a stream of 'Array Word8'. {-# INLINE chunkReader #-} chunkReader :: Monad m => Unfold m Text (Array Word8) chunkReader = Unfold step seed @@ -42,26 +67,25 @@ step (Chunk bs bl) = return $ Yield (Strict.toArray bs) bl step Empty = return Stop --- | Unfold a lazy 'Text' to a stream of Word8 +-- | Unfold a lazy 'Text' to a stream of 'Word8'. {-# INLINE reader #-} reader :: Monad m => Unfold m Text Word8 reader = UNFOLD_EACH Array.reader chunkReader -- XXX Should this be called readChunks? --- | Convert a lazy 'Text' to a serial stream of 'Array' 'Word8'. +-- | Convert a lazy 'Text' to a stream of 'Array Word8'. {-# INLINE toChunks #-} toChunks :: Monad m => Text -> Stream m (Array Word8) toChunks = Stream.unfold chunkReader --- | Convert a serial stream of 'Array' 'Word8' to a lazy 'Text'. +-- | IMPORTANT NOTE: This function is lazy only for lazy monads (e.g. +-- Identity). For strict monads (e.g. /IO/) it consumes the entire input before +-- generating the output. For /IO/ monad use 'unsafeFromChunksIO' instead. -- --- This function is unsafe: the caller must ensure that each 'Array' 'Word8' --- element in the stream is a valid UTF-8 encoding. +-- Convert a stream of 'Array' 'Word8' to a lazy 'Text'. -- --- IMPORTANT NOTE: This function is lazy only for lazy monads --- (e.g. Identity). For strict monads (e.g. /IO/) it consumes the entire input --- before generating the output. For /IO/ monad please use unsafeFromChunksIO --- instead. +-- Unsafe because the caller must ensure that each 'Array Word8' +-- in the stream is UTF-8 encoded and terminates at Char boundary. -- -- For strict monads like /IO/ you could create a newtype wrapper to make the -- monad bind operation lazy and lift the stream to that type using hoist, then @@ -80,6 +104,7 @@ -- @ -- -- /unsafeFromChunks/ can then be used as, +-- -- @ -- {-# INLINE unsafeFromChunksIO #-} -- unsafeFromChunksIO :: Stream IO (Array Word8) -> IO Text @@ -89,8 +114,11 @@ unsafeFromChunks :: Monad m => Stream m (Array Word8) -> m Text unsafeFromChunks = Stream.foldr chunk Empty . fmap Strict.unsafeFromArray --- | Convert a serial stream of 'Array' 'Word8' to a lazy 'Text' in the +-- | Convert a stream of 'Array Word8' to a lazy 'Text' in the -- /IO/ monad. +-- +-- Unsafe because the caller must ensure that each 'Array Word8' +-- in the stream is UTF-8 encoded and terminates at Char boundary. {-# INLINE unsafeFromChunksIO #-} unsafeFromChunksIO :: Stream IO (Array Word8) -> IO Text unsafeFromChunksIO = diff --git a/streamly-text.cabal b/streamly-text.cabal --- a/streamly-text.cabal +++ b/streamly-text.cabal @@ -1,15 +1,30 @@ cabal-version: 2.4 name: streamly-text -version: 0.1.0 -synopsis: Library for streamly and text interoperation. -description: Please see the README on GitHub at +version: 0.1.1 +synopsis: Efficient conversion between Streamly Arrays and Text +description: + Zero-overhead interoperability between the @Array@ type from the + @streamly-core@ package and the @Text@ type from the @text@ package. + . + Provides efficient conversion in both directions without copying. + . + Please see the README at + for more details. category: Streamly, Stream, Array, Text license: BSD-3-Clause license-file: LICENSE author: Composewell Technologies maintainer: streamly@composewell.com -extra-source-files: CHANGELOG.md -extra-doc-files: README.md +extra-doc-files: README.md, CHANGELOG.md +tested-with: + GHC==9.14.1 + , GHC==9.12.4 + , GHC==9.10.3 + , GHC==9.8.4 + , GHC==9.6.3 + , GHC==9.4.8 + , GHC==9.2.8 + , GHC==8.10.7 common compile-options default-language: Haskell2010 @@ -31,9 +46,13 @@ import: compile-options exposed-modules: Streamly.Compat.Text , Streamly.Compat.Text.Lazy - build-depends: base >=4.7 && <5 - , streamly-core >=0.2.0 && <0.3.1 - , text >=2.0 && <2.1.2 + build-depends: base >=4.7 && < 5 + -- Should use minor versions as we depend on text + -- internals, but in practice it is more hassle and + -- little benefit. We can revise it on hackage if + -- it ever becomes a problem. + , streamly-core >=0.2.0 && < 0.4 + , text >=2.0 && < 2.2 hs-source-dirs: src default-language: Haskell2010