packages feed

streamly-text (empty) → 0.1.0

raw patch · 8 files changed

+397/−0 lines, 8 filesdep +basedep +hspecdep +quickcheck-instancessetup-changed

Dependencies added: base, hspec, quickcheck-instances, random, streamly-core, streamly-text, temporary, text

Files

+ CHANGELOG.md view
@@ -0,0 +1,3 @@+# Revision history for streamly-text++* First version. Released on an unsuspecting world.
+ LICENSE view
@@ -0,0 +1,30 @@+Copyright (c) 2023, Composewell Technologies++All rights reserved.++Redistribution and use in source and binary forms, with or without+modification, are permitted provided that the following conditions are met:++    * Redistributions of source code must retain the above copyright+      notice, this list of conditions and the following disclaimer.++    * Redistributions in binary form must reproduce the above+      copyright notice, this list of conditions and the following+      disclaimer in the documentation and/or other materials provided+      with the distribution.++    * Neither the name of Composewell Techonologies nor the names of other+      contributors may be used to endorse or promote products derived+      from this software without specific prior written permission.++THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ README.md view
@@ -0,0 +1,12 @@+# streamly-text++Library for streamly and text interoperation.++This library is to enable interoperation of streamly with existing code that+uses `Text`.++The package provides APIs to interconvert between strict `Text` and streamly+`Array Word8` and between lazy `Text` and stream of `Array Word8`.++The interconversion in the case of strict `Text` and streamly `Array Word8` has+no overhead.
+ Setup.hs view
@@ -0,0 +1,2 @@+import Distribution.Simple+main = defaultMain
+ src/Streamly/Compat/Text.hs view
@@ -0,0 +1,93 @@+{-# LANGUAGE CPP             #-}+{-# LANGUAGE RecordWildCards #-}+{-# LANGUAGE MagicHash #-}+{-# LANGUAGE BangPatterns #-}++module Streamly.Compat.Text+  ( toArray+  , unsafeFromArray++  , reader++  -- , unsafeCreateOf+  , unsafeCreate+  )+where++import Control.Monad.IO.Class (MonadIO)+import Data.Word (Word8)+import GHC.Exts (unsafeCoerce#)+import Streamly.Data.Fold (Fold)+import Streamly.Data.Unfold (Unfold, lmap)++import qualified Data.Text as T+import qualified Data.Text.Array as TArr++-- Internal imports+import Data.Text.Internal (Text(..))+import Streamly.Internal.Data.Array (Array(..))+import Streamly.Internal.Data.MutByteArray (MutByteArray(..))++import qualified Streamly.Data.Array as Array+import qualified Streamly.Internal.Data.MutByteArray as MBArr++import Prelude hiding (read)++-- This module currently only supports text >= 2.+-- For text < 2 we need to consider utf16 encoding.++#if MIN_VERSION_streamly_core(0,2,2)+#define EMPTY MBArr.empty+#else+#define EMPTY MBArr.nil+#endif++#if MIN_VERSION_streamly_core(0,3,0)+#define CREATE_OF Array.createOf+#define CREATE Array.create+#else+#define CREATE_OF Array.writeN+#define CREATE Array.write+#endif++-- | Convert a 'Text' to an array of 'Word8'. It can be done in constant time.+{-# INLINE toArray #-}+toArray :: Text -> Array Word8+toArray (Text (TArr.ByteArray _) _ len)+    | len == 0 = Array EMPTY 0 0+toArray (Text (TArr.ByteArray barr#) off8 len8) =+    Array (MutByteArray (unsafeCoerce# barr#)) off8 (off8 + len8)++-- | Treat an an array of 'Word8' as 'Text'.+--+-- This function is unsafe: the caller must ensure that the 'Array' 'Word8' is a+-- valid UTF-8 encoding.+--+-- This function unwraps the 'Array' and wraps it with 'Text' constructors and+-- hence the operation is performed in constant time.+{-# INLINE unsafeFromArray #-}+unsafeFromArray :: Array Word8 -> Text+unsafeFromArray Array {..}+    | len8 == 0 = T.empty+    | otherwise = Text (TArr.ByteArray (unsafeCoerce# marr#)) off8 len8++    where++    len8 = arrEnd - arrStart+    off8 = arrStart+    !(MutByteArray marr#) = arrContents++-- | Unfold a 'Text' to a stream of Word8.+{-# INLINE reader #-}+reader :: Monad m => Unfold m Text Word8+reader = lmap toArray Array.reader++-- | Fold a stream of Word8 to a 'Text' of given size in bytes.+{-# INLINE _unsafeCreateOf #-}+_unsafeCreateOf :: MonadIO m => Int -> Fold m Word8 Text+_unsafeCreateOf i = unsafeFromArray <$> CREATE_OF i++-- | Fold a stream of Word8 to a 'Text' of appropriate size.+{-# INLINE unsafeCreate #-}+unsafeCreate :: MonadIO m => Fold m Word8 Text+unsafeCreate = unsafeFromArray <$> CREATE
+ src/Streamly/Compat/Text/Lazy.hs view
@@ -0,0 +1,100 @@+{-# LANGUAGE CPP #-}++module Streamly.Compat.Text.Lazy+  ( chunkReader+  , reader++  , toChunks+  , unsafeFromChunks+  , unsafeFromChunksIO+  )+where++import Data.Word (Word8)+import Streamly.Data.Array (Array)+import System.IO.Unsafe (unsafeInterleaveIO)+import Streamly.Data.Stream (Stream)++-- Internal imports+import Data.Text.Internal.Lazy (Text(..), chunk)+import Streamly.Internal.Data.Stream (Step(..))+import Streamly.Internal.Data.Unfold (Unfold(..))++import qualified Streamly.Compat.Text as Strict+import qualified Streamly.Data.Array as Array+import qualified Streamly.Data.Unfold as Unfold+import qualified Streamly.Data.Stream as Stream++import Prelude hiding (read)++#if MIN_VERSION_streamly_core(0,3,0)+#define UNFOLD_EACH Unfold.unfoldEach+#else+#define UNFOLD_EACH Unfold.many+#endif++-- | Unfold a lazy 'Text' to a stream of 'Array' 'Words'.+{-# INLINE  chunkReader #-}+chunkReader :: Monad m => Unfold m Text (Array Word8)+chunkReader = Unfold step seed+  where+    seed = return+    step (Chunk bs bl) = return $ Yield (Strict.toArray bs) bl+    step Empty = return Stop++-- | Unfold a lazy 'Text' to a stream of Word8+{-# INLINE reader #-}+reader :: Monad m => Unfold m Text Word8+reader = UNFOLD_EACH Array.reader chunkReader++-- XXX Should this be called readChunks?+-- | Convert a lazy 'Text' to a serial stream of 'Array' 'Word8'.+{-# INLINE toChunks #-}+toChunks :: Monad m => Text -> Stream m (Array Word8)+toChunks = Stream.unfold chunkReader++-- | Convert a serial stream of 'Array' 'Word8' to a lazy 'Text'.+--+-- This function is unsafe: the caller must ensure that each 'Array' 'Word8'+-- element in the stream is a valid UTF-8 encoding.+--+-- IMPORTANT NOTE: This function is lazy only for lazy monads+-- (e.g. Identity). For strict monads (e.g. /IO/) it consumes the entire input+-- before generating the output. For /IO/ monad please use unsafeFromChunksIO+-- instead.+--+-- For strict monads like /IO/ you could create a newtype wrapper to make the+-- monad bind operation lazy and lift the stream to that type using hoist, then+-- you can use this function to generate the text lazily. For example you can+-- wrap the /IO/ type to make the bind lazy like this:+--+-- @+-- newtype LazyIO a = LazyIO { runLazy :: IO a } deriving (Functor, Applicative)+--+-- liftToLazy :: IO a -> LazyIO a+-- liftToLazy = LazyIO+--+-- instance Monad LazyIO where+--   return = pure+--   LazyIO a >>= f = LazyIO (unsafeInterleaveIO a >>= unsafeInterleaveIO . runLazy . f)+-- @+--+-- /unsafeFromChunks/ can then be used as,+-- @+-- {-# INLINE unsafeFromChunksIO #-}+-- unsafeFromChunksIO :: Stream IO (Array Word8) -> IO Text+-- unsafeFromChunksIO str = runLazy (unsafeFromChunks (Stream.hoist liftToLazy str))+-- @+{-# INLINE unsafeFromChunks #-}+unsafeFromChunks :: Monad m => Stream m (Array Word8) -> m Text+unsafeFromChunks = Stream.foldr chunk Empty . fmap Strict.unsafeFromArray++-- | Convert a serial stream of 'Array' 'Word8' to a lazy 'Text' in the+-- /IO/ monad.+{-# INLINE unsafeFromChunksIO #-}+unsafeFromChunksIO :: Stream IO (Array Word8) -> IO Text+unsafeFromChunksIO =+    -- Although the /IO/ monad is strict in nature we emulate laziness using+    -- 'unsafeInterleaveIO'.+    Stream.foldrM (\x b -> chunk x <$> unsafeInterleaveIO b) (pure Empty)+        . fmap Strict.unsafeFromArray
+ streamly-text.cabal view
@@ -0,0 +1,54 @@+cabal-version:       2.4+name:                streamly-text+version:             0.1.0+synopsis:            Library for streamly and text interoperation.+description:         Please see the README on GitHub at <https://github.com/composewell/streamly-text>+category:            Streamly, Stream, Array, Text+license:             BSD-3-Clause+license-file:        LICENSE+author:              Composewell Technologies+maintainer:          streamly@composewell.com+extra-source-files:  CHANGELOG.md+extra-doc-files:     README.md++common compile-options+    default-language: Haskell2010+    ghc-options:    -Weverything+                    -Wno-implicit-prelude+                    -Wno-missing-deriving-strategies+                    -Wno-missing-exported-signatures+                    -Wno-missing-import-lists+                    -Wno-missing-local-signatures+                    -Wno-missing-safe-haskell-mode+                    -Wno-missed-specialisations+                    -Wno-all-missed-specialisations+                    -Wno-monomorphism-restriction+                    -Wno-prepositive-qualified-module+                    -Wno-unsafe+                    -Rghc-timing++library+  import:              compile-options+  exposed-modules:     Streamly.Compat.Text+                     , Streamly.Compat.Text.Lazy+  build-depends:       base          >=4.7   && <5+                     , streamly-core >=0.2.0 && <0.3.1+                     , text          >=2.0   && <2.1.2+  hs-source-dirs:      src+  default-language:    Haskell2010++test-suite test+  import:              compile-options+  hs-source-dirs:     test+  type:               exitcode-stdio-1.0+  main-is:            Main.hs+  build-depends:+      base >=4.7 && <5+    , text+    , hspec+    , quickcheck-instances+    , random+    , streamly-core+    , streamly-text+    , temporary+  default-language:   Haskell2010
+ test/Main.hs view
@@ -0,0 +1,103 @@+{-# LANGUAGE RankNTypes #-}++module Main (main) where++import System.Random (randomIO)+import System.IO.Temp (withSystemTempFile)+import Test.Hspec+import Test.Hspec.QuickCheck+import Test.QuickCheck.Instances.Text ()+import System.IO (hClose)+import Data.Functor.Identity (Identity(..))++import qualified Streamly.Internal.Data.Array as Array (unsafeCast)++import qualified Data.Text as BS+import qualified Data.Text.Lazy as BSL+import qualified Streamly.FileSystem.FileIO as File+import qualified Streamly.FileSystem.Path as Path+import qualified Streamly.Compat.Text as Strict+import qualified Streamly.Compat.Text.Lazy as Lazy+import qualified Streamly.Data.Stream as Stream++pipeline ::+       (Eq t, Show t)+    => t+    -> (t -> a)+    -> (a -> m t)+    -> (forall b. m b -> IO b)+    -> IO ()+pipeline txt fromTxt toTxt runMonad = do+    txt1 <-+        runMonad $ do+            let a = fromTxt txt+            toTxt a+    txt1 `shouldBe` txt++pipelinePropStrict ::+       String+    -> (BS.Text -> a)+    -> (a -> m BS.Text)+    -> (forall b. m b -> IO b)+    -> Spec+pipelinePropStrict info fromTxt toTxt runMonad =+    prop info $ \txt -> pipeline txt fromTxt toTxt runMonad++pipelinePropLazy ::+       String+    -> (BSL.Text -> a)+    -> (a -> m BSL.Text)+    -> (forall b. m b -> IO b)+    -> Spec+pipelinePropLazy info fromTxt toTxt runMonad =+    prop info $ \txt -> pipeline txt fromTxt toTxt runMonad++writeRead :: Int -> IO ()+writeRead n = do+    str <- sequence $ replicate n (randomIO :: IO Char)+    let txt = BSL.pack str+    withSystemTempFile "temp" $ \fp0 hdl -> do+        fp <- Path.fromString fp0+        hClose hdl+        let strm = fmap Array.unsafeCast $ Lazy.toChunks txt+        Stream.fold (File.writeChunks fp) strm+        let strm1 = fmap Array.unsafeCast $ File.readChunks fp+        txt1 <- Lazy.unsafeFromChunksIO strm1+        txt1 `shouldBe` txt++main :: IO ()+main = do+    hspec $ do+        describe "IO"+            $ it "write & read (100000)" $ writeRead 100000+        describe "Text tests" $ do+            pipelinePropStrict+                "Strict.unsafeFromArray . Strict.toArray"+                Strict.toArray+                (pure . Strict.unsafeFromArray)+                (pure . runIdentity)+            pipelinePropStrict+                "Strict.unsafeCreate . Strict.reader"+                (Stream.unfold Strict.reader)+                (Stream.fold Strict.unsafeCreate)+                id+            pipelinePropLazy+                "Lazy.unsafeFromChunks . Lazy.toChunks"+                Lazy.toChunks+                Lazy.unsafeFromChunks+                (pure . runIdentity)+            pipelinePropLazy+                "Lazy.unsafeFromChunksIO . Lazy.toChunks"+                Lazy.toChunks+                Lazy.unsafeFromChunksIO+                id+            pipelinePropLazy+                "Lazy.unsafeFromChunks . Lazy.chunkReader"+                (Stream.unfold Lazy.chunkReader)+                Lazy.unsafeFromChunks+                (pure . runIdentity)+            pipelinePropLazy+                "fmap Text.fromStrict . Strict.unsafeCreate . Lazy.reader"+                (Stream.unfold Lazy.reader)+                (fmap BSL.fromStrict . Stream.fold Strict.unsafeCreate)+                id