tokenizer-monad 0.2.0.0 → 0.2.1.0
raw patch · 4 files changed
+184/−95 lines, 4 files
Files
- README.md +16/−2
- src/Control/Monad/Tokenizer.hs +54/−91
- src/Control/Monad/Tokenizer/Class.hs +107/−0
- tokenizer-monad.cabal +7/−2
README.md view
@@ -11,9 +11,23 @@ should be suppressed, you can 'discard' them instead (or filter afterwards). This package supports Strings, strict and lazy Text, as well as strict and-lazy ASCII ByteStrings.+lazy ASCII ByteStrings. The generic `Tokenizer` type from module `Control.Monad.Tokenizer` takes the text type as its first argument. In the other modules, `Tokenizer` is+already specialized to a specific text type. It is recommendable to avoid+importing more than one module from this package. Instead, you could just switch to a more general one. -**Examples**:+## Provided functions++The functions provided by this package can be divided into three categories:++ - *tests* peek characters from the input text or check a condition; they have+ no effect on the turtle position nor on the emissions. Examples: `peek`,+ `isEOT`, `lookAhead`+ - *walkers* modify the turtle position, but have no effect on the emissions.+ Examples: `walk`, `walkBack`, `pop`, `restore`+ - *commits* cut off the input text at the current position, and emit or+ discard the visited part. Examples: `emit`, `discard`++## Examples This tokenizer is equivalent to `words` from Prelude:
src/Control/Monad/Tokenizer.hs view
@@ -42,6 +42,13 @@ emit, discard, restore,+ -- * Embedding+ embed,+ embed_,+ discardAndEmbed,+ -- * Conversion+ convert,+ convertWith, -- * Text types Tokenizable(..) ) where@@ -49,45 +56,10 @@ import Control.Monad import Data.Char import Data.Monoid+import Data.String import qualified Data.Text as T import qualified Data.Text.Lazy as LT---- | Text types that can be split by the Tokenizer monad. In this module,--- instances are provided for String, strict Text, and lazy Text.--- If you are dealing with ASCII ByteStrings, you can find instances in--- the modules "Control.Monad.Tokenizer.Char8.Strict" and--- "Control.Monad.Tokenizer.Char8.Lazy"-class Tokenizable t where- tnull :: t -> Bool- thead :: t -> Char- ttail :: t -> t- ttake :: Int -> t -> t- tdrop :: Int -> t -> t- tlower :: t -> t--instance Tokenizable T.Text where- tnull = T.null- thead = T.head- ttail = T.tail- ttake = T.take- tdrop = T.drop- tlower = T.toLower--instance Tokenizable LT.Text where- tnull = LT.null- thead = LT.head- ttail = LT.tail- ttake = LT.take . fromIntegral- tdrop = LT.drop . fromIntegral- tlower = LT.toLower--instance Tokenizable [Char] where- tnull = null- thead = head- ttail = tail- ttake = take- tdrop = drop- tlower = map toLower+import Control.Monad.Tokenizer.Class -- | Tokenizer monad. Use runTokenizer or runTokenizerCS to run it newtype Tokenizer t a = Tokenizer { runTokenizer' :: (t, Int, t) -> (a,[t] -> [t],t,Int,t) }@@ -108,75 +80,66 @@ (a2,o2,w2,!c2,t2) = runTokenizer' (f a1) (w1,c1,t1) in (a2,o1.o2,w2,c2,t2) --- | Check if the next input chars agree with the given string-lookAhead :: Tokenizable t => [Char] -> Tokenizer t Bool-lookAhead chars =+instance Tokenizable t => MonadTokenizer (Tokenizer t) where+ lookAhead chars = Tokenizer $ \(whole,count,tail) -> let h = unpack $ ttake (length chars) tail in (h == chars, id, whole, count, tail)- where unpack t | tnull t = []- | otherwise = thead t : unpack (ttail t)+ where unpack t | tnull t = []+ | otherwise = thead t : unpack (ttail t) --- | Proceed to the next character-walk :: Tokenizable t => Tokenizer t ()-walk = Tokenizer $ \(whole,count,tail) ->- if tnull tail- then ((),id,whole,count,tail)- else ((),id,whole,count+1,ttail tail)+ walk = Tokenizer $ \(whole,count,tail) ->+ if tnull tail+ then ((),id,whole,count,tail)+ else ((),id,whole,count+1,ttail tail) --- | Walk back to the previous character, unless it was discarded/emitted.-walkBack :: Tokenizable t => Tokenizer t ()-walkBack = Tokenizer $ \(whole,count,_) ->- if count > 0- then ((),id,whole,count-1,tdrop (count-1) whole)- else ((),id,whole,0,whole)+ walkBack = Tokenizer $ \(whole,count,_) ->+ if count > 0+ then ((),id,whole,count-1,tdrop (count-1) whole)+ else ((),id,whole,0,whole) --- | Restore the state after the last emit/discard.-restore :: Tokenizer t ()-restore = Tokenizer $ \(whole,_,_) -> ((),id,whole,0,whole)+ restore = Tokenizer $ \(whole,_,_) -> ((),id,whole,0,whole) --- | Peek the current character-peek :: Tokenizable t => Tokenizer t Char-peek = Tokenizer $ \(whole,count,tail) -> (th tail,id,whole,count,tail)+ peek = Tokenizer $ \(whole,count,tail) -> (th tail,id,whole,count,tail) where th t | tnull t = '\0' | otherwise = thead t---- | Peek the current character and proceed-pop :: Tokenizable t => Tokenizer t Char-pop = peek <* walk --- | Break at the current position and emit the scanned token-emit :: Tokenizable t => Tokenizer t ()-emit = Tokenizer $ \(whole,count,tail) -> ((),(ttake count whole:),tail,0,tail)+ emit = Tokenizer $ \(whole,count,tail) -> ((),(ttake count whole:),tail,0,tail) --- | Break at the current position and discard the scanned token-discard :: Tokenizer t ()-discard = Tokenizer $ \(whole,count,tail) -> ((),id,tail,0,tail)+ discard = Tokenizer $ \(whole,count,tail) -> ((),id,tail,0,tail) --- | Have I reached the end of the input text?-isEOT :: Tokenizable t => Tokenizer t Bool-isEOT = Tokenizer $ \(whole, count, tail) -> (tnull tail, id, whole, count, tail)+ isEOT = Tokenizer $ \(whole, count, tail) -> (tnull tail, id, whole, count, tail) --- | Proceed as long as a given function succeeds-walkWhile :: Tokenizable t => (Char -> Bool) -> Tokenizer t ()-walkWhile f = do- c <- peek- when (c /= '\0' && f c) $- walk >> walkWhile f+-- | Embed a pure tokenizer into the monad. The arguments to the function are (visited string, remaining string), and the return value is expected to be (result, emitted tokens, remaining string).+embed :: Tokenizable t => ((t,t) -> (a,[t],t)) -> Tokenizer t a+embed f = Tokenizer $ \(whole,count,tail) ->+ let (a, rs, rem) = f (tdrop count whole, tail)+ in (a, (rs++), rem, 0, rem) --- | Proceed as long as a given fold returns Just (generalization of walkWhile)-walkFold :: Tokenizable t => a -> (Char -> a -> Maybe a) -> Tokenizer t ()-walkFold s0 f = do- c <- peek- unless (c == '\0') $ case f c s0 of- Nothing -> return ()- Just s -> walk >> walkFold s f+-- | Embed a pure tokenizer into the monad without a result.+embed_ :: Tokenizable t => ((t,t) -> ([t],t)) -> Tokenizer t ()+embed_ f = embed f'+ where f' (v,u) = let (rs,rem) = f (v,u) in ((),rs,rem) --- | Repeat a given tokenizer as long as the end of text is not reached-untilEOT :: Tokenizable t => Tokenizer t () -> Tokenizer t ()-untilEOT f = do- eot <- isEOT- unless eot $ f >> untilEOT f+-- | Embed a pure tokenizer into the monad. The visited string is discarded, and+-- the given function is run on the unvisited part. The return value is expected+-- to be (result, emitted tokens, remaining string).+discardAndEmbed :: Tokenizable t => (t -> (a,[t],t)) -> Tokenizer t a+discardAndEmbed f = discard >> embed (f . snd)++-- | Natural transformation to convert between tokenizers of different text types. Note that this operation does not perform encoding\/decoding (i.e. converting from ByteString to Text does not decode Unicode characters). To do so, use convertWith a provide the correct encoding\/decoding functions.+convert :: (Tokenizable t, IsString t, Tokenizable s, IsString s) =>+ Tokenizer s a -> Tokenizer t a+convert = convertWith fromString unpack . convertWith unpack fromString+ where unpack t | tnull t = []+ | otherwise = thead t : unpack (ttail t)++-- | Natural transformation to convert between tokenizers of different text types, using the given conversion functions.+convertWith :: (s -> t) -> (t -> s) -> Tokenizer s a -> Tokenizer t a+convertWith fw bw (Tokenizer runTok) = Tokenizer $ \(whole,count,tail) ->+ let (a, o, w, c, t) = runTok (bw whole, count, bw tail)+ o' = map fw $ o []+ in (a, (o'++), fw w, c, fw t) -- | Split a string into tokens using the given tokenizer runTokenizer :: Tokenizable t => Tokenizer t () -> t -> [t]
+ src/Control/Monad/Tokenizer/Class.hs view
@@ -0,0 +1,107 @@+{-# LANGUAGE FlexibleInstances #-}++-- | A monad class for writing pure tokenizers in an imperative-looking way.+--+-- Main idea: You 'walk' through the input string like a turtle, and everytime+-- you find a token boundary, you call 'emit'. If some specific kinds of tokens+-- should be suppressed, you can 'discard' them instead (or filter afterwards).+--+-- An concrete instance of this class is provided in "Control.Monad.Tokenizer".++module Control.Monad.Tokenizer.Class (+ -- * Monad class+ MonadTokenizer(..),+ untilEOT,+ -- * Utilities+ pop,+ walkWhile,+ walkFold,+ -- * Text types+ Tokenizable(..)+ ) where++import Control.Monad+import Data.Char+import qualified Data.Text as T+import qualified Data.Text.Lazy as LT++-- | Text types that can be split by the Tokenizer monad. In this module,+-- instances are provided for String, strict Text, and lazy Text.+-- If you are dealing with ASCII ByteStrings, you can find instances in+-- the modules "Control.Monad.Tokenizer.Char8.Strict" and+-- "Control.Monad.Tokenizer.Char8.Lazy"+class Tokenizable t where+ tnull :: t -> Bool+ thead :: t -> Char+ ttail :: t -> t+ ttake :: Int -> t -> t+ tdrop :: Int -> t -> t+ tlower :: t -> t++instance Tokenizable T.Text where+ tnull = T.null+ thead = T.head+ ttail = T.tail+ ttake = T.take+ tdrop = T.drop+ tlower = T.toLower++instance Tokenizable LT.Text where+ tnull = LT.null+ thead = LT.head+ ttail = LT.tail+ ttake = LT.take . fromIntegral+ tdrop = LT.drop . fromIntegral+ tlower = LT.toLower++instance Tokenizable [Char] where+ tnull = null+ thead = head+ ttail = tail+ ttake = take+ tdrop = drop+ tlower = map toLower++-- | A monad for turtle tokenization.+class Monad m => MonadTokenizer m where+ -- | Proceed to the next character+ walk :: m ()+ -- | Walk back to the previous character, unless it was discarded/emitted.+ walkBack :: m ()+ -- | Peek the current character+ peek :: m Char+ -- | Restore the state after the last emit/discard.+ restore :: m ()+ -- | Break at the current position and emit the scanned token+ emit :: m ()+ -- | Break at the current position and discard the scanned token+ discard :: m ()+ -- | Have I reached the end of the input text?+ isEOT :: m Bool+ -- | Check if the next input chars agree with the given string+ lookAhead :: [Char] -> m Bool++-- | Peek the current character and proceed+pop :: MonadTokenizer m => m Char+pop = peek <* walk++-- | Proceed as long as a given function succeeds+walkWhile :: MonadTokenizer m => (Char -> Bool) -> m ()+walkWhile f = do+ c <- peek+ when (c /= '\0' && f c) $+ walk >> walkWhile f++-- | Proceed as long as a given fold returns Just (generalization of walkWhile)+walkFold :: MonadTokenizer m => a -> (Char -> a -> Maybe a) -> m ()+walkFold s0 f = do+ c <- peek+ unless (c == '\0') $ case f c s0 of+ Nothing -> return ()+ Just s -> walk >> walkFold s f++-- | Repeat a given tokenizer as long as the end of text is not reached+untilEOT :: MonadTokenizer m => m () -> m ()+untilEOT f = do+ eot <- isEOT+ unless eot $ f >> untilEOT f
tokenizer-monad.cabal view
@@ -10,7 +10,7 @@ -- PVP summary: +-+------- breaking API changes -- | | +----- non-breaking API additions -- | | | +--- code changes with no API change-version: 0.2.0.0+version: 0.2.1.0 -- A short (one-line) description of the package. synopsis: An efficient and easy-to-use tokenizer monad.@@ -47,6 +47,10 @@ cabal-version: >=1.10 +source-repository head+ type: darcs+ location: https://hub.darcs.net/enum/tokenizer-monad+ library -- Modules exported by the library. exposed-modules: Control.Monad.Tokenizer@@ -55,6 +59,7 @@ Control.Monad.Tokenizer.String Control.Monad.Tokenizer.Char8.Strict Control.Monad.Tokenizer.Char8.Lazy+ Control.Monad.Tokenizer.Class -- Modules included in this library but not exported. -- other-modules: @@ -63,7 +68,7 @@ other-extensions: OverloadedStrings, BangPatterns -- Other library packages from which modules are imported.- build-depends: base >=4.9 && <4.11,+ build-depends: base >=4.9 && <5, text >=1.2, bytestring