lexer-applicative 2.0 → 2.1

raw patch · 4 files changed

+27/−52 lines, 4 files

Files

• CHANGELOG.md +6/−0
• lexer-applicative.cabal +1/−1
• src/Language/Lexer/Applicative.hs +15/−41
• tests/test.hs +5/−10

@@ -1,6 +1,12 @@ Changelog ========= +2.1+---++* Restore compatibility with older GHCs+* Change the type of `longestShortest`+ 2.0 --- 

@@ -2,7 +2,7 @@ -- documentation, see http://haskell.org/cabal/users-guide/  name:                lexer-applicative-version:             2.0+version:             2.1 synopsis:            Simple lexer based on applicative regular expressions description:         Simple lexer based on applicative regular expressions homepage:            https://github.com/feuerbach/lexer-applicative

@@ -1,4 +1,4 @@-{-# LANGUAGE ScopedTypeVariables, DeriveDataTypeable, DeriveFunctor, TypeFamilies #-}+{-# LANGUAGE ScopedTypeVariables, DeriveDataTypeable, DeriveFunctor #-} -- | For some background, see -- <https://ro-che.info/articles/2015-01-02-lexical-analysis> module Language.Lexer.Applicative@@ -27,7 +27,6 @@ import Data.Monoid import Data.Function import Control.Exception-import GHC.Exts  ---------------------------------------------------------------------- --                             Lexer@@ -44,7 +43,7 @@ --  myLexer = 'mconcat' --    [ 'token'      ('longest' myToken) --    , 'whitespace' ('longest' myWhiteSpace)---    , 'whitespace' ('longestShortest' myComment)+--    , 'whitespace' ('longestShortest' myCommentPrefix myCommentSuffix) --    ] -- @ data Lexer tok = Lexer@@ -102,11 +101,11 @@ -- -- * @'longest' (r1 '<|>' r2) = 'longest' r1 '<>' 'longest' r2@ ----- * @'longest' r = 'longestShortest' 'const' r ('const' '$' 'pure' ())@+-- * @'longest' r = 'longestShortest' r 'pure'@ longest   :: RE Char tok   -> Recognizer tok-longest re = longestShortest const re (const $ pure ())+longest re = longestShortest re pure  -- | This is a more sophisticated recognizer than 'longest'. --@@ -131,7 +130,6 @@ -- -- @ -- 'longestShortest'---    (\\_ _ -> ()) -- don't care about the comment text --    ('string' "\/*") --    (\\_ -> 'many' 'anySym' '*>' 'string' "*\/") -- @@@ -151,52 +149,33 @@ -- must match or else a 'LexicalError' is thrown. Therefore, -- -- @--- 'longestShortest' f pref suff1+-- 'longestShortest' pref suff1 --          '<>'--- 'longestShortest' f pref suff2+-- 'longestShortest' pref suff2 --          =--- 'longestShortest' f pref suff1+-- 'longestShortest' pref suff1 -- @ -- -- and is not the same as ----- @'longestShortest' f pref (suff1 '<|>' suff2)@+-- @'longestShortest' pref (suff1 '<|>' suff2)@ -- -- The following holds, however: -- -- @--- 'longestShortest' f pref1 suff+-- 'longestShortest' pref1 suff --          '<>'--- 'longestShortest' f pref2 suff+-- 'longestShortest' pref2 suff --          =--- 'longestShortest' f (pref1 '<|>' pref2) suff+-- 'longestShortest' (pref1 '<|>' pref2) suff -- @------ \* * *------ Passing the result of prefix into both suffix and combining function may--- seem superfluous; indeed we could get away with------ @'RE' 'Char' pref -> (pref -> 'RE' 'Char' tok) -> 'Recognizer' tok@------ or even------ @'RE' 'Char' ('RE' 'Char' tok) -> 'Recognizer' tok@------ This is done purely for convenience and readability; the intention is--- that @pref@ passed into suffix is used to customize the regular--- expression which would still return only its part of the token, and then--- the function will combine the two parts. Of course, you don't need to--- follow this recommendation. Thanks to parametricity, all three versions--- are equivalent. longestShortest-  :: (pref -> suff -> tok)-  -> RE Char pref -- ^ regex for the longest prefix-  -> (pref -> RE Char suff) -- ^ regex for the shortest suffix+  :: RE Char pref -- ^ regex for the longest prefix+  -> (pref -> RE Char tok) -- ^ regex for the shortest suffix   -> Recognizer tok-longestShortest f prefRE suffRE =+longestShortest prefRE suffRE =   Recognizer $-    (\pref -> f pref <$> suffRE pref) <$> prefRE+    suffRE <$> prefRE  ---------------------------------------------------------------------- --                           Running a Lexer@@ -216,11 +195,6 @@   | TsEof   | TsError LexicalError   deriving (Eq, Functor, Show)--instance IsList (TokenStream tok) where-  type Item (TokenStream tok) = tok-  toList = streamToList-  fromList = foldr TsToken TsEof  -- | Convert a 'TokenStream' to a list of tokens. Turn 'TsError' into -- a runtime 'LexicalError' exception.

@@ -1,4 +1,5 @@-{-# LANGUAGE ScopedTypeVariables, OverloadedLists #-}+{-# LANGUAGE ScopedTypeVariables #-}+ import Test.Tasty import Test.Tasty.HUnit @@ -28,9 +29,8 @@ -- but also matching delimiters like /*** ... ***/ (to make it more fun) blockComment = token $   longestShortest-    (,)     ((++) <$> string "/" <*> many (sym '*'))-    (\start -> (++) <$> many anySym <*> string (reverse start))+    (\start -> (,) start <$> ((++) <$> many anySym <*> string (reverse start)))  main = defaultMain $ testGroup "Tests"   [ testCase "Empty string" $@@ -67,22 +67,17 @@       @?=         Right [Left ("/*"," xxx */"),Right "yyy",Left ("/***"," abc ***/"),Right "ef"]   , testCase "longestShortest (failure of shortest; end of stream)" $-      (tokensEither (whitespace $ longestShortest (\_ _ -> ()) (string "abc") (const empty))+      (tokensEither (whitespace $ longestShortest (string "abc") (const empty))         "-"         "abc" :: Either LexicalError [L ()])       @?=         Left (LexicalError (Pos "-" 1 3 2))   , testCase "longestShortest (failure of shortest; not end of stream)" $-      (tokensEither (whitespace $ longestShortest (\_ _ -> ()) (string "abc") (const empty))+      (tokensEither (whitespace $ longestShortest (string "abc") (const empty))         "-"         "abc " :: Either LexicalError [L ()])       @?=         Left (LexicalError (Pos "-" 1 3 2))-  , testCase "instance IsList TokenStream" $ do-      let r :: TokenStream Int-          r = fmap unLoc $ runLexer (longestToken decimal <> ws) "-" "1 2 3"-      [1,2,3] <- return r -- testing pattern match, i.e. toList-      r @?= [1,2,3] -- testing fromList   ]  -- orphan