diff --git a/CHANGELOG.md b/CHANGELOG.md --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,12 @@ Changelog ========= +2.1 +--- + +* Restore compatibility with older GHCs +* Change the type of `longestShortest` + 2.0 --- diff --git a/lexer-applicative.cabal b/lexer-applicative.cabal --- a/lexer-applicative.cabal +++ b/lexer-applicative.cabal @@ -2,7 +2,7 @@ -- documentation, see http://haskell.org/cabal/users-guide/ name: lexer-applicative -version: 2.0 +version: 2.1 synopsis: Simple lexer based on applicative regular expressions description: Simple lexer based on applicative regular expressions homepage: https://github.com/feuerbach/lexer-applicative diff --git a/src/Language/Lexer/Applicative.hs b/src/Language/Lexer/Applicative.hs --- a/src/Language/Lexer/Applicative.hs +++ b/src/Language/Lexer/Applicative.hs @@ -1,4 +1,4 @@ -{-# LANGUAGE ScopedTypeVariables, DeriveDataTypeable, DeriveFunctor, TypeFamilies #-} +{-# LANGUAGE ScopedTypeVariables, DeriveDataTypeable, DeriveFunctor #-} -- | For some background, see -- module Language.Lexer.Applicative @@ -27,7 +27,6 @@ import Data.Monoid import Data.Function import Control.Exception -import GHC.Exts ---------------------------------------------------------------------- -- Lexer @@ -44,7 +43,7 @@ -- myLexer = 'mconcat' -- [ 'token' ('longest' myToken) -- , 'whitespace' ('longest' myWhiteSpace) --- , 'whitespace' ('longestShortest' myComment) +-- , 'whitespace' ('longestShortest' myCommentPrefix myCommentSuffix) -- ] -- @ data Lexer tok = Lexer @@ -102,11 +101,11 @@ -- -- * @'longest' (r1 '<|>' r2) = 'longest' r1 '<>' 'longest' r2@ -- --- * @'longest' r = 'longestShortest' 'const' r ('const' '$' 'pure' ())@ +-- * @'longest' r = 'longestShortest' r 'pure'@ longest :: RE Char tok -> Recognizer tok -longest re = longestShortest const re (const $ pure ()) +longest re = longestShortest re pure -- | This is a more sophisticated recognizer than 'longest'. -- @@ -131,7 +130,6 @@ -- -- @ -- 'longestShortest' --- (\\_ _ -> ()) -- don't care about the comment text -- ('string' "\/*") -- (\\_ -> 'many' 'anySym' '*>' 'string' "*\/") -- @ @@ -151,52 +149,33 @@ -- must match or else a 'LexicalError' is thrown. Therefore, -- -- @ --- 'longestShortest' f pref suff1 +-- 'longestShortest' pref suff1 -- '<>' --- 'longestShortest' f pref suff2 +-- 'longestShortest' pref suff2 -- = --- 'longestShortest' f pref suff1 +-- 'longestShortest' pref suff1 -- @ -- -- and is not the same as -- --- @'longestShortest' f pref (suff1 '<|>' suff2)@ +-- @'longestShortest' pref (suff1 '<|>' suff2)@ -- -- The following holds, however: -- -- @ --- 'longestShortest' f pref1 suff +-- 'longestShortest' pref1 suff -- '<>' --- 'longestShortest' f pref2 suff +-- 'longestShortest' pref2 suff -- = --- 'longestShortest' f (pref1 '<|>' pref2) suff +-- 'longestShortest' (pref1 '<|>' pref2) suff -- @ --- --- \* * * --- --- Passing the result of prefix into both suffix and combining function may --- seem superfluous; indeed we could get away with --- --- @'RE' 'Char' pref -> (pref -> 'RE' 'Char' tok) -> 'Recognizer' tok@ --- --- or even --- --- @'RE' 'Char' ('RE' 'Char' tok) -> 'Recognizer' tok@ --- --- This is done purely for convenience and readability; the intention is --- that @pref@ passed into suffix is used to customize the regular --- expression which would still return only its part of the token, and then --- the function will combine the two parts. Of course, you don't need to --- follow this recommendation. Thanks to parametricity, all three versions --- are equivalent. longestShortest - :: (pref -> suff -> tok) - -> RE Char pref -- ^ regex for the longest prefix - -> (pref -> RE Char suff) -- ^ regex for the shortest suffix + :: RE Char pref -- ^ regex for the longest prefix + -> (pref -> RE Char tok) -- ^ regex for the shortest suffix -> Recognizer tok -longestShortest f prefRE suffRE = +longestShortest prefRE suffRE = Recognizer $ - (\pref -> f pref <$> suffRE pref) <$> prefRE + suffRE <$> prefRE ---------------------------------------------------------------------- -- Running a Lexer @@ -216,11 +195,6 @@ | TsEof | TsError LexicalError deriving (Eq, Functor, Show) - -instance IsList (TokenStream tok) where - type Item (TokenStream tok) = tok - toList = streamToList - fromList = foldr TsToken TsEof -- | Convert a 'TokenStream' to a list of tokens. Turn 'TsError' into -- a runtime 'LexicalError' exception. diff --git a/tests/test.hs b/tests/test.hs --- a/tests/test.hs +++ b/tests/test.hs @@ -1,4 +1,5 @@ -{-# LANGUAGE ScopedTypeVariables, OverloadedLists #-} +{-# LANGUAGE ScopedTypeVariables #-} + import Test.Tasty import Test.Tasty.HUnit @@ -28,9 +29,8 @@ -- but also matching delimiters like /*** ... ***/ (to make it more fun) blockComment = token $ longestShortest - (,) ((++) <$> string "/" <*> many (sym '*')) - (\start -> (++) <$> many anySym <*> string (reverse start)) + (\start -> (,) start <$> ((++) <$> many anySym <*> string (reverse start))) main = defaultMain $ testGroup "Tests" [ testCase "Empty string" $ @@ -67,22 +67,17 @@ @?= Right [Left ("/*"," xxx */"),Right "yyy",Left ("/***"," abc ***/"),Right "ef"] , testCase "longestShortest (failure of shortest; end of stream)" $ - (tokensEither (whitespace $ longestShortest (\_ _ -> ()) (string "abc") (const empty)) + (tokensEither (whitespace $ longestShortest (string "abc") (const empty)) "-" "abc" :: Either LexicalError [L ()]) @?= Left (LexicalError (Pos "-" 1 3 2)) , testCase "longestShortest (failure of shortest; not end of stream)" $ - (tokensEither (whitespace $ longestShortest (\_ _ -> ()) (string "abc") (const empty)) + (tokensEither (whitespace $ longestShortest (string "abc") (const empty)) "-" "abc " :: Either LexicalError [L ()]) @?= Left (LexicalError (Pos "-" 1 3 2)) - , testCase "instance IsList TokenStream" $ do - let r :: TokenStream Int - r = fmap unLoc $ runLexer (longestToken decimal <> ws) "-" "1 2 3" - [1,2,3] <- return r -- testing pattern match, i.e. toList - r @?= [1,2,3] -- testing fromList ] -- orphan