hlex (empty) → 0.1.0

raw patch · 11 files changed

+323/−0 lines, 11 filesdep +HUnitdep +basedep +hlexsetup-changed

Dependencies added: HUnit, base, hlex, regex-tdfa

Files

• CHANGELOG.md +7/−0
• LICENSE +21/−0
• README.md +8/−0
• Setup.hs +2/−0
• hlex.cabal +77/−0
• src/Hlex.hs +123/−0
• test/ExampleLang.hs +21/−0
• test/Exceptions.hs +17/−0
• test/Spec.hs +14/−0
• test/Successes.hs +16/−0
• test/TestResources.hs +17/−0

@@ -0,0 +1,7 @@+For the package version policy (PVP), see  http://pvp.haskell.org/faq .++### 0.1.0++_2023-05-06, Sebastian Tee_++Initial version

@@ -0,0 +1,21 @@+MIT License++Copyright (c) 2023 Sebastian Tee++Permission is hereby granted, free of charge, to any person obtaining a copy+of this software and associated documentation files (the "Software"), to deal+in the Software without restriction, including without limitation the rights+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell+copies of the Software, and to permit persons to whom the Software is+furnished to do so, subject to the following conditions:++The above copyright notice and this permission notice shall be included in+all copies or substantial portions of the Software.++THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN+THE SOFTWARE.

@@ -0,0 +1,8 @@+# hlex++![ci](https://github.com/SebTee/Hlex/actions/workflows/ci.yml/badge.svg)++The tools needed to create a lexer from a lexical grammar in Haskell.++You can see an [example language](https://hackage.haskell.org/package/hlex/docs/Hlex.html#g:1)+in the [documentation](https://hackage.haskell.org/package/hlex/docs/doc-index.html).

@@ -0,0 +1,2 @@+import Distribution.Simple+main = defaultMain

@@ -0,0 +1,77 @@+cabal-version: 1.12++name:           hlex+version:        0.1.0+synopsis:       Simple Lexer Creation+description:    This package provides the tools to create a simple lexer.+category:       lexer+homepage:       https://github.com/SebTee/hlex#readme+bug-reports:    https://github.com/SebTee/hlex/issues+author:         Sebastian Tee+maintainer:     Sebastian Tee+copyright:      2023 Sebastian Tee+license:        MIT+license-file:   LICENSE+build-type:     Simple+++extra-source-files:+  README.md+  CHANGELOG.md++source-repository head+  type: git+  location: https://github.com/SebTee/hlex++library+  exposed-modules:+      Hlex+  other-modules:+      Paths_hlex+  hs-source-dirs:+      src+  ghc-options:+      -Wall+      -Wcompat+      -Widentities+      -Wincomplete-record-updates+      -Wincomplete-uni-patterns+      -Wmissing-export-lists+      -Wmissing-home-modules+      -Wpartial-fields+      -Wredundant-constraints+  build-depends:+      base >=4.7 && <5+    , regex-tdfa ==1.3.*+  default-language: Haskell2010++test-suite Hlex-test+  type: exitcode-stdio-1.0+  main-is: Spec.hs+  other-modules:+      ExampleLang+      Exceptions+      Successes+      TestResources+      Paths_hlex+  hs-source-dirs:+      test+  ghc-options:+      -Wall+      -Wcompat+      -Widentities+      -Wincomplete-record-updates+      -Wincomplete-uni-patterns+      -Wmissing-export-lists+      -Wmissing-home-modules+      -Wpartial-fields+      -Wredundant-constraints+      -threaded+      -rtsopts+      -with-rtsopts=-N+  build-depends:+      HUnit ==1.6.*+    , hlex+    , base >=4.7 && <5+    , regex-tdfa ==1.3.*+  default-language: Haskell2010

@@ -0,0 +1,123 @@+{-|+Module      : Hlex+Description : Lexer creation tools+Copyright   : (c) Sebastian Tee, 2023+License     : MIT++Tools needed to create a 'Lexer' from a lexical 'Grammar'.+-}+module Hlex+     ( -- * Example+       -- $example++       -- * Types+       Grammar+     , TokenSyntax(..)+     , Lexer+       -- ** Exceptions+     , LexException(..)+       -- * Functions+     , hlex+     ) where++import Text.Regex.TDFA ((=~))++-- | Exception thrown when a 'Lexer' is unable to lex a string.+data LexException = LexException+  Int -- ^ The line number where the string that couldn't be lexed is located.+  Int -- ^ The column where the string that couldn't be lexed is located.+  String -- ^ The String that couldn't be lexed.+  deriving(Read, Show, Eq)++-- | These are the individual rules that make up a 'Grammar'.+--+-- Takes a __POSIX regular expression__ then converts it to a token or skips it.+data TokenSyntax token+  = Skip -- ^ Skips over any matches.+    String -- ^ Regular expression.+  | Tokenize -- ^ Takes a function that converts the matched string to a token.+    String -- ^ Regular expression.+    (String -> token) -- ^ Function that converts the matched string into a token.+  | JustToken -- ^ Converts any regular expression matches to a given token.+    String -- ^ Regular expression.+    token -- ^ Given token.++type InternalToken token = (String, Maybe (String -> token))++-- | Lexical grammar made up of 'TokenSyntax' rules.+--+-- The __order is important__. The 'Lexer' will apply each 'TokenSyntax' rule in the order listed.+type Grammar token = [TokenSyntax token]++-- | Converts a string into a list of tokens.+-- If the string does not follow the Lexer's 'Grammar' a 'LexException' will be returned.+type Lexer token = String -> Either LexException [token]++tokenizerToInternalToken :: TokenSyntax a -> InternalToken a+tokenizerToInternalToken (Skip regex) = (regex, Nothing)+tokenizerToInternalToken (Tokenize regex toToken) = (regex, Just toToken)+tokenizerToInternalToken (JustToken regex token) = (regex, Just $ \_ -> token)++-- | Takes a given 'Grammar' and turns it into a 'Lexer'.+hlex :: Grammar token -> Lexer token+hlex = lexInternal 1 1 . map tokenizerToInternalToken++lexInternal :: Int -> Int -> [InternalToken token] -> Lexer token+lexInternal _ _ _ "" = Right []+lexInternal row col ((regex, t):grammar) program = if null matchedText+  then lexInternal row col grammar program+  else do+    before <- parsedBefore+    after <- parsedAfter+    case t of+      Nothing -> Right $ before ++ after+      Just tk -> Right $ before ++ tk matchedText : after+  where+    (beforeProgram, matchedText, afterProgram) = program =~ regex :: (String, String, String)+    (afterRow, afterCol) = getLastCharPos row col (beforeProgram ++ matchedText)+    parsedBefore = lexInternal row col grammar beforeProgram+    parsedAfter = lexInternal afterRow afterCol ((regex, t):grammar) afterProgram+lexInternal row col _ invalidString = Left $ LexException row col invalidString++getLastCharPos :: Int -> Int -> String -> (Int, Int)+getLastCharPos startRow startCol x = (startRow + addRow, addCol + if addRow == 0 then startCol else 1)+  where+    ls = lines x+    addRow = length ls - 1+    addCol = length $ last ls++{- $example+Here is an example module for a simple language.++@+  module ExampleLang+       ( MyToken(..) -- Export the language's tokens and the lexer+       , myLexer+       ) where++  import Hlex++  data MyToken = Ident String -- String identifier token+               | Number Float -- Number token and numeric value+               | Assign       -- Assignment operator token+               deriving(Show)++  myGrammar :: Grammar MyToken+  myGrammar = [ JustToken "=" Assign                                       -- "=" Operator becomes the assign token+              , Tokenize "[a-zA-Z]+" (\match -> Ident match)                -- Identifier token with string+              , Tokenize "[0-9]+(\\.[0-9]+)?" (\match -> Number (read match) -- Number token with the parsed numeric value stored as a Float+              , Skip "[ \\n\\r\\t]+"                                          -- Skip whitespace+              ]++  myLexer :: Lexer MyToken+  myLexer = hlex myGrammar -- hlex turns a Grammar into a Lexer+@++Here is the lexer being used on a simple program.++>>> lexer "x = 1.2"+Right [Ident "x", Assign, Number 1.2]++The lexer uses 'Either'. Right means the lexer successfully parsed the program to a list of MyTokens.+If Left was returned it would be a 'LexException'.+-}

@@ -0,0 +1,21 @@+module ExampleLang+     ( Token(..)+     , lexer+     ) where++import Hlex++data Token = Ident String+           | Number Float+           | Assign+           deriving(Read, Show, Eq)++grammar :: Grammar Token+grammar = [ JustToken "=" Assign+          , Tokenize "[a-zA-Z]+" Ident+          , Tokenize "[0-9]+(\\.[0-9]+)?" $ Number . read+          , Skip "[ \n\r\t]+"+          ]++lexer :: Lexer Token+lexer = hlex grammar

@@ -0,0 +1,17 @@+module Exceptions (exceptions) where++import Test.HUnit+import Hlex+import TestResources+import ExampleLang++exceptions :: Test+exceptions = TestList [ TestLabel "Location Exception" exceptionLocation+                      , TestLabel "Number Parse Exception" exceptionNumParse+                      ]++exceptionLocation :: Test+exceptionLocation = TestCase $ assertLexException lexer "aaaa\n\naaaaa\naaa\naaa//bbbaa\naaaaa" $ LexException 5 4 "//"++exceptionNumParse :: Test+exceptionNumParse = TestCase $ assertLexException lexer "10.2.3" $ LexException 1 5 "."

@@ -0,0 +1,14 @@+import System.Exit+import Test.HUnit+import Exceptions+import Successes++main :: IO ()+main = do+  results <- runTestTT tests+  case results of+    Counts {errors = 0, failures = 0} -> exitSuccess+    _ -> exitFailure++tests :: Test+tests = TestList [TestLabel "Successes" successes, TestLabel "Exceptions" exceptions]

@@ -0,0 +1,16 @@+module Successes (successes) where++import Test.HUnit+import TestResources+import ExampleLang++successes :: Test+successes = TestList [ TestLabel "Parse Number" parseNum+                     , TestLabel "Assign Number" assignNumber+                     ]++parseNum :: Test+parseNum = TestCase $ assertLexResult lexer "12 3.5" [Number 12, Number 3.5]++assignNumber :: Test+assignNumber = TestCase $ assertLexResult lexer "x = 1" [Ident "x", Assign, Number 1]

@@ -0,0 +1,17 @@+module TestResources+     ( assertLexResult+     , assertLexException+     ) where++import Test.HUnit+import Hlex++assertLexResult :: (Eq a, Show a) => Lexer a -> String -> [a] -> IO()+assertLexResult lexer program expectedTokens = case lexer program of+  Right tokens -> assertEqual "Incorrectly parsed" expectedTokens tokens+  Left err -> assertFailure $ "Failed with the following exception " ++ (show err)++assertLexException :: (Show a) => Lexer a -> String -> LexException -> IO ()+assertLexException lexer program expectedException = case lexer program of+  Right tokens -> assertFailure $ "Successfully parsed to: " ++ show tokens+  Left err -> assertEqual "Incorrect exception" expectedException err