alex 3.0 → 3.0.1
raw patch · 6 files changed
+139/−41 lines, 6 files
Files
- alex.cabal +3/−1
- templates/wrappers.hs +32/−20
- tests/Makefile +1/−1
- tests/tokens_bytestring.x +15/−19
- tests/tokens_posn_bytestring.x +46/−0
- tests/tokens_strict_bytestring.x +42/−0
alex.cabal view
@@ -1,5 +1,5 @@ name: alex-version: 3.0+version: 3.0.1 license: BSD3 license-file: LICENSE copyright: (c) Chis Dornan, Simon Marlow@@ -52,6 +52,8 @@ tests/tokens_gscan.x tests/tokens_posn.x tests/tokens_bytestring.x+ tests/tokens_posn_bytestring.x+ tests/tokens_strict_bytestring.x tests/unicode.x source-repository head
templates/wrappers.hs view
@@ -8,7 +8,8 @@ #if defined(ALEX_BASIC_BYTESTRING) || defined(ALEX_POSN_BYTESTRING) || defined(ALEX_MONAD_BYTESTRING) import qualified Data.Char-import qualified Data.ByteString.Lazy as ByteString+import qualified Data.ByteString.Lazy as ByteString+import qualified Data.ByteString.Internal as ByteString (w2c) #elif defined(ALEX_STRICT_BYTESTRING) @@ -80,11 +81,40 @@ alexGetByte (p,_,cs) | ByteString.null cs = Nothing | otherwise = let b = ByteString.head cs cs' = ByteString.tail cs- c = Data.Char.chr (fromIntegral b)+ c = ByteString.w2c b p' = alexMove p c in p' `seq` cs' `seq` Just (b, (p', c, cs')) #endif +#ifdef ALEX_BASIC_BYTESTRING+type AlexInput = (Char,+ ByteString.ByteString)++alexInputPrevChar :: AlexInput -> Char+alexInputPrevChar (c,_) = c++alexGetByte (_, cs)+ | ByteString.null cs = Nothing+ | otherwise = Just (ByteString.head cs,+ (ByteString.w2c $ ByteString.head cs,+ ByteString.tail cs))+#endif++#ifdef ALEX_STRICT_BYTESTRING+data AlexInput = AlexInput { alexChar :: {-# UNPACK #-}!Char+ , alexStr :: {-# UNPACK #-}!ByteString.ByteString }++alexInputPrevChar :: AlexInput -> Char+alexInputPrevChar = alexChar++alexGetByte (AlexInput _ cs)+ | ByteString.null cs = Nothing+ | otherwise = Just $! (ByteString.head cs, AlexInput c cs')+ where+ (c,cs') = (ByteString.w2c (ByteString.unsafeHead cs)+ , ByteString.unsafeTail cs)+#endif+ -- ----------------------------------------------------------------------------- -- Token positions @@ -318,13 +348,7 @@ -- Basic wrapper, ByteString version #ifdef ALEX_BASIC_BYTESTRING-type AlexInput = (Char,ByteString.ByteString) -alexGetByte (_, cs) | ByteString.null cs = Nothing- | otherwise = Just (ByteString.head cs, (ByteString.head cs, ByteString.tail cs))--alexInputPrevChar (c,_) = c- -- alexScanTokens :: String -> [token] alexScanTokens str = go ('\n',str) where go inp@(_,str) =@@ -338,18 +362,6 @@ #endif #ifdef ALEX_STRICT_BYTESTRING--data AlexInput = AlexInput { alexChar :: {-# UNPACK #-}!Char- , alexStr :: {-# UNPACK #-}!ByteString.ByteString }--alexGetByte (AlexInput _ cs)- | ByteString.null cs = Nothing- | otherwise = Just $! (ByteString.head cs, AlexInput c cs')- where- (c,cs') = (ByteString.w2c (ByteString.unsafeHead cs)- , ByteString.unsafeTail cs)--alexInputPrevChar = alexChar -- alexScanTokens :: String -> [token] alexScanTokens str = go (AlexInput '\n' str)
tests/Makefile view
@@ -9,7 +9,7 @@ HS_PROG_EXT = .bin endif -TESTS = unicode.x simple.x tokens.x tokens_posn.x tokens_gscan.x tokens_bytestring.x+TESTS = unicode.x simple.x tokens.x tokens_posn.x tokens_gscan.x tokens_bytestring.x tokens_posn_bytestring.x tokens_strict_bytestring.x TEST_ALEX_OPTS = --template=..
tests/tokens_bytestring.x view
@@ -5,7 +5,7 @@ import Data.ByteString.Lazy.Char8 (unpack) } -%wrapper "posn-bytestring"+%wrapper "basic-bytestring" $digit = 0-9 -- digits $alpha = [a-zA-Z] -- alphabetic characters@@ -14,33 +14,29 @@ $white+ ; "--".* ;- let { tok (\p s -> Let p) }- in { tok (\p s -> In p) }- $digit+ { tok (\p s -> Int p (read (unpack s))) }- [\=\+\-\*\/\(\)] { tok (\p s -> Sym p (head (unpack s))) }- $alpha [$alpha $digit \_ \']* { tok (\p s -> Var p (unpack s)) }+ let { \s -> Let }+ in { \s -> In }+ $digit+ { \s -> Int (read (unpack s)) }+ [\=\+\-\*\/\(\)] { \s -> Sym (head (unpack s)) }+ $alpha [$alpha $digit \_ \']* { \s -> Var (unpack s) } {--- Each right-hand side has type :: AlexPosn -> String -> Token---- Some action helpers:-tok f p s = f p s+-- Each right-hand side has type :: ByteString -> Token -- The token type: data Token =- Let AlexPosn |- In AlexPosn |- Sym AlexPosn Char |- Var AlexPosn String |- Int AlexPosn Int |- Err AlexPosn+ Let |+ In |+ Sym Char |+ Var String |+ Int Int |+ Err deriving (Eq,Show) main = if test1 /= result1 then exitFailure- else exitWith ExitSuccess+ else exitWith ExitSuccess test1 = alexScanTokens " let in 012334\n=+*foo bar__'"-result1 = [Let (AlexPn 2 1 3),In (AlexPn 6 1 7),Int (AlexPn 9 1 10) 12334,Sym (AlexPn 16 2 1) '=',Sym (AlexPn 17 2 2) '+',Sym (AlexPn 18 2 3) '*',Var (AlexPn 19 2 4) "foo",Var (AlexPn 23 2 8) "bar__'"]-+result1 = [Let,In,Int 12334,Sym '=',Sym '+',Sym '*',Var "foo",Var "bar__'"] }
+ tests/tokens_posn_bytestring.x view
@@ -0,0 +1,46 @@+{+{-# LANGUAGE OverloadedStrings #-}+module Main (main) where+import System.Exit+import Data.ByteString.Lazy.Char8 (unpack)+}++%wrapper "posn-bytestring"++$digit = 0-9 -- digits+$alpha = [a-zA-Z] -- alphabetic characters++tokens :-++ $white+ ;+ "--".* ;+ let { tok (\p s -> Let p) }+ in { tok (\p s -> In p) }+ $digit+ { tok (\p s -> Int p (read (unpack s))) }+ [\=\+\-\*\/\(\)] { tok (\p s -> Sym p (head (unpack s))) }+ $alpha [$alpha $digit \_ \']* { tok (\p s -> Var p (unpack s)) }++{+-- Each right-hand side has type :: AlexPosn -> String -> Token++-- Some action helpers:+tok f p s = f p s++-- The token type:+data Token =+ Let AlexPosn |+ In AlexPosn |+ Sym AlexPosn Char |+ Var AlexPosn String |+ Int AlexPosn Int |+ Err AlexPosn+ deriving (Eq,Show)++main = if test1 /= result1 then exitFailure+ else exitWith ExitSuccess++test1 = alexScanTokens " let in 012334\n=+*foo bar__'"+result1 = [Let (AlexPn 2 1 3),In (AlexPn 6 1 7),Int (AlexPn 9 1 10) 12334,Sym (AlexPn 16 2 1) '=',Sym (AlexPn 17 2 2) '+',Sym (AlexPn 18 2 3) '*',Var (AlexPn 19 2 4) "foo",Var (AlexPn 23 2 8) "bar__'"]+++}
+ tests/tokens_strict_bytestring.x view
@@ -0,0 +1,42 @@+{+{-# LANGUAGE OverloadedStrings #-}+module Main (main) where+import System.Exit+import Data.ByteString.Char8 (unpack)+}++%wrapper "strict-bytestring"++$digit = 0-9 -- digits+$alpha = [a-zA-Z] -- alphabetic characters++tokens :-++ $white+ ;+ "--".* ;+ let { \s -> Let }+ in { \s -> In }+ $digit+ { \s -> Int (read (unpack s)) }+ [\=\+\-\*\/\(\)] { \s -> Sym (head (unpack s)) }+ $alpha [$alpha $digit \_ \']* { \s -> Var (unpack s) }++{+-- Each right-hand side has type :: ByteString -> Token++-- The token type:+data Token =+ Let |+ In |+ Sym Char |+ Var String |+ Int Int |+ Err + deriving (Eq,Show)++main = if test1 /= result1 then exitFailure+ else exitWith ExitSuccess++test1 = alexScanTokens " let in 012334\n=+*foo bar__'"+result1 = [Let,In,Int 12334,Sym '=',Sym '+',Sym '*',Var "foo",Var "bar__'"]++}