tlex-th 0.2.0.0 → 0.2.0.1
raw patch · 5 files changed
+43/−38 lines, 5 filesPVP: major bump suggested
API removals or changes: PVP suggests a major version bump
API changes (from Hackage documentation)
- Language.Lexer.Tlex.Data.Addr: addrCodeUnitsLE :: Bits a => Integral a => Int -> a -> [Word8]
+ Language.Lexer.Tlex.Output.TH: addrCodeUnitsLE :: Bits a => Integral a => Int -> a -> [Word8]
Files
- CHANGELOG.md +4/−0
- src/Language/Lexer/Tlex/Data/Addr.hs +0/−23
- src/Language/Lexer/Tlex/Output/TH.hs +34/−11
- test/doctest/Doctest.hs +4/−2
- tlex-th.cabal +1/−2
CHANGELOG.md view
@@ -1,3 +1,7 @@+## 0.2.0.1 -- 2021-01-21++* Fix bug raised index accessing errors with rules having big states.+ ## 0.2.0.0 -- 2021-01-13 * Lax tlex constraints.
− src/Language/Lexer/Tlex/Data/Addr.hs
@@ -1,23 +0,0 @@-module Language.Lexer.Tlex.Data.Addr (- addrCodeUnitsLE,-) where--import Language.Lexer.Tlex.Prelude--import qualified Data.Bits as Bits---addrCodeUnitsLE :: Bits.Bits a => Integral a => Int -> a -> [Word8]-addrCodeUnitsLE us n- | n >= 0 = take us- do map- do \m -> fromInteger do fromIntegral do mod8bit m- do iterate (`Bits.shiftR` 8) n- | n == -1 = replicate us 0xFF- | otherwise = error "unsupported"- where- mod8bit = case Bits.bitSizeMaybe n of- Nothing -> \x -> x `mod` 256- Just bs- | bs <= 8 -> \x -> x- | otherwise -> \x -> x `mod` 256
src/Language/Lexer/Tlex/Output/TH.hs view
@@ -13,6 +13,9 @@ tlexArrayIndex, OutputContext (..), outputDfa,++ -- for tests+ addrCodeUnitsLE, ) where import Language.Lexer.Tlex.Prelude@@ -25,7 +28,6 @@ import qualified GHC.Types as Types import qualified Language.Haskell.TH as TH import qualified Language.Haskell.TH.Syntax as TH-import qualified Language.Lexer.Tlex.Data.Addr as Addr import qualified Language.Lexer.Tlex.Data.Bits as Bits import qualified Language.Lexer.Tlex.Data.EnumMap as EnumMap import qualified Language.Lexer.Tlex.Machine.DFA as DFA@@ -46,12 +48,17 @@ tlexLookupTlexTransTable offset unitSize table# s c = let !(Types.I# i#) = s `Bits.shiftL` offset + c in ST.runST- do ST.ST \s0# ->- let !(# s1#, r# #) = case unitSize of- TlexTransStateSize8 -> Prim.readInt8OffAddr# table# i# s0#- TlexTransStateSize16 -> Prim.readInt16OffAddr# table# i# s0#- TlexTransStateSize32 -> Prim.readInt32OffAddr# table# i# s0#- in (# s1#, Types.I# r# #)+ do ST.ST \s0# -> case unitSize of+ TlexTransStateSize8 -> case Prim.readWord8OffAddr# table# i# s0# of+ (# s1#, r# #) -> case r# of+ 255## -> (# s1#, -1 #)+ _ -> (# s1#, Types.I# do Prim.word2Int# r# #)+ TlexTransStateSize16 -> case Prim.readWord16OffAddr# table# i# s0# of+ (# s1#, r# #) -> case r# of+ 65535## -> (# s1#, -1 #)+ _ -> (# s1#, Types.I# do Prim.word2Int# r# #)+ TlexTransStateSize32 -> case Prim.readInt32OffAddr# table# i# s0# of+ (# s1#, r# #) -> (# s1#, Types.I# r# #) type TlexArray = Array.Array Int @@ -192,8 +199,7 @@ outputTlexTransFn :: DFA.DFA a -> (Int, Int) -> TH.Name -> TH.Q TH.Dec outputTlexTransFn DFA.DFA{ dfaTrans } (minUnitB, maxUnitB) fnName = let ubs = Bits.maxBitSize do maxUnitB - minUnitB- um =- do 1 `Bits.shiftL` ubs+ um = do 1 `Bits.shiftL` ubs - 1 l = concatMap do \dstState ->@@ -207,7 +213,8 @@ Nothing -> smDef [0..um] do toList dfaTrans- sbs = Bits.maxBitSize do length dfaTrans - 1+ -- count of states + count of specials (i.e. -1)+ sbs = Bits.maxBitSize do (length dfaTrans - 1) + 1 sbsEnum = if | ubs + sbs > 29 -> error "exceed over bit size limited" | otherwise -> stateSize sbs@@ -239,9 +246,25 @@ do TH.LitE do TH.StringPrimL do concatMap- do \sn -> Addr.addrCodeUnitsLE us+ do \sn -> addrCodeUnitsLE us do fromEnum sn do l++-- | Should correspond @tlexLookupTlexTransTable@+addrCodeUnitsLE :: Bits.Bits a => Integral a => Int -> a -> [Word8]+addrCodeUnitsLE us n+ | n >= 0 = take us+ do map+ do \m -> fromInteger do toInteger do mod8bit m+ do iterate (`Bits.shiftR` 8) n+ | n == -1 = replicate us 0xFF+ | otherwise = error "unsupported"+ where+ mod8bit = case Bits.bitSizeMaybe n of+ Nothing -> \x -> x Bits..&. 0xFF+ Just bs+ | bs <= 8 -> \x -> x+ | otherwise -> \x -> x Bits..&. 0xFF outputTlexAcceptFn :: DFA.DFA (TH.Q TH.Exp) -> (TH.Q TH.Type) -> TH.Name -> TH.Q TH.Dec
test/doctest/Doctest.hs view
@@ -3,10 +3,11 @@ import Prelude import qualified Build_doctests as BuildF+import qualified Control.Exception as Exception import Control.Monad import qualified System.Environment as IO import qualified System.IO as IO-import Test.DocTest (doctest)+import qualified Test.DocTest as DocTest main :: IO () main = forM_ BuildF.components \(BuildF.Component name flags pkgs sources) -> do@@ -16,6 +17,7 @@ IO.hFlush IO.stdout let args = flags ++ pkgs ++ sources IO.unsetEnv "GHC_ENVIRONMENT"- doctest args+ DocTest.doctest args `Exception.catch`+ \(e :: Exception.SomeException) -> print e putStrLn "=============================================" IO.hFlush IO.stdout
tlex-th.cabal view
@@ -2,7 +2,7 @@ build-type: Custom name: tlex-th-version: 0.2.0.0+version: 0.2.0.1 license: Apache-2.0 OR MPL-2.0 license-file: LICENSE copyright: (c) 2021 Mizunashi Mana@@ -121,7 +121,6 @@ -- internals Language.Lexer.Tlex.Output.TH- Language.Lexer.Tlex.Data.Addr Language.Lexer.Tlex.Data.Bits test-suite doctest