yapb 0.1.1 → 0.1.3
raw patch · 17 files changed
+548/−256 lines, 17 filesdep +hspecdep ~process
Dependencies added: hspec
Dependency ranges changed: process
Files
- README.md +8/−3
- app/parser/Main.hs +14/−13
- app/parser/ParserSpec.hs +15/−0
- app/parser/Run.hs +21/−0
- app/parser/Token.hs +4/−4
- app/syntaxcompletion/Main.hs +8/−22
- app/syntaxcompletion/SyntaxCompletion.hs +44/−0
- app/syntaxcompletion/SyntaxCompletionSpec.hs +97/−0
- app/syntaxcompletion/Token.hs +4/−4
- src/parserlib/AutomatonType.hs +1/−1
- src/parserlib/CommonParserUtil.hs +211/−127
- src/parserlib/Terminal.hs +18/−4
- src/parserlib/TokenInterface.hs +1/−1
- src/syncomplib/EmacsServer.hs +72/−66
- src/syncomplib/SynCompInterface.hs +8/−6
- test/Spec.hs +7/−1
- yapb.cabal +15/−4
README.md view
@@ -12,19 +12,24 @@ - Examples: - parser-exe: an arithmetic parser - syncomp-exe: a syntax completion server for Emacs- - (polyrpc)[https://github.com/kwanghoon/polyrpc]: a polyrpc programming language system including a parser, a poly rpc type checker, a slicing compiler, a poly cs type checker, and a poly cs interpter. -### Download and build+### Download, build, and test ~~~ $ git clone https://github.com/kwanghoon/yapb $ cd yapb $ stack build+ $ stack test ~~~ ### Tutorial - [How to write and run a parser](https://github.com/kwanghoon/yapb/blob/master/doc/Tutorial-parser.md) - [How to write and run a syntax completion server for Emacs](https://github.com/kwanghoon/yapb/blob/master/doc/Tutorial-syntax-completion.md)-- [A top-down approach to writing a compiler for arithmetic expressions](https://github.com/kwanghoon/swlab_parser_builder/blob/master/doc/tutorial_swlab_parser_builder.txt) Written in Korean.+- [For AST and interpreter: A top-down approach to writing a compiler for arithmetic expressions](https://github.com/kwanghoon/swlab_parser_builder/blob/master/doc/tutorial_swlab_parser_builder.txt) Written in Korean.+- [For parser: Parser generators sharing LR automaton generators and accepting general purpose programming language-based specifications](http://swlab.jnu.ac.kr/paper/kiise202001.pdf) Written in Korean.+- [For syntax complection with YAPB-0.1.2: A text-based syntax completion method using LR parsing (PEPM 2021)](http://swlab.jnu.ac.kr/paper/pepm2021final.pdf).+++ ### Reference - [References](https://github.com/kwanghoon/yapb/blob/master/doc/Reference.md)
app/parser/Main.hs view
@@ -7,25 +7,26 @@ import Parser import Expr +import Run(doProcess)+import ParserSpec (spec)+ import System.IO+import System.Environment (getArgs, withArgs) main :: IO () main = do- fileName <- readline "Enter your file: "+ args <- getArgs+ _main args++-- Todo: Can I fix to have "test" as a command in stack exec?++_main [] = return ()+_main (fileName:args) = case fileName of- "exit" -> return ()- line -> doProcess line+ "test" -> withArgs [] spec+ _ -> do _ <- doProcess True fileName+ _main args -doProcess line = do- text <- readFile line - putStrLn "Lexing..."- terminalList <- lexing lexerSpec text- putStrLn "Parsing..."- exprSeqAst <- parsing parserSpec terminalList- putStrLn "Pretty Printing..."- putStrLn (pprintAst exprSeqAst)- - readline msg = do putStr msg hFlush stdout
+ app/parser/ParserSpec.hs view
@@ -0,0 +1,15 @@+module ParserSpec where++import Run++import Test.Hspec++spec = hspec $ do+ describe "parsing yapb/app/parser" $ do+ it "one-line example" $ do+ result <- doProcess False "./app/parser/example/oneline.arith"+ result `shouldBe` "((1 + 2) - ((3 * 4) / 5))"++ it "multi-line example" $ do+ result <- doProcess False "./app/parser/example/multiline.arith"+ result `shouldBe` "(x = 123); (x = (x + 1)); (y = x); (y = (y - ((1 * 2) / 3))); (z = (y = x))"
+ app/parser/Run.hs view
@@ -0,0 +1,21 @@+module Run where++import CommonParserUtil++import Lexer+import Terminal+import Parser+import Expr++import Control.Monad (when)+import System.IO++doProcess verbose fileName = do+ text <- readFile fileName+ when (verbose) $ putStrLn "Lexing..."+ terminalList <- lexing lexerSpec text+ when (verbose) $ putStrLn "Parsing..."+ exprSeqAst <- parsing False parserSpec terminalList+ when (verbose) $ putStrLn "Pretty Printing..."+ when (verbose) $ putStrLn (pprintAst exprSeqAst)+ return (pprintAst exprSeqAst)
app/parser/Token.hs view
@@ -31,10 +31,10 @@ | otherwise = findStr str list instance TokenInterface Token where- toToken str =- case findStr str tokenStrList of- Nothing -> error ("toToken: " ++ str)- Just tok -> tok+ -- toToken str =+ -- case findStr str tokenStrList of+ -- Nothing -> error ("toToken: " ++ str)+ -- Just tok -> tok fromToken tok = case findTok tok tokenStrList of Nothing -> error ("fromToken: " ++ show tok)
app/syntaxcompletion/Main.hs view
@@ -1,29 +1,15 @@ module Main where -import CommonParserUtil +import EmacsServer -import Lexer-import Terminal-import Parser-import System.IO+import SyntaxCompletion (computeCand)+import SyntaxCompletionSpec (spec) --- for syntax completion-import Token-import Expr-import EmacsServer-import SynCompInterface-import Control.Exception+import System.Environment (getArgs, withArgs) main :: IO () main = do- emacsServer computeCand- -computeCand :: String -> Bool -> IO [EmacsDataItem]-computeCand programTextUptoCursor isSimpleMode = ((do- terminalList <- lexing lexerSpec programTextUptoCursor - ast <- parsing parserSpec terminalList - successfullyParsed)- `catch` \e -> case e :: LexError of _ -> handleLexError)- `catch` \e -> case e :: ParseError Token AST of _ -> handleParseError isSimpleMode e--+ args <- getArgs+ if "test" `elem` args+ then withArgs [] spec+ else emacsServer (computeCand False)
+ app/syntaxcompletion/SyntaxCompletion.hs view
@@ -0,0 +1,44 @@+module SyntaxCompletion (computeCand) where++import CommonParserUtil ++import TokenInterface+import Terminal+import Lexer (lexerSpec)+import Parser (parserSpec)+import System.IO++-- for syntax completion+import Token+import Expr+import SynCompInterface+import Control.Exception+import Data.Typeable++-- Todo: The following part should be moved to the library.+-- Arguments: lexerSpec, parserSpec+-- isSimpleMode+-- programTextUptoCursor, programTextAfterCursor++maxLevel = 10000++-- | computeCand+computeCand :: Bool -> String -> String -> Bool -> IO [EmacsDataItem]+computeCand debug programTextUptoCursor programTextAfterCursor isSimpleMode = (do+ {- 1. Lexing -} + (line, column, terminalListUptoCursor) <-+ lexingWithLineColumn lexerSpec 1 1 programTextUptoCursor++ {- 2. Parsing -}+ ((do ast <- parsing debug parserSpec terminalListUptoCursor+ successfullyParsed)++ `catch` \parseError ->+ case parseError :: ParseError Token AST of+ _ ->+ {- 3. Lexing the rest and computing candidates with it -}+ do (_, _, terminalListAfterCursor) <-+ lexingWithLineColumn lexerSpec line column programTextAfterCursor+ handleParseError debug maxLevel isSimpleMode terminalListAfterCursor parseError))++ `catch` \lexError -> case lexError :: LexError of _ -> handleLexError
+ app/syntaxcompletion/SyntaxCompletionSpec.hs view
@@ -0,0 +1,97 @@+module SyntaxCompletionSpec where++import SyntaxCompletion (computeCand)+import SynCompInterface+import Test.Hspec++import System.IO (readFile)++spec = hspec $ do+ describe "syntax complection yapb/app/syntaxcompletion" $ do+ let ex1_sml = "let val add = fn x =>"+ it ("[ex1.sml:simple] " ++ ex1_sml) $ do+ results <- computeCand False ex1_sml "" True+ results `shouldBe` [Candidate "white ..."]++ it ("[ex1.sml:nested] " ++ ex1_sml) $ do+ results <- computeCand False ex1_sml "" False+ results `shouldBe` [Candidate "white ...",Candidate "white ... white in white ... white end"]++ let ex2_sml = "let val add = fn y"+ it ("[ex2.sml:simple] " ++ ex2_sml) $ do+ results <- computeCand False ex2_sml "" True+ results `shouldBe` [Candidate "white => white ..."]++ let ex2_sml = "let val add = fn y"+ it ("[ex2.sml:nested] " ++ ex2_sml) $ do+ results <- computeCand False ex2_sml "" False+ results `shouldBe`+ [Candidate "white => white ...",Candidate "white => white ... white in white ... white end"]++ let test1_sml = "let val app = fn f => fn x => f x in let val add = fn y => y in"+ let test1_sml_end = " end"+ it ("[test1.sml:simple] " ++ test1_sml ++ " [cursor] " ++ test1_sml_end) $ do+ results <- computeCand False test1_sml test1_sml_end True+ results `shouldBe` [Candidate "white ... gray end 1 66 "]++ let test1_sml = "let val app = fn f => fn x => f x in let val add = fn y => y in"+ let test1_sml_end = " end"+ it ("[test1.sml:nested] " ++ test1_sml ++ " [cursor] " ++ test1_sml_end) $ do+ results <- computeCand False test1_sml test1_sml_end False+ results `shouldBe`+ [Candidate "white ... gray end 1 66 ",Candidate "white ... gray end 1 66 white end"]++ let test1_sml = "let val app = fn f => fn x => f x in let val add = fn y => y in"+ let test1_sml_end = " end !="+ it ("[test1.sml:simple:invalid tokens] " ++ test1_sml ++ " [cursor] " ++ test1_sml_end) $ do+ results <- computeCand False test1_sml test1_sml_end True+ results `shouldBe` [LexError]++ let test1_sml = "let val app = fn f => fn x => f x in let val add = fn y => y in"+ let test1_sml_end = " end !="+ it ("[test1.sml:nested:invalid tokens] " ++ test1_sml ++ " [cursor] " ++ test1_sml_end) $ do+ results <- computeCand False test1_sml test1_sml_end False+ results `shouldBe` [LexError]++ let test2_sml = "fn x => f (f (f (f x"+ it ("[test2.sml:simple] " ++ test2_sml) $ do+ results <- computeCand False test2_sml "" True+ results `shouldBe` [Candidate "white )"]++ let test2_sml = "fn x => f (f (f (f x"+ it ("[test2.sml:nested] " ++ test2_sml) $ do+ results <- computeCand False test2_sml "" False+ results `shouldBe`+ [Candidate "white )",Candidate "white ) white )",Candidate "white ) white ) white )"]++ let test3_sml = "let val add = x "+ it ("[test3.sml:simple] " ++ test3_sml) $ do+ results <- computeCand False test3_sml "" True+ results `shouldBe` []++ let test3_sml = "let val add = x "+ it ("[test3.sml:nested] " ++ test3_sml) $ do+ results <- computeCand False test3_sml "" False+ results `shouldBe` [Candidate "white in white ... white end"]++ let test4_sml = "fn x y => "+ it ("[test4.sml:simple] " ++ test4_sml) $ do+ results <- computeCand False test4_sml "" True+ results `shouldBe`+ [ParseError ["y at (1, 6): identifier","=> at (1, 8): =>","$ at (1, 11): $"]]++ let test4_sml = "fn x y => "+ it ("[test4.sml:nested] " ++ test4_sml) $ do+ results <- computeCand False test4_sml "" False+ results `shouldBe`+ [ParseError ["y at (1, 6): identifier","=> at (1, 8): =>","$ at (1, 11): $"]]++ let lexerror = "x != y "+ it ("[lex error:simple] " ++ lexerror) $ do+ results <- computeCand False lexerror "" True+ results `shouldBe` [LexError]++ let lexerror = "x != y "+ it ("[lex error:nested] " ++ lexerror) $ do+ results <- computeCand False lexerror "" False+ results `shouldBe` [LexError]
app/syntaxcompletion/Token.hs view
@@ -29,10 +29,10 @@ | otherwise = findStr str list instance TokenInterface Token where- toToken str =- case findStr str tokenStrList of- Nothing -> error ("toToken: " ++ str)- Just tok -> tok+ -- toToken str =+ -- case findStr str tokenStrList of+ -- Nothing -> error ("toToken: " ++ str)+ -- Just tok -> tok fromToken tok = case findTok tok tokenStrList of Nothing -> error ("fromToken: " ++ show tok)
src/parserlib/AutomatonType.hs view
@@ -1,6 +1,6 @@ module AutomatonType where -data Action = Shift Int | Reduce Int | Accept deriving Eq+data Action = Shift Int | Reduce Int | Accept deriving (Eq, Show) type ActionTable = [((Int, String), Action)] -- key: (Int,String), value: Action type GotoTable = [((Int, String), Int)] -- key: (Int,String), value: Int
src/parserlib/CommonParserUtil.hs view
@@ -1,5 +1,10 @@ {-# LANGUAGE GADTs #-}-module CommonParserUtil where+module CommonParserUtil+ ( LexerSpec(..), ParserSpec(..)+ , lexing, lexingWithLineColumn, parsing, runAutomaton+ , get, getText+ , LexError(..), ParseError(..)+ , successfullyParsed, handleLexError, handleParseError) where import Terminal import TokenInterface@@ -17,6 +22,7 @@ import LoadAutomaton import Data.List (nub)+import Data.Maybe import SynCompInterface @@ -67,34 +73,36 @@ instance Exception LexError -prLexError (CommonParserUtil.LexError line col text) = do- putStr $ "No matching lexer spec at "- putStr $ "Line " ++ show line- putStr $ "Column " ++ show col- putStr $ " : "- putStr $ take 10 text+-- prLexError (CommonParserUtil.LexError line col text) = do+-- putStr $ "No matching lexer spec at "+-- putStr $ "Line " ++ show line+-- putStr $ "Column " ++ show col+-- putStr $ " : "+-- putStr $ take 10 text -- lexing :: TokenInterface token => LexerSpec token -> String -> IO [Terminal token]-lexing lexerspec text = lexing_ lexerspec 1 1 text+lexing lexerspec text = do+ (line, col, terminalList) <- lexingWithLineColumn lexerspec 1 1 text+ return terminalList -lexing_ :: TokenInterface token =>- LexerSpec token -> Line -> Column -> String -> IO [Terminal token]-lexing_ lexerspec line col [] = do+lexingWithLineColumn :: TokenInterface token =>+ LexerSpec token -> Line -> Column -> String -> IO (Line, Column, [Terminal token])+lexingWithLineColumn lexerspec line col [] = do let eot = endOfToken lexerspec - return [Terminal (fromToken eot) line col eot]+ return (line, col, [Terminal (fromToken eot) line col (Just eot)]) -lexing_ lexerspec line col text = do+lexingWithLineColumn lexerspec line col text = do --Todo: make it tail-recursive! (matchedText, theRestText, maybeTok) <- matchLexSpec line col (lexerSpecList lexerspec) text let (line_, col_) = moveLineCol line col matchedText- terminalList <- lexing_ lexerspec line_ col_ theRestText+ (line__, col__, terminalList) <- lexingWithLineColumn lexerspec line_ col_ theRestText case maybeTok of- Nothing -> return terminalList+ Nothing -> return (line__, col__, terminalList) Just tok -> do- let terminal = Terminal matchedText line col tok- return (terminal:terminalList)+ let terminal = Terminal matchedText line col (Just tok)+ return (line__, col__, terminal:terminalList) matchLexSpec :: TokenInterface token => Line -> Column -> LexerSpecList token -> String@@ -124,48 +132,55 @@ -- The parsing machine -------------------------------------------------------------------------------- +type CurrentState = Int+type StateOnStackTop = Int+type LhsSymbol = String++type AutomatonSnapshot token ast = -- TODO: Refactoring+ (Stack token ast, ActionTable, GotoTable, ProdRules)+ -- data ParseError token ast where -- teminal, state, stack actiontbl, gototbl NotFoundAction :: (TokenInterface token, Typeable token, Typeable ast, Show token, Show ast) =>- (Terminal token) -> Int -> (Stack token ast) -> ActionTable -> GotoTable -> ProdRules -> [Terminal token] -> ParseError token ast+ (Terminal token) -> CurrentState -> (Stack token ast) -> ActionTable -> GotoTable -> ProdRules -> [Terminal token] -> ParseError token ast -- topState, lhs, stack, actiontbl, gototbl, NotFoundGoto :: (TokenInterface token, Typeable token, Typeable ast, Show token, Show ast) =>- Int -> String -> (Stack token ast) -> ActionTable -> GotoTable -> ProdRules -> [Terminal token] -> ParseError token ast+ StateOnStackTop -> LhsSymbol -> (Stack token ast) -> ActionTable -> GotoTable -> ProdRules -> [Terminal token] -> ParseError token ast deriving (Typeable) instance (Show token, Show ast) => Show (ParseError token ast) where showsPrec p (NotFoundAction terminal state stack _ _ _ _) =- (++) "NotFoundAction" . (++) (terminalToString terminal) . (++) (show state) -- . (++) (show stack)+ (++) "NotFoundAction: " . (++) (show state) . (++) " " . (++) (terminalToString terminal) -- (++) (show $ length stack) showsPrec p (NotFoundGoto topstate lhs stack _ _ _ _) =- (++) "NotFoundGoto" . (++) (show topstate) . (++) lhs -- . (++) (show stack)+ (++) "NotFoundGoto: " . (++) (show topstate) . (++) " " . (++) lhs -- . (++) (show stack) instance (TokenInterface token, Typeable token, Show token, Typeable ast, Show ast) => Exception (ParseError token ast) -prParseError (NotFoundAction terminal state stack actiontbl gototbl prodRules terminalList) = do- putStrLn $- ("Not found in the action table: "- ++ terminalToString terminal)- ++ " : "- ++ show (state, tokenTextFromTerminal terminal)- ++ " (" ++ show (length terminalList) ++ ")"- ++ "\n" ++ prStack stack ++ "\n"+-- prParseError (NotFoundAction terminal state stack actiontbl gototbl prodRules terminalList) = do+-- putStrLn $+-- ("Not found in the action table: "+-- ++ terminalToString terminal)+-- ++ " : "+-- ++ show (state, tokenTextFromTerminal terminal)+-- ++ " (" ++ show (length terminalList) ++ ")"+-- ++ "\n" ++ prStack stack ++ "\n" -prParseError (NotFoundGoto topState lhs stack actiontbl gototbl prodRules terminalList) = do- putStrLn $- ("Not found in the goto table: ")- ++ " : "- ++ show (topState,lhs) ++ "\n"- ++ " (" ++ show (length terminalList) ++ ")"- ++ prStack stack ++ "\n"+-- prParseError (NotFoundGoto topState lhs stack actiontbl gototbl prodRules terminalList) = do+-- putStrLn $+-- ("Not found in the goto table: ")+-- ++ " : "+-- ++ show (topState,lhs) ++ "\n"+-- ++ " (" ++ show (length terminalList) ++ ")"+-- ++ prStack stack ++ "\n" -- parsing :: (TokenInterface token, Typeable token, Typeable ast, Show token, Show ast) =>- ParserSpec token ast -> [Terminal token] -> IO ast-parsing parserSpec terminalList = do+ Bool -> ParserSpec token ast -> [Terminal token] -> IO ast+parsing flag parserSpec terminalList = do -- 1. Save the production rules in the parser spec (Parser.hs). writtenBool <- saveProdRules specFileName sSym pSpecList @@ -184,7 +199,7 @@ putStrLn $ "Delete " ++ hashFile removeIfExists hashFile error $ "Error: Empty automation: please rerun"- else do ast <- runAutomaton actionTbl gotoTbl prodRules pFunList terminalList+ else do ast <- runAutomaton flag initState actionTbl gotoTbl prodRules pFunList terminalList -- putStrLn "done." -- It was for the interafce with Java-version RPC calculus interpreter. return ast @@ -217,7 +232,6 @@ | isDoesNotExistError e = return () | otherwise = throwIO e - -- Stack data StkElem token ast =@@ -257,10 +271,12 @@ prStack :: TokenInterface token => Stack token ast -> String prStack [] = "STACK END" prStack (StkState i : stack) = "S" ++ show i ++ " : " ++ prStack stack-prStack (StkTerminal (Terminal text _ _ token) : stack) =+prStack (StkTerminal (Terminal text _ _ (Just token)) : stack) = let str_token = fromToken token in (if str_token == text then str_token else (fromToken token ++ " i.e. " ++ text)) ++ " : " ++ prStack stack+prStack (StkTerminal (Terminal text _ _ Nothing) : stack) =+ (token_na ++ " " ++ text) ++ " : " ++ prStack stack prStack (StkNonterminal _ str : stack) = str ++ " : " ++ prStack stack -- Utility for Automation@@ -269,7 +285,8 @@ currentState _ = error "No state found in the stack top" tokenTextFromTerminal :: TokenInterface token => Terminal token -> String-tokenTextFromTerminal (Terminal _ _ _ token) = fromToken token+tokenTextFromTerminal (Terminal _ _ _ (Just token)) = fromToken token+tokenTextFromTerminal (Terminal _ _ _ Nothing) = token_na lookupActionTable :: TokenInterface token => ActionTable -> Int -> (Terminal token) -> Maybe Action lookupActionTable actionTbl state terminal =@@ -302,13 +319,14 @@ type ParseFunList token ast = [ParseFun token ast] runAutomaton :: (TokenInterface token, Typeable token, Typeable ast, Show token, Show ast) =>+ Bool -> Int -> {- static part -} ActionTable -> GotoTable -> ProdRules -> ParseFunList token ast -> {- dynamic part -} [Terminal token] -> {- AST -} IO ast-runAutomaton actionTbl gotoTbl prodRules pFunList terminalList = do+runAutomaton flag initState actionTbl gotoTbl prodRules pFunList terminalList = do let initStack = push (StkState initState) emptyStack run terminalList initStack @@ -328,13 +346,13 @@ -- ++ show (state, tokenTextFromTerminal terminal) -- ++ "\n" ++ prStack stack ++ "\n" - debug ("\nState " ++ show state)- debug ("Token " ++ text)- debug ("Stack " ++ prStack stack)+ debug flag ("\nState " ++ show state)+ debug flag ("Token " ++ text)+ debug flag ("Stack " ++ prStack stack) case action of Accept -> do- debug "Accept"+ debug flag "Accept" case stack !! 1 of StkNonterminal (Just ast) _ -> return ast@@ -342,18 +360,18 @@ _ -> fail "Not Stknontermianl on Accept" Shift toState -> do- debug ("Shift " ++ show toState)+ debug flag ("Shift " ++ show toState) let stack1 = push (StkTerminal (head terminalList)) stack let stack2 = push (StkState toState) stack1 run (tail terminalList) stack2 Reduce n -> do- debug ("Reduce " ++ show n)+ debug flag ("Reduce " ++ show n) let prodrule = prodRules !! n - debug ("\t" ++ show prodrule)+ debug flag ("\t" ++ show prodrule) let builderFun = pFunList !! n let lhs = fst prodrule@@ -375,15 +393,14 @@ let stack3 = push (StkState toState) stack2 run terminalList stack3 -flag = True--debug :: String -> IO ()-debug msg = if flag then putStrLn msg else return ()+debug :: Bool -> String -> IO ()+debug flag msg = if flag then putStrLn msg else return () prlevel n = take n (let spaces = ' ' : spaces in spaces) ----data Candidate =+-- | Computing candidates++data Candidate = -- Todo: data Candidate vs. data EmacsDataItem = ... | Candidate String TerminalSymbol String | NonterminalSymbol String deriving (Show,Eq)@@ -394,28 +411,44 @@ gotoTbl :: GotoTable, prodRules :: ProdRules }+ +compCandidates+ :: (TokenInterface token, Typeable token, Typeable ast, Show token, Show ast) =>+ Bool -- debug+ -> Int -- maximum search depth level+ -> Bool -- simple or nested+ -> Int+ -> [Candidate]+ -> Int+ -> Automaton token ast+ -> Stack token ast+ -> IO [[Candidate]] -compCandidates isSimple level symbols state automaton stk = do- compGammas isSimple level symbols state automaton stk []--- gammas <- compGammas isSimple level symbols state automaton stk []+compCandidates flag maxLevel isSimple level symbols state automaton stk = do+ compGammasDfs flag maxLevel isSimple level symbols state automaton stk []+-- gammas <- compGammasDfs isSimple level symbols state automaton stk [] -- if isSimple -- then return gammas -- else return $ tail $ scanl (++) [] (filter (not . null) gammas) -compGammas :: (TokenInterface token, Typeable token, Typeable ast, Show token, Show ast) =>- Bool -> Int -> [Candidate] -> Int -> Automaton token ast -> Stack token ast -> [(Int, Stack token ast, String)]-> IO [[Candidate]]--checkCycle flag level state stk action history cont =- if flag && (state,stk,action) `elem` history- then do debug $ prlevel level ++ "CYCLE is detected !!"- debug $ prlevel level ++ show state ++ " " ++ action- debug $ prlevel level ++ prStack stk- debug $ ""- return []- else cont ( (state,stk,action) : history )+compGammasDfs+ :: (TokenInterface token, Typeable token, Typeable ast, Show token, Show ast) =>+ Bool+ -> Int+ -> Bool+ -> Int+ -> [Candidate]+ -> Int+ -> Automaton token ast+ -> Stack token ast+ -> [(Int, Stack token ast, String)]+ -> IO [[Candidate]] -compGammas isSimple level symbols state automaton stk history = - checkCycle False level state stk "" history+compGammasDfs flag maxLevel isSimple level symbols state automaton stk history =+ if level > maxLevel then+ return (if null symbols then [] else [symbols])+ else+ checkCycle flag False level state stk "" history (\history -> case nub [prnum | ((s,lookahead),Reduce prnum) <- actTbl automaton, state==s] of [] ->@@ -431,20 +464,20 @@ _ -> do listOfList <- mapM (\ ((terminal,snext),i)->- let stk1 = push (StkTerminal (Terminal terminal 0 0 (toToken terminal))) stk+ let stk1 = push (StkTerminal (Terminal terminal 0 0 Nothing)) stk -- Todo: ??? (toToken terminal) stk2 = push (StkState snext) stk1 in -- checkCycle False level snext stk2 ("SHIFT " ++ show snext ++ " " ++ terminal) history -- checkCycle True level state stk terminal history- checkCycle True level snext stk2 terminal history+ checkCycle flag True level snext stk2 terminal history (\history1 -> do- debug $ prlevel level ++ "SHIFT [" ++ show i ++ "/" ++ show len ++ "]: "+ debug flag $ prlevel level ++ "SHIFT [" ++ show i ++ "/" ++ show len ++ "]: " ++ show state ++ " -> " ++ terminal ++ " -> " ++ show snext- debug $ prlevel level ++ "Goto/Shift symbols: " ++ show (symbols++[TerminalSymbol terminal])- debug $ prlevel level ++ "Stack " ++ prStack stk2- debug $ ""- compGammas isSimple (level+1) (symbols++[TerminalSymbol terminal]) snext automaton stk2 history1) )+ debug flag $ prlevel level ++ "Goto/Shift symbols: " ++ show (symbols++[TerminalSymbol terminal])+ debug flag $ prlevel level ++ "Stack " ++ prStack stk2+ debug flag $ ""+ compGammasDfs flag maxLevel isSimple (level+1) (symbols++[TerminalSymbol terminal]) snext automaton stk2 history1) ) (zip cand2 [1..]) return $ concat listOfList nontermStateList -> do@@ -457,24 +490,24 @@ in -- checkCycle False level snext stk2 ("GOTO " ++ show snext ++ " " ++ nonterminal) history -- checkCycle True level state stk nonterminal history- checkCycle True level snext stk2 nonterminal history+ checkCycle flag True level snext stk2 nonterminal history (\history1 -> do- debug $ prlevel level ++ "GOTO [" ++ show i ++ "/" ++ show len ++ "] at "+ debug flag $ prlevel level ++ "GOTO [" ++ show i ++ "/" ++ show len ++ "] at " ++ show state ++ " -> " ++ show nonterminal ++ " -> " ++ show snext- debug $ prlevel level ++ "Goto/Shift symbols:" ++ show (symbols++[NonterminalSymbol nonterminal])- debug $ prlevel level ++ "Stack " ++ prStack stk2- debug $ ""+ debug flag $ prlevel level ++ "Goto/Shift symbols:" ++ show (symbols++[NonterminalSymbol nonterminal])+ debug flag $ prlevel level ++ "Stack " ++ prStack stk2+ debug flag $ "" - compGammas isSimple (level+1) (symbols++[NonterminalSymbol nonterminal]) snext automaton stk2 history1) )+ compGammasDfs flag maxLevel isSimple (level+1) (symbols++[NonterminalSymbol nonterminal]) snext automaton stk2 history1) ) (zip nontermStateList [1..]) return $ concat listOfList prnumList -> do let len = length prnumList - debug $ prlevel level ++ "# of prNumList to reduce: " ++ show len ++ " at State " ++ show state- debug $ prlevel (level+1) ++ show [ (prodRules automaton) !! prnum | prnum <- prnumList ]+ debug flag $ prlevel level ++ "# of prNumList to reduce: " ++ show len ++ " at State " ++ show state+ debug flag $ prlevel (level+1) ++ show [ (prodRules automaton) !! prnum | prnum <- prnumList ] -- let aCandidate = if null symbols then [] else [symbols] -- if isSimple@@ -483,22 +516,19 @@ do listOfList <- mapM (\ (prnum,i) -> ( -- checkCycle False level state stk ("REDUCE " ++ show prnum) history- checkCycle True level state stk (show prnum) history+ checkCycle flag True level state stk (show prnum) history (\history1 -> do- debug $ prlevel level ++ "State " ++ show state ++ "[" ++ show i ++ "/" ++ show len ++ "]" - debug $ prlevel level ++ "REDUCE" ++ " prod #" ++ show prnum- debug $ prlevel level ++ show ((prodRules automaton) !! prnum)- debug $ prlevel level ++ "Goto/Shift symbols: " ++ show symbols- debug $ prlevel level ++ "Stack " ++ prStack stk- debug $ ""- compGammasForReduce level isSimple symbols state automaton stk history1 prnum)) )+ debug flag $ prlevel level ++ "State " ++ show state ++ "[" ++ show i ++ "/" ++ show len ++ "]" + debug flag $ prlevel level ++ "REDUCE" ++ " prod #" ++ show prnum+ debug flag $ prlevel level ++ show ((prodRules automaton) !! prnum)+ debug flag $ prlevel level ++ "Goto/Shift symbols: " ++ show symbols+ debug flag $ prlevel level ++ "Stack " ++ prStack stk+ debug flag $ ""+ compGammasDfsForReduce flag maxLevel level isSimple symbols state automaton stk history1 prnum)) ) (zip prnumList [1..]) return $ concat listOfList ) -noCycleCheck :: Bool-noCycleCheck = True--compGammasForReduce level isSimple symbols state automaton stk history prnum = +compGammasDfsForReduce flag maxLevel level isSimple symbols state automaton stk history prnum = let prodrule = (prodRules automaton) !! prnum lhs = fst prodrule rhs = snd prodrule@@ -507,62 +537,116 @@ in if ( {- rhsLength == 0 || -} (rhsLength > length symbols) ) == False then do- debug $ prlevel level ++ "[LEN COND: False] length rhs > length symbols: NOT " ++ show rhsLength ++ ">" ++ show (length symbols)- debug $ prlevel (level+1) ++ show symbols- debug $ prlevel level- return []+ debug flag $ prlevel level ++ "[LEN COND: False] length rhs > length symbols: NOT " ++ show rhsLength ++ ">" ++ show (length symbols)+ debug flag $ prlevel (level+1) ++ show symbols+ debug flag $ prlevel level+ return [] -- Todo: (if null symbols then [] else [symbols]) else do let stk1 = drop (rhsLength*2) stk let topState = currentState stk1 let toState = case lookupGotoTable (gotoTbl automaton) topState lhs of Just state -> state- Nothing -> error $ "[compGammasForReduce] Must not happen: lhs: " ++ lhs ++ " state: " ++ show topState+ Nothing -> error $ "[compGammasDfsForReduce] Must not happen: lhs: " ++ lhs ++ " state: " ++ show topState let stk2 = push (StkNonterminal Nothing lhs) stk1 -- ast let stk3 = push (StkState toState) stk2- debug $ prlevel level ++ "GOTO after REDUCE: " ++ show topState ++ " " ++ lhs ++ " " ++ show toState- debug $ prlevel level ++ "Goto/Shift symbols: " ++ "[]"- debug $ prlevel level ++ "Stack " ++ prStack stk3- debug $ ""+ debug flag $ prlevel level ++ "GOTO after REDUCE: " ++ show topState ++ " " ++ lhs ++ " " ++ show toState+ debug flag $ prlevel level ++ "Goto/Shift symbols: " ++ "[]"+ debug flag $ prlevel level ++ "Stack " ++ prStack stk3+ debug flag $ "" - debug $ prlevel level ++ "Found a gamma: " ++ show symbols- debug $ ""+ debug flag $ prlevel level ++ "Found a gamma: " ++ show symbols+ debug flag $ "" if isSimple then return (if null symbols then [] else [symbols])- else do listOfList <- compGammas isSimple (level+1) [] toState automaton stk3 history+ else do listOfList <- compGammasDfs flag maxLevel isSimple (level+1) [] toState automaton stk3 history return (if null symbols then listOfList else (symbols : map (symbols ++) listOfList)) ---+-- | Cycle checking+noCycleCheck :: Bool+noCycleCheck = True++checkCycle debugflag flag level state stk action history cont =+ if flag && (state,stk,action) `elem` history+ then do+ debug debugflag $ prlevel level ++ "CYCLE is detected !!"+ debug debugflag $ prlevel level ++ show state ++ " " ++ action+ debug debugflag $ prlevel level ++ prStack stk+ debug debugflag $ ""+ return []+ else cont ( (state,stk,action) : history )++-- | Parsing programming interfaces++-- | successfullyParsed successfullyParsed :: IO [EmacsDataItem] successfullyParsed = return [SynCompInterface.SuccessfullyParsed] +-- | handleLexError handleLexError :: IO [EmacsDataItem] handleLexError = return [SynCompInterface.LexError]- -handleParseError isSimple (NotFoundAction _ state stk actTbl gotoTbl prodRules terminalList) =- _handleParseError isSimple state stk actTbl gotoTbl prodRules terminalList-handleParseError isSimple (NotFoundGoto state _ stk actTbl gotoTbl prodRules terminalList) =- _handleParseError isSimple state stk actTbl gotoTbl prodRules terminalList +-- | handleParseError+handleParseError :: TokenInterface token => Bool -> Int -> Bool -> [Terminal token] -> ParseError token ast -> IO [EmacsDataItem]+handleParseError flag maxLevel isSimple terminalListAfterCursor parseError =+ unwrapParseError flag maxLevel isSimple terminalListAfterCursor parseError+ +unwrapParseError flag maxLevel isSimple terminalListAfterCursor (NotFoundAction _ state stk actTbl gotoTbl prodRules terminalList) =+ arrivedAtTheEndOfSymbol flag maxLevel isSimple terminalListAfterCursor state stk actTbl gotoTbl prodRules terminalList+unwrapParseError flag maxLevel isSimple terminalListAfterCursor (NotFoundGoto state _ stk actTbl gotoTbl prodRules terminalList) =+ arrivedAtTheEndOfSymbol flag maxLevel isSimple terminalListAfterCursor state stk actTbl gotoTbl prodRules terminalList -_handleParseError isSimple state stk _actTbl _gotoTbl _prodRules terminalList = - if length terminalList == 1 then do -- [$]- let automaton = Automaton {actTbl=_actTbl, gotoTbl=_gotoTbl, prodRules=_prodRules}- candidates <- compCandidates isSimple 0 [] state automaton stk- let cands = candidates- let strs = nub [ concatStrList strList | strList <- map (map showSymbol) cands ]- let rawStrs = nub [ strList | strList <- map (map showRawSymbol) cands ]- mapM_ (putStrLn . show) rawStrs- return $ map Candidate strs- else+arrivedAtTheEndOfSymbol flag maxLevel isSimple terminalListAfterCursor state stk _actTbl _gotoTbl _prodRules terminalList =+ if length terminalList == 1 then do -- [$]+ _handleParseError flag maxLevel isSimple terminalListAfterCursor state stk _actTbl _gotoTbl _prodRules+ else return [SynCompInterface.ParseError (map terminalToString terminalList)] +_handleParseError flag maxLevel isSimple terminalListAfterCursor state stk _actTbl _gotoTbl _prodRules = do+ let automaton = Automaton {actTbl=_actTbl, gotoTbl=_gotoTbl, prodRules=_prodRules}+ candidateListList <- compCandidates flag maxLevel isSimple 0 [] state automaton stk+ let colorListList =+ [ filterCandidates candidateList terminalListAfterCursor | candidateList <- candidateListList ]+ let strList = nub [ concatStrList strList | strList <- map (map showEmacsColor) colorListList ]+ let rawStrListList = nub [ strList | strList <- map (map showRawEmacsColor) colorListList ]+ debug flag $ show $ map (\x -> (show x ++ "\n")) rawStrListList -- mapM_ (putStrLn . show) rawStrListList+ return $ map Candidate strList++-- | Filter the given candidates with the following texts+data EmacsColor =+ Gray String Line Column -- Overlapping with some in the following text+ | White String -- Not overlapping+ deriving Show++filterCandidates :: (TokenInterface token) => [Candidate] -> [Terminal token] -> [EmacsColor]+filterCandidates candidates terminalListAfterCursor =+ f candidates terminalListAfterCursor []+ where+ f (a:alpha) (b:beta) accm+ | equal a b = f alpha beta (Gray (strCandidate a) (terminalToLine b) (terminalToCol b) : accm)+ | otherwise = f alpha (b:beta) (White (strCandidate a) : accm)+ f [] beta accm = reverse accm+ f (a:alpha) [] accm = f alpha [] (White (strCandidate a) : accm)++ equal (TerminalSymbol s1) (Terminal s2 _ _ _) = s1==s2+ equal (NonterminalSymbol s1) _ = False++ strCandidate (TerminalSymbol s) = s+ strCandidate (NonterminalSymbol s) = "..."++-- | Utilities showSymbol (TerminalSymbol s) = s showSymbol (NonterminalSymbol _) = "..." showRawSymbol (TerminalSymbol s) = s showRawSymbol (NonterminalSymbol s) = s++showEmacsColor (Gray s line col) = "gray " ++ s ++ " " ++ show line ++ " " ++ show col ++ " "+showEmacsColor (White s) = "white " ++ s++showRawEmacsColor (Gray s line col) = s ++ "@" ++ show line ++ "," ++ show col ++ " "+showRawEmacsColor (White s) = s concatStrList [] = "" -- error "The empty candidate?" concatStrList [str] = str
src/parserlib/Terminal.hs view
@@ -1,14 +1,28 @@ {-# LANGUAGE GADTs #-}-module Terminal(Terminal(..), terminalToString) where+module Terminal(Terminal(..), terminalToString, terminalToLine, terminalToCol, token_na) where import TokenInterface +import Data.Maybe+ type Line = Int type Column = Int -data Terminal token where- Terminal :: TokenInterface token => String -> Line -> Column -> token -> Terminal token+data Terminal token where -- Todo: data Terminal token vs. data Symbol = ... | Terminal String in CFG.hs ??+ Terminal :: TokenInterface token => String -> Line -> Column -> Maybe token -> Terminal token+ -- Todo: In Maybe token, Just token for parsing, and Nothing is for syntax complection! +token_na = "token n/a"+ terminalToString :: TokenInterface token => Terminal token -> String-terminalToString (Terminal text line col tok) =+terminalToString (Terminal text line col (Just tok)) = text ++ " at (" ++ show line ++ ", " ++ show col ++ "): " ++ fromToken tok++terminalToString (Terminal text line col Nothing) =+ text ++ " at (" ++ show line ++ ", " ++ show col ++ "): " ++ token_na++terminalToLine :: TokenInterface token => Terminal token -> Int+terminalToLine (Terminal text line col tok) = line++terminalToCol :: TokenInterface token => Terminal token -> Int+terminalToCol (Terminal text line col tok) = col
src/parserlib/TokenInterface.hs view
@@ -1,7 +1,7 @@ module TokenInterface where class TokenInterface token where- toToken :: String -> token+ -- toToken :: String -> token fromToken :: token -> String
src/syncomplib/EmacsServer.hs view
@@ -1,66 +1,72 @@-module EmacsServer where--import SynCompInterface- -import Network.Socket hiding (recv,send)-import Network.Socket.ByteString-import Data.ByteString.Char8-import Control.Monad-import Control.Exception--type ComputeCandidate = String -> Bool -> {- Int -> -} IO [EmacsDataItem]--emacsServer :: ComputeCandidate -> IO ()-emacsServer computeCand = do- sock <- socket AF_INET Stream defaultProtocol- setSocketOption sock ReuseAddr 1- bind sock (SockAddrInet 50000 0)- listen sock 5- acceptLoop computeCand sock `finally` close sock--acceptLoop :: ComputeCandidate -> Socket -> IO ()-acceptLoop computeCand sock = forever $ do- (conn, _) <- accept sock- (cursorPos, isSimple) <- getCursorPos_and_isSimple conn- print (cursorPos, isSimple)- (conn, _) <- accept sock- str <- getSource conn- print str- candidateList <- computeCand str isSimple {- cursorPos -} -- What is cursorPos useful for?- print (Prelude.map show candidateList)- (conn, _) <- accept sock- sendCandidateList conn candidateList- close conn--str2cursorPos_and_isSimple :: String -> (Int,Bool)-str2cursorPos_and_isSimple str =- let [s1,s2] = Prelude.words str- in (read s1 :: Int, read s2 :: Bool)--getCursorPos_and_isSimple :: Socket -> IO (Int, Bool)-getCursorPos_and_isSimple conn = do- str <- recv conn 64- return (str2cursorPos_and_isSimple (unpack str))--getSource :: Socket -> IO String-getSource conn = do- str <- recv conn 64- if Data.ByteString.Char8.length str == 0 then- return (unpack str)- else do- aaa <- getSource conn- return ((unpack str) ++ aaa)--sendCandidateList :: Socket -> [EmacsDataItem] -> IO ()-sendCandidateList conn xs = do- let- f [] = ""- f ((Candidate x) : xs) = "\n" ++ x ++ f xs- f (LexError : xs) = "LexError" ++ f xs- f ((ParseError _) : xs) = "ParseError" ++ f xs- f (SuccessfullyParsed : xs) = "SuccessfullyParsed" ++ f xs- let- s = f xs- do- _ <- send conn (pack s)- print s+module EmacsServer where + +import SynCompInterface + +import Network.Socket hiding (recv,send) +import Network.Socket.ByteString +import Data.ByteString.Char8 +import Control.Monad +import Control.Exception + +type ComputeCandidate = String -> String -> Bool -> {- Int -> -} IO [EmacsDataItem] + +emacsServer :: ComputeCandidate -> IO () +emacsServer computeCand = do + sock <- socket AF_INET Stream defaultProtocol + setSocketOption sock ReuseAddr 1 + bind sock (SockAddrInet 50000 0) + listen sock 5 + acceptLoop computeCand sock `finally` close sock + +acceptLoop :: ComputeCandidate -> Socket -> IO () +acceptLoop computeCand sock = forever $ do + (conn, _) <- accept sock + (cursorPos, isSimple) <- getCursorPos_and_isSimple conn + print (cursorPos, isSimple) + close conn + (conn, _) <- accept sock + str <- getSource conn + print str + close conn + (conn, _) <- accept sock + strAfterCursor <- getSource conn + print strAfterCursor + candidateList <- computeCand str strAfterCursor isSimple + print (Prelude.map show candidateList) + close conn + (conn, _) <- accept sock + sendCandidateList conn candidateList + close conn + +str2cursorPos_and_isSimple :: String -> (Int,Bool) +str2cursorPos_and_isSimple str = + let [s1,s2] = Prelude.words str + in (read s1 :: Int, read s2 :: Bool) + +getCursorPos_and_isSimple :: Socket -> IO (Int, Bool) +getCursorPos_and_isSimple conn = do + str <- recv conn 64 + return (str2cursorPos_and_isSimple (unpack str)) + +getSource :: Socket -> IO String +getSource conn = do + str <- recv conn 64 + if Data.ByteString.Char8.length str == 0 then + return (unpack str) + else do + aaa <- getSource conn + return ((unpack str) ++ aaa) + +sendCandidateList :: Socket -> [EmacsDataItem] -> IO () +sendCandidateList conn xs = do + let + f [] = "" + f ((Candidate x) : xs) = "\n" ++ x ++ f xs + f (LexError : xs) = "LexError" ++ f xs + f ((ParseError _) : xs) = "ParseError" ++ f xs + f (SuccessfullyParsed : xs) = "SuccessfullyParsed" ++ f xs + let + s = f xs + do + _ <- send conn (pack s) + print s
src/syncomplib/SynCompInterface.hs view
@@ -1,8 +1,10 @@-module SynCompInterface where+module SynCompInterface (EmacsDataItem(..)) where data EmacsDataItem =- LexError- | ParseError [String]- | SuccessfullyParsed- | Candidate String- deriving Show+ LexError -- Lex error at some terminal (not $)+ | ParseError [String] -- Parse error at some terminal (not $)+ | Candidate String -- Parse error at the cursor position returning a candidate string + | SuccessfullyParsed -- Successfully parsed until the cursor position+ deriving (Eq, Show)++
test/Spec.hs view
@@ -1,2 +1,8 @@+import System.Process+ main :: IO ()-main = putStrLn "Test suite not yet implemented. Refer to app/parser and app/polyrpc for your testing"+main = do+ text <- readProcess "test.sh" [] ""+ putStrLn text+ return ()+
yapb.cabal view
@@ -1,13 +1,13 @@ cabal-version: 1.12 --- This file has been generated from package.yaml by hpack version 0.31.2.+-- This file has been generated from package.yaml by hpack version 0.34.4. -- -- see: https://github.com/sol/hpack ----- hash: f28a1347c7ddd84cdcba33d0d1afcd5bbea2d63c57749607f36a45874e4aeb3f+-- hash: 859129ea79d8fa58a86693919544d22f7730f727ef3cc4358ce3f31c25db35fb name: yapb-version: 0.1.1+version: 0.1.3 synopsis: Yet Another Parser Builder (YAPB) description: A programmable LALR(1) parser builder system. Please see the README on GitHub at <https://github.com/kwanghoon/yapb#readme> category: parser builder@@ -15,7 +15,7 @@ bug-reports: https://github.com/kwanghoon/yapb/issues author: Kwanghoon Choi maintainer: lazyswamp@gmail.com-copyright: 2020 Kwanghoon Choi+copyright: 2020-2021 Kwanghoon Choi license: BSD3 license-file: LICENSE build-type: Simple@@ -56,6 +56,7 @@ , bytestring >=0.10.8 && <0.11 , directory >=1.3.3 && <1.4 , hashable >=1.3.0 && <1.4+ , hspec , network >=3.1.1 && <3.2 , process >=1.6.5 && <1.7 , regex-tdfa >=1.3.1 && <1.4@@ -70,6 +71,7 @@ ghc-options: -threaded -rtsopts -with-rtsopts=-N build-depends: base >=4.7 && <5+ , hspec , yapb default-language: Haskell2010 @@ -78,6 +80,8 @@ other-modules: Lexer Parser+ ParserSpec+ Run Token Expr Paths_yapb@@ -87,6 +91,7 @@ ghc-options: -threaded -rtsopts -with-rtsopts=-N build-depends: base >=4.7 && <5+ , hspec , regex-tdfa , yapb default-language: Haskell2010@@ -96,6 +101,8 @@ other-modules: Lexer Parser+ SyntaxCompletion+ SyntaxCompletionSpec Token Expr Paths_yapb@@ -105,6 +112,7 @@ ghc-options: -threaded -rtsopts -with-rtsopts=-N build-depends: base >=4.7 && <5+ , hspec , regex-tdfa , yapb default-language: Haskell2010@@ -118,6 +126,7 @@ ghc-options: -threaded -rtsopts -with-rtsopts=-N build-depends: base >=4.7 && <5+ , hspec , yapb default-language: Haskell2010 @@ -131,5 +140,7 @@ ghc-options: -threaded -rtsopts -with-rtsopts=-N build-depends: base >=4.7 && <5+ , hspec+ , process , yapb default-language: Haskell2010