Lexer.hs

{-# OPTIONS  #-}
-----------------------------------------------------------------------------
-- |
-- Module      : Language.Python.Version2.Lexer
-- Copyright   : (c) 2009 Bernie Pope 
-- License     : BSD-style
-- Maintainer  : bjpop@csse.unimelb.edu.au
-- Stability   : experimental
-- Portability : ghc
--
-- Lexical analysis for Python version 2.x programs. 
-- See: <http://docs.python.org/2.6/reference/lexical_analysis.html>.
-----------------------------------------------------------------------------

module Language.Python.Version2.Lexer (
   -- * Lexical analysis
   lex, 
   lexOneToken) where

import Prelude hiding (lex)
import Language.Python.Version2.Parser.Lexer (lexToken, initStartCodeStack)
import Language.Python.Common.Token as Token 
import Language.Python.Common.SrcLocation (initialSrcLocation)
import Language.Python.Common.ParserMonad 
       (ParseState (input), P, runParser, execParser, ParseError, initialState)

-- | Parse a string into a list of Python Tokens, or return an error. 
lex :: String -- ^ The input stream (python source code). 
    -> String -- ^ The name of the python source (filename or input device).
    -> Either ParseError [Token] -- ^ An error or a list of tokens.
lex input srcName =
   execParser lexer state
   where
   initLoc = initialSrcLocation srcName
   state = initialState initLoc input initStartCodeStack

-- | Try to lex the first token in an input string. Return either a parse error
-- or a pair containing the next token and the rest of the input after the token.
lexOneToken :: String -- ^ The input stream (python source code).
         -> String -- ^ The name of the python source (filename or input device).
         -> Either ParseError (Token, String) -- ^ An error or the next token and the rest of the input after the token. 
lexOneToken source srcName =
   case runParser lexToken state of
      Left err -> Left err
      Right (tok, state) -> Right (tok, input state)
   where
   initLoc = initialSrcLocation srcName
   state = initialState initLoc source initStartCodeStack

lexer :: P [Token]
lexer = loop []
   where
   loop toks = do
      tok <- lexToken
      case tok of
         EOFToken {} -> return (reverse toks)
         other -> loop (tok:toks)