language-python 0.4.1 → 0.5.8
raw patch · 26 files changed
Files
- ghc_6_12-13/Language/Python/Common/AST.hs +0/−734
- ghc_normal/Language/Python/Common/AST.hs +0/−733
- language-python.cabal +16/−18
- src/Language/Python/Common.hs +3/−3
- src/Language/Python/Common/AST.hs +795/−0
- src/Language/Python/Common/LexerUtils.hs +37/−14
- src/Language/Python/Common/ParseError.hs +0/−6
- src/Language/Python/Common/ParserMonad.hs +4/−4
- src/Language/Python/Common/ParserUtils.hs +47/−39
- src/Language/Python/Common/Pretty.hs +9/−1
- src/Language/Python/Common/PrettyAST.hs +47/−16
- src/Language/Python/Common/PrettyParseError.hs +1/−1
- src/Language/Python/Common/PrettyToken.hs +6/−1
- src/Language/Python/Common/SrcLocation.hs +7/−0
- src/Language/Python/Common/StringEscape.hs +2/−2
- src/Language/Python/Common/Token.hs +24/−12
- src/Language/Python/Version2.hs +3/−3
- src/Language/Python/Version2/Lexer.hs +1/−1
- src/Language/Python/Version2/Parser.hs +4/−4
- src/Language/Python/Version2/Parser/Lexer.x +3/−6
- src/Language/Python/Version2/Parser/Parser.y +80/−17
- src/Language/Python/Version3.hs +3/−3
- src/Language/Python/Version3/Lexer.hs +1/−1
- src/Language/Python/Version3/Parser.hs +4/−4
- src/Language/Python/Version3/Parser/Lexer.x +30/−16
- src/Language/Python/Version3/Parser/Parser.y +80/−27
− ghc_6_12-13/Language/Python/Common/AST.hs
@@ -1,734 +0,0 @@-{-# LANGUAGE FlexibleInstances, TypeSynonymInstances, CPP, DeriveDataTypeable #-}-{-# OPTIONS_GHC -fomit-interface-pragmas #-}--------------------------------------------------------------------------------- |--- Module : Language.Python.Version2.Syntax.AST --- Copyright : (c) 2009 Bernie Pope --- License : BSD-style--- Maintainer : bjpop@csse.unimelb.edu.au--- Stability : experimental--- Portability : ghc------ Representation of the Python abstract syntax tree (AST). The representation is--- a superset of versions 2.x and 3.x of Python. In many cases they are --- identical. The documentation in this module indicates where they are--- different.------ All the data types have a (polymorphic) parameter which allows the AST to--- be annotated by an arbitrary type (for example source locations). Specialised--- instances of the types are provided for source spans. For example @Module a@ is--- the type of modules, and @ModuleSpan@ is the type of modules annoted with source--- span information.------ Note: there are cases where the AST is more liberal than the formal grammar--- of the language. Therefore some care must be taken when constructing--- Python programs using the raw AST. --------------------------------------------------------------------------------module Language.Python.Common.AST ( - -- * Annotation projection- Annotated (..)- -- * Modules- , Module (..), ModuleSpan- -- * Identifiers and dotted names- , Ident (..), IdentSpan- , DottedName, DottedNameSpan- -- * Statements, suites, parameters, decorators and assignment operators- , Statement (..), StatementSpan- , Suite, SuiteSpan- , Parameter (..), ParameterSpan- , ParamTuple (..), ParamTupleSpan- , Decorator (..), DecoratorSpan- , AssignOp (..), AssignOpSpan- -- * Expressions, operators, arguments and slices- , Expr (..), ExprSpan- , Op (..), OpSpan- , Argument (..), ArgumentSpan- , Slice (..), SliceSpan- -- * Imports- , ImportItem (..), ImportItemSpan- , FromItem (..), FromItemSpan- , FromItems (..), FromItemsSpan- , ImportRelative (..), ImportRelativeSpan- -- * Exceptions- , Handler (..), HandlerSpan- , ExceptClause (..), ExceptClauseSpan- , RaiseExpr (..), RaiseExprSpan- -- * Comprehensions- , Comprehension (..), ComprehensionSpan- , CompFor (..), CompForSpan- , CompIf (..), CompIfSpan- , CompIter (..), CompIterSpan- )- where--import Language.Python.Common.SrcLocation ( Span (getSpan), SrcSpan (..) ) -import Data.Data-------------------------------------------------------------------------------------- | Convenient access to annotations in annotated types. -class Annotated t where- -- | Given an annotated type, project out its annotation value.- annot :: t annot -> annot---- | Identifier.-data Ident annot = Ident { ident_string :: !String, ident_annot :: annot }- deriving (Eq,Ord,Show,Typeable,Data)--type IdentSpan = Ident SrcSpan--instance Span IdentSpan where- getSpan = annot --instance Annotated Ident where- annot = ident_annot---- | A module (Python source file). ------ * Version 2.6 <http://www.python.org/doc/2.6/reference/toplevel_components.html>--- --- * Version 3.1 <http://www.python.org/doc/3.1/reference/toplevel_components.html> --- -newtype Module annot = Module [Statement annot] -- ^ A module is just a sequence of top-level statements.- deriving (Eq,Ord,Show,Typeable,Data)--type ModuleSpan = Module SrcSpan---- | A block of statements. A suite is a group of statements controlled by a clause, --- for example, the body of a loop. ------ * Version 2.6 <http://www.python.org/doc/2.6/reference/compound_stmts.html>--- --- * Version 3.1 <http://www.python.org/doc/3.1/reference/compound_stmts.html>----type Suite annot = [Statement annot] --type SuiteSpan = Suite SrcSpan---- | A compound name constructed with the dot operator.-type DottedName annot = [Ident annot]--type DottedNameSpan = DottedName SrcSpan ---- | An entity imported using the \'import\' keyword.--- --- * Version 2.6 <http://www.python.org/doc/2.6/reference/simple_stmts.html#the-import-statement>------ * Version 3.1 <http://www.python.org/doc/3.1/reference/simple_stmts.html#the-import-statement> ----data ImportItem annot = - ImportItem - { import_item_name :: DottedName annot -- ^ The name of module to import.- , import_as_name :: Maybe (Ident annot) -- ^ An optional name to refer to the entity (the \'as\' name). - , import_item_annot :: annot- }- deriving (Eq,Ord,Show,Typeable,Data)--type ImportItemSpan = ImportItem SrcSpan--instance Span ImportItemSpan where- getSpan = annot --instance Annotated ImportItem where- annot = import_item_annot ---- | An entity imported using the \'from ... import\' construct.------ * Version 2.6 <http://www.python.org/doc/2.6/reference/simple_stmts.html#the-import-statement>--- --- * Version 3.1 <http://www.python.org/doc/3.1/reference/simple_stmts.html#the-import-statement>----data FromItem annot = - FromItem - { from_item_name :: Ident annot -- ^ The name of the entity imported. - , from_as_name :: Maybe (Ident annot) -- ^ An optional name to refer to the entity (the \'as\' name).- , from_item_annot :: annot- }- deriving (Eq,Ord,Show,Typeable,Data)--type FromItemSpan = FromItem SrcSpan--instance Span FromItemSpan where- getSpan = annot --instance Annotated FromItem where- annot = from_item_annot ---- | Items imported using the \'from ... import\' construct.-data FromItems annot - = ImportEverything { from_items_annot :: annot } -- ^ Import everything exported from the module.- | FromItems { from_items_items :: [FromItem annot], from_items_annot :: annot } -- ^ Import a specific list of items from the module.- deriving (Eq,Ord,Show,Typeable,Data)--type FromItemsSpan = FromItems SrcSpan--instance Span FromItemsSpan where- getSpan = annot --instance Annotated FromItems where- annot = from_items_annot ---- | A reference to the module to import from using the \'from ... import\' construct.-data ImportRelative annot - = ImportRelative - { import_relative_dots :: Int- , import_relative_module :: Maybe (DottedName annot) - , import_relative_annot :: annot - }- deriving (Eq,Ord,Show,Typeable,Data)--type ImportRelativeSpan = ImportRelative SrcSpan--instance Span ImportRelativeSpan where- getSpan = annot --instance Annotated ImportRelative where- annot = import_relative_annot ---- | Statements.------ * Simple statements:------ * Version 2.6 <http://www.python.org/doc/2.6/reference/simple_stmts.html>--- --- * Version 3.1 <http://www.python.org/doc/3.1/reference/simple_stmts.html>------ * Compound statements:------ * Version 2.6 <http://www.python.org/doc/2.6/reference/compound_stmts.html>------ * Version 3.1 <http://www.python.org/doc/3.1/reference/compound_stmts.html>----data Statement annot - -- | Import statement.- = Import - { import_items :: [ImportItem annot] -- ^ Items to import.- , stmt_annot :: annot - } - -- | From ... import statement.- | FromImport - { from_module :: ImportRelative annot -- ^ Module to import from.- , from_items :: FromItems annot -- ^ Items to import.- , stmt_annot :: annot- }- -- | While loop. - | While - { while_cond :: Expr annot -- ^ Loop condition.- , while_body :: Suite annot -- ^ Loop body.- , while_else :: Suite annot -- ^ Else clause.- , stmt_annot :: annot- }- -- | For loop. - | For - { for_targets :: [Expr annot] -- ^ Loop variables.- , for_generator :: Expr annot -- ^ Loop generator. - , for_body :: Suite annot -- ^ Loop body- , for_else :: Suite annot -- ^ Else clause.- , stmt_annot :: annot- }- -- | Function definition. - | Fun - { fun_name :: Ident annot -- ^ Function name.- , fun_args :: [Parameter annot] -- ^ Function parameter list.- , fun_result_annotation :: Maybe (Expr annot) -- ^ Optional result annotation.- , fun_body :: Suite annot -- ^ Function body.- , stmt_annot :: annot - }- -- | Class definition. - | Class - { class_name :: Ident annot -- ^ Class name.- , class_args :: [Argument annot] -- ^ Class argument list. In version 2.x this is only ArgExprs. - , class_body :: Suite annot -- ^ Class body.- , stmt_annot :: annot- }- -- | Conditional statement (if-elif-else). - | Conditional - { cond_guards :: [(Expr annot, Suite annot)] -- ^ Sequence of if-elif conditional clauses.- , cond_else :: Suite annot -- ^ Possibly empty unconditional else clause.- , stmt_annot :: annot- }- -- | Assignment statement. - | Assign - { assign_to :: [Expr annot] -- ^ Entity to assign to. - , assign_expr :: Expr annot -- ^ Expression to evaluate.- , stmt_annot :: annot- }- -- | Augmented assignment statement. - | AugmentedAssign - { aug_assign_to :: Expr annot -- ^ Entity to assign to.- , aug_assign_op :: AssignOp annot -- ^ Assignment operator (for example \'+=\').- , aug_assign_expr :: Expr annot -- ^ Expression to evaluate.- , stmt_annot :: annot- }- -- | Decorated definition of a function or class.- | Decorated - { decorated_decorators :: [Decorator annot] -- ^ Decorators.- , decorated_def :: Statement annot -- ^ Function or class definition to be decorated.- , stmt_annot :: annot - }- -- | Return statement (may only occur syntactically nested in a function definition). - | Return - { return_expr :: Maybe (Expr annot) -- ^ Optional expression to evaluate and return to caller.- , stmt_annot :: annot - }- -- | Try statement (exception handling). - | Try - { try_body :: Suite annot -- ^ Try clause.- , try_excepts :: [Handler annot] -- ^ Exception handlers.- , try_else :: Suite annot -- ^ Possibly empty else clause, executed if and when control flows off the end of the try clause.- , try_finally :: Suite annot -- ^ Possibly empty finally clause.- , stmt_annot :: annot- }- -- | Raise statement (exception throwing). - | Raise - { raise_expr :: RaiseExpr annot - , stmt_annot :: annot- }- -- | With statement (context management). - | With - { with_context :: [(Expr annot, Maybe (Expr annot))] -- ^ Context expression(s) (yields a context manager).- , with_body :: Suite annot -- ^ Suite to be managed.- , stmt_annot :: annot- }- -- | Pass statement (null operation). - | Pass { stmt_annot :: annot }- -- | Break statement (may only occur syntactically nested in a for or while loop, but not nested in a function or class definition within that loop). - | Break { stmt_annot :: annot }- -- | Continue statement (may only occur syntactically nested in a for or while loop, but not nested in a function or class definition or finally clause within that loop). - | Continue { stmt_annot :: annot }- -- | Del statement (delete). - | Delete - { del_exprs :: [Expr annot] -- ^ Items to delete.- , stmt_annot :: annot - }- -- | Expression statement. - | StmtExpr { stmt_expr :: Expr annot, stmt_annot :: annot }- -- | Global declaration. - | Global - { global_vars :: [Ident annot] -- ^ Variables declared global in the current block.- , stmt_annot :: annot- }- -- | Nonlocal declaration. /Version 3.x only/. - | NonLocal - { nonLocal_vars :: [Ident annot] -- ^ Variables declared nonlocal in the current block (their binding comes from bound the nearest enclosing scope).- , stmt_annot :: annot- }- -- | Assertion. - | Assert - { assert_exprs :: [Expr annot] -- ^ Expressions being asserted.- , stmt_annot :: annot- }- -- | Print statement. /Version 2 only/. - | Print - { print_chevron :: Bool -- ^ Optional chevron (>>)- , print_exprs :: [Expr annot] -- ^ Arguments to print- , print_trailing_comma :: Bool -- ^ Does it end in a comma?- , stmt_annot :: annot - }- -- | Exec statement. /Version 2 only/. - | Exec- { exec_expr :: Expr annot -- ^ Expression to exec.- , exec_globals_locals :: Maybe (Expr annot, Maybe (Expr annot)) -- ^ Global and local environments to evaluate the expression within.- , stmt_annot :: annot - }- deriving (Eq,Ord,Show,Typeable,Data)--type StatementSpan = Statement SrcSpan--instance Span StatementSpan where- getSpan = annot --instance Annotated Statement where- annot = stmt_annot ---- | The argument for a @raise@ statement.-data RaiseExpr annot- = RaiseV3 (Maybe (Expr annot, Maybe (Expr annot))) -- ^ Optional expression to evaluate, and optional \'from\' clause. /Version 3 only/.- | RaiseV2 (Maybe (Expr annot, (Maybe (Expr annot, Maybe (Expr annot))))) -- ^ /Version 2 only/.- deriving (Eq,Ord,Show,Typeable,Data)--type RaiseExprSpan = RaiseExpr SrcSpan---- | Decorator.-data Decorator annot = - Decorator - { decorator_name :: DottedName annot -- ^ Decorator name.- , decorator_args :: [Argument annot] -- ^ Decorator arguments.- , decorator_annot :: annot - }- deriving (Eq,Ord,Show,Typeable,Data)--type DecoratorSpan = Decorator SrcSpan--instance Span DecoratorSpan where- getSpan = annot --instance Annotated Decorator where- annot = decorator_annot ---- | Formal parameter of function definitions and lambda expressions.--- --- * Version 2.6: ------ * <http://www.python.org/doc/2.6/reference/compound_stmts.html#function-definitions>------ * <http://www.python.org/doc/2.6/reference/expressions.html#calls>------ * Version 3.1: ------ * <http://www.python.org/doc/3.1/reference/compound_stmts.html#function-definitions>------ * <http://www.python.org/doc/3.1/reference/expressions.html#calls>----data Parameter annot- -- | Ordinary named parameter.- = Param - { param_name :: Ident annot -- ^ Parameter name.- , param_py_annotation :: Maybe (Expr annot) -- ^ Optional annotation.- , param_default :: Maybe (Expr annot) -- ^ Optional default value.- , param_annot :: annot- }- -- | Excess positional parameter (single asterisk before its name in the concrete syntax). - | VarArgsPos - { param_name :: Ident annot -- ^ Parameter name.- , param_py_annotation :: Maybe (Expr annot) -- ^ Optional annotation.- , param_annot :: annot- }- -- | Excess keyword parameter (double asterisk before its name in the concrete syntax).- | VarArgsKeyword - { param_name :: Ident annot -- ^ Parameter name.- , param_py_annotation :: Maybe (Expr annot) -- ^ Optional annotation.- , param_annot :: annot- }- -- | Marker for the end of positional parameters (not a parameter itself).- | EndPositional { param_annot :: annot }- -- | Tuple unpack. /Version 2 only/.- | UnPackTuple - { param_unpack_tuple :: ParamTuple annot -- ^ The tuple to unpack.- , param_default :: Maybe (Expr annot) -- ^ Optional default value.- , param_annot :: annot- }- deriving (Eq,Ord,Show,Typeable,Data)--type ParameterSpan = Parameter SrcSpan--instance Span ParameterSpan where- getSpan = annot --instance Annotated Parameter where- annot = param_annot ---- | Tuple unpack parameter. /Version 2 only/.-data ParamTuple annot- = ParamTupleName { param_tuple_name :: Ident annot, param_tuple_annot :: annot } -- ^ A variable name.- | ParamTuple { param_tuple :: [ParamTuple annot], param_tuple_annot :: annot } -- ^ A (possibly nested) tuple parameter.- deriving (Eq,Ord,Show,Typeable,Data)--type ParamTupleSpan = ParamTuple SrcSpan--instance Span ParamTupleSpan where- getSpan = annot--instance Annotated ParamTuple where- annot = param_tuple_annot---- | Arguments to function calls, class declarations and decorators.-data Argument annot- -- | Ordinary argument expression.- = ArgExpr { arg_expr :: Expr annot, arg_annot :: annot }- -- | Excess positional argument.- | ArgVarArgsPos { arg_expr :: Expr annot, arg_annot :: annot }- -- | Excess keyword argument.- | ArgVarArgsKeyword { arg_expr :: Expr annot, arg_annot :: annot }- -- | Keyword argument.- | ArgKeyword - { arg_keyword :: Ident annot -- ^ Keyword name.- , arg_expr :: Expr annot -- ^ Argument expression.- , arg_annot :: annot- }- deriving (Eq,Ord,Show,Typeable,Data)--type ArgumentSpan = Argument SrcSpan--instance Span ArgumentSpan where- getSpan = annot --instance Annotated Argument where- annot = arg_annot ---- | Exception handler. -data Handler annot- = Handler - { handler_clause :: ExceptClause annot- , handler_suite :: Suite annot- , handler_annot :: annot - }- deriving (Eq,Ord,Show,Typeable,Data)--type HandlerSpan = Handler SrcSpan--instance Span HandlerSpan where- getSpan = annot --instance Annotated Handler where- annot = handler_annot ---- | Exception clause. -data ExceptClause annot- = ExceptClause - -- NB: difference with version 3 (has NAME as target, but looks like bug in grammar)- { except_clause :: Maybe (Expr annot, Maybe (Expr annot))- , except_clause_annot :: annot - }- deriving (Eq,Ord,Show,Typeable,Data)--type ExceptClauseSpan = ExceptClause SrcSpan--instance Span ExceptClauseSpan where- getSpan = annot --instance Annotated ExceptClause where- annot = except_clause_annot ---- | Comprehension. In version 3.x this can be used for lists, sets, dictionaries and generators. -data Comprehension e annot- = Comprehension - { comprehension_expr :: e- , comprehension_for :: CompFor annot- , comprehension_annot :: annot - }- deriving (Eq,Ord,Show,Typeable,Data)--type ComprehensionSpan e = Comprehension e SrcSpan--instance Span (ComprehensionSpan e) where- getSpan = annot --instance Annotated (Comprehension e) where- annot = comprehension_annot ---- | Comprehension \'for\' component. -data CompFor annot = - CompFor - { comp_for_exprs :: [Expr annot]- , comp_in_expr :: Expr annot- , comp_for_iter :: Maybe (CompIter annot) - , comp_for_annot :: annot- }- deriving (Eq,Ord,Show,Typeable,Data)--type CompForSpan = CompFor SrcSpan--instance Span CompForSpan where- getSpan = annot --instance Annotated CompFor where- annot = comp_for_annot ---- | Comprehension guard. -data CompIf annot = - CompIf - { comp_if :: Expr annot- , comp_if_iter :: Maybe (CompIter annot)- , comp_if_annot :: annot - }- deriving (Eq,Ord,Show,Typeable,Data)--type CompIfSpan = CompIf SrcSpan--instance Span CompIfSpan where- getSpan = annot --instance Annotated CompIf where- annot = comp_if_annot ---- | Comprehension iterator (either a \'for\' or an \'if\'). -data CompIter annot - = IterFor { comp_iter_for :: CompFor annot, comp_iter_annot :: annot }- | IterIf { comp_iter_if :: CompIf annot, comp_iter_annot :: annot }- deriving (Eq,Ord,Show,Typeable,Data)--type CompIterSpan = CompIter SrcSpan--instance Span CompIterSpan where- getSpan = annot --instance Annotated CompIter where- annot = comp_iter_annot ---- | Expressions.--- --- * Version 2.6 <http://www.python.org/doc/2.6/reference/expressions.html>.--- --- * Version 3.1 <http://www.python.org/doc/3.1/reference/expressions.html>.--- -data Expr annot- -- | Variable.- = Var { var_ident :: Ident annot, expr_annot :: annot }- -- | Literal integer.- | Int { int_value :: Integer, expr_literal :: String, expr_annot :: annot }- -- | Long literal integer. /Version 2 only/.- | LongInt { int_value :: Integer, expr_literal :: String, expr_annot :: annot }- -- | Literal floating point number.- | Float { float_value :: Double, expr_literal :: String, expr_annot :: annot }- -- | Literal imaginary number.- | Imaginary { imaginary_value :: Double, expr_literal :: String, expr_annot :: annot } - -- | Literal boolean.- | Bool { bool_value :: Bool, expr_annot :: annot }- -- | Literal \'None\' value.- | None { expr_annot :: annot } - -- | Ellipsis \'...\'.- | Ellipsis { expr_annot :: annot }- -- | Literal byte string.- | ByteStrings { byte_string_strings :: [String], expr_annot :: annot }- -- | Literal strings (to be concatentated together).- | Strings { strings_strings :: [String], expr_annot :: annot }- -- | Unicode literal strings (to be concatentated together). Version 2 only.- | UnicodeStrings { unicodestrings_strings :: [String], expr_annot :: annot }- -- | Function call. - | Call - { call_fun :: Expr annot -- ^ Expression yielding a callable object (such as a function).- , call_args :: [Argument annot] -- ^ Call arguments.- , expr_annot :: annot- }- -- | Subscription, for example \'x [y]\'. - | Subscript { subscriptee :: Expr annot, subscript_expr :: Expr annot, expr_annot :: annot }- -- | Slicing, for example \'w [x:y:z]\'. - | SlicedExpr { slicee :: Expr annot, slices :: [Slice annot], expr_annot :: annot } - -- | Conditional expresison. - | CondExpr - { ce_true_branch :: Expr annot -- ^ Expression to evaluate if condition is True.- , ce_condition :: Expr annot -- ^ Boolean condition.- , ce_false_branch :: Expr annot -- ^ Expression to evaluate if condition is False.- , expr_annot :: annot- }- -- | Binary operator application.- | BinaryOp { operator :: Op annot, left_op_arg :: Expr annot, right_op_arg :: Expr annot, expr_annot :: annot }- -- | Unary operator application.- | UnaryOp { operator :: Op annot, op_arg :: Expr annot, expr_annot :: annot }- -- | Anonymous function definition (lambda). - | Lambda { lambda_args :: [Parameter annot], lambda_body :: Expr annot, expr_annot :: annot }- -- | Tuple. Can be empty. - | Tuple { tuple_exprs :: [Expr annot], expr_annot :: annot }- -- | Generator yield. - | Yield - { yield_expr :: Maybe (Expr annot) -- ^ Optional expression to yield.- , expr_annot :: annot- }- -- | Generator. - | Generator { gen_comprehension :: Comprehension (Expr annot) annot, expr_annot :: annot }- -- | List comprehension. - | ListComp { list_comprehension :: Comprehension (Expr annot) annot, expr_annot :: annot }- -- | List. - | List { list_exprs :: [Expr annot], expr_annot :: annot }- -- | Dictionary. - | Dictionary { dict_mappings :: [(Expr annot, Expr annot)], expr_annot :: annot }- -- | Dictionary comprehension. /Version 3 only/. - | DictComp { dict_comprehension :: Comprehension (Expr annot, Expr annot) annot, expr_annot :: annot }- -- | Set. - | Set { set_exprs :: [Expr annot], expr_annot :: annot } - -- | Set comprehension. /Version 3 only/. - | SetComp { set_comprehension :: Comprehension (Expr annot) annot, expr_annot :: annot }- -- | Starred expression. /Version 3 only/.- | Starred { starred_expr :: Expr annot, expr_annot :: annot }- -- | Parenthesised expression.- | Paren { paren_expr :: Expr annot, expr_annot :: annot }- -- | String conversion (backquoted expression). Version 2 only. - | StringConversion { backquoted_expr :: Expr annot, expr_anot :: annot }- deriving (Eq,Ord,Show,Typeable,Data)--type ExprSpan = Expr SrcSpan--instance Span ExprSpan where- getSpan = annot --instance Annotated Expr where- annot = expr_annot ---- | Slice compenent.-data Slice annot- = SliceProper - { slice_lower :: Maybe (Expr annot)- , slice_upper :: Maybe (Expr annot)- , slice_stride :: Maybe (Maybe (Expr annot)) - , slice_annot :: annot- } - | SliceExpr - { slice_expr :: Expr annot- , slice_annot :: annot - }- | SliceEllipsis { slice_annot :: annot }- deriving (Eq,Ord,Show,Typeable,Data)--type SliceSpan = Slice SrcSpan--instance Span SliceSpan where- getSpan = annot --instance Annotated Slice where- annot = slice_annot ---- | Operators.-data Op annot- = And { op_annot :: annot } -- ^ \'and\'- | Or { op_annot :: annot } -- ^ \'or\'- | Not { op_annot :: annot } -- ^ \'not\'- | Exponent { op_annot :: annot } -- ^ \'**\'- | LessThan { op_annot :: annot } -- ^ \'<\'- | GreaterThan { op_annot :: annot } -- ^ \'>\'- | Equality { op_annot :: annot } -- ^ \'==\'- | GreaterThanEquals { op_annot :: annot } -- ^ \'>=\'- | LessThanEquals { op_annot :: annot } -- ^ \'<=\'- | NotEquals { op_annot :: annot } -- ^ \'!=\'- | NotEqualsV2 { op_annot :: annot } -- ^ \'<>\'. Version 2 only.- | In { op_annot :: annot } -- ^ \'in\'- | Is { op_annot :: annot } -- ^ \'is\'- | IsNot { op_annot :: annot } -- ^ \'is not\'- | NotIn { op_annot :: annot } -- ^ \'not in\'- | BinaryOr { op_annot :: annot } -- ^ \'|\'- | Xor { op_annot :: annot } -- ^ \'^\'- | BinaryAnd { op_annot :: annot } -- ^ \'&\'- | ShiftLeft { op_annot :: annot } -- ^ \'<<\'- | ShiftRight { op_annot :: annot } -- ^ \'>>\'- | Multiply { op_annot :: annot } -- ^ \'*\'- | Plus { op_annot :: annot } -- ^ \'+\'- | Minus { op_annot :: annot } -- ^ \'-\'- | Divide { op_annot :: annot } -- ^ \'\/\'- | FloorDivide { op_annot :: annot } -- ^ \'\/\/\'- | Invert { op_annot :: annot } -- ^ \'~\' (bitwise inversion of its integer argument)- | Modulo { op_annot :: annot } -- ^ \'%\'- | Dot { op_annot :: annot } -- ^ \'.\'- deriving (Eq,Ord,Show,Typeable,Data)--type OpSpan = Op SrcSpan--instance Span OpSpan where- getSpan = annot --instance Annotated Op where- annot = op_annot ---- | Augmented assignment operators.-data AssignOp annot- = PlusAssign { assignOp_annot :: annot } -- ^ \'+=\'- | MinusAssign { assignOp_annot :: annot } -- ^ \'-=\'- | MultAssign { assignOp_annot :: annot } -- ^ \'*=\'- | DivAssign { assignOp_annot :: annot } -- ^ \'\/=\'- | ModAssign { assignOp_annot :: annot } -- ^ \'%=\'- | PowAssign { assignOp_annot :: annot } -- ^ \'*=\'- | BinAndAssign { assignOp_annot :: annot } -- ^ \'&=\'- | BinOrAssign { assignOp_annot :: annot } -- ^ \'|=\'- | BinXorAssign { assignOp_annot :: annot } -- ^ \'^=\' - | LeftShiftAssign { assignOp_annot :: annot } -- ^ \'<<=\'- | RightShiftAssign { assignOp_annot :: annot } -- ^ \'>>=\'- | FloorDivAssign { assignOp_annot :: annot } -- ^ \'\/\/=\'- deriving (Eq,Ord,Show,Typeable,Data)--type AssignOpSpan = AssignOp SrcSpan--instance Span AssignOpSpan where- getSpan = annot --instance Annotated AssignOp where- annot = assignOp_annot
− ghc_normal/Language/Python/Common/AST.hs
@@ -1,733 +0,0 @@-{-# LANGUAGE FlexibleInstances, TypeSynonymInstances, CPP, DeriveDataTypeable #-}--------------------------------------------------------------------------------- |--- Module : Language.Python.Version2.Syntax.AST --- Copyright : (c) 2009 Bernie Pope --- License : BSD-style--- Maintainer : bjpop@csse.unimelb.edu.au--- Stability : experimental--- Portability : ghc------ Representation of the Python abstract syntax tree (AST). The representation is--- a superset of versions 2.x and 3.x of Python. In many cases they are --- identical. The documentation in this module indicates where they are--- different.------ All the data types have a (polymorphic) parameter which allows the AST to--- be annotated by an arbitrary type (for example source locations). Specialised--- instances of the types are provided for source spans. For example @Module a@ is--- the type of modules, and @ModuleSpan@ is the type of modules annoted with source--- span information.------ Note: there are cases where the AST is more liberal than the formal grammar--- of the language. Therefore some care must be taken when constructing--- Python programs using the raw AST. --------------------------------------------------------------------------------module Language.Python.Common.AST ( - -- * Annotation projection- Annotated (..)- -- * Modules- , Module (..), ModuleSpan- -- * Identifiers and dotted names- , Ident (..), IdentSpan- , DottedName, DottedNameSpan- -- * Statements, suites, parameters, decorators and assignment operators- , Statement (..), StatementSpan- , Suite, SuiteSpan- , Parameter (..), ParameterSpan- , ParamTuple (..), ParamTupleSpan- , Decorator (..), DecoratorSpan- , AssignOp (..), AssignOpSpan- -- * Expressions, operators, arguments and slices- , Expr (..), ExprSpan- , Op (..), OpSpan- , Argument (..), ArgumentSpan- , Slice (..), SliceSpan- -- * Imports- , ImportItem (..), ImportItemSpan- , FromItem (..), FromItemSpan- , FromItems (..), FromItemsSpan- , ImportRelative (..), ImportRelativeSpan- -- * Exceptions- , Handler (..), HandlerSpan- , ExceptClause (..), ExceptClauseSpan- , RaiseExpr (..), RaiseExprSpan- -- * Comprehensions- , Comprehension (..), ComprehensionSpan- , CompFor (..), CompForSpan- , CompIf (..), CompIfSpan- , CompIter (..), CompIterSpan- )- where--import Language.Python.Common.SrcLocation ( Span (getSpan), SrcSpan (..) ) -import Data.Data-------------------------------------------------------------------------------------- | Convenient access to annotations in annotated types. -class Annotated t where- -- | Given an annotated type, project out its annotation value.- annot :: t annot -> annot---- | Identifier.-data Ident annot = Ident { ident_string :: !String, ident_annot :: annot }- deriving (Eq,Ord,Show,Typeable,Data)--type IdentSpan = Ident SrcSpan--instance Span IdentSpan where- getSpan = annot --instance Annotated Ident where- annot = ident_annot---- | A module (Python source file). ------ * Version 2.6 <http://www.python.org/doc/2.6/reference/toplevel_components.html>--- --- * Version 3.1 <http://www.python.org/doc/3.1/reference/toplevel_components.html> --- -newtype Module annot = Module [Statement annot] -- ^ A module is just a sequence of top-level statements.- deriving (Eq,Ord,Show,Typeable,Data)--type ModuleSpan = Module SrcSpan---- | A block of statements. A suite is a group of statements controlled by a clause, --- for example, the body of a loop. ------ * Version 2.6 <http://www.python.org/doc/2.6/reference/compound_stmts.html>--- --- * Version 3.1 <http://www.python.org/doc/3.1/reference/compound_stmts.html>----type Suite annot = [Statement annot] --type SuiteSpan = Suite SrcSpan---- | A compound name constructed with the dot operator.-type DottedName annot = [Ident annot]--type DottedNameSpan = DottedName SrcSpan ---- | An entity imported using the \'import\' keyword.--- --- * Version 2.6 <http://www.python.org/doc/2.6/reference/simple_stmts.html#the-import-statement>------ * Version 3.1 <http://www.python.org/doc/3.1/reference/simple_stmts.html#the-import-statement> ----data ImportItem annot = - ImportItem - { import_item_name :: DottedName annot -- ^ The name of module to import.- , import_as_name :: Maybe (Ident annot) -- ^ An optional name to refer to the entity (the \'as\' name). - , import_item_annot :: annot- }- deriving (Eq,Ord,Show,Typeable,Data)--type ImportItemSpan = ImportItem SrcSpan--instance Span ImportItemSpan where- getSpan = annot --instance Annotated ImportItem where- annot = import_item_annot ---- | An entity imported using the \'from ... import\' construct.------ * Version 2.6 <http://www.python.org/doc/2.6/reference/simple_stmts.html#the-import-statement>--- --- * Version 3.1 <http://www.python.org/doc/3.1/reference/simple_stmts.html#the-import-statement>----data FromItem annot = - FromItem - { from_item_name :: Ident annot -- ^ The name of the entity imported. - , from_as_name :: Maybe (Ident annot) -- ^ An optional name to refer to the entity (the \'as\' name).- , from_item_annot :: annot- }- deriving (Eq,Ord,Show,Typeable,Data)--type FromItemSpan = FromItem SrcSpan--instance Span FromItemSpan where- getSpan = annot --instance Annotated FromItem where- annot = from_item_annot ---- | Items imported using the \'from ... import\' construct.-data FromItems annot - = ImportEverything { from_items_annot :: annot } -- ^ Import everything exported from the module.- | FromItems { from_items_items :: [FromItem annot], from_items_annot :: annot } -- ^ Import a specific list of items from the module.- deriving (Eq,Ord,Show,Typeable,Data)--type FromItemsSpan = FromItems SrcSpan--instance Span FromItemsSpan where- getSpan = annot --instance Annotated FromItems where- annot = from_items_annot ---- | A reference to the module to import from using the \'from ... import\' construct.-data ImportRelative annot - = ImportRelative - { import_relative_dots :: Int- , import_relative_module :: Maybe (DottedName annot) - , import_relative_annot :: annot - }- deriving (Eq,Ord,Show,Typeable,Data)--type ImportRelativeSpan = ImportRelative SrcSpan--instance Span ImportRelativeSpan where- getSpan = annot --instance Annotated ImportRelative where- annot = import_relative_annot ---- | Statements.------ * Simple statements:------ * Version 2.6 <http://www.python.org/doc/2.6/reference/simple_stmts.html>--- --- * Version 3.1 <http://www.python.org/doc/3.1/reference/simple_stmts.html>------ * Compound statements:------ * Version 2.6 <http://www.python.org/doc/2.6/reference/compound_stmts.html>------ * Version 3.1 <http://www.python.org/doc/3.1/reference/compound_stmts.html>----data Statement annot - -- | Import statement.- = Import - { import_items :: [ImportItem annot] -- ^ Items to import.- , stmt_annot :: annot - } - -- | From ... import statement.- | FromImport - { from_module :: ImportRelative annot -- ^ Module to import from.- , from_items :: FromItems annot -- ^ Items to import.- , stmt_annot :: annot- }- -- | While loop. - | While - { while_cond :: Expr annot -- ^ Loop condition.- , while_body :: Suite annot -- ^ Loop body.- , while_else :: Suite annot -- ^ Else clause.- , stmt_annot :: annot- }- -- | For loop. - | For - { for_targets :: [Expr annot] -- ^ Loop variables.- , for_generator :: Expr annot -- ^ Loop generator. - , for_body :: Suite annot -- ^ Loop body- , for_else :: Suite annot -- ^ Else clause.- , stmt_annot :: annot- }- -- | Function definition. - | Fun - { fun_name :: Ident annot -- ^ Function name.- , fun_args :: [Parameter annot] -- ^ Function parameter list.- , fun_result_annotation :: Maybe (Expr annot) -- ^ Optional result annotation.- , fun_body :: Suite annot -- ^ Function body.- , stmt_annot :: annot - }- -- | Class definition. - | Class - { class_name :: Ident annot -- ^ Class name.- , class_args :: [Argument annot] -- ^ Class argument list. In version 2.x this is only ArgExprs. - , class_body :: Suite annot -- ^ Class body.- , stmt_annot :: annot- }- -- | Conditional statement (if-elif-else). - | Conditional - { cond_guards :: [(Expr annot, Suite annot)] -- ^ Sequence of if-elif conditional clauses.- , cond_else :: Suite annot -- ^ Possibly empty unconditional else clause.- , stmt_annot :: annot- }- -- | Assignment statement. - | Assign - { assign_to :: [Expr annot] -- ^ Entity to assign to. - , assign_expr :: Expr annot -- ^ Expression to evaluate.- , stmt_annot :: annot- }- -- | Augmented assignment statement. - | AugmentedAssign - { aug_assign_to :: Expr annot -- ^ Entity to assign to.- , aug_assign_op :: AssignOp annot -- ^ Assignment operator (for example \'+=\').- , aug_assign_expr :: Expr annot -- ^ Expression to evaluate.- , stmt_annot :: annot- }- -- | Decorated definition of a function or class.- | Decorated - { decorated_decorators :: [Decorator annot] -- ^ Decorators.- , decorated_def :: Statement annot -- ^ Function or class definition to be decorated.- , stmt_annot :: annot - }- -- | Return statement (may only occur syntactically nested in a function definition). - | Return - { return_expr :: Maybe (Expr annot) -- ^ Optional expression to evaluate and return to caller.- , stmt_annot :: annot - }- -- | Try statement (exception handling). - | Try - { try_body :: Suite annot -- ^ Try clause.- , try_excepts :: [Handler annot] -- ^ Exception handlers.- , try_else :: Suite annot -- ^ Possibly empty else clause, executed if and when control flows off the end of the try clause.- , try_finally :: Suite annot -- ^ Possibly empty finally clause.- , stmt_annot :: annot- }- -- | Raise statement (exception throwing). - | Raise - { raise_expr :: RaiseExpr annot - , stmt_annot :: annot- }- -- | With statement (context management). - | With - { with_context :: [(Expr annot, Maybe (Expr annot))] -- ^ Context expression(s) (yields a context manager).- , with_body :: Suite annot -- ^ Suite to be managed.- , stmt_annot :: annot- }- -- | Pass statement (null operation). - | Pass { stmt_annot :: annot }- -- | Break statement (may only occur syntactically nested in a for or while loop, but not nested in a function or class definition within that loop). - | Break { stmt_annot :: annot }- -- | Continue statement (may only occur syntactically nested in a for or while loop, but not nested in a function or class definition or finally clause within that loop). - | Continue { stmt_annot :: annot }- -- | Del statement (delete). - | Delete - { del_exprs :: [Expr annot] -- ^ Items to delete.- , stmt_annot :: annot - }- -- | Expression statement. - | StmtExpr { stmt_expr :: Expr annot, stmt_annot :: annot }- -- | Global declaration. - | Global - { global_vars :: [Ident annot] -- ^ Variables declared global in the current block.- , stmt_annot :: annot- }- -- | Nonlocal declaration. /Version 3.x only/. - | NonLocal - { nonLocal_vars :: [Ident annot] -- ^ Variables declared nonlocal in the current block (their binding comes from bound the nearest enclosing scope).- , stmt_annot :: annot- }- -- | Assertion. - | Assert - { assert_exprs :: [Expr annot] -- ^ Expressions being asserted.- , stmt_annot :: annot- }- -- | Print statement. /Version 2 only/. - | Print - { print_chevron :: Bool -- ^ Optional chevron (>>)- , print_exprs :: [Expr annot] -- ^ Arguments to print- , print_trailing_comma :: Bool -- ^ Does it end in a comma?- , stmt_annot :: annot - }- -- | Exec statement. /Version 2 only/. - | Exec- { exec_expr :: Expr annot -- ^ Expression to exec.- , exec_globals_locals :: Maybe (Expr annot, Maybe (Expr annot)) -- ^ Global and local environments to evaluate the expression within.- , stmt_annot :: annot - }- deriving (Eq,Ord,Show,Typeable,Data)--type StatementSpan = Statement SrcSpan--instance Span StatementSpan where- getSpan = annot --instance Annotated Statement where- annot = stmt_annot ---- | The argument for a @raise@ statement.-data RaiseExpr annot- = RaiseV3 (Maybe (Expr annot, Maybe (Expr annot))) -- ^ Optional expression to evaluate, and optional \'from\' clause. /Version 3 only/.- | RaiseV2 (Maybe (Expr annot, (Maybe (Expr annot, Maybe (Expr annot))))) -- ^ /Version 2 only/.- deriving (Eq,Ord,Show,Typeable,Data)--type RaiseExprSpan = RaiseExpr SrcSpan---- | Decorator.-data Decorator annot = - Decorator - { decorator_name :: DottedName annot -- ^ Decorator name.- , decorator_args :: [Argument annot] -- ^ Decorator arguments.- , decorator_annot :: annot - }- deriving (Eq,Ord,Show,Typeable,Data)--type DecoratorSpan = Decorator SrcSpan--instance Span DecoratorSpan where- getSpan = annot --instance Annotated Decorator where- annot = decorator_annot ---- | Formal parameter of function definitions and lambda expressions.--- --- * Version 2.6: ------ * <http://www.python.org/doc/2.6/reference/compound_stmts.html#function-definitions>------ * <http://www.python.org/doc/2.6/reference/expressions.html#calls>------ * Version 3.1: ------ * <http://www.python.org/doc/3.1/reference/compound_stmts.html#function-definitions>------ * <http://www.python.org/doc/3.1/reference/expressions.html#calls>----data Parameter annot- -- | Ordinary named parameter.- = Param - { param_name :: Ident annot -- ^ Parameter name.- , param_py_annotation :: Maybe (Expr annot) -- ^ Optional annotation.- , param_default :: Maybe (Expr annot) -- ^ Optional default value.- , param_annot :: annot- }- -- | Excess positional parameter (single asterisk before its name in the concrete syntax). - | VarArgsPos - { param_name :: Ident annot -- ^ Parameter name.- , param_py_annotation :: Maybe (Expr annot) -- ^ Optional annotation.- , param_annot :: annot- }- -- | Excess keyword parameter (double asterisk before its name in the concrete syntax).- | VarArgsKeyword - { param_name :: Ident annot -- ^ Parameter name.- , param_py_annotation :: Maybe (Expr annot) -- ^ Optional annotation.- , param_annot :: annot- }- -- | Marker for the end of positional parameters (not a parameter itself).- | EndPositional { param_annot :: annot }- -- | Tuple unpack. /Version 2 only/.- | UnPackTuple - { param_unpack_tuple :: ParamTuple annot -- ^ The tuple to unpack.- , param_default :: Maybe (Expr annot) -- ^ Optional default value.- , param_annot :: annot- }- deriving (Eq,Ord,Show,Typeable,Data)--type ParameterSpan = Parameter SrcSpan--instance Span ParameterSpan where- getSpan = annot --instance Annotated Parameter where- annot = param_annot ---- | Tuple unpack parameter. /Version 2 only/.-data ParamTuple annot- = ParamTupleName { param_tuple_name :: Ident annot, param_tuple_annot :: annot } -- ^ A variable name.- | ParamTuple { param_tuple :: [ParamTuple annot], param_tuple_annot :: annot } -- ^ A (possibly nested) tuple parameter.- deriving (Eq,Ord,Show,Typeable,Data)--type ParamTupleSpan = ParamTuple SrcSpan--instance Span ParamTupleSpan where- getSpan = annot--instance Annotated ParamTuple where- annot = param_tuple_annot---- | Arguments to function calls, class declarations and decorators.-data Argument annot- -- | Ordinary argument expression.- = ArgExpr { arg_expr :: Expr annot, arg_annot :: annot }- -- | Excess positional argument.- | ArgVarArgsPos { arg_expr :: Expr annot, arg_annot :: annot }- -- | Excess keyword argument.- | ArgVarArgsKeyword { arg_expr :: Expr annot, arg_annot :: annot }- -- | Keyword argument.- | ArgKeyword - { arg_keyword :: Ident annot -- ^ Keyword name.- , arg_expr :: Expr annot -- ^ Argument expression.- , arg_annot :: annot- }- deriving (Eq,Ord,Show,Typeable,Data)--type ArgumentSpan = Argument SrcSpan--instance Span ArgumentSpan where- getSpan = annot --instance Annotated Argument where- annot = arg_annot ---- | Exception handler. -data Handler annot- = Handler - { handler_clause :: ExceptClause annot- , handler_suite :: Suite annot- , handler_annot :: annot - }- deriving (Eq,Ord,Show,Typeable,Data)--type HandlerSpan = Handler SrcSpan--instance Span HandlerSpan where- getSpan = annot --instance Annotated Handler where- annot = handler_annot ---- | Exception clause. -data ExceptClause annot- = ExceptClause - -- NB: difference with version 3 (has NAME as target, but looks like bug in grammar)- { except_clause :: Maybe (Expr annot, Maybe (Expr annot))- , except_clause_annot :: annot - }- deriving (Eq,Ord,Show,Typeable,Data)--type ExceptClauseSpan = ExceptClause SrcSpan--instance Span ExceptClauseSpan where- getSpan = annot --instance Annotated ExceptClause where- annot = except_clause_annot ---- | Comprehension. In version 3.x this can be used for lists, sets, dictionaries and generators. -data Comprehension e annot- = Comprehension - { comprehension_expr :: e- , comprehension_for :: CompFor annot- , comprehension_annot :: annot - }- deriving (Eq,Ord,Show,Typeable,Data)--type ComprehensionSpan e = Comprehension e SrcSpan--instance Span (ComprehensionSpan e) where- getSpan = annot --instance Annotated (Comprehension e) where- annot = comprehension_annot ---- | Comprehension \'for\' component. -data CompFor annot = - CompFor - { comp_for_exprs :: [Expr annot]- , comp_in_expr :: Expr annot- , comp_for_iter :: Maybe (CompIter annot) - , comp_for_annot :: annot- }- deriving (Eq,Ord,Show,Typeable,Data)--type CompForSpan = CompFor SrcSpan--instance Span CompForSpan where- getSpan = annot --instance Annotated CompFor where- annot = comp_for_annot ---- | Comprehension guard. -data CompIf annot = - CompIf - { comp_if :: Expr annot- , comp_if_iter :: Maybe (CompIter annot)- , comp_if_annot :: annot - }- deriving (Eq,Ord,Show,Typeable,Data)--type CompIfSpan = CompIf SrcSpan--instance Span CompIfSpan where- getSpan = annot --instance Annotated CompIf where- annot = comp_if_annot ---- | Comprehension iterator (either a \'for\' or an \'if\'). -data CompIter annot - = IterFor { comp_iter_for :: CompFor annot, comp_iter_annot :: annot }- | IterIf { comp_iter_if :: CompIf annot, comp_iter_annot :: annot }- deriving (Eq,Ord,Show,Typeable,Data)--type CompIterSpan = CompIter SrcSpan--instance Span CompIterSpan where- getSpan = annot --instance Annotated CompIter where- annot = comp_iter_annot ---- | Expressions.--- --- * Version 2.6 <http://www.python.org/doc/2.6/reference/expressions.html>.--- --- * Version 3.1 <http://www.python.org/doc/3.1/reference/expressions.html>.--- -data Expr annot- -- | Variable.- = Var { var_ident :: Ident annot, expr_annot :: annot }- -- | Literal integer.- | Int { int_value :: Integer, expr_literal :: String, expr_annot :: annot }- -- | Long literal integer. /Version 2 only/.- | LongInt { int_value :: Integer, expr_literal :: String, expr_annot :: annot }- -- | Literal floating point number.- | Float { float_value :: Double, expr_literal :: String, expr_annot :: annot }- -- | Literal imaginary number.- | Imaginary { imaginary_value :: Double, expr_literal :: String, expr_annot :: annot } - -- | Literal boolean.- | Bool { bool_value :: Bool, expr_annot :: annot }- -- | Literal \'None\' value.- | None { expr_annot :: annot } - -- | Ellipsis \'...\'.- | Ellipsis { expr_annot :: annot }- -- | Literal byte string.- | ByteStrings { byte_string_strings :: [String], expr_annot :: annot }- -- | Literal strings (to be concatentated together).- | Strings { strings_strings :: [String], expr_annot :: annot }- -- | Unicode literal strings (to be concatentated together). Version 2 only.- | UnicodeStrings { unicodestrings_strings :: [String], expr_annot :: annot }- -- | Function call. - | Call - { call_fun :: Expr annot -- ^ Expression yielding a callable object (such as a function).- , call_args :: [Argument annot] -- ^ Call arguments.- , expr_annot :: annot- }- -- | Subscription, for example \'x [y]\'. - | Subscript { subscriptee :: Expr annot, subscript_expr :: Expr annot, expr_annot :: annot }- -- | Slicing, for example \'w [x:y:z]\'. - | SlicedExpr { slicee :: Expr annot, slices :: [Slice annot], expr_annot :: annot } - -- | Conditional expresison. - | CondExpr - { ce_true_branch :: Expr annot -- ^ Expression to evaluate if condition is True.- , ce_condition :: Expr annot -- ^ Boolean condition.- , ce_false_branch :: Expr annot -- ^ Expression to evaluate if condition is False.- , expr_annot :: annot- }- -- | Binary operator application.- | BinaryOp { operator :: Op annot, left_op_arg :: Expr annot, right_op_arg :: Expr annot, expr_annot :: annot }- -- | Unary operator application.- | UnaryOp { operator :: Op annot, op_arg :: Expr annot, expr_annot :: annot }- -- | Anonymous function definition (lambda). - | Lambda { lambda_args :: [Parameter annot], lambda_body :: Expr annot, expr_annot :: annot }- -- | Tuple. Can be empty. - | Tuple { tuple_exprs :: [Expr annot], expr_annot :: annot }- -- | Generator yield. - | Yield - { yield_expr :: Maybe (Expr annot) -- ^ Optional expression to yield.- , expr_annot :: annot- }- -- | Generator. - | Generator { gen_comprehension :: Comprehension (Expr annot) annot, expr_annot :: annot }- -- | List comprehension. - | ListComp { list_comprehension :: Comprehension (Expr annot) annot, expr_annot :: annot }- -- | List. - | List { list_exprs :: [Expr annot], expr_annot :: annot }- -- | Dictionary. - | Dictionary { dict_mappings :: [(Expr annot, Expr annot)], expr_annot :: annot }- -- | Dictionary comprehension. /Version 3 only/. - | DictComp { dict_comprehension :: Comprehension (Expr annot, Expr annot) annot, expr_annot :: annot }- -- | Set. - | Set { set_exprs :: [Expr annot], expr_annot :: annot } - -- | Set comprehension. /Version 3 only/. - | SetComp { set_comprehension :: Comprehension (Expr annot) annot, expr_annot :: annot }- -- | Starred expression. /Version 3 only/.- | Starred { starred_expr :: Expr annot, expr_annot :: annot }- -- | Parenthesised expression.- | Paren { paren_expr :: Expr annot, expr_annot :: annot }- -- | String conversion (backquoted expression). Version 2 only. - | StringConversion { backquoted_expr :: Expr annot, expr_anot :: annot }- deriving (Eq,Ord,Show,Typeable,Data)--type ExprSpan = Expr SrcSpan--instance Span ExprSpan where- getSpan = annot --instance Annotated Expr where- annot = expr_annot ---- | Slice compenent.-data Slice annot- = SliceProper - { slice_lower :: Maybe (Expr annot)- , slice_upper :: Maybe (Expr annot)- , slice_stride :: Maybe (Maybe (Expr annot)) - , slice_annot :: annot- } - | SliceExpr - { slice_expr :: Expr annot- , slice_annot :: annot - }- | SliceEllipsis { slice_annot :: annot }- deriving (Eq,Ord,Show,Typeable,Data)--type SliceSpan = Slice SrcSpan--instance Span SliceSpan where- getSpan = annot --instance Annotated Slice where- annot = slice_annot ---- | Operators.-data Op annot- = And { op_annot :: annot } -- ^ \'and\'- | Or { op_annot :: annot } -- ^ \'or\'- | Not { op_annot :: annot } -- ^ \'not\'- | Exponent { op_annot :: annot } -- ^ \'**\'- | LessThan { op_annot :: annot } -- ^ \'<\'- | GreaterThan { op_annot :: annot } -- ^ \'>\'- | Equality { op_annot :: annot } -- ^ \'==\'- | GreaterThanEquals { op_annot :: annot } -- ^ \'>=\'- | LessThanEquals { op_annot :: annot } -- ^ \'<=\'- | NotEquals { op_annot :: annot } -- ^ \'!=\'- | NotEqualsV2 { op_annot :: annot } -- ^ \'<>\'. Version 2 only.- | In { op_annot :: annot } -- ^ \'in\'- | Is { op_annot :: annot } -- ^ \'is\'- | IsNot { op_annot :: annot } -- ^ \'is not\'- | NotIn { op_annot :: annot } -- ^ \'not in\'- | BinaryOr { op_annot :: annot } -- ^ \'|\'- | Xor { op_annot :: annot } -- ^ \'^\'- | BinaryAnd { op_annot :: annot } -- ^ \'&\'- | ShiftLeft { op_annot :: annot } -- ^ \'<<\'- | ShiftRight { op_annot :: annot } -- ^ \'>>\'- | Multiply { op_annot :: annot } -- ^ \'*\'- | Plus { op_annot :: annot } -- ^ \'+\'- | Minus { op_annot :: annot } -- ^ \'-\'- | Divide { op_annot :: annot } -- ^ \'\/\'- | FloorDivide { op_annot :: annot } -- ^ \'\/\/\'- | Invert { op_annot :: annot } -- ^ \'~\' (bitwise inversion of its integer argument)- | Modulo { op_annot :: annot } -- ^ \'%\'- | Dot { op_annot :: annot } -- ^ \'.\'- deriving (Eq,Ord,Show,Typeable,Data)--type OpSpan = Op SrcSpan--instance Span OpSpan where- getSpan = annot --instance Annotated Op where- annot = op_annot ---- | Augmented assignment operators.-data AssignOp annot- = PlusAssign { assignOp_annot :: annot } -- ^ \'+=\'- | MinusAssign { assignOp_annot :: annot } -- ^ \'-=\'- | MultAssign { assignOp_annot :: annot } -- ^ \'*=\'- | DivAssign { assignOp_annot :: annot } -- ^ \'\/=\'- | ModAssign { assignOp_annot :: annot } -- ^ \'%=\'- | PowAssign { assignOp_annot :: annot } -- ^ \'*=\'- | BinAndAssign { assignOp_annot :: annot } -- ^ \'&=\'- | BinOrAssign { assignOp_annot :: annot } -- ^ \'|=\'- | BinXorAssign { assignOp_annot :: annot } -- ^ \'^=\' - | LeftShiftAssign { assignOp_annot :: annot } -- ^ \'<<=\'- | RightShiftAssign { assignOp_annot :: annot } -- ^ \'>>=\'- | FloorDivAssign { assignOp_annot :: annot } -- ^ \'\/\/=\'- deriving (Eq,Ord,Show,Typeable,Data)--type AssignOpSpan = AssignOp SrcSpan--instance Span AssignOpSpan where- getSpan = annot --instance Annotated AssignOp where- annot = assignOp_annot
language-python.cabal view
@@ -1,43 +1,41 @@ name: language-python-version: 0.4.1-cabal-version: >= 1.6+version: 0.5.8+cabal-version: >= 1.10 synopsis: Parsing and pretty printing of Python code. description: language-python is a Haskell library for lexical analysis, parsing and pretty printing Python code. It supports versions 2.x and 3.x of Python. category: Language license: BSD3 license-file: LICENSE-copyright: (c) 2008-2014 Bernard James Pope+copyright: (c) 2008-2019 Bernard James Pope author: Bernard James Pope (Bernie Pope) maintainer: florbitous@gmail.com homepage: http://github.com/bjpop/language-python -build-depends: base build-type: Simple stability: experimental-extra-source-files: src/Language/Python/Version3/Parser/Parser.y - src/Language/Python/Version3/Parser/Lexer.x - ghc_6_12-13/Language/Python/Common/AST.hs - ghc_normal/Language/Python/Common/AST.hs +extra-source-files: src/Language/Python/Version3/Parser/Parser.y+ src/Language/Python/Version3/Parser/Lexer.x+ src/Language/Python/Version2/Parser/Parser.y+ src/Language/Python/Version2/Parser/Lexer.x+tested-with: GHC ==7.8.4, GHC ==7.10.3, GHC ==8.0.2, GHC ==8.2.1, GHC ==8.6.3 source-repository head type: git location: https://github.com/bjpop/language-python Library- -- work around the memory usage bug in ghc 6.12.x- if impl(ghc < 6.12) || impl(ghc >= 6.14)- hs-source-dirs: src ghc_normal- else - hs-source-dirs: src ghc_6_12-13-+ default-language: Haskell2010+ hs-source-dirs: src + ghc-options: -fwarn-incomplete-patterns -fwarn-unused-imports -fwarn-warnings-deprecations build-depends: base == 4.*,- containers == 0.5.*,+ containers >= 0.5 && < 0.7, pretty == 1.1.*, array >= 0.4 && < 0.6,- transformers >= 0.3 && < 0.5,- monads-tf == 0.1.*- build-tools: happy, alex+ transformers >= 0.3 && < 0.6,+ monads-tf == 0.1.*,+ utf8-string >= 1 && < 2+ build-tools: happy, alex exposed-modules: Language.Python.Common Language.Python.Common.ParseError
src/Language/Python/Common.hs view
@@ -33,8 +33,8 @@ import Language.Python.Common.Pretty import Language.Python.Common.Token import Language.Python.Common.AST -import Language.Python.Common.PrettyAST -import Language.Python.Common.PrettyToken +import Language.Python.Common.PrettyAST ()+import Language.Python.Common.PrettyToken () import Language.Python.Common.SrcLocation -import Language.Python.Common.PrettyParseError +import Language.Python.Common.PrettyParseError () import Language.Python.Common.ParseError
+ src/Language/Python/Common/AST.hs view
@@ -0,0 +1,795 @@+{-# LANGUAGE FlexibleInstances, TypeSynonymInstances, CPP, DeriveDataTypeable, DeriveFunctor #-}+-----------------------------------------------------------------------------+-- |+-- Module : Language.Python.Version2.Syntax.AST +-- Copyright : (c) 2009 Bernie Pope +-- License : BSD-style+-- Maintainer : bjpop@csse.unimelb.edu.au+-- Stability : experimental+-- Portability : ghc+--+-- Representation of the Python abstract syntax tree (AST). The representation is+-- a superset of versions 2.x and 3.x of Python. In many cases they are +-- identical. The documentation in this module indicates where they are+-- different.+--+-- All the data types have a (polymorphic) parameter which allows the AST to+-- be annotated by an arbitrary type (for example source locations). Specialised+-- instances of the types are provided for source spans. For example @Module a@ is+-- the type of modules, and @ModuleSpan@ is the type of modules annoted with source+-- span information.+--+-- Note: there are cases where the AST is more liberal than the formal grammar+-- of the language. Therefore some care must be taken when constructing+-- Python programs using the raw AST. +-----------------------------------------------------------------------------++module Language.Python.Common.AST ( + -- * Annotation projection+ Annotated (..)+ -- * Modules+ , Module (..), ModuleSpan+ -- * Identifiers and dotted names+ , Ident (..), IdentSpan+ , DottedName, DottedNameSpan+ -- * Statements, suites, parameters, decorators and assignment operators+ , Statement (..), StatementSpan+ , Suite, SuiteSpan+ , Parameter (..), ParameterSpan+ , ParamTuple (..), ParamTupleSpan+ , Decorator (..), DecoratorSpan+ , AssignOp (..), AssignOpSpan+ -- * Expressions, operators, arguments and slices+ , Expr (..), ExprSpan+ , Op (..), OpSpan+ , Argument (..), ArgumentSpan+ , Slice (..), SliceSpan+ , DictKeyDatumList (..), DictKeyDatumListSpan+ , YieldArg (..), YieldArgSpan+ -- * Imports+ , ImportItem (..), ImportItemSpan+ , FromItem (..), FromItemSpan+ , FromItems (..), FromItemsSpan+ , ImportRelative (..), ImportRelativeSpan+ -- * Exceptions+ , Handler (..), HandlerSpan+ , ExceptClause (..), ExceptClauseSpan+ , RaiseExpr (..), RaiseExprSpan+ -- * Comprehensions+ , Comprehension (..), ComprehensionSpan+ , ComprehensionExpr (..), ComprehensionExprSpan+ , CompFor (..), CompForSpan+ , CompIf (..), CompIfSpan+ , CompIter (..), CompIterSpan+ )+ where++import Language.Python.Common.SrcLocation ( Span (getSpan), SrcSpan (..), spanning ) +import Data.Data++--------------------------------------------------------------------------------++-- | Convenient access to annotations in annotated types. +class Annotated t where+ -- | Given an annotated type, project out its annotation value.+ annot :: t annot -> annot++-- | Identifier.+data Ident annot = Ident { ident_string :: !String, ident_annot :: annot }+ deriving (Eq,Ord,Show,Typeable,Data,Functor)++type IdentSpan = Ident SrcSpan++instance Span IdentSpan where+ getSpan = annot ++instance Annotated Ident where+ annot = ident_annot++-- | A module (Python source file). +--+-- * Version 2.6 <http://docs.python.org/2.6/reference/toplevel_components.html>+-- +-- * Version 3.1 <http://docs.python.org/3.1/reference/toplevel_components.html> +-- +newtype Module annot = Module [Statement annot] -- ^ A module is just a sequence of top-level statements.+ deriving (Eq,Ord,Show,Typeable,Data,Functor)++type ModuleSpan = Module SrcSpan++-- | A block of statements. A suite is a group of statements controlled by a clause, +-- for example, the body of a loop. +--+-- * Version 2.6 <http://docs.python.org/2.6/reference/compound_stmts.html>+-- +-- * Version 3.1 <http://docs.python.org/3.1/reference/compound_stmts.html>+--+type Suite annot = [Statement annot] ++type SuiteSpan = Suite SrcSpan++-- | A compound name constructed with the dot operator.+type DottedName annot = [Ident annot]++type DottedNameSpan = DottedName SrcSpan ++-- | An entity imported using the \'import\' keyword.+-- +-- * Version 2.6 <http://docs.python.org/2.6/reference/simple_stmts.html#the-import-statement>+--+-- * Version 3.1 <http://docs.python.org/3.1/reference/simple_stmts.html#the-import-statement> +--+data ImportItem annot = + ImportItem + { import_item_name :: DottedName annot -- ^ The name of module to import.+ , import_as_name :: Maybe (Ident annot) -- ^ An optional name to refer to the entity (the \'as\' name). + , import_item_annot :: annot+ }+ deriving (Eq,Ord,Show,Typeable,Data,Functor)++type ImportItemSpan = ImportItem SrcSpan++instance Span ImportItemSpan where+ getSpan = annot ++instance Annotated ImportItem where+ annot = import_item_annot ++-- | An entity imported using the \'from ... import\' construct.+--+-- * Version 2.6 <http://docs.python.org/2.6/reference/simple_stmts.html#the-import-statement>+-- +-- * Version 3.1 <http://docs.python.org/3.1/reference/simple_stmts.html#the-import-statement>+--+data FromItem annot = + FromItem + { from_item_name :: Ident annot -- ^ The name of the entity imported. + , from_as_name :: Maybe (Ident annot) -- ^ An optional name to refer to the entity (the \'as\' name).+ , from_item_annot :: annot+ }+ deriving (Eq,Ord,Show,Typeable,Data,Functor)++type FromItemSpan = FromItem SrcSpan++instance Span FromItemSpan where+ getSpan = annot ++instance Annotated FromItem where+ annot = from_item_annot ++-- | Items imported using the \'from ... import\' construct.+data FromItems annot + = ImportEverything { from_items_annot :: annot } -- ^ Import everything exported from the module.+ | FromItems { from_items_items :: [FromItem annot], from_items_annot :: annot } -- ^ Import a specific list of items from the module.+ deriving (Eq,Ord,Show,Typeable,Data,Functor)++type FromItemsSpan = FromItems SrcSpan++instance Span FromItemsSpan where+ getSpan = annot ++instance Annotated FromItems where+ annot = from_items_annot ++-- | A reference to the module to import from using the \'from ... import\' construct.+data ImportRelative annot + = ImportRelative + { import_relative_dots :: Int+ , import_relative_module :: Maybe (DottedName annot) + , import_relative_annot :: annot + }+ deriving (Eq,Ord,Show,Typeable,Data,Functor)++type ImportRelativeSpan = ImportRelative SrcSpan++instance Span ImportRelativeSpan where+ getSpan = annot ++instance Annotated ImportRelative where+ annot = import_relative_annot ++-- | Statements.+--+-- * Simple statements:+--+-- * Version 2.6 <http://docs.python.org/2.6/reference/simple_stmts.html>+-- +-- * Version 3.1 <http://docs.python.org/3.1/reference/simple_stmts.html>+--+-- * Compound statements:+--+-- * Version 2.6 <http://docs.python.org/2.6/reference/compound_stmts.html>+--+-- * Version 3.1 <http://docs.python.org/3.1/reference/compound_stmts.html>+--+data Statement annot + -- | Import statement.+ = Import + { import_items :: [ImportItem annot] -- ^ Items to import.+ , stmt_annot :: annot + } + -- | From ... import statement.+ | FromImport + { from_module :: ImportRelative annot -- ^ Module to import from.+ , from_items :: FromItems annot -- ^ Items to import.+ , stmt_annot :: annot+ }+ -- | While loop. + | While + { while_cond :: Expr annot -- ^ Loop condition.+ , while_body :: Suite annot -- ^ Loop body.+ , while_else :: Suite annot -- ^ Else clause.+ , stmt_annot :: annot+ }+ -- | For loop. + | For + { for_targets :: [Expr annot] -- ^ Loop variables.+ , for_generator :: Expr annot -- ^ Loop generator. + , for_body :: Suite annot -- ^ Loop body+ , for_else :: Suite annot -- ^ Else clause.+ , stmt_annot :: annot+ }+ | AsyncFor+ { for_stmt :: Statement annot -- ^ For statement+ , stmt_annot :: annot+ }+ -- | Function definition. + | Fun + { fun_name :: Ident annot -- ^ Function name.+ , fun_args :: [Parameter annot] -- ^ Function parameter list.+ , fun_result_annotation :: Maybe (Expr annot) -- ^ Optional result annotation.+ , fun_body :: Suite annot -- ^ Function body.+ , stmt_annot :: annot + }+ | AsyncFun+ { fun_def :: Statement annot -- ^ Function definition (Fun)+ , stmt_annot :: annot+ }+ -- | Class definition. + | Class + { class_name :: Ident annot -- ^ Class name.+ , class_args :: [Argument annot] -- ^ Class argument list. In version 2.x this is only ArgExprs. + , class_body :: Suite annot -- ^ Class body.+ , stmt_annot :: annot+ }+ -- | Conditional statement (if-elif-else). + | Conditional + { cond_guards :: [(Expr annot, Suite annot)] -- ^ Sequence of if-elif conditional clauses.+ , cond_else :: Suite annot -- ^ Possibly empty unconditional else clause.+ , stmt_annot :: annot+ }+ -- | Assignment statement. + | Assign + { assign_to :: [Expr annot] -- ^ Entity to assign to. + , assign_expr :: Expr annot -- ^ Expression to evaluate.+ , stmt_annot :: annot+ }+ -- | Augmented assignment statement. + | AugmentedAssign + { aug_assign_to :: Expr annot -- ^ Entity to assign to.+ , aug_assign_op :: AssignOp annot -- ^ Assignment operator (for example \'+=\').+ , aug_assign_expr :: Expr annot -- ^ Expression to evaluate.+ , stmt_annot :: annot+ }+ | AnnotatedAssign+ { ann_assign_annotation :: Expr annot+ , ann_assign_to :: Expr annot+ , ann_assign_expr :: Maybe (Expr annot)+ , stmt_annot :: annot+ }+ -- | Decorated definition of a function or class.+ | Decorated + { decorated_decorators :: [Decorator annot] -- ^ Decorators.+ , decorated_def :: Statement annot -- ^ Function or class definition to be decorated.+ , stmt_annot :: annot + }+ -- | Return statement (may only occur syntactically nested in a function definition). + | Return + { return_expr :: Maybe (Expr annot) -- ^ Optional expression to evaluate and return to caller.+ , stmt_annot :: annot + }+ -- | Try statement (exception handling). + | Try + { try_body :: Suite annot -- ^ Try clause.+ , try_excepts :: [Handler annot] -- ^ Exception handlers.+ , try_else :: Suite annot -- ^ Possibly empty else clause, executed if and when control flows off the end of the try clause.+ , try_finally :: Suite annot -- ^ Possibly empty finally clause.+ , stmt_annot :: annot+ }+ -- | Raise statement (exception throwing). + | Raise + { raise_expr :: RaiseExpr annot + , stmt_annot :: annot+ }+ -- | With statement (context management). + | With + { with_context :: [(Expr annot, Maybe (Expr annot))] -- ^ Context expression(s) (yields a context manager).+ , with_body :: Suite annot -- ^ Suite to be managed.+ , stmt_annot :: annot+ }+ | AsyncWith+ { with_stmt :: Statement annot -- ^ With statement+ , stmt_annot :: annot+ }+ -- | Pass statement (null operation). + | Pass { stmt_annot :: annot }+ -- | Break statement (may only occur syntactically nested in a for or while loop, but not nested in a function or class definition within that loop). + | Break { stmt_annot :: annot }+ -- | Continue statement (may only occur syntactically nested in a for or while loop, but not nested in a function or class definition or finally clause within that loop). + | Continue { stmt_annot :: annot }+ -- | Del statement (delete). + | Delete + { del_exprs :: [Expr annot] -- ^ Items to delete.+ , stmt_annot :: annot + }+ -- | Expression statement. + | StmtExpr { stmt_expr :: Expr annot, stmt_annot :: annot }+ -- | Global declaration. + | Global + { global_vars :: [Ident annot] -- ^ Variables declared global in the current block.+ , stmt_annot :: annot+ }+ -- | Nonlocal declaration. /Version 3.x only/. + | NonLocal + { nonLocal_vars :: [Ident annot] -- ^ Variables declared nonlocal in the current block (their binding comes from bound the nearest enclosing scope).+ , stmt_annot :: annot+ }+ -- | Assertion. + | Assert + { assert_exprs :: [Expr annot] -- ^ Expressions being asserted.+ , stmt_annot :: annot+ }+ -- | Print statement. /Version 2 only/. + | Print + { print_chevron :: Bool -- ^ Optional chevron (>>)+ , print_exprs :: [Expr annot] -- ^ Arguments to print+ , print_trailing_comma :: Bool -- ^ Does it end in a comma?+ , stmt_annot :: annot + }+ -- | Exec statement. /Version 2 only/. + | Exec+ { exec_expr :: Expr annot -- ^ Expression to exec.+ , exec_globals_locals :: Maybe (Expr annot, Maybe (Expr annot)) -- ^ Global and local environments to evaluate the expression within.+ , stmt_annot :: annot + }+ deriving (Eq,Ord,Show,Typeable,Data,Functor)++type StatementSpan = Statement SrcSpan++instance Span StatementSpan where+ getSpan = annot ++instance Annotated Statement where+ annot = stmt_annot ++-- | The argument for a @raise@ statement.+data RaiseExpr annot+ = RaiseV3 (Maybe (Expr annot, Maybe (Expr annot))) -- ^ Optional expression to evaluate, and optional \'from\' clause. /Version 3 only/.+ | RaiseV2 (Maybe (Expr annot, (Maybe (Expr annot, Maybe (Expr annot))))) -- ^ /Version 2 only/.+ deriving (Eq,Ord,Show,Typeable,Data,Functor)++type RaiseExprSpan = RaiseExpr SrcSpan++-- | Decorator.+data Decorator annot = + Decorator + { decorator_name :: DottedName annot -- ^ Decorator name.+ , decorator_args :: [Argument annot] -- ^ Decorator arguments.+ , decorator_annot :: annot + }+ deriving (Eq,Ord,Show,Typeable,Data,Functor)++type DecoratorSpan = Decorator SrcSpan++instance Span DecoratorSpan where+ getSpan = annot ++instance Annotated Decorator where+ annot = decorator_annot ++-- | Formal parameter of function definitions and lambda expressions.+-- +-- * Version 2.6: +--+-- * <http://docs.python.org/2.6/reference/compound_stmts.html#function-definitions>+--+-- * <http://docs.python.org/2.6/reference/expressions.html#calls>+--+-- * Version 3.1: +--+-- * <http://docs.python.org/3.1/reference/compound_stmts.html#function-definitions>+--+-- * <http://docs.python.org/3.1/reference/expressions.html#calls>+--+data Parameter annot+ -- | Ordinary named parameter.+ = Param + { param_name :: Ident annot -- ^ Parameter name.+ , param_py_annotation :: Maybe (Expr annot) -- ^ Optional annotation.+ , param_default :: Maybe (Expr annot) -- ^ Optional default value.+ , param_annot :: annot+ }+ -- | Excess positional parameter (single asterisk before its name in the concrete syntax). + | VarArgsPos + { param_name :: Ident annot -- ^ Parameter name.+ , param_py_annotation :: Maybe (Expr annot) -- ^ Optional annotation.+ , param_annot :: annot+ }+ -- | Excess keyword parameter (double asterisk before its name in the concrete syntax).+ | VarArgsKeyword + { param_name :: Ident annot -- ^ Parameter name.+ , param_py_annotation :: Maybe (Expr annot) -- ^ Optional annotation.+ , param_annot :: annot+ }+ -- | Marker for the end of positional parameters (not a parameter itself).+ | EndPositional { param_annot :: annot }+ -- | Tuple unpack. /Version 2 only/.+ | UnPackTuple + { param_unpack_tuple :: ParamTuple annot -- ^ The tuple to unpack.+ , param_default :: Maybe (Expr annot) -- ^ Optional default value.+ , param_annot :: annot+ }+ deriving (Eq,Ord,Show,Typeable,Data,Functor)++type ParameterSpan = Parameter SrcSpan++instance Span ParameterSpan where+ getSpan = annot ++instance Annotated Parameter where+ annot = param_annot ++-- | Tuple unpack parameter. /Version 2 only/.+data ParamTuple annot+ = ParamTupleName { param_tuple_name :: Ident annot, param_tuple_annot :: annot } -- ^ A variable name.+ | ParamTuple { param_tuple :: [ParamTuple annot], param_tuple_annot :: annot } -- ^ A (possibly nested) tuple parameter.+ deriving (Eq,Ord,Show,Typeable,Data,Functor)++type ParamTupleSpan = ParamTuple SrcSpan++instance Span ParamTupleSpan where+ getSpan = annot++instance Annotated ParamTuple where+ annot = param_tuple_annot++-- | Arguments to function calls, class declarations and decorators.+data Argument annot+ -- | Ordinary argument expression.+ = ArgExpr { arg_expr :: Expr annot, arg_annot :: annot }+ -- | Excess positional argument.+ | ArgVarArgsPos { arg_expr :: Expr annot, arg_annot :: annot }+ -- | Excess keyword argument.+ | ArgVarArgsKeyword { arg_expr :: Expr annot, arg_annot :: annot }+ -- | Keyword argument.+ | ArgKeyword + { arg_keyword :: Ident annot -- ^ Keyword name.+ , arg_expr :: Expr annot -- ^ Argument expression.+ , arg_annot :: annot+ }+ deriving (Eq,Ord,Show,Typeable,Data,Functor)++type ArgumentSpan = Argument SrcSpan++instance Span ArgumentSpan where+ getSpan = annot ++instance Annotated Argument where+ annot = arg_annot ++-- | Exception handler. +data Handler annot+ = Handler + { handler_clause :: ExceptClause annot+ , handler_suite :: Suite annot+ , handler_annot :: annot + }+ deriving (Eq,Ord,Show,Typeable,Data,Functor)++type HandlerSpan = Handler SrcSpan++instance Span HandlerSpan where+ getSpan = annot ++instance Annotated Handler where+ annot = handler_annot ++-- | Exception clause. +data ExceptClause annot+ = ExceptClause + -- NB: difference with version 3 (has NAME as target, but looks like bug in grammar)+ { except_clause :: Maybe (Expr annot, Maybe (Expr annot))+ , except_clause_annot :: annot + }+ deriving (Eq,Ord,Show,Typeable,Data,Functor)++type ExceptClauseSpan = ExceptClause SrcSpan++instance Span ExceptClauseSpan where+ getSpan = annot ++instance Annotated ExceptClause where+ annot = except_clause_annot ++-- | Comprehension. In version 3.x this can be used for lists, sets, dictionaries and generators. +-- data Comprehension e annot+data Comprehension annot+ = Comprehension + { comprehension_expr :: ComprehensionExpr annot+ , comprehension_for :: CompFor annot+ , comprehension_annot :: annot + }+ deriving (Eq,Ord,Show,Typeable,Data,Functor)++type ComprehensionSpan = Comprehension SrcSpan++instance Span ComprehensionSpan where+ getSpan = annot ++instance Annotated Comprehension where+ annot = comprehension_annot ++data ComprehensionExpr annot+ = ComprehensionExpr (Expr annot)+ | ComprehensionDict (DictKeyDatumList annot)+ deriving (Eq,Ord,Show,Typeable,Data,Functor)++type ComprehensionExprSpan = ComprehensionExpr SrcSpan++instance Span ComprehensionExprSpan where+ getSpan (ComprehensionExpr e) = getSpan e+ getSpan (ComprehensionDict d) = getSpan d++-- | Comprehension \'for\' component. +data CompFor annot = + CompFor + { comp_for_async :: Bool+ , comp_for_exprs :: [Expr annot]+ , comp_in_expr :: Expr annot+ , comp_for_iter :: Maybe (CompIter annot) + , comp_for_annot :: annot+ }+ deriving (Eq,Ord,Show,Typeable,Data,Functor)++type CompForSpan = CompFor SrcSpan++instance Span CompForSpan where+ getSpan = annot ++instance Annotated CompFor where+ annot = comp_for_annot ++-- | Comprehension guard. +data CompIf annot = + CompIf + { comp_if :: Expr annot+ , comp_if_iter :: Maybe (CompIter annot)+ , comp_if_annot :: annot + }+ deriving (Eq,Ord,Show,Typeable,Data,Functor)++type CompIfSpan = CompIf SrcSpan++instance Span CompIfSpan where+ getSpan = annot ++instance Annotated CompIf where+ annot = comp_if_annot ++-- | Comprehension iterator (either a \'for\' or an \'if\'). +data CompIter annot + = IterFor { comp_iter_for :: CompFor annot, comp_iter_annot :: annot }+ | IterIf { comp_iter_if :: CompIf annot, comp_iter_annot :: annot }+ deriving (Eq,Ord,Show,Typeable,Data,Functor)++type CompIterSpan = CompIter SrcSpan++instance Span CompIterSpan where+ getSpan = annot ++instance Annotated CompIter where+ annot = comp_iter_annot ++-- | Expressions.+-- +-- * Version 2.6 <http://docs.python.org/2.6/reference/expressions.html>.+-- +-- * Version 3.1 <http://docs.python.org/3.1/reference/expressions.html>.+-- +data Expr annot+ -- | Variable.+ = Var { var_ident :: Ident annot, expr_annot :: annot }+ -- | Literal integer.+ | Int { int_value :: Integer, expr_literal :: String, expr_annot :: annot }+ -- | Long literal integer. /Version 2 only/.+ | LongInt { int_value :: Integer, expr_literal :: String, expr_annot :: annot }+ -- | Literal floating point number.+ | Float { float_value :: Double, expr_literal :: String, expr_annot :: annot }+ -- | Literal imaginary number.+ | Imaginary { imaginary_value :: Double, expr_literal :: String, expr_annot :: annot } + -- | Literal boolean.+ | Bool { bool_value :: Bool, expr_annot :: annot }+ -- | Literal \'None\' value.+ | None { expr_annot :: annot } + -- | Ellipsis \'...\'.+ | Ellipsis { expr_annot :: annot }+ -- | Literal byte string.+ | ByteStrings { byte_string_strings :: [String], expr_annot :: annot }+ -- | Literal strings (to be concatentated together).+ | Strings { strings_strings :: [String], expr_annot :: annot }+ -- | Unicode literal strings (to be concatentated together). Version 2 only.+ | UnicodeStrings { unicodestrings_strings :: [String], expr_annot :: annot }+ -- | Function call. + | Call + { call_fun :: Expr annot -- ^ Expression yielding a callable object (such as a function).+ , call_args :: [Argument annot] -- ^ Call arguments.+ , expr_annot :: annot+ }+ -- | Subscription, for example \'x [y]\'. + | Subscript { subscriptee :: Expr annot, subscript_expr :: Expr annot, expr_annot :: annot }+ -- | Slicing, for example \'w [x:y:z]\'. + | SlicedExpr { slicee :: Expr annot, slices :: [Slice annot], expr_annot :: annot } + -- | Conditional expresison. + | CondExpr + { ce_true_branch :: Expr annot -- ^ Expression to evaluate if condition is True.+ , ce_condition :: Expr annot -- ^ Boolean condition.+ , ce_false_branch :: Expr annot -- ^ Expression to evaluate if condition is False.+ , expr_annot :: annot+ }+ -- | Binary operator application.+ | BinaryOp { operator :: Op annot, left_op_arg :: Expr annot, right_op_arg :: Expr annot, expr_annot :: annot }+ -- | Unary operator application.+ | UnaryOp { operator :: Op annot, op_arg :: Expr annot, expr_annot :: annot }+ -- Dot operator (attribute selection)+ | Dot { dot_expr :: Expr annot, dot_attribute :: Ident annot, expr_annot :: annot }+ -- | Anonymous function definition (lambda). + | Lambda { lambda_args :: [Parameter annot], lambda_body :: Expr annot, expr_annot :: annot }+ -- | Tuple. Can be empty. + | Tuple { tuple_exprs :: [Expr annot], expr_annot :: annot }+ -- | Generator yield. + | Yield + -- { yield_expr :: Maybe (Expr annot) -- ^ Optional expression to yield.+ { yield_arg :: Maybe (YieldArg annot) -- ^ Optional Yield argument.+ , expr_annot :: annot+ }+ -- | Generator. + | Generator { gen_comprehension :: Comprehension annot, expr_annot :: annot }+ -- | Await+ | Await { await_expr :: Expr annot, expr_annot :: annot }+ -- | List comprehension. + | ListComp { list_comprehension :: Comprehension annot, expr_annot :: annot }+ -- | List. + | List { list_exprs :: [Expr annot], expr_annot :: annot }+ -- | Dictionary. + | Dictionary { dict_mappings :: [DictKeyDatumList annot], expr_annot :: annot }+ -- | Dictionary comprehension. /Version 3 only/. + | DictComp { dict_comprehension :: Comprehension annot, expr_annot :: annot }+ -- | Set. + | Set { set_exprs :: [Expr annot], expr_annot :: annot } + -- | Set comprehension. /Version 3 only/. + | SetComp { set_comprehension :: Comprehension annot, expr_annot :: annot }+ -- | Starred expression. /Version 3 only/.+ | Starred { starred_expr :: Expr annot, expr_annot :: annot }+ -- | Parenthesised expression.+ | Paren { paren_expr :: Expr annot, expr_annot :: annot }+ -- | String conversion (backquoted expression). Version 2 only. + | StringConversion { backquoted_expr :: Expr annot, expr_anot :: annot }+ deriving (Eq,Ord,Show,Typeable,Data,Functor)++type ExprSpan = Expr SrcSpan++instance Span ExprSpan where+ getSpan = annot ++data YieldArg annot+ = YieldFrom (Expr annot) annot -- ^ Yield from a generator (Version 3 only)+ | YieldExpr (Expr annot) -- ^ Yield value of an expression+ deriving (Eq,Ord,Show,Typeable,Data,Functor)++type YieldArgSpan = YieldArg SrcSpan++instance Span YieldArgSpan where+ getSpan (YieldFrom _e span) = span+ getSpan (YieldExpr e) = getSpan e++instance Annotated Expr where+ annot = expr_annot ++data DictKeyDatumList annot =+ DictMappingPair (Expr annot) (Expr annot)+ | DictUnpacking (Expr annot)+ deriving (Eq,Ord,Show,Typeable,Data,Functor)++type DictKeyDatumListSpan = DictKeyDatumList SrcSpan++instance Span DictKeyDatumListSpan where+ getSpan (DictMappingPair e1 e2) = spanning e1 e2+ getSpan (DictUnpacking e) = getSpan e++-- | Slice compenent.+data Slice annot+ = SliceProper + { slice_lower :: Maybe (Expr annot)+ , slice_upper :: Maybe (Expr annot)+ , slice_stride :: Maybe (Maybe (Expr annot)) + , slice_annot :: annot+ } + | SliceExpr + { slice_expr :: Expr annot+ , slice_annot :: annot + }+ | SliceEllipsis { slice_annot :: annot }+ deriving (Eq,Ord,Show,Typeable,Data,Functor)++type SliceSpan = Slice SrcSpan++instance Span SliceSpan where+ getSpan = annot ++instance Annotated Slice where+ annot = slice_annot ++-- | Operators.+data Op annot+ = And { op_annot :: annot } -- ^ \'and\'+ | Or { op_annot :: annot } -- ^ \'or\'+ | Not { op_annot :: annot } -- ^ \'not\'+ | Exponent { op_annot :: annot } -- ^ \'**\'+ | LessThan { op_annot :: annot } -- ^ \'<\'+ | GreaterThan { op_annot :: annot } -- ^ \'>\'+ | Equality { op_annot :: annot } -- ^ \'==\'+ | GreaterThanEquals { op_annot :: annot } -- ^ \'>=\'+ | LessThanEquals { op_annot :: annot } -- ^ \'<=\'+ | NotEquals { op_annot :: annot } -- ^ \'!=\'+ | NotEqualsV2 { op_annot :: annot } -- ^ \'<>\'. Version 2 only.+ | In { op_annot :: annot } -- ^ \'in\'+ | Is { op_annot :: annot } -- ^ \'is\'+ | IsNot { op_annot :: annot } -- ^ \'is not\'+ | NotIn { op_annot :: annot } -- ^ \'not in\'+ | BinaryOr { op_annot :: annot } -- ^ \'|\'+ | Xor { op_annot :: annot } -- ^ \'^\'+ | BinaryAnd { op_annot :: annot } -- ^ \'&\'+ | ShiftLeft { op_annot :: annot } -- ^ \'<<\'+ | ShiftRight { op_annot :: annot } -- ^ \'>>\'+ | Multiply { op_annot :: annot } -- ^ \'*\'+ | Plus { op_annot :: annot } -- ^ \'+\'+ | Minus { op_annot :: annot } -- ^ \'-\'+ | Divide { op_annot :: annot } -- ^ \'\/\'+ | FloorDivide { op_annot :: annot } -- ^ \'\/\/\'+ | MatrixMult { op_annot :: annot } -- ^ \'@\'+ | Invert { op_annot :: annot } -- ^ \'~\' (bitwise inversion of its integer argument)+ | Modulo { op_annot :: annot } -- ^ \'%\'+ deriving (Eq,Ord,Show,Typeable,Data,Functor)++type OpSpan = Op SrcSpan++instance Span OpSpan where+ getSpan = annot ++instance Annotated Op where+ annot = op_annot ++-- | Augmented assignment operators.+data AssignOp annot+ = PlusAssign { assignOp_annot :: annot } -- ^ \'+=\'+ | MinusAssign { assignOp_annot :: annot } -- ^ \'-=\'+ | MultAssign { assignOp_annot :: annot } -- ^ \'*=\'+ | DivAssign { assignOp_annot :: annot } -- ^ \'\/=\'+ | ModAssign { assignOp_annot :: annot } -- ^ \'%=\'+ | PowAssign { assignOp_annot :: annot } -- ^ \'*=\'+ | BinAndAssign { assignOp_annot :: annot } -- ^ \'&=\'+ | BinOrAssign { assignOp_annot :: annot } -- ^ \'|=\'+ | BinXorAssign { assignOp_annot :: annot } -- ^ \'^=\' + | LeftShiftAssign { assignOp_annot :: annot } -- ^ \'<<=\'+ | RightShiftAssign { assignOp_annot :: annot } -- ^ \'>>=\'+ | FloorDivAssign { assignOp_annot :: annot } -- ^ \'\/\/=\'+ | MatrixMultAssign { assignOp_annot :: annot } -- ^ \'@=\'+ deriving (Eq,Ord,Show,Typeable,Data,Functor)++type AssignOpSpan = AssignOp SrcSpan++instance Span AssignOpSpan where+ getSpan = annot ++instance Annotated AssignOp where+ annot = assignOp_annot
src/Language/Python/Common/LexerUtils.hs view
@@ -14,16 +14,15 @@ module Language.Python.Common.LexerUtils where import Control.Monad (liftM)-import Control.Monad.Error.Class (throwError) import Data.List (foldl')-import Data.Map as Map hiding (null, map, foldl') import Data.Word (Word8)-import Data.Char (ord)-import Numeric (readHex, readOct)-import Language.Python.Common.Token as Token +import Language.Python.Common.Token as Token import Language.Python.Common.ParserMonad hiding (location) import Language.Python.Common.SrcLocation +import Codec.Binary.UTF8.String as UTF8 (encode) +type Byte = Word8+ -- Beginning of. BOF = beginning of file, BOL = beginning of line data BO = BOF | BOL @@ -106,15 +105,20 @@ -- Test if we are at the end of the line or file atEOLorEOF :: a -> AlexInput -> Int -> AlexInput -> Bool-atEOLorEOF _user _inputBeforeToken _tokenLength (_loc, inputAfterToken) +atEOLorEOF _user _inputBeforeToken _tokenLength (_loc, _bs, inputAfterToken) = null inputAfterToken || nextChar == '\n' || nextChar == '\r' where nextChar = head inputAfterToken notEOF :: a -> AlexInput -> Int -> AlexInput -> Bool-notEOF _user _inputBeforeToken _tokenLength (_loc, inputAfterToken) +notEOF _user _inputBeforeToken _tokenLength (_loc, _bs, inputAfterToken) = not (null inputAfterToken) +delUnderscores :: String -> String+delUnderscores [] = []+delUnderscores ('_':xs) = delUnderscores xs+delUnderscores (x :xs) = x : delUnderscores xs+ readBinary :: String -> Integer readBinary = toBinary . drop 2 @@ -122,6 +126,7 @@ toBinary = foldl' acc 0 acc b '0' = 2 * b acc b '1' = 2 * b + 1+ acc _ _ = error "Lexer ensures all digits passed to readBinary are 0 or 1." readFloat :: String -> Double readFloat str@('.':cs) = read ('0':readFloatRest str)@@ -144,6 +149,12 @@ byteStringToken :: SrcSpan -> String -> Token byteStringToken = ByteStringToken +formatStringToken :: SrcSpan -> String -> Token+formatStringToken = StringToken++formatRawStringToken :: SrcSpan -> String -> Token+formatRawStringToken = StringToken+ unicodeStringToken :: SrcSpan -> String -> Token unicodeStringToken = UnicodeStringToken @@ -179,25 +190,36 @@ -- ----------------------------------------------------------------------------- -- Functionality required by Alex -type AlexInput = (SrcLocation, String)+type AlexInput = (SrcLocation, -- current src location+ [Byte], -- byte buffer for next character+ String) -- input string alexInputPrevChar :: AlexInput -> Char alexInputPrevChar _ = error "alexInputPrevChar not used" +-- byte buffer should be empty here alexGetChar :: AlexInput -> Maybe (Char, AlexInput)-alexGetChar (loc, input) +alexGetChar (loc, [], input) | null input = Nothing- | otherwise = Just (nextChar, (nextLoc, rest))+ | otherwise = seq nextLoc (Just (nextChar, (nextLoc, [], rest))) where nextChar = head input rest = tail input nextLoc = moveChar nextChar loc+alexGetChar (loc, _:_, _) = error "alexGetChar called with non-empty byte buffer" -mapFst :: (a -> b) -> (a, c) -> (b, c)-mapFst f (a, c) = (f a, c)+-- mapFst :: (a -> b) -> (a, c) -> (b, c)+-- mapFst f (a, c) = (f a, c) -alexGetByte :: AlexInput -> Maybe (Word8, AlexInput)-alexGetByte = fmap (mapFst (fromIntegral . ord)) . alexGetChar+alexGetByte :: AlexInput -> Maybe (Byte, AlexInput)+-- alexGetByte = fmap (mapFst (fromIntegral . ord)) . alexGetChar+alexGetByte (loc, b:bs, input) = Just (b, (loc, bs, input))+alexGetByte (loc, [], []) = Nothing+alexGetByte (loc, [], nextChar:rest) =+ seq nextLoc (Just (byte, (nextLoc, restBytes, rest)))+ where+ nextLoc = moveChar nextChar loc+ byte:restBytes = UTF8.encode [nextChar] moveChar :: Char -> SrcLocation -> SrcLocation moveChar '\n' = incLine 1 @@ -213,3 +235,4 @@ readOctNoO :: String -> Integer readOctNoO (zero:rest) = read (zero:'O':rest)+readOctNoO [] = error "Lexer ensures readOctNoO is never called on an empty string"
src/Language/Python/Common/ParseError.hs view
@@ -12,17 +12,11 @@ module Language.Python.Common.ParseError ( ParseError (..) ) where -import Language.Python.Common.Pretty import Language.Python.Common.SrcLocation (SrcLocation) import Language.Python.Common.Token (Token)-import Control.Monad.Error.Class data ParseError = UnexpectedToken Token -- ^ An error from the parser. Token found where it should not be. Note: tokens contain their own source span. | UnexpectedChar Char SrcLocation -- ^ An error from the lexer. Character found where it should not be. | StrError String -- ^ A generic error containing a string message. No source location. deriving (Eq, Ord, Show)--instance Error ParseError where- noMsg = StrError ""- strMsg = StrError
src/Language/Python/Common/ParserMonad.hs view
@@ -43,6 +43,7 @@ , addComment , getComments , spanError+ , throwError ) where import Language.Python.Common.SrcLocation (SrcLocation (..), SrcSpan (..), Span (..))@@ -51,10 +52,6 @@ import Control.Applicative ((<$>)) import Control.Monad.State.Class import Control.Monad.State.Strict as State-import Control.Monad.Error as Error-import Control.Monad.Error.Class-import Control.Monad.Identity as Identity-import Control.Monad.Trans as Trans import Language.Python.Common.Pretty internalError :: String -> P a @@ -93,6 +90,9 @@ } type P a = StateT ParseState (Either ParseError) a++throwError :: ParseError -> P a+throwError = lift . Left execParser :: P a -> ParseState -> Either ParseError a execParser = evalStateT
src/Language/Python/Common/ParserUtils.hs view
@@ -15,7 +15,6 @@ import Data.List (foldl') import Data.Maybe (isJust)-import Control.Monad.Error.Class (throwError) import Language.Python.Common.AST as AST import Language.Python.Common.Token as Token import Language.Python.Common.ParserMonad hiding (location)@@ -58,6 +57,7 @@ isProperSlice :: Subscript -> Bool isProperSlice (SubscriptSlice {}) = True+isProperSlice (SubscriptSliceEllipsis {}) = True isProperSlice other = False subscriptToSlice :: Subscript -> SliceSpan@@ -78,12 +78,6 @@ | length subs == 1 = subscriptToExpr $ head subs | otherwise = error "subscriptsToExpr: empty subscript list" -{-- = TrailerCall { trailer_call_args :: [ArgumentSpan], trailer_span :: SrcSpan }- | TrailerSubscript { trailer_subs :: [Subscript], trailer_span :: SrcSpan }- | TrailerDot { trailer_dot_ident :: IdentSpan, dot_span :: SrcSpan, trailer_span :: SrcSpan }--}- addTrailer :: ExprSpan -> [Trailer] -> ExprSpan addTrailer = foldl' trail@@ -97,31 +91,35 @@ | otherwise = Subscript e (subscriptsToExpr subs) (spanning e trail) trail e trail@(TrailerDot { trailer_dot_ident = ident, dot_span = ds })- = BinaryOp (AST.Dot ds) e (Var ident (getSpan ident)) (spanning e trail)+ = Dot { dot_expr = e, dot_attribute = ident, expr_annot = spanning e trail } makeTupleOrExpr :: [ExprSpan] -> Maybe Token -> ExprSpan makeTupleOrExpr [e] Nothing = e makeTupleOrExpr es@(_:_) (Just t) = Tuple es (spanning es t) makeTupleOrExpr es@(_:_) Nothing = Tuple es (getSpan es)+makeTupleOrExpr [] _ = error "makeTupleOrExpr should never be called with an empty list" makeAssignmentOrExpr :: ExprSpan -> Either [ExprSpan] (AssignOpSpan, ExprSpan) -> StatementSpan makeAssignmentOrExpr e (Left es) = makeNormalAssignment e es- where- makeNormalAssignment :: ExprSpan -> [ExprSpan] -> StatementSpan- makeNormalAssignment e [] = StmtExpr e (getSpan e)- makeNormalAssignment e es - = AST.Assign (e : front) (head back) (spanning e es)- where- (front, back) = splitAt (len - 1) es- len = length es -makeAssignmentOrExpr e1 (Right (op, e2)) - = makeAugAssignment e1 op e2- where- makeAugAssignment :: ExprSpan -> AssignOpSpan -> ExprSpan -> StatementSpan- makeAugAssignment e1 op e2- = AST.AugmentedAssign e1 op e2 (spanning e1 e2)+makeAssignmentOrExpr e (Right ope2)+ = makeAugAssignment e ope2 +makeAugAssignment :: ExprSpan -> (AssignOpSpan, ExprSpan) -> StatementSpan+makeAugAssignment e1 (op, e2)+ = AST.AugmentedAssign e1 op e2 (spanning e1 e2)++makeNormalAssignment :: ExprSpan -> [ExprSpan] -> StatementSpan+makeNormalAssignment e [] = StmtExpr e (getSpan e)+makeNormalAssignment e es+ = AST.Assign (e : front) (head back) (spanning e es)+ where+ (front, back) = splitAt (len - 1) es+ len = length es++makeAnnAssignment :: ExprSpan -> (ExprSpan, Maybe ExprSpan) -> StatementSpan+makeAnnAssignment ato (annotation, ae) = AST.AnnotatedAssign annotation ato ae (spanning ae ato)+ makeTry :: Token -> SuiteSpan -> ([HandlerSpan], [StatementSpan], [StatementSpan]) -> StatementSpan makeTry t1 body (handlers, elses, finally) = AST.Try body handlers elses finally @@ -151,22 +149,34 @@ makeTupleParam p@(ParamTuple { param_tuple_annot = span }) optDefault = UnPackTuple p optDefault span -makeComprehension :: ExprSpan -> CompForSpan -> ComprehensionSpan ExprSpan-makeComprehension e for = Comprehension e for (spanning e for)+makeComprehension :: ExprSpan -> CompForSpan -> ComprehensionSpan+makeComprehension e for = Comprehension (ComprehensionExpr e) for (spanning e for) -makeListForm :: SrcSpan -> Either ExprSpan (ComprehensionSpan ExprSpan) -> ExprSpan+makeListForm :: SrcSpan -> Either ExprSpan ComprehensionSpan -> ExprSpan makeListForm span (Left tuple@(Tuple {})) = List (tuple_exprs tuple) span makeListForm span (Left other) = List [other] span makeListForm span (Right comprehension) = ListComp comprehension span makeSet :: ExprSpan -> Either CompForSpan [ExprSpan] -> SrcSpan -> ExprSpan-makeSet e (Left compFor) = SetComp (Comprehension e compFor (spanning e compFor))+makeSet e (Left compFor) = SetComp (Comprehension (ComprehensionExpr e) compFor (spanning e compFor)) makeSet e (Right es) = Set (e:es) -makeDictionary :: (ExprSpan, ExprSpan) -> Either CompForSpan [(ExprSpan,ExprSpan)] -> SrcSpan -> ExprSpan-makeDictionary e (Left compFor) = DictComp (Comprehension e compFor (spanning e compFor))-makeDictionary e (Right es) = Dictionary (e:es)+-- The Either (ExprSpan, ExprSpan) ExprSpan refers to a (key, value) pair or a dictionary unpacking expression.+makeDictionary :: Either (ExprSpan, ExprSpan) ExprSpan -> Either CompForSpan [Either (ExprSpan, ExprSpan) ExprSpan] -> SrcSpan -> ExprSpan+makeDictionary (Left mapping@(key, val)) (Left compFor) =+ DictComp (Comprehension (ComprehensionDict (DictMappingPair key val)) compFor (spanning mapping compFor))+-- This is allowed by the grammar, but will produce a runtime syntax error:+-- dict unpacking cannot be used in dict comprehension+makeDictionary (Right unpacking) (Left compFor) =+ DictComp (Comprehension (ComprehensionDict (DictUnpacking unpacking)) compFor (spanning unpacking compFor))+makeDictionary item (Right es) = Dictionary $ toKeyDatumList <$> item : es ++toKeyDatumList :: Either (ExprSpan, ExprSpan) ExprSpan -> DictKeyDatumList SrcSpan+toKeyDatumList (Left (key, value)) = DictMappingPair key value+toKeyDatumList (Right unpacking) = DictUnpacking unpacking++ fromEither :: Either a a -> a fromEither (Left x) = x fromEither (Right x) = x@@ -178,6 +188,7 @@ -- parser guarantees that the first list is non-empty makeDecorated :: [DecoratorSpan] -> StatementSpan -> StatementSpan makeDecorated ds@(d:_) def = Decorated ds def (spanning d def)+makeDecorated [] _ = error "parser guarantees that makeDecorated's first argument is non-empty" -- suite can't be empty so it is safe to take span over it makeFun :: Token -> IdentSpan -> [ParameterSpan] -> Maybe ExprSpan -> SuiteSpan -> StatementSpan@@ -188,7 +199,7 @@ makeReturn t1 Nothing = AST.Return Nothing (getSpan t1) makeReturn t1 expr@(Just e) = AST.Return expr (spanning t1 e) -makeParenOrGenerator :: Either ExprSpan (ComprehensionSpan ExprSpan) -> SrcSpan -> ExprSpan+makeParenOrGenerator :: Either ExprSpan ComprehensionSpan -> SrcSpan -> ExprSpan makeParenOrGenerator (Left e) span = Paren e span makeParenOrGenerator (Right comp) span = Generator comp span @@ -197,14 +208,6 @@ makePrint chevron (Just (args, last_comma)) span = AST.Print chevron args (isJust last_comma) span -{--makeRelative :: Int -> ImportRelativeSpan -> SrcSpan -> ImportRelativeSpan-makeRelative dots importRelative span- = importRelative { import_relative_dots = dots + oldDots, import_relative_annot = span }- where- oldDots = import_relative_dots importRelative--}- makeRelative :: [Either Token DottedNameSpan] -> ImportRelativeSpan makeRelative items = ImportRelative ndots maybeName (getSpan items) @@ -218,9 +221,10 @@ countDots count (Left token:rest) = countDots (count + dots token) rest dots (DotToken {}) = 1 dots (EllipsisToken {}) = 3+ dots _ = error "Parser ensures dots is only called on DotToken or EllipsisToken." {-- See: http://www.python.org/doc/3.0/reference/expressions.html#calls+ See: http://docs.python.org/3.0/reference/expressions.html#calls arglist: (argument ',')* (argument [','] |'*' test (',' argument)* [',' '**' test]@@ -252,9 +256,11 @@ ArgKeyword {} | state `elem` [1,2] -> check 2 rest | state `elem` [3,4] -> check 4 rest+ | otherwise -> error "state should always be in range 1..4 here" ArgVarArgsPos {} | state `elem` [1,2] -> check 3 rest | state `elem` [3,4] -> spanError arg "there must not be two *arguments in an argument list"+ | otherwise -> error "state should always be in range 1..4 here" ArgVarArgsKeyword {} -> check 5 rest {-@@ -288,9 +294,11 @@ UnPackTuple {} | state `elem` [1,3] -> check state rest | state == 2 -> check 3 rest + | otherwise -> error "state should always be in range 1..3 here" Param {} | state `elem` [1,3] -> check state rest | state == 2 -> check 3 rest + | otherwise -> error "state should always be in range 1..3 here" EndPositional {} | state == 1 -> check 2 rest | otherwise -> spanError param "there must not be two *parameters in a parameter list"
src/Language/Python/Common/Pretty.hs view
@@ -1,4 +1,4 @@-{-# LANGUAGE TypeSynonymInstances, FlexibleInstances #-}+{-# LANGUAGE TypeSynonymInstances, FlexibleInstances, CPP #-} ----------------------------------------------------------------------------- -- | -- Module : Language.Python.Common.Pretty@@ -13,6 +13,10 @@ module Language.Python.Common.Pretty (module TextPP, module Language.Python.Common.Pretty) where +#if __GLASGOW_HASKELL__ >= 803+import Prelude hiding ((<>))+#endif+ import Text.PrettyPrint as TextPP --------------------------------------------------------------------------------@@ -47,6 +51,10 @@ -- | A list of things separated by commas. commaList :: Pretty a => [a] -> Doc commaList = hsep . punctuate comma . map pretty ++-- | A list of things separated by equals signs.+equalsList :: Pretty a => [a] -> Doc+equalsList = hsep . punctuate (space <> equals) . map pretty instance Pretty Int where pretty = int
src/Language/Python/Common/PrettyAST.hs view
@@ -1,3 +1,4 @@+{-# LANGUAGE CPP #-} ----------------------------------------------------------------------------- -- | -- Module : Language.Python.Version2.Syntax.PrettyAST@@ -12,8 +13,13 @@ module Language.Python.Common.PrettyAST () where +#if __GLASGOW_HASKELL__ >= 803+import Prelude hiding ((<>))+#endif+ import Language.Python.Common.Pretty-import Language.Python.Common.AST +import Language.Python.Common.AST+import Data.Maybe (fromMaybe) -------------------------------------------------------------------------------- @@ -96,10 +102,12 @@ pretty stmt@(For {}) = text "for" <+> commaList (for_targets stmt) <+> text "in" <+> pretty (for_generator stmt) <> colon $+$ indent (prettySuite (for_body stmt)) $+$ optionalKeywordSuite "else" (for_else stmt)+ pretty (AsyncFor { for_stmt = fs }) = zeroWidthText "async " <> pretty fs pretty stmt@(Fun {}) = text "def" <+> pretty (fun_name stmt) <> parens (commaList (fun_args stmt)) <+> perhaps (fun_result_annotation stmt) (text "->") <+> pretty (fun_result_annotation stmt) <> colon $+$ indent (prettySuite (fun_body stmt)) + pretty (AsyncFun { fun_def = fd }) = zeroWidthText "async " <+> pretty fd pretty stmt@(Class {}) = text "class" <+> pretty (class_name stmt) <> prettyOptionalList (class_args stmt) <> colon $+$ indent (prettySuite (class_body stmt)) @@ -109,11 +117,14 @@ text "if" <+> pretty cond <> colon $+$ indent (prettySuite body) $+$ prettyGuards xs $+$ optionalKeywordSuite "else" optionalElse+ [] -> error "Attempt to pretty print conditional statement with empty guards" -- XXX is the assign_to always a singleton? pretty (Assign { assign_to = pattern, assign_expr = e })- = commaList pattern <+> equals <+> pretty e+ = equalsList pattern <+> equals <+> pretty e pretty (AugmentedAssign { aug_assign_to = to_expr, aug_assign_op = op, aug_assign_expr = e})- = pretty to_expr <+> pretty op <+> pretty e + = pretty to_expr <+> pretty op <+> pretty e+ pretty (AnnotatedAssign { ann_assign_annotation = ann_annotate, ann_assign_to = ann_to, ann_assign_expr = ann_expr})+ = pretty ann_to <+> text ":" <+> pretty ann_annotate <+> fromMaybe empty (((text "=" <+>) . pretty) <$> ann_expr) pretty (Decorated { decorated_decorators = decs, decorated_def = stmt}) = vcat (map pretty decs) $+$ pretty stmt pretty (Return { return_expr = e }) = text "return" <+> pretty e@@ -126,6 +137,7 @@ pretty (With { with_context = context, with_body = body }) = text "with" <+> hcat (punctuate comma (map prettyWithContext context)) <+> colon $+$ indent (prettySuite body)+ pretty (AsyncWith { with_stmt = ws }) = zeroWidthText "async " <+> pretty ws pretty Pass {} = text "pass" pretty Break {} = text "break" pretty Continue {} = text "continue"@@ -195,13 +207,18 @@ pretty (ArgKeyword { arg_keyword = ident, arg_expr = e }) = pretty ident <> equals <> pretty e -instance Pretty t => Pretty (Comprehension t a) where+instance Pretty (Comprehension a) where pretty (Comprehension { comprehension_expr = e, comprehension_for = for }) = pretty e <+> pretty for +instance Pretty (ComprehensionExpr a) where+ pretty (ComprehensionExpr e) = pretty e+ pretty (ComprehensionDict d) = pretty d+ instance Pretty (CompFor a) where- pretty (CompFor { comp_for_exprs = es, comp_in_expr = e, comp_for_iter = iter }) - = text "for" <+> commaList es <+> text "in" <+> pretty e <+> pretty iter+ pretty (CompFor { comp_for_async = ca, comp_for_exprs = es, comp_in_expr = e, comp_for_iter = iter })+ = (text $ if ca then "async for" else "for") <+> commaList es+ <+> text "in" <+> pretty e <+> pretty iter instance Pretty (CompIf a) where pretty (CompIf { comp_if = e, comp_if_iter = iter }) @@ -231,11 +248,10 @@ pretty (CondExpr { ce_true_branch = trueBranch, ce_condition = cond, ce_false_branch = falseBranch }) = pretty trueBranch <+> text "if" <+> pretty cond <+> text "else" <+> pretty falseBranch pretty (BinaryOp { operator = op, left_op_arg = left, right_op_arg = right })- = pretty left <> (if isDot op then dot else space <> pretty op <> space) <> pretty right- where- isDot (Dot {}) = True- isDot _other = False+ = pretty left <+> pretty op <+> pretty right pretty (UnaryOp { operator = op, op_arg = e }) = pretty op <+> pretty e+ pretty (Dot { dot_expr = e, dot_attribute = a }) =+ pretty e <> dot <> pretty a pretty (Lambda { lambda_args = args, lambda_body = body }) = text "lambda" <+> commaList args <> colon <+> pretty body pretty (Tuple { tuple_exprs = es }) =@@ -243,20 +259,34 @@ [] -> text "()" [e] -> pretty e <> comma _ -> commaList es- pretty (Yield { yield_expr = e })- = text "yield" <+> pretty e+ pretty (Yield { yield_arg = arg })+ = text "yield" <+> pretty arg + pretty (Generator { gen_comprehension = gc }) = parens $ pretty gc+ pretty (Await { await_expr = ae }) = text "await" <+> pretty ae+ pretty (ListComp { list_comprehension = lc }) = brackets $ pretty lc pretty (List { list_exprs = es }) = brackets (commaList es) pretty (Dictionary { dict_mappings = mappings })- = braces (hsep (punctuate comma $ map (\ (e1,e2) -> pretty e1 <> colon <> pretty e2) mappings))+ = braces (hsep (punctuate comma $ map pretty mappings))+ pretty (DictComp { dict_comprehension = dc }) = braces $ pretty dc pretty (Set { set_exprs = es }) = braces $ commaList es- pretty (ListComp { list_comprehension = lc }) = brackets $ pretty lc- pretty (Generator { gen_comprehension = gc }) = parens $ pretty gc+ pretty (SetComp { set_comprehension = sc }) = braces $ pretty sc pretty (Paren { paren_expr = e }) = parens $ pretty e+ pretty (StringConversion { backquoted_expr = e }) = char '`' <> pretty e <> char '`'+ pretty (Starred { starred_expr = e }) = char '*' <> pretty e +instance Pretty (YieldArg a) where+ pretty (YieldFrom e _annot) = text "from" <+> pretty e+ pretty (YieldExpr e) = pretty e++instance Pretty (DictKeyDatumList a) where+ pretty (DictMappingPair key val) = pretty key <> colon <+> pretty val+ pretty (DictUnpacking expr) = text "**" <> pretty expr+ instance Pretty (Slice a) where pretty (SliceProper { slice_lower = lower, slice_upper = upper, slice_stride = stride }) = pretty lower <> colon <> pretty upper <> (maybe empty (\s -> colon <> pretty s) stride) pretty (SliceExpr { slice_expr = e }) = pretty e+ pretty (SliceEllipsis {}) = text "..." instance Pretty (Op a) where pretty (And {}) = text "and"@@ -284,9 +314,9 @@ pretty (Minus {}) = text "-" pretty (Divide {}) = text "/" pretty (FloorDivide {}) = text "//"+ pretty (MatrixMult {}) = text "@" pretty (Invert {}) = text "~" pretty (Modulo {}) = text "%"- pretty (Dot {}) = dot instance Pretty (AssignOp a) where pretty (PlusAssign {}) = text "+="@@ -301,3 +331,4 @@ pretty (LeftShiftAssign {}) = text "<<=" pretty (RightShiftAssign {}) = text ">>=" pretty (FloorDivAssign {}) = text "//="+ pretty (MatrixMultAssign {}) = text "@="
src/Language/Python/Common/PrettyParseError.hs view
@@ -15,7 +15,7 @@ import Language.Python.Common.Pretty import Language.Python.Common.ParseError (ParseError (..)) import Language.Python.Common.SrcLocation -import Language.Python.Common.PrettyToken+import Language.Python.Common.PrettyToken() instance Pretty ParseError where pretty (UnexpectedToken t) = pretty (getSpan t) <+> text "unexpected token:" <+> pretty t
src/Language/Python/Common/PrettyToken.hs view
@@ -24,6 +24,7 @@ IndentToken {} -> text "indentation" DedentToken {} -> text "dedentation" NewlineToken {} -> text "end of line" + LineJoinToken {} -> text "line join" CommentToken { token_literal = str } -> text "comment:" <+> prettyPrefix 10 str IdentifierToken { token_literal = str } ->@@ -32,6 +33,8 @@ text "string:" <+> prettyPrefix 10 str ByteStringToken { token_literal = str } -> text "byte string:" <+> prettyPrefix 10 str+ UnicodeStringToken { token_literal = str } ->+ text "unicode string:" <+> prettyPrefix 10 str IntegerToken { token_literal = str } -> text "integer:" <+> text str LongIntegerToken { token_literal = str } ->@@ -62,6 +65,8 @@ AsToken {} -> text "as" ElifToken {} -> text "elif" YieldToken {} -> text "yield"+ AsyncToken {} -> text "async"+ AwaitToken {} -> text "await" AssertToken {} -> text "assert" ImportToken {} -> text "import" PassToken {} -> text "pass"@@ -101,6 +106,7 @@ LeftShiftAssignToken {} -> text "<<=" RightShiftAssignToken {} -> text ">>=" FloorDivAssignToken {} -> text "//="+ MatrixMultAssignToken {} -> text "@=" BackQuoteToken {} -> text "` (back quote)" PlusToken {} -> text "+" MinusToken {} -> text "-"@@ -123,4 +129,3 @@ NotEqualsToken {} -> text "!=" NotEqualsV2Token {} -> text "<>" EOFToken {} -> text "end of input"- LineJoinToken {} -> text "line join"
src/Language/Python/Common/SrcLocation.hs view
@@ -34,6 +34,10 @@ startRow ) where +#if __GLASGOW_HASKELL__ >= 803+import Prelude hiding ((<>))+#endif+ import Language.Python.Common.Pretty import Data.Data @@ -100,6 +104,7 @@ incColumn :: Int -> SrcLocation -> SrcLocation incColumn n loc@(Sloc { sloc_column = col }) = loc { sloc_column = col + n }+incColumn _ NoLocation = NoLocation -- | Increment the column of a location by one tab stop. incTab :: SrcLocation -> SrcLocation@@ -107,11 +112,13 @@ = loc { sloc_column = newCol } where newCol = col + 8 - (col - 1) `mod` 8+incTab NoLocation = NoLocation -- | Increment the line number (row) of a location by one. incLine :: Int -> SrcLocation -> SrcLocation incLine n loc@(Sloc { sloc_row = row }) = loc { sloc_column = 1, sloc_row = row + n }+incLine _ NoLocation = NoLocation {- Inspired heavily by compiler/basicTypes/SrcLoc.lhs
src/Language/Python/Common/StringEscape.hs view
@@ -13,9 +13,9 @@ -- -- See: -- --- * Version 2.6 <http://www.python.org/doc/2.6/reference/lexical_analysis.html#string-literals>+-- * Version 2.6 <http://docs.python.org/2.6/reference/lexical_analysis.html#string-literals> -- --- * Version 3.1 <http://www.python.org/doc/3.1/reference/lexical_analysis.html#string-and-bytes-literals> +-- * Version 3.1 <http://docs.python.org/3.1/reference/lexical_analysis.html#string-and-bytes-literals> ----------------------------------------------------------------------------- module Language.Python.Common.StringEscape (
src/Language/Python/Common/Token.hs view
@@ -25,7 +25,7 @@ ) where import Language.Python.Common.Pretty-import Language.Python.Common.SrcLocation (SrcSpan (..), SrcLocation (..), Span(getSpan))+import Language.Python.Common.SrcLocation (SrcSpan (..), Span(getSpan)) import Data.Data -- | Lexical tokens.@@ -86,6 +86,8 @@ | OrToken { token_span :: !SrcSpan } -- ^ Keyword: boolean disjunction \'or\'. -- Version 3.x only: | NonLocalToken { token_span :: !SrcSpan } -- ^ Keyword: \'nonlocal\' (Python 3.x only)+ | AsyncToken { token_span :: !SrcSpan } -- ^ Keyword: \'async\' (Python 3.x only)+ | AwaitToken { token_span :: !SrcSpan } -- ^ Keyword: \'await\' (Python 3.x only) -- Version 2.x only: | PrintToken { token_span :: !SrcSpan } -- ^ Keyword: \'print\'. (Python 2.x only) | ExecToken { token_span :: !SrcSpan } -- ^ Keyword: \'exec\'. (Python 2.x only)@@ -117,6 +119,7 @@ | LeftShiftAssignToken { token_span :: !SrcSpan } -- ^ Delimiter: binary-left-shift assignment \'<<=\'. | RightShiftAssignToken { token_span :: !SrcSpan } -- ^ Delimiter: binary-right-shift assignment \'>>=\'. | FloorDivAssignToken { token_span :: !SrcSpan } -- ^ Delimiter: floor-divide assignment \'//=\'.+ | MatrixMultAssignToken { token_span :: !SrcSpan } -- ^ Delimiter: matrix multiplication assignment \'@=\'. | BackQuoteToken { token_span :: !SrcSpan } -- ^ Delimiter: back quote character \'`\'. -- Operators@@ -158,15 +161,16 @@ hasLiteral :: Token -> Bool hasLiteral token = case token of- CommentToken {} -> True - IdentifierToken {} -> True - StringToken {} -> True - ByteStringToken {} -> True - IntegerToken {} -> True - LongIntegerToken {} -> True - FloatToken {} -> True - ImaginaryToken {} -> True - other -> False+ CommentToken {} -> True + IdentifierToken {} -> True + StringToken {} -> True + ByteStringToken {} -> True+ UnicodeStringToken {} -> True+ IntegerToken {} -> True + LongIntegerToken {} -> True + FloatToken {} -> True + ImaginaryToken {} -> True + other -> False -- | Classification of tokens data TokenClass@@ -191,7 +195,8 @@ CommentToken {} -> Comment IdentifierToken {} -> Identifier StringToken {} -> String - ByteStringToken {} -> String + ByteStringToken {} -> String+ UnicodeStringToken {} -> String IntegerToken {} -> Number LongIntegerToken {} -> Number FloatToken {} -> Number @@ -218,6 +223,8 @@ AsToken {} -> Keyword ElifToken {} -> Keyword YieldToken {} -> Keyword + AsyncToken {} -> Keyword+ AwaitToken {} -> Keyword AssertToken {} -> Keyword ImportToken {} -> Keyword PassToken {} -> Keyword @@ -257,6 +264,7 @@ LeftShiftAssignToken {} -> Assignment RightShiftAssignToken {} -> Assignment FloorDivAssignToken {} -> Assignment + MatrixMultAssignToken {} -> Assignment BackQuoteToken {} -> Punctuation PlusToken {} -> Operator MinusToken {} -> Operator @@ -292,7 +300,8 @@ CommentToken {} -> token_literal token IdentifierToken {} -> token_literal token StringToken {} -> token_literal token- ByteStringToken {} -> token_literal token + ByteStringToken {} -> token_literal token+ UnicodeStringToken {} -> token_literal token IntegerToken {} -> token_literal token LongIntegerToken {} -> token_literal token FloatToken {} -> token_literal token@@ -319,6 +328,8 @@ AsToken {} -> "as" ElifToken {} -> "elif" YieldToken {} -> "yield"+ AsyncToken {} -> "async"+ AwaitToken {} -> "await" AssertToken {} -> "assert" ImportToken {} -> "import" PassToken {} -> "pass" @@ -358,6 +369,7 @@ LeftShiftAssignToken {} -> "<<=" RightShiftAssignToken {} -> ">>=" FloorDivAssignToken {} -> "//=" + MatrixMultAssignToken {} -> "@=" BackQuoteToken {} -> "`" PlusToken {} -> "+" MinusToken {} -> "-"
src/Language/Python/Version2.hs view
@@ -11,11 +11,11 @@ -- -- See: ----- * <http://www.python.org/doc/2.6/reference/index.html> for an overview of the language. +-- * <http://docs.python.org/2.6/reference/index.html> for an overview of the language. ----- * <http://www.python.org/doc/2.6/reference/grammar.html> for the full grammar.+-- * <http://docs.python.org/2.6/reference/grammar.html> for the full grammar. -- --- * <http://www.python.org/doc/2.6/reference/toplevel_components.html> for a description of +-- * <http://docs.python.org/2.6/reference/toplevel_components.html> for a description of -- the various Python top-levels, which correspond to the parsers provided here. -----------------------------------------------------------------------------
src/Language/Python/Version2/Lexer.hs view
@@ -9,7 +9,7 @@ -- Portability : ghc -- -- Lexical analysis for Python version 2.x programs. --- See: <http://www.python.org/doc/2.6/reference/lexical_analysis.html>.+-- See: <http://docs.python.org/2.6/reference/lexical_analysis.html>. ----------------------------------------------------------------------------- module Language.Python.Version2.Lexer (
src/Language/Python/Version2/Parser.hs view
@@ -14,11 +14,11 @@ -- -- See: ----- * <http://www.python.org/doc/2.6/reference/index.html> for an overview of the language. +-- * <http://docs.python.org/2.6/reference/index.html> for an overview of the language. ----- * <http://www.python.org/doc/2.6/reference/grammar.html> for the full grammar.+-- * <http://docs.python.org/2.6/reference/grammar.html> for the full grammar. -- --- * <http://www.python.org/doc/2.6/reference/toplevel_components.html> for a description of +-- * <http://docs.python.org/2.6/reference/toplevel_components.html> for a description of -- the various Python top-levels, which correspond to the parsers provided here. ----------------------------------------------------------------------------- @@ -35,7 +35,7 @@ import Language.Python.Common.AST (ModuleSpan, StatementSpan, ExprSpan) import Language.Python.Common.Token (Token) import Language.Python.Common.SrcLocation (initialSrcLocation)-import Language.Python.Common.ParserMonad (execParser, execParserKeepComments, ParseError, initialState)+import Language.Python.Common.ParserMonad (execParserKeepComments, ParseError, initialState) -- | Parse a whole Python source file. Return comments in addition to the parsed module. parseModule :: String -- ^ The input stream (python module source code).
src/Language/Python/Version2/Parser/Lexer.x view
@@ -20,9 +20,6 @@ import Language.Python.Common.SrcLocation import Language.Python.Common.LexerUtils import qualified Data.Map as Map-import Control.Monad (liftM)-import Data.List (foldl')-import Numeric (readHex, readOct) } -- character sets@@ -225,7 +222,7 @@ location <- getLocation input <- getInput startCode <- getStartCode- case alexScan (location, input) startCode of+ case alexScan (location, [], input) startCode of AlexEOF -> do -- Ensure there is a newline token before the EOF previousToken <- getLastToken@@ -245,11 +242,11 @@ setLastToken insertedNewlineToken return insertedNewlineToken AlexError _ -> lexicalError- AlexSkip (nextLocation, rest) len -> do+ AlexSkip (nextLocation, _bytes, rest) len -> do setLocation nextLocation setInput rest lexToken- AlexToken (nextLocation, rest) len action -> do+ AlexToken (nextLocation, _bytes, rest) len action -> do setLocation nextLocation setInput rest token <- action (mkSrcSpan location $ decColumn 1 nextLocation) len input
src/Language/Python/Version2/Parser/Parser.y view
@@ -1,4 +1,8 @@ {+{-# LANGUAGE CPP #-}++#undef __GLASGOW_HASKELL__+#define __GLASGOW_HASKELL__ 709 ----------------------------------------------------------------------------- -- | -- Module : Language.Python.Version2.Parser.Parser @@ -64,6 +68,7 @@ '!=' { NotEqualsToken {} } '<>' { NotEqualsV2Token {} } '.' { DotToken {} }+ '...' { EllipsisToken {} } '`' { BackQuoteToken {} } '+=' { PlusAssignToken {} } '-=' { MinusAssignToken {} }@@ -203,10 +208,10 @@ -- decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE opt_paren_arg_list :: { [ArgumentSpan] }-opt_paren_arg_list: opt(paren_arg_list) { concat (maybeToList $1) }+opt_paren_arg_list: opt(paren_arg_list) { (concat (maybeToList (fmap fst $1))) } -paren_arg_list :: { [ArgumentSpan] }-paren_arg_list : '(' optional_arg_list ')' { $2 }+paren_arg_list :: { ([ArgumentSpan], SrcSpan) }+paren_arg_list : '(' optional_arg_list ')' { ($2, spanning $1 $3) } decorator :: { DecoratorSpan } decorator @@ -746,10 +751,10 @@ exponent_op :: { OpSpan } exponent_op: '**' { AST.Exponent (getSpan $1) } -{- +{- atom: ('(' [yield_expr|testlist_gexp] ')' | '[' [listmaker] ']' |- '{' [dictmaker] '}' |+ '{' [dictorsetmaker] '}' | '`' testlist1 '`' | NAME | NUMBER | STRING+) -}@@ -758,7 +763,7 @@ atom : '(' yield_or_testlist_gexp ')' { $2 (spanning $1 $3) } | list_atom { $1 }- | '{' opt(dictmaker) '}' { AST.Dictionary (concat (maybeToList $2)) (spanning $1 $3) }+ | dict_or_set_atom { $1 } | '`' testlist1 '`' { AST.StringConversion $2 (spanning $1 $3) } | NAME { AST.Var $1 (getSpan $1) } | 'integer' { AST.Int (token_integer $1) (token_literal $1) (getSpan $1) }@@ -776,7 +781,12 @@ : '[' ']' { List [] (spanning $1 $2) } | '[' testlistfor ']' { makeListForm (spanning $1 $3) $2 } -testlistfor :: { Either ExprSpan (ComprehensionSpan ExprSpan) }+dict_or_set_atom :: { ExprSpan }+dict_or_set_atom+ : '{' '}' { Dictionary [] (spanning $1 $2) }+ | '{' dictorsetmaker '}' { $2 (spanning $1 $3) }++testlistfor :: { Either ExprSpan ComprehensionSpan } testlistfor : testlist { Left $1 } | test list_for { Right (makeComprehension $1 $2) }@@ -789,7 +799,7 @@ -- testlist_gexp: test ( gen_for | (',' test)* [','] ) -testlist_gexp :: { Either ExprSpan (ComprehensionSpan ExprSpan) }+testlist_gexp :: { Either ExprSpan ComprehensionSpan } testlist_gexp : testlist { Left $1 } | test gen_for { Right (makeComprehension $1 $2) }@@ -798,7 +808,7 @@ trailer :: { Trailer } trailer - : paren_arg_list { TrailerCall $1 (getSpan $1) }+ : paren_arg_list { TrailerCall (fst $1) (snd $1) } | '[' subscriptlist ']' { TrailerSubscript $2 (spanning $1 $3) } | '.' NAME { TrailerDot $2 (getSpan $1) (spanning $1 $2) } @@ -811,7 +821,8 @@ subscript :: { Subscript } subscript- : '.' '.' '.' { SubscriptSliceEllipsis (spanning $1 $3) }+ -- : '.' '.' '.' { SubscriptSliceEllipsis (spanning $1 $3) }+ : '...' { SubscriptSliceEllipsis (getSpan $1) } | test { SubscriptExpr $1 (getSpan $1) } | opt(test) ':' opt(test) opt(sliceop) { SubscriptSlice $1 $3 $4 (spanning (spanning (spanning $1 $2) $3) $4) }@@ -847,15 +858,40 @@ testlist : testlistrev opt_comma { makeTupleOrExpr (reverse $1) $2 } testlistrev :: { [ExprSpan] }-testlistrev +testlistrev : test { [$1] } | testlistrev ',' test { $3 : $1 } --- dictmaker: test ':' test (',' test ':' test)* [',']+{-+ dictorsetmaker: ( (test ':' test (comp_for | (',' test ':' test)* [','])) |+ (test (comp_for | (',' test)* [','])) )+-} -dictmaker :: { [(ExprSpan, ExprSpan)] }-dictmaker: sepOptEndBy(pair(test,right(':',test)), ',') { $1 }+dictorsetmaker :: { SrcSpan -> ExprSpan }+dictorsetmaker+ : test ':' test dict_rest { makeDictionary (Left ($1, $3)) $4 }+ | test set_rest { makeSet $1 $2 } +dict_rest :: { Either CompForSpan [Either (ExprSpan, ExprSpan) ExprSpan] }+dict_rest+ : comp_for { Left $1 }+ | zero_or_more_dict_mappings_rev opt_comma { Right (reverse $1) }++zero_or_more_dict_mappings_rev :: { [Either (ExprSpan, ExprSpan) ExprSpan] }+zero_or_more_dict_mappings_rev+ : {- empty -} { [] }+ | zero_or_more_dict_mappings_rev ',' test ':' test { (Left ($3,$5)) : $1 }++set_rest :: { Either CompForSpan [ExprSpan] }+set_rest+ : comp_for { Left $1 }+ | zero_or_more_comma_test_rev opt_comma { Right (reverse $1) }++zero_or_more_comma_test_rev :: { [ExprSpan] }+zero_or_more_comma_test_rev+ : {- empty -} { [] }+ | zero_or_more_comma_test_rev ',' test { $3 : $1 }+ -- classdef: 'class' NAME ['(' [testlist] ')'] ':' suite classdef :: { StatementSpan }@@ -905,6 +941,27 @@ | test gen_for { let span = spanning $1 $1 in ArgExpr (Generator (makeComprehension $1 $2) span) span } +-- comp_iter: comp_for | comp_if++comp_iter :: { CompIterSpan }+comp_iter+ : comp_for { IterFor $1 (getSpan $1) }+ | comp_if { IterIf $1 (getSpan $1) }++-- comp_for: 'for' exprlist 'in' or_test [comp_iter]++comp_for :: { CompForSpan }+comp_for+ : 'for' exprlist 'in' or_test opt(comp_iter)+ { CompFor False $2 $4 $5 (spanning (spanning $1 $4) $5) }++-- comp_if: 'if' old_trest [comp_iter]++comp_if :: { CompIfSpan }+comp_if+ : 'if' old_test opt(comp_iter)+ { CompIf $2 $3 (spanning (spanning $1 $2) $3) }+ -- list_iter: list_for | list_if list_iter :: { CompIterSpan } list_iter@@ -914,7 +971,7 @@ -- list_for: 'for' exprlist 'in' testlist_safe [list_iter] list_for :: { CompForSpan } list_for: 'for' exprlist 'in' testlist_safe opt(list_iter)- { AST.CompFor $2 $4 $5 (spanning (spanning $1 $4) $5) }+ { AST.CompFor False $2 $4 $5 (spanning (spanning $1 $4) $5) } -- list_if: 'if' old_test [list_iter] @@ -932,7 +989,7 @@ gen_for :: { CompForSpan } gen_for: 'for' exprlist 'in' or_test opt(gen_iter)- { AST.CompFor $2 $4 $5 (spanning (spanning $1 $4) $5) }+ { AST.CompFor False $2 $4 $5 (spanning (spanning $1 $4) $5) } -- gen_if: 'if' old_test [gen_iter] @@ -950,10 +1007,16 @@ -- yield_expr: 'yield' [testlist] yield_expr :: { ExprSpan }-yield_expr : 'yield' optional_testlist { AST.Yield $2 (spanning $1 $2) }+yield_expr : 'yield' optional_yieldarg { AST.Yield $2 (spanning $1 $2) } optional_testlist :: { Maybe ExprSpan } optional_testlist: opt(testlist) { $1 }++optional_yieldarg :: { Maybe YieldArgSpan }+optional_yieldarg : opt(yieldarg) { $1 }++yieldarg :: { YieldArgSpan }+yieldarg: testlist { YieldExpr $1 } { -- Put additional Haskell code in here if needed.
src/Language/Python/Version3.hs view
@@ -11,11 +11,11 @@ -- -- See: ----- * <http://www.python.org/doc/3.1/reference/index.html> for an overview of the language. +-- * <http://docs.python.org/3.1/reference/index.html> for an overview of the language. ----- * <http://www.python.org/doc/3.1/reference/grammar.html> for the full grammar.+-- * <http://docs.python.org/3.1/reference/grammar.html> for the full grammar. -- --- * <http://www.python.org/doc/3.1/reference/toplevel_components.html> for a description of +-- * <http://docs.python.org/3.1/reference/toplevel_components.html> for a description of -- the various Python top-levels, which correspond to the parsers provided here. -----------------------------------------------------------------------------
src/Language/Python/Version3/Lexer.hs view
@@ -9,7 +9,7 @@ -- Portability : ghc -- -- Lexical analysis for Python version 3.x programs. --- See: <http://www.python.org/doc/3.1/reference/lexical_analysis.html>.+-- See: <http://docs.python.org/3.1/reference/lexical_analysis.html>. ----------------------------------------------------------------------------- module Language.Python.Version3.Lexer (
src/Language/Python/Version3/Parser.hs view
@@ -14,11 +14,11 @@ -- -- See: ----- * <http://www.python.org/doc/3.1/reference/index.html> for an overview of the language. +-- * <http://docs.python.org/3.1/reference/index.html> for an overview of the language. ----- * <http://www.python.org/doc/3.1/reference/grammar.html> for the full grammar.+-- * <http://docs.python.org/3.1/reference/grammar.html> for the full grammar. -- --- * <http://www.python.org/doc/3.1/reference/toplevel_components.html> for a description of +-- * <http://docs.python.org/3.1/reference/toplevel_components.html> for a description of -- the various Python top-levels, which correspond to the parsers provided here. ----------------------------------------------------------------------------- @@ -35,7 +35,7 @@ import Language.Python.Common.AST (ModuleSpan, StatementSpan, ExprSpan) import Language.Python.Common.Token (Token) import Language.Python.Common.SrcLocation (initialSrcLocation)-import Language.Python.Common.ParserMonad (execParser, execParserKeepComments, ParseError, initialState)+import Language.Python.Common.ParserMonad (execParserKeepComments, ParseError, initialState) -- | Parse a whole Python source file. Return comments in addition to the parsed module. parseModule :: String -- ^ The input stream (python module source code).
src/Language/Python/Version3/Parser/Lexer.x view
@@ -9,7 +9,7 @@ -- Portability : ghc -- -- Implementation of a lexer for Python version 3.x programs. Generated by--- alex. +-- alex. Edited by Curran McConnell to conform to PEP515. ----------------------------------------------------------------------------- module Language.Python.Version3.Parser.Lexer @@ -20,9 +20,6 @@ import Language.Python.Common.SrcLocation import Language.Python.Common.LexerUtils import qualified Data.Map as Map-import Control.Monad (liftM)-import Data.List (foldl')-import Numeric (readHex, readOct) } -- character sets@@ -46,9 +43,9 @@ $not_double_quote = [. \n] # \" -- macro definitions-@exponent = (e | E) (\+ | \-)? $digit+ -@fraction = \. $digit+-@int_part = $digit++@exponent = (e | E) (\+ | \-)? $digit(_?$digit+)*+@fraction = \. $digit(_?$digit+)*+@int_part = $digit(_?$digit+)* @point_float = (@int_part? @fraction) | @int_part \. @exponent_float = (@int_part | @point_float) @exponent @float_number = @point_float | @exponent_float@@ -59,7 +56,10 @@ @two_double_quotes = \"\" $not_double_quote @byte_str_prefix = b | B @raw_str_prefix = r | R+@unicode_str_prefix = u | U+@format_str_prefix = f | F @raw_byte_str_prefix = @byte_str_prefix @raw_str_prefix | @raw_str_prefix @byte_str_prefix+@format_raw_str_prefix = @format_str_prefix @raw_str_prefix | @raw_str_prefix @format_str_prefix @backslash_pair = \\ (\\|'|\"|@eol_pattern|$short_str_char) @backslash_pair_bs = \\ (\\|'|\"|@eol_pattern|$short_byte_str_char) @short_str_item_single = $short_str_char|@backslash_pair|\"@@ -85,13 +85,13 @@ \\ @eol_pattern { lineJoin } -- line join <0> {- @float_number { token FloatToken readFloat }- $non_zero_digit $digit* { token IntegerToken read } + @float_number { token FloatToken (readFloat.delUnderscores) }+ $non_zero_digit (_?$digit)* { token IntegerToken (read.delUnderscores) } (@float_number | @int_part) (j | J) { token ImaginaryToken (readFloat.init) }- 0+ { token IntegerToken read } - 0 (o | O) $oct_digit+ { token IntegerToken read }- 0 (x | X) $hex_digit+ { token IntegerToken read }- 0 (b | B) $bin_digit+ { token IntegerToken readBinary }+ 0+(_?0+)* { token IntegerToken (read.delUnderscores) }+ 0 (o | O) (_?$oct_digit+)+ { token IntegerToken (read.delUnderscores) }+ 0 (x | X) (_?$hex_digit+)+ { token IntegerToken (read.delUnderscores) }+ 0 (b | B) (_?$bin_digit+)+ { token IntegerToken (readBinary.delUnderscores) } } -- String literals @@ -99,23 +99,35 @@ <0> { ' @short_str_item_single* ' { mkString stringToken } @raw_str_prefix ' @short_str_item_single* ' { mkString rawStringToken }+ @format_str_prefix ' @short_str_item_single* ' { mkString formatStringToken } @byte_str_prefix ' @short_byte_str_item_single* ' { mkString byteStringToken } @raw_byte_str_prefix ' @short_byte_str_item_single* ' { mkString rawByteStringToken }+ @format_raw_str_prefix ' @short_str_item_single* ' { mkString formatRawStringToken }+ @unicode_str_prefix ' @short_str_item_single* ' { mkString unicodeStringToken } \" @short_str_item_double* \" { mkString stringToken } @raw_str_prefix \" @short_str_item_double* \" { mkString rawStringToken }+ @format_str_prefix \" @short_str_item_double* \" { mkString formatStringToken } @byte_str_prefix \" @short_byte_str_item_double* \" { mkString byteStringToken } @raw_byte_str_prefix \" @short_byte_str_item_double* \" { mkString rawByteStringToken }+ @format_raw_str_prefix \" @short_str_item_double* \" { mkString formatRawStringToken }+ @unicode_str_prefix \" @short_str_item_double* \" { mkString unicodeStringToken } ''' @long_str_item_single* ''' { mkString stringToken } @raw_str_prefix ''' @long_str_item_single* ''' { mkString rawStringToken }+ @format_str_prefix ''' @long_str_item_single* ''' { mkString formatStringToken } @byte_str_prefix ''' @long_byte_str_item_single* ''' { mkString byteStringToken } @raw_byte_str_prefix ''' @long_byte_str_item_single* ''' { mkString rawByteStringToken }+ @format_raw_str_prefix ''' @long_str_item_single* ''' { mkString formatRawStringToken }+ @unicode_str_prefix ''' @long_str_item_single* ''' { mkString unicodeStringToken } \"\"\" @long_str_item_double* \"\"\" { mkString stringToken } @raw_str_prefix \"\"\" @long_str_item_double* \"\"\" { mkString rawStringToken }+ @format_str_prefix \"\"\" @long_str_item_double* \"\"\" { mkString formatStringToken } @byte_str_prefix \"\"\" @long_byte_str_item_double* \"\"\" { mkString byteStringToken } @raw_byte_str_prefix \"\"\" @long_byte_str_item_double* \"\"\" { mkString rawByteStringToken }+ @format_raw_str_prefix \"\"\" @long_str_item_double* \"\"\" { mkString formatRawStringToken }+ @unicode_str_prefix \"\"\" @long_str_item_double* \"\"\" { mkString unicodeStringToken } } -- NOTE: we pass lexToken into some functions as an argument.@@ -196,6 +208,7 @@ "<<=" { symbolToken LeftShiftAssignToken } ">>=" { symbolToken RightShiftAssignToken } "//=" { symbolToken FloorDivAssignToken } + "@=" { symbolToken MatrixMultAssignToken } "," { symbolToken CommaToken } "@" { symbolToken AtToken } \; { symbolToken SemiColonToken }@@ -211,7 +224,7 @@ location <- getLocation input <- getInput startCode <- getStartCode- case alexScan (location, input) startCode of+ case alexScan (location, [], input) startCode of AlexEOF -> do -- Ensure there is a newline token before the EOF previousToken <- getLastToken@@ -231,11 +244,11 @@ setLastToken insertedNewlineToken return insertedNewlineToken AlexError _ -> lexicalError- AlexSkip (nextLocation, rest) len -> do+ AlexSkip (nextLocation, _bs, rest) len -> do setLocation nextLocation setInput rest lexToken- AlexToken (nextLocation, rest) len action -> do+ AlexToken (nextLocation, _bs, rest) len action -> do setLocation nextLocation setInput rest token <- action (mkSrcSpan location $ decColumn 1 nextLocation) len input @@ -277,5 +290,6 @@ , ("as", AsToken), ("elif", ElifToken), ("if", IfToken), ("or", OrToken), ("yield", YieldToken) , ("assert", AssertToken), ("else", ElseToken), ("import", ImportToken), ("pass", PassToken) , ("break", BreakToken), ("except", ExceptToken), ("in", InToken), ("raise", RaiseToken)+ , ("async", AsyncToken), ("await", AwaitToken) ] }
src/Language/Python/Version3/Parser/Parser.y view
@@ -1,4 +1,8 @@ {+{-# LANGUAGE CPP #-}++#undef __GLASGOW_HASKELL__+#define __GLASGOW_HASKELL__ 709 ----------------------------------------------------------------------------- -- | -- Module : Language.Python.Version3.Parser.Parser @@ -20,7 +24,7 @@ import Language.Python.Common.ParserMonad import Language.Python.Common.SrcLocation import Data.Either (rights, either)-import Data.Maybe (maybeToList)+import Data.Maybe (isJust, maybeToList) } %name parseFileInput file_input @@ -76,11 +80,14 @@ '<<=' { LeftShiftAssignToken {} } '>>=' { RightShiftAssignToken {} } '//=' { FloorDivAssignToken {} } + '@=' { MatrixMultAssignToken {} } '@' { AtToken {} } '->' { RightArrowToken {} } 'and' { AndToken {} } 'as' { AsToken {} } 'assert' { AssertToken {} }+ 'async' { AsyncToken {} }+ 'await' { AwaitToken {} } 'break' { BreakToken {} } 'bytestring' { ByteStringToken {} } 'class' { ClassToken {} }@@ -117,6 +124,7 @@ 'string' { StringToken {} } 'True' { TrueToken {} } 'try' { TryToken {} }+ 'unicodestring' { UnicodeStringToken {} } 'while' { WhileToken {} } 'with' { WithToken {} } 'yield' { YieldToken {} }@@ -203,10 +211,10 @@ -- decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE opt_paren_arg_list :: { [ArgumentSpan] }-opt_paren_arg_list: opt(paren_arg_list) { concat (maybeToList $1) }+opt_paren_arg_list: opt(paren_arg_list) { (concat (maybeToList (fmap fst $1))) } -paren_arg_list :: { [ArgumentSpan] }-paren_arg_list : '(' optional_arg_list ')' { $2 }+paren_arg_list :: { ([ArgumentSpan], SrcSpan) }+paren_arg_list : '(' optional_arg_list ')' { ($2, spanning $1 $3) } decorator :: { DecoratorSpan } decorator @@ -222,8 +230,9 @@ decorated :: { StatementSpan } decorated - : decorators or(classdef,funcdef) - { makeDecorated $1 $2 } + : decorators classdef { makeDecorated $1 $2 }+ | decorators funcdef { makeDecorated $1 $2 }+ | decorators async_funcdef { makeDecorated $1 $2 } -- funcdef: 'def' NAME parameters ['->' test] ':' suite @@ -334,12 +343,15 @@ | nonlocal_stmt { $1 } | assert_stmt { $1 } --- expr_stmt: testlist_star_expr (augassign (yield_expr|testlist) | ('=' (yield_expr|testlist_star_expr))*)+{- expr_stmt: testlist_star_expr (annassign | augassign (yield_expr|testlist) |+ ('=' (yield_expr|testlist_star_expr))*)+-} expr_stmt :: { StatementSpan }-expr_stmt - : testlist_star_expr either(many_assign, augassign_yield_or_test_list) - { makeAssignmentOrExpr $1 $2 }+expr_stmt+ : testlist_star_expr many_assign { makeNormalAssignment $1 $2 }+ | testlist_star_expr augassign_yield_or_test_list { makeAugAssignment $1 $2 }+ | testlist_star_expr annassign { makeAnnAssignment $1 $2 } many_assign :: { [ExprSpan] } many_assign : many0(right('=', yield_or_test_list_star)) { $1 }@@ -367,7 +379,7 @@ {- augassign: ('+=' | '-=' | '*=' | '/=' | '%=' | '&=' | '|=' | '^=' |- '<<=' | '>>=' | '**=' | '//=') + '<<=' | '>>=' | '**=' | '//=' | '@=') -} augassign :: { AssignOpSpan }@@ -384,7 +396,13 @@ | '<<=' { AST.LeftShiftAssign (getSpan $1) } | '>>=' { AST.RightShiftAssign (getSpan $1) } | '//=' { AST.FloorDivAssign (getSpan $1) } + | '@=' { AST.MatrixMultAssign (getSpan $1) } +-- annassign: ':' test ['=' test]++annassign :: { (ExprSpan, Maybe ExprSpan) }+annassign : ':' test opt(right('=', test)) { ($2, $3) }+ -- del_stmt: 'del' exprlist del_stmt :: { StatementSpan }@@ -514,7 +532,7 @@ assert_stmt : 'assert' sepBy(test,',') { AST.Assert $2 (spanning $1 $2) } --- compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | with_stmt | funcdef | classdef | decorated +-- compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | with_stmt | funcdef | classdef | decorated | async_stmt compound_stmt :: { StatementSpan } compound_stmt @@ -526,6 +544,7 @@ | funcdef { $1 } | classdef { $1 } | decorated { $1 }+ | async_stmt { $1 } -- if_stmt: 'if' test ':' suite ('elif' test ':' suite)* ['else' ':' suite] @@ -555,6 +574,20 @@ : 'for' exprlist 'in' testlist ':' suite optional_else { AST.For $2 $4 $6 $7 (spanning (spanning $1 $6) $7) } +-- async_stmt: ASYNC (funcdef | with_stmt | for_stmt)++async_stmt :: { StatementSpan }+async_stmt+ : 'async' funcdef { AST.AsyncFun $2 (spanning $1 $2) }+ | 'async' with_stmt { AST.AsyncWith $2 (spanning $1 $2) }+ | 'async' for_stmt { AST.AsyncFor $2 (spanning $1 $2) }++-- async_fundef: ASYNC funcdef++async_funcdef :: { StatementSpan }+async_funcdef+ : 'async' funcdef { AST.AsyncFun $2 (spanning $1 $2) }+ {- try_stmt: ('try' ':' suite ((except_clause ':' suite)+ ['else' ':' suite] ['finally' ':' suite] | 'finally' ':' suite))@@ -742,6 +775,7 @@ | '/' { AST.Divide (getSpan $1) } | '%' { AST.Modulo (getSpan $1) } | '//' { AST.FloorDivide (getSpan $1) }+ | '@' { AST.MatrixMult (getSpan $1) } -- factor: ('+'|'-'|'~') factor | power @@ -753,10 +787,14 @@ tilde_op :: { OpSpan } tilde_op: '~' { AST.Invert (getSpan $1) } +-- await_expr: 'await' primary+await_expr :: { ExprSpan }+await_expr : 'await' atom { AST.Await $2 (spanning $1 $2) }+ -- power: atom trailer* ['**' factor] power :: { ExprSpan }-power : atom many0(trailer) opt(pair(exponent_op, factor)) +power : or(await_expr, atom) many0(trailer) opt(pair(exponent_op, factor)) { makeBinOp (addTrailer $1 $2) (maybeToList $3) } exponent_op :: { OpSpan }@@ -779,6 +817,7 @@ | 'imaginary' { AST.Imaginary (token_double $1) (token_literal $1) (getSpan $1) } | many1('string') { AST.Strings (map token_literal $1) (getSpan $1) } | many1('bytestring') { AST.ByteStrings (map token_literal $1) (getSpan $1) }+ | many1('unicodestring') { AST.UnicodeStrings (map token_literal $1) (getSpan $1) } | '...' { AST.Ellipsis (getSpan $1) } | 'None' { AST.None (getSpan $1) } | 'True' { AST.Bool Prelude.True (getSpan $1) }@@ -802,7 +841,7 @@ -- testlist_comp: (test|star_expr) ( comp_for | (',' (test|star_expr))* [','] ) -testlist_comp :: { Either ExprSpan (ComprehensionSpan ExprSpan) }+testlist_comp :: { Either ExprSpan ComprehensionSpan } testlist_comp : testlist_star_expr { Left $1 } | or(test,star_expr) comp_for { Right (makeComprehension $1 $2) }@@ -811,7 +850,7 @@ trailer :: { Trailer } trailer - : paren_arg_list { TrailerCall $1 (getSpan $1) }+ : paren_arg_list { TrailerCall (fst $1) (snd $1) } | '[' subscriptlist ']' { TrailerSubscript $2 (spanning $1 $3) } | '.' NAME { TrailerDot $2 (getSpan $1) (spanning $1 $2) } @@ -863,25 +902,29 @@ : test { [$1] } | testlistrev ',' test { $3 : $1 } -{- - dictorsetmaker: ( (test ':' test (comp_for | (',' test ':' test)* [','])) |- (test (comp_for | (',' test)* [','])) )+{-+ dictorsetmaker: ( ((test ':' test | '**' expr)+ (comp_for | (',' (test ':' test | '**' expr))* [','])) |+ ((test | star_expr)+ (comp_for | (',' (test | star_expr))* [','])) ) -} dictorsetmaker :: { SrcSpan -> ExprSpan } dictorsetmaker- : test ':' test dict_rest { makeDictionary ($1, $3) $4 } - | test set_rest { makeSet $1 $2 } + : test ':' test dict_rest { makeDictionary (Left ($1, $3)) $4 }+ | '**' expr dict_rest { makeDictionary (Right $2) $3 }+ | test set_rest { makeSet $1 $2 } -dict_rest :: { Either CompForSpan [(ExprSpan, ExprSpan)] }+dict_rest :: { Either CompForSpan [Either (ExprSpan, ExprSpan) ExprSpan] } dict_rest : comp_for { Left $1 } | zero_or_more_dict_mappings_rev opt_comma { Right (reverse $1) } -zero_or_more_dict_mappings_rev :: { [(ExprSpan, ExprSpan)] }+zero_or_more_dict_mappings_rev :: { [Either (ExprSpan, ExprSpan) ExprSpan] } zero_or_more_dict_mappings_rev : {- empty -} { [] }- | zero_or_more_dict_mappings_rev ',' test ':' test { ($3,$5) : $1 }+ | zero_or_more_dict_mappings_rev ',' test ':' test { Left ($3,$5) : $1 }+ | zero_or_more_dict_mappings_rev ',' '**' expr { Right $4 : $1 } set_rest :: { Either CompForSpan [ExprSpan] } set_rest@@ -946,8 +989,8 @@ comp_for :: { CompForSpan } comp_for - : 'for' exprlist 'in' or_test opt(comp_iter) - { CompFor $2 $4 $5 (spanning (spanning $1 $4) $5) }+ : opt('async') 'for' exprlist 'in' or_test opt(comp_iter)+ { CompFor (isJust $1) $3 $5 $6 (spanning $1 (spanning (spanning $2 $5) $6)) } -- comp_if: 'if' test_nocond [comp_iter] @@ -959,13 +1002,23 @@ -- encoding_decl: NAME -- Not used in the rest of the grammar! --- yield_expr: 'yield' [testlist] +-- yield_expr: 'yield' [yield_arg] yield_expr :: { ExprSpan }-yield_expr : 'yield' optional_testlist { AST.Yield $2 (spanning $1 $2) }+yield_expr : 'yield' optional_yieldarg { AST.Yield $2 (spanning $1 $2) } optional_testlist :: { Maybe ExprSpan } optional_testlist: opt(testlist) { $1 }++optional_yieldarg :: { Maybe YieldArgSpan } +optional_yieldarg: opt(yieldarg) { $1 }++-- yield_arg: 'from' test | testlist++yieldarg :: { YieldArgSpan }+yieldarg+ : 'from' test { YieldFrom $2 (spanning $1 $2) }+ | testlist { YieldExpr $1 } { -- Put additional Haskell code in here if needed.