diff --git a/ghc_6_12-13/Language/Python/Common/AST.hs b/ghc_6_12-13/Language/Python/Common/AST.hs deleted file mode 100644 --- a/ghc_6_12-13/Language/Python/Common/AST.hs +++ /dev/null @@ -1,734 +0,0 @@ -{-# LANGUAGE FlexibleInstances, TypeSynonymInstances, CPP, DeriveDataTypeable #-} -{-# OPTIONS_GHC -fomit-interface-pragmas #-} ------------------------------------------------------------------------------ --- | --- Module : Language.Python.Version2.Syntax.AST --- Copyright : (c) 2009 Bernie Pope --- License : BSD-style --- Maintainer : bjpop@csse.unimelb.edu.au --- Stability : experimental --- Portability : ghc --- --- Representation of the Python abstract syntax tree (AST). The representation is --- a superset of versions 2.x and 3.x of Python. In many cases they are --- identical. The documentation in this module indicates where they are --- different. --- --- All the data types have a (polymorphic) parameter which allows the AST to --- be annotated by an arbitrary type (for example source locations). Specialised --- instances of the types are provided for source spans. For example @Module a@ is --- the type of modules, and @ModuleSpan@ is the type of modules annoted with source --- span information. --- --- Note: there are cases where the AST is more liberal than the formal grammar --- of the language. Therefore some care must be taken when constructing --- Python programs using the raw AST. ------------------------------------------------------------------------------ - -module Language.Python.Common.AST ( - -- * Annotation projection - Annotated (..) - -- * Modules - , Module (..), ModuleSpan - -- * Identifiers and dotted names - , Ident (..), IdentSpan - , DottedName, DottedNameSpan - -- * Statements, suites, parameters, decorators and assignment operators - , Statement (..), StatementSpan - , Suite, SuiteSpan - , Parameter (..), ParameterSpan - , ParamTuple (..), ParamTupleSpan - , Decorator (..), DecoratorSpan - , AssignOp (..), AssignOpSpan - -- * Expressions, operators, arguments and slices - , Expr (..), ExprSpan - , Op (..), OpSpan - , Argument (..), ArgumentSpan - , Slice (..), SliceSpan - -- * Imports - , ImportItem (..), ImportItemSpan - , FromItem (..), FromItemSpan - , FromItems (..), FromItemsSpan - , ImportRelative (..), ImportRelativeSpan - -- * Exceptions - , Handler (..), HandlerSpan - , ExceptClause (..), ExceptClauseSpan - , RaiseExpr (..), RaiseExprSpan - -- * Comprehensions - , Comprehension (..), ComprehensionSpan - , CompFor (..), CompForSpan - , CompIf (..), CompIfSpan - , CompIter (..), CompIterSpan - ) - where - -import Language.Python.Common.SrcLocation ( Span (getSpan), SrcSpan (..) ) -import Data.Data - --------------------------------------------------------------------------------- - --- | Convenient access to annotations in annotated types. -class Annotated t where - -- | Given an annotated type, project out its annotation value. - annot :: t annot -> annot - --- | Identifier. -data Ident annot = Ident { ident_string :: !String, ident_annot :: annot } - deriving (Eq,Ord,Show,Typeable,Data) - -type IdentSpan = Ident SrcSpan - -instance Span IdentSpan where - getSpan = annot - -instance Annotated Ident where - annot = ident_annot - --- | A module (Python source file). --- --- * Version 2.6 --- --- * Version 3.1 --- -newtype Module annot = Module [Statement annot] -- ^ A module is just a sequence of top-level statements. - deriving (Eq,Ord,Show,Typeable,Data) - -type ModuleSpan = Module SrcSpan - --- | A block of statements. A suite is a group of statements controlled by a clause, --- for example, the body of a loop. --- --- * Version 2.6 --- --- * Version 3.1 --- -type Suite annot = [Statement annot] - -type SuiteSpan = Suite SrcSpan - --- | A compound name constructed with the dot operator. -type DottedName annot = [Ident annot] - -type DottedNameSpan = DottedName SrcSpan - --- | An entity imported using the \'import\' keyword. --- --- * Version 2.6 --- --- * Version 3.1 --- -data ImportItem annot = - ImportItem - { import_item_name :: DottedName annot -- ^ The name of module to import. - , import_as_name :: Maybe (Ident annot) -- ^ An optional name to refer to the entity (the \'as\' name). - , import_item_annot :: annot - } - deriving (Eq,Ord,Show,Typeable,Data) - -type ImportItemSpan = ImportItem SrcSpan - -instance Span ImportItemSpan where - getSpan = annot - -instance Annotated ImportItem where - annot = import_item_annot - --- | An entity imported using the \'from ... import\' construct. --- --- * Version 2.6 --- --- * Version 3.1 --- -data FromItem annot = - FromItem - { from_item_name :: Ident annot -- ^ The name of the entity imported. - , from_as_name :: Maybe (Ident annot) -- ^ An optional name to refer to the entity (the \'as\' name). - , from_item_annot :: annot - } - deriving (Eq,Ord,Show,Typeable,Data) - -type FromItemSpan = FromItem SrcSpan - -instance Span FromItemSpan where - getSpan = annot - -instance Annotated FromItem where - annot = from_item_annot - --- | Items imported using the \'from ... import\' construct. -data FromItems annot - = ImportEverything { from_items_annot :: annot } -- ^ Import everything exported from the module. - | FromItems { from_items_items :: [FromItem annot], from_items_annot :: annot } -- ^ Import a specific list of items from the module. - deriving (Eq,Ord,Show,Typeable,Data) - -type FromItemsSpan = FromItems SrcSpan - -instance Span FromItemsSpan where - getSpan = annot - -instance Annotated FromItems where - annot = from_items_annot - --- | A reference to the module to import from using the \'from ... import\' construct. -data ImportRelative annot - = ImportRelative - { import_relative_dots :: Int - , import_relative_module :: Maybe (DottedName annot) - , import_relative_annot :: annot - } - deriving (Eq,Ord,Show,Typeable,Data) - -type ImportRelativeSpan = ImportRelative SrcSpan - -instance Span ImportRelativeSpan where - getSpan = annot - -instance Annotated ImportRelative where - annot = import_relative_annot - --- | Statements. --- --- * Simple statements: --- --- * Version 2.6 --- --- * Version 3.1 --- --- * Compound statements: --- --- * Version 2.6 --- --- * Version 3.1 --- -data Statement annot - -- | Import statement. - = Import - { import_items :: [ImportItem annot] -- ^ Items to import. - , stmt_annot :: annot - } - -- | From ... import statement. - | FromImport - { from_module :: ImportRelative annot -- ^ Module to import from. - , from_items :: FromItems annot -- ^ Items to import. - , stmt_annot :: annot - } - -- | While loop. - | While - { while_cond :: Expr annot -- ^ Loop condition. - , while_body :: Suite annot -- ^ Loop body. - , while_else :: Suite annot -- ^ Else clause. - , stmt_annot :: annot - } - -- | For loop. - | For - { for_targets :: [Expr annot] -- ^ Loop variables. - , for_generator :: Expr annot -- ^ Loop generator. - , for_body :: Suite annot -- ^ Loop body - , for_else :: Suite annot -- ^ Else clause. - , stmt_annot :: annot - } - -- | Function definition. - | Fun - { fun_name :: Ident annot -- ^ Function name. - , fun_args :: [Parameter annot] -- ^ Function parameter list. - , fun_result_annotation :: Maybe (Expr annot) -- ^ Optional result annotation. - , fun_body :: Suite annot -- ^ Function body. - , stmt_annot :: annot - } - -- | Class definition. - | Class - { class_name :: Ident annot -- ^ Class name. - , class_args :: [Argument annot] -- ^ Class argument list. In version 2.x this is only ArgExprs. - , class_body :: Suite annot -- ^ Class body. - , stmt_annot :: annot - } - -- | Conditional statement (if-elif-else). - | Conditional - { cond_guards :: [(Expr annot, Suite annot)] -- ^ Sequence of if-elif conditional clauses. - , cond_else :: Suite annot -- ^ Possibly empty unconditional else clause. - , stmt_annot :: annot - } - -- | Assignment statement. - | Assign - { assign_to :: [Expr annot] -- ^ Entity to assign to. - , assign_expr :: Expr annot -- ^ Expression to evaluate. - , stmt_annot :: annot - } - -- | Augmented assignment statement. - | AugmentedAssign - { aug_assign_to :: Expr annot -- ^ Entity to assign to. - , aug_assign_op :: AssignOp annot -- ^ Assignment operator (for example \'+=\'). - , aug_assign_expr :: Expr annot -- ^ Expression to evaluate. - , stmt_annot :: annot - } - -- | Decorated definition of a function or class. - | Decorated - { decorated_decorators :: [Decorator annot] -- ^ Decorators. - , decorated_def :: Statement annot -- ^ Function or class definition to be decorated. - , stmt_annot :: annot - } - -- | Return statement (may only occur syntactically nested in a function definition). - | Return - { return_expr :: Maybe (Expr annot) -- ^ Optional expression to evaluate and return to caller. - , stmt_annot :: annot - } - -- | Try statement (exception handling). - | Try - { try_body :: Suite annot -- ^ Try clause. - , try_excepts :: [Handler annot] -- ^ Exception handlers. - , try_else :: Suite annot -- ^ Possibly empty else clause, executed if and when control flows off the end of the try clause. - , try_finally :: Suite annot -- ^ Possibly empty finally clause. - , stmt_annot :: annot - } - -- | Raise statement (exception throwing). - | Raise - { raise_expr :: RaiseExpr annot - , stmt_annot :: annot - } - -- | With statement (context management). - | With - { with_context :: [(Expr annot, Maybe (Expr annot))] -- ^ Context expression(s) (yields a context manager). - , with_body :: Suite annot -- ^ Suite to be managed. - , stmt_annot :: annot - } - -- | Pass statement (null operation). - | Pass { stmt_annot :: annot } - -- | Break statement (may only occur syntactically nested in a for or while loop, but not nested in a function or class definition within that loop). - | Break { stmt_annot :: annot } - -- | Continue statement (may only occur syntactically nested in a for or while loop, but not nested in a function or class definition or finally clause within that loop). - | Continue { stmt_annot :: annot } - -- | Del statement (delete). - | Delete - { del_exprs :: [Expr annot] -- ^ Items to delete. - , stmt_annot :: annot - } - -- | Expression statement. - | StmtExpr { stmt_expr :: Expr annot, stmt_annot :: annot } - -- | Global declaration. - | Global - { global_vars :: [Ident annot] -- ^ Variables declared global in the current block. - , stmt_annot :: annot - } - -- | Nonlocal declaration. /Version 3.x only/. - | NonLocal - { nonLocal_vars :: [Ident annot] -- ^ Variables declared nonlocal in the current block (their binding comes from bound the nearest enclosing scope). - , stmt_annot :: annot - } - -- | Assertion. - | Assert - { assert_exprs :: [Expr annot] -- ^ Expressions being asserted. - , stmt_annot :: annot - } - -- | Print statement. /Version 2 only/. - | Print - { print_chevron :: Bool -- ^ Optional chevron (>>) - , print_exprs :: [Expr annot] -- ^ Arguments to print - , print_trailing_comma :: Bool -- ^ Does it end in a comma? - , stmt_annot :: annot - } - -- | Exec statement. /Version 2 only/. - | Exec - { exec_expr :: Expr annot -- ^ Expression to exec. - , exec_globals_locals :: Maybe (Expr annot, Maybe (Expr annot)) -- ^ Global and local environments to evaluate the expression within. - , stmt_annot :: annot - } - deriving (Eq,Ord,Show,Typeable,Data) - -type StatementSpan = Statement SrcSpan - -instance Span StatementSpan where - getSpan = annot - -instance Annotated Statement where - annot = stmt_annot - --- | The argument for a @raise@ statement. -data RaiseExpr annot - = RaiseV3 (Maybe (Expr annot, Maybe (Expr annot))) -- ^ Optional expression to evaluate, and optional \'from\' clause. /Version 3 only/. - | RaiseV2 (Maybe (Expr annot, (Maybe (Expr annot, Maybe (Expr annot))))) -- ^ /Version 2 only/. - deriving (Eq,Ord,Show,Typeable,Data) - -type RaiseExprSpan = RaiseExpr SrcSpan - --- | Decorator. -data Decorator annot = - Decorator - { decorator_name :: DottedName annot -- ^ Decorator name. - , decorator_args :: [Argument annot] -- ^ Decorator arguments. - , decorator_annot :: annot - } - deriving (Eq,Ord,Show,Typeable,Data) - -type DecoratorSpan = Decorator SrcSpan - -instance Span DecoratorSpan where - getSpan = annot - -instance Annotated Decorator where - annot = decorator_annot - --- | Formal parameter of function definitions and lambda expressions. --- --- * Version 2.6: --- --- * --- --- * --- --- * Version 3.1: --- --- * --- --- * --- -data Parameter annot - -- | Ordinary named parameter. - = Param - { param_name :: Ident annot -- ^ Parameter name. - , param_py_annotation :: Maybe (Expr annot) -- ^ Optional annotation. - , param_default :: Maybe (Expr annot) -- ^ Optional default value. - , param_annot :: annot - } - -- | Excess positional parameter (single asterisk before its name in the concrete syntax). - | VarArgsPos - { param_name :: Ident annot -- ^ Parameter name. - , param_py_annotation :: Maybe (Expr annot) -- ^ Optional annotation. - , param_annot :: annot - } - -- | Excess keyword parameter (double asterisk before its name in the concrete syntax). - | VarArgsKeyword - { param_name :: Ident annot -- ^ Parameter name. - , param_py_annotation :: Maybe (Expr annot) -- ^ Optional annotation. - , param_annot :: annot - } - -- | Marker for the end of positional parameters (not a parameter itself). - | EndPositional { param_annot :: annot } - -- | Tuple unpack. /Version 2 only/. - | UnPackTuple - { param_unpack_tuple :: ParamTuple annot -- ^ The tuple to unpack. - , param_default :: Maybe (Expr annot) -- ^ Optional default value. - , param_annot :: annot - } - deriving (Eq,Ord,Show,Typeable,Data) - -type ParameterSpan = Parameter SrcSpan - -instance Span ParameterSpan where - getSpan = annot - -instance Annotated Parameter where - annot = param_annot - --- | Tuple unpack parameter. /Version 2 only/. -data ParamTuple annot - = ParamTupleName { param_tuple_name :: Ident annot, param_tuple_annot :: annot } -- ^ A variable name. - | ParamTuple { param_tuple :: [ParamTuple annot], param_tuple_annot :: annot } -- ^ A (possibly nested) tuple parameter. - deriving (Eq,Ord,Show,Typeable,Data) - -type ParamTupleSpan = ParamTuple SrcSpan - -instance Span ParamTupleSpan where - getSpan = annot - -instance Annotated ParamTuple where - annot = param_tuple_annot - --- | Arguments to function calls, class declarations and decorators. -data Argument annot - -- | Ordinary argument expression. - = ArgExpr { arg_expr :: Expr annot, arg_annot :: annot } - -- | Excess positional argument. - | ArgVarArgsPos { arg_expr :: Expr annot, arg_annot :: annot } - -- | Excess keyword argument. - | ArgVarArgsKeyword { arg_expr :: Expr annot, arg_annot :: annot } - -- | Keyword argument. - | ArgKeyword - { arg_keyword :: Ident annot -- ^ Keyword name. - , arg_expr :: Expr annot -- ^ Argument expression. - , arg_annot :: annot - } - deriving (Eq,Ord,Show,Typeable,Data) - -type ArgumentSpan = Argument SrcSpan - -instance Span ArgumentSpan where - getSpan = annot - -instance Annotated Argument where - annot = arg_annot - --- | Exception handler. -data Handler annot - = Handler - { handler_clause :: ExceptClause annot - , handler_suite :: Suite annot - , handler_annot :: annot - } - deriving (Eq,Ord,Show,Typeable,Data) - -type HandlerSpan = Handler SrcSpan - -instance Span HandlerSpan where - getSpan = annot - -instance Annotated Handler where - annot = handler_annot - --- | Exception clause. -data ExceptClause annot - = ExceptClause - -- NB: difference with version 3 (has NAME as target, but looks like bug in grammar) - { except_clause :: Maybe (Expr annot, Maybe (Expr annot)) - , except_clause_annot :: annot - } - deriving (Eq,Ord,Show,Typeable,Data) - -type ExceptClauseSpan = ExceptClause SrcSpan - -instance Span ExceptClauseSpan where - getSpan = annot - -instance Annotated ExceptClause where - annot = except_clause_annot - --- | Comprehension. In version 3.x this can be used for lists, sets, dictionaries and generators. -data Comprehension e annot - = Comprehension - { comprehension_expr :: e - , comprehension_for :: CompFor annot - , comprehension_annot :: annot - } - deriving (Eq,Ord,Show,Typeable,Data) - -type ComprehensionSpan e = Comprehension e SrcSpan - -instance Span (ComprehensionSpan e) where - getSpan = annot - -instance Annotated (Comprehension e) where - annot = comprehension_annot - --- | Comprehension \'for\' component. -data CompFor annot = - CompFor - { comp_for_exprs :: [Expr annot] - , comp_in_expr :: Expr annot - , comp_for_iter :: Maybe (CompIter annot) - , comp_for_annot :: annot - } - deriving (Eq,Ord,Show,Typeable,Data) - -type CompForSpan = CompFor SrcSpan - -instance Span CompForSpan where - getSpan = annot - -instance Annotated CompFor where - annot = comp_for_annot - --- | Comprehension guard. -data CompIf annot = - CompIf - { comp_if :: Expr annot - , comp_if_iter :: Maybe (CompIter annot) - , comp_if_annot :: annot - } - deriving (Eq,Ord,Show,Typeable,Data) - -type CompIfSpan = CompIf SrcSpan - -instance Span CompIfSpan where - getSpan = annot - -instance Annotated CompIf where - annot = comp_if_annot - --- | Comprehension iterator (either a \'for\' or an \'if\'). -data CompIter annot - = IterFor { comp_iter_for :: CompFor annot, comp_iter_annot :: annot } - | IterIf { comp_iter_if :: CompIf annot, comp_iter_annot :: annot } - deriving (Eq,Ord,Show,Typeable,Data) - -type CompIterSpan = CompIter SrcSpan - -instance Span CompIterSpan where - getSpan = annot - -instance Annotated CompIter where - annot = comp_iter_annot - --- | Expressions. --- --- * Version 2.6 . --- --- * Version 3.1 . --- -data Expr annot - -- | Variable. - = Var { var_ident :: Ident annot, expr_annot :: annot } - -- | Literal integer. - | Int { int_value :: Integer, expr_literal :: String, expr_annot :: annot } - -- | Long literal integer. /Version 2 only/. - | LongInt { int_value :: Integer, expr_literal :: String, expr_annot :: annot } - -- | Literal floating point number. - | Float { float_value :: Double, expr_literal :: String, expr_annot :: annot } - -- | Literal imaginary number. - | Imaginary { imaginary_value :: Double, expr_literal :: String, expr_annot :: annot } - -- | Literal boolean. - | Bool { bool_value :: Bool, expr_annot :: annot } - -- | Literal \'None\' value. - | None { expr_annot :: annot } - -- | Ellipsis \'...\'. - | Ellipsis { expr_annot :: annot } - -- | Literal byte string. - | ByteStrings { byte_string_strings :: [String], expr_annot :: annot } - -- | Literal strings (to be concatentated together). - | Strings { strings_strings :: [String], expr_annot :: annot } - -- | Unicode literal strings (to be concatentated together). Version 2 only. - | UnicodeStrings { unicodestrings_strings :: [String], expr_annot :: annot } - -- | Function call. - | Call - { call_fun :: Expr annot -- ^ Expression yielding a callable object (such as a function). - , call_args :: [Argument annot] -- ^ Call arguments. - , expr_annot :: annot - } - -- | Subscription, for example \'x [y]\'. - | Subscript { subscriptee :: Expr annot, subscript_expr :: Expr annot, expr_annot :: annot } - -- | Slicing, for example \'w [x:y:z]\'. - | SlicedExpr { slicee :: Expr annot, slices :: [Slice annot], expr_annot :: annot } - -- | Conditional expresison. - | CondExpr - { ce_true_branch :: Expr annot -- ^ Expression to evaluate if condition is True. - , ce_condition :: Expr annot -- ^ Boolean condition. - , ce_false_branch :: Expr annot -- ^ Expression to evaluate if condition is False. - , expr_annot :: annot - } - -- | Binary operator application. - | BinaryOp { operator :: Op annot, left_op_arg :: Expr annot, right_op_arg :: Expr annot, expr_annot :: annot } - -- | Unary operator application. - | UnaryOp { operator :: Op annot, op_arg :: Expr annot, expr_annot :: annot } - -- | Anonymous function definition (lambda). - | Lambda { lambda_args :: [Parameter annot], lambda_body :: Expr annot, expr_annot :: annot } - -- | Tuple. Can be empty. - | Tuple { tuple_exprs :: [Expr annot], expr_annot :: annot } - -- | Generator yield. - | Yield - { yield_expr :: Maybe (Expr annot) -- ^ Optional expression to yield. - , expr_annot :: annot - } - -- | Generator. - | Generator { gen_comprehension :: Comprehension (Expr annot) annot, expr_annot :: annot } - -- | List comprehension. - | ListComp { list_comprehension :: Comprehension (Expr annot) annot, expr_annot :: annot } - -- | List. - | List { list_exprs :: [Expr annot], expr_annot :: annot } - -- | Dictionary. - | Dictionary { dict_mappings :: [(Expr annot, Expr annot)], expr_annot :: annot } - -- | Dictionary comprehension. /Version 3 only/. - | DictComp { dict_comprehension :: Comprehension (Expr annot, Expr annot) annot, expr_annot :: annot } - -- | Set. - | Set { set_exprs :: [Expr annot], expr_annot :: annot } - -- | Set comprehension. /Version 3 only/. - | SetComp { set_comprehension :: Comprehension (Expr annot) annot, expr_annot :: annot } - -- | Starred expression. /Version 3 only/. - | Starred { starred_expr :: Expr annot, expr_annot :: annot } - -- | Parenthesised expression. - | Paren { paren_expr :: Expr annot, expr_annot :: annot } - -- | String conversion (backquoted expression). Version 2 only. - | StringConversion { backquoted_expr :: Expr annot, expr_anot :: annot } - deriving (Eq,Ord,Show,Typeable,Data) - -type ExprSpan = Expr SrcSpan - -instance Span ExprSpan where - getSpan = annot - -instance Annotated Expr where - annot = expr_annot - --- | Slice compenent. -data Slice annot - = SliceProper - { slice_lower :: Maybe (Expr annot) - , slice_upper :: Maybe (Expr annot) - , slice_stride :: Maybe (Maybe (Expr annot)) - , slice_annot :: annot - } - | SliceExpr - { slice_expr :: Expr annot - , slice_annot :: annot - } - | SliceEllipsis { slice_annot :: annot } - deriving (Eq,Ord,Show,Typeable,Data) - -type SliceSpan = Slice SrcSpan - -instance Span SliceSpan where - getSpan = annot - -instance Annotated Slice where - annot = slice_annot - --- | Operators. -data Op annot - = And { op_annot :: annot } -- ^ \'and\' - | Or { op_annot :: annot } -- ^ \'or\' - | Not { op_annot :: annot } -- ^ \'not\' - | Exponent { op_annot :: annot } -- ^ \'**\' - | LessThan { op_annot :: annot } -- ^ \'<\' - | GreaterThan { op_annot :: annot } -- ^ \'>\' - | Equality { op_annot :: annot } -- ^ \'==\' - | GreaterThanEquals { op_annot :: annot } -- ^ \'>=\' - | LessThanEquals { op_annot :: annot } -- ^ \'<=\' - | NotEquals { op_annot :: annot } -- ^ \'!=\' - | NotEqualsV2 { op_annot :: annot } -- ^ \'<>\'. Version 2 only. - | In { op_annot :: annot } -- ^ \'in\' - | Is { op_annot :: annot } -- ^ \'is\' - | IsNot { op_annot :: annot } -- ^ \'is not\' - | NotIn { op_annot :: annot } -- ^ \'not in\' - | BinaryOr { op_annot :: annot } -- ^ \'|\' - | Xor { op_annot :: annot } -- ^ \'^\' - | BinaryAnd { op_annot :: annot } -- ^ \'&\' - | ShiftLeft { op_annot :: annot } -- ^ \'<<\' - | ShiftRight { op_annot :: annot } -- ^ \'>>\' - | Multiply { op_annot :: annot } -- ^ \'*\' - | Plus { op_annot :: annot } -- ^ \'+\' - | Minus { op_annot :: annot } -- ^ \'-\' - | Divide { op_annot :: annot } -- ^ \'\/\' - | FloorDivide { op_annot :: annot } -- ^ \'\/\/\' - | Invert { op_annot :: annot } -- ^ \'~\' (bitwise inversion of its integer argument) - | Modulo { op_annot :: annot } -- ^ \'%\' - | Dot { op_annot :: annot } -- ^ \'.\' - deriving (Eq,Ord,Show,Typeable,Data) - -type OpSpan = Op SrcSpan - -instance Span OpSpan where - getSpan = annot - -instance Annotated Op where - annot = op_annot - --- | Augmented assignment operators. -data AssignOp annot - = PlusAssign { assignOp_annot :: annot } -- ^ \'+=\' - | MinusAssign { assignOp_annot :: annot } -- ^ \'-=\' - | MultAssign { assignOp_annot :: annot } -- ^ \'*=\' - | DivAssign { assignOp_annot :: annot } -- ^ \'\/=\' - | ModAssign { assignOp_annot :: annot } -- ^ \'%=\' - | PowAssign { assignOp_annot :: annot } -- ^ \'*=\' - | BinAndAssign { assignOp_annot :: annot } -- ^ \'&=\' - | BinOrAssign { assignOp_annot :: annot } -- ^ \'|=\' - | BinXorAssign { assignOp_annot :: annot } -- ^ \'^=\' - | LeftShiftAssign { assignOp_annot :: annot } -- ^ \'<<=\' - | RightShiftAssign { assignOp_annot :: annot } -- ^ \'>>=\' - | FloorDivAssign { assignOp_annot :: annot } -- ^ \'\/\/=\' - deriving (Eq,Ord,Show,Typeable,Data) - -type AssignOpSpan = AssignOp SrcSpan - -instance Span AssignOpSpan where - getSpan = annot - -instance Annotated AssignOp where - annot = assignOp_annot diff --git a/ghc_normal/Language/Python/Common/AST.hs b/ghc_normal/Language/Python/Common/AST.hs deleted file mode 100644 --- a/ghc_normal/Language/Python/Common/AST.hs +++ /dev/null @@ -1,733 +0,0 @@ -{-# LANGUAGE FlexibleInstances, TypeSynonymInstances, CPP, DeriveDataTypeable #-} ------------------------------------------------------------------------------ --- | --- Module : Language.Python.Version2.Syntax.AST --- Copyright : (c) 2009 Bernie Pope --- License : BSD-style --- Maintainer : bjpop@csse.unimelb.edu.au --- Stability : experimental --- Portability : ghc --- --- Representation of the Python abstract syntax tree (AST). The representation is --- a superset of versions 2.x and 3.x of Python. In many cases they are --- identical. The documentation in this module indicates where they are --- different. --- --- All the data types have a (polymorphic) parameter which allows the AST to --- be annotated by an arbitrary type (for example source locations). Specialised --- instances of the types are provided for source spans. For example @Module a@ is --- the type of modules, and @ModuleSpan@ is the type of modules annoted with source --- span information. --- --- Note: there are cases where the AST is more liberal than the formal grammar --- of the language. Therefore some care must be taken when constructing --- Python programs using the raw AST. ------------------------------------------------------------------------------ - -module Language.Python.Common.AST ( - -- * Annotation projection - Annotated (..) - -- * Modules - , Module (..), ModuleSpan - -- * Identifiers and dotted names - , Ident (..), IdentSpan - , DottedName, DottedNameSpan - -- * Statements, suites, parameters, decorators and assignment operators - , Statement (..), StatementSpan - , Suite, SuiteSpan - , Parameter (..), ParameterSpan - , ParamTuple (..), ParamTupleSpan - , Decorator (..), DecoratorSpan - , AssignOp (..), AssignOpSpan - -- * Expressions, operators, arguments and slices - , Expr (..), ExprSpan - , Op (..), OpSpan - , Argument (..), ArgumentSpan - , Slice (..), SliceSpan - -- * Imports - , ImportItem (..), ImportItemSpan - , FromItem (..), FromItemSpan - , FromItems (..), FromItemsSpan - , ImportRelative (..), ImportRelativeSpan - -- * Exceptions - , Handler (..), HandlerSpan - , ExceptClause (..), ExceptClauseSpan - , RaiseExpr (..), RaiseExprSpan - -- * Comprehensions - , Comprehension (..), ComprehensionSpan - , CompFor (..), CompForSpan - , CompIf (..), CompIfSpan - , CompIter (..), CompIterSpan - ) - where - -import Language.Python.Common.SrcLocation ( Span (getSpan), SrcSpan (..) ) -import Data.Data - --------------------------------------------------------------------------------- - --- | Convenient access to annotations in annotated types. -class Annotated t where - -- | Given an annotated type, project out its annotation value. - annot :: t annot -> annot - --- | Identifier. -data Ident annot = Ident { ident_string :: !String, ident_annot :: annot } - deriving (Eq,Ord,Show,Typeable,Data) - -type IdentSpan = Ident SrcSpan - -instance Span IdentSpan where - getSpan = annot - -instance Annotated Ident where - annot = ident_annot - --- | A module (Python source file). --- --- * Version 2.6 --- --- * Version 3.1 --- -newtype Module annot = Module [Statement annot] -- ^ A module is just a sequence of top-level statements. - deriving (Eq,Ord,Show,Typeable,Data) - -type ModuleSpan = Module SrcSpan - --- | A block of statements. A suite is a group of statements controlled by a clause, --- for example, the body of a loop. --- --- * Version 2.6 --- --- * Version 3.1 --- -type Suite annot = [Statement annot] - -type SuiteSpan = Suite SrcSpan - --- | A compound name constructed with the dot operator. -type DottedName annot = [Ident annot] - -type DottedNameSpan = DottedName SrcSpan - --- | An entity imported using the \'import\' keyword. --- --- * Version 2.6 --- --- * Version 3.1 --- -data ImportItem annot = - ImportItem - { import_item_name :: DottedName annot -- ^ The name of module to import. - , import_as_name :: Maybe (Ident annot) -- ^ An optional name to refer to the entity (the \'as\' name). - , import_item_annot :: annot - } - deriving (Eq,Ord,Show,Typeable,Data) - -type ImportItemSpan = ImportItem SrcSpan - -instance Span ImportItemSpan where - getSpan = annot - -instance Annotated ImportItem where - annot = import_item_annot - --- | An entity imported using the \'from ... import\' construct. --- --- * Version 2.6 --- --- * Version 3.1 --- -data FromItem annot = - FromItem - { from_item_name :: Ident annot -- ^ The name of the entity imported. - , from_as_name :: Maybe (Ident annot) -- ^ An optional name to refer to the entity (the \'as\' name). - , from_item_annot :: annot - } - deriving (Eq,Ord,Show,Typeable,Data) - -type FromItemSpan = FromItem SrcSpan - -instance Span FromItemSpan where - getSpan = annot - -instance Annotated FromItem where - annot = from_item_annot - --- | Items imported using the \'from ... import\' construct. -data FromItems annot - = ImportEverything { from_items_annot :: annot } -- ^ Import everything exported from the module. - | FromItems { from_items_items :: [FromItem annot], from_items_annot :: annot } -- ^ Import a specific list of items from the module. - deriving (Eq,Ord,Show,Typeable,Data) - -type FromItemsSpan = FromItems SrcSpan - -instance Span FromItemsSpan where - getSpan = annot - -instance Annotated FromItems where - annot = from_items_annot - --- | A reference to the module to import from using the \'from ... import\' construct. -data ImportRelative annot - = ImportRelative - { import_relative_dots :: Int - , import_relative_module :: Maybe (DottedName annot) - , import_relative_annot :: annot - } - deriving (Eq,Ord,Show,Typeable,Data) - -type ImportRelativeSpan = ImportRelative SrcSpan - -instance Span ImportRelativeSpan where - getSpan = annot - -instance Annotated ImportRelative where - annot = import_relative_annot - --- | Statements. --- --- * Simple statements: --- --- * Version 2.6 --- --- * Version 3.1 --- --- * Compound statements: --- --- * Version 2.6 --- --- * Version 3.1 --- -data Statement annot - -- | Import statement. - = Import - { import_items :: [ImportItem annot] -- ^ Items to import. - , stmt_annot :: annot - } - -- | From ... import statement. - | FromImport - { from_module :: ImportRelative annot -- ^ Module to import from. - , from_items :: FromItems annot -- ^ Items to import. - , stmt_annot :: annot - } - -- | While loop. - | While - { while_cond :: Expr annot -- ^ Loop condition. - , while_body :: Suite annot -- ^ Loop body. - , while_else :: Suite annot -- ^ Else clause. - , stmt_annot :: annot - } - -- | For loop. - | For - { for_targets :: [Expr annot] -- ^ Loop variables. - , for_generator :: Expr annot -- ^ Loop generator. - , for_body :: Suite annot -- ^ Loop body - , for_else :: Suite annot -- ^ Else clause. - , stmt_annot :: annot - } - -- | Function definition. - | Fun - { fun_name :: Ident annot -- ^ Function name. - , fun_args :: [Parameter annot] -- ^ Function parameter list. - , fun_result_annotation :: Maybe (Expr annot) -- ^ Optional result annotation. - , fun_body :: Suite annot -- ^ Function body. - , stmt_annot :: annot - } - -- | Class definition. - | Class - { class_name :: Ident annot -- ^ Class name. - , class_args :: [Argument annot] -- ^ Class argument list. In version 2.x this is only ArgExprs. - , class_body :: Suite annot -- ^ Class body. - , stmt_annot :: annot - } - -- | Conditional statement (if-elif-else). - | Conditional - { cond_guards :: [(Expr annot, Suite annot)] -- ^ Sequence of if-elif conditional clauses. - , cond_else :: Suite annot -- ^ Possibly empty unconditional else clause. - , stmt_annot :: annot - } - -- | Assignment statement. - | Assign - { assign_to :: [Expr annot] -- ^ Entity to assign to. - , assign_expr :: Expr annot -- ^ Expression to evaluate. - , stmt_annot :: annot - } - -- | Augmented assignment statement. - | AugmentedAssign - { aug_assign_to :: Expr annot -- ^ Entity to assign to. - , aug_assign_op :: AssignOp annot -- ^ Assignment operator (for example \'+=\'). - , aug_assign_expr :: Expr annot -- ^ Expression to evaluate. - , stmt_annot :: annot - } - -- | Decorated definition of a function or class. - | Decorated - { decorated_decorators :: [Decorator annot] -- ^ Decorators. - , decorated_def :: Statement annot -- ^ Function or class definition to be decorated. - , stmt_annot :: annot - } - -- | Return statement (may only occur syntactically nested in a function definition). - | Return - { return_expr :: Maybe (Expr annot) -- ^ Optional expression to evaluate and return to caller. - , stmt_annot :: annot - } - -- | Try statement (exception handling). - | Try - { try_body :: Suite annot -- ^ Try clause. - , try_excepts :: [Handler annot] -- ^ Exception handlers. - , try_else :: Suite annot -- ^ Possibly empty else clause, executed if and when control flows off the end of the try clause. - , try_finally :: Suite annot -- ^ Possibly empty finally clause. - , stmt_annot :: annot - } - -- | Raise statement (exception throwing). - | Raise - { raise_expr :: RaiseExpr annot - , stmt_annot :: annot - } - -- | With statement (context management). - | With - { with_context :: [(Expr annot, Maybe (Expr annot))] -- ^ Context expression(s) (yields a context manager). - , with_body :: Suite annot -- ^ Suite to be managed. - , stmt_annot :: annot - } - -- | Pass statement (null operation). - | Pass { stmt_annot :: annot } - -- | Break statement (may only occur syntactically nested in a for or while loop, but not nested in a function or class definition within that loop). - | Break { stmt_annot :: annot } - -- | Continue statement (may only occur syntactically nested in a for or while loop, but not nested in a function or class definition or finally clause within that loop). - | Continue { stmt_annot :: annot } - -- | Del statement (delete). - | Delete - { del_exprs :: [Expr annot] -- ^ Items to delete. - , stmt_annot :: annot - } - -- | Expression statement. - | StmtExpr { stmt_expr :: Expr annot, stmt_annot :: annot } - -- | Global declaration. - | Global - { global_vars :: [Ident annot] -- ^ Variables declared global in the current block. - , stmt_annot :: annot - } - -- | Nonlocal declaration. /Version 3.x only/. - | NonLocal - { nonLocal_vars :: [Ident annot] -- ^ Variables declared nonlocal in the current block (their binding comes from bound the nearest enclosing scope). - , stmt_annot :: annot - } - -- | Assertion. - | Assert - { assert_exprs :: [Expr annot] -- ^ Expressions being asserted. - , stmt_annot :: annot - } - -- | Print statement. /Version 2 only/. - | Print - { print_chevron :: Bool -- ^ Optional chevron (>>) - , print_exprs :: [Expr annot] -- ^ Arguments to print - , print_trailing_comma :: Bool -- ^ Does it end in a comma? - , stmt_annot :: annot - } - -- | Exec statement. /Version 2 only/. - | Exec - { exec_expr :: Expr annot -- ^ Expression to exec. - , exec_globals_locals :: Maybe (Expr annot, Maybe (Expr annot)) -- ^ Global and local environments to evaluate the expression within. - , stmt_annot :: annot - } - deriving (Eq,Ord,Show,Typeable,Data) - -type StatementSpan = Statement SrcSpan - -instance Span StatementSpan where - getSpan = annot - -instance Annotated Statement where - annot = stmt_annot - --- | The argument for a @raise@ statement. -data RaiseExpr annot - = RaiseV3 (Maybe (Expr annot, Maybe (Expr annot))) -- ^ Optional expression to evaluate, and optional \'from\' clause. /Version 3 only/. - | RaiseV2 (Maybe (Expr annot, (Maybe (Expr annot, Maybe (Expr annot))))) -- ^ /Version 2 only/. - deriving (Eq,Ord,Show,Typeable,Data) - -type RaiseExprSpan = RaiseExpr SrcSpan - --- | Decorator. -data Decorator annot = - Decorator - { decorator_name :: DottedName annot -- ^ Decorator name. - , decorator_args :: [Argument annot] -- ^ Decorator arguments. - , decorator_annot :: annot - } - deriving (Eq,Ord,Show,Typeable,Data) - -type DecoratorSpan = Decorator SrcSpan - -instance Span DecoratorSpan where - getSpan = annot - -instance Annotated Decorator where - annot = decorator_annot - --- | Formal parameter of function definitions and lambda expressions. --- --- * Version 2.6: --- --- * --- --- * --- --- * Version 3.1: --- --- * --- --- * --- -data Parameter annot - -- | Ordinary named parameter. - = Param - { param_name :: Ident annot -- ^ Parameter name. - , param_py_annotation :: Maybe (Expr annot) -- ^ Optional annotation. - , param_default :: Maybe (Expr annot) -- ^ Optional default value. - , param_annot :: annot - } - -- | Excess positional parameter (single asterisk before its name in the concrete syntax). - | VarArgsPos - { param_name :: Ident annot -- ^ Parameter name. - , param_py_annotation :: Maybe (Expr annot) -- ^ Optional annotation. - , param_annot :: annot - } - -- | Excess keyword parameter (double asterisk before its name in the concrete syntax). - | VarArgsKeyword - { param_name :: Ident annot -- ^ Parameter name. - , param_py_annotation :: Maybe (Expr annot) -- ^ Optional annotation. - , param_annot :: annot - } - -- | Marker for the end of positional parameters (not a parameter itself). - | EndPositional { param_annot :: annot } - -- | Tuple unpack. /Version 2 only/. - | UnPackTuple - { param_unpack_tuple :: ParamTuple annot -- ^ The tuple to unpack. - , param_default :: Maybe (Expr annot) -- ^ Optional default value. - , param_annot :: annot - } - deriving (Eq,Ord,Show,Typeable,Data) - -type ParameterSpan = Parameter SrcSpan - -instance Span ParameterSpan where - getSpan = annot - -instance Annotated Parameter where - annot = param_annot - --- | Tuple unpack parameter. /Version 2 only/. -data ParamTuple annot - = ParamTupleName { param_tuple_name :: Ident annot, param_tuple_annot :: annot } -- ^ A variable name. - | ParamTuple { param_tuple :: [ParamTuple annot], param_tuple_annot :: annot } -- ^ A (possibly nested) tuple parameter. - deriving (Eq,Ord,Show,Typeable,Data) - -type ParamTupleSpan = ParamTuple SrcSpan - -instance Span ParamTupleSpan where - getSpan = annot - -instance Annotated ParamTuple where - annot = param_tuple_annot - --- | Arguments to function calls, class declarations and decorators. -data Argument annot - -- | Ordinary argument expression. - = ArgExpr { arg_expr :: Expr annot, arg_annot :: annot } - -- | Excess positional argument. - | ArgVarArgsPos { arg_expr :: Expr annot, arg_annot :: annot } - -- | Excess keyword argument. - | ArgVarArgsKeyword { arg_expr :: Expr annot, arg_annot :: annot } - -- | Keyword argument. - | ArgKeyword - { arg_keyword :: Ident annot -- ^ Keyword name. - , arg_expr :: Expr annot -- ^ Argument expression. - , arg_annot :: annot - } - deriving (Eq,Ord,Show,Typeable,Data) - -type ArgumentSpan = Argument SrcSpan - -instance Span ArgumentSpan where - getSpan = annot - -instance Annotated Argument where - annot = arg_annot - --- | Exception handler. -data Handler annot - = Handler - { handler_clause :: ExceptClause annot - , handler_suite :: Suite annot - , handler_annot :: annot - } - deriving (Eq,Ord,Show,Typeable,Data) - -type HandlerSpan = Handler SrcSpan - -instance Span HandlerSpan where - getSpan = annot - -instance Annotated Handler where - annot = handler_annot - --- | Exception clause. -data ExceptClause annot - = ExceptClause - -- NB: difference with version 3 (has NAME as target, but looks like bug in grammar) - { except_clause :: Maybe (Expr annot, Maybe (Expr annot)) - , except_clause_annot :: annot - } - deriving (Eq,Ord,Show,Typeable,Data) - -type ExceptClauseSpan = ExceptClause SrcSpan - -instance Span ExceptClauseSpan where - getSpan = annot - -instance Annotated ExceptClause where - annot = except_clause_annot - --- | Comprehension. In version 3.x this can be used for lists, sets, dictionaries and generators. -data Comprehension e annot - = Comprehension - { comprehension_expr :: e - , comprehension_for :: CompFor annot - , comprehension_annot :: annot - } - deriving (Eq,Ord,Show,Typeable,Data) - -type ComprehensionSpan e = Comprehension e SrcSpan - -instance Span (ComprehensionSpan e) where - getSpan = annot - -instance Annotated (Comprehension e) where - annot = comprehension_annot - --- | Comprehension \'for\' component. -data CompFor annot = - CompFor - { comp_for_exprs :: [Expr annot] - , comp_in_expr :: Expr annot - , comp_for_iter :: Maybe (CompIter annot) - , comp_for_annot :: annot - } - deriving (Eq,Ord,Show,Typeable,Data) - -type CompForSpan = CompFor SrcSpan - -instance Span CompForSpan where - getSpan = annot - -instance Annotated CompFor where - annot = comp_for_annot - --- | Comprehension guard. -data CompIf annot = - CompIf - { comp_if :: Expr annot - , comp_if_iter :: Maybe (CompIter annot) - , comp_if_annot :: annot - } - deriving (Eq,Ord,Show,Typeable,Data) - -type CompIfSpan = CompIf SrcSpan - -instance Span CompIfSpan where - getSpan = annot - -instance Annotated CompIf where - annot = comp_if_annot - --- | Comprehension iterator (either a \'for\' or an \'if\'). -data CompIter annot - = IterFor { comp_iter_for :: CompFor annot, comp_iter_annot :: annot } - | IterIf { comp_iter_if :: CompIf annot, comp_iter_annot :: annot } - deriving (Eq,Ord,Show,Typeable,Data) - -type CompIterSpan = CompIter SrcSpan - -instance Span CompIterSpan where - getSpan = annot - -instance Annotated CompIter where - annot = comp_iter_annot - --- | Expressions. --- --- * Version 2.6 . --- --- * Version 3.1 . --- -data Expr annot - -- | Variable. - = Var { var_ident :: Ident annot, expr_annot :: annot } - -- | Literal integer. - | Int { int_value :: Integer, expr_literal :: String, expr_annot :: annot } - -- | Long literal integer. /Version 2 only/. - | LongInt { int_value :: Integer, expr_literal :: String, expr_annot :: annot } - -- | Literal floating point number. - | Float { float_value :: Double, expr_literal :: String, expr_annot :: annot } - -- | Literal imaginary number. - | Imaginary { imaginary_value :: Double, expr_literal :: String, expr_annot :: annot } - -- | Literal boolean. - | Bool { bool_value :: Bool, expr_annot :: annot } - -- | Literal \'None\' value. - | None { expr_annot :: annot } - -- | Ellipsis \'...\'. - | Ellipsis { expr_annot :: annot } - -- | Literal byte string. - | ByteStrings { byte_string_strings :: [String], expr_annot :: annot } - -- | Literal strings (to be concatentated together). - | Strings { strings_strings :: [String], expr_annot :: annot } - -- | Unicode literal strings (to be concatentated together). Version 2 only. - | UnicodeStrings { unicodestrings_strings :: [String], expr_annot :: annot } - -- | Function call. - | Call - { call_fun :: Expr annot -- ^ Expression yielding a callable object (such as a function). - , call_args :: [Argument annot] -- ^ Call arguments. - , expr_annot :: annot - } - -- | Subscription, for example \'x [y]\'. - | Subscript { subscriptee :: Expr annot, subscript_expr :: Expr annot, expr_annot :: annot } - -- | Slicing, for example \'w [x:y:z]\'. - | SlicedExpr { slicee :: Expr annot, slices :: [Slice annot], expr_annot :: annot } - -- | Conditional expresison. - | CondExpr - { ce_true_branch :: Expr annot -- ^ Expression to evaluate if condition is True. - , ce_condition :: Expr annot -- ^ Boolean condition. - , ce_false_branch :: Expr annot -- ^ Expression to evaluate if condition is False. - , expr_annot :: annot - } - -- | Binary operator application. - | BinaryOp { operator :: Op annot, left_op_arg :: Expr annot, right_op_arg :: Expr annot, expr_annot :: annot } - -- | Unary operator application. - | UnaryOp { operator :: Op annot, op_arg :: Expr annot, expr_annot :: annot } - -- | Anonymous function definition (lambda). - | Lambda { lambda_args :: [Parameter annot], lambda_body :: Expr annot, expr_annot :: annot } - -- | Tuple. Can be empty. - | Tuple { tuple_exprs :: [Expr annot], expr_annot :: annot } - -- | Generator yield. - | Yield - { yield_expr :: Maybe (Expr annot) -- ^ Optional expression to yield. - , expr_annot :: annot - } - -- | Generator. - | Generator { gen_comprehension :: Comprehension (Expr annot) annot, expr_annot :: annot } - -- | List comprehension. - | ListComp { list_comprehension :: Comprehension (Expr annot) annot, expr_annot :: annot } - -- | List. - | List { list_exprs :: [Expr annot], expr_annot :: annot } - -- | Dictionary. - | Dictionary { dict_mappings :: [(Expr annot, Expr annot)], expr_annot :: annot } - -- | Dictionary comprehension. /Version 3 only/. - | DictComp { dict_comprehension :: Comprehension (Expr annot, Expr annot) annot, expr_annot :: annot } - -- | Set. - | Set { set_exprs :: [Expr annot], expr_annot :: annot } - -- | Set comprehension. /Version 3 only/. - | SetComp { set_comprehension :: Comprehension (Expr annot) annot, expr_annot :: annot } - -- | Starred expression. /Version 3 only/. - | Starred { starred_expr :: Expr annot, expr_annot :: annot } - -- | Parenthesised expression. - | Paren { paren_expr :: Expr annot, expr_annot :: annot } - -- | String conversion (backquoted expression). Version 2 only. - | StringConversion { backquoted_expr :: Expr annot, expr_anot :: annot } - deriving (Eq,Ord,Show,Typeable,Data) - -type ExprSpan = Expr SrcSpan - -instance Span ExprSpan where - getSpan = annot - -instance Annotated Expr where - annot = expr_annot - --- | Slice compenent. -data Slice annot - = SliceProper - { slice_lower :: Maybe (Expr annot) - , slice_upper :: Maybe (Expr annot) - , slice_stride :: Maybe (Maybe (Expr annot)) - , slice_annot :: annot - } - | SliceExpr - { slice_expr :: Expr annot - , slice_annot :: annot - } - | SliceEllipsis { slice_annot :: annot } - deriving (Eq,Ord,Show,Typeable,Data) - -type SliceSpan = Slice SrcSpan - -instance Span SliceSpan where - getSpan = annot - -instance Annotated Slice where - annot = slice_annot - --- | Operators. -data Op annot - = And { op_annot :: annot } -- ^ \'and\' - | Or { op_annot :: annot } -- ^ \'or\' - | Not { op_annot :: annot } -- ^ \'not\' - | Exponent { op_annot :: annot } -- ^ \'**\' - | LessThan { op_annot :: annot } -- ^ \'<\' - | GreaterThan { op_annot :: annot } -- ^ \'>\' - | Equality { op_annot :: annot } -- ^ \'==\' - | GreaterThanEquals { op_annot :: annot } -- ^ \'>=\' - | LessThanEquals { op_annot :: annot } -- ^ \'<=\' - | NotEquals { op_annot :: annot } -- ^ \'!=\' - | NotEqualsV2 { op_annot :: annot } -- ^ \'<>\'. Version 2 only. - | In { op_annot :: annot } -- ^ \'in\' - | Is { op_annot :: annot } -- ^ \'is\' - | IsNot { op_annot :: annot } -- ^ \'is not\' - | NotIn { op_annot :: annot } -- ^ \'not in\' - | BinaryOr { op_annot :: annot } -- ^ \'|\' - | Xor { op_annot :: annot } -- ^ \'^\' - | BinaryAnd { op_annot :: annot } -- ^ \'&\' - | ShiftLeft { op_annot :: annot } -- ^ \'<<\' - | ShiftRight { op_annot :: annot } -- ^ \'>>\' - | Multiply { op_annot :: annot } -- ^ \'*\' - | Plus { op_annot :: annot } -- ^ \'+\' - | Minus { op_annot :: annot } -- ^ \'-\' - | Divide { op_annot :: annot } -- ^ \'\/\' - | FloorDivide { op_annot :: annot } -- ^ \'\/\/\' - | Invert { op_annot :: annot } -- ^ \'~\' (bitwise inversion of its integer argument) - | Modulo { op_annot :: annot } -- ^ \'%\' - | Dot { op_annot :: annot } -- ^ \'.\' - deriving (Eq,Ord,Show,Typeable,Data) - -type OpSpan = Op SrcSpan - -instance Span OpSpan where - getSpan = annot - -instance Annotated Op where - annot = op_annot - --- | Augmented assignment operators. -data AssignOp annot - = PlusAssign { assignOp_annot :: annot } -- ^ \'+=\' - | MinusAssign { assignOp_annot :: annot } -- ^ \'-=\' - | MultAssign { assignOp_annot :: annot } -- ^ \'*=\' - | DivAssign { assignOp_annot :: annot } -- ^ \'\/=\' - | ModAssign { assignOp_annot :: annot } -- ^ \'%=\' - | PowAssign { assignOp_annot :: annot } -- ^ \'*=\' - | BinAndAssign { assignOp_annot :: annot } -- ^ \'&=\' - | BinOrAssign { assignOp_annot :: annot } -- ^ \'|=\' - | BinXorAssign { assignOp_annot :: annot } -- ^ \'^=\' - | LeftShiftAssign { assignOp_annot :: annot } -- ^ \'<<=\' - | RightShiftAssign { assignOp_annot :: annot } -- ^ \'>>=\' - | FloorDivAssign { assignOp_annot :: annot } -- ^ \'\/\/=\' - deriving (Eq,Ord,Show,Typeable,Data) - -type AssignOpSpan = AssignOp SrcSpan - -instance Span AssignOpSpan where - getSpan = annot - -instance Annotated AssignOp where - annot = assignOp_annot diff --git a/language-python.cabal b/language-python.cabal --- a/language-python.cabal +++ b/language-python.cabal @@ -1,43 +1,41 @@ name: language-python -version: 0.4.1 -cabal-version: >= 1.6 +version: 0.5.8 +cabal-version: >= 1.10 synopsis: Parsing and pretty printing of Python code. description: language-python is a Haskell library for lexical analysis, parsing and pretty printing Python code. It supports versions 2.x and 3.x of Python. category: Language license: BSD3 license-file: LICENSE -copyright: (c) 2008-2014 Bernard James Pope +copyright: (c) 2008-2019 Bernard James Pope author: Bernard James Pope (Bernie Pope) maintainer: florbitous@gmail.com homepage: http://github.com/bjpop/language-python -build-depends: base build-type: Simple stability: experimental -extra-source-files: src/Language/Python/Version3/Parser/Parser.y - src/Language/Python/Version3/Parser/Lexer.x - ghc_6_12-13/Language/Python/Common/AST.hs - ghc_normal/Language/Python/Common/AST.hs +extra-source-files: src/Language/Python/Version3/Parser/Parser.y + src/Language/Python/Version3/Parser/Lexer.x + src/Language/Python/Version2/Parser/Parser.y + src/Language/Python/Version2/Parser/Lexer.x +tested-with: GHC ==7.8.4, GHC ==7.10.3, GHC ==8.0.2, GHC ==8.2.1, GHC ==8.6.3 source-repository head type: git location: https://github.com/bjpop/language-python Library - -- work around the memory usage bug in ghc 6.12.x - if impl(ghc < 6.12) || impl(ghc >= 6.14) - hs-source-dirs: src ghc_normal - else - hs-source-dirs: src ghc_6_12-13 - + default-language: Haskell2010 + hs-source-dirs: src + ghc-options: -fwarn-incomplete-patterns -fwarn-unused-imports -fwarn-warnings-deprecations build-depends: base == 4.*, - containers == 0.5.*, + containers >= 0.5 && < 0.7, pretty == 1.1.*, array >= 0.4 && < 0.6, - transformers >= 0.3 && < 0.5, - monads-tf == 0.1.* - build-tools: happy, alex + transformers >= 0.3 && < 0.6, + monads-tf == 0.1.*, + utf8-string >= 1 && < 2 + build-tools: happy, alex exposed-modules: Language.Python.Common Language.Python.Common.ParseError diff --git a/src/Language/Python/Common.hs b/src/Language/Python/Common.hs --- a/src/Language/Python/Common.hs +++ b/src/Language/Python/Common.hs @@ -33,8 +33,8 @@ import Language.Python.Common.Pretty import Language.Python.Common.Token import Language.Python.Common.AST -import Language.Python.Common.PrettyAST -import Language.Python.Common.PrettyToken +import Language.Python.Common.PrettyAST () +import Language.Python.Common.PrettyToken () import Language.Python.Common.SrcLocation -import Language.Python.Common.PrettyParseError +import Language.Python.Common.PrettyParseError () import Language.Python.Common.ParseError diff --git a/src/Language/Python/Common/AST.hs b/src/Language/Python/Common/AST.hs new file mode 100644 --- /dev/null +++ b/src/Language/Python/Common/AST.hs @@ -0,0 +1,795 @@ +{-# LANGUAGE FlexibleInstances, TypeSynonymInstances, CPP, DeriveDataTypeable, DeriveFunctor #-} +----------------------------------------------------------------------------- +-- | +-- Module : Language.Python.Version2.Syntax.AST +-- Copyright : (c) 2009 Bernie Pope +-- License : BSD-style +-- Maintainer : bjpop@csse.unimelb.edu.au +-- Stability : experimental +-- Portability : ghc +-- +-- Representation of the Python abstract syntax tree (AST). The representation is +-- a superset of versions 2.x and 3.x of Python. In many cases they are +-- identical. The documentation in this module indicates where they are +-- different. +-- +-- All the data types have a (polymorphic) parameter which allows the AST to +-- be annotated by an arbitrary type (for example source locations). Specialised +-- instances of the types are provided for source spans. For example @Module a@ is +-- the type of modules, and @ModuleSpan@ is the type of modules annoted with source +-- span information. +-- +-- Note: there are cases where the AST is more liberal than the formal grammar +-- of the language. Therefore some care must be taken when constructing +-- Python programs using the raw AST. +----------------------------------------------------------------------------- + +module Language.Python.Common.AST ( + -- * Annotation projection + Annotated (..) + -- * Modules + , Module (..), ModuleSpan + -- * Identifiers and dotted names + , Ident (..), IdentSpan + , DottedName, DottedNameSpan + -- * Statements, suites, parameters, decorators and assignment operators + , Statement (..), StatementSpan + , Suite, SuiteSpan + , Parameter (..), ParameterSpan + , ParamTuple (..), ParamTupleSpan + , Decorator (..), DecoratorSpan + , AssignOp (..), AssignOpSpan + -- * Expressions, operators, arguments and slices + , Expr (..), ExprSpan + , Op (..), OpSpan + , Argument (..), ArgumentSpan + , Slice (..), SliceSpan + , DictKeyDatumList (..), DictKeyDatumListSpan + , YieldArg (..), YieldArgSpan + -- * Imports + , ImportItem (..), ImportItemSpan + , FromItem (..), FromItemSpan + , FromItems (..), FromItemsSpan + , ImportRelative (..), ImportRelativeSpan + -- * Exceptions + , Handler (..), HandlerSpan + , ExceptClause (..), ExceptClauseSpan + , RaiseExpr (..), RaiseExprSpan + -- * Comprehensions + , Comprehension (..), ComprehensionSpan + , ComprehensionExpr (..), ComprehensionExprSpan + , CompFor (..), CompForSpan + , CompIf (..), CompIfSpan + , CompIter (..), CompIterSpan + ) + where + +import Language.Python.Common.SrcLocation ( Span (getSpan), SrcSpan (..), spanning ) +import Data.Data + +-------------------------------------------------------------------------------- + +-- | Convenient access to annotations in annotated types. +class Annotated t where + -- | Given an annotated type, project out its annotation value. + annot :: t annot -> annot + +-- | Identifier. +data Ident annot = Ident { ident_string :: !String, ident_annot :: annot } + deriving (Eq,Ord,Show,Typeable,Data,Functor) + +type IdentSpan = Ident SrcSpan + +instance Span IdentSpan where + getSpan = annot + +instance Annotated Ident where + annot = ident_annot + +-- | A module (Python source file). +-- +-- * Version 2.6 +-- +-- * Version 3.1 +-- +newtype Module annot = Module [Statement annot] -- ^ A module is just a sequence of top-level statements. + deriving (Eq,Ord,Show,Typeable,Data,Functor) + +type ModuleSpan = Module SrcSpan + +-- | A block of statements. A suite is a group of statements controlled by a clause, +-- for example, the body of a loop. +-- +-- * Version 2.6 +-- +-- * Version 3.1 +-- +type Suite annot = [Statement annot] + +type SuiteSpan = Suite SrcSpan + +-- | A compound name constructed with the dot operator. +type DottedName annot = [Ident annot] + +type DottedNameSpan = DottedName SrcSpan + +-- | An entity imported using the \'import\' keyword. +-- +-- * Version 2.6 +-- +-- * Version 3.1 +-- +data ImportItem annot = + ImportItem + { import_item_name :: DottedName annot -- ^ The name of module to import. + , import_as_name :: Maybe (Ident annot) -- ^ An optional name to refer to the entity (the \'as\' name). + , import_item_annot :: annot + } + deriving (Eq,Ord,Show,Typeable,Data,Functor) + +type ImportItemSpan = ImportItem SrcSpan + +instance Span ImportItemSpan where + getSpan = annot + +instance Annotated ImportItem where + annot = import_item_annot + +-- | An entity imported using the \'from ... import\' construct. +-- +-- * Version 2.6 +-- +-- * Version 3.1 +-- +data FromItem annot = + FromItem + { from_item_name :: Ident annot -- ^ The name of the entity imported. + , from_as_name :: Maybe (Ident annot) -- ^ An optional name to refer to the entity (the \'as\' name). + , from_item_annot :: annot + } + deriving (Eq,Ord,Show,Typeable,Data,Functor) + +type FromItemSpan = FromItem SrcSpan + +instance Span FromItemSpan where + getSpan = annot + +instance Annotated FromItem where + annot = from_item_annot + +-- | Items imported using the \'from ... import\' construct. +data FromItems annot + = ImportEverything { from_items_annot :: annot } -- ^ Import everything exported from the module. + | FromItems { from_items_items :: [FromItem annot], from_items_annot :: annot } -- ^ Import a specific list of items from the module. + deriving (Eq,Ord,Show,Typeable,Data,Functor) + +type FromItemsSpan = FromItems SrcSpan + +instance Span FromItemsSpan where + getSpan = annot + +instance Annotated FromItems where + annot = from_items_annot + +-- | A reference to the module to import from using the \'from ... import\' construct. +data ImportRelative annot + = ImportRelative + { import_relative_dots :: Int + , import_relative_module :: Maybe (DottedName annot) + , import_relative_annot :: annot + } + deriving (Eq,Ord,Show,Typeable,Data,Functor) + +type ImportRelativeSpan = ImportRelative SrcSpan + +instance Span ImportRelativeSpan where + getSpan = annot + +instance Annotated ImportRelative where + annot = import_relative_annot + +-- | Statements. +-- +-- * Simple statements: +-- +-- * Version 2.6 +-- +-- * Version 3.1 +-- +-- * Compound statements: +-- +-- * Version 2.6 +-- +-- * Version 3.1 +-- +data Statement annot + -- | Import statement. + = Import + { import_items :: [ImportItem annot] -- ^ Items to import. + , stmt_annot :: annot + } + -- | From ... import statement. + | FromImport + { from_module :: ImportRelative annot -- ^ Module to import from. + , from_items :: FromItems annot -- ^ Items to import. + , stmt_annot :: annot + } + -- | While loop. + | While + { while_cond :: Expr annot -- ^ Loop condition. + , while_body :: Suite annot -- ^ Loop body. + , while_else :: Suite annot -- ^ Else clause. + , stmt_annot :: annot + } + -- | For loop. + | For + { for_targets :: [Expr annot] -- ^ Loop variables. + , for_generator :: Expr annot -- ^ Loop generator. + , for_body :: Suite annot -- ^ Loop body + , for_else :: Suite annot -- ^ Else clause. + , stmt_annot :: annot + } + | AsyncFor + { for_stmt :: Statement annot -- ^ For statement + , stmt_annot :: annot + } + -- | Function definition. + | Fun + { fun_name :: Ident annot -- ^ Function name. + , fun_args :: [Parameter annot] -- ^ Function parameter list. + , fun_result_annotation :: Maybe (Expr annot) -- ^ Optional result annotation. + , fun_body :: Suite annot -- ^ Function body. + , stmt_annot :: annot + } + | AsyncFun + { fun_def :: Statement annot -- ^ Function definition (Fun) + , stmt_annot :: annot + } + -- | Class definition. + | Class + { class_name :: Ident annot -- ^ Class name. + , class_args :: [Argument annot] -- ^ Class argument list. In version 2.x this is only ArgExprs. + , class_body :: Suite annot -- ^ Class body. + , stmt_annot :: annot + } + -- | Conditional statement (if-elif-else). + | Conditional + { cond_guards :: [(Expr annot, Suite annot)] -- ^ Sequence of if-elif conditional clauses. + , cond_else :: Suite annot -- ^ Possibly empty unconditional else clause. + , stmt_annot :: annot + } + -- | Assignment statement. + | Assign + { assign_to :: [Expr annot] -- ^ Entity to assign to. + , assign_expr :: Expr annot -- ^ Expression to evaluate. + , stmt_annot :: annot + } + -- | Augmented assignment statement. + | AugmentedAssign + { aug_assign_to :: Expr annot -- ^ Entity to assign to. + , aug_assign_op :: AssignOp annot -- ^ Assignment operator (for example \'+=\'). + , aug_assign_expr :: Expr annot -- ^ Expression to evaluate. + , stmt_annot :: annot + } + | AnnotatedAssign + { ann_assign_annotation :: Expr annot + , ann_assign_to :: Expr annot + , ann_assign_expr :: Maybe (Expr annot) + , stmt_annot :: annot + } + -- | Decorated definition of a function or class. + | Decorated + { decorated_decorators :: [Decorator annot] -- ^ Decorators. + , decorated_def :: Statement annot -- ^ Function or class definition to be decorated. + , stmt_annot :: annot + } + -- | Return statement (may only occur syntactically nested in a function definition). + | Return + { return_expr :: Maybe (Expr annot) -- ^ Optional expression to evaluate and return to caller. + , stmt_annot :: annot + } + -- | Try statement (exception handling). + | Try + { try_body :: Suite annot -- ^ Try clause. + , try_excepts :: [Handler annot] -- ^ Exception handlers. + , try_else :: Suite annot -- ^ Possibly empty else clause, executed if and when control flows off the end of the try clause. + , try_finally :: Suite annot -- ^ Possibly empty finally clause. + , stmt_annot :: annot + } + -- | Raise statement (exception throwing). + | Raise + { raise_expr :: RaiseExpr annot + , stmt_annot :: annot + } + -- | With statement (context management). + | With + { with_context :: [(Expr annot, Maybe (Expr annot))] -- ^ Context expression(s) (yields a context manager). + , with_body :: Suite annot -- ^ Suite to be managed. + , stmt_annot :: annot + } + | AsyncWith + { with_stmt :: Statement annot -- ^ With statement + , stmt_annot :: annot + } + -- | Pass statement (null operation). + | Pass { stmt_annot :: annot } + -- | Break statement (may only occur syntactically nested in a for or while loop, but not nested in a function or class definition within that loop). + | Break { stmt_annot :: annot } + -- | Continue statement (may only occur syntactically nested in a for or while loop, but not nested in a function or class definition or finally clause within that loop). + | Continue { stmt_annot :: annot } + -- | Del statement (delete). + | Delete + { del_exprs :: [Expr annot] -- ^ Items to delete. + , stmt_annot :: annot + } + -- | Expression statement. + | StmtExpr { stmt_expr :: Expr annot, stmt_annot :: annot } + -- | Global declaration. + | Global + { global_vars :: [Ident annot] -- ^ Variables declared global in the current block. + , stmt_annot :: annot + } + -- | Nonlocal declaration. /Version 3.x only/. + | NonLocal + { nonLocal_vars :: [Ident annot] -- ^ Variables declared nonlocal in the current block (their binding comes from bound the nearest enclosing scope). + , stmt_annot :: annot + } + -- | Assertion. + | Assert + { assert_exprs :: [Expr annot] -- ^ Expressions being asserted. + , stmt_annot :: annot + } + -- | Print statement. /Version 2 only/. + | Print + { print_chevron :: Bool -- ^ Optional chevron (>>) + , print_exprs :: [Expr annot] -- ^ Arguments to print + , print_trailing_comma :: Bool -- ^ Does it end in a comma? + , stmt_annot :: annot + } + -- | Exec statement. /Version 2 only/. + | Exec + { exec_expr :: Expr annot -- ^ Expression to exec. + , exec_globals_locals :: Maybe (Expr annot, Maybe (Expr annot)) -- ^ Global and local environments to evaluate the expression within. + , stmt_annot :: annot + } + deriving (Eq,Ord,Show,Typeable,Data,Functor) + +type StatementSpan = Statement SrcSpan + +instance Span StatementSpan where + getSpan = annot + +instance Annotated Statement where + annot = stmt_annot + +-- | The argument for a @raise@ statement. +data RaiseExpr annot + = RaiseV3 (Maybe (Expr annot, Maybe (Expr annot))) -- ^ Optional expression to evaluate, and optional \'from\' clause. /Version 3 only/. + | RaiseV2 (Maybe (Expr annot, (Maybe (Expr annot, Maybe (Expr annot))))) -- ^ /Version 2 only/. + deriving (Eq,Ord,Show,Typeable,Data,Functor) + +type RaiseExprSpan = RaiseExpr SrcSpan + +-- | Decorator. +data Decorator annot = + Decorator + { decorator_name :: DottedName annot -- ^ Decorator name. + , decorator_args :: [Argument annot] -- ^ Decorator arguments. + , decorator_annot :: annot + } + deriving (Eq,Ord,Show,Typeable,Data,Functor) + +type DecoratorSpan = Decorator SrcSpan + +instance Span DecoratorSpan where + getSpan = annot + +instance Annotated Decorator where + annot = decorator_annot + +-- | Formal parameter of function definitions and lambda expressions. +-- +-- * Version 2.6: +-- +-- * +-- +-- * +-- +-- * Version 3.1: +-- +-- * +-- +-- * +-- +data Parameter annot + -- | Ordinary named parameter. + = Param + { param_name :: Ident annot -- ^ Parameter name. + , param_py_annotation :: Maybe (Expr annot) -- ^ Optional annotation. + , param_default :: Maybe (Expr annot) -- ^ Optional default value. + , param_annot :: annot + } + -- | Excess positional parameter (single asterisk before its name in the concrete syntax). + | VarArgsPos + { param_name :: Ident annot -- ^ Parameter name. + , param_py_annotation :: Maybe (Expr annot) -- ^ Optional annotation. + , param_annot :: annot + } + -- | Excess keyword parameter (double asterisk before its name in the concrete syntax). + | VarArgsKeyword + { param_name :: Ident annot -- ^ Parameter name. + , param_py_annotation :: Maybe (Expr annot) -- ^ Optional annotation. + , param_annot :: annot + } + -- | Marker for the end of positional parameters (not a parameter itself). + | EndPositional { param_annot :: annot } + -- | Tuple unpack. /Version 2 only/. + | UnPackTuple + { param_unpack_tuple :: ParamTuple annot -- ^ The tuple to unpack. + , param_default :: Maybe (Expr annot) -- ^ Optional default value. + , param_annot :: annot + } + deriving (Eq,Ord,Show,Typeable,Data,Functor) + +type ParameterSpan = Parameter SrcSpan + +instance Span ParameterSpan where + getSpan = annot + +instance Annotated Parameter where + annot = param_annot + +-- | Tuple unpack parameter. /Version 2 only/. +data ParamTuple annot + = ParamTupleName { param_tuple_name :: Ident annot, param_tuple_annot :: annot } -- ^ A variable name. + | ParamTuple { param_tuple :: [ParamTuple annot], param_tuple_annot :: annot } -- ^ A (possibly nested) tuple parameter. + deriving (Eq,Ord,Show,Typeable,Data,Functor) + +type ParamTupleSpan = ParamTuple SrcSpan + +instance Span ParamTupleSpan where + getSpan = annot + +instance Annotated ParamTuple where + annot = param_tuple_annot + +-- | Arguments to function calls, class declarations and decorators. +data Argument annot + -- | Ordinary argument expression. + = ArgExpr { arg_expr :: Expr annot, arg_annot :: annot } + -- | Excess positional argument. + | ArgVarArgsPos { arg_expr :: Expr annot, arg_annot :: annot } + -- | Excess keyword argument. + | ArgVarArgsKeyword { arg_expr :: Expr annot, arg_annot :: annot } + -- | Keyword argument. + | ArgKeyword + { arg_keyword :: Ident annot -- ^ Keyword name. + , arg_expr :: Expr annot -- ^ Argument expression. + , arg_annot :: annot + } + deriving (Eq,Ord,Show,Typeable,Data,Functor) + +type ArgumentSpan = Argument SrcSpan + +instance Span ArgumentSpan where + getSpan = annot + +instance Annotated Argument where + annot = arg_annot + +-- | Exception handler. +data Handler annot + = Handler + { handler_clause :: ExceptClause annot + , handler_suite :: Suite annot + , handler_annot :: annot + } + deriving (Eq,Ord,Show,Typeable,Data,Functor) + +type HandlerSpan = Handler SrcSpan + +instance Span HandlerSpan where + getSpan = annot + +instance Annotated Handler where + annot = handler_annot + +-- | Exception clause. +data ExceptClause annot + = ExceptClause + -- NB: difference with version 3 (has NAME as target, but looks like bug in grammar) + { except_clause :: Maybe (Expr annot, Maybe (Expr annot)) + , except_clause_annot :: annot + } + deriving (Eq,Ord,Show,Typeable,Data,Functor) + +type ExceptClauseSpan = ExceptClause SrcSpan + +instance Span ExceptClauseSpan where + getSpan = annot + +instance Annotated ExceptClause where + annot = except_clause_annot + +-- | Comprehension. In version 3.x this can be used for lists, sets, dictionaries and generators. +-- data Comprehension e annot +data Comprehension annot + = Comprehension + { comprehension_expr :: ComprehensionExpr annot + , comprehension_for :: CompFor annot + , comprehension_annot :: annot + } + deriving (Eq,Ord,Show,Typeable,Data,Functor) + +type ComprehensionSpan = Comprehension SrcSpan + +instance Span ComprehensionSpan where + getSpan = annot + +instance Annotated Comprehension where + annot = comprehension_annot + +data ComprehensionExpr annot + = ComprehensionExpr (Expr annot) + | ComprehensionDict (DictKeyDatumList annot) + deriving (Eq,Ord,Show,Typeable,Data,Functor) + +type ComprehensionExprSpan = ComprehensionExpr SrcSpan + +instance Span ComprehensionExprSpan where + getSpan (ComprehensionExpr e) = getSpan e + getSpan (ComprehensionDict d) = getSpan d + +-- | Comprehension \'for\' component. +data CompFor annot = + CompFor + { comp_for_async :: Bool + , comp_for_exprs :: [Expr annot] + , comp_in_expr :: Expr annot + , comp_for_iter :: Maybe (CompIter annot) + , comp_for_annot :: annot + } + deriving (Eq,Ord,Show,Typeable,Data,Functor) + +type CompForSpan = CompFor SrcSpan + +instance Span CompForSpan where + getSpan = annot + +instance Annotated CompFor where + annot = comp_for_annot + +-- | Comprehension guard. +data CompIf annot = + CompIf + { comp_if :: Expr annot + , comp_if_iter :: Maybe (CompIter annot) + , comp_if_annot :: annot + } + deriving (Eq,Ord,Show,Typeable,Data,Functor) + +type CompIfSpan = CompIf SrcSpan + +instance Span CompIfSpan where + getSpan = annot + +instance Annotated CompIf where + annot = comp_if_annot + +-- | Comprehension iterator (either a \'for\' or an \'if\'). +data CompIter annot + = IterFor { comp_iter_for :: CompFor annot, comp_iter_annot :: annot } + | IterIf { comp_iter_if :: CompIf annot, comp_iter_annot :: annot } + deriving (Eq,Ord,Show,Typeable,Data,Functor) + +type CompIterSpan = CompIter SrcSpan + +instance Span CompIterSpan where + getSpan = annot + +instance Annotated CompIter where + annot = comp_iter_annot + +-- | Expressions. +-- +-- * Version 2.6 . +-- +-- * Version 3.1 . +-- +data Expr annot + -- | Variable. + = Var { var_ident :: Ident annot, expr_annot :: annot } + -- | Literal integer. + | Int { int_value :: Integer, expr_literal :: String, expr_annot :: annot } + -- | Long literal integer. /Version 2 only/. + | LongInt { int_value :: Integer, expr_literal :: String, expr_annot :: annot } + -- | Literal floating point number. + | Float { float_value :: Double, expr_literal :: String, expr_annot :: annot } + -- | Literal imaginary number. + | Imaginary { imaginary_value :: Double, expr_literal :: String, expr_annot :: annot } + -- | Literal boolean. + | Bool { bool_value :: Bool, expr_annot :: annot } + -- | Literal \'None\' value. + | None { expr_annot :: annot } + -- | Ellipsis \'...\'. + | Ellipsis { expr_annot :: annot } + -- | Literal byte string. + | ByteStrings { byte_string_strings :: [String], expr_annot :: annot } + -- | Literal strings (to be concatentated together). + | Strings { strings_strings :: [String], expr_annot :: annot } + -- | Unicode literal strings (to be concatentated together). Version 2 only. + | UnicodeStrings { unicodestrings_strings :: [String], expr_annot :: annot } + -- | Function call. + | Call + { call_fun :: Expr annot -- ^ Expression yielding a callable object (such as a function). + , call_args :: [Argument annot] -- ^ Call arguments. + , expr_annot :: annot + } + -- | Subscription, for example \'x [y]\'. + | Subscript { subscriptee :: Expr annot, subscript_expr :: Expr annot, expr_annot :: annot } + -- | Slicing, for example \'w [x:y:z]\'. + | SlicedExpr { slicee :: Expr annot, slices :: [Slice annot], expr_annot :: annot } + -- | Conditional expresison. + | CondExpr + { ce_true_branch :: Expr annot -- ^ Expression to evaluate if condition is True. + , ce_condition :: Expr annot -- ^ Boolean condition. + , ce_false_branch :: Expr annot -- ^ Expression to evaluate if condition is False. + , expr_annot :: annot + } + -- | Binary operator application. + | BinaryOp { operator :: Op annot, left_op_arg :: Expr annot, right_op_arg :: Expr annot, expr_annot :: annot } + -- | Unary operator application. + | UnaryOp { operator :: Op annot, op_arg :: Expr annot, expr_annot :: annot } + -- Dot operator (attribute selection) + | Dot { dot_expr :: Expr annot, dot_attribute :: Ident annot, expr_annot :: annot } + -- | Anonymous function definition (lambda). + | Lambda { lambda_args :: [Parameter annot], lambda_body :: Expr annot, expr_annot :: annot } + -- | Tuple. Can be empty. + | Tuple { tuple_exprs :: [Expr annot], expr_annot :: annot } + -- | Generator yield. + | Yield + -- { yield_expr :: Maybe (Expr annot) -- ^ Optional expression to yield. + { yield_arg :: Maybe (YieldArg annot) -- ^ Optional Yield argument. + , expr_annot :: annot + } + -- | Generator. + | Generator { gen_comprehension :: Comprehension annot, expr_annot :: annot } + -- | Await + | Await { await_expr :: Expr annot, expr_annot :: annot } + -- | List comprehension. + | ListComp { list_comprehension :: Comprehension annot, expr_annot :: annot } + -- | List. + | List { list_exprs :: [Expr annot], expr_annot :: annot } + -- | Dictionary. + | Dictionary { dict_mappings :: [DictKeyDatumList annot], expr_annot :: annot } + -- | Dictionary comprehension. /Version 3 only/. + | DictComp { dict_comprehension :: Comprehension annot, expr_annot :: annot } + -- | Set. + | Set { set_exprs :: [Expr annot], expr_annot :: annot } + -- | Set comprehension. /Version 3 only/. + | SetComp { set_comprehension :: Comprehension annot, expr_annot :: annot } + -- | Starred expression. /Version 3 only/. + | Starred { starred_expr :: Expr annot, expr_annot :: annot } + -- | Parenthesised expression. + | Paren { paren_expr :: Expr annot, expr_annot :: annot } + -- | String conversion (backquoted expression). Version 2 only. + | StringConversion { backquoted_expr :: Expr annot, expr_anot :: annot } + deriving (Eq,Ord,Show,Typeable,Data,Functor) + +type ExprSpan = Expr SrcSpan + +instance Span ExprSpan where + getSpan = annot + +data YieldArg annot + = YieldFrom (Expr annot) annot -- ^ Yield from a generator (Version 3 only) + | YieldExpr (Expr annot) -- ^ Yield value of an expression + deriving (Eq,Ord,Show,Typeable,Data,Functor) + +type YieldArgSpan = YieldArg SrcSpan + +instance Span YieldArgSpan where + getSpan (YieldFrom _e span) = span + getSpan (YieldExpr e) = getSpan e + +instance Annotated Expr where + annot = expr_annot + +data DictKeyDatumList annot = + DictMappingPair (Expr annot) (Expr annot) + | DictUnpacking (Expr annot) + deriving (Eq,Ord,Show,Typeable,Data,Functor) + +type DictKeyDatumListSpan = DictKeyDatumList SrcSpan + +instance Span DictKeyDatumListSpan where + getSpan (DictMappingPair e1 e2) = spanning e1 e2 + getSpan (DictUnpacking e) = getSpan e + +-- | Slice compenent. +data Slice annot + = SliceProper + { slice_lower :: Maybe (Expr annot) + , slice_upper :: Maybe (Expr annot) + , slice_stride :: Maybe (Maybe (Expr annot)) + , slice_annot :: annot + } + | SliceExpr + { slice_expr :: Expr annot + , slice_annot :: annot + } + | SliceEllipsis { slice_annot :: annot } + deriving (Eq,Ord,Show,Typeable,Data,Functor) + +type SliceSpan = Slice SrcSpan + +instance Span SliceSpan where + getSpan = annot + +instance Annotated Slice where + annot = slice_annot + +-- | Operators. +data Op annot + = And { op_annot :: annot } -- ^ \'and\' + | Or { op_annot :: annot } -- ^ \'or\' + | Not { op_annot :: annot } -- ^ \'not\' + | Exponent { op_annot :: annot } -- ^ \'**\' + | LessThan { op_annot :: annot } -- ^ \'<\' + | GreaterThan { op_annot :: annot } -- ^ \'>\' + | Equality { op_annot :: annot } -- ^ \'==\' + | GreaterThanEquals { op_annot :: annot } -- ^ \'>=\' + | LessThanEquals { op_annot :: annot } -- ^ \'<=\' + | NotEquals { op_annot :: annot } -- ^ \'!=\' + | NotEqualsV2 { op_annot :: annot } -- ^ \'<>\'. Version 2 only. + | In { op_annot :: annot } -- ^ \'in\' + | Is { op_annot :: annot } -- ^ \'is\' + | IsNot { op_annot :: annot } -- ^ \'is not\' + | NotIn { op_annot :: annot } -- ^ \'not in\' + | BinaryOr { op_annot :: annot } -- ^ \'|\' + | Xor { op_annot :: annot } -- ^ \'^\' + | BinaryAnd { op_annot :: annot } -- ^ \'&\' + | ShiftLeft { op_annot :: annot } -- ^ \'<<\' + | ShiftRight { op_annot :: annot } -- ^ \'>>\' + | Multiply { op_annot :: annot } -- ^ \'*\' + | Plus { op_annot :: annot } -- ^ \'+\' + | Minus { op_annot :: annot } -- ^ \'-\' + | Divide { op_annot :: annot } -- ^ \'\/\' + | FloorDivide { op_annot :: annot } -- ^ \'\/\/\' + | MatrixMult { op_annot :: annot } -- ^ \'@\' + | Invert { op_annot :: annot } -- ^ \'~\' (bitwise inversion of its integer argument) + | Modulo { op_annot :: annot } -- ^ \'%\' + deriving (Eq,Ord,Show,Typeable,Data,Functor) + +type OpSpan = Op SrcSpan + +instance Span OpSpan where + getSpan = annot + +instance Annotated Op where + annot = op_annot + +-- | Augmented assignment operators. +data AssignOp annot + = PlusAssign { assignOp_annot :: annot } -- ^ \'+=\' + | MinusAssign { assignOp_annot :: annot } -- ^ \'-=\' + | MultAssign { assignOp_annot :: annot } -- ^ \'*=\' + | DivAssign { assignOp_annot :: annot } -- ^ \'\/=\' + | ModAssign { assignOp_annot :: annot } -- ^ \'%=\' + | PowAssign { assignOp_annot :: annot } -- ^ \'*=\' + | BinAndAssign { assignOp_annot :: annot } -- ^ \'&=\' + | BinOrAssign { assignOp_annot :: annot } -- ^ \'|=\' + | BinXorAssign { assignOp_annot :: annot } -- ^ \'^=\' + | LeftShiftAssign { assignOp_annot :: annot } -- ^ \'<<=\' + | RightShiftAssign { assignOp_annot :: annot } -- ^ \'>>=\' + | FloorDivAssign { assignOp_annot :: annot } -- ^ \'\/\/=\' + | MatrixMultAssign { assignOp_annot :: annot } -- ^ \'@=\' + deriving (Eq,Ord,Show,Typeable,Data,Functor) + +type AssignOpSpan = AssignOp SrcSpan + +instance Span AssignOpSpan where + getSpan = annot + +instance Annotated AssignOp where + annot = assignOp_annot diff --git a/src/Language/Python/Common/LexerUtils.hs b/src/Language/Python/Common/LexerUtils.hs --- a/src/Language/Python/Common/LexerUtils.hs +++ b/src/Language/Python/Common/LexerUtils.hs @@ -14,16 +14,15 @@ module Language.Python.Common.LexerUtils where import Control.Monad (liftM) -import Control.Monad.Error.Class (throwError) import Data.List (foldl') -import Data.Map as Map hiding (null, map, foldl') import Data.Word (Word8) -import Data.Char (ord) -import Numeric (readHex, readOct) -import Language.Python.Common.Token as Token +import Language.Python.Common.Token as Token import Language.Python.Common.ParserMonad hiding (location) import Language.Python.Common.SrcLocation +import Codec.Binary.UTF8.String as UTF8 (encode) +type Byte = Word8 + -- Beginning of. BOF = beginning of file, BOL = beginning of line data BO = BOF | BOL @@ -106,15 +105,20 @@ -- Test if we are at the end of the line or file atEOLorEOF :: a -> AlexInput -> Int -> AlexInput -> Bool -atEOLorEOF _user _inputBeforeToken _tokenLength (_loc, inputAfterToken) +atEOLorEOF _user _inputBeforeToken _tokenLength (_loc, _bs, inputAfterToken) = null inputAfterToken || nextChar == '\n' || nextChar == '\r' where nextChar = head inputAfterToken notEOF :: a -> AlexInput -> Int -> AlexInput -> Bool -notEOF _user _inputBeforeToken _tokenLength (_loc, inputAfterToken) +notEOF _user _inputBeforeToken _tokenLength (_loc, _bs, inputAfterToken) = not (null inputAfterToken) +delUnderscores :: String -> String +delUnderscores [] = [] +delUnderscores ('_':xs) = delUnderscores xs +delUnderscores (x :xs) = x : delUnderscores xs + readBinary :: String -> Integer readBinary = toBinary . drop 2 @@ -122,6 +126,7 @@ toBinary = foldl' acc 0 acc b '0' = 2 * b acc b '1' = 2 * b + 1 + acc _ _ = error "Lexer ensures all digits passed to readBinary are 0 or 1." readFloat :: String -> Double readFloat str@('.':cs) = read ('0':readFloatRest str) @@ -144,6 +149,12 @@ byteStringToken :: SrcSpan -> String -> Token byteStringToken = ByteStringToken +formatStringToken :: SrcSpan -> String -> Token +formatStringToken = StringToken + +formatRawStringToken :: SrcSpan -> String -> Token +formatRawStringToken = StringToken + unicodeStringToken :: SrcSpan -> String -> Token unicodeStringToken = UnicodeStringToken @@ -179,25 +190,36 @@ -- ----------------------------------------------------------------------------- -- Functionality required by Alex -type AlexInput = (SrcLocation, String) +type AlexInput = (SrcLocation, -- current src location + [Byte], -- byte buffer for next character + String) -- input string alexInputPrevChar :: AlexInput -> Char alexInputPrevChar _ = error "alexInputPrevChar not used" +-- byte buffer should be empty here alexGetChar :: AlexInput -> Maybe (Char, AlexInput) -alexGetChar (loc, input) +alexGetChar (loc, [], input) | null input = Nothing - | otherwise = Just (nextChar, (nextLoc, rest)) + | otherwise = seq nextLoc (Just (nextChar, (nextLoc, [], rest))) where nextChar = head input rest = tail input nextLoc = moveChar nextChar loc +alexGetChar (loc, _:_, _) = error "alexGetChar called with non-empty byte buffer" -mapFst :: (a -> b) -> (a, c) -> (b, c) -mapFst f (a, c) = (f a, c) +-- mapFst :: (a -> b) -> (a, c) -> (b, c) +-- mapFst f (a, c) = (f a, c) -alexGetByte :: AlexInput -> Maybe (Word8, AlexInput) -alexGetByte = fmap (mapFst (fromIntegral . ord)) . alexGetChar +alexGetByte :: AlexInput -> Maybe (Byte, AlexInput) +-- alexGetByte = fmap (mapFst (fromIntegral . ord)) . alexGetChar +alexGetByte (loc, b:bs, input) = Just (b, (loc, bs, input)) +alexGetByte (loc, [], []) = Nothing +alexGetByte (loc, [], nextChar:rest) = + seq nextLoc (Just (byte, (nextLoc, restBytes, rest))) + where + nextLoc = moveChar nextChar loc + byte:restBytes = UTF8.encode [nextChar] moveChar :: Char -> SrcLocation -> SrcLocation moveChar '\n' = incLine 1 @@ -213,3 +235,4 @@ readOctNoO :: String -> Integer readOctNoO (zero:rest) = read (zero:'O':rest) +readOctNoO [] = error "Lexer ensures readOctNoO is never called on an empty string" diff --git a/src/Language/Python/Common/ParseError.hs b/src/Language/Python/Common/ParseError.hs --- a/src/Language/Python/Common/ParseError.hs +++ b/src/Language/Python/Common/ParseError.hs @@ -12,17 +12,11 @@ module Language.Python.Common.ParseError ( ParseError (..) ) where -import Language.Python.Common.Pretty import Language.Python.Common.SrcLocation (SrcLocation) import Language.Python.Common.Token (Token) -import Control.Monad.Error.Class data ParseError = UnexpectedToken Token -- ^ An error from the parser. Token found where it should not be. Note: tokens contain their own source span. | UnexpectedChar Char SrcLocation -- ^ An error from the lexer. Character found where it should not be. | StrError String -- ^ A generic error containing a string message. No source location. deriving (Eq, Ord, Show) - -instance Error ParseError where - noMsg = StrError "" - strMsg = StrError diff --git a/src/Language/Python/Common/ParserMonad.hs b/src/Language/Python/Common/ParserMonad.hs --- a/src/Language/Python/Common/ParserMonad.hs +++ b/src/Language/Python/Common/ParserMonad.hs @@ -43,6 +43,7 @@ , addComment , getComments , spanError + , throwError ) where import Language.Python.Common.SrcLocation (SrcLocation (..), SrcSpan (..), Span (..)) @@ -51,10 +52,6 @@ import Control.Applicative ((<$>)) import Control.Monad.State.Class import Control.Monad.State.Strict as State -import Control.Monad.Error as Error -import Control.Monad.Error.Class -import Control.Monad.Identity as Identity -import Control.Monad.Trans as Trans import Language.Python.Common.Pretty internalError :: String -> P a @@ -93,6 +90,9 @@ } type P a = StateT ParseState (Either ParseError) a + +throwError :: ParseError -> P a +throwError = lift . Left execParser :: P a -> ParseState -> Either ParseError a execParser = evalStateT diff --git a/src/Language/Python/Common/ParserUtils.hs b/src/Language/Python/Common/ParserUtils.hs --- a/src/Language/Python/Common/ParserUtils.hs +++ b/src/Language/Python/Common/ParserUtils.hs @@ -15,7 +15,6 @@ import Data.List (foldl') import Data.Maybe (isJust) -import Control.Monad.Error.Class (throwError) import Language.Python.Common.AST as AST import Language.Python.Common.Token as Token import Language.Python.Common.ParserMonad hiding (location) @@ -58,6 +57,7 @@ isProperSlice :: Subscript -> Bool isProperSlice (SubscriptSlice {}) = True +isProperSlice (SubscriptSliceEllipsis {}) = True isProperSlice other = False subscriptToSlice :: Subscript -> SliceSpan @@ -78,12 +78,6 @@ | length subs == 1 = subscriptToExpr $ head subs | otherwise = error "subscriptsToExpr: empty subscript list" -{- - = TrailerCall { trailer_call_args :: [ArgumentSpan], trailer_span :: SrcSpan } - | TrailerSubscript { trailer_subs :: [Subscript], trailer_span :: SrcSpan } - | TrailerDot { trailer_dot_ident :: IdentSpan, dot_span :: SrcSpan, trailer_span :: SrcSpan } --} - addTrailer :: ExprSpan -> [Trailer] -> ExprSpan addTrailer = foldl' trail @@ -97,31 +91,35 @@ | otherwise = Subscript e (subscriptsToExpr subs) (spanning e trail) trail e trail@(TrailerDot { trailer_dot_ident = ident, dot_span = ds }) - = BinaryOp (AST.Dot ds) e (Var ident (getSpan ident)) (spanning e trail) + = Dot { dot_expr = e, dot_attribute = ident, expr_annot = spanning e trail } makeTupleOrExpr :: [ExprSpan] -> Maybe Token -> ExprSpan makeTupleOrExpr [e] Nothing = e makeTupleOrExpr es@(_:_) (Just t) = Tuple es (spanning es t) makeTupleOrExpr es@(_:_) Nothing = Tuple es (getSpan es) +makeTupleOrExpr [] _ = error "makeTupleOrExpr should never be called with an empty list" makeAssignmentOrExpr :: ExprSpan -> Either [ExprSpan] (AssignOpSpan, ExprSpan) -> StatementSpan makeAssignmentOrExpr e (Left es) = makeNormalAssignment e es - where - makeNormalAssignment :: ExprSpan -> [ExprSpan] -> StatementSpan - makeNormalAssignment e [] = StmtExpr e (getSpan e) - makeNormalAssignment e es - = AST.Assign (e : front) (head back) (spanning e es) - where - (front, back) = splitAt (len - 1) es - len = length es -makeAssignmentOrExpr e1 (Right (op, e2)) - = makeAugAssignment e1 op e2 - where - makeAugAssignment :: ExprSpan -> AssignOpSpan -> ExprSpan -> StatementSpan - makeAugAssignment e1 op e2 - = AST.AugmentedAssign e1 op e2 (spanning e1 e2) +makeAssignmentOrExpr e (Right ope2) + = makeAugAssignment e ope2 +makeAugAssignment :: ExprSpan -> (AssignOpSpan, ExprSpan) -> StatementSpan +makeAugAssignment e1 (op, e2) + = AST.AugmentedAssign e1 op e2 (spanning e1 e2) + +makeNormalAssignment :: ExprSpan -> [ExprSpan] -> StatementSpan +makeNormalAssignment e [] = StmtExpr e (getSpan e) +makeNormalAssignment e es + = AST.Assign (e : front) (head back) (spanning e es) + where + (front, back) = splitAt (len - 1) es + len = length es + +makeAnnAssignment :: ExprSpan -> (ExprSpan, Maybe ExprSpan) -> StatementSpan +makeAnnAssignment ato (annotation, ae) = AST.AnnotatedAssign annotation ato ae (spanning ae ato) + makeTry :: Token -> SuiteSpan -> ([HandlerSpan], [StatementSpan], [StatementSpan]) -> StatementSpan makeTry t1 body (handlers, elses, finally) = AST.Try body handlers elses finally @@ -151,22 +149,34 @@ makeTupleParam p@(ParamTuple { param_tuple_annot = span }) optDefault = UnPackTuple p optDefault span -makeComprehension :: ExprSpan -> CompForSpan -> ComprehensionSpan ExprSpan -makeComprehension e for = Comprehension e for (spanning e for) +makeComprehension :: ExprSpan -> CompForSpan -> ComprehensionSpan +makeComprehension e for = Comprehension (ComprehensionExpr e) for (spanning e for) -makeListForm :: SrcSpan -> Either ExprSpan (ComprehensionSpan ExprSpan) -> ExprSpan +makeListForm :: SrcSpan -> Either ExprSpan ComprehensionSpan -> ExprSpan makeListForm span (Left tuple@(Tuple {})) = List (tuple_exprs tuple) span makeListForm span (Left other) = List [other] span makeListForm span (Right comprehension) = ListComp comprehension span makeSet :: ExprSpan -> Either CompForSpan [ExprSpan] -> SrcSpan -> ExprSpan -makeSet e (Left compFor) = SetComp (Comprehension e compFor (spanning e compFor)) +makeSet e (Left compFor) = SetComp (Comprehension (ComprehensionExpr e) compFor (spanning e compFor)) makeSet e (Right es) = Set (e:es) -makeDictionary :: (ExprSpan, ExprSpan) -> Either CompForSpan [(ExprSpan,ExprSpan)] -> SrcSpan -> ExprSpan -makeDictionary e (Left compFor) = DictComp (Comprehension e compFor (spanning e compFor)) -makeDictionary e (Right es) = Dictionary (e:es) +-- The Either (ExprSpan, ExprSpan) ExprSpan refers to a (key, value) pair or a dictionary unpacking expression. +makeDictionary :: Either (ExprSpan, ExprSpan) ExprSpan -> Either CompForSpan [Either (ExprSpan, ExprSpan) ExprSpan] -> SrcSpan -> ExprSpan +makeDictionary (Left mapping@(key, val)) (Left compFor) = + DictComp (Comprehension (ComprehensionDict (DictMappingPair key val)) compFor (spanning mapping compFor)) +-- This is allowed by the grammar, but will produce a runtime syntax error: +-- dict unpacking cannot be used in dict comprehension +makeDictionary (Right unpacking) (Left compFor) = + DictComp (Comprehension (ComprehensionDict (DictUnpacking unpacking)) compFor (spanning unpacking compFor)) +makeDictionary item (Right es) = Dictionary $ toKeyDatumList <$> item : es + +toKeyDatumList :: Either (ExprSpan, ExprSpan) ExprSpan -> DictKeyDatumList SrcSpan +toKeyDatumList (Left (key, value)) = DictMappingPair key value +toKeyDatumList (Right unpacking) = DictUnpacking unpacking + + fromEither :: Either a a -> a fromEither (Left x) = x fromEither (Right x) = x @@ -178,6 +188,7 @@ -- parser guarantees that the first list is non-empty makeDecorated :: [DecoratorSpan] -> StatementSpan -> StatementSpan makeDecorated ds@(d:_) def = Decorated ds def (spanning d def) +makeDecorated [] _ = error "parser guarantees that makeDecorated's first argument is non-empty" -- suite can't be empty so it is safe to take span over it makeFun :: Token -> IdentSpan -> [ParameterSpan] -> Maybe ExprSpan -> SuiteSpan -> StatementSpan @@ -188,7 +199,7 @@ makeReturn t1 Nothing = AST.Return Nothing (getSpan t1) makeReturn t1 expr@(Just e) = AST.Return expr (spanning t1 e) -makeParenOrGenerator :: Either ExprSpan (ComprehensionSpan ExprSpan) -> SrcSpan -> ExprSpan +makeParenOrGenerator :: Either ExprSpan ComprehensionSpan -> SrcSpan -> ExprSpan makeParenOrGenerator (Left e) span = Paren e span makeParenOrGenerator (Right comp) span = Generator comp span @@ -197,14 +208,6 @@ makePrint chevron (Just (args, last_comma)) span = AST.Print chevron args (isJust last_comma) span -{- -makeRelative :: Int -> ImportRelativeSpan -> SrcSpan -> ImportRelativeSpan -makeRelative dots importRelative span - = importRelative { import_relative_dots = dots + oldDots, import_relative_annot = span } - where - oldDots = import_relative_dots importRelative --} - makeRelative :: [Either Token DottedNameSpan] -> ImportRelativeSpan makeRelative items = ImportRelative ndots maybeName (getSpan items) @@ -218,9 +221,10 @@ countDots count (Left token:rest) = countDots (count + dots token) rest dots (DotToken {}) = 1 dots (EllipsisToken {}) = 3 + dots _ = error "Parser ensures dots is only called on DotToken or EllipsisToken." {- - See: http://www.python.org/doc/3.0/reference/expressions.html#calls + See: http://docs.python.org/3.0/reference/expressions.html#calls arglist: (argument ',')* (argument [','] |'*' test (',' argument)* [',' '**' test] @@ -252,9 +256,11 @@ ArgKeyword {} | state `elem` [1,2] -> check 2 rest | state `elem` [3,4] -> check 4 rest + | otherwise -> error "state should always be in range 1..4 here" ArgVarArgsPos {} | state `elem` [1,2] -> check 3 rest | state `elem` [3,4] -> spanError arg "there must not be two *arguments in an argument list" + | otherwise -> error "state should always be in range 1..4 here" ArgVarArgsKeyword {} -> check 5 rest {- @@ -288,9 +294,11 @@ UnPackTuple {} | state `elem` [1,3] -> check state rest | state == 2 -> check 3 rest + | otherwise -> error "state should always be in range 1..3 here" Param {} | state `elem` [1,3] -> check state rest | state == 2 -> check 3 rest + | otherwise -> error "state should always be in range 1..3 here" EndPositional {} | state == 1 -> check 2 rest | otherwise -> spanError param "there must not be two *parameters in a parameter list" diff --git a/src/Language/Python/Common/Pretty.hs b/src/Language/Python/Common/Pretty.hs --- a/src/Language/Python/Common/Pretty.hs +++ b/src/Language/Python/Common/Pretty.hs @@ -1,4 +1,4 @@ -{-# LANGUAGE TypeSynonymInstances, FlexibleInstances #-} +{-# LANGUAGE TypeSynonymInstances, FlexibleInstances, CPP #-} ----------------------------------------------------------------------------- -- | -- Module : Language.Python.Common.Pretty @@ -13,6 +13,10 @@ module Language.Python.Common.Pretty (module TextPP, module Language.Python.Common.Pretty) where +#if __GLASGOW_HASKELL__ >= 803 +import Prelude hiding ((<>)) +#endif + import Text.PrettyPrint as TextPP -------------------------------------------------------------------------------- @@ -47,6 +51,10 @@ -- | A list of things separated by commas. commaList :: Pretty a => [a] -> Doc commaList = hsep . punctuate comma . map pretty + +-- | A list of things separated by equals signs. +equalsList :: Pretty a => [a] -> Doc +equalsList = hsep . punctuate (space <> equals) . map pretty instance Pretty Int where pretty = int diff --git a/src/Language/Python/Common/PrettyAST.hs b/src/Language/Python/Common/PrettyAST.hs --- a/src/Language/Python/Common/PrettyAST.hs +++ b/src/Language/Python/Common/PrettyAST.hs @@ -1,3 +1,4 @@ +{-# LANGUAGE CPP #-} ----------------------------------------------------------------------------- -- | -- Module : Language.Python.Version2.Syntax.PrettyAST @@ -12,8 +13,13 @@ module Language.Python.Common.PrettyAST () where +#if __GLASGOW_HASKELL__ >= 803 +import Prelude hiding ((<>)) +#endif + import Language.Python.Common.Pretty -import Language.Python.Common.AST +import Language.Python.Common.AST +import Data.Maybe (fromMaybe) -------------------------------------------------------------------------------- @@ -96,10 +102,12 @@ pretty stmt@(For {}) = text "for" <+> commaList (for_targets stmt) <+> text "in" <+> pretty (for_generator stmt) <> colon $+$ indent (prettySuite (for_body stmt)) $+$ optionalKeywordSuite "else" (for_else stmt) + pretty (AsyncFor { for_stmt = fs }) = zeroWidthText "async " <> pretty fs pretty stmt@(Fun {}) = text "def" <+> pretty (fun_name stmt) <> parens (commaList (fun_args stmt)) <+> perhaps (fun_result_annotation stmt) (text "->") <+> pretty (fun_result_annotation stmt) <> colon $+$ indent (prettySuite (fun_body stmt)) + pretty (AsyncFun { fun_def = fd }) = zeroWidthText "async " <+> pretty fd pretty stmt@(Class {}) = text "class" <+> pretty (class_name stmt) <> prettyOptionalList (class_args stmt) <> colon $+$ indent (prettySuite (class_body stmt)) @@ -109,11 +117,14 @@ text "if" <+> pretty cond <> colon $+$ indent (prettySuite body) $+$ prettyGuards xs $+$ optionalKeywordSuite "else" optionalElse + [] -> error "Attempt to pretty print conditional statement with empty guards" -- XXX is the assign_to always a singleton? pretty (Assign { assign_to = pattern, assign_expr = e }) - = commaList pattern <+> equals <+> pretty e + = equalsList pattern <+> equals <+> pretty e pretty (AugmentedAssign { aug_assign_to = to_expr, aug_assign_op = op, aug_assign_expr = e}) - = pretty to_expr <+> pretty op <+> pretty e + = pretty to_expr <+> pretty op <+> pretty e + pretty (AnnotatedAssign { ann_assign_annotation = ann_annotate, ann_assign_to = ann_to, ann_assign_expr = ann_expr}) + = pretty ann_to <+> text ":" <+> pretty ann_annotate <+> fromMaybe empty (((text "=" <+>) . pretty) <$> ann_expr) pretty (Decorated { decorated_decorators = decs, decorated_def = stmt}) = vcat (map pretty decs) $+$ pretty stmt pretty (Return { return_expr = e }) = text "return" <+> pretty e @@ -126,6 +137,7 @@ pretty (With { with_context = context, with_body = body }) = text "with" <+> hcat (punctuate comma (map prettyWithContext context)) <+> colon $+$ indent (prettySuite body) + pretty (AsyncWith { with_stmt = ws }) = zeroWidthText "async " <+> pretty ws pretty Pass {} = text "pass" pretty Break {} = text "break" pretty Continue {} = text "continue" @@ -195,13 +207,18 @@ pretty (ArgKeyword { arg_keyword = ident, arg_expr = e }) = pretty ident <> equals <> pretty e -instance Pretty t => Pretty (Comprehension t a) where +instance Pretty (Comprehension a) where pretty (Comprehension { comprehension_expr = e, comprehension_for = for }) = pretty e <+> pretty for +instance Pretty (ComprehensionExpr a) where + pretty (ComprehensionExpr e) = pretty e + pretty (ComprehensionDict d) = pretty d + instance Pretty (CompFor a) where - pretty (CompFor { comp_for_exprs = es, comp_in_expr = e, comp_for_iter = iter }) - = text "for" <+> commaList es <+> text "in" <+> pretty e <+> pretty iter + pretty (CompFor { comp_for_async = ca, comp_for_exprs = es, comp_in_expr = e, comp_for_iter = iter }) + = (text $ if ca then "async for" else "for") <+> commaList es + <+> text "in" <+> pretty e <+> pretty iter instance Pretty (CompIf a) where pretty (CompIf { comp_if = e, comp_if_iter = iter }) @@ -231,11 +248,10 @@ pretty (CondExpr { ce_true_branch = trueBranch, ce_condition = cond, ce_false_branch = falseBranch }) = pretty trueBranch <+> text "if" <+> pretty cond <+> text "else" <+> pretty falseBranch pretty (BinaryOp { operator = op, left_op_arg = left, right_op_arg = right }) - = pretty left <> (if isDot op then dot else space <> pretty op <> space) <> pretty right - where - isDot (Dot {}) = True - isDot _other = False + = pretty left <+> pretty op <+> pretty right pretty (UnaryOp { operator = op, op_arg = e }) = pretty op <+> pretty e + pretty (Dot { dot_expr = e, dot_attribute = a }) = + pretty e <> dot <> pretty a pretty (Lambda { lambda_args = args, lambda_body = body }) = text "lambda" <+> commaList args <> colon <+> pretty body pretty (Tuple { tuple_exprs = es }) = @@ -243,20 +259,34 @@ [] -> text "()" [e] -> pretty e <> comma _ -> commaList es - pretty (Yield { yield_expr = e }) - = text "yield" <+> pretty e + pretty (Yield { yield_arg = arg }) + = text "yield" <+> pretty arg + pretty (Generator { gen_comprehension = gc }) = parens $ pretty gc + pretty (Await { await_expr = ae }) = text "await" <+> pretty ae + pretty (ListComp { list_comprehension = lc }) = brackets $ pretty lc pretty (List { list_exprs = es }) = brackets (commaList es) pretty (Dictionary { dict_mappings = mappings }) - = braces (hsep (punctuate comma $ map (\ (e1,e2) -> pretty e1 <> colon <> pretty e2) mappings)) + = braces (hsep (punctuate comma $ map pretty mappings)) + pretty (DictComp { dict_comprehension = dc }) = braces $ pretty dc pretty (Set { set_exprs = es }) = braces $ commaList es - pretty (ListComp { list_comprehension = lc }) = brackets $ pretty lc - pretty (Generator { gen_comprehension = gc }) = parens $ pretty gc + pretty (SetComp { set_comprehension = sc }) = braces $ pretty sc pretty (Paren { paren_expr = e }) = parens $ pretty e + pretty (StringConversion { backquoted_expr = e }) = char '`' <> pretty e <> char '`' + pretty (Starred { starred_expr = e }) = char '*' <> pretty e +instance Pretty (YieldArg a) where + pretty (YieldFrom e _annot) = text "from" <+> pretty e + pretty (YieldExpr e) = pretty e + +instance Pretty (DictKeyDatumList a) where + pretty (DictMappingPair key val) = pretty key <> colon <+> pretty val + pretty (DictUnpacking expr) = text "**" <> pretty expr + instance Pretty (Slice a) where pretty (SliceProper { slice_lower = lower, slice_upper = upper, slice_stride = stride }) = pretty lower <> colon <> pretty upper <> (maybe empty (\s -> colon <> pretty s) stride) pretty (SliceExpr { slice_expr = e }) = pretty e + pretty (SliceEllipsis {}) = text "..." instance Pretty (Op a) where pretty (And {}) = text "and" @@ -284,9 +314,9 @@ pretty (Minus {}) = text "-" pretty (Divide {}) = text "/" pretty (FloorDivide {}) = text "//" + pretty (MatrixMult {}) = text "@" pretty (Invert {}) = text "~" pretty (Modulo {}) = text "%" - pretty (Dot {}) = dot instance Pretty (AssignOp a) where pretty (PlusAssign {}) = text "+=" @@ -301,3 +331,4 @@ pretty (LeftShiftAssign {}) = text "<<=" pretty (RightShiftAssign {}) = text ">>=" pretty (FloorDivAssign {}) = text "//=" + pretty (MatrixMultAssign {}) = text "@=" diff --git a/src/Language/Python/Common/PrettyParseError.hs b/src/Language/Python/Common/PrettyParseError.hs --- a/src/Language/Python/Common/PrettyParseError.hs +++ b/src/Language/Python/Common/PrettyParseError.hs @@ -15,7 +15,7 @@ import Language.Python.Common.Pretty import Language.Python.Common.ParseError (ParseError (..)) import Language.Python.Common.SrcLocation -import Language.Python.Common.PrettyToken +import Language.Python.Common.PrettyToken() instance Pretty ParseError where pretty (UnexpectedToken t) = pretty (getSpan t) <+> text "unexpected token:" <+> pretty t diff --git a/src/Language/Python/Common/PrettyToken.hs b/src/Language/Python/Common/PrettyToken.hs --- a/src/Language/Python/Common/PrettyToken.hs +++ b/src/Language/Python/Common/PrettyToken.hs @@ -24,6 +24,7 @@ IndentToken {} -> text "indentation" DedentToken {} -> text "dedentation" NewlineToken {} -> text "end of line" + LineJoinToken {} -> text "line join" CommentToken { token_literal = str } -> text "comment:" <+> prettyPrefix 10 str IdentifierToken { token_literal = str } -> @@ -32,6 +33,8 @@ text "string:" <+> prettyPrefix 10 str ByteStringToken { token_literal = str } -> text "byte string:" <+> prettyPrefix 10 str + UnicodeStringToken { token_literal = str } -> + text "unicode string:" <+> prettyPrefix 10 str IntegerToken { token_literal = str } -> text "integer:" <+> text str LongIntegerToken { token_literal = str } -> @@ -62,6 +65,8 @@ AsToken {} -> text "as" ElifToken {} -> text "elif" YieldToken {} -> text "yield" + AsyncToken {} -> text "async" + AwaitToken {} -> text "await" AssertToken {} -> text "assert" ImportToken {} -> text "import" PassToken {} -> text "pass" @@ -101,6 +106,7 @@ LeftShiftAssignToken {} -> text "<<=" RightShiftAssignToken {} -> text ">>=" FloorDivAssignToken {} -> text "//=" + MatrixMultAssignToken {} -> text "@=" BackQuoteToken {} -> text "` (back quote)" PlusToken {} -> text "+" MinusToken {} -> text "-" @@ -123,4 +129,3 @@ NotEqualsToken {} -> text "!=" NotEqualsV2Token {} -> text "<>" EOFToken {} -> text "end of input" - LineJoinToken {} -> text "line join" diff --git a/src/Language/Python/Common/SrcLocation.hs b/src/Language/Python/Common/SrcLocation.hs --- a/src/Language/Python/Common/SrcLocation.hs +++ b/src/Language/Python/Common/SrcLocation.hs @@ -34,6 +34,10 @@ startRow ) where +#if __GLASGOW_HASKELL__ >= 803 +import Prelude hiding ((<>)) +#endif + import Language.Python.Common.Pretty import Data.Data @@ -100,6 +104,7 @@ incColumn :: Int -> SrcLocation -> SrcLocation incColumn n loc@(Sloc { sloc_column = col }) = loc { sloc_column = col + n } +incColumn _ NoLocation = NoLocation -- | Increment the column of a location by one tab stop. incTab :: SrcLocation -> SrcLocation @@ -107,11 +112,13 @@ = loc { sloc_column = newCol } where newCol = col + 8 - (col - 1) `mod` 8 +incTab NoLocation = NoLocation -- | Increment the line number (row) of a location by one. incLine :: Int -> SrcLocation -> SrcLocation incLine n loc@(Sloc { sloc_row = row }) = loc { sloc_column = 1, sloc_row = row + n } +incLine _ NoLocation = NoLocation {- Inspired heavily by compiler/basicTypes/SrcLoc.lhs diff --git a/src/Language/Python/Common/StringEscape.hs b/src/Language/Python/Common/StringEscape.hs --- a/src/Language/Python/Common/StringEscape.hs +++ b/src/Language/Python/Common/StringEscape.hs @@ -13,9 +13,9 @@ -- -- See: -- --- * Version 2.6 +-- * Version 2.6 -- --- * Version 3.1 +-- * Version 3.1 ----------------------------------------------------------------------------- module Language.Python.Common.StringEscape ( diff --git a/src/Language/Python/Common/Token.hs b/src/Language/Python/Common/Token.hs --- a/src/Language/Python/Common/Token.hs +++ b/src/Language/Python/Common/Token.hs @@ -25,7 +25,7 @@ ) where import Language.Python.Common.Pretty -import Language.Python.Common.SrcLocation (SrcSpan (..), SrcLocation (..), Span(getSpan)) +import Language.Python.Common.SrcLocation (SrcSpan (..), Span(getSpan)) import Data.Data -- | Lexical tokens. @@ -86,6 +86,8 @@ | OrToken { token_span :: !SrcSpan } -- ^ Keyword: boolean disjunction \'or\'. -- Version 3.x only: | NonLocalToken { token_span :: !SrcSpan } -- ^ Keyword: \'nonlocal\' (Python 3.x only) + | AsyncToken { token_span :: !SrcSpan } -- ^ Keyword: \'async\' (Python 3.x only) + | AwaitToken { token_span :: !SrcSpan } -- ^ Keyword: \'await\' (Python 3.x only) -- Version 2.x only: | PrintToken { token_span :: !SrcSpan } -- ^ Keyword: \'print\'. (Python 2.x only) | ExecToken { token_span :: !SrcSpan } -- ^ Keyword: \'exec\'. (Python 2.x only) @@ -117,6 +119,7 @@ | LeftShiftAssignToken { token_span :: !SrcSpan } -- ^ Delimiter: binary-left-shift assignment \'<<=\'. | RightShiftAssignToken { token_span :: !SrcSpan } -- ^ Delimiter: binary-right-shift assignment \'>>=\'. | FloorDivAssignToken { token_span :: !SrcSpan } -- ^ Delimiter: floor-divide assignment \'//=\'. + | MatrixMultAssignToken { token_span :: !SrcSpan } -- ^ Delimiter: matrix multiplication assignment \'@=\'. | BackQuoteToken { token_span :: !SrcSpan } -- ^ Delimiter: back quote character \'`\'. -- Operators @@ -158,15 +161,16 @@ hasLiteral :: Token -> Bool hasLiteral token = case token of - CommentToken {} -> True - IdentifierToken {} -> True - StringToken {} -> True - ByteStringToken {} -> True - IntegerToken {} -> True - LongIntegerToken {} -> True - FloatToken {} -> True - ImaginaryToken {} -> True - other -> False + CommentToken {} -> True + IdentifierToken {} -> True + StringToken {} -> True + ByteStringToken {} -> True + UnicodeStringToken {} -> True + IntegerToken {} -> True + LongIntegerToken {} -> True + FloatToken {} -> True + ImaginaryToken {} -> True + other -> False -- | Classification of tokens data TokenClass @@ -191,7 +195,8 @@ CommentToken {} -> Comment IdentifierToken {} -> Identifier StringToken {} -> String - ByteStringToken {} -> String + ByteStringToken {} -> String + UnicodeStringToken {} -> String IntegerToken {} -> Number LongIntegerToken {} -> Number FloatToken {} -> Number @@ -218,6 +223,8 @@ AsToken {} -> Keyword ElifToken {} -> Keyword YieldToken {} -> Keyword + AsyncToken {} -> Keyword + AwaitToken {} -> Keyword AssertToken {} -> Keyword ImportToken {} -> Keyword PassToken {} -> Keyword @@ -257,6 +264,7 @@ LeftShiftAssignToken {} -> Assignment RightShiftAssignToken {} -> Assignment FloorDivAssignToken {} -> Assignment + MatrixMultAssignToken {} -> Assignment BackQuoteToken {} -> Punctuation PlusToken {} -> Operator MinusToken {} -> Operator @@ -292,7 +300,8 @@ CommentToken {} -> token_literal token IdentifierToken {} -> token_literal token StringToken {} -> token_literal token - ByteStringToken {} -> token_literal token + ByteStringToken {} -> token_literal token + UnicodeStringToken {} -> token_literal token IntegerToken {} -> token_literal token LongIntegerToken {} -> token_literal token FloatToken {} -> token_literal token @@ -319,6 +328,8 @@ AsToken {} -> "as" ElifToken {} -> "elif" YieldToken {} -> "yield" + AsyncToken {} -> "async" + AwaitToken {} -> "await" AssertToken {} -> "assert" ImportToken {} -> "import" PassToken {} -> "pass" @@ -358,6 +369,7 @@ LeftShiftAssignToken {} -> "<<=" RightShiftAssignToken {} -> ">>=" FloorDivAssignToken {} -> "//=" + MatrixMultAssignToken {} -> "@=" BackQuoteToken {} -> "`" PlusToken {} -> "+" MinusToken {} -> "-" diff --git a/src/Language/Python/Version2.hs b/src/Language/Python/Version2.hs --- a/src/Language/Python/Version2.hs +++ b/src/Language/Python/Version2.hs @@ -11,11 +11,11 @@ -- -- See: -- --- * for an overview of the language. +-- * for an overview of the language. -- --- * for the full grammar. +-- * for the full grammar. -- --- * for a description of +-- * for a description of -- the various Python top-levels, which correspond to the parsers provided here. ----------------------------------------------------------------------------- diff --git a/src/Language/Python/Version2/Lexer.hs b/src/Language/Python/Version2/Lexer.hs --- a/src/Language/Python/Version2/Lexer.hs +++ b/src/Language/Python/Version2/Lexer.hs @@ -9,7 +9,7 @@ -- Portability : ghc -- -- Lexical analysis for Python version 2.x programs. --- See: . +-- See: . ----------------------------------------------------------------------------- module Language.Python.Version2.Lexer ( diff --git a/src/Language/Python/Version2/Parser.hs b/src/Language/Python/Version2/Parser.hs --- a/src/Language/Python/Version2/Parser.hs +++ b/src/Language/Python/Version2/Parser.hs @@ -14,11 +14,11 @@ -- -- See: -- --- * for an overview of the language. +-- * for an overview of the language. -- --- * for the full grammar. +-- * for the full grammar. -- --- * for a description of +-- * for a description of -- the various Python top-levels, which correspond to the parsers provided here. ----------------------------------------------------------------------------- @@ -35,7 +35,7 @@ import Language.Python.Common.AST (ModuleSpan, StatementSpan, ExprSpan) import Language.Python.Common.Token (Token) import Language.Python.Common.SrcLocation (initialSrcLocation) -import Language.Python.Common.ParserMonad (execParser, execParserKeepComments, ParseError, initialState) +import Language.Python.Common.ParserMonad (execParserKeepComments, ParseError, initialState) -- | Parse a whole Python source file. Return comments in addition to the parsed module. parseModule :: String -- ^ The input stream (python module source code). diff --git a/src/Language/Python/Version2/Parser/Lexer.x b/src/Language/Python/Version2/Parser/Lexer.x --- a/src/Language/Python/Version2/Parser/Lexer.x +++ b/src/Language/Python/Version2/Parser/Lexer.x @@ -20,9 +20,6 @@ import Language.Python.Common.SrcLocation import Language.Python.Common.LexerUtils import qualified Data.Map as Map -import Control.Monad (liftM) -import Data.List (foldl') -import Numeric (readHex, readOct) } -- character sets @@ -225,7 +222,7 @@ location <- getLocation input <- getInput startCode <- getStartCode - case alexScan (location, input) startCode of + case alexScan (location, [], input) startCode of AlexEOF -> do -- Ensure there is a newline token before the EOF previousToken <- getLastToken @@ -245,11 +242,11 @@ setLastToken insertedNewlineToken return insertedNewlineToken AlexError _ -> lexicalError - AlexSkip (nextLocation, rest) len -> do + AlexSkip (nextLocation, _bytes, rest) len -> do setLocation nextLocation setInput rest lexToken - AlexToken (nextLocation, rest) len action -> do + AlexToken (nextLocation, _bytes, rest) len action -> do setLocation nextLocation setInput rest token <- action (mkSrcSpan location $ decColumn 1 nextLocation) len input diff --git a/src/Language/Python/Version2/Parser/Parser.y b/src/Language/Python/Version2/Parser/Parser.y --- a/src/Language/Python/Version2/Parser/Parser.y +++ b/src/Language/Python/Version2/Parser/Parser.y @@ -1,4 +1,8 @@ { +{-# LANGUAGE CPP #-} + +#undef __GLASGOW_HASKELL__ +#define __GLASGOW_HASKELL__ 709 ----------------------------------------------------------------------------- -- | -- Module : Language.Python.Version2.Parser.Parser @@ -64,6 +68,7 @@ '!=' { NotEqualsToken {} } '<>' { NotEqualsV2Token {} } '.' { DotToken {} } + '...' { EllipsisToken {} } '`' { BackQuoteToken {} } '+=' { PlusAssignToken {} } '-=' { MinusAssignToken {} } @@ -203,10 +208,10 @@ -- decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE opt_paren_arg_list :: { [ArgumentSpan] } -opt_paren_arg_list: opt(paren_arg_list) { concat (maybeToList $1) } +opt_paren_arg_list: opt(paren_arg_list) { (concat (maybeToList (fmap fst $1))) } -paren_arg_list :: { [ArgumentSpan] } -paren_arg_list : '(' optional_arg_list ')' { $2 } +paren_arg_list :: { ([ArgumentSpan], SrcSpan) } +paren_arg_list : '(' optional_arg_list ')' { ($2, spanning $1 $3) } decorator :: { DecoratorSpan } decorator @@ -746,10 +751,10 @@ exponent_op :: { OpSpan } exponent_op: '**' { AST.Exponent (getSpan $1) } -{- +{- atom: ('(' [yield_expr|testlist_gexp] ')' | '[' [listmaker] ']' | - '{' [dictmaker] '}' | + '{' [dictorsetmaker] '}' | '`' testlist1 '`' | NAME | NUMBER | STRING+) -} @@ -758,7 +763,7 @@ atom : '(' yield_or_testlist_gexp ')' { $2 (spanning $1 $3) } | list_atom { $1 } - | '{' opt(dictmaker) '}' { AST.Dictionary (concat (maybeToList $2)) (spanning $1 $3) } + | dict_or_set_atom { $1 } | '`' testlist1 '`' { AST.StringConversion $2 (spanning $1 $3) } | NAME { AST.Var $1 (getSpan $1) } | 'integer' { AST.Int (token_integer $1) (token_literal $1) (getSpan $1) } @@ -776,7 +781,12 @@ : '[' ']' { List [] (spanning $1 $2) } | '[' testlistfor ']' { makeListForm (spanning $1 $3) $2 } -testlistfor :: { Either ExprSpan (ComprehensionSpan ExprSpan) } +dict_or_set_atom :: { ExprSpan } +dict_or_set_atom + : '{' '}' { Dictionary [] (spanning $1 $2) } + | '{' dictorsetmaker '}' { $2 (spanning $1 $3) } + +testlistfor :: { Either ExprSpan ComprehensionSpan } testlistfor : testlist { Left $1 } | test list_for { Right (makeComprehension $1 $2) } @@ -789,7 +799,7 @@ -- testlist_gexp: test ( gen_for | (',' test)* [','] ) -testlist_gexp :: { Either ExprSpan (ComprehensionSpan ExprSpan) } +testlist_gexp :: { Either ExprSpan ComprehensionSpan } testlist_gexp : testlist { Left $1 } | test gen_for { Right (makeComprehension $1 $2) } @@ -798,7 +808,7 @@ trailer :: { Trailer } trailer - : paren_arg_list { TrailerCall $1 (getSpan $1) } + : paren_arg_list { TrailerCall (fst $1) (snd $1) } | '[' subscriptlist ']' { TrailerSubscript $2 (spanning $1 $3) } | '.' NAME { TrailerDot $2 (getSpan $1) (spanning $1 $2) } @@ -811,7 +821,8 @@ subscript :: { Subscript } subscript - : '.' '.' '.' { SubscriptSliceEllipsis (spanning $1 $3) } + -- : '.' '.' '.' { SubscriptSliceEllipsis (spanning $1 $3) } + : '...' { SubscriptSliceEllipsis (getSpan $1) } | test { SubscriptExpr $1 (getSpan $1) } | opt(test) ':' opt(test) opt(sliceop) { SubscriptSlice $1 $3 $4 (spanning (spanning (spanning $1 $2) $3) $4) } @@ -847,15 +858,40 @@ testlist : testlistrev opt_comma { makeTupleOrExpr (reverse $1) $2 } testlistrev :: { [ExprSpan] } -testlistrev +testlistrev : test { [$1] } | testlistrev ',' test { $3 : $1 } --- dictmaker: test ':' test (',' test ':' test)* [','] +{- + dictorsetmaker: ( (test ':' test (comp_for | (',' test ':' test)* [','])) | + (test (comp_for | (',' test)* [','])) ) +-} -dictmaker :: { [(ExprSpan, ExprSpan)] } -dictmaker: sepOptEndBy(pair(test,right(':',test)), ',') { $1 } +dictorsetmaker :: { SrcSpan -> ExprSpan } +dictorsetmaker + : test ':' test dict_rest { makeDictionary (Left ($1, $3)) $4 } + | test set_rest { makeSet $1 $2 } +dict_rest :: { Either CompForSpan [Either (ExprSpan, ExprSpan) ExprSpan] } +dict_rest + : comp_for { Left $1 } + | zero_or_more_dict_mappings_rev opt_comma { Right (reverse $1) } + +zero_or_more_dict_mappings_rev :: { [Either (ExprSpan, ExprSpan) ExprSpan] } +zero_or_more_dict_mappings_rev + : {- empty -} { [] } + | zero_or_more_dict_mappings_rev ',' test ':' test { (Left ($3,$5)) : $1 } + +set_rest :: { Either CompForSpan [ExprSpan] } +set_rest + : comp_for { Left $1 } + | zero_or_more_comma_test_rev opt_comma { Right (reverse $1) } + +zero_or_more_comma_test_rev :: { [ExprSpan] } +zero_or_more_comma_test_rev + : {- empty -} { [] } + | zero_or_more_comma_test_rev ',' test { $3 : $1 } + -- classdef: 'class' NAME ['(' [testlist] ')'] ':' suite classdef :: { StatementSpan } @@ -905,6 +941,27 @@ | test gen_for { let span = spanning $1 $1 in ArgExpr (Generator (makeComprehension $1 $2) span) span } +-- comp_iter: comp_for | comp_if + +comp_iter :: { CompIterSpan } +comp_iter + : comp_for { IterFor $1 (getSpan $1) } + | comp_if { IterIf $1 (getSpan $1) } + +-- comp_for: 'for' exprlist 'in' or_test [comp_iter] + +comp_for :: { CompForSpan } +comp_for + : 'for' exprlist 'in' or_test opt(comp_iter) + { CompFor False $2 $4 $5 (spanning (spanning $1 $4) $5) } + +-- comp_if: 'if' old_trest [comp_iter] + +comp_if :: { CompIfSpan } +comp_if + : 'if' old_test opt(comp_iter) + { CompIf $2 $3 (spanning (spanning $1 $2) $3) } + -- list_iter: list_for | list_if list_iter :: { CompIterSpan } list_iter @@ -914,7 +971,7 @@ -- list_for: 'for' exprlist 'in' testlist_safe [list_iter] list_for :: { CompForSpan } list_for: 'for' exprlist 'in' testlist_safe opt(list_iter) - { AST.CompFor $2 $4 $5 (spanning (spanning $1 $4) $5) } + { AST.CompFor False $2 $4 $5 (spanning (spanning $1 $4) $5) } -- list_if: 'if' old_test [list_iter] @@ -932,7 +989,7 @@ gen_for :: { CompForSpan } gen_for: 'for' exprlist 'in' or_test opt(gen_iter) - { AST.CompFor $2 $4 $5 (spanning (spanning $1 $4) $5) } + { AST.CompFor False $2 $4 $5 (spanning (spanning $1 $4) $5) } -- gen_if: 'if' old_test [gen_iter] @@ -950,10 +1007,16 @@ -- yield_expr: 'yield' [testlist] yield_expr :: { ExprSpan } -yield_expr : 'yield' optional_testlist { AST.Yield $2 (spanning $1 $2) } +yield_expr : 'yield' optional_yieldarg { AST.Yield $2 (spanning $1 $2) } optional_testlist :: { Maybe ExprSpan } optional_testlist: opt(testlist) { $1 } + +optional_yieldarg :: { Maybe YieldArgSpan } +optional_yieldarg : opt(yieldarg) { $1 } + +yieldarg :: { YieldArgSpan } +yieldarg: testlist { YieldExpr $1 } { -- Put additional Haskell code in here if needed. diff --git a/src/Language/Python/Version3.hs b/src/Language/Python/Version3.hs --- a/src/Language/Python/Version3.hs +++ b/src/Language/Python/Version3.hs @@ -11,11 +11,11 @@ -- -- See: -- --- * for an overview of the language. +-- * for an overview of the language. -- --- * for the full grammar. +-- * for the full grammar. -- --- * for a description of +-- * for a description of -- the various Python top-levels, which correspond to the parsers provided here. ----------------------------------------------------------------------------- diff --git a/src/Language/Python/Version3/Lexer.hs b/src/Language/Python/Version3/Lexer.hs --- a/src/Language/Python/Version3/Lexer.hs +++ b/src/Language/Python/Version3/Lexer.hs @@ -9,7 +9,7 @@ -- Portability : ghc -- -- Lexical analysis for Python version 3.x programs. --- See: . +-- See: . ----------------------------------------------------------------------------- module Language.Python.Version3.Lexer ( diff --git a/src/Language/Python/Version3/Parser.hs b/src/Language/Python/Version3/Parser.hs --- a/src/Language/Python/Version3/Parser.hs +++ b/src/Language/Python/Version3/Parser.hs @@ -14,11 +14,11 @@ -- -- See: -- --- * for an overview of the language. +-- * for an overview of the language. -- --- * for the full grammar. +-- * for the full grammar. -- --- * for a description of +-- * for a description of -- the various Python top-levels, which correspond to the parsers provided here. ----------------------------------------------------------------------------- @@ -35,7 +35,7 @@ import Language.Python.Common.AST (ModuleSpan, StatementSpan, ExprSpan) import Language.Python.Common.Token (Token) import Language.Python.Common.SrcLocation (initialSrcLocation) -import Language.Python.Common.ParserMonad (execParser, execParserKeepComments, ParseError, initialState) +import Language.Python.Common.ParserMonad (execParserKeepComments, ParseError, initialState) -- | Parse a whole Python source file. Return comments in addition to the parsed module. parseModule :: String -- ^ The input stream (python module source code). diff --git a/src/Language/Python/Version3/Parser/Lexer.x b/src/Language/Python/Version3/Parser/Lexer.x --- a/src/Language/Python/Version3/Parser/Lexer.x +++ b/src/Language/Python/Version3/Parser/Lexer.x @@ -9,7 +9,7 @@ -- Portability : ghc -- -- Implementation of a lexer for Python version 3.x programs. Generated by --- alex. +-- alex. Edited by Curran McConnell to conform to PEP515. ----------------------------------------------------------------------------- module Language.Python.Version3.Parser.Lexer @@ -20,9 +20,6 @@ import Language.Python.Common.SrcLocation import Language.Python.Common.LexerUtils import qualified Data.Map as Map -import Control.Monad (liftM) -import Data.List (foldl') -import Numeric (readHex, readOct) } -- character sets @@ -46,9 +43,9 @@ $not_double_quote = [. \n] # \" -- macro definitions -@exponent = (e | E) (\+ | \-)? $digit+ -@fraction = \. $digit+ -@int_part = $digit+ +@exponent = (e | E) (\+ | \-)? $digit(_?$digit+)* +@fraction = \. $digit(_?$digit+)* +@int_part = $digit(_?$digit+)* @point_float = (@int_part? @fraction) | @int_part \. @exponent_float = (@int_part | @point_float) @exponent @float_number = @point_float | @exponent_float @@ -59,7 +56,10 @@ @two_double_quotes = \"\" $not_double_quote @byte_str_prefix = b | B @raw_str_prefix = r | R +@unicode_str_prefix = u | U +@format_str_prefix = f | F @raw_byte_str_prefix = @byte_str_prefix @raw_str_prefix | @raw_str_prefix @byte_str_prefix +@format_raw_str_prefix = @format_str_prefix @raw_str_prefix | @raw_str_prefix @format_str_prefix @backslash_pair = \\ (\\|'|\"|@eol_pattern|$short_str_char) @backslash_pair_bs = \\ (\\|'|\"|@eol_pattern|$short_byte_str_char) @short_str_item_single = $short_str_char|@backslash_pair|\" @@ -85,13 +85,13 @@ \\ @eol_pattern { lineJoin } -- line join <0> { - @float_number { token FloatToken readFloat } - $non_zero_digit $digit* { token IntegerToken read } + @float_number { token FloatToken (readFloat.delUnderscores) } + $non_zero_digit (_?$digit)* { token IntegerToken (read.delUnderscores) } (@float_number | @int_part) (j | J) { token ImaginaryToken (readFloat.init) } - 0+ { token IntegerToken read } - 0 (o | O) $oct_digit+ { token IntegerToken read } - 0 (x | X) $hex_digit+ { token IntegerToken read } - 0 (b | B) $bin_digit+ { token IntegerToken readBinary } + 0+(_?0+)* { token IntegerToken (read.delUnderscores) } + 0 (o | O) (_?$oct_digit+)+ { token IntegerToken (read.delUnderscores) } + 0 (x | X) (_?$hex_digit+)+ { token IntegerToken (read.delUnderscores) } + 0 (b | B) (_?$bin_digit+)+ { token IntegerToken (readBinary.delUnderscores) } } -- String literals @@ -99,23 +99,35 @@ <0> { ' @short_str_item_single* ' { mkString stringToken } @raw_str_prefix ' @short_str_item_single* ' { mkString rawStringToken } + @format_str_prefix ' @short_str_item_single* ' { mkString formatStringToken } @byte_str_prefix ' @short_byte_str_item_single* ' { mkString byteStringToken } @raw_byte_str_prefix ' @short_byte_str_item_single* ' { mkString rawByteStringToken } + @format_raw_str_prefix ' @short_str_item_single* ' { mkString formatRawStringToken } + @unicode_str_prefix ' @short_str_item_single* ' { mkString unicodeStringToken } \" @short_str_item_double* \" { mkString stringToken } @raw_str_prefix \" @short_str_item_double* \" { mkString rawStringToken } + @format_str_prefix \" @short_str_item_double* \" { mkString formatStringToken } @byte_str_prefix \" @short_byte_str_item_double* \" { mkString byteStringToken } @raw_byte_str_prefix \" @short_byte_str_item_double* \" { mkString rawByteStringToken } + @format_raw_str_prefix \" @short_str_item_double* \" { mkString formatRawStringToken } + @unicode_str_prefix \" @short_str_item_double* \" { mkString unicodeStringToken } ''' @long_str_item_single* ''' { mkString stringToken } @raw_str_prefix ''' @long_str_item_single* ''' { mkString rawStringToken } + @format_str_prefix ''' @long_str_item_single* ''' { mkString formatStringToken } @byte_str_prefix ''' @long_byte_str_item_single* ''' { mkString byteStringToken } @raw_byte_str_prefix ''' @long_byte_str_item_single* ''' { mkString rawByteStringToken } + @format_raw_str_prefix ''' @long_str_item_single* ''' { mkString formatRawStringToken } + @unicode_str_prefix ''' @long_str_item_single* ''' { mkString unicodeStringToken } \"\"\" @long_str_item_double* \"\"\" { mkString stringToken } @raw_str_prefix \"\"\" @long_str_item_double* \"\"\" { mkString rawStringToken } + @format_str_prefix \"\"\" @long_str_item_double* \"\"\" { mkString formatStringToken } @byte_str_prefix \"\"\" @long_byte_str_item_double* \"\"\" { mkString byteStringToken } @raw_byte_str_prefix \"\"\" @long_byte_str_item_double* \"\"\" { mkString rawByteStringToken } + @format_raw_str_prefix \"\"\" @long_str_item_double* \"\"\" { mkString formatRawStringToken } + @unicode_str_prefix \"\"\" @long_str_item_double* \"\"\" { mkString unicodeStringToken } } -- NOTE: we pass lexToken into some functions as an argument. @@ -196,6 +208,7 @@ "<<=" { symbolToken LeftShiftAssignToken } ">>=" { symbolToken RightShiftAssignToken } "//=" { symbolToken FloorDivAssignToken } + "@=" { symbolToken MatrixMultAssignToken } "," { symbolToken CommaToken } "@" { symbolToken AtToken } \; { symbolToken SemiColonToken } @@ -211,7 +224,7 @@ location <- getLocation input <- getInput startCode <- getStartCode - case alexScan (location, input) startCode of + case alexScan (location, [], input) startCode of AlexEOF -> do -- Ensure there is a newline token before the EOF previousToken <- getLastToken @@ -231,11 +244,11 @@ setLastToken insertedNewlineToken return insertedNewlineToken AlexError _ -> lexicalError - AlexSkip (nextLocation, rest) len -> do + AlexSkip (nextLocation, _bs, rest) len -> do setLocation nextLocation setInput rest lexToken - AlexToken (nextLocation, rest) len action -> do + AlexToken (nextLocation, _bs, rest) len action -> do setLocation nextLocation setInput rest token <- action (mkSrcSpan location $ decColumn 1 nextLocation) len input @@ -277,5 +290,6 @@ , ("as", AsToken), ("elif", ElifToken), ("if", IfToken), ("or", OrToken), ("yield", YieldToken) , ("assert", AssertToken), ("else", ElseToken), ("import", ImportToken), ("pass", PassToken) , ("break", BreakToken), ("except", ExceptToken), ("in", InToken), ("raise", RaiseToken) + , ("async", AsyncToken), ("await", AwaitToken) ] } diff --git a/src/Language/Python/Version3/Parser/Parser.y b/src/Language/Python/Version3/Parser/Parser.y --- a/src/Language/Python/Version3/Parser/Parser.y +++ b/src/Language/Python/Version3/Parser/Parser.y @@ -1,4 +1,8 @@ { +{-# LANGUAGE CPP #-} + +#undef __GLASGOW_HASKELL__ +#define __GLASGOW_HASKELL__ 709 ----------------------------------------------------------------------------- -- | -- Module : Language.Python.Version3.Parser.Parser @@ -20,7 +24,7 @@ import Language.Python.Common.ParserMonad import Language.Python.Common.SrcLocation import Data.Either (rights, either) -import Data.Maybe (maybeToList) +import Data.Maybe (isJust, maybeToList) } %name parseFileInput file_input @@ -76,11 +80,14 @@ '<<=' { LeftShiftAssignToken {} } '>>=' { RightShiftAssignToken {} } '//=' { FloorDivAssignToken {} } + '@=' { MatrixMultAssignToken {} } '@' { AtToken {} } '->' { RightArrowToken {} } 'and' { AndToken {} } 'as' { AsToken {} } 'assert' { AssertToken {} } + 'async' { AsyncToken {} } + 'await' { AwaitToken {} } 'break' { BreakToken {} } 'bytestring' { ByteStringToken {} } 'class' { ClassToken {} } @@ -117,6 +124,7 @@ 'string' { StringToken {} } 'True' { TrueToken {} } 'try' { TryToken {} } + 'unicodestring' { UnicodeStringToken {} } 'while' { WhileToken {} } 'with' { WithToken {} } 'yield' { YieldToken {} } @@ -203,10 +211,10 @@ -- decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE opt_paren_arg_list :: { [ArgumentSpan] } -opt_paren_arg_list: opt(paren_arg_list) { concat (maybeToList $1) } +opt_paren_arg_list: opt(paren_arg_list) { (concat (maybeToList (fmap fst $1))) } -paren_arg_list :: { [ArgumentSpan] } -paren_arg_list : '(' optional_arg_list ')' { $2 } +paren_arg_list :: { ([ArgumentSpan], SrcSpan) } +paren_arg_list : '(' optional_arg_list ')' { ($2, spanning $1 $3) } decorator :: { DecoratorSpan } decorator @@ -222,8 +230,9 @@ decorated :: { StatementSpan } decorated - : decorators or(classdef,funcdef) - { makeDecorated $1 $2 } + : decorators classdef { makeDecorated $1 $2 } + | decorators funcdef { makeDecorated $1 $2 } + | decorators async_funcdef { makeDecorated $1 $2 } -- funcdef: 'def' NAME parameters ['->' test] ':' suite @@ -334,12 +343,15 @@ | nonlocal_stmt { $1 } | assert_stmt { $1 } --- expr_stmt: testlist_star_expr (augassign (yield_expr|testlist) | ('=' (yield_expr|testlist_star_expr))*) +{- expr_stmt: testlist_star_expr (annassign | augassign (yield_expr|testlist) | + ('=' (yield_expr|testlist_star_expr))*) +-} expr_stmt :: { StatementSpan } -expr_stmt - : testlist_star_expr either(many_assign, augassign_yield_or_test_list) - { makeAssignmentOrExpr $1 $2 } +expr_stmt + : testlist_star_expr many_assign { makeNormalAssignment $1 $2 } + | testlist_star_expr augassign_yield_or_test_list { makeAugAssignment $1 $2 } + | testlist_star_expr annassign { makeAnnAssignment $1 $2 } many_assign :: { [ExprSpan] } many_assign : many0(right('=', yield_or_test_list_star)) { $1 } @@ -367,7 +379,7 @@ {- augassign: ('+=' | '-=' | '*=' | '/=' | '%=' | '&=' | '|=' | '^=' | - '<<=' | '>>=' | '**=' | '//=') + '<<=' | '>>=' | '**=' | '//=' | '@=') -} augassign :: { AssignOpSpan } @@ -384,7 +396,13 @@ | '<<=' { AST.LeftShiftAssign (getSpan $1) } | '>>=' { AST.RightShiftAssign (getSpan $1) } | '//=' { AST.FloorDivAssign (getSpan $1) } + | '@=' { AST.MatrixMultAssign (getSpan $1) } +-- annassign: ':' test ['=' test] + +annassign :: { (ExprSpan, Maybe ExprSpan) } +annassign : ':' test opt(right('=', test)) { ($2, $3) } + -- del_stmt: 'del' exprlist del_stmt :: { StatementSpan } @@ -514,7 +532,7 @@ assert_stmt : 'assert' sepBy(test,',') { AST.Assert $2 (spanning $1 $2) } --- compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | with_stmt | funcdef | classdef | decorated +-- compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | with_stmt | funcdef | classdef | decorated | async_stmt compound_stmt :: { StatementSpan } compound_stmt @@ -526,6 +544,7 @@ | funcdef { $1 } | classdef { $1 } | decorated { $1 } + | async_stmt { $1 } -- if_stmt: 'if' test ':' suite ('elif' test ':' suite)* ['else' ':' suite] @@ -555,6 +574,20 @@ : 'for' exprlist 'in' testlist ':' suite optional_else { AST.For $2 $4 $6 $7 (spanning (spanning $1 $6) $7) } +-- async_stmt: ASYNC (funcdef | with_stmt | for_stmt) + +async_stmt :: { StatementSpan } +async_stmt + : 'async' funcdef { AST.AsyncFun $2 (spanning $1 $2) } + | 'async' with_stmt { AST.AsyncWith $2 (spanning $1 $2) } + | 'async' for_stmt { AST.AsyncFor $2 (spanning $1 $2) } + +-- async_fundef: ASYNC funcdef + +async_funcdef :: { StatementSpan } +async_funcdef + : 'async' funcdef { AST.AsyncFun $2 (spanning $1 $2) } + {- try_stmt: ('try' ':' suite ((except_clause ':' suite)+ ['else' ':' suite] ['finally' ':' suite] | 'finally' ':' suite)) @@ -742,6 +775,7 @@ | '/' { AST.Divide (getSpan $1) } | '%' { AST.Modulo (getSpan $1) } | '//' { AST.FloorDivide (getSpan $1) } + | '@' { AST.MatrixMult (getSpan $1) } -- factor: ('+'|'-'|'~') factor | power @@ -753,10 +787,14 @@ tilde_op :: { OpSpan } tilde_op: '~' { AST.Invert (getSpan $1) } +-- await_expr: 'await' primary +await_expr :: { ExprSpan } +await_expr : 'await' atom { AST.Await $2 (spanning $1 $2) } + -- power: atom trailer* ['**' factor] power :: { ExprSpan } -power : atom many0(trailer) opt(pair(exponent_op, factor)) +power : or(await_expr, atom) many0(trailer) opt(pair(exponent_op, factor)) { makeBinOp (addTrailer $1 $2) (maybeToList $3) } exponent_op :: { OpSpan } @@ -779,6 +817,7 @@ | 'imaginary' { AST.Imaginary (token_double $1) (token_literal $1) (getSpan $1) } | many1('string') { AST.Strings (map token_literal $1) (getSpan $1) } | many1('bytestring') { AST.ByteStrings (map token_literal $1) (getSpan $1) } + | many1('unicodestring') { AST.UnicodeStrings (map token_literal $1) (getSpan $1) } | '...' { AST.Ellipsis (getSpan $1) } | 'None' { AST.None (getSpan $1) } | 'True' { AST.Bool Prelude.True (getSpan $1) } @@ -802,7 +841,7 @@ -- testlist_comp: (test|star_expr) ( comp_for | (',' (test|star_expr))* [','] ) -testlist_comp :: { Either ExprSpan (ComprehensionSpan ExprSpan) } +testlist_comp :: { Either ExprSpan ComprehensionSpan } testlist_comp : testlist_star_expr { Left $1 } | or(test,star_expr) comp_for { Right (makeComprehension $1 $2) } @@ -811,7 +850,7 @@ trailer :: { Trailer } trailer - : paren_arg_list { TrailerCall $1 (getSpan $1) } + : paren_arg_list { TrailerCall (fst $1) (snd $1) } | '[' subscriptlist ']' { TrailerSubscript $2 (spanning $1 $3) } | '.' NAME { TrailerDot $2 (getSpan $1) (spanning $1 $2) } @@ -863,25 +902,29 @@ : test { [$1] } | testlistrev ',' test { $3 : $1 } -{- - dictorsetmaker: ( (test ':' test (comp_for | (',' test ':' test)* [','])) | - (test (comp_for | (',' test)* [','])) ) +{- + dictorsetmaker: ( ((test ':' test | '**' expr) + (comp_for | (',' (test ':' test | '**' expr))* [','])) | + ((test | star_expr) + (comp_for | (',' (test | star_expr))* [','])) ) -} dictorsetmaker :: { SrcSpan -> ExprSpan } dictorsetmaker - : test ':' test dict_rest { makeDictionary ($1, $3) $4 } - | test set_rest { makeSet $1 $2 } + : test ':' test dict_rest { makeDictionary (Left ($1, $3)) $4 } + | '**' expr dict_rest { makeDictionary (Right $2) $3 } + | test set_rest { makeSet $1 $2 } -dict_rest :: { Either CompForSpan [(ExprSpan, ExprSpan)] } +dict_rest :: { Either CompForSpan [Either (ExprSpan, ExprSpan) ExprSpan] } dict_rest : comp_for { Left $1 } | zero_or_more_dict_mappings_rev opt_comma { Right (reverse $1) } -zero_or_more_dict_mappings_rev :: { [(ExprSpan, ExprSpan)] } +zero_or_more_dict_mappings_rev :: { [Either (ExprSpan, ExprSpan) ExprSpan] } zero_or_more_dict_mappings_rev : {- empty -} { [] } - | zero_or_more_dict_mappings_rev ',' test ':' test { ($3,$5) : $1 } + | zero_or_more_dict_mappings_rev ',' test ':' test { Left ($3,$5) : $1 } + | zero_or_more_dict_mappings_rev ',' '**' expr { Right $4 : $1 } set_rest :: { Either CompForSpan [ExprSpan] } set_rest @@ -946,8 +989,8 @@ comp_for :: { CompForSpan } comp_for - : 'for' exprlist 'in' or_test opt(comp_iter) - { CompFor $2 $4 $5 (spanning (spanning $1 $4) $5) } + : opt('async') 'for' exprlist 'in' or_test opt(comp_iter) + { CompFor (isJust $1) $3 $5 $6 (spanning $1 (spanning (spanning $2 $5) $6)) } -- comp_if: 'if' test_nocond [comp_iter] @@ -959,13 +1002,23 @@ -- encoding_decl: NAME -- Not used in the rest of the grammar! --- yield_expr: 'yield' [testlist] +-- yield_expr: 'yield' [yield_arg] yield_expr :: { ExprSpan } -yield_expr : 'yield' optional_testlist { AST.Yield $2 (spanning $1 $2) } +yield_expr : 'yield' optional_yieldarg { AST.Yield $2 (spanning $1 $2) } optional_testlist :: { Maybe ExprSpan } optional_testlist: opt(testlist) { $1 } + +optional_yieldarg :: { Maybe YieldArgSpan } +optional_yieldarg: opt(yieldarg) { $1 } + +-- yield_arg: 'from' test | testlist + +yieldarg :: { YieldArgSpan } +yieldarg + : 'from' test { YieldFrom $2 (spanning $1 $2) } + | testlist { YieldExpr $1 } { -- Put additional Haskell code in here if needed.