diff --git a/ChangeLog.md b/ChangeLog.md new file mode 100644 --- /dev/null +++ b/ChangeLog.md @@ -0,0 +1,7 @@ +# Revision history for hpython + +## 0.1 + +*2018-01-07* + +Initial release diff --git a/LICENCE b/LICENCE new file mode 100644 --- /dev/null +++ b/LICENCE @@ -0,0 +1,31 @@ +Copyright (c) 2017, Commonwealth Scientific and Industrial Research Organisation +(CSIRO) ABN 41 687 119 230. + +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials provided + with the distribution. + + * Neither the name of Data61 nor the names of other + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/Setup.hs b/Setup.hs new file mode 100644 --- /dev/null +++ b/Setup.hs @@ -0,0 +1,2 @@ +import Distribution.Simple +main = defaultMain diff --git a/benchmarks/Main.hs b/benchmarks/Main.hs new file mode 100644 --- /dev/null +++ b/benchmarks/Main.hs @@ -0,0 +1,48 @@ +{-# language DataKinds #-} +{-# options_ghc -ddump-to-file -ddump-simpl #-} +module Main where + +import Criterion.Main + +import Data.List.NonEmpty (NonEmpty) +import Data.Validation (Validation(..)) +import System.Exit (exitFailure) + +import qualified Data.Text.IO as StrictText + +import Language.Python.Parse (parseModule) +import Language.Python.Parse.Error (ParseError) +import Language.Python.Internal.Lexer (SrcInfo, tokenize) +import Language.Python.Validate + +parseCheckSeq :: FilePath -> IO () +parseCheckSeq name = do + file <- StrictText.readFile name + py <- + case parseModule name file of + Failure e -> print (e :: NonEmpty (ParseError SrcInfo)) *> exitFailure + Success a -> pure a + case runValidateIndentation $ validateModuleIndentation py of + Failure errs -> + print (errs :: NonEmpty (IndentationError SrcInfo)) *> exitFailure + Success res -> + case runValidateSyntax (validateModuleSyntax res) of + Failure errs' -> + print (errs' :: (NonEmpty (SyntaxError SrcInfo))) *> exitFailure + Success a -> pure $ seq a () + +tokenizeSeq :: FilePath -> IO () +tokenizeSeq name = do + file <- StrictText.readFile name + case tokenize name file of + Left e -> print (e :: (ParseError SrcInfo)) *> exitFailure + Right a -> pure $ seq (length a) () + +main :: IO () +main = + defaultMain + [ bench "tokenize 9000 lines of correct python" $ + nfIO (tokenizeSeq "./benchmarks/pypy.py") + , bench "9000 lines of correct python" $ + nfIO (parseCheckSeq "./benchmarks/pypy.py") + ] diff --git a/benchmarks/pypy.py b/benchmarks/pypy.py new file mode 100644 --- /dev/null +++ b/benchmarks/pypy.py @@ -0,0 +1,6399 @@ +# Copyright (c) 2004 Python Software Foundation. +# All rights reserved. + +# Written by Eric Price +# and Facundo Batista +# and Raymond Hettinger +# and Aahz +# and Tim Peters + +# This module should be kept in sync with the latest updates of the +# IBM specification as it evolves. Those updates will be treated +# as bug fixes (deviation from the spec is a compatibility, usability +# bug) and will be backported. At this point the spec is stabilizing +# and the updates are becoming fewer, smaller, and less significant. + +""" +This is an implementation of decimal floating point arithmetic based on +the General Decimal Arithmetic Specification: + + http://speleotrove.com/decimal/decarith.html + +and IEEE standard 854-1987: + + http://en.wikipedia.org/wiki/IEEE_854-1987 + +Decimal floating point has finite precision with arbitrarily large bounds. + +The purpose of this module is to support arithmetic using familiar +"schoolhouse" rules and to avoid some of the tricky representation +issues associated with binary floating point. The package is especially +useful for financial applications or for contexts where users have +expectations that are at odds with binary floating point (for instance, +in binary floating point, 1.00 % 0.1 gives 0.09999999999999995 instead +of 0.0; Decimal('1.00') % Decimal('0.1') returns the expected +Decimal('0.00')). + +Here are some examples of using the decimal module: + +>>> from decimal import * +>>> setcontext(ExtendedContext) +>>> Decimal(0) +Decimal('0') +>>> Decimal('1') +Decimal('1') +>>> Decimal('-.0123') +Decimal('-0.0123') +>>> Decimal(123456) +Decimal('123456') +>>> Decimal('123.45e12345678') +Decimal('1.2345E+12345680') +>>> Decimal('1.33') + Decimal('1.27') +Decimal('2.60') +>>> Decimal('12.34') + Decimal('3.87') - Decimal('18.41') +Decimal('-2.20') +>>> dig = Decimal(1) +>>> print(dig / Decimal(3)) +0.333333333 +>>> getcontext().prec = 18 +>>> print(dig / Decimal(3)) +0.333333333333333333 +>>> print(dig.sqrt()) +1 +>>> print(Decimal(3).sqrt()) +1.73205080756887729 +>>> print(Decimal(3) ** 123) +4.85192780976896427E+58 +>>> inf = Decimal(1) / Decimal(0) +>>> print(inf) +Infinity +>>> neginf = Decimal(-1) / Decimal(0) +>>> print(neginf) +-Infinity +>>> print(neginf + inf) +NaN +>>> print(neginf * inf) +-Infinity +>>> print(dig / 0) +Infinity +>>> getcontext().traps[DivisionByZero] = 1 +>>> print(dig / 0) +Traceback (most recent call last): + ... + ... + ... +decimal.DivisionByZero: x / 0 +>>> c = Context() +>>> c.traps[InvalidOperation] = 0 +>>> print(c.flags[InvalidOperation]) +0 +>>> c.divide(Decimal(0), Decimal(0)) +Decimal('NaN') +>>> c.traps[InvalidOperation] = 1 +>>> print(c.flags[InvalidOperation]) +1 +>>> c.flags[InvalidOperation] = 0 +>>> print(c.flags[InvalidOperation]) +0 +>>> print(c.divide(Decimal(0), Decimal(0))) +Traceback (most recent call last): + ... + ... + ... +decimal.InvalidOperation: 0 / 0 +>>> print(c.flags[InvalidOperation]) +1 +>>> c.flags[InvalidOperation] = 0 +>>> c.traps[InvalidOperation] = 0 +>>> print(c.divide(Decimal(0), Decimal(0))) +NaN +>>> print(c.flags[InvalidOperation]) +1 +>>> +""" + +__all__ = [ + # Two major classes + 'Decimal', 'Context', + + # Named tuple representation + 'DecimalTuple', + + # Contexts + 'DefaultContext', 'BasicContext', 'ExtendedContext', + + # Exceptions + 'DecimalException', 'Clamped', 'InvalidOperation', 'DivisionByZero', + 'Inexact', 'Rounded', 'Subnormal', 'Overflow', 'Underflow', + 'FloatOperation', + + # Exceptional conditions that trigger InvalidOperation + 'DivisionImpossible', 'InvalidContext', 'ConversionSyntax', 'DivisionUndefined', + + # Constants for use in setting up contexts + 'ROUND_DOWN', 'ROUND_HALF_UP', 'ROUND_HALF_EVEN', 'ROUND_CEILING', + 'ROUND_FLOOR', 'ROUND_UP', 'ROUND_HALF_DOWN', 'ROUND_05UP', + + # Functions for manipulating contexts + 'setcontext', 'getcontext', 'localcontext', + + # Limits for the C version for compatibility + 'MAX_PREC', 'MAX_EMAX', 'MIN_EMIN', 'MIN_ETINY', + + # C version: compile time choice that enables the thread local context + 'HAVE_THREADS' +] + +__xname__ = __name__ # sys.modules lookup (--without-threads) +__name__ = 'decimal' # For pickling +__version__ = '1.70' # Highest version of the spec this complies with + # See http://speleotrove.com/decimal/ +__libmpdec_version__ = "2.4.1" # compatible libmpdec version + +import math as _math +import numbers as _numbers +import sys + +try: + from collections import namedtuple as _namedtuple + DecimalTuple = _namedtuple('DecimalTuple', 'sign digits exponent') +except ImportError: + DecimalTuple = lambda *args: args + +# Rounding +ROUND_DOWN = 'ROUND_DOWN' +ROUND_HALF_UP = 'ROUND_HALF_UP' +ROUND_HALF_EVEN = 'ROUND_HALF_EVEN' +ROUND_CEILING = 'ROUND_CEILING' +ROUND_FLOOR = 'ROUND_FLOOR' +ROUND_UP = 'ROUND_UP' +ROUND_HALF_DOWN = 'ROUND_HALF_DOWN' +ROUND_05UP = 'ROUND_05UP' + +# Compatibility with the C version +HAVE_THREADS = True +if sys.maxsize == 2**63-1: + MAX_PREC = 999999999999999999 + MAX_EMAX = 999999999999999999 + MIN_EMIN = -999999999999999999 +else: + MAX_PREC = 425000000 + MAX_EMAX = 425000000 + MIN_EMIN = -425000000 + +MIN_ETINY = MIN_EMIN - (MAX_PREC-1) + +# Errors + +class DecimalException(ArithmeticError): + """Base exception class. + + Used exceptions derive from this. + If an exception derives from another exception besides this (such as + Underflow (Inexact, Rounded, Subnormal) that indicates that it is only + called if the others are present. This isn't actually used for + anything, though. + + handle -- Called when context._raise_error is called and the + trap_enabler is not set. First argument is self, second is the + context. More arguments can be given, those being after + the explanation in _raise_error (For example, + context._raise_error(NewError, '(-x)!', self._sign) would + call NewError().handle(context, self._sign).) + + To define a new exception, it should be sufficient to have it derive + from DecimalException. + """ + def handle(self, context, *args): + pass + + +class Clamped(DecimalException): + """Exponent of a 0 changed to fit bounds. + + This occurs and signals clamped if the exponent of a result has been + altered in order to fit the constraints of a specific concrete + representation. This may occur when the exponent of a zero result would + be outside the bounds of a representation, or when a large normal + number would have an encoded exponent that cannot be represented. In + this latter case, the exponent is reduced to fit and the corresponding + number of zero digits are appended to the coefficient ("fold-down"). + """ + +class InvalidOperation(DecimalException): + """An invalid operation was performed. + + Various bad things cause this: + + Something creates a signaling NaN + -INF + INF + 0 * (+-)INF + (+-)INF / (+-)INF + x % 0 + (+-)INF % x + x._rescale( non-integer ) + sqrt(-x) , x > 0 + 0 ** 0 + x ** (non-integer) + x ** (+-)INF + An operand is invalid + + The result of the operation after these is a quiet positive NaN, + except when the cause is a signaling NaN, in which case the result is + also a quiet NaN, but with the original sign, and an optional + diagnostic information. + """ + def handle(self, context, *args): + if args: + ans = _dec_from_triple(args[0]._sign, args[0]._int, 'n', True) + return ans._fix_nan(context) + return _NaN + +class ConversionSyntax(InvalidOperation): + """Trying to convert badly formed string. + + This occurs and signals invalid-operation if a string is being + converted to a number and it does not conform to the numeric string + syntax. The result is [0,qNaN]. + """ + def handle(self, context, *args): + return _NaN + +class DivisionByZero(DecimalException, ZeroDivisionError): + """Division by 0. + + This occurs and signals division-by-zero if division of a finite number + by zero was attempted (during a divide-integer or divide operation, or a + power operation with negative right-hand operand), and the dividend was + not zero. + + The result of the operation is [sign,inf], where sign is the exclusive + or of the signs of the operands for divide, or is 1 for an odd power of + -0, for power. + """ + + def handle(self, context, sign, *args): + return _SignedInfinity[sign] + +class DivisionImpossible(InvalidOperation): + """Cannot perform the division adequately. + + This occurs and signals invalid-operation if the integer result of a + divide-integer or remainder operation had too many digits (would be + longer than precision). The result is [0,qNaN]. + """ + + def handle(self, context, *args): + return _NaN + +class DivisionUndefined(InvalidOperation, ZeroDivisionError): + """Undefined result of division. + + This occurs and signals invalid-operation if division by zero was + attempted (during a divide-integer, divide, or remainder operation), and + the dividend is also zero. The result is [0,qNaN]. + """ + + def handle(self, context, *args): + return _NaN + +class Inexact(DecimalException): + """Had to round, losing information. + + This occurs and signals inexact whenever the result of an operation is + not exact (that is, it needed to be rounded and any discarded digits + were non-zero), or if an overflow or underflow condition occurs. The + result in all cases is unchanged. + + The inexact signal may be tested (or trapped) to determine if a given + operation (or sequence of operations) was inexact. + """ + +class InvalidContext(InvalidOperation): + """Invalid context. Unknown rounding, for example. + + This occurs and signals invalid-operation if an invalid context was + detected during an operation. This can occur if contexts are not checked + on creation and either the precision exceeds the capability of the + underlying concrete representation or an unknown or unsupported rounding + was specified. These aspects of the context need only be checked when + the values are required to be used. The result is [0,qNaN]. + """ + + def handle(self, context, *args): + return _NaN + +class Rounded(DecimalException): + """Number got rounded (not necessarily changed during rounding). + + This occurs and signals rounded whenever the result of an operation is + rounded (that is, some zero or non-zero digits were discarded from the + coefficient), or if an overflow or underflow condition occurs. The + result in all cases is unchanged. + + The rounded signal may be tested (or trapped) to determine if a given + operation (or sequence of operations) caused a loss of precision. + """ + +class Subnormal(DecimalException): + """Exponent < Emin before rounding. + + This occurs and signals subnormal whenever the result of a conversion or + operation is subnormal (that is, its adjusted exponent is less than + Emin, before any rounding). The result in all cases is unchanged. + + The subnormal signal may be tested (or trapped) to determine if a given + or operation (or sequence of operations) yielded a subnormal result. + """ + +class Overflow(Inexact, Rounded): + """Numerical overflow. + + This occurs and signals overflow if the adjusted exponent of a result + (from a conversion or from an operation that is not an attempt to divide + by zero), after rounding, would be greater than the largest value that + can be handled by the implementation (the value Emax). + + The result depends on the rounding mode: + + For round-half-up and round-half-even (and for round-half-down and + round-up, if implemented), the result of the operation is [sign,inf], + where sign is the sign of the intermediate result. For round-down, the + result is the largest finite number that can be represented in the + current precision, with the sign of the intermediate result. For + round-ceiling, the result is the same as for round-down if the sign of + the intermediate result is 1, or is [0,inf] otherwise. For round-floor, + the result is the same as for round-down if the sign of the intermediate + result is 0, or is [1,inf] otherwise. In all cases, Inexact and Rounded + will also be raised. + """ + + def handle(self, context, sign, *args): + if context.rounding in (ROUND_HALF_UP, ROUND_HALF_EVEN, + ROUND_HALF_DOWN, ROUND_UP): + return _SignedInfinity[sign] + if sign == 0: + if context.rounding == ROUND_CEILING: + return _SignedInfinity[sign] + return _dec_from_triple(sign, '9'*context.prec, + context.Emax-context.prec+1) + if sign == 1: + if context.rounding == ROUND_FLOOR: + return _SignedInfinity[sign] + return _dec_from_triple(sign, '9'*context.prec, + context.Emax-context.prec+1) + + +class Underflow(Inexact, Rounded, Subnormal): + """Numerical underflow with result rounded to 0. + + This occurs and signals underflow if a result is inexact and the + adjusted exponent of the result would be smaller (more negative) than + the smallest value that can be handled by the implementation (the value + Emin). That is, the result is both inexact and subnormal. + + The result after an underflow will be a subnormal number rounded, if + necessary, so that its exponent is not less than Etiny. This may result + in 0 with the sign of the intermediate result and an exponent of Etiny. + + In all cases, Inexact, Rounded, and Subnormal will also be raised. + """ + +class FloatOperation(DecimalException, TypeError): + """Enable stricter semantics for mixing floats and Decimals. + + If the signal is not trapped (default), mixing floats and Decimals is + permitted in the Decimal() constructor, context.create_decimal() and + all comparison operators. Both conversion and comparisons are exact. + Any occurrence of a mixed operation is silently recorded by setting + FloatOperation in the context flags. Explicit conversions with + Decimal.from_float() or context.create_decimal_from_float() do not + set the flag. + + Otherwise (the signal is trapped), only equality comparisons and explicit + conversions are silent. All other mixed operations raise FloatOperation. + """ + +# List of public traps and flags +_signals = [Clamped, DivisionByZero, Inexact, Overflow, Rounded, + Underflow, InvalidOperation, Subnormal, FloatOperation] + +# Map conditions (per the spec) to signals +_condition_map = {ConversionSyntax:InvalidOperation, + DivisionImpossible:InvalidOperation, + DivisionUndefined:InvalidOperation, + InvalidContext:InvalidOperation} + +# Valid rounding modes +_rounding_modes = (ROUND_DOWN, ROUND_HALF_UP, ROUND_HALF_EVEN, ROUND_CEILING, + ROUND_FLOOR, ROUND_UP, ROUND_HALF_DOWN, ROUND_05UP) + +##### Context Functions ################################################## + +# The getcontext() and setcontext() function manage access to a thread-local +# current context. Py2.4 offers direct support for thread locals. If that +# is not available, use threading.current_thread() which is slower but will +# work for older Pythons. If threads are not part of the build, create a +# mock threading object with threading.local() returning the module namespace. + +try: + import threading +except ImportError: + # Python was compiled without threads; create a mock object instead + class MockThreading(object): + def local(self, sys=sys): + return sys.modules[__xname__] + threading = MockThreading() + del MockThreading + +try: + threading.local + +except AttributeError: + + # To fix reloading, force it to create a new context + # Old contexts have different exceptions in their dicts, making problems. + if hasattr(threading.current_thread(), '__decimal_context__'): + del threading.current_thread().__decimal_context__ + + def setcontext(context): + """Set this thread's context to context.""" + if context in (DefaultContext, BasicContext, ExtendedContext): + context = context.copy() + context.clear_flags() + threading.current_thread().__decimal_context__ = context + + def getcontext(): + """Returns this thread's context. + + If this thread does not yet have a context, returns + a new context and sets this thread's context. + New contexts are copies of DefaultContext. + """ + try: + return threading.current_thread().__decimal_context__ + except AttributeError: + context = Context() + threading.current_thread().__decimal_context__ = context + return context + +else: + + local = threading.local() + if hasattr(local, '__decimal_context__'): + del local.__decimal_context__ + + def getcontext(_local=local): + """Returns this thread's context. + + If this thread does not yet have a context, returns + a new context and sets this thread's context. + New contexts are copies of DefaultContext. + """ + try: + return _local.__decimal_context__ + except AttributeError: + context = Context() + _local.__decimal_context__ = context + return context + + def setcontext(context, _local=local): + """Set this thread's context to context.""" + if context in (DefaultContext, BasicContext, ExtendedContext): + context = context.copy() + context.clear_flags() + _local.__decimal_context__ = context + + del threading, local # Don't contaminate the namespace + +def localcontext(ctx=None): + """Return a context manager for a copy of the supplied context + + Uses a copy of the current context if no context is specified + The returned context manager creates a local decimal context + in a with statement: + def sin(x): + with localcontext() as ctx: + ctx.prec += 2 + # Rest of sin calculation algorithm + # uses a precision 2 greater than normal + return +s # Convert result to normal precision + + def sin(x): + with localcontext(ExtendedContext): + # Rest of sin calculation algorithm + # uses the Extended Context from the + # General Decimal Arithmetic Specification + return +s # Convert result to normal context + + >>> setcontext(DefaultContext) + >>> print(getcontext().prec) + 28 + >>> with localcontext(): + ... ctx = getcontext() + ... ctx.prec += 2 + ... print(ctx.prec) + ... + 30 + >>> with localcontext(ExtendedContext): + ... print(getcontext().prec) + ... + 9 + >>> print(getcontext().prec) + 28 + """ + if ctx is None: ctx = getcontext() + return _ContextManager(ctx) + + +##### Decimal class ####################################################### + +# Do not subclass Decimal from numbers.Real and do not register it as such +# (because Decimals are not interoperable with floats). See the notes in +# numbers.py for more detail. + +class Decimal(object): + """Floating point class for decimal arithmetic.""" + + __slots__ = ('_exp','_int','_sign', '_is_special') + # Generally, the value of the Decimal instance is given by + # (-1)**_sign * _int * 10**_exp + # Special values are signified by _is_special == True + + # We're immutable, so use __new__ not __init__ + def __new__(cls, value="0", context=None): + """Create a decimal point instance. + + >>> Decimal('3.14') # string input + Decimal('3.14') + >>> Decimal((0, (3, 1, 4), -2)) # tuple (sign, digit_tuple, exponent) + Decimal('3.14') + >>> Decimal(314) # int + Decimal('314') + >>> Decimal(Decimal(314)) # another decimal instance + Decimal('314') + >>> Decimal(' 3.14 \\n') # leading and trailing whitespace okay + Decimal('3.14') + """ + + # Note that the coefficient, self._int, is actually stored as + # a string rather than as a tuple of digits. This speeds up + # the "digits to integer" and "integer to digits" conversions + # that are used in almost every arithmetic operation on + # Decimals. This is an internal detail: the as_tuple function + # and the Decimal constructor still deal with tuples of + # digits. + + self = object.__new__(cls) + + # From a string + # REs insist on real strings, so we can too. + if isinstance(value, str): + m = _parser(value.strip()) + if m is None: + if context is None: + context = getcontext() + return context._raise_error(ConversionSyntax, + "Invalid literal for Decimal: %r" % value) + + if m.group('sign') == "-": + self._sign = 1 + else: + self._sign = 0 + intpart = m.group('int') + if intpart is not None: + # finite number + fracpart = m.group('frac') or '' + exp = int(m.group('exp') or '0') + self._int = str(int(intpart+fracpart)) + self._exp = exp - len(fracpart) + self._is_special = False + else: + diag = m.group('diag') + if diag is not None: + # NaN + self._int = str(int(diag or '0')).lstrip('0') + if m.group('signal'): + self._exp = 'N' + else: + self._exp = 'n' + else: + # infinity + self._int = '0' + self._exp = 'F' + self._is_special = True + return self + + # From an integer + if isinstance(value, int): + if value >= 0: + self._sign = 0 + else: + self._sign = 1 + self._exp = 0 + self._int = str(abs(value)) + self._is_special = False + return self + + # From another decimal + if isinstance(value, Decimal): + self._exp = value._exp + self._sign = value._sign + self._int = value._int + self._is_special = value._is_special + return self + + # From an internal working value + if isinstance(value, _WorkRep): + self._sign = value.sign + self._int = str(value.int) + self._exp = int(value.exp) + self._is_special = False + return self + + # tuple/list conversion (possibly from as_tuple()) + if isinstance(value, (list,tuple)): + if len(value) != 3: + raise ValueError('Invalid tuple size in creation of Decimal ' + 'from list or tuple. The list or tuple ' + 'should have exactly three elements.') + # process sign. The isinstance test rejects floats + if not (isinstance(value[0], int) and value[0] in (0,1)): + raise ValueError("Invalid sign. The first value in the tuple " + "should be an integer; either 0 for a " + "positive number or 1 for a negative number.") + self._sign = value[0] + if value[2] == 'F': + # infinity: value[1] is ignored + self._int = '0' + self._exp = value[2] + self._is_special = True + else: + # process and validate the digits in value[1] + digits = [] + for digit in value[1]: + if isinstance(digit, int) and 0 <= digit <= 9: + # skip leading zeros + if digits or digit != 0: + digits.append(digit) + else: + raise ValueError("The second value in the tuple must " + "be composed of integers in the range " + "0 through 9.") + if value[2] in ('n', 'N'): + # NaN: digits form the diagnostic + self._int = ''.join(map(str, digits)) + self._exp = value[2] + self._is_special = True + elif isinstance(value[2], int): + # finite number: digits give the coefficient + self._int = ''.join(map(str, digits or [0])) + self._exp = value[2] + self._is_special = False + else: + raise ValueError("The third value in the tuple must " + "be an integer, or one of the " + "strings 'F', 'n', 'N'.") + return self + + if isinstance(value, float): + if context is None: + context = getcontext() + context._raise_error(FloatOperation, + "strict semantics for mixing floats and Decimals are " + "enabled") + value = Decimal.from_float(value) + self._exp = value._exp + self._sign = value._sign + self._int = value._int + self._is_special = value._is_special + return self + + raise TypeError("Cannot convert %r to Decimal" % value) + + @classmethod + def from_float(cls, f): + """Converts a float to a decimal number, exactly. + + Note that Decimal.from_float(0.1) is not the same as Decimal('0.1'). + Since 0.1 is not exactly representable in binary floating point, the + value is stored as the nearest representable value which is + 0x1.999999999999ap-4. The exact equivalent of the value in decimal + is 0.1000000000000000055511151231257827021181583404541015625. + + >>> Decimal.from_float(0.1) + Decimal('0.1000000000000000055511151231257827021181583404541015625') + >>> Decimal.from_float(float('nan')) + Decimal('NaN') + >>> Decimal.from_float(float('inf')) + Decimal('Infinity') + >>> Decimal.from_float(-float('inf')) + Decimal('-Infinity') + >>> Decimal.from_float(-0.0) + Decimal('-0') + + """ + if isinstance(f, int): # handle integer inputs + return cls(f) + if not isinstance(f, float): + raise TypeError("argument must be int or float.") + if _math.isinf(f) or _math.isnan(f): + return cls(repr(f)) + if _math.copysign(1.0, f) == 1.0: + sign = 0 + else: + sign = 1 + n, d = abs(f).as_integer_ratio() + k = d.bit_length() - 1 + result = _dec_from_triple(sign, str(n*5**k), -k) + if cls is Decimal: + return result + else: + return cls(result) + + def _isnan(self): + """Returns whether the number is not actually one. + + 0 if a number + 1 if NaN + 2 if sNaN + """ + if self._is_special: + exp = self._exp + if exp == 'n': + return 1 + elif exp == 'N': + return 2 + return 0 + + def _isinfinity(self): + """Returns whether the number is infinite + + 0 if finite or not a number + 1 if +INF + -1 if -INF + """ + if self._exp == 'F': + if self._sign: + return -1 + return 1 + return 0 + + def _check_nans(self, other=None, context=None): + """Returns whether the number is not actually one. + + if self, other are sNaN, signal + if self, other are NaN return nan + return 0 + + Done before operations. + """ + + self_is_nan = self._isnan() + if other is None: + other_is_nan = False + else: + other_is_nan = other._isnan() + + if self_is_nan or other_is_nan: + if context is None: + context = getcontext() + + if self_is_nan == 2: + return context._raise_error(InvalidOperation, 'sNaN', + self) + if other_is_nan == 2: + return context._raise_error(InvalidOperation, 'sNaN', + other) + if self_is_nan: + return self._fix_nan(context) + + return other._fix_nan(context) + return 0 + + def _compare_check_nans(self, other, context): + """Version of _check_nans used for the signaling comparisons + compare_signal, __le__, __lt__, __ge__, __gt__. + + Signal InvalidOperation if either self or other is a (quiet + or signaling) NaN. Signaling NaNs take precedence over quiet + NaNs. + + Return 0 if neither operand is a NaN. + + """ + if context is None: + context = getcontext() + + if self._is_special or other._is_special: + if self.is_snan(): + return context._raise_error(InvalidOperation, + 'comparison involving sNaN', + self) + elif other.is_snan(): + return context._raise_error(InvalidOperation, + 'comparison involving sNaN', + other) + elif self.is_qnan(): + return context._raise_error(InvalidOperation, + 'comparison involving NaN', + self) + elif other.is_qnan(): + return context._raise_error(InvalidOperation, + 'comparison involving NaN', + other) + return 0 + + def __bool__(self): + """Return True if self is nonzero; otherwise return False. + + NaNs and infinities are considered nonzero. + """ + return self._is_special or self._int != '0' + + def _cmp(self, other): + """Compare the two non-NaN decimal instances self and other. + + Returns -1 if self < other, 0 if self == other and 1 + if self > other. This routine is for internal use only.""" + + if self._is_special or other._is_special: + self_inf = self._isinfinity() + other_inf = other._isinfinity() + if self_inf == other_inf: + return 0 + elif self_inf < other_inf: + return -1 + else: + return 1 + + # check for zeros; Decimal('0') == Decimal('-0') + if not self: + if not other: + return 0 + else: + return -((-1)**other._sign) + if not other: + return (-1)**self._sign + + # If different signs, neg one is less + if other._sign < self._sign: + return -1 + if self._sign < other._sign: + return 1 + + self_adjusted = self.adjusted() + other_adjusted = other.adjusted() + if self_adjusted == other_adjusted: + self_padded = self._int + '0'*(self._exp - other._exp) + other_padded = other._int + '0'*(other._exp - self._exp) + if self_padded == other_padded: + return 0 + elif self_padded < other_padded: + return -(-1)**self._sign + else: + return (-1)**self._sign + elif self_adjusted > other_adjusted: + return (-1)**self._sign + else: # self_adjusted < other_adjusted + return -((-1)**self._sign) + + # Note: The Decimal standard doesn't cover rich comparisons for + # Decimals. In particular, the specification is silent on the + # subject of what should happen for a comparison involving a NaN. + # We take the following approach: + # + # == comparisons involving a quiet NaN always return False + # != comparisons involving a quiet NaN always return True + # == or != comparisons involving a signaling NaN signal + # InvalidOperation, and return False or True as above if the + # InvalidOperation is not trapped. + # <, >, <= and >= comparisons involving a (quiet or signaling) + # NaN signal InvalidOperation, and return False if the + # InvalidOperation is not trapped. + # + # This behavior is designed to conform as closely as possible to + # that specified by IEEE 754. + + def __eq__(self, other, context=None): + self, other = _convert_for_comparison(self, other, equality_op=True) + if other is NotImplemented: + return other + if self._check_nans(other, context): + return False + return self._cmp(other) == 0 + + def __lt__(self, other, context=None): + self, other = _convert_for_comparison(self, other) + if other is NotImplemented: + return other + ans = self._compare_check_nans(other, context) + if ans: + return False + return self._cmp(other) < 0 + + def __le__(self, other, context=None): + self, other = _convert_for_comparison(self, other) + if other is NotImplemented: + return other + ans = self._compare_check_nans(other, context) + if ans: + return False + return self._cmp(other) <= 0 + + def __gt__(self, other, context=None): + self, other = _convert_for_comparison(self, other) + if other is NotImplemented: + return other + ans = self._compare_check_nans(other, context) + if ans: + return False + return self._cmp(other) > 0 + + def __ge__(self, other, context=None): + self, other = _convert_for_comparison(self, other) + if other is NotImplemented: + return other + ans = self._compare_check_nans(other, context) + if ans: + return False + return self._cmp(other) >= 0 + + def compare(self, other, context=None): + """Compare self to other. Return a decimal value: + + a or b is a NaN ==> Decimal('NaN') + a < b ==> Decimal('-1') + a == b ==> Decimal('0') + a > b ==> Decimal('1') + """ + other = _convert_other(other, raiseit=True) + + # Compare(NaN, NaN) = NaN + if (self._is_special or other and other._is_special): + ans = self._check_nans(other, context) + if ans: + return ans + + return Decimal(self._cmp(other)) + + def __hash__(self): + """x.__hash__() <==> hash(x)""" + + # In order to make sure that the hash of a Decimal instance + # agrees with the hash of a numerically equal integer, float + # or Fraction, we follow the rules for numeric hashes outlined + # in the documentation. (See library docs, 'Built-in Types'). + if self._is_special: + if self.is_snan(): + raise TypeError('Cannot hash a signaling NaN value.') + elif self.is_nan(): + return _PyHASH_NAN + else: + if self._sign: + return -_PyHASH_INF + else: + return _PyHASH_INF + + if self._exp >= 0: + exp_hash = pow(10, self._exp, _PyHASH_MODULUS) + else: + exp_hash = pow(_PyHASH_10INV, -self._exp, _PyHASH_MODULUS) + hash_ = int(self._int) * exp_hash % _PyHASH_MODULUS + ans = hash_ if self >= 0 else -hash_ + return -2 if ans == -1 else ans + + def as_tuple(self): + """Represents the number as a triple tuple. + + To show the internals exactly as they are. + """ + return DecimalTuple(self._sign, tuple(map(int, self._int)), self._exp) + + def __repr__(self): + """Represents the number as an instance of Decimal.""" + # Invariant: eval(repr(d)) == d + return "Decimal('%s')" % str(self) + + def __str__(self, eng=False, context=None): + """Return string representation of the number in scientific notation. + + Captures all of the information in the underlying representation. + """ + + sign = ['', '-'][self._sign] + if self._is_special: + if self._exp == 'F': + return sign + 'Infinity' + elif self._exp == 'n': + return sign + 'NaN' + self._int + else: # self._exp == 'N' + return sign + 'sNaN' + self._int + + # number of digits of self._int to left of decimal point + leftdigits = self._exp + len(self._int) + + # dotplace is number of digits of self._int to the left of the + # decimal point in the mantissa of the output string (that is, + # after adjusting the exponent) + if self._exp <= 0 and leftdigits > -6: + # no exponent required + dotplace = leftdigits + elif not eng: + # usual scientific notation: 1 digit on left of the point + dotplace = 1 + elif self._int == '0': + # engineering notation, zero + dotplace = (leftdigits + 1) % 3 - 1 + else: + # engineering notation, nonzero + dotplace = (leftdigits - 1) % 3 + 1 + + if dotplace <= 0: + intpart = '0' + fracpart = '.' + '0'*(-dotplace) + self._int + elif dotplace >= len(self._int): + intpart = self._int+'0'*(dotplace-len(self._int)) + fracpart = '' + else: + intpart = self._int[:dotplace] + fracpart = '.' + self._int[dotplace:] + if leftdigits == dotplace: + exp = '' + else: + if context is None: + context = getcontext() + exp = ['e', 'E'][context.capitals] + "%+d" % (leftdigits-dotplace) + + return sign + intpart + fracpart + exp + + def to_eng_string(self, context=None): + """Convert to a string, using engineering notation if an exponent is needed. + + Engineering notation has an exponent which is a multiple of 3. This + can leave up to 3 digits to the left of the decimal place and may + require the addition of either one or two trailing zeros. + """ + return self.__str__(eng=True, context=context) + + def __neg__(self, context=None): + """Returns a copy with the sign switched. + + Rounds, if it has reason. + """ + if self._is_special: + ans = self._check_nans(context=context) + if ans: + return ans + + if context is None: + context = getcontext() + + if not self and context.rounding != ROUND_FLOOR: + # -Decimal('0') is Decimal('0'), not Decimal('-0'), except + # in ROUND_FLOOR rounding mode. + ans = self.copy_abs() + else: + ans = self.copy_negate() + + return ans._fix(context) + + def __pos__(self, context=None): + """Returns a copy, unless it is a sNaN. + + Rounds the number (if more than precision digits) + """ + if self._is_special: + ans = self._check_nans(context=context) + if ans: + return ans + + if context is None: + context = getcontext() + + if not self and context.rounding != ROUND_FLOOR: + # + (-0) = 0, except in ROUND_FLOOR rounding mode. + ans = self.copy_abs() + else: + ans = Decimal(self) + + return ans._fix(context) + + def __abs__(self, round=True, context=None): + """Returns the absolute value of self. + + If the keyword argument 'round' is false, do not round. The + expression self.__abs__(round=False) is equivalent to + self.copy_abs(). + """ + if not round: + return self.copy_abs() + + if self._is_special: + ans = self._check_nans(context=context) + if ans: + return ans + + if self._sign: + ans = self.__neg__(context=context) + else: + ans = self.__pos__(context=context) + + return ans + + def __add__(self, other, context=None): + """Returns self + other. + + -INF + INF (or the reverse) cause InvalidOperation errors. + """ + other = _convert_other(other) + if other is NotImplemented: + return other + + if context is None: + context = getcontext() + + if self._is_special or other._is_special: + ans = self._check_nans(other, context) + if ans: + return ans + + if self._isinfinity(): + # If both INF, same sign => same as both, opposite => error. + if self._sign != other._sign and other._isinfinity(): + return context._raise_error(InvalidOperation, '-INF + INF') + return Decimal(self) + if other._isinfinity(): + return Decimal(other) # Can't both be infinity here + + exp = min(self._exp, other._exp) + negativezero = 0 + if context.rounding == ROUND_FLOOR and self._sign != other._sign: + # If the answer is 0, the sign should be negative, in this case. + negativezero = 1 + + if not self and not other: + sign = min(self._sign, other._sign) + if negativezero: + sign = 1 + ans = _dec_from_triple(sign, '0', exp) + ans = ans._fix(context) + return ans + if not self: + exp = max(exp, other._exp - context.prec-1) + ans = other._rescale(exp, context.rounding) + ans = ans._fix(context) + return ans + if not other: + exp = max(exp, self._exp - context.prec-1) + ans = self._rescale(exp, context.rounding) + ans = ans._fix(context) + return ans + + op1 = _WorkRep(self) + op2 = _WorkRep(other) + op1, op2 = _normalize(op1, op2, context.prec) + + result = _WorkRep() + if op1.sign != op2.sign: + # Equal and opposite + if op1.int == op2.int: + ans = _dec_from_triple(negativezero, '0', exp) + ans = ans._fix(context) + return ans + if op1.int < op2.int: + op1, op2 = op2, op1 + # OK, now abs(op1) > abs(op2) + if op1.sign == 1: + result.sign = 1 + op1.sign, op2.sign = op2.sign, op1.sign + else: + result.sign = 0 + # So we know the sign, and op1 > 0. + elif op1.sign == 1: + result.sign = 1 + op1.sign, op2.sign = (0, 0) + else: + result.sign = 0 + # Now, op1 > abs(op2) > 0 + + if op2.sign == 0: + result.int = op1.int + op2.int + else: + result.int = op1.int - op2.int + + result.exp = op1.exp + ans = Decimal(result) + ans = ans._fix(context) + return ans + + __radd__ = __add__ + + def __sub__(self, other, context=None): + """Return self - other""" + other = _convert_other(other) + if other is NotImplemented: + return other + + if self._is_special or other._is_special: + ans = self._check_nans(other, context=context) + if ans: + return ans + + # self - other is computed as self + other.copy_negate() + return self.__add__(other.copy_negate(), context=context) + + def __rsub__(self, other, context=None): + """Return other - self""" + other = _convert_other(other) + if other is NotImplemented: + return other + + return other.__sub__(self, context=context) + + def __mul__(self, other, context=None): + """Return self * other. + + (+-) INF * 0 (or its reverse) raise InvalidOperation. + """ + other = _convert_other(other) + if other is NotImplemented: + return other + + if context is None: + context = getcontext() + + resultsign = self._sign ^ other._sign + + if self._is_special or other._is_special: + ans = self._check_nans(other, context) + if ans: + return ans + + if self._isinfinity(): + if not other: + return context._raise_error(InvalidOperation, '(+-)INF * 0') + return _SignedInfinity[resultsign] + + if other._isinfinity(): + if not self: + return context._raise_error(InvalidOperation, '0 * (+-)INF') + return _SignedInfinity[resultsign] + + resultexp = self._exp + other._exp + + # Special case for multiplying by zero + if not self or not other: + ans = _dec_from_triple(resultsign, '0', resultexp) + # Fixing in case the exponent is out of bounds + ans = ans._fix(context) + return ans + + # Special case for multiplying by power of 10 + if self._int == '1': + ans = _dec_from_triple(resultsign, other._int, resultexp) + ans = ans._fix(context) + return ans + if other._int == '1': + ans = _dec_from_triple(resultsign, self._int, resultexp) + ans = ans._fix(context) + return ans + + op1 = _WorkRep(self) + op2 = _WorkRep(other) + + ans = _dec_from_triple(resultsign, str(op1.int * op2.int), resultexp) + ans = ans._fix(context) + + return ans + __rmul__ = __mul__ + + def __truediv__(self, other, context=None): + """Return self / other.""" + other = _convert_other(other) + if other is NotImplemented: + return NotImplemented + + if context is None: + context = getcontext() + + sign = self._sign ^ other._sign + + if self._is_special or other._is_special: + ans = self._check_nans(other, context) + if ans: + return ans + + if self._isinfinity() and other._isinfinity(): + return context._raise_error(InvalidOperation, '(+-)INF/(+-)INF') + + if self._isinfinity(): + return _SignedInfinity[sign] + + if other._isinfinity(): + context._raise_error(Clamped, 'Division by infinity') + return _dec_from_triple(sign, '0', context.Etiny()) + + # Special cases for zeroes + if not other: + if not self: + return context._raise_error(DivisionUndefined, '0 / 0') + return context._raise_error(DivisionByZero, 'x / 0', sign) + + if not self: + exp = self._exp - other._exp + coeff = 0 + else: + # OK, so neither = 0, INF or NaN + shift = len(other._int) - len(self._int) + context.prec + 1 + exp = self._exp - other._exp - shift + op1 = _WorkRep(self) + op2 = _WorkRep(other) + if shift >= 0: + coeff, remainder = divmod(op1.int * 10**shift, op2.int) + else: + coeff, remainder = divmod(op1.int, op2.int * 10**-shift) + if remainder: + # result is not exact; adjust to ensure correct rounding + if coeff % 5 == 0: + coeff += 1 + else: + # result is exact; get as close to ideal exponent as possible + ideal_exp = self._exp - other._exp + while exp < ideal_exp and coeff % 10 == 0: + coeff //= 10 + exp += 1 + + ans = _dec_from_triple(sign, str(coeff), exp) + return ans._fix(context) + + def _divide(self, other, context): + """Return (self // other, self % other), to context.prec precision. + + Assumes that neither self nor other is a NaN, that self is not + infinite and that other is nonzero. + """ + sign = self._sign ^ other._sign + if other._isinfinity(): + ideal_exp = self._exp + else: + ideal_exp = min(self._exp, other._exp) + + expdiff = self.adjusted() - other.adjusted() + if not self or other._isinfinity() or expdiff <= -2: + return (_dec_from_triple(sign, '0', 0), + self._rescale(ideal_exp, context.rounding)) + if expdiff <= context.prec: + op1 = _WorkRep(self) + op2 = _WorkRep(other) + if op1.exp >= op2.exp: + op1.int *= 10**(op1.exp - op2.exp) + else: + op2.int *= 10**(op2.exp - op1.exp) + q, r = divmod(op1.int, op2.int) + if q < 10**context.prec: + return (_dec_from_triple(sign, str(q), 0), + _dec_from_triple(self._sign, str(r), ideal_exp)) + + # Here the quotient is too large to be representable + ans = context._raise_error(DivisionImpossible, + 'quotient too large in //, % or divmod') + return ans, ans + + def __rtruediv__(self, other, context=None): + """Swaps self/other and returns __truediv__.""" + other = _convert_other(other) + if other is NotImplemented: + return other + return other.__truediv__(self, context=context) + + def __divmod__(self, other, context=None): + """ + Return (self // other, self % other) + """ + other = _convert_other(other) + if other is NotImplemented: + return other + + if context is None: + context = getcontext() + + ans = self._check_nans(other, context) + if ans: + return (ans, ans) + + sign = self._sign ^ other._sign + if self._isinfinity(): + if other._isinfinity(): + ans = context._raise_error(InvalidOperation, 'divmod(INF, INF)') + return ans, ans + else: + return (_SignedInfinity[sign], + context._raise_error(InvalidOperation, 'INF % x')) + + if not other: + if not self: + ans = context._raise_error(DivisionUndefined, 'divmod(0, 0)') + return ans, ans + else: + return (context._raise_error(DivisionByZero, 'x // 0', sign), + context._raise_error(InvalidOperation, 'x % 0')) + + quotient, remainder = self._divide(other, context) + remainder = remainder._fix(context) + return quotient, remainder + + def __rdivmod__(self, other, context=None): + """Swaps self/other and returns __divmod__.""" + other = _convert_other(other) + if other is NotImplemented: + return other + return other.__divmod__(self, context=context) + + def __mod__(self, other, context=None): + """ + self % other + """ + other = _convert_other(other) + if other is NotImplemented: + return other + + if context is None: + context = getcontext() + + ans = self._check_nans(other, context) + if ans: + return ans + + if self._isinfinity(): + return context._raise_error(InvalidOperation, 'INF % x') + elif not other: + if self: + return context._raise_error(InvalidOperation, 'x % 0') + else: + return context._raise_error(DivisionUndefined, '0 % 0') + + remainder = self._divide(other, context)[1] + remainder = remainder._fix(context) + return remainder + + def __rmod__(self, other, context=None): + """Swaps self/other and returns __mod__.""" + other = _convert_other(other) + if other is NotImplemented: + return other + return other.__mod__(self, context=context) + + def remainder_near(self, other, context=None): + """ + Remainder nearest to 0- abs(remainder-near) <= other/2 + """ + if context is None: + context = getcontext() + + other = _convert_other(other, raiseit=True) + + ans = self._check_nans(other, context) + if ans: + return ans + + # self == +/-infinity -> InvalidOperation + if self._isinfinity(): + return context._raise_error(InvalidOperation, + 'remainder_near(infinity, x)') + + # other == 0 -> either InvalidOperation or DivisionUndefined + if not other: + if self: + return context._raise_error(InvalidOperation, + 'remainder_near(x, 0)') + else: + return context._raise_error(DivisionUndefined, + 'remainder_near(0, 0)') + + # other = +/-infinity -> remainder = self + if other._isinfinity(): + ans = Decimal(self) + return ans._fix(context) + + # self = 0 -> remainder = self, with ideal exponent + ideal_exponent = min(self._exp, other._exp) + if not self: + ans = _dec_from_triple(self._sign, '0', ideal_exponent) + return ans._fix(context) + + # catch most cases of large or small quotient + expdiff = self.adjusted() - other.adjusted() + if expdiff >= context.prec + 1: + # expdiff >= prec+1 => abs(self/other) > 10**prec + return context._raise_error(DivisionImpossible) + if expdiff <= -2: + # expdiff <= -2 => abs(self/other) < 0.1 + ans = self._rescale(ideal_exponent, context.rounding) + return ans._fix(context) + + # adjust both arguments to have the same exponent, then divide + op1 = _WorkRep(self) + op2 = _WorkRep(other) + if op1.exp >= op2.exp: + op1.int *= 10**(op1.exp - op2.exp) + else: + op2.int *= 10**(op2.exp - op1.exp) + q, r = divmod(op1.int, op2.int) + # remainder is r*10**ideal_exponent; other is +/-op2.int * + # 10**ideal_exponent. Apply correction to ensure that + # abs(remainder) <= abs(other)/2 + if 2*r + (q&1) > op2.int: + r -= op2.int + q += 1 + + if q >= 10**context.prec: + return context._raise_error(DivisionImpossible) + + # result has same sign as self unless r is negative + sign = self._sign + if r < 0: + sign = 1-sign + r = -r + + ans = _dec_from_triple(sign, str(r), ideal_exponent) + return ans._fix(context) + + def __floordiv__(self, other, context=None): + """self // other""" + other = _convert_other(other) + if other is NotImplemented: + return other + + if context is None: + context = getcontext() + + ans = self._check_nans(other, context) + if ans: + return ans + + if self._isinfinity(): + if other._isinfinity(): + return context._raise_error(InvalidOperation, 'INF // INF') + else: + return _SignedInfinity[self._sign ^ other._sign] + + if not other: + if self: + return context._raise_error(DivisionByZero, 'x // 0', + self._sign ^ other._sign) + else: + return context._raise_error(DivisionUndefined, '0 // 0') + + return self._divide(other, context)[0] + + def __rfloordiv__(self, other, context=None): + """Swaps self/other and returns __floordiv__.""" + other = _convert_other(other) + if other is NotImplemented: + return other + return other.__floordiv__(self, context=context) + + def __float__(self): + """Float representation.""" + if self._isnan(): + if self.is_snan(): + raise ValueError("Cannot convert signaling NaN to float") + s = "-nan" if self._sign else "nan" + else: + s = str(self) + return float(s) + + def __int__(self): + """Converts self to an int, truncating if necessary.""" + if self._is_special: + if self._isnan(): + raise ValueError("Cannot convert NaN to integer") + elif self._isinfinity(): + raise OverflowError("Cannot convert infinity to integer") + s = (-1)**self._sign + if self._exp >= 0: + return s*int(self._int)*10**self._exp + else: + return s*int(self._int[:self._exp] or '0') + + __trunc__ = __int__ + + def real(self): + return self + real = property(real) + + def imag(self): + return Decimal(0) + imag = property(imag) + + def conjugate(self): + return self + + def __complex__(self): + return complex(float(self)) + + def _fix_nan(self, context): + """Decapitate the payload of a NaN to fit the context""" + payload = self._int + + # maximum length of payload is precision if clamp=0, + # precision-1 if clamp=1. + max_payload_len = context.prec - context.clamp + if len(payload) > max_payload_len: + payload = payload[len(payload)-max_payload_len:].lstrip('0') + return _dec_from_triple(self._sign, payload, self._exp, True) + return Decimal(self) + + def _fix(self, context): + """Round if it is necessary to keep self within prec precision. + + Rounds and fixes the exponent. Does not raise on a sNaN. + + Arguments: + self - Decimal instance + context - context used. + """ + + if self._is_special: + if self._isnan(): + # decapitate payload if necessary + return self._fix_nan(context) + else: + # self is +/-Infinity; return unaltered + return Decimal(self) + + # if self is zero then exponent should be between Etiny and + # Emax if clamp==0, and between Etiny and Etop if clamp==1. + Etiny = context.Etiny() + Etop = context.Etop() + if not self: + exp_max = [context.Emax, Etop][context.clamp] + new_exp = min(max(self._exp, Etiny), exp_max) + if new_exp != self._exp: + context._raise_error(Clamped) + return _dec_from_triple(self._sign, '0', new_exp) + else: + return Decimal(self) + + # exp_min is the smallest allowable exponent of the result, + # equal to max(self.adjusted()-context.prec+1, Etiny) + exp_min = len(self._int) + self._exp - context.prec + if exp_min > Etop: + # overflow: exp_min > Etop iff self.adjusted() > Emax + ans = context._raise_error(Overflow, 'above Emax', self._sign) + context._raise_error(Inexact) + context._raise_error(Rounded) + return ans + + self_is_subnormal = exp_min < Etiny + if self_is_subnormal: + exp_min = Etiny + + # round if self has too many digits + if self._exp < exp_min: + digits = len(self._int) + self._exp - exp_min + if digits < 0: + self = _dec_from_triple(self._sign, '1', exp_min-1) + digits = 0 + rounding_method = self._pick_rounding_function[context.rounding] + changed = rounding_method(self, digits) + coeff = self._int[:digits] or '0' + if changed > 0: + coeff = str(int(coeff)+1) + if len(coeff) > context.prec: + coeff = coeff[:-1] + exp_min += 1 + + # check whether the rounding pushed the exponent out of range + if exp_min > Etop: + ans = context._raise_error(Overflow, 'above Emax', self._sign) + else: + ans = _dec_from_triple(self._sign, coeff, exp_min) + + # raise the appropriate signals, taking care to respect + # the precedence described in the specification + if changed and self_is_subnormal: + context._raise_error(Underflow) + if self_is_subnormal: + context._raise_error(Subnormal) + if changed: + context._raise_error(Inexact) + context._raise_error(Rounded) + if not ans: + # raise Clamped on underflow to 0 + context._raise_error(Clamped) + return ans + + if self_is_subnormal: + context._raise_error(Subnormal) + + # fold down if clamp == 1 and self has too few digits + if context.clamp == 1 and self._exp > Etop: + context._raise_error(Clamped) + self_padded = self._int + '0'*(self._exp - Etop) + return _dec_from_triple(self._sign, self_padded, Etop) + + # here self was representable to begin with; return unchanged + return Decimal(self) + + # for each of the rounding functions below: + # self is a finite, nonzero Decimal + # prec is an integer satisfying 0 <= prec < len(self._int) + # + # each function returns either -1, 0, or 1, as follows: + # 1 indicates that self should be rounded up (away from zero) + # 0 indicates that self should be truncated, and that all the + # digits to be truncated are zeros (so the value is unchanged) + # -1 indicates that there are nonzero digits to be truncated + + def _round_down(self, prec): + """Also known as round-towards-0, truncate.""" + if _all_zeros(self._int, prec): + return 0 + else: + return -1 + + def _round_up(self, prec): + """Rounds away from 0.""" + return -self._round_down(prec) + + def _round_half_up(self, prec): + """Rounds 5 up (away from 0)""" + if self._int[prec] in '56789': + return 1 + elif _all_zeros(self._int, prec): + return 0 + else: + return -1 + + def _round_half_down(self, prec): + """Round 5 down""" + if _exact_half(self._int, prec): + return -1 + else: + return self._round_half_up(prec) + + def _round_half_even(self, prec): + """Round 5 to even, rest to nearest.""" + if _exact_half(self._int, prec) and \ + (prec == 0 or self._int[prec-1] in '02468'): + return -1 + else: + return self._round_half_up(prec) + + def _round_ceiling(self, prec): + """Rounds up (not away from 0 if negative.)""" + if self._sign: + return self._round_down(prec) + else: + return -self._round_down(prec) + + def _round_floor(self, prec): + """Rounds down (not towards 0 if negative)""" + if not self._sign: + return self._round_down(prec) + else: + return -self._round_down(prec) + + def _round_05up(self, prec): + """Round down unless digit prec-1 is 0 or 5.""" + if prec and self._int[prec-1] not in '05': + return self._round_down(prec) + else: + return -self._round_down(prec) + + _pick_rounding_function = dict( + ROUND_DOWN = _round_down, + ROUND_UP = _round_up, + ROUND_HALF_UP = _round_half_up, + ROUND_HALF_DOWN = _round_half_down, + ROUND_HALF_EVEN = _round_half_even, + ROUND_CEILING = _round_ceiling, + ROUND_FLOOR = _round_floor, + ROUND_05UP = _round_05up, + ) + + def __round__(self, n=None): + """Round self to the nearest integer, or to a given precision. + + If only one argument is supplied, round a finite Decimal + instance self to the nearest integer. If self is infinite or + a NaN then a Python exception is raised. If self is finite + and lies exactly halfway between two integers then it is + rounded to the integer with even last digit. + + >>> round(Decimal('123.456')) + 123 + >>> round(Decimal('-456.789')) + -457 + >>> round(Decimal('-3.0')) + -3 + >>> round(Decimal('2.5')) + 2 + >>> round(Decimal('3.5')) + 4 + >>> round(Decimal('Inf')) + Traceback (most recent call last): + ... + OverflowError: cannot round an infinity + >>> round(Decimal('NaN')) + Traceback (most recent call last): + ... + ValueError: cannot round a NaN + + If a second argument n is supplied, self is rounded to n + decimal places using the rounding mode for the current + context. + + For an integer n, round(self, -n) is exactly equivalent to + self.quantize(Decimal('1En')). + + >>> round(Decimal('123.456'), 0) + Decimal('123') + >>> round(Decimal('123.456'), 2) + Decimal('123.46') + >>> round(Decimal('123.456'), -2) + Decimal('1E+2') + >>> round(Decimal('-Infinity'), 37) + Decimal('NaN') + >>> round(Decimal('sNaN123'), 0) + Decimal('NaN123') + + """ + if n is not None: + # two-argument form: use the equivalent quantize call + if not isinstance(n, int): + raise TypeError('Second argument to round should be integral') + exp = _dec_from_triple(0, '1', -n) + return self.quantize(exp) + + # one-argument form + if self._is_special: + if self.is_nan(): + raise ValueError("cannot round a NaN") + else: + raise OverflowError("cannot round an infinity") + return int(self._rescale(0, ROUND_HALF_EVEN)) + + def __floor__(self): + """Return the floor of self, as an integer. + + For a finite Decimal instance self, return the greatest + integer n such that n <= self. If self is infinite or a NaN + then a Python exception is raised. + + """ + if self._is_special: + if self.is_nan(): + raise ValueError("cannot round a NaN") + else: + raise OverflowError("cannot round an infinity") + return int(self._rescale(0, ROUND_FLOOR)) + + def __ceil__(self): + """Return the ceiling of self, as an integer. + + For a finite Decimal instance self, return the least integer n + such that n >= self. If self is infinite or a NaN then a + Python exception is raised. + + """ + if self._is_special: + if self.is_nan(): + raise ValueError("cannot round a NaN") + else: + raise OverflowError("cannot round an infinity") + return int(self._rescale(0, ROUND_CEILING)) + + def fma(self, other, third, context=None): + """Fused multiply-add. + + Returns self*other+third with no rounding of the intermediate + product self*other. + + self and other are multiplied together, with no rounding of + the result. The third operand is then added to the result, + and a single final rounding is performed. + """ + + other = _convert_other(other, raiseit=True) + third = _convert_other(third, raiseit=True) + + # compute product; raise InvalidOperation if either operand is + # a signaling NaN or if the product is zero times infinity. + if self._is_special or other._is_special: + if context is None: + context = getcontext() + if self._exp == 'N': + return context._raise_error(InvalidOperation, 'sNaN', self) + if other._exp == 'N': + return context._raise_error(InvalidOperation, 'sNaN', other) + if self._exp == 'n': + product = self + elif other._exp == 'n': + product = other + elif self._exp == 'F': + if not other: + return context._raise_error(InvalidOperation, + 'INF * 0 in fma') + product = _SignedInfinity[self._sign ^ other._sign] + elif other._exp == 'F': + if not self: + return context._raise_error(InvalidOperation, + '0 * INF in fma') + product = _SignedInfinity[self._sign ^ other._sign] + else: + product = _dec_from_triple(self._sign ^ other._sign, + str(int(self._int) * int(other._int)), + self._exp + other._exp) + + return product.__add__(third, context) + + def _power_modulo(self, other, modulo, context=None): + """Three argument version of __pow__""" + + other = _convert_other(other) + if other is NotImplemented: + return other + modulo = _convert_other(modulo) + if modulo is NotImplemented: + return modulo + + if context is None: + context = getcontext() + + # deal with NaNs: if there are any sNaNs then first one wins, + # (i.e. behaviour for NaNs is identical to that of fma) + self_is_nan = self._isnan() + other_is_nan = other._isnan() + modulo_is_nan = modulo._isnan() + if self_is_nan or other_is_nan or modulo_is_nan: + if self_is_nan == 2: + return context._raise_error(InvalidOperation, 'sNaN', + self) + if other_is_nan == 2: + return context._raise_error(InvalidOperation, 'sNaN', + other) + if modulo_is_nan == 2: + return context._raise_error(InvalidOperation, 'sNaN', + modulo) + if self_is_nan: + return self._fix_nan(context) + if other_is_nan: + return other._fix_nan(context) + return modulo._fix_nan(context) + + # check inputs: we apply same restrictions as Python's pow() + if not (self._isinteger() and + other._isinteger() and + modulo._isinteger()): + return context._raise_error(InvalidOperation, + 'pow() 3rd argument not allowed ' + 'unless all arguments are integers') + if other < 0: + return context._raise_error(InvalidOperation, + 'pow() 2nd argument cannot be ' + 'negative when 3rd argument specified') + if not modulo: + return context._raise_error(InvalidOperation, + 'pow() 3rd argument cannot be 0') + + # additional restriction for decimal: the modulus must be less + # than 10**prec in absolute value + if modulo.adjusted() >= context.prec: + return context._raise_error(InvalidOperation, + 'insufficient precision: pow() 3rd ' + 'argument must not have more than ' + 'precision digits') + + # define 0**0 == NaN, for consistency with two-argument pow + # (even though it hurts!) + if not other and not self: + return context._raise_error(InvalidOperation, + 'at least one of pow() 1st argument ' + 'and 2nd argument must be nonzero ;' + '0**0 is not defined') + + # compute sign of result + if other._iseven(): + sign = 0 + else: + sign = self._sign + + # convert modulo to a Python integer, and self and other to + # Decimal integers (i.e. force their exponents to be >= 0) + modulo = abs(int(modulo)) + base = _WorkRep(self.to_integral_value()) + exponent = _WorkRep(other.to_integral_value()) + + # compute result using integer pow() + base = (base.int % modulo * pow(10, base.exp, modulo)) % modulo + for i in range(exponent.exp): + base = pow(base, 10, modulo) + base = pow(base, exponent.int, modulo) + + return _dec_from_triple(sign, str(base), 0) + + def _power_exact(self, other, p): + """Attempt to compute self**other exactly. + + Given Decimals self and other and an integer p, attempt to + compute an exact result for the power self**other, with p + digits of precision. Return None if self**other is not + exactly representable in p digits. + + Assumes that elimination of special cases has already been + performed: self and other must both be nonspecial; self must + be positive and not numerically equal to 1; other must be + nonzero. For efficiency, other._exp should not be too large, + so that 10**abs(other._exp) is a feasible calculation.""" + + # In the comments below, we write x for the value of self and y for the + # value of other. Write x = xc*10**xe and abs(y) = yc*10**ye, with xc + # and yc positive integers not divisible by 10. + + # The main purpose of this method is to identify the *failure* + # of x**y to be exactly representable with as little effort as + # possible. So we look for cheap and easy tests that + # eliminate the possibility of x**y being exact. Only if all + # these tests are passed do we go on to actually compute x**y. + + # Here's the main idea. Express y as a rational number m/n, with m and + # n relatively prime and n>0. Then for x**y to be exactly + # representable (at *any* precision), xc must be the nth power of a + # positive integer and xe must be divisible by n. If y is negative + # then additionally xc must be a power of either 2 or 5, hence a power + # of 2**n or 5**n. + # + # There's a limit to how small |y| can be: if y=m/n as above + # then: + # + # (1) if xc != 1 then for the result to be representable we + # need xc**(1/n) >= 2, and hence also xc**|y| >= 2. So + # if |y| <= 1/nbits(xc) then xc < 2**nbits(xc) <= + # 2**(1/|y|), hence xc**|y| < 2 and the result is not + # representable. + # + # (2) if xe != 0, |xe|*(1/n) >= 1, so |xe|*|y| >= 1. Hence if + # |y| < 1/|xe| then the result is not representable. + # + # Note that since x is not equal to 1, at least one of (1) and + # (2) must apply. Now |y| < 1/nbits(xc) iff |yc|*nbits(xc) < + # 10**-ye iff len(str(|yc|*nbits(xc)) <= -ye. + # + # There's also a limit to how large y can be, at least if it's + # positive: the normalized result will have coefficient xc**y, + # so if it's representable then xc**y < 10**p, and y < + # p/log10(xc). Hence if y*log10(xc) >= p then the result is + # not exactly representable. + + # if len(str(abs(yc*xe)) <= -ye then abs(yc*xe) < 10**-ye, + # so |y| < 1/xe and the result is not representable. + # Similarly, len(str(abs(yc)*xc_bits)) <= -ye implies |y| + # < 1/nbits(xc). + + x = _WorkRep(self) + xc, xe = x.int, x.exp + while xc % 10 == 0: + xc //= 10 + xe += 1 + + y = _WorkRep(other) + yc, ye = y.int, y.exp + while yc % 10 == 0: + yc //= 10 + ye += 1 + + # case where xc == 1: result is 10**(xe*y), with xe*y + # required to be an integer + if xc == 1: + xe *= yc + # result is now 10**(xe * 10**ye); xe * 10**ye must be integral + while xe % 10 == 0: + xe //= 10 + ye += 1 + if ye < 0: + return None + exponent = xe * 10**ye + if y.sign == 1: + exponent = -exponent + # if other is a nonnegative integer, use ideal exponent + if other._isinteger() and other._sign == 0: + ideal_exponent = self._exp*int(other) + zeros = min(exponent-ideal_exponent, p-1) + else: + zeros = 0 + return _dec_from_triple(0, '1' + '0'*zeros, exponent-zeros) + + # case where y is negative: xc must be either a power + # of 2 or a power of 5. + if y.sign == 1: + last_digit = xc % 10 + if last_digit in (2,4,6,8): + # quick test for power of 2 + if xc & -xc != xc: + return None + # now xc is a power of 2; e is its exponent + e = _nbits(xc)-1 + + # We now have: + # + # x = 2**e * 10**xe, e > 0, and y < 0. + # + # The exact result is: + # + # x**y = 5**(-e*y) * 10**(e*y + xe*y) + # + # provided that both e*y and xe*y are integers. Note that if + # 5**(-e*y) >= 10**p, then the result can't be expressed + # exactly with p digits of precision. + # + # Using the above, we can guard against large values of ye. + # 93/65 is an upper bound for log(10)/log(5), so if + # + # ye >= len(str(93*p//65)) + # + # then + # + # -e*y >= -y >= 10**ye > 93*p/65 > p*log(10)/log(5), + # + # so 5**(-e*y) >= 10**p, and the coefficient of the result + # can't be expressed in p digits. + + # emax >= largest e such that 5**e < 10**p. + emax = p*93//65 + if ye >= len(str(emax)): + return None + + # Find -e*y and -xe*y; both must be integers + e = _decimal_lshift_exact(e * yc, ye) + xe = _decimal_lshift_exact(xe * yc, ye) + if e is None or xe is None: + return None + + if e > emax: + return None + xc = 5**e + + elif last_digit == 5: + # e >= log_5(xc) if xc is a power of 5; we have + # equality all the way up to xc=5**2658 + e = _nbits(xc)*28//65 + xc, remainder = divmod(5**e, xc) + if remainder: + return None + while xc % 5 == 0: + xc //= 5 + e -= 1 + + # Guard against large values of ye, using the same logic as in + # the 'xc is a power of 2' branch. 10/3 is an upper bound for + # log(10)/log(2). + emax = p*10//3 + if ye >= len(str(emax)): + return None + + e = _decimal_lshift_exact(e * yc, ye) + xe = _decimal_lshift_exact(xe * yc, ye) + if e is None or xe is None: + return None + + if e > emax: + return None + xc = 2**e + else: + return None + + if xc >= 10**p: + return None + xe = -e-xe + return _dec_from_triple(0, str(xc), xe) + + # now y is positive; find m and n such that y = m/n + if ye >= 0: + m, n = yc*10**ye, 1 + else: + if xe != 0 and len(str(abs(yc*xe))) <= -ye: + return None + xc_bits = _nbits(xc) + if xc != 1 and len(str(abs(yc)*xc_bits)) <= -ye: + return None + m, n = yc, 10**(-ye) + while m % 2 == n % 2 == 0: + m //= 2 + n //= 2 + while m % 5 == n % 5 == 0: + m //= 5 + n //= 5 + + # compute nth root of xc*10**xe + if n > 1: + # if 1 < xc < 2**n then xc isn't an nth power + if xc != 1 and xc_bits <= n: + return None + + xe, rem = divmod(xe, n) + if rem != 0: + return None + + # compute nth root of xc using Newton's method + a = 1 << -(-_nbits(xc)//n) # initial estimate + while True: + q, r = divmod(xc, a**(n-1)) + if a <= q: + break + else: + a = (a*(n-1) + q)//n + if not (a == q and r == 0): + return None + xc = a + + # now xc*10**xe is the nth root of the original xc*10**xe + # compute mth power of xc*10**xe + + # if m > p*100//_log10_lb(xc) then m > p/log10(xc), hence xc**m > + # 10**p and the result is not representable. + if xc > 1 and m > p*100//_log10_lb(xc): + return None + xc = xc**m + xe *= m + if xc > 10**p: + return None + + # by this point the result *is* exactly representable + # adjust the exponent to get as close as possible to the ideal + # exponent, if necessary + str_xc = str(xc) + if other._isinteger() and other._sign == 0: + ideal_exponent = self._exp*int(other) + zeros = min(xe-ideal_exponent, p-len(str_xc)) + else: + zeros = 0 + return _dec_from_triple(0, str_xc+'0'*zeros, xe-zeros) + + def __pow__(self, other, modulo=None, context=None): + """Return self ** other [ % modulo]. + + With two arguments, compute self**other. + + With three arguments, compute (self**other) % modulo. For the + three argument form, the following restrictions on the + arguments hold: + + - all three arguments must be integral + - other must be nonnegative + - either self or other (or both) must be nonzero + - modulo must be nonzero and must have at most p digits, + where p is the context precision. + + If any of these restrictions is violated the InvalidOperation + flag is raised. + + The result of pow(self, other, modulo) is identical to the + result that would be obtained by computing (self**other) % + modulo with unbounded precision, but is computed more + efficiently. It is always exact. + """ + + if modulo is not None: + return self._power_modulo(other, modulo, context) + + other = _convert_other(other) + if other is NotImplemented: + return other + + if context is None: + context = getcontext() + + # either argument is a NaN => result is NaN + ans = self._check_nans(other, context) + if ans: + return ans + + # 0**0 = NaN (!), x**0 = 1 for nonzero x (including +/-Infinity) + if not other: + if not self: + return context._raise_error(InvalidOperation, '0 ** 0') + else: + return _One + + # result has sign 1 iff self._sign is 1 and other is an odd integer + result_sign = 0 + if self._sign == 1: + if other._isinteger(): + if not other._iseven(): + result_sign = 1 + else: + # -ve**noninteger = NaN + # (-0)**noninteger = 0**noninteger + if self: + return context._raise_error(InvalidOperation, + 'x ** y with x negative and y not an integer') + # negate self, without doing any unwanted rounding + self = self.copy_negate() + + # 0**(+ve or Inf)= 0; 0**(-ve or -Inf) = Infinity + if not self: + if other._sign == 0: + return _dec_from_triple(result_sign, '0', 0) + else: + return _SignedInfinity[result_sign] + + # Inf**(+ve or Inf) = Inf; Inf**(-ve or -Inf) = 0 + if self._isinfinity(): + if other._sign == 0: + return _SignedInfinity[result_sign] + else: + return _dec_from_triple(result_sign, '0', 0) + + # 1**other = 1, but the choice of exponent and the flags + # depend on the exponent of self, and on whether other is a + # positive integer, a negative integer, or neither + if self == _One: + if other._isinteger(): + # exp = max(self._exp*max(int(other), 0), + # 1-context.prec) but evaluating int(other) directly + # is dangerous until we know other is small (other + # could be 1e999999999) + if other._sign == 1: + multiplier = 0 + elif other > context.prec: + multiplier = context.prec + else: + multiplier = int(other) + + exp = self._exp * multiplier + if exp < 1-context.prec: + exp = 1-context.prec + context._raise_error(Rounded) + else: + context._raise_error(Inexact) + context._raise_error(Rounded) + exp = 1-context.prec + + return _dec_from_triple(result_sign, '1'+'0'*-exp, exp) + + # compute adjusted exponent of self + self_adj = self.adjusted() + + # self ** infinity is infinity if self > 1, 0 if self < 1 + # self ** -infinity is infinity if self < 1, 0 if self > 1 + if other._isinfinity(): + if (other._sign == 0) == (self_adj < 0): + return _dec_from_triple(result_sign, '0', 0) + else: + return _SignedInfinity[result_sign] + + # from here on, the result always goes through the call + # to _fix at the end of this function. + ans = None + exact = False + + # crude test to catch cases of extreme overflow/underflow. If + # log10(self)*other >= 10**bound and bound >= len(str(Emax)) + # then 10**bound >= 10**len(str(Emax)) >= Emax+1 and hence + # self**other >= 10**(Emax+1), so overflow occurs. The test + # for underflow is similar. + bound = self._log10_exp_bound() + other.adjusted() + if (self_adj >= 0) == (other._sign == 0): + # self > 1 and other +ve, or self < 1 and other -ve + # possibility of overflow + if bound >= len(str(context.Emax)): + ans = _dec_from_triple(result_sign, '1', context.Emax+1) + else: + # self > 1 and other -ve, or self < 1 and other +ve + # possibility of underflow to 0 + Etiny = context.Etiny() + if bound >= len(str(-Etiny)): + ans = _dec_from_triple(result_sign, '1', Etiny-1) + + # try for an exact result with precision +1 + if ans is None: + ans = self._power_exact(other, context.prec + 1) + if ans is not None: + if result_sign == 1: + ans = _dec_from_triple(1, ans._int, ans._exp) + exact = True + + # usual case: inexact result, x**y computed directly as exp(y*log(x)) + if ans is None: + p = context.prec + x = _WorkRep(self) + xc, xe = x.int, x.exp + y = _WorkRep(other) + yc, ye = y.int, y.exp + if y.sign == 1: + yc = -yc + + # compute correctly rounded result: start with precision +3, + # then increase precision until result is unambiguously roundable + extra = 3 + while True: + coeff, exp = _dpower(xc, xe, yc, ye, p+extra) + if coeff % (5*10**(len(str(coeff))-p-1)): + break + extra += 3 + + ans = _dec_from_triple(result_sign, str(coeff), exp) + + # unlike exp, ln and log10, the power function respects the + # rounding mode; no need to switch to ROUND_HALF_EVEN here + + # There's a difficulty here when 'other' is not an integer and + # the result is exact. In this case, the specification + # requires that the Inexact flag be raised (in spite of + # exactness), but since the result is exact _fix won't do this + # for us. (Correspondingly, the Underflow signal should also + # be raised for subnormal results.) We can't directly raise + # these signals either before or after calling _fix, since + # that would violate the precedence for signals. So we wrap + # the ._fix call in a temporary context, and reraise + # afterwards. + if exact and not other._isinteger(): + # pad with zeros up to length context.prec+1 if necessary; this + # ensures that the Rounded signal will be raised. + if len(ans._int) <= context.prec: + expdiff = context.prec + 1 - len(ans._int) + ans = _dec_from_triple(ans._sign, ans._int+'0'*expdiff, + ans._exp-expdiff) + + # create a copy of the current context, with cleared flags/traps + newcontext = context.copy() + newcontext.clear_flags() + for exception in _signals: + newcontext.traps[exception] = 0 + + # round in the new context + ans = ans._fix(newcontext) + + # raise Inexact, and if necessary, Underflow + newcontext._raise_error(Inexact) + if newcontext.flags[Subnormal]: + newcontext._raise_error(Underflow) + + # propagate signals to the original context; _fix could + # have raised any of Overflow, Underflow, Subnormal, + # Inexact, Rounded, Clamped. Overflow needs the correct + # arguments. Note that the order of the exceptions is + # important here. + if newcontext.flags[Overflow]: + context._raise_error(Overflow, 'above Emax', ans._sign) + for exception in Underflow, Subnormal, Inexact, Rounded, Clamped: + if newcontext.flags[exception]: + context._raise_error(exception) + + else: + ans = ans._fix(context) + + return ans + + def __rpow__(self, other, context=None): + """Swaps self/other and returns __pow__.""" + other = _convert_other(other) + if other is NotImplemented: + return other + return other.__pow__(self, context=context) + + def normalize(self, context=None): + """Normalize- strip trailing 0s, change anything equal to 0 to 0e0""" + + if context is None: + context = getcontext() + + if self._is_special: + ans = self._check_nans(context=context) + if ans: + return ans + + dup = self._fix(context) + if dup._isinfinity(): + return dup + + if not dup: + return _dec_from_triple(dup._sign, '0', 0) + exp_max = [context.Emax, context.Etop()][context.clamp] + end = len(dup._int) + exp = dup._exp + while dup._int[end-1] == '0' and exp < exp_max: + exp += 1 + end -= 1 + return _dec_from_triple(dup._sign, dup._int[:end], exp) + + def quantize(self, exp, rounding=None, context=None): + """Quantize self so its exponent is the same as that of exp. + + Similar to self._rescale(exp._exp) but with error checking. + """ + exp = _convert_other(exp, raiseit=True) + + if context is None: + context = getcontext() + if rounding is None: + rounding = context.rounding + + if self._is_special or exp._is_special: + ans = self._check_nans(exp, context) + if ans: + return ans + + if exp._isinfinity() or self._isinfinity(): + if exp._isinfinity() and self._isinfinity(): + return Decimal(self) # if both are inf, it is OK + return context._raise_error(InvalidOperation, + 'quantize with one INF') + + # exp._exp should be between Etiny and Emax + if not (context.Etiny() <= exp._exp <= context.Emax): + return context._raise_error(InvalidOperation, + 'target exponent out of bounds in quantize') + + if not self: + ans = _dec_from_triple(self._sign, '0', exp._exp) + return ans._fix(context) + + self_adjusted = self.adjusted() + if self_adjusted > context.Emax: + return context._raise_error(InvalidOperation, + 'exponent of quantize result too large for current context') + if self_adjusted - exp._exp + 1 > context.prec: + return context._raise_error(InvalidOperation, + 'quantize result has too many digits for current context') + + ans = self._rescale(exp._exp, rounding) + if ans.adjusted() > context.Emax: + return context._raise_error(InvalidOperation, + 'exponent of quantize result too large for current context') + if len(ans._int) > context.prec: + return context._raise_error(InvalidOperation, + 'quantize result has too many digits for current context') + + # raise appropriate flags + if ans and ans.adjusted() < context.Emin: + context._raise_error(Subnormal) + if ans._exp > self._exp: + if ans != self: + context._raise_error(Inexact) + context._raise_error(Rounded) + + # call to fix takes care of any necessary folddown, and + # signals Clamped if necessary + ans = ans._fix(context) + return ans + + def same_quantum(self, other, context=None): + """Return True if self and other have the same exponent; otherwise + return False. + + If either operand is a special value, the following rules are used: + * return True if both operands are infinities + * return True if both operands are NaNs + * otherwise, return False. + """ + other = _convert_other(other, raiseit=True) + if self._is_special or other._is_special: + return (self.is_nan() and other.is_nan() or + self.is_infinite() and other.is_infinite()) + return self._exp == other._exp + + def _rescale(self, exp, rounding): + """Rescale self so that the exponent is exp, either by padding with zeros + or by truncating digits, using the given rounding mode. + + Specials are returned without change. This operation is + quiet: it raises no flags, and uses no information from the + context. + + exp = exp to scale to (an integer) + rounding = rounding mode + """ + if self._is_special: + return Decimal(self) + if not self: + return _dec_from_triple(self._sign, '0', exp) + + if self._exp >= exp: + # pad answer with zeros if necessary + return _dec_from_triple(self._sign, + self._int + '0'*(self._exp - exp), exp) + + # too many digits; round and lose data. If self.adjusted() < + # exp-1, replace self by 10**(exp-1) before rounding + digits = len(self._int) + self._exp - exp + if digits < 0: + self = _dec_from_triple(self._sign, '1', exp-1) + digits = 0 + this_function = self._pick_rounding_function[rounding] + changed = this_function(self, digits) + coeff = self._int[:digits] or '0' + if changed == 1: + coeff = str(int(coeff)+1) + return _dec_from_triple(self._sign, coeff, exp) + + def _round(self, places, rounding): + """Round a nonzero, nonspecial Decimal to a fixed number of + significant figures, using the given rounding mode. + + Infinities, NaNs and zeros are returned unaltered. + + This operation is quiet: it raises no flags, and uses no + information from the context. + + """ + if places <= 0: + raise ValueError("argument should be at least 1 in _round") + if self._is_special or not self: + return Decimal(self) + ans = self._rescale(self.adjusted()+1-places, rounding) + # it can happen that the rescale alters the adjusted exponent; + # for example when rounding 99.97 to 3 significant figures. + # When this happens we end up with an extra 0 at the end of + # the number; a second rescale fixes this. + if ans.adjusted() != self.adjusted(): + ans = ans._rescale(ans.adjusted()+1-places, rounding) + return ans + + def to_integral_exact(self, rounding=None, context=None): + """Rounds to a nearby integer. + + If no rounding mode is specified, take the rounding mode from + the context. This method raises the Rounded and Inexact flags + when appropriate. + + See also: to_integral_value, which does exactly the same as + this method except that it doesn't raise Inexact or Rounded. + """ + if self._is_special: + ans = self._check_nans(context=context) + if ans: + return ans + return Decimal(self) + if self._exp >= 0: + return Decimal(self) + if not self: + return _dec_from_triple(self._sign, '0', 0) + if context is None: + context = getcontext() + if rounding is None: + rounding = context.rounding + ans = self._rescale(0, rounding) + if ans != self: + context._raise_error(Inexact) + context._raise_error(Rounded) + return ans + + def to_integral_value(self, rounding=None, context=None): + """Rounds to the nearest integer, without raising inexact, rounded.""" + if context is None: + context = getcontext() + if rounding is None: + rounding = context.rounding + if self._is_special: + ans = self._check_nans(context=context) + if ans: + return ans + return Decimal(self) + if self._exp >= 0: + return Decimal(self) + else: + return self._rescale(0, rounding) + + # the method name changed, but we provide also the old one, for compatibility + to_integral = to_integral_value + + def sqrt(self, context=None): + """Return the square root of self.""" + if context is None: + context = getcontext() + + if self._is_special: + ans = self._check_nans(context=context) + if ans: + return ans + + if self._isinfinity() and self._sign == 0: + return Decimal(self) + + if not self: + # exponent = self._exp // 2. sqrt(-0) = -0 + ans = _dec_from_triple(self._sign, '0', self._exp // 2) + return ans._fix(context) + + if self._sign == 1: + return context._raise_error(InvalidOperation, 'sqrt(-x), x > 0') + + # At this point self represents a positive number. Let p be + # the desired precision and express self in the form c*100**e + # with c a positive real number and e an integer, c and e + # being chosen so that 100**(p-1) <= c < 100**p. Then the + # (exact) square root of self is sqrt(c)*10**e, and 10**(p-1) + # <= sqrt(c) < 10**p, so the closest representable Decimal at + # precision p is n*10**e where n = round_half_even(sqrt(c)), + # the closest integer to sqrt(c) with the even integer chosen + # in the case of a tie. + # + # To ensure correct rounding in all cases, we use the + # following trick: we compute the square root to an extra + # place (precision p+1 instead of precision p), rounding down. + # Then, if the result is inexact and its last digit is 0 or 5, + # we increase the last digit to 1 or 6 respectively; if it's + # exact we leave the last digit alone. Now the final round to + # p places (or fewer in the case of underflow) will round + # correctly and raise the appropriate flags. + + # use an extra digit of precision + prec = context.prec+1 + + # write argument in the form c*100**e where e = self._exp//2 + # is the 'ideal' exponent, to be used if the square root is + # exactly representable. l is the number of 'digits' of c in + # base 100, so that 100**(l-1) <= c < 100**l. + op = _WorkRep(self) + e = op.exp >> 1 + if op.exp & 1: + c = op.int * 10 + l = (len(self._int) >> 1) + 1 + else: + c = op.int + l = len(self._int)+1 >> 1 + + # rescale so that c has exactly prec base 100 'digits' + shift = prec-l + if shift >= 0: + c *= 100**shift + exact = True + else: + c, remainder = divmod(c, 100**-shift) + exact = not remainder + e -= shift + + # find n = floor(sqrt(c)) using Newton's method + n = 10**prec + while True: + q = c//n + if n <= q: + break + else: + n = n + q >> 1 + exact = exact and n*n == c + + if exact: + # result is exact; rescale to use ideal exponent e + if shift >= 0: + # assert n % 10**shift == 0 + n //= 10**shift + else: + n *= 10**-shift + e += shift + else: + # result is not exact; fix last digit as described above + if n % 5 == 0: + n += 1 + + ans = _dec_from_triple(0, str(n), e) + + # round, and fit to current context + context = context._shallow_copy() + rounding = context._set_rounding(ROUND_HALF_EVEN) + ans = ans._fix(context) + context.rounding = rounding + + return ans + + def max(self, other, context=None): + """Returns the larger value. + + Like max(self, other) except if one is not a number, returns + NaN (and signals if one is sNaN). Also rounds. + """ + other = _convert_other(other, raiseit=True) + + if context is None: + context = getcontext() + + if self._is_special or other._is_special: + # If one operand is a quiet NaN and the other is number, then the + # number is always returned + sn = self._isnan() + on = other._isnan() + if sn or on: + if on == 1 and sn == 0: + return self._fix(context) + if sn == 1 and on == 0: + return other._fix(context) + return self._check_nans(other, context) + + c = self._cmp(other) + if c == 0: + # If both operands are finite and equal in numerical value + # then an ordering is applied: + # + # If the signs differ then max returns the operand with the + # positive sign and min returns the operand with the negative sign + # + # If the signs are the same then the exponent is used to select + # the result. This is exactly the ordering used in compare_total. + c = self.compare_total(other) + + if c == -1: + ans = other + else: + ans = self + + return ans._fix(context) + + def min(self, other, context=None): + """Returns the smaller value. + + Like min(self, other) except if one is not a number, returns + NaN (and signals if one is sNaN). Also rounds. + """ + other = _convert_other(other, raiseit=True) + + if context is None: + context = getcontext() + + if self._is_special or other._is_special: + # If one operand is a quiet NaN and the other is number, then the + # number is always returned + sn = self._isnan() + on = other._isnan() + if sn or on: + if on == 1 and sn == 0: + return self._fix(context) + if sn == 1 and on == 0: + return other._fix(context) + return self._check_nans(other, context) + + c = self._cmp(other) + if c == 0: + c = self.compare_total(other) + + if c == -1: + ans = self + else: + ans = other + + return ans._fix(context) + + def _isinteger(self): + """Returns whether self is an integer""" + if self._is_special: + return False + if self._exp >= 0: + return True + rest = self._int[self._exp:] + return rest == '0'*len(rest) + + def _iseven(self): + """Returns True if self is even. Assumes self is an integer.""" + if not self or self._exp > 0: + return True + return self._int[-1+self._exp] in '02468' + + def adjusted(self): + """Return the adjusted exponent of self""" + try: + return self._exp + len(self._int) - 1 + # If NaN or Infinity, self._exp is string + except TypeError: + return 0 + + def canonical(self): + """Returns the same Decimal object. + + As we do not have different encodings for the same number, the + received object already is in its canonical form. + """ + return self + + def compare_signal(self, other, context=None): + """Compares self to the other operand numerically. + + It's pretty much like compare(), but all NaNs signal, with signaling + NaNs taking precedence over quiet NaNs. + """ + other = _convert_other(other, raiseit = True) + ans = self._compare_check_nans(other, context) + if ans: + return ans + return self.compare(other, context=context) + + def compare_total(self, other, context=None): + """Compares self to other using the abstract representations. + + This is not like the standard compare, which use their numerical + value. Note that a total ordering is defined for all possible abstract + representations. + """ + other = _convert_other(other, raiseit=True) + + # if one is negative and the other is positive, it's easy + if self._sign and not other._sign: + return _NegativeOne + if not self._sign and other._sign: + return _One + sign = self._sign + + # let's handle both NaN types + self_nan = self._isnan() + other_nan = other._isnan() + if self_nan or other_nan: + if self_nan == other_nan: + # compare payloads as though they're integers + self_key = len(self._int), self._int + other_key = len(other._int), other._int + if self_key < other_key: + if sign: + return _One + else: + return _NegativeOne + if self_key > other_key: + if sign: + return _NegativeOne + else: + return _One + return _Zero + + if sign: + if self_nan == 1: + return _NegativeOne + if other_nan == 1: + return _One + if self_nan == 2: + return _NegativeOne + if other_nan == 2: + return _One + else: + if self_nan == 1: + return _One + if other_nan == 1: + return _NegativeOne + if self_nan == 2: + return _One + if other_nan == 2: + return _NegativeOne + + if self < other: + return _NegativeOne + if self > other: + return _One + + if self._exp < other._exp: + if sign: + return _One + else: + return _NegativeOne + if self._exp > other._exp: + if sign: + return _NegativeOne + else: + return _One + return _Zero + + + def compare_total_mag(self, other, context=None): + """Compares self to other using abstract repr., ignoring sign. + + Like compare_total, but with operand's sign ignored and assumed to be 0. + """ + other = _convert_other(other, raiseit=True) + + s = self.copy_abs() + o = other.copy_abs() + return s.compare_total(o) + + def copy_abs(self): + """Returns a copy with the sign set to 0. """ + return _dec_from_triple(0, self._int, self._exp, self._is_special) + + def copy_negate(self): + """Returns a copy with the sign inverted.""" + if self._sign: + return _dec_from_triple(0, self._int, self._exp, self._is_special) + else: + return _dec_from_triple(1, self._int, self._exp, self._is_special) + + def copy_sign(self, other, context=None): + """Returns self with the sign of other.""" + other = _convert_other(other, raiseit=True) + return _dec_from_triple(other._sign, self._int, + self._exp, self._is_special) + + def exp(self, context=None): + """Returns e ** self.""" + + if context is None: + context = getcontext() + + # exp(NaN) = NaN + ans = self._check_nans(context=context) + if ans: + return ans + + # exp(-Infinity) = 0 + if self._isinfinity() == -1: + return _Zero + + # exp(0) = 1 + if not self: + return _One + + # exp(Infinity) = Infinity + if self._isinfinity() == 1: + return Decimal(self) + + # the result is now guaranteed to be inexact (the true + # mathematical result is transcendental). There's no need to + # raise Rounded and Inexact here---they'll always be raised as + # a result of the call to _fix. + p = context.prec + adj = self.adjusted() + + # we only need to do any computation for quite a small range + # of adjusted exponents---for example, -29 <= adj <= 10 for + # the default context. For smaller exponent the result is + # indistinguishable from 1 at the given precision, while for + # larger exponent the result either overflows or underflows. + if self._sign == 0 and adj > len(str((context.Emax+1)*3)): + # overflow + ans = _dec_from_triple(0, '1', context.Emax+1) + elif self._sign == 1 and adj > len(str((-context.Etiny()+1)*3)): + # underflow to 0 + ans = _dec_from_triple(0, '1', context.Etiny()-1) + elif self._sign == 0 and adj < -p: + # p+1 digits; final round will raise correct flags + ans = _dec_from_triple(0, '1' + '0'*(p-1) + '1', -p) + elif self._sign == 1 and adj < -p-1: + # p+1 digits; final round will raise correct flags + ans = _dec_from_triple(0, '9'*(p+1), -p-1) + # general case + else: + op = _WorkRep(self) + c, e = op.int, op.exp + if op.sign == 1: + c = -c + + # compute correctly rounded result: increase precision by + # 3 digits at a time until we get an unambiguously + # roundable result + extra = 3 + while True: + coeff, exp = _dexp(c, e, p+extra) + if coeff % (5*10**(len(str(coeff))-p-1)): + break + extra += 3 + + ans = _dec_from_triple(0, str(coeff), exp) + + # at this stage, ans should round correctly with *any* + # rounding mode, not just with ROUND_HALF_EVEN + context = context._shallow_copy() + rounding = context._set_rounding(ROUND_HALF_EVEN) + ans = ans._fix(context) + context.rounding = rounding + + return ans + + def is_canonical(self): + """Return True if self is canonical; otherwise return False. + + Currently, the encoding of a Decimal instance is always + canonical, so this method returns True for any Decimal. + """ + return True + + def is_finite(self): + """Return True if self is finite; otherwise return False. + + A Decimal instance is considered finite if it is neither + infinite nor a NaN. + """ + return not self._is_special + + def is_infinite(self): + """Return True if self is infinite; otherwise return False.""" + return self._exp == 'F' + + def is_nan(self): + """Return True if self is a qNaN or sNaN; otherwise return False.""" + return self._exp in ('n', 'N') + + def is_normal(self, context=None): + """Return True if self is a normal number; otherwise return False.""" + if self._is_special or not self: + return False + if context is None: + context = getcontext() + return context.Emin <= self.adjusted() + + def is_qnan(self): + """Return True if self is a quiet NaN; otherwise return False.""" + return self._exp == 'n' + + def is_signed(self): + """Return True if self is negative; otherwise return False.""" + return self._sign == 1 + + def is_snan(self): + """Return True if self is a signaling NaN; otherwise return False.""" + return self._exp == 'N' + + def is_subnormal(self, context=None): + """Return True if self is subnormal; otherwise return False.""" + if self._is_special or not self: + return False + if context is None: + context = getcontext() + return self.adjusted() < context.Emin + + def is_zero(self): + """Return True if self is a zero; otherwise return False.""" + return not self._is_special and self._int == '0' + + def _ln_exp_bound(self): + """Compute a lower bound for the adjusted exponent of self.ln(). + In other words, compute r such that self.ln() >= 10**r. Assumes + that self is finite and positive and that self != 1. + """ + + # for 0.1 <= x <= 10 we use the inequalities 1-1/x <= ln(x) <= x-1 + adj = self._exp + len(self._int) - 1 + if adj >= 1: + # argument >= 10; we use 23/10 = 2.3 as a lower bound for ln(10) + return len(str(adj*23//10)) - 1 + if adj <= -2: + # argument <= 0.1 + return len(str((-1-adj)*23//10)) - 1 + op = _WorkRep(self) + c, e = op.int, op.exp + if adj == 0: + # 1 < self < 10 + num = str(c-10**-e) + den = str(c) + return len(num) - len(den) - (num < den) + # adj == -1, 0.1 <= self < 1 + return e + len(str(10**-e - c)) - 1 + + + def ln(self, context=None): + """Returns the natural (base e) logarithm of self.""" + + if context is None: + context = getcontext() + + # ln(NaN) = NaN + ans = self._check_nans(context=context) + if ans: + return ans + + # ln(0.0) == -Infinity + if not self: + return _NegativeInfinity + + # ln(Infinity) = Infinity + if self._isinfinity() == 1: + return _Infinity + + # ln(1.0) == 0.0 + if self == _One: + return _Zero + + # ln(negative) raises InvalidOperation + if self._sign == 1: + return context._raise_error(InvalidOperation, + 'ln of a negative value') + + # result is irrational, so necessarily inexact + op = _WorkRep(self) + c, e = op.int, op.exp + p = context.prec + + # correctly rounded result: repeatedly increase precision by 3 + # until we get an unambiguously roundable result + places = p - self._ln_exp_bound() + 2 # at least p+3 places + while True: + coeff = _dlog(c, e, places) + # assert len(str(abs(coeff)))-p >= 1 + if coeff % (5*10**(len(str(abs(coeff)))-p-1)): + break + places += 3 + ans = _dec_from_triple(int(coeff<0), str(abs(coeff)), -places) + + context = context._shallow_copy() + rounding = context._set_rounding(ROUND_HALF_EVEN) + ans = ans._fix(context) + context.rounding = rounding + return ans + + def _log10_exp_bound(self): + """Compute a lower bound for the adjusted exponent of self.log10(). + In other words, find r such that self.log10() >= 10**r. + Assumes that self is finite and positive and that self != 1. + """ + + # For x >= 10 or x < 0.1 we only need a bound on the integer + # part of log10(self), and this comes directly from the + # exponent of x. For 0.1 <= x <= 10 we use the inequalities + # 1-1/x <= log(x) <= x-1. If x > 1 we have |log10(x)| > + # (1-1/x)/2.31 > 0. If x < 1 then |log10(x)| > (1-x)/2.31 > 0 + + adj = self._exp + len(self._int) - 1 + if adj >= 1: + # self >= 10 + return len(str(adj))-1 + if adj <= -2: + # self < 0.1 + return len(str(-1-adj))-1 + op = _WorkRep(self) + c, e = op.int, op.exp + if adj == 0: + # 1 < self < 10 + num = str(c-10**-e) + den = str(231*c) + return len(num) - len(den) - (num < den) + 2 + # adj == -1, 0.1 <= self < 1 + num = str(10**-e-c) + return len(num) + e - (num < "231") - 1 + + def log10(self, context=None): + """Returns the base 10 logarithm of self.""" + + if context is None: + context = getcontext() + + # log10(NaN) = NaN + ans = self._check_nans(context=context) + if ans: + return ans + + # log10(0.0) == -Infinity + if not self: + return _NegativeInfinity + + # log10(Infinity) = Infinity + if self._isinfinity() == 1: + return _Infinity + + # log10(negative or -Infinity) raises InvalidOperation + if self._sign == 1: + return context._raise_error(InvalidOperation, + 'log10 of a negative value') + + # log10(10**n) = n + if self._int[0] == '1' and self._int[1:] == '0'*(len(self._int) - 1): + # answer may need rounding + ans = Decimal(self._exp + len(self._int) - 1) + else: + # result is irrational, so necessarily inexact + op = _WorkRep(self) + c, e = op.int, op.exp + p = context.prec + + # correctly rounded result: repeatedly increase precision + # until result is unambiguously roundable + places = p-self._log10_exp_bound()+2 + while True: + coeff = _dlog10(c, e, places) + # assert len(str(abs(coeff)))-p >= 1 + if coeff % (5*10**(len(str(abs(coeff)))-p-1)): + break + places += 3 + ans = _dec_from_triple(int(coeff<0), str(abs(coeff)), -places) + + context = context._shallow_copy() + rounding = context._set_rounding(ROUND_HALF_EVEN) + ans = ans._fix(context) + context.rounding = rounding + return ans + + def logb(self, context=None): + """ Returns the exponent of the magnitude of self's MSD. + + The result is the integer which is the exponent of the magnitude + of the most significant digit of self (as though it were truncated + to a single digit while maintaining the value of that digit and + without limiting the resulting exponent). + """ + # logb(NaN) = NaN + ans = self._check_nans(context=context) + if ans: + return ans + + if context is None: + context = getcontext() + + # logb(+/-Inf) = +Inf + if self._isinfinity(): + return _Infinity + + # logb(0) = -Inf, DivisionByZero + if not self: + return context._raise_error(DivisionByZero, 'logb(0)', 1) + + # otherwise, simply return the adjusted exponent of self, as a + # Decimal. Note that no attempt is made to fit the result + # into the current context. + ans = Decimal(self.adjusted()) + return ans._fix(context) + + def _islogical(self): + """Return True if self is a logical operand. + + For being logical, it must be a finite number with a sign of 0, + an exponent of 0, and a coefficient whose digits must all be + either 0 or 1. + """ + if self._sign != 0 or self._exp != 0: + return False + for dig in self._int: + if dig not in '01': + return False + return True + + def _fill_logical(self, context, opa, opb): + dif = context.prec - len(opa) + if dif > 0: + opa = '0'*dif + opa + elif dif < 0: + opa = opa[-context.prec:] + dif = context.prec - len(opb) + if dif > 0: + opb = '0'*dif + opb + elif dif < 0: + opb = opb[-context.prec:] + return opa, opb + + def logical_and(self, other, context=None): + """Applies an 'and' operation between self and other's digits.""" + if context is None: + context = getcontext() + + other = _convert_other(other, raiseit=True) + + if not self._islogical() or not other._islogical(): + return context._raise_error(InvalidOperation) + + # fill to context.prec + (opa, opb) = self._fill_logical(context, self._int, other._int) + + # make the operation, and clean starting zeroes + result = "".join([str(int(a)&int(b)) for a,b in zip(opa,opb)]) + return _dec_from_triple(0, result.lstrip('0') or '0', 0) + + def logical_invert(self, context=None): + """Invert all its digits.""" + if context is None: + context = getcontext() + return self.logical_xor(_dec_from_triple(0,'1'*context.prec,0), + context) + + def logical_or(self, other, context=None): + """Applies an 'or' operation between self and other's digits.""" + if context is None: + context = getcontext() + + other = _convert_other(other, raiseit=True) + + if not self._islogical() or not other._islogical(): + return context._raise_error(InvalidOperation) + + # fill to context.prec + (opa, opb) = self._fill_logical(context, self._int, other._int) + + # make the operation, and clean starting zeroes + result = "".join([str(int(a)|int(b)) for a,b in zip(opa,opb)]) + return _dec_from_triple(0, result.lstrip('0') or '0', 0) + + def logical_xor(self, other, context=None): + """Applies an 'xor' operation between self and other's digits.""" + if context is None: + context = getcontext() + + other = _convert_other(other, raiseit=True) + + if not self._islogical() or not other._islogical(): + return context._raise_error(InvalidOperation) + + # fill to context.prec + (opa, opb) = self._fill_logical(context, self._int, other._int) + + # make the operation, and clean starting zeroes + result = "".join([str(int(a)^int(b)) for a,b in zip(opa,opb)]) + return _dec_from_triple(0, result.lstrip('0') or '0', 0) + + def max_mag(self, other, context=None): + """Compares the values numerically with their sign ignored.""" + other = _convert_other(other, raiseit=True) + + if context is None: + context = getcontext() + + if self._is_special or other._is_special: + # If one operand is a quiet NaN and the other is number, then the + # number is always returned + sn = self._isnan() + on = other._isnan() + if sn or on: + if on == 1 and sn == 0: + return self._fix(context) + if sn == 1 and on == 0: + return other._fix(context) + return self._check_nans(other, context) + + c = self.copy_abs()._cmp(other.copy_abs()) + if c == 0: + c = self.compare_total(other) + + if c == -1: + ans = other + else: + ans = self + + return ans._fix(context) + + def min_mag(self, other, context=None): + """Compares the values numerically with their sign ignored.""" + other = _convert_other(other, raiseit=True) + + if context is None: + context = getcontext() + + if self._is_special or other._is_special: + # If one operand is a quiet NaN and the other is number, then the + # number is always returned + sn = self._isnan() + on = other._isnan() + if sn or on: + if on == 1 and sn == 0: + return self._fix(context) + if sn == 1 and on == 0: + return other._fix(context) + return self._check_nans(other, context) + + c = self.copy_abs()._cmp(other.copy_abs()) + if c == 0: + c = self.compare_total(other) + + if c == -1: + ans = self + else: + ans = other + + return ans._fix(context) + + def next_minus(self, context=None): + """Returns the largest representable number smaller than itself.""" + if context is None: + context = getcontext() + + ans = self._check_nans(context=context) + if ans: + return ans + + if self._isinfinity() == -1: + return _NegativeInfinity + if self._isinfinity() == 1: + return _dec_from_triple(0, '9'*context.prec, context.Etop()) + + context = context.copy() + context._set_rounding(ROUND_FLOOR) + context._ignore_all_flags() + new_self = self._fix(context) + if new_self != self: + return new_self + return self.__sub__(_dec_from_triple(0, '1', context.Etiny()-1), + context) + + def next_plus(self, context=None): + """Returns the smallest representable number larger than itself.""" + if context is None: + context = getcontext() + + ans = self._check_nans(context=context) + if ans: + return ans + + if self._isinfinity() == 1: + return _Infinity + if self._isinfinity() == -1: + return _dec_from_triple(1, '9'*context.prec, context.Etop()) + + context = context.copy() + context._set_rounding(ROUND_CEILING) + context._ignore_all_flags() + new_self = self._fix(context) + if new_self != self: + return new_self + return self.__add__(_dec_from_triple(0, '1', context.Etiny()-1), + context) + + def next_toward(self, other, context=None): + """Returns the number closest to self, in the direction towards other. + + The result is the closest representable number to self + (excluding self) that is in the direction towards other, + unless both have the same value. If the two operands are + numerically equal, then the result is a copy of self with the + sign set to be the same as the sign of other. + """ + other = _convert_other(other, raiseit=True) + + if context is None: + context = getcontext() + + ans = self._check_nans(other, context) + if ans: + return ans + + comparison = self._cmp(other) + if comparison == 0: + return self.copy_sign(other) + + if comparison == -1: + ans = self.next_plus(context) + else: # comparison == 1 + ans = self.next_minus(context) + + # decide which flags to raise using value of ans + if ans._isinfinity(): + context._raise_error(Overflow, + 'Infinite result from next_toward', + ans._sign) + context._raise_error(Inexact) + context._raise_error(Rounded) + elif ans.adjusted() < context.Emin: + context._raise_error(Underflow) + context._raise_error(Subnormal) + context._raise_error(Inexact) + context._raise_error(Rounded) + # if precision == 1 then we don't raise Clamped for a + # result 0E-Etiny. + if not ans: + context._raise_error(Clamped) + + return ans + + def number_class(self, context=None): + """Returns an indication of the class of self. + + The class is one of the following strings: + sNaN + NaN + -Infinity + -Normal + -Subnormal + -Zero + +Zero + +Subnormal + +Normal + +Infinity + """ + if self.is_snan(): + return "sNaN" + if self.is_qnan(): + return "NaN" + inf = self._isinfinity() + if inf == 1: + return "+Infinity" + if inf == -1: + return "-Infinity" + if self.is_zero(): + if self._sign: + return "-Zero" + else: + return "+Zero" + if context is None: + context = getcontext() + if self.is_subnormal(context=context): + if self._sign: + return "-Subnormal" + else: + return "+Subnormal" + # just a normal, regular, boring number, :) + if self._sign: + return "-Normal" + else: + return "+Normal" + + def radix(self): + """Just returns 10, as this is Decimal, :)""" + return Decimal(10) + + def rotate(self, other, context=None): + """Returns a rotated copy of self, value-of-other times.""" + if context is None: + context = getcontext() + + other = _convert_other(other, raiseit=True) + + ans = self._check_nans(other, context) + if ans: + return ans + + if other._exp != 0: + return context._raise_error(InvalidOperation) + if not (-context.prec <= int(other) <= context.prec): + return context._raise_error(InvalidOperation) + + if self._isinfinity(): + return Decimal(self) + + # get values, pad if necessary + torot = int(other) + rotdig = self._int + topad = context.prec - len(rotdig) + if topad > 0: + rotdig = '0'*topad + rotdig + elif topad < 0: + rotdig = rotdig[-topad:] + + # let's rotate! + rotated = rotdig[torot:] + rotdig[:torot] + return _dec_from_triple(self._sign, + rotated.lstrip('0') or '0', self._exp) + + def scaleb(self, other, context=None): + """Returns self operand after adding the second value to its exp.""" + if context is None: + context = getcontext() + + other = _convert_other(other, raiseit=True) + + ans = self._check_nans(other, context) + if ans: + return ans + + if other._exp != 0: + return context._raise_error(InvalidOperation) + liminf = -2 * (context.Emax + context.prec) + limsup = 2 * (context.Emax + context.prec) + if not (liminf <= int(other) <= limsup): + return context._raise_error(InvalidOperation) + + if self._isinfinity(): + return Decimal(self) + + d = _dec_from_triple(self._sign, self._int, self._exp + int(other)) + d = d._fix(context) + return d + + def shift(self, other, context=None): + """Returns a shifted copy of self, value-of-other times.""" + if context is None: + context = getcontext() + + other = _convert_other(other, raiseit=True) + + ans = self._check_nans(other, context) + if ans: + return ans + + if other._exp != 0: + return context._raise_error(InvalidOperation) + if not (-context.prec <= int(other) <= context.prec): + return context._raise_error(InvalidOperation) + + if self._isinfinity(): + return Decimal(self) + + # get values, pad if necessary + torot = int(other) + rotdig = self._int + topad = context.prec - len(rotdig) + if topad > 0: + rotdig = '0'*topad + rotdig + elif topad < 0: + rotdig = rotdig[-topad:] + + # let's shift! + if torot < 0: + shifted = rotdig[:torot] + else: + shifted = rotdig + '0'*torot + shifted = shifted[-context.prec:] + + return _dec_from_triple(self._sign, + shifted.lstrip('0') or '0', self._exp) + + # Support for pickling, copy, and deepcopy + def __reduce__(self): + return (self.__class__, (str(self),)) + + def __copy__(self): + if type(self) is Decimal: + return self # I'm immutable; therefore I am my own clone + return self.__class__(str(self)) + + def __deepcopy__(self, memo): + if type(self) is Decimal: + return self # My components are also immutable + return self.__class__(str(self)) + + # PEP 3101 support. the _localeconv keyword argument should be + # considered private: it's provided for ease of testing only. + def __format__(self, specifier, context=None, _localeconv=None): + """Format a Decimal instance according to the given specifier. + + The specifier should be a standard format specifier, with the + form described in PEP 3101. Formatting types 'e', 'E', 'f', + 'F', 'g', 'G', 'n' and '%' are supported. If the formatting + type is omitted it defaults to 'g' or 'G', depending on the + value of context.capitals. + """ + + # Note: PEP 3101 says that if the type is not present then + # there should be at least one digit after the decimal point. + # We take the liberty of ignoring this requirement for + # Decimal---it's presumably there to make sure that + # format(float, '') behaves similarly to str(float). + if context is None: + context = getcontext() + + spec = _parse_format_specifier(specifier, _localeconv=_localeconv) + + # special values don't care about the type or precision + if self._is_special: + sign = _format_sign(self._sign, spec) + body = str(self.copy_abs()) + if spec['type'] == '%': + body += '%' + return _format_align(sign, body, spec) + + # a type of None defaults to 'g' or 'G', depending on context + if spec['type'] is None: + spec['type'] = ['g', 'G'][context.capitals] + + # if type is '%', adjust exponent of self accordingly + if spec['type'] == '%': + self = _dec_from_triple(self._sign, self._int, self._exp+2) + + # round if necessary, taking rounding mode from the context + rounding = context.rounding + precision = spec['precision'] + if precision is not None: + if spec['type'] in 'eE': + self = self._round(precision+1, rounding) + elif spec['type'] in 'fF%': + self = self._rescale(-precision, rounding) + elif spec['type'] in 'gG' and len(self._int) > precision: + self = self._round(precision, rounding) + # special case: zeros with a positive exponent can't be + # represented in fixed point; rescale them to 0e0. + if not self and self._exp > 0 and spec['type'] in 'fF%': + self = self._rescale(0, rounding) + + # figure out placement of the decimal point + leftdigits = self._exp + len(self._int) + if spec['type'] in 'eE': + if not self and precision is not None: + dotplace = 1 - precision + else: + dotplace = 1 + elif spec['type'] in 'fF%': + dotplace = leftdigits + elif spec['type'] in 'gG': + if self._exp <= 0 and leftdigits > -6: + dotplace = leftdigits + else: + dotplace = 1 + + # find digits before and after decimal point, and get exponent + if dotplace < 0: + intpart = '0' + fracpart = '0'*(-dotplace) + self._int + elif dotplace > len(self._int): + intpart = self._int + '0'*(dotplace-len(self._int)) + fracpart = '' + else: + intpart = self._int[:dotplace] or '0' + fracpart = self._int[dotplace:] + exp = leftdigits-dotplace + + # done with the decimal-specific stuff; hand over the rest + # of the formatting to the _format_number function + return _format_number(self._sign, intpart, fracpart, exp, spec) + +def _dec_from_triple(sign, coefficient, exponent, special=False): + """Create a decimal instance directly, without any validation, + normalization (e.g. removal of leading zeros) or argument + conversion. + + This function is for *internal use only*. + """ + + self = object.__new__(Decimal) + self._sign = sign + self._int = coefficient + self._exp = exponent + self._is_special = special + + return self + +# Register Decimal as a kind of Number (an abstract base class). +# However, do not register it as Real (because Decimals are not +# interoperable with floats). +_numbers.Number.register(Decimal) + + +##### Context class ####################################################### + +class _ContextManager(object): + """Context manager class to support localcontext(). + + Sets a copy of the supplied context in __enter__() and restores + the previous decimal context in __exit__() + """ + def __init__(self, new_context): + self.new_context = new_context.copy() + def __enter__(self): + self.saved_context = getcontext() + setcontext(self.new_context) + return self.new_context + def __exit__(self, t, v, tb): + setcontext(self.saved_context) + +class Context(object): + """Contains the context for a Decimal instance. + + Contains: + prec - precision (for use in rounding, division, square roots..) + rounding - rounding type (how you round) + traps - If traps[exception] = 1, then the exception is + raised when it is caused. Otherwise, a value is + substituted in. + flags - When an exception is caused, flags[exception] is set. + (Whether or not the trap_enabler is set) + Should be reset by user of Decimal instance. + Emin - Minimum exponent + Emax - Maximum exponent + capitals - If 1, 1*10^1 is printed as 1E+1. + If 0, printed as 1e1 + clamp - If 1, change exponents if too high (Default 0) + """ + + def __init__(self, prec=None, rounding=None, Emin=None, Emax=None, + capitals=None, clamp=None, flags=None, traps=None, + _ignored_flags=None): + # Set defaults; for everything except flags and _ignored_flags, + # inherit from DefaultContext. + try: + dc = DefaultContext + except NameError: + pass + + self.prec = prec if prec is not None else dc.prec + self.rounding = rounding if rounding is not None else dc.rounding + self.Emin = Emin if Emin is not None else dc.Emin + self.Emax = Emax if Emax is not None else dc.Emax + self.capitals = capitals if capitals is not None else dc.capitals + self.clamp = clamp if clamp is not None else dc.clamp + + if _ignored_flags is None: + self._ignored_flags = [] + else: + self._ignored_flags = _ignored_flags + + if traps is None: + self.traps = dc.traps.copy() + elif not isinstance(traps, dict): + self.traps = dict((s, int(s in traps)) for s in _signals + traps) + else: + self.traps = traps + + if flags is None: + self.flags = dict.fromkeys(_signals, 0) + elif not isinstance(flags, dict): + self.flags = dict((s, int(s in flags)) for s in _signals + flags) + else: + self.flags = flags + + def _set_integer_check(self, name, value, vmin, vmax): + if not isinstance(value, int): + raise TypeError("%s must be an integer" % name) + if vmin == '-inf': + if value > vmax: + raise ValueError("%s must be in [%s, %d]. got: %s" % (name, vmin, vmax, value)) + elif vmax == 'inf': + if value < vmin: + raise ValueError("%s must be in [%d, %s]. got: %s" % (name, vmin, vmax, value)) + else: + if value < vmin or value > vmax: + raise ValueError("%s must be in [%d, %d]. got %s" % (name, vmin, vmax, value)) + return object.__setattr__(self, name, value) + + def _set_signal_dict(self, name, d): + if not isinstance(d, dict): + raise TypeError("%s must be a signal dict" % d) + for key in d: + if not key in _signals: + raise KeyError("%s is not a valid signal dict" % d) + for key in _signals: + if not key in d: + raise KeyError("%s is not a valid signal dict" % d) + return object.__setattr__(self, name, d) + + def __setattr__(self, name, value): + if name == 'prec': + return self._set_integer_check(name, value, 1, 'inf') + elif name == 'Emin': + return self._set_integer_check(name, value, '-inf', 0) + elif name == 'Emax': + return self._set_integer_check(name, value, 0, 'inf') + elif name == 'capitals': + return self._set_integer_check(name, value, 0, 1) + elif name == 'clamp': + return self._set_integer_check(name, value, 0, 1) + elif name == 'rounding': + if not value in _rounding_modes: + # raise TypeError even for strings to have consistency + # among various implementations. + raise TypeError("%s: invalid rounding mode" % value) + return object.__setattr__(self, name, value) + elif name == 'flags' or name == 'traps': + return self._set_signal_dict(name, value) + elif name == '_ignored_flags': + return object.__setattr__(self, name, value) + else: + raise AttributeError( + "'decimal.Context' object has no attribute '%s'" % name) + + def __delattr__(self, name): + raise AttributeError("%s cannot be deleted" % name) + + # Support for pickling, copy, and deepcopy + def __reduce__(self): + flags = [sig for sig, v in self.flags.items() if v] + traps = [sig for sig, v in self.traps.items() if v] + return (self.__class__, + (self.prec, self.rounding, self.Emin, self.Emax, + self.capitals, self.clamp, flags, traps)) + + def __repr__(self): + """Show the current context.""" + s = [] + s.append('Context(prec=%(prec)d, rounding=%(rounding)s, ' + 'Emin=%(Emin)d, Emax=%(Emax)d, capitals=%(capitals)d, ' + 'clamp=%(clamp)d' + % vars(self)) + names = [f.__name__ for f, v in self.flags.items() if v] + s.append('flags=[' + ', '.join(names) + ']') + names = [t.__name__ for t, v in self.traps.items() if v] + s.append('traps=[' + ', '.join(names) + ']') + return ', '.join(s) + ')' + + def clear_flags(self): + """Reset all flags to zero""" + for flag in self.flags: + self.flags[flag] = 0 + + def clear_traps(self): + """Reset all traps to zero""" + for flag in self.traps: + self.traps[flag] = 0 + + def _shallow_copy(self): + """Returns a shallow copy from self.""" + nc = Context(self.prec, self.rounding, self.Emin, self.Emax, + self.capitals, self.clamp, self.flags, self.traps, + self._ignored_flags) + return nc + + def copy(self): + """Returns a deep copy from self.""" + nc = Context(self.prec, self.rounding, self.Emin, self.Emax, + self.capitals, self.clamp, + self.flags.copy(), self.traps.copy(), + self._ignored_flags) + return nc + __copy__ = copy + + def _raise_error(self, condition, explanation = None, *args): + """Handles an error + + If the flag is in _ignored_flags, returns the default response. + Otherwise, it sets the flag, then, if the corresponding + trap_enabler is set, it reraises the exception. Otherwise, it returns + the default value after setting the flag. + """ + error = _condition_map.get(condition, condition) + if error in self._ignored_flags: + # Don't touch the flag + return error().handle(self, *args) + + self.flags[error] = 1 + if not self.traps[error]: + # The errors define how to handle themselves. + return condition().handle(self, *args) + + # Errors should only be risked on copies of the context + # self._ignored_flags = [] + raise error(explanation) + + def _ignore_all_flags(self): + """Ignore all flags, if they are raised""" + return self._ignore_flags(*_signals) + + def _ignore_flags(self, *flags): + """Ignore the flags, if they are raised""" + # Do not mutate-- This way, copies of a context leave the original + # alone. + self._ignored_flags = (self._ignored_flags + list(flags)) + return list(flags) + + def _regard_flags(self, *flags): + """Stop ignoring the flags, if they are raised""" + if flags and isinstance(flags[0], (tuple,list)): + flags = flags[0] + for flag in flags: + self._ignored_flags.remove(flag) + + # We inherit object.__hash__, so we must deny this explicitly + __hash__ = None + + def Etiny(self): + """Returns Etiny (= Emin - prec + 1)""" + return int(self.Emin - self.prec + 1) + + def Etop(self): + """Returns maximum exponent (= Emax - prec + 1)""" + return int(self.Emax - self.prec + 1) + + def _set_rounding(self, type): + """Sets the rounding type. + + Sets the rounding type, and returns the current (previous) + rounding type. Often used like: + + context = context.copy() + # so you don't change the calling context + # if an error occurs in the middle. + rounding = context._set_rounding(ROUND_UP) + val = self.__sub__(other, context=context) + context._set_rounding(rounding) + + This will make it round up for that operation. + """ + rounding = self.rounding + self.rounding= type + return rounding + + def create_decimal(self, num='0'): + """Creates a new Decimal instance but using self as context. + + This method implements the to-number operation of the + IBM Decimal specification.""" + + if isinstance(num, str) and num != num.strip(): + return self._raise_error(ConversionSyntax, + "no trailing or leading whitespace is " + "permitted.") + + d = Decimal(num, context=self) + if d._isnan() and len(d._int) > self.prec - self.clamp: + return self._raise_error(ConversionSyntax, + "diagnostic info too long in NaN") + return d._fix(self) + + def create_decimal_from_float(self, f): + """Creates a new Decimal instance from a float but rounding using self + as the context. + + >>> context = Context(prec=5, rounding=ROUND_DOWN) + >>> context.create_decimal_from_float(3.1415926535897932) + Decimal('3.1415') + >>> context = Context(prec=5, traps=[Inexact]) + >>> context.create_decimal_from_float(3.1415926535897932) + Traceback (most recent call last): + ... + decimal.Inexact: None + + """ + d = Decimal.from_float(f) # An exact conversion + return d._fix(self) # Apply the context rounding + + # Methods + def abs(self, a): + """Returns the absolute value of the operand. + + If the operand is negative, the result is the same as using the minus + operation on the operand. Otherwise, the result is the same as using + the plus operation on the operand. + + >>> ExtendedContext.abs(Decimal('2.1')) + Decimal('2.1') + >>> ExtendedContext.abs(Decimal('-100')) + Decimal('100') + >>> ExtendedContext.abs(Decimal('101.5')) + Decimal('101.5') + >>> ExtendedContext.abs(Decimal('-101.5')) + Decimal('101.5') + >>> ExtendedContext.abs(-1) + Decimal('1') + """ + a = _convert_other(a, raiseit=True) + return a.__abs__(context=self) + + def add(self, a, b): + """Return the sum of the two operands. + + >>> ExtendedContext.add(Decimal('12'), Decimal('7.00')) + Decimal('19.00') + >>> ExtendedContext.add(Decimal('1E+2'), Decimal('1.01E+4')) + Decimal('1.02E+4') + >>> ExtendedContext.add(1, Decimal(2)) + Decimal('3') + >>> ExtendedContext.add(Decimal(8), 5) + Decimal('13') + >>> ExtendedContext.add(5, 5) + Decimal('10') + """ + a = _convert_other(a, raiseit=True) + r = a.__add__(b, context=self) + if r is NotImplemented: + raise TypeError("Unable to convert %s to Decimal" % b) + else: + return r + + def _apply(self, a): + return str(a._fix(self)) + + def canonical(self, a): + """Returns the same Decimal object. + + As we do not have different encodings for the same number, the + received object already is in its canonical form. + + >>> ExtendedContext.canonical(Decimal('2.50')) + Decimal('2.50') + """ + if not isinstance(a, Decimal): + raise TypeError("canonical requires a Decimal as an argument.") + return a.canonical() + + def compare(self, a, b): + """Compares values numerically. + + If the signs of the operands differ, a value representing each operand + ('-1' if the operand is less than zero, '0' if the operand is zero or + negative zero, or '1' if the operand is greater than zero) is used in + place of that operand for the comparison instead of the actual + operand. + + The comparison is then effected by subtracting the second operand from + the first and then returning a value according to the result of the + subtraction: '-1' if the result is less than zero, '0' if the result is + zero or negative zero, or '1' if the result is greater than zero. + + >>> ExtendedContext.compare(Decimal('2.1'), Decimal('3')) + Decimal('-1') + >>> ExtendedContext.compare(Decimal('2.1'), Decimal('2.1')) + Decimal('0') + >>> ExtendedContext.compare(Decimal('2.1'), Decimal('2.10')) + Decimal('0') + >>> ExtendedContext.compare(Decimal('3'), Decimal('2.1')) + Decimal('1') + >>> ExtendedContext.compare(Decimal('2.1'), Decimal('-3')) + Decimal('1') + >>> ExtendedContext.compare(Decimal('-3'), Decimal('2.1')) + Decimal('-1') + >>> ExtendedContext.compare(1, 2) + Decimal('-1') + >>> ExtendedContext.compare(Decimal(1), 2) + Decimal('-1') + >>> ExtendedContext.compare(1, Decimal(2)) + Decimal('-1') + """ + a = _convert_other(a, raiseit=True) + return a.compare(b, context=self) + + def compare_signal(self, a, b): + """Compares the values of the two operands numerically. + + It's pretty much like compare(), but all NaNs signal, with signaling + NaNs taking precedence over quiet NaNs. + + >>> c = ExtendedContext + >>> c.compare_signal(Decimal('2.1'), Decimal('3')) + Decimal('-1') + >>> c.compare_signal(Decimal('2.1'), Decimal('2.1')) + Decimal('0') + >>> c.flags[InvalidOperation] = 0 + >>> print(c.flags[InvalidOperation]) + 0 + >>> c.compare_signal(Decimal('NaN'), Decimal('2.1')) + Decimal('NaN') + >>> print(c.flags[InvalidOperation]) + 1 + >>> c.flags[InvalidOperation] = 0 + >>> print(c.flags[InvalidOperation]) + 0 + >>> c.compare_signal(Decimal('sNaN'), Decimal('2.1')) + Decimal('NaN') + >>> print(c.flags[InvalidOperation]) + 1 + >>> c.compare_signal(-1, 2) + Decimal('-1') + >>> c.compare_signal(Decimal(-1), 2) + Decimal('-1') + >>> c.compare_signal(-1, Decimal(2)) + Decimal('-1') + """ + a = _convert_other(a, raiseit=True) + return a.compare_signal(b, context=self) + + def compare_total(self, a, b): + """Compares two operands using their abstract representation. + + This is not like the standard compare, which use their numerical + value. Note that a total ordering is defined for all possible abstract + representations. + + >>> ExtendedContext.compare_total(Decimal('12.73'), Decimal('127.9')) + Decimal('-1') + >>> ExtendedContext.compare_total(Decimal('-127'), Decimal('12')) + Decimal('-1') + >>> ExtendedContext.compare_total(Decimal('12.30'), Decimal('12.3')) + Decimal('-1') + >>> ExtendedContext.compare_total(Decimal('12.30'), Decimal('12.30')) + Decimal('0') + >>> ExtendedContext.compare_total(Decimal('12.3'), Decimal('12.300')) + Decimal('1') + >>> ExtendedContext.compare_total(Decimal('12.3'), Decimal('NaN')) + Decimal('-1') + >>> ExtendedContext.compare_total(1, 2) + Decimal('-1') + >>> ExtendedContext.compare_total(Decimal(1), 2) + Decimal('-1') + >>> ExtendedContext.compare_total(1, Decimal(2)) + Decimal('-1') + """ + a = _convert_other(a, raiseit=True) + return a.compare_total(b) + + def compare_total_mag(self, a, b): + """Compares two operands using their abstract representation ignoring sign. + + Like compare_total, but with operand's sign ignored and assumed to be 0. + """ + a = _convert_other(a, raiseit=True) + return a.compare_total_mag(b) + + def copy_abs(self, a): + """Returns a copy of the operand with the sign set to 0. + + >>> ExtendedContext.copy_abs(Decimal('2.1')) + Decimal('2.1') + >>> ExtendedContext.copy_abs(Decimal('-100')) + Decimal('100') + >>> ExtendedContext.copy_abs(-1) + Decimal('1') + """ + a = _convert_other(a, raiseit=True) + return a.copy_abs() + + def copy_decimal(self, a): + """Returns a copy of the decimal object. + + >>> ExtendedContext.copy_decimal(Decimal('2.1')) + Decimal('2.1') + >>> ExtendedContext.copy_decimal(Decimal('-1.00')) + Decimal('-1.00') + >>> ExtendedContext.copy_decimal(1) + Decimal('1') + """ + a = _convert_other(a, raiseit=True) + return Decimal(a) + + def copy_negate(self, a): + """Returns a copy of the operand with the sign inverted. + + >>> ExtendedContext.copy_negate(Decimal('101.5')) + Decimal('-101.5') + >>> ExtendedContext.copy_negate(Decimal('-101.5')) + Decimal('101.5') + >>> ExtendedContext.copy_negate(1) + Decimal('-1') + """ + a = _convert_other(a, raiseit=True) + return a.copy_negate() + + def copy_sign(self, a, b): + """Copies the second operand's sign to the first one. + + In detail, it returns a copy of the first operand with the sign + equal to the sign of the second operand. + + >>> ExtendedContext.copy_sign(Decimal( '1.50'), Decimal('7.33')) + Decimal('1.50') + >>> ExtendedContext.copy_sign(Decimal('-1.50'), Decimal('7.33')) + Decimal('1.50') + >>> ExtendedContext.copy_sign(Decimal( '1.50'), Decimal('-7.33')) + Decimal('-1.50') + >>> ExtendedContext.copy_sign(Decimal('-1.50'), Decimal('-7.33')) + Decimal('-1.50') + >>> ExtendedContext.copy_sign(1, -2) + Decimal('-1') + >>> ExtendedContext.copy_sign(Decimal(1), -2) + Decimal('-1') + >>> ExtendedContext.copy_sign(1, Decimal(-2)) + Decimal('-1') + """ + a = _convert_other(a, raiseit=True) + return a.copy_sign(b) + + def divide(self, a, b): + """Decimal division in a specified context. + + >>> ExtendedContext.divide(Decimal('1'), Decimal('3')) + Decimal('0.333333333') + >>> ExtendedContext.divide(Decimal('2'), Decimal('3')) + Decimal('0.666666667') + >>> ExtendedContext.divide(Decimal('5'), Decimal('2')) + Decimal('2.5') + >>> ExtendedContext.divide(Decimal('1'), Decimal('10')) + Decimal('0.1') + >>> ExtendedContext.divide(Decimal('12'), Decimal('12')) + Decimal('1') + >>> ExtendedContext.divide(Decimal('8.00'), Decimal('2')) + Decimal('4.00') + >>> ExtendedContext.divide(Decimal('2.400'), Decimal('2.0')) + Decimal('1.20') + >>> ExtendedContext.divide(Decimal('1000'), Decimal('100')) + Decimal('10') + >>> ExtendedContext.divide(Decimal('1000'), Decimal('1')) + Decimal('1000') + >>> ExtendedContext.divide(Decimal('2.40E+6'), Decimal('2')) + Decimal('1.20E+6') + >>> ExtendedContext.divide(5, 5) + Decimal('1') + >>> ExtendedContext.divide(Decimal(5), 5) + Decimal('1') + >>> ExtendedContext.divide(5, Decimal(5)) + Decimal('1') + """ + a = _convert_other(a, raiseit=True) + r = a.__truediv__(b, context=self) + if r is NotImplemented: + raise TypeError("Unable to convert %s to Decimal" % b) + else: + return r + + def divide_int(self, a, b): + """Divides two numbers and returns the integer part of the result. + + >>> ExtendedContext.divide_int(Decimal('2'), Decimal('3')) + Decimal('0') + >>> ExtendedContext.divide_int(Decimal('10'), Decimal('3')) + Decimal('3') + >>> ExtendedContext.divide_int(Decimal('1'), Decimal('0.3')) + Decimal('3') + >>> ExtendedContext.divide_int(10, 3) + Decimal('3') + >>> ExtendedContext.divide_int(Decimal(10), 3) + Decimal('3') + >>> ExtendedContext.divide_int(10, Decimal(3)) + Decimal('3') + """ + a = _convert_other(a, raiseit=True) + r = a.__floordiv__(b, context=self) + if r is NotImplemented: + raise TypeError("Unable to convert %s to Decimal" % b) + else: + return r + + def divmod(self, a, b): + """Return (a // b, a % b). + + >>> ExtendedContext.divmod(Decimal(8), Decimal(3)) + (Decimal('2'), Decimal('2')) + >>> ExtendedContext.divmod(Decimal(8), Decimal(4)) + (Decimal('2'), Decimal('0')) + >>> ExtendedContext.divmod(8, 4) + (Decimal('2'), Decimal('0')) + >>> ExtendedContext.divmod(Decimal(8), 4) + (Decimal('2'), Decimal('0')) + >>> ExtendedContext.divmod(8, Decimal(4)) + (Decimal('2'), Decimal('0')) + """ + a = _convert_other(a, raiseit=True) + r = a.__divmod__(b, context=self) + if r is NotImplemented: + raise TypeError("Unable to convert %s to Decimal" % b) + else: + return r + + def exp(self, a): + """Returns e ** a. + + >>> c = ExtendedContext.copy() + >>> c.Emin = -999 + >>> c.Emax = 999 + >>> c.exp(Decimal('-Infinity')) + Decimal('0') + >>> c.exp(Decimal('-1')) + Decimal('0.367879441') + >>> c.exp(Decimal('0')) + Decimal('1') + >>> c.exp(Decimal('1')) + Decimal('2.71828183') + >>> c.exp(Decimal('0.693147181')) + Decimal('2.00000000') + >>> c.exp(Decimal('+Infinity')) + Decimal('Infinity') + >>> c.exp(10) + Decimal('22026.4658') + """ + a =_convert_other(a, raiseit=True) + return a.exp(context=self) + + def fma(self, a, b, c): + """Returns a multiplied by b, plus c. + + The first two operands are multiplied together, using multiply, + the third operand is then added to the result of that + multiplication, using add, all with only one final rounding. + + >>> ExtendedContext.fma(Decimal('3'), Decimal('5'), Decimal('7')) + Decimal('22') + >>> ExtendedContext.fma(Decimal('3'), Decimal('-5'), Decimal('7')) + Decimal('-8') + >>> ExtendedContext.fma(Decimal('888565290'), Decimal('1557.96930'), Decimal('-86087.7578')) + Decimal('1.38435736E+12') + >>> ExtendedContext.fma(1, 3, 4) + Decimal('7') + >>> ExtendedContext.fma(1, Decimal(3), 4) + Decimal('7') + >>> ExtendedContext.fma(1, 3, Decimal(4)) + Decimal('7') + """ + a = _convert_other(a, raiseit=True) + return a.fma(b, c, context=self) + + def is_canonical(self, a): + """Return True if the operand is canonical; otherwise return False. + + Currently, the encoding of a Decimal instance is always + canonical, so this method returns True for any Decimal. + + >>> ExtendedContext.is_canonical(Decimal('2.50')) + True + """ + if not isinstance(a, Decimal): + raise TypeError("is_canonical requires a Decimal as an argument.") + return a.is_canonical() + + def is_finite(self, a): + """Return True if the operand is finite; otherwise return False. + + A Decimal instance is considered finite if it is neither + infinite nor a NaN. + + >>> ExtendedContext.is_finite(Decimal('2.50')) + True + >>> ExtendedContext.is_finite(Decimal('-0.3')) + True + >>> ExtendedContext.is_finite(Decimal('0')) + True + >>> ExtendedContext.is_finite(Decimal('Inf')) + False + >>> ExtendedContext.is_finite(Decimal('NaN')) + False + >>> ExtendedContext.is_finite(1) + True + """ + a = _convert_other(a, raiseit=True) + return a.is_finite() + + def is_infinite(self, a): + """Return True if the operand is infinite; otherwise return False. + + >>> ExtendedContext.is_infinite(Decimal('2.50')) + False + >>> ExtendedContext.is_infinite(Decimal('-Inf')) + True + >>> ExtendedContext.is_infinite(Decimal('NaN')) + False + >>> ExtendedContext.is_infinite(1) + False + """ + a = _convert_other(a, raiseit=True) + return a.is_infinite() + + def is_nan(self, a): + """Return True if the operand is a qNaN or sNaN; + otherwise return False. + + >>> ExtendedContext.is_nan(Decimal('2.50')) + False + >>> ExtendedContext.is_nan(Decimal('NaN')) + True + >>> ExtendedContext.is_nan(Decimal('-sNaN')) + True + >>> ExtendedContext.is_nan(1) + False + """ + a = _convert_other(a, raiseit=True) + return a.is_nan() + + def is_normal(self, a): + """Return True if the operand is a normal number; + otherwise return False. + + >>> c = ExtendedContext.copy() + >>> c.Emin = -999 + >>> c.Emax = 999 + >>> c.is_normal(Decimal('2.50')) + True + >>> c.is_normal(Decimal('0.1E-999')) + False + >>> c.is_normal(Decimal('0.00')) + False + >>> c.is_normal(Decimal('-Inf')) + False + >>> c.is_normal(Decimal('NaN')) + False + >>> c.is_normal(1) + True + """ + a = _convert_other(a, raiseit=True) + return a.is_normal(context=self) + + def is_qnan(self, a): + """Return True if the operand is a quiet NaN; otherwise return False. + + >>> ExtendedContext.is_qnan(Decimal('2.50')) + False + >>> ExtendedContext.is_qnan(Decimal('NaN')) + True + >>> ExtendedContext.is_qnan(Decimal('sNaN')) + False + >>> ExtendedContext.is_qnan(1) + False + """ + a = _convert_other(a, raiseit=True) + return a.is_qnan() + + def is_signed(self, a): + """Return True if the operand is negative; otherwise return False. + + >>> ExtendedContext.is_signed(Decimal('2.50')) + False + >>> ExtendedContext.is_signed(Decimal('-12')) + True + >>> ExtendedContext.is_signed(Decimal('-0')) + True + >>> ExtendedContext.is_signed(8) + False + >>> ExtendedContext.is_signed(-8) + True + """ + a = _convert_other(a, raiseit=True) + return a.is_signed() + + def is_snan(self, a): + """Return True if the operand is a signaling NaN; + otherwise return False. + + >>> ExtendedContext.is_snan(Decimal('2.50')) + False + >>> ExtendedContext.is_snan(Decimal('NaN')) + False + >>> ExtendedContext.is_snan(Decimal('sNaN')) + True + >>> ExtendedContext.is_snan(1) + False + """ + a = _convert_other(a, raiseit=True) + return a.is_snan() + + def is_subnormal(self, a): + """Return True if the operand is subnormal; otherwise return False. + + >>> c = ExtendedContext.copy() + >>> c.Emin = -999 + >>> c.Emax = 999 + >>> c.is_subnormal(Decimal('2.50')) + False + >>> c.is_subnormal(Decimal('0.1E-999')) + True + >>> c.is_subnormal(Decimal('0.00')) + False + >>> c.is_subnormal(Decimal('-Inf')) + False + >>> c.is_subnormal(Decimal('NaN')) + False + >>> c.is_subnormal(1) + False + """ + a = _convert_other(a, raiseit=True) + return a.is_subnormal(context=self) + + def is_zero(self, a): + """Return True if the operand is a zero; otherwise return False. + + >>> ExtendedContext.is_zero(Decimal('0')) + True + >>> ExtendedContext.is_zero(Decimal('2.50')) + False + >>> ExtendedContext.is_zero(Decimal('-0E+2')) + True + >>> ExtendedContext.is_zero(1) + False + >>> ExtendedContext.is_zero(0) + True + """ + a = _convert_other(a, raiseit=True) + return a.is_zero() + + def ln(self, a): + """Returns the natural (base e) logarithm of the operand. + + >>> c = ExtendedContext.copy() + >>> c.Emin = -999 + >>> c.Emax = 999 + >>> c.ln(Decimal('0')) + Decimal('-Infinity') + >>> c.ln(Decimal('1.000')) + Decimal('0') + >>> c.ln(Decimal('2.71828183')) + Decimal('1.00000000') + >>> c.ln(Decimal('10')) + Decimal('2.30258509') + >>> c.ln(Decimal('+Infinity')) + Decimal('Infinity') + >>> c.ln(1) + Decimal('0') + """ + a = _convert_other(a, raiseit=True) + return a.ln(context=self) + + def log10(self, a): + """Returns the base 10 logarithm of the operand. + + >>> c = ExtendedContext.copy() + >>> c.Emin = -999 + >>> c.Emax = 999 + >>> c.log10(Decimal('0')) + Decimal('-Infinity') + >>> c.log10(Decimal('0.001')) + Decimal('-3') + >>> c.log10(Decimal('1.000')) + Decimal('0') + >>> c.log10(Decimal('2')) + Decimal('0.301029996') + >>> c.log10(Decimal('10')) + Decimal('1') + >>> c.log10(Decimal('70')) + Decimal('1.84509804') + >>> c.log10(Decimal('+Infinity')) + Decimal('Infinity') + >>> c.log10(0) + Decimal('-Infinity') + >>> c.log10(1) + Decimal('0') + """ + a = _convert_other(a, raiseit=True) + return a.log10(context=self) + + def logb(self, a): + """ Returns the exponent of the magnitude of the operand's MSD. + + The result is the integer which is the exponent of the magnitude + of the most significant digit of the operand (as though the + operand were truncated to a single digit while maintaining the + value of that digit and without limiting the resulting exponent). + + >>> ExtendedContext.logb(Decimal('250')) + Decimal('2') + >>> ExtendedContext.logb(Decimal('2.50')) + Decimal('0') + >>> ExtendedContext.logb(Decimal('0.03')) + Decimal('-2') + >>> ExtendedContext.logb(Decimal('0')) + Decimal('-Infinity') + >>> ExtendedContext.logb(1) + Decimal('0') + >>> ExtendedContext.logb(10) + Decimal('1') + >>> ExtendedContext.logb(100) + Decimal('2') + """ + a = _convert_other(a, raiseit=True) + return a.logb(context=self) + + def logical_and(self, a, b): + """Applies the logical operation 'and' between each operand's digits. + + The operands must be both logical numbers. + + >>> ExtendedContext.logical_and(Decimal('0'), Decimal('0')) + Decimal('0') + >>> ExtendedContext.logical_and(Decimal('0'), Decimal('1')) + Decimal('0') + >>> ExtendedContext.logical_and(Decimal('1'), Decimal('0')) + Decimal('0') + >>> ExtendedContext.logical_and(Decimal('1'), Decimal('1')) + Decimal('1') + >>> ExtendedContext.logical_and(Decimal('1100'), Decimal('1010')) + Decimal('1000') + >>> ExtendedContext.logical_and(Decimal('1111'), Decimal('10')) + Decimal('10') + >>> ExtendedContext.logical_and(110, 1101) + Decimal('100') + >>> ExtendedContext.logical_and(Decimal(110), 1101) + Decimal('100') + >>> ExtendedContext.logical_and(110, Decimal(1101)) + Decimal('100') + """ + a = _convert_other(a, raiseit=True) + return a.logical_and(b, context=self) + + def logical_invert(self, a): + """Invert all the digits in the operand. + + The operand must be a logical number. + + >>> ExtendedContext.logical_invert(Decimal('0')) + Decimal('111111111') + >>> ExtendedContext.logical_invert(Decimal('1')) + Decimal('111111110') + >>> ExtendedContext.logical_invert(Decimal('111111111')) + Decimal('0') + >>> ExtendedContext.logical_invert(Decimal('101010101')) + Decimal('10101010') + >>> ExtendedContext.logical_invert(1101) + Decimal('111110010') + """ + a = _convert_other(a, raiseit=True) + return a.logical_invert(context=self) + + def logical_or(self, a, b): + """Applies the logical operation 'or' between each operand's digits. + + The operands must be both logical numbers. + + >>> ExtendedContext.logical_or(Decimal('0'), Decimal('0')) + Decimal('0') + >>> ExtendedContext.logical_or(Decimal('0'), Decimal('1')) + Decimal('1') + >>> ExtendedContext.logical_or(Decimal('1'), Decimal('0')) + Decimal('1') + >>> ExtendedContext.logical_or(Decimal('1'), Decimal('1')) + Decimal('1') + >>> ExtendedContext.logical_or(Decimal('1100'), Decimal('1010')) + Decimal('1110') + >>> ExtendedContext.logical_or(Decimal('1110'), Decimal('10')) + Decimal('1110') + >>> ExtendedContext.logical_or(110, 1101) + Decimal('1111') + >>> ExtendedContext.logical_or(Decimal(110), 1101) + Decimal('1111') + >>> ExtendedContext.logical_or(110, Decimal(1101)) + Decimal('1111') + """ + a = _convert_other(a, raiseit=True) + return a.logical_or(b, context=self) + + def logical_xor(self, a, b): + """Applies the logical operation 'xor' between each operand's digits. + + The operands must be both logical numbers. + + >>> ExtendedContext.logical_xor(Decimal('0'), Decimal('0')) + Decimal('0') + >>> ExtendedContext.logical_xor(Decimal('0'), Decimal('1')) + Decimal('1') + >>> ExtendedContext.logical_xor(Decimal('1'), Decimal('0')) + Decimal('1') + >>> ExtendedContext.logical_xor(Decimal('1'), Decimal('1')) + Decimal('0') + >>> ExtendedContext.logical_xor(Decimal('1100'), Decimal('1010')) + Decimal('110') + >>> ExtendedContext.logical_xor(Decimal('1111'), Decimal('10')) + Decimal('1101') + >>> ExtendedContext.logical_xor(110, 1101) + Decimal('1011') + >>> ExtendedContext.logical_xor(Decimal(110), 1101) + Decimal('1011') + >>> ExtendedContext.logical_xor(110, Decimal(1101)) + Decimal('1011') + """ + a = _convert_other(a, raiseit=True) + return a.logical_xor(b, context=self) + + def max(self, a, b): + """max compares two values numerically and returns the maximum. + + If either operand is a NaN then the general rules apply. + Otherwise, the operands are compared as though by the compare + operation. If they are numerically equal then the left-hand operand + is chosen as the result. Otherwise the maximum (closer to positive + infinity) of the two operands is chosen as the result. + + >>> ExtendedContext.max(Decimal('3'), Decimal('2')) + Decimal('3') + >>> ExtendedContext.max(Decimal('-10'), Decimal('3')) + Decimal('3') + >>> ExtendedContext.max(Decimal('1.0'), Decimal('1')) + Decimal('1') + >>> ExtendedContext.max(Decimal('7'), Decimal('NaN')) + Decimal('7') + >>> ExtendedContext.max(1, 2) + Decimal('2') + >>> ExtendedContext.max(Decimal(1), 2) + Decimal('2') + >>> ExtendedContext.max(1, Decimal(2)) + Decimal('2') + """ + a = _convert_other(a, raiseit=True) + return a.max(b, context=self) + + def max_mag(self, a, b): + """Compares the values numerically with their sign ignored. + + >>> ExtendedContext.max_mag(Decimal('7'), Decimal('NaN')) + Decimal('7') + >>> ExtendedContext.max_mag(Decimal('7'), Decimal('-10')) + Decimal('-10') + >>> ExtendedContext.max_mag(1, -2) + Decimal('-2') + >>> ExtendedContext.max_mag(Decimal(1), -2) + Decimal('-2') + >>> ExtendedContext.max_mag(1, Decimal(-2)) + Decimal('-2') + """ + a = _convert_other(a, raiseit=True) + return a.max_mag(b, context=self) + + def min(self, a, b): + """min compares two values numerically and returns the minimum. + + If either operand is a NaN then the general rules apply. + Otherwise, the operands are compared as though by the compare + operation. If they are numerically equal then the left-hand operand + is chosen as the result. Otherwise the minimum (closer to negative + infinity) of the two operands is chosen as the result. + + >>> ExtendedContext.min(Decimal('3'), Decimal('2')) + Decimal('2') + >>> ExtendedContext.min(Decimal('-10'), Decimal('3')) + Decimal('-10') + >>> ExtendedContext.min(Decimal('1.0'), Decimal('1')) + Decimal('1.0') + >>> ExtendedContext.min(Decimal('7'), Decimal('NaN')) + Decimal('7') + >>> ExtendedContext.min(1, 2) + Decimal('1') + >>> ExtendedContext.min(Decimal(1), 2) + Decimal('1') + >>> ExtendedContext.min(1, Decimal(29)) + Decimal('1') + """ + a = _convert_other(a, raiseit=True) + return a.min(b, context=self) + + def min_mag(self, a, b): + """Compares the values numerically with their sign ignored. + + >>> ExtendedContext.min_mag(Decimal('3'), Decimal('-2')) + Decimal('-2') + >>> ExtendedContext.min_mag(Decimal('-3'), Decimal('NaN')) + Decimal('-3') + >>> ExtendedContext.min_mag(1, -2) + Decimal('1') + >>> ExtendedContext.min_mag(Decimal(1), -2) + Decimal('1') + >>> ExtendedContext.min_mag(1, Decimal(-2)) + Decimal('1') + """ + a = _convert_other(a, raiseit=True) + return a.min_mag(b, context=self) + + def minus(self, a): + """Minus corresponds to unary prefix minus in Python. + + The operation is evaluated using the same rules as subtract; the + operation minus(a) is calculated as subtract('0', a) where the '0' + has the same exponent as the operand. + + >>> ExtendedContext.minus(Decimal('1.3')) + Decimal('-1.3') + >>> ExtendedContext.minus(Decimal('-1.3')) + Decimal('1.3') + >>> ExtendedContext.minus(1) + Decimal('-1') + """ + a = _convert_other(a, raiseit=True) + return a.__neg__(context=self) + + def multiply(self, a, b): + """multiply multiplies two operands. + + If either operand is a special value then the general rules apply. + Otherwise, the operands are multiplied together + ('long multiplication'), resulting in a number which may be as long as + the sum of the lengths of the two operands. + + >>> ExtendedContext.multiply(Decimal('1.20'), Decimal('3')) + Decimal('3.60') + >>> ExtendedContext.multiply(Decimal('7'), Decimal('3')) + Decimal('21') + >>> ExtendedContext.multiply(Decimal('0.9'), Decimal('0.8')) + Decimal('0.72') + >>> ExtendedContext.multiply(Decimal('0.9'), Decimal('-0')) + Decimal('-0.0') + >>> ExtendedContext.multiply(Decimal('654321'), Decimal('654321')) + Decimal('4.28135971E+11') + >>> ExtendedContext.multiply(7, 7) + Decimal('49') + >>> ExtendedContext.multiply(Decimal(7), 7) + Decimal('49') + >>> ExtendedContext.multiply(7, Decimal(7)) + Decimal('49') + """ + a = _convert_other(a, raiseit=True) + r = a.__mul__(b, context=self) + if r is NotImplemented: + raise TypeError("Unable to convert %s to Decimal" % b) + else: + return r + + def next_minus(self, a): + """Returns the largest representable number smaller than a. + + >>> c = ExtendedContext.copy() + >>> c.Emin = -999 + >>> c.Emax = 999 + >>> ExtendedContext.next_minus(Decimal('1')) + Decimal('0.999999999') + >>> c.next_minus(Decimal('1E-1007')) + Decimal('0E-1007') + >>> ExtendedContext.next_minus(Decimal('-1.00000003')) + Decimal('-1.00000004') + >>> c.next_minus(Decimal('Infinity')) + Decimal('9.99999999E+999') + >>> c.next_minus(1) + Decimal('0.999999999') + """ + a = _convert_other(a, raiseit=True) + return a.next_minus(context=self) + + def next_plus(self, a): + """Returns the smallest representable number larger than a. + + >>> c = ExtendedContext.copy() + >>> c.Emin = -999 + >>> c.Emax = 999 + >>> ExtendedContext.next_plus(Decimal('1')) + Decimal('1.00000001') + >>> c.next_plus(Decimal('-1E-1007')) + Decimal('-0E-1007') + >>> ExtendedContext.next_plus(Decimal('-1.00000003')) + Decimal('-1.00000002') + >>> c.next_plus(Decimal('-Infinity')) + Decimal('-9.99999999E+999') + >>> c.next_plus(1) + Decimal('1.00000001') + """ + a = _convert_other(a, raiseit=True) + return a.next_plus(context=self) + + def next_toward(self, a, b): + """Returns the number closest to a, in direction towards b. + + The result is the closest representable number from the first + operand (but not the first operand) that is in the direction + towards the second operand, unless the operands have the same + value. + + >>> c = ExtendedContext.copy() + >>> c.Emin = -999 + >>> c.Emax = 999 + >>> c.next_toward(Decimal('1'), Decimal('2')) + Decimal('1.00000001') + >>> c.next_toward(Decimal('-1E-1007'), Decimal('1')) + Decimal('-0E-1007') + >>> c.next_toward(Decimal('-1.00000003'), Decimal('0')) + Decimal('-1.00000002') + >>> c.next_toward(Decimal('1'), Decimal('0')) + Decimal('0.999999999') + >>> c.next_toward(Decimal('1E-1007'), Decimal('-100')) + Decimal('0E-1007') + >>> c.next_toward(Decimal('-1.00000003'), Decimal('-10')) + Decimal('-1.00000004') + >>> c.next_toward(Decimal('0.00'), Decimal('-0.0000')) + Decimal('-0.00') + >>> c.next_toward(0, 1) + Decimal('1E-1007') + >>> c.next_toward(Decimal(0), 1) + Decimal('1E-1007') + >>> c.next_toward(0, Decimal(1)) + Decimal('1E-1007') + """ + a = _convert_other(a, raiseit=True) + return a.next_toward(b, context=self) + + def normalize(self, a): + """normalize reduces an operand to its simplest form. + + Essentially a plus operation with all trailing zeros removed from the + result. + + >>> ExtendedContext.normalize(Decimal('2.1')) + Decimal('2.1') + >>> ExtendedContext.normalize(Decimal('-2.0')) + Decimal('-2') + >>> ExtendedContext.normalize(Decimal('1.200')) + Decimal('1.2') + >>> ExtendedContext.normalize(Decimal('-120')) + Decimal('-1.2E+2') + >>> ExtendedContext.normalize(Decimal('120.00')) + Decimal('1.2E+2') + >>> ExtendedContext.normalize(Decimal('0.00')) + Decimal('0') + >>> ExtendedContext.normalize(6) + Decimal('6') + """ + a = _convert_other(a, raiseit=True) + return a.normalize(context=self) + + def number_class(self, a): + """Returns an indication of the class of the operand. + + The class is one of the following strings: + -sNaN + -NaN + -Infinity + -Normal + -Subnormal + -Zero + +Zero + +Subnormal + +Normal + +Infinity + + >>> c = ExtendedContext.copy() + >>> c.Emin = -999 + >>> c.Emax = 999 + >>> c.number_class(Decimal('Infinity')) + '+Infinity' + >>> c.number_class(Decimal('1E-10')) + '+Normal' + >>> c.number_class(Decimal('2.50')) + '+Normal' + >>> c.number_class(Decimal('0.1E-999')) + '+Subnormal' + >>> c.number_class(Decimal('0')) + '+Zero' + >>> c.number_class(Decimal('-0')) + '-Zero' + >>> c.number_class(Decimal('-0.1E-999')) + '-Subnormal' + >>> c.number_class(Decimal('-1E-10')) + '-Normal' + >>> c.number_class(Decimal('-2.50')) + '-Normal' + >>> c.number_class(Decimal('-Infinity')) + '-Infinity' + >>> c.number_class(Decimal('NaN')) + 'NaN' + >>> c.number_class(Decimal('-NaN')) + 'NaN' + >>> c.number_class(Decimal('sNaN')) + 'sNaN' + >>> c.number_class(123) + '+Normal' + """ + a = _convert_other(a, raiseit=True) + return a.number_class(context=self) + + def plus(self, a): + """Plus corresponds to unary prefix plus in Python. + + The operation is evaluated using the same rules as add; the + operation plus(a) is calculated as add('0', a) where the '0' + has the same exponent as the operand. + + >>> ExtendedContext.plus(Decimal('1.3')) + Decimal('1.3') + >>> ExtendedContext.plus(Decimal('-1.3')) + Decimal('-1.3') + >>> ExtendedContext.plus(-1) + Decimal('-1') + """ + a = _convert_other(a, raiseit=True) + return a.__pos__(context=self) + + def power(self, a, b, modulo=None): + """Raises a to the power of b, to modulo if given. + + With two arguments, compute a**b. If a is negative then b + must be integral. The result will be inexact unless b is + integral and the result is finite and can be expressed exactly + in 'precision' digits. + + With three arguments, compute (a**b) % modulo. For the + three argument form, the following restrictions on the + arguments hold: + + - all three arguments must be integral + - b must be nonnegative + - at least one of a or b must be nonzero + - modulo must be nonzero and have at most 'precision' digits + + The result of pow(a, b, modulo) is identical to the result + that would be obtained by computing (a**b) % modulo with + unbounded precision, but is computed more efficiently. It is + always exact. + + >>> c = ExtendedContext.copy() + >>> c.Emin = -999 + >>> c.Emax = 999 + >>> c.power(Decimal('2'), Decimal('3')) + Decimal('8') + >>> c.power(Decimal('-2'), Decimal('3')) + Decimal('-8') + >>> c.power(Decimal('2'), Decimal('-3')) + Decimal('0.125') + >>> c.power(Decimal('1.7'), Decimal('8')) + Decimal('69.7575744') + >>> c.power(Decimal('10'), Decimal('0.301029996')) + Decimal('2.00000000') + >>> c.power(Decimal('Infinity'), Decimal('-1')) + Decimal('0') + >>> c.power(Decimal('Infinity'), Decimal('0')) + Decimal('1') + >>> c.power(Decimal('Infinity'), Decimal('1')) + Decimal('Infinity') + >>> c.power(Decimal('-Infinity'), Decimal('-1')) + Decimal('-0') + >>> c.power(Decimal('-Infinity'), Decimal('0')) + Decimal('1') + >>> c.power(Decimal('-Infinity'), Decimal('1')) + Decimal('-Infinity') + >>> c.power(Decimal('-Infinity'), Decimal('2')) + Decimal('Infinity') + >>> c.power(Decimal('0'), Decimal('0')) + Decimal('NaN') + + >>> c.power(Decimal('3'), Decimal('7'), Decimal('16')) + Decimal('11') + >>> c.power(Decimal('-3'), Decimal('7'), Decimal('16')) + Decimal('-11') + >>> c.power(Decimal('-3'), Decimal('8'), Decimal('16')) + Decimal('1') + >>> c.power(Decimal('3'), Decimal('7'), Decimal('-16')) + Decimal('11') + >>> c.power(Decimal('23E12345'), Decimal('67E189'), Decimal('123456789')) + Decimal('11729830') + >>> c.power(Decimal('-0'), Decimal('17'), Decimal('1729')) + Decimal('-0') + >>> c.power(Decimal('-23'), Decimal('0'), Decimal('65537')) + Decimal('1') + >>> ExtendedContext.power(7, 7) + Decimal('823543') + >>> ExtendedContext.power(Decimal(7), 7) + Decimal('823543') + >>> ExtendedContext.power(7, Decimal(7), 2) + Decimal('1') + """ + a = _convert_other(a, raiseit=True) + r = a.__pow__(b, modulo, context=self) + if r is NotImplemented: + raise TypeError("Unable to convert %s to Decimal" % b) + else: + return r + + def quantize(self, a, b): + """Returns a value equal to 'a' (rounded), having the exponent of 'b'. + + The coefficient of the result is derived from that of the left-hand + operand. It may be rounded using the current rounding setting (if the + exponent is being increased), multiplied by a positive power of ten (if + the exponent is being decreased), or is unchanged (if the exponent is + already equal to that of the right-hand operand). + + Unlike other operations, if the length of the coefficient after the + quantize operation would be greater than precision then an Invalid + operation condition is raised. This guarantees that, unless there is + an error condition, the exponent of the result of a quantize is always + equal to that of the right-hand operand. + + Also unlike other operations, quantize will never raise Underflow, even + if the result is subnormal and inexact. + + >>> ExtendedContext.quantize(Decimal('2.17'), Decimal('0.001')) + Decimal('2.170') + >>> ExtendedContext.quantize(Decimal('2.17'), Decimal('0.01')) + Decimal('2.17') + >>> ExtendedContext.quantize(Decimal('2.17'), Decimal('0.1')) + Decimal('2.2') + >>> ExtendedContext.quantize(Decimal('2.17'), Decimal('1e+0')) + Decimal('2') + >>> ExtendedContext.quantize(Decimal('2.17'), Decimal('1e+1')) + Decimal('0E+1') + >>> ExtendedContext.quantize(Decimal('-Inf'), Decimal('Infinity')) + Decimal('-Infinity') + >>> ExtendedContext.quantize(Decimal('2'), Decimal('Infinity')) + Decimal('NaN') + >>> ExtendedContext.quantize(Decimal('-0.1'), Decimal('1')) + Decimal('-0') + >>> ExtendedContext.quantize(Decimal('-0'), Decimal('1e+5')) + Decimal('-0E+5') + >>> ExtendedContext.quantize(Decimal('+35236450.6'), Decimal('1e-2')) + Decimal('NaN') + >>> ExtendedContext.quantize(Decimal('-35236450.6'), Decimal('1e-2')) + Decimal('NaN') + >>> ExtendedContext.quantize(Decimal('217'), Decimal('1e-1')) + Decimal('217.0') + >>> ExtendedContext.quantize(Decimal('217'), Decimal('1e-0')) + Decimal('217') + >>> ExtendedContext.quantize(Decimal('217'), Decimal('1e+1')) + Decimal('2.2E+2') + >>> ExtendedContext.quantize(Decimal('217'), Decimal('1e+2')) + Decimal('2E+2') + >>> ExtendedContext.quantize(1, 2) + Decimal('1') + >>> ExtendedContext.quantize(Decimal(1), 2) + Decimal('1') + >>> ExtendedContext.quantize(1, Decimal(2)) + Decimal('1') + """ + a = _convert_other(a, raiseit=True) + return a.quantize(b, context=self) + + def radix(self): + """Just returns 10, as this is Decimal, :) + + >>> ExtendedContext.radix() + Decimal('10') + """ + return Decimal(10) + + def remainder(self, a, b): + """Returns the remainder from integer division. + + The result is the residue of the dividend after the operation of + calculating integer division as described for divide-integer, rounded + to precision digits if necessary. The sign of the result, if + non-zero, is the same as that of the original dividend. + + This operation will fail under the same conditions as integer division + (that is, if integer division on the same two operands would fail, the + remainder cannot be calculated). + + >>> ExtendedContext.remainder(Decimal('2.1'), Decimal('3')) + Decimal('2.1') + >>> ExtendedContext.remainder(Decimal('10'), Decimal('3')) + Decimal('1') + >>> ExtendedContext.remainder(Decimal('-10'), Decimal('3')) + Decimal('-1') + >>> ExtendedContext.remainder(Decimal('10.2'), Decimal('1')) + Decimal('0.2') + >>> ExtendedContext.remainder(Decimal('10'), Decimal('0.3')) + Decimal('0.1') + >>> ExtendedContext.remainder(Decimal('3.6'), Decimal('1.3')) + Decimal('1.0') + >>> ExtendedContext.remainder(22, 6) + Decimal('4') + >>> ExtendedContext.remainder(Decimal(22), 6) + Decimal('4') + >>> ExtendedContext.remainder(22, Decimal(6)) + Decimal('4') + """ + a = _convert_other(a, raiseit=True) + r = a.__mod__(b, context=self) + if r is NotImplemented: + raise TypeError("Unable to convert %s to Decimal" % b) + else: + return r + + def remainder_near(self, a, b): + """Returns to be "a - b * n", where n is the integer nearest the exact + value of "x / b" (if two integers are equally near then the even one + is chosen). If the result is equal to 0 then its sign will be the + sign of a. + + This operation will fail under the same conditions as integer division + (that is, if integer division on the same two operands would fail, the + remainder cannot be calculated). + + >>> ExtendedContext.remainder_near(Decimal('2.1'), Decimal('3')) + Decimal('-0.9') + >>> ExtendedContext.remainder_near(Decimal('10'), Decimal('6')) + Decimal('-2') + >>> ExtendedContext.remainder_near(Decimal('10'), Decimal('3')) + Decimal('1') + >>> ExtendedContext.remainder_near(Decimal('-10'), Decimal('3')) + Decimal('-1') + >>> ExtendedContext.remainder_near(Decimal('10.2'), Decimal('1')) + Decimal('0.2') + >>> ExtendedContext.remainder_near(Decimal('10'), Decimal('0.3')) + Decimal('0.1') + >>> ExtendedContext.remainder_near(Decimal('3.6'), Decimal('1.3')) + Decimal('-0.3') + >>> ExtendedContext.remainder_near(3, 11) + Decimal('3') + >>> ExtendedContext.remainder_near(Decimal(3), 11) + Decimal('3') + >>> ExtendedContext.remainder_near(3, Decimal(11)) + Decimal('3') + """ + a = _convert_other(a, raiseit=True) + return a.remainder_near(b, context=self) + + def rotate(self, a, b): + """Returns a rotated copy of a, b times. + + The coefficient of the result is a rotated copy of the digits in + the coefficient of the first operand. The number of places of + rotation is taken from the absolute value of the second operand, + with the rotation being to the left if the second operand is + positive or to the right otherwise. + + >>> ExtendedContext.rotate(Decimal('34'), Decimal('8')) + Decimal('400000003') + >>> ExtendedContext.rotate(Decimal('12'), Decimal('9')) + Decimal('12') + >>> ExtendedContext.rotate(Decimal('123456789'), Decimal('-2')) + Decimal('891234567') + >>> ExtendedContext.rotate(Decimal('123456789'), Decimal('0')) + Decimal('123456789') + >>> ExtendedContext.rotate(Decimal('123456789'), Decimal('+2')) + Decimal('345678912') + >>> ExtendedContext.rotate(1333333, 1) + Decimal('13333330') + >>> ExtendedContext.rotate(Decimal(1333333), 1) + Decimal('13333330') + >>> ExtendedContext.rotate(1333333, Decimal(1)) + Decimal('13333330') + """ + a = _convert_other(a, raiseit=True) + return a.rotate(b, context=self) + + def same_quantum(self, a, b): + """Returns True if the two operands have the same exponent. + + The result is never affected by either the sign or the coefficient of + either operand. + + >>> ExtendedContext.same_quantum(Decimal('2.17'), Decimal('0.001')) + False + >>> ExtendedContext.same_quantum(Decimal('2.17'), Decimal('0.01')) + True + >>> ExtendedContext.same_quantum(Decimal('2.17'), Decimal('1')) + False + >>> ExtendedContext.same_quantum(Decimal('Inf'), Decimal('-Inf')) + True + >>> ExtendedContext.same_quantum(10000, -1) + True + >>> ExtendedContext.same_quantum(Decimal(10000), -1) + True + >>> ExtendedContext.same_quantum(10000, Decimal(-1)) + True + """ + a = _convert_other(a, raiseit=True) + return a.same_quantum(b) + + def scaleb (self, a, b): + """Returns the first operand after adding the second value its exp. + + >>> ExtendedContext.scaleb(Decimal('7.50'), Decimal('-2')) + Decimal('0.0750') + >>> ExtendedContext.scaleb(Decimal('7.50'), Decimal('0')) + Decimal('7.50') + >>> ExtendedContext.scaleb(Decimal('7.50'), Decimal('3')) + Decimal('7.50E+3') + >>> ExtendedContext.scaleb(1, 4) + Decimal('1E+4') + >>> ExtendedContext.scaleb(Decimal(1), 4) + Decimal('1E+4') + >>> ExtendedContext.scaleb(1, Decimal(4)) + Decimal('1E+4') + """ + a = _convert_other(a, raiseit=True) + return a.scaleb(b, context=self) + + def shift(self, a, b): + """Returns a shifted copy of a, b times. + + The coefficient of the result is a shifted copy of the digits + in the coefficient of the first operand. The number of places + to shift is taken from the absolute value of the second operand, + with the shift being to the left if the second operand is + positive or to the right otherwise. Digits shifted into the + coefficient are zeros. + + >>> ExtendedContext.shift(Decimal('34'), Decimal('8')) + Decimal('400000000') + >>> ExtendedContext.shift(Decimal('12'), Decimal('9')) + Decimal('0') + >>> ExtendedContext.shift(Decimal('123456789'), Decimal('-2')) + Decimal('1234567') + >>> ExtendedContext.shift(Decimal('123456789'), Decimal('0')) + Decimal('123456789') + >>> ExtendedContext.shift(Decimal('123456789'), Decimal('+2')) + Decimal('345678900') + >>> ExtendedContext.shift(88888888, 2) + Decimal('888888800') + >>> ExtendedContext.shift(Decimal(88888888), 2) + Decimal('888888800') + >>> ExtendedContext.shift(88888888, Decimal(2)) + Decimal('888888800') + """ + a = _convert_other(a, raiseit=True) + return a.shift(b, context=self) + + def sqrt(self, a): + """Square root of a non-negative number to context precision. + + If the result must be inexact, it is rounded using the round-half-even + algorithm. + + >>> ExtendedContext.sqrt(Decimal('0')) + Decimal('0') + >>> ExtendedContext.sqrt(Decimal('-0')) + Decimal('-0') + >>> ExtendedContext.sqrt(Decimal('0.39')) + Decimal('0.624499800') + >>> ExtendedContext.sqrt(Decimal('100')) + Decimal('10') + >>> ExtendedContext.sqrt(Decimal('1')) + Decimal('1') + >>> ExtendedContext.sqrt(Decimal('1.0')) + Decimal('1.0') + >>> ExtendedContext.sqrt(Decimal('1.00')) + Decimal('1.0') + >>> ExtendedContext.sqrt(Decimal('7')) + Decimal('2.64575131') + >>> ExtendedContext.sqrt(Decimal('10')) + Decimal('3.16227766') + >>> ExtendedContext.sqrt(2) + Decimal('1.41421356') + >>> ExtendedContext.prec + 9 + """ + a = _convert_other(a, raiseit=True) + return a.sqrt(context=self) + + def subtract(self, a, b): + """Return the difference between the two operands. + + >>> ExtendedContext.subtract(Decimal('1.3'), Decimal('1.07')) + Decimal('0.23') + >>> ExtendedContext.subtract(Decimal('1.3'), Decimal('1.30')) + Decimal('0.00') + >>> ExtendedContext.subtract(Decimal('1.3'), Decimal('2.07')) + Decimal('-0.77') + >>> ExtendedContext.subtract(8, 5) + Decimal('3') + >>> ExtendedContext.subtract(Decimal(8), 5) + Decimal('3') + >>> ExtendedContext.subtract(8, Decimal(5)) + Decimal('3') + """ + a = _convert_other(a, raiseit=True) + r = a.__sub__(b, context=self) + if r is NotImplemented: + raise TypeError("Unable to convert %s to Decimal" % b) + else: + return r + + def to_eng_string(self, a): + """Convert to a string, using engineering notation if an exponent is needed. + + Engineering notation has an exponent which is a multiple of 3. This + can leave up to 3 digits to the left of the decimal place and may + require the addition of either one or two trailing zeros. + + The operation is not affected by the context. + + >>> ExtendedContext.to_eng_string(Decimal('123E+1')) + '1.23E+3' + >>> ExtendedContext.to_eng_string(Decimal('123E+3')) + '123E+3' + >>> ExtendedContext.to_eng_string(Decimal('123E-10')) + '12.3E-9' + >>> ExtendedContext.to_eng_string(Decimal('-123E-12')) + '-123E-12' + >>> ExtendedContext.to_eng_string(Decimal('7E-7')) + '700E-9' + >>> ExtendedContext.to_eng_string(Decimal('7E+1')) + '70' + >>> ExtendedContext.to_eng_string(Decimal('0E+1')) + '0.00E+3' + + """ + a = _convert_other(a, raiseit=True) + return a.to_eng_string(context=self) + + def to_sci_string(self, a): + """Converts a number to a string, using scientific notation. + + The operation is not affected by the context. + """ + a = _convert_other(a, raiseit=True) + return a.__str__(context=self) + + def to_integral_exact(self, a): + """Rounds to an integer. + + When the operand has a negative exponent, the result is the same + as using the quantize() operation using the given operand as the + left-hand-operand, 1E+0 as the right-hand-operand, and the precision + of the operand as the precision setting; Inexact and Rounded flags + are allowed in this operation. The rounding mode is taken from the + context. + + >>> ExtendedContext.to_integral_exact(Decimal('2.1')) + Decimal('2') + >>> ExtendedContext.to_integral_exact(Decimal('100')) + Decimal('100') + >>> ExtendedContext.to_integral_exact(Decimal('100.0')) + Decimal('100') + >>> ExtendedContext.to_integral_exact(Decimal('101.5')) + Decimal('102') + >>> ExtendedContext.to_integral_exact(Decimal('-101.5')) + Decimal('-102') + >>> ExtendedContext.to_integral_exact(Decimal('10E+5')) + Decimal('1.0E+6') + >>> ExtendedContext.to_integral_exact(Decimal('7.89E+77')) + Decimal('7.89E+77') + >>> ExtendedContext.to_integral_exact(Decimal('-Inf')) + Decimal('-Infinity') + """ + a = _convert_other(a, raiseit=True) + return a.to_integral_exact(context=self) + + def to_integral_value(self, a): + """Rounds to an integer. + + When the operand has a negative exponent, the result is the same + as using the quantize() operation using the given operand as the + left-hand-operand, 1E+0 as the right-hand-operand, and the precision + of the operand as the precision setting, except that no flags will + be set. The rounding mode is taken from the context. + + >>> ExtendedContext.to_integral_value(Decimal('2.1')) + Decimal('2') + >>> ExtendedContext.to_integral_value(Decimal('100')) + Decimal('100') + >>> ExtendedContext.to_integral_value(Decimal('100.0')) + Decimal('100') + >>> ExtendedContext.to_integral_value(Decimal('101.5')) + Decimal('102') + >>> ExtendedContext.to_integral_value(Decimal('-101.5')) + Decimal('-102') + >>> ExtendedContext.to_integral_value(Decimal('10E+5')) + Decimal('1.0E+6') + >>> ExtendedContext.to_integral_value(Decimal('7.89E+77')) + Decimal('7.89E+77') + >>> ExtendedContext.to_integral_value(Decimal('-Inf')) + Decimal('-Infinity') + """ + a = _convert_other(a, raiseit=True) + return a.to_integral_value(context=self) + + # the method name changed, but we provide also the old one, for compatibility + to_integral = to_integral_value + +class _WorkRep(object): + __slots__ = ('sign','int','exp') + # sign: 0 or 1 + # int: int + # exp: None, int, or string + + def __init__(self, value=None): + if value is None: + self.sign = None + self.int = 0 + self.exp = None + elif isinstance(value, Decimal): + self.sign = value._sign + self.int = int(value._int) + self.exp = value._exp + else: + # assert isinstance(value, tuple) + self.sign = value[0] + self.int = value[1] + self.exp = value[2] + + def __repr__(self): + return "(%r, %r, %r)" % (self.sign, self.int, self.exp) + + __str__ = __repr__ + + + +def _normalize(op1, op2, prec = 0): + """Normalizes op1, op2 to have the same exp and length of coefficient. + + Done during addition. + """ + if op1.exp < op2.exp: + tmp = op2 + other = op1 + else: + tmp = op1 + other = op2 + + # Let exp = min(tmp.exp - 1, tmp.adjusted() - precision - 1). + # Then adding 10**exp to tmp has the same effect (after rounding) + # as adding any positive quantity smaller than 10**exp; similarly + # for subtraction. So if other is smaller than 10**exp we replace + # it with 10**exp. This avoids tmp.exp - other.exp getting too large. + tmp_len = len(str(tmp.int)) + other_len = len(str(other.int)) + exp = tmp.exp + min(-1, tmp_len - prec - 2) + if other_len + other.exp - 1 < exp: + other.int = 1 + other.exp = exp + + tmp.int *= 10 ** (tmp.exp - other.exp) + tmp.exp = other.exp + return op1, op2 + +##### Integer arithmetic functions used by ln, log10, exp and __pow__ ##### + +_nbits = int.bit_length + +def _decimal_lshift_exact(n, e): + """ Given integers n and e, return n * 10**e if it's an integer, else None. + + The computation is designed to avoid computing large powers of 10 + unnecessarily. + + >>> _decimal_lshift_exact(3, 4) + 30000 + >>> _decimal_lshift_exact(300, -999999999) # returns None + + """ + if n == 0: + return 0 + elif e >= 0: + return n * 10**e + else: + # val_n = largest power of 10 dividing n. + str_n = str(abs(n)) + val_n = len(str_n) - len(str_n.rstrip('0')) + return None if val_n < -e else n // 10**-e + +def _sqrt_nearest(n, a): + """Closest integer to the square root of the positive integer n. a is + an initial approximation to the square root. Any positive integer + will do for a, but the closer a is to the square root of n the + faster convergence will be. + + """ + if n <= 0 or a <= 0: + raise ValueError("Both arguments to _sqrt_nearest should be positive.") + + b=0 + while a != b: + b, a = a, a--n//a>>1 + return a + +def _rshift_nearest(x, shift): + """Given an integer x and a nonnegative integer shift, return closest + integer to x / 2**shift; use round-to-even in case of a tie. + + """ + b, q = 1 << shift, x >> shift + return q + (2*(x & (b-1)) + (q&1) > b) + +def _div_nearest(a, b): + """Closest integer to a/b, a and b positive integers; rounds to even + in the case of a tie. + + """ + q, r = divmod(a, b) + return q + (2*r + (q&1) > b) + +def _ilog(x, M, L = 8): + """Integer approximation to M*log(x/M), with absolute error boundable + in terms only of x/M. + + Given positive integers x and M, return an integer approximation to + M * log(x/M). For L = 8 and 0.1 <= x/M <= 10 the difference + between the approximation and the exact result is at most 22. For + L = 8 and 1.0 <= x/M <= 10.0 the difference is at most 15. In + both cases these are upper bounds on the error; it will usually be + much smaller.""" + + # The basic algorithm is the following: let log1p be the function + # log1p(x) = log(1+x). Then log(x/M) = log1p((x-M)/M). We use + # the reduction + # + # log1p(y) = 2*log1p(y/(1+sqrt(1+y))) + # + # repeatedly until the argument to log1p is small (< 2**-L in + # absolute value). For small y we can use the Taylor series + # expansion + # + # log1p(y) ~ y - y**2/2 + y**3/3 - ... - (-y)**T/T + # + # truncating at T such that y**T is small enough. The whole + # computation is carried out in a form of fixed-point arithmetic, + # with a real number z being represented by an integer + # approximation to z*M. To avoid loss of precision, the y below + # is actually an integer approximation to 2**R*y*M, where R is the + # number of reductions performed so far. + + y = x-M + # argument reduction; R = number of reductions performed + R = 0 + while (R <= L and abs(y) << L-R >= M or + R > L and abs(y) >> R-L >= M): + y = _div_nearest((M*y) << 1, + M + _sqrt_nearest(M*(M+_rshift_nearest(y, R)), M)) + R += 1 + + # Taylor series with T terms + T = -int(-10*len(str(M))//(3*L)) + yshift = _rshift_nearest(y, R) + w = _div_nearest(M, T) + for k in range(T-1, 0, -1): + w = _div_nearest(M, k) - _div_nearest(yshift*w, M) + + return _div_nearest(w*y, M) + +def _dlog10(c, e, p): + """Given integers c, e and p with c > 0, p >= 0, compute an integer + approximation to 10**p * log10(c*10**e), with an absolute error of + at most 1. Assumes that c*10**e is not exactly 1.""" + + # increase precision by 2; compensate for this by dividing + # final result by 100 + p += 2 + + # write c*10**e as d*10**f with either: + # f >= 0 and 1 <= d <= 10, or + # f <= 0 and 0.1 <= d <= 1. + # Thus for c*10**e close to 1, f = 0 + l = len(str(c)) + f = e+l - (e+l >= 1) + + if p > 0: + M = 10**p + k = e+p-f + if k >= 0: + c *= 10**k + else: + c = _div_nearest(c, 10**-k) + + log_d = _ilog(c, M) # error < 5 + 22 = 27 + log_10 = _log10_digits(p) # error < 1 + log_d = _div_nearest(log_d*M, log_10) + log_tenpower = f*M # exact + else: + log_d = 0 # error < 2.31 + log_tenpower = _div_nearest(f, 10**-p) # error < 0.5 + + return _div_nearest(log_tenpower+log_d, 100) + +def _dlog(c, e, p): + """Given integers c, e and p with c > 0, compute an integer + approximation to 10**p * log(c*10**e), with an absolute error of + at most 1. Assumes that c*10**e is not exactly 1.""" + + # Increase precision by 2. The precision increase is compensated + # for at the end with a division by 100. + p += 2 + + # rewrite c*10**e as d*10**f with either f >= 0 and 1 <= d <= 10, + # or f <= 0 and 0.1 <= d <= 1. Then we can compute 10**p * log(c*10**e) + # as 10**p * log(d) + 10**p*f * log(10). + l = len(str(c)) + f = e+l - (e+l >= 1) + + # compute approximation to 10**p*log(d), with error < 27 + if p > 0: + k = e+p-f + if k >= 0: + c *= 10**k + else: + c = _div_nearest(c, 10**-k) # error of <= 0.5 in c + + # _ilog magnifies existing error in c by a factor of at most 10 + log_d = _ilog(c, 10**p) # error < 5 + 22 = 27 + else: + # p <= 0: just approximate the whole thing by 0; error < 2.31 + log_d = 0 + + # compute approximation to f*10**p*log(10), with error < 11. + if f: + extra = len(str(abs(f)))-1 + if p + extra >= 0: + # error in f * _log10_digits(p+extra) < |f| * 1 = |f| + # after division, error < |f|/10**extra + 0.5 < 10 + 0.5 < 11 + f_log_ten = _div_nearest(f*_log10_digits(p+extra), 10**extra) + else: + f_log_ten = 0 + else: + f_log_ten = 0 + + # error in sum < 11+27 = 38; error after division < 0.38 + 0.5 < 1 + return _div_nearest(f_log_ten + log_d, 100) + +class _Log10Memoize(object): + """Class to compute, store, and allow retrieval of, digits of the + constant log(10) = 2.302585.... This constant is needed by + Decimal.ln, Decimal.log10, Decimal.exp and Decimal.__pow__.""" + def __init__(self): + self.digits = "23025850929940456840179914546843642076011014886" + + def getdigits(self, p): + """Given an integer p >= 0, return floor(10**p)*log(10). + + For example, self.getdigits(3) returns 2302. + """ + # digits are stored as a string, for quick conversion to + # integer in the case that we've already computed enough + # digits; the stored digits should always be correct + # (truncated, not rounded to nearest). + if p < 0: + raise ValueError("p should be nonnegative") + + if p >= len(self.digits): + # compute p+3, p+6, p+9, ... digits; continue until at + # least one of the extra digits is nonzero + extra = 3 + while True: + # compute p+extra digits, correct to within 1ulp + M = 10**(p+extra+2) + digits = str(_div_nearest(_ilog(10*M, M), 100)) + if digits[-extra:] != '0'*extra: + break + extra += 3 + # keep all reliable digits so far; remove trailing zeros + # and next nonzero digit + self.digits = digits.rstrip('0')[:-1] + return int(self.digits[:p+1]) + +_log10_digits = _Log10Memoize().getdigits + +def _iexp(x, M, L=8): + """Given integers x and M, M > 0, such that x/M is small in absolute + value, compute an integer approximation to M*exp(x/M). For 0 <= + x/M <= 2.4, the absolute error in the result is bounded by 60 (and + is usually much smaller).""" + + # Algorithm: to compute exp(z) for a real number z, first divide z + # by a suitable power R of 2 so that |z/2**R| < 2**-L. Then + # compute expm1(z/2**R) = exp(z/2**R) - 1 using the usual Taylor + # series + # + # expm1(x) = x + x**2/2! + x**3/3! + ... + # + # Now use the identity + # + # expm1(2x) = expm1(x)*(expm1(x)+2) + # + # R times to compute the sequence expm1(z/2**R), + # expm1(z/2**(R-1)), ... , exp(z/2), exp(z). + + # Find R such that x/2**R/M <= 2**-L + R = _nbits((x< M + T = -int(-10*len(str(M))//(3*L)) + y = _div_nearest(x, T) + Mshift = M<= 0: + cshift = c*10**shift + else: + cshift = c//10**-shift + quot, rem = divmod(cshift, _log10_digits(q)) + + # reduce remainder back to original precision + rem = _div_nearest(rem, 10**extra) + + # error in result of _iexp < 120; error after division < 0.62 + return _div_nearest(_iexp(rem, 10**p), 1000), quot - p + 3 + +def _dpower(xc, xe, yc, ye, p): + """Given integers xc, xe, yc and ye representing Decimals x = xc*10**xe and + y = yc*10**ye, compute x**y. Returns a pair of integers (c, e) such that: + + 10**(p-1) <= c <= 10**p, and + (c-1)*10**e < x**y < (c+1)*10**e + + in other words, c*10**e is an approximation to x**y with p digits + of precision, and with an error in c of at most 1. (This is + almost, but not quite, the same as the error being < 1ulp: when c + == 10**(p-1) we can only guarantee error < 10ulp.) + + We assume that: x is positive and not equal to 1, and y is nonzero. + """ + + # Find b such that 10**(b-1) <= |y| <= 10**b + b = len(str(abs(yc))) + ye + + # log(x) = lxc*10**(-p-b-1), to p+b+1 places after the decimal point + lxc = _dlog(xc, xe, p+b+1) + + # compute product y*log(x) = yc*lxc*10**(-p-b-1+ye) = pc*10**(-p-1) + shift = ye-b + if shift >= 0: + pc = lxc*yc*10**shift + else: + pc = _div_nearest(lxc*yc, 10**-shift) + + if pc == 0: + # we prefer a result that isn't exactly 1; this makes it + # easier to compute a correctly rounded result in __pow__ + if ((len(str(xc)) + xe >= 1) == (yc > 0)): # if x**y > 1: + coeff, exp = 10**(p-1)+1, 1-p + else: + coeff, exp = 10**p-1, -p + else: + coeff, exp = _dexp(pc, -(p+1), p+1) + coeff = _div_nearest(coeff, 10) + exp += 1 + + return coeff, exp + +def _log10_lb(c, correction = { + '1': 100, '2': 70, '3': 53, '4': 40, '5': 31, + '6': 23, '7': 16, '8': 10, '9': 5}): + """Compute a lower bound for 100*log10(c) for a positive integer c.""" + if c <= 0: + raise ValueError("The argument to _log10_lb should be nonnegative.") + str_c = str(c) + return 100*len(str_c) - correction[str_c[0]] + +##### Helper Functions #################################################### + +def _convert_other(other, raiseit=False, allow_float=False): + """Convert other to Decimal. + + Verifies that it's ok to use in an implicit construction. + If allow_float is true, allow conversion from float; this + is used in the comparison methods (__eq__ and friends). + + """ + if isinstance(other, Decimal): + return other + if isinstance(other, int): + return Decimal(other) + if allow_float and isinstance(other, float): + return Decimal.from_float(other) + + if raiseit: + raise TypeError("Unable to convert %s to Decimal" % other) + return NotImplemented + +def _convert_for_comparison(self, other, equality_op=False): + """Given a Decimal instance self and a Python object other, return + a pair (s, o) of Decimal instances such that "s op o" is + equivalent to "self op other" for any of the 6 comparison + operators "op". + + """ + if isinstance(other, Decimal): + return self, other + + # Comparison with a Rational instance (also includes integers): + # self op n/d <=> self*d op n (for n and d integers, d positive). + # A NaN or infinity can be left unchanged without affecting the + # comparison result. + if isinstance(other, _numbers.Rational): + if not self._is_special: + self = _dec_from_triple(self._sign, + str(int(self._int) * other.denominator), + self._exp) + return self, Decimal(other.numerator) + + # Comparisons with float and complex types. == and != comparisons + # with complex numbers should succeed, returning either True or False + # as appropriate. Other comparisons return NotImplemented. + if equality_op and isinstance(other, _numbers.Complex) and other.imag == 0: + other = other.real + if isinstance(other, float): + context = getcontext() + if equality_op: + context.flags[FloatOperation] = 1 + else: + context._raise_error(FloatOperation, + "strict semantics for mixing floats and Decimals are enabled") + return self, Decimal.from_float(other) + return NotImplemented, NotImplemented + + +##### Setup Specific Contexts ############################################ + +# The default context prototype used by Context() +# Is mutable, so that new contexts can have different default values + +DefaultContext = Context( + prec=28, rounding=ROUND_HALF_EVEN, + traps=[DivisionByZero, Overflow, InvalidOperation], + flags=[], + Emax=999999, + Emin=-999999, + capitals=1, + clamp=0 +) + +# Pre-made alternate contexts offered by the specification +# Don't change these; the user should be able to select these +# contexts and be able to reproduce results from other implementations +# of the spec. + +BasicContext = Context( + prec=9, rounding=ROUND_HALF_UP, + traps=[DivisionByZero, Overflow, InvalidOperation, Clamped, Underflow], + flags=[], +) + +ExtendedContext = Context( + prec=9, rounding=ROUND_HALF_EVEN, + traps=[], + flags=[], +) + + +##### crud for parsing strings ############################################# +# +# Regular expression used for parsing numeric strings. Additional +# comments: +# +# 1. Uncomment the two '\s*' lines to allow leading and/or trailing +# whitespace. But note that the specification disallows whitespace in +# a numeric string. +# +# 2. For finite numbers (not infinities and NaNs) the body of the +# number between the optional sign and the optional exponent must have +# at least one decimal digit, possibly after the decimal point. The +# lookahead expression '(?=\d|\.\d)' checks this. + +import re +_parser = re.compile(r""" # A numeric string consists of: +# \s* + (?P[-+])? # an optional sign, followed by either... + ( + (?=\d|\.\d) # ...a number (with at least one digit) + (?P\d*) # having a (possibly empty) integer part + (\.(?P\d*))? # followed by an optional fractional part + (E(?P[-+]?\d+))? # followed by an optional exponent, or... + | + Inf(inity)? # ...an infinity, or... + | + (?Ps)? # ...an (optionally signaling) + NaN # NaN + (?P\d*) # with (possibly empty) diagnostic info. + ) +# \s* + \Z +""", re.VERBOSE | re.IGNORECASE).match + +_all_zeros = re.compile('0*$').match +_exact_half = re.compile('50*$').match + +##### PEP3101 support functions ############################################## +# The functions in this section have little to do with the Decimal +# class, and could potentially be reused or adapted for other pure +# Python numeric classes that want to implement __format__ +# +# A format specifier for Decimal looks like: +# +# [[fill]align][sign][#][0][minimumwidth][,][.precision][type] + +_parse_format_specifier_regex = re.compile(r"""\A +(?: + (?P.)? + (?P[<>=^]) +)? +(?P[-+ ])? +(?P\#)? +(?P0)? +(?P(?!0)\d+)? +(?P,)? +(?:\.(?P0|(?!0)\d+))? +(?P[eEfFgGn%])? +\Z +""", re.VERBOSE|re.DOTALL) + +del re + +# The locale module is only needed for the 'n' format specifier. The +# rest of the PEP 3101 code functions quite happily without it, so we +# don't care too much if locale isn't present. +try: + import locale as _locale +except ImportError: + pass + +def _parse_format_specifier(format_spec, _localeconv=None): + """Parse and validate a format specifier. + + Turns a standard numeric format specifier into a dict, with the + following entries: + + fill: fill character to pad field to minimum width + align: alignment type, either '<', '>', '=' or '^' + sign: either '+', '-' or ' ' + minimumwidth: nonnegative integer giving minimum width + zeropad: boolean, indicating whether to pad with zeros + thousands_sep: string to use as thousands separator, or '' + grouping: grouping for thousands separators, in format + used by localeconv + decimal_point: string to use for decimal point + precision: nonnegative integer giving precision, or None + type: one of the characters 'eEfFgG%', or None + + """ + m = _parse_format_specifier_regex.match(format_spec) + if m is None: + raise ValueError("Invalid format specifier: " + format_spec) + + # get the dictionary + format_dict = m.groupdict() + + # zeropad; defaults for fill and alignment. If zero padding + # is requested, the fill and align fields should be absent. + fill = format_dict['fill'] + align = format_dict['align'] + format_dict['zeropad'] = (format_dict['zeropad'] is not None) + if format_dict['zeropad']: + if fill is not None: + raise ValueError("Fill character conflicts with '0'" + " in format specifier: " + format_spec) + if align is not None: + raise ValueError("Alignment conflicts with '0' in " + "format specifier: " + format_spec) + format_dict['fill'] = fill or ' ' + # PEP 3101 originally specified that the default alignment should + # be left; it was later agreed that right-aligned makes more sense + # for numeric types. See http://bugs.python.org/issue6857. + format_dict['align'] = align or '>' + + # default sign handling: '-' for negative, '' for positive + if format_dict['sign'] is None: + format_dict['sign'] = '-' + + # minimumwidth defaults to 0; precision remains None if not given + format_dict['minimumwidth'] = int(format_dict['minimumwidth'] or '0') + if format_dict['precision'] is not None: + format_dict['precision'] = int(format_dict['precision']) + + # if format type is 'g' or 'G' then a precision of 0 makes little + # sense; convert it to 1. Same if format type is unspecified. + if format_dict['precision'] == 0: + if format_dict['type'] is None or format_dict['type'] in 'gGn': + format_dict['precision'] = 1 + + # determine thousands separator, grouping, and decimal separator, and + # add appropriate entries to format_dict + if format_dict['type'] == 'n': + # apart from separators, 'n' behaves just like 'g' + format_dict['type'] = 'g' + if _localeconv is None: + _localeconv = _locale.localeconv() + if format_dict['thousands_sep'] is not None: + raise ValueError("Explicit thousands separator conflicts with " + "'n' type in format specifier: " + format_spec) + format_dict['thousands_sep'] = _localeconv['thousands_sep'] + format_dict['grouping'] = _localeconv['grouping'] + format_dict['decimal_point'] = _localeconv['decimal_point'] + else: + if format_dict['thousands_sep'] is None: + format_dict['thousands_sep'] = '' + format_dict['grouping'] = [3, 0] + format_dict['decimal_point'] = '.' + + return format_dict + +def _format_align(sign, body, spec): + """Given an unpadded, non-aligned numeric string 'body' and sign + string 'sign', add padding and alignment conforming to the given + format specifier dictionary 'spec' (as produced by + parse_format_specifier). + + """ + # how much extra space do we have to play with? + minimumwidth = spec['minimumwidth'] + fill = spec['fill'] + padding = fill*(minimumwidth - len(sign) - len(body)) + + align = spec['align'] + if align == '<': + result = sign + body + padding + elif align == '>': + result = padding + sign + body + elif align == '=': + result = sign + padding + body + elif align == '^': + half = len(padding)//2 + result = padding[:half] + sign + body + padding[half:] + else: + raise ValueError('Unrecognised alignment field') + + return result + +def _group_lengths(grouping): + """Convert a localeconv-style grouping into a (possibly infinite) + iterable of integers representing group lengths. + + """ + # The result from localeconv()['grouping'], and the input to this + # function, should be a list of integers in one of the + # following three forms: + # + # (1) an empty list, or + # (2) nonempty list of positive integers + [0] + # (3) list of positive integers + [locale.CHAR_MAX], or + + from itertools import chain, repeat + if not grouping: + return [] + elif grouping[-1] == 0 and len(grouping) >= 2: + return chain(grouping[:-1], repeat(grouping[-2])) + elif grouping[-1] == _locale.CHAR_MAX: + return grouping[:-1] + else: + raise ValueError('unrecognised format for grouping') + +def _insert_thousands_sep(digits, spec, min_width=1): + """Insert thousands separators into a digit string. + + spec is a dictionary whose keys should include 'thousands_sep' and + 'grouping'; typically it's the result of parsing the format + specifier using _parse_format_specifier. + + The min_width keyword argument gives the minimum length of the + result, which will be padded on the left with zeros if necessary. + + If necessary, the zero padding adds an extra '0' on the left to + avoid a leading thousands separator. For example, inserting + commas every three digits in '123456', with min_width=8, gives + '0,123,456', even though that has length 9. + + """ + + sep = spec['thousands_sep'] + grouping = spec['grouping'] + + groups = [] + for l in _group_lengths(grouping): + if l <= 0: + raise ValueError("group length should be positive") + # max(..., 1) forces at least 1 digit to the left of a separator + l = min(max(len(digits), min_width, 1), l) + groups.append('0'*(l - len(digits)) + digits[-l:]) + digits = digits[:-l] + min_width -= l + if not digits and min_width <= 0: + break + min_width -= len(sep) + else: + l = max(len(digits), min_width, 1) + groups.append('0'*(l - len(digits)) + digits[-l:]) + return sep.join(reversed(groups)) + +def _format_sign(is_negative, spec): + """Determine sign character.""" + + if is_negative: + return '-' + elif spec['sign'] in ' +': + return spec['sign'] + else: + return '' + +def _format_number(is_negative, intpart, fracpart, exp, spec): + """Format a number, given the following data: + + is_negative: true if the number is negative, else false + intpart: string of digits that must appear before the decimal point + fracpart: string of digits that must come after the point + exp: exponent, as an integer + spec: dictionary resulting from parsing the format specifier + + This function uses the information in spec to: + insert separators (decimal separator and thousands separators) + format the sign + format the exponent + add trailing '%' for the '%' type + zero-pad if necessary + fill and align if necessary + """ + + sign = _format_sign(is_negative, spec) + + if fracpart or spec['alt']: + fracpart = spec['decimal_point'] + fracpart + + if exp != 0 or spec['type'] in 'eE': + echar = {'E': 'E', 'e': 'e', 'G': 'E', 'g': 'e'}[spec['type']] + fracpart += "{0}{1:+}".format(echar, exp) + if spec['type'] == '%': + fracpart += '%' + + if spec['zeropad']: + min_width = spec['minimumwidth'] - len(fracpart) - len(sign) + else: + min_width = 0 + intpart = _insert_thousands_sep(intpart, spec, min_width) + + return _format_align(sign, intpart+fracpart, spec) + + +##### Useful Constants (internal use only) ################################ + +# Reusable defaults +_Infinity = Decimal('Inf') +_NegativeInfinity = Decimal('-Inf') +_NaN = Decimal('NaN') +_Zero = Decimal(0) +_One = Decimal(1) +_NegativeOne = Decimal(-1) + +# _SignedInfinity[sign] is infinity w/ that sign +_SignedInfinity = (_Infinity, _NegativeInfinity) + +# Constants related to the hash implementation; hash(x) is based +# on the reduction of x modulo _PyHASH_MODULUS +_PyHASH_MODULUS = sys.hash_info.modulus +# hash values to use for positive and negative infinities, and nans +_PyHASH_INF = sys.hash_info.inf +_PyHASH_NAN = sys.hash_info.nan + +# _PyHASH_10INV is the inverse of 10 modulo the prime _PyHASH_MODULUS +_PyHASH_10INV = pow(10, _PyHASH_MODULUS - 2, _PyHASH_MODULUS) +del sys diff --git a/example/FixMutableDefaultArguments.hs b/example/FixMutableDefaultArguments.hs new file mode 100644 --- /dev/null +++ b/example/FixMutableDefaultArguments.hs @@ -0,0 +1,201 @@ +{-# language DataKinds #-} +module FixMutableDefaultArguments where + +import Control.Lens.Cons (_head) +import Control.Lens.Fold ((^..), (^?), filtered, folded, anyOf, has) +import Control.Lens.Getter ((^.)) +import Control.Lens.Review ((#)) +import Control.Lens.Setter ((.~), (%~)) +import Control.Monad (guard) +import Data.Function ((&)) +import Data.Semigroup ((<>)) + +import Language.Python.DSL +import Language.Python.Optics +import Language.Python.Syntax.Expr (Expr(..), _Exprs) + +{- + +I want to write a function that fixes the 'mutable default argument' pattern. + +It takes a function like this: + +def a(b=[]): + b.push(1) + +to this: + +def a(b=None): + if b is None: + b = [] + b.push(1) + +if the parameter is a 'mutable' thing. + +-} +fixMutableDefaultArguments :: Raw Statement -> Maybe (Raw Statement) +fixMutableDefaultArguments input = do + {- + + Firstly, this transformation only applies to function definitions. So we use the '_Fundef' + prism to pull out the function definition from a statement. + + -} + function <- input ^? _Fundef + + let + + {- + + I want the parameters of the function as a list. I can use the 'parameters' lens to get + at them. + + -} + paramsList = function ^.. parameters.folded + + {- + + I also want to know which parameters meet the 'mutable default argument' pattern. + + The '_KeywordParam' prism only matches keyword parameters, and '_kpExpr' is the field + accessor for the right hand side of an '=' in a keyword parameter. + + This expression gets 'all the keyword parameters that have mutable value on their RHS' + + -} + targetParams = paramsList ^.. folded._KeywordParam.filtered (isMutable._kpExpr) + + {- + + If the list of targetParams is empty, then we don't need to do the transformation + + -} + guard $ has _head targetParams + + let + {- + + Let's generate some 'if' statements + + -} + conditionalAssignments = + {- + + for each 'param' in our list of keyword parameters + + -} + (\param -> + let + {- + + let be the left hand side of the '=' + + -} + pName = var_ (param ^. kpName.identValue) + + {- + + let be the right hand side of the '=' + + -} + pValue = param ^. kpExpr + + in + {- + + output a new line, which says... + + -} + line_ $ + {- + + if is None: + = + + -} + if_ (pName `is_` none_) [ line_ (pName .= pValue) ]) <$> + targetParams + + {- + + For each parameter in the original parameter list, set the right hand sides of + the target parameters to 'None', but leave all of the others as they were. + + -} + newparams = + paramsList & traverse._KeywordParam.kpExpr.filtered isMutable .~ none_ + + pure $ + {- + + Return a new function defition statement + + -} + _Fundef # + {- + + that consists of the original function + + -} + (function & + {- + + with its problem parameters set to 'None' + + -} + parameters_ .~ newparams & + + {- + + and for each problem parameter, there is a corresponding if statement at the + start of the function definition + + -} + body_ %~ (conditionalAssignments <>)) + + where + {- + + This function decides whether or not an expression is mutable + + -} + isMutable :: Raw Expr -> Bool + isMutable Unit{} = False + isMutable None{} = False + isMutable Ellipsis{} = False + isMutable Lambda{} = False + isMutable Float{} = False + isMutable Imag{} = False + isMutable Int{} = False + isMutable Bool{} = False + isMutable String{} = False + + isMutable List{} = True + isMutable ListComp{} = True + isMutable Deref{} = True + isMutable Call{} = True + isMutable BinOp{} = True + isMutable UnOp{} = True + isMutable Not{} = True + isMutable DictComp{} = True + isMutable Dict{} = True + isMutable Ident{} = True + isMutable Yield{} = True + isMutable Await{} = True + isMutable YieldFrom{} = True + isMutable SetComp{} = True + isMutable Set{} = True + isMutable Subscript{} = True + isMutable Generator{} = True + + isMutable (Ternary _ _ _ a _ b) = isMutable a || isMutable b + isMutable (Parens _ _ a _) = isMutable a + isMutable (Tuple _ a _ as) = + {- + + Tuples contain many expressions, and are mutable if any of the sub-expressions + are mutable. The '_Exprs' traversal can get at all these sub-expressions + + -} + anyOf _Exprs isMutable a || + anyOf (folded.folded._Exprs) isMutable as diff --git a/example/Indentation.hs b/example/Indentation.hs new file mode 100644 --- /dev/null +++ b/example/Indentation.hs @@ -0,0 +1,27 @@ +{-# language DataKinds #-} +module Indentation where + +import Control.Lens.Setter ((.~)) +import Control.Lens.Plated (transform) +import GHC.Natural (Natural) + +import Language.Python.Optics +import Language.Python.Syntax.Statement (Statement) +import Language.Python.Syntax.Whitespace (Whitespace (Space, Tab)) + +{- + +These functions show how we can use Control.Lens.Plated to perform +whole-program transformations. + +They're illustrative only, because these functions aren't enough to re-indent +all Python code properly. The _Indent optic is limited in where it can reach. See +Language.Python.Optics.Indents for more info. + +-} + +indentSpaces :: Natural -> Statement '[] a -> Statement '[] a +indentSpaces n = transform (_Indent .~ replicate (fromIntegral n) Space) + +indentTabs :: Statement '[] a -> Statement '[] a +indentTabs = transform (_Indent .~ [Tab]) diff --git a/example/Main.hs b/example/Main.hs new file mode 100644 --- /dev/null +++ b/example/Main.hs @@ -0,0 +1,49 @@ +{-# language DataKinds #-} +module Main where + +import Control.Lens + +import Programs +import FixMutableDefaultArguments +import OptimizeTailRecursion +import Indentation +import Validation + +import Language.Python.Render (showModule) +import Language.Python.Syntax.Statement (_Statements) + +import qualified Data.Text.IO as StrictText + +section a = do + putStrLn "**********" + a + putStrLn "\n**********\n" + +main = do + section $ do + putStrLn "Before\n" + StrictText.putStrLn $ showModule everything + + section $ do + putStrLn "Spaced\n" + StrictText.putStrLn . + showModule $ + everything & _Statements %~ indentSpaces 2 + + section $ do + putStrLn "Tabbed\n" + StrictText.putStrLn . + showModule $ + everything & _Statements %~ indentTabs + + section $ do + putStrLn "Refactored\n" + StrictText.putStrLn . + showModule . + rewriteOn _Statements fixMutableDefaultArguments . + rewriteOn _Statements optimizeTailRecursion $ + everything + + section $ do + putStrLn "Validated\n" + doValidating diff --git a/example/OptimizeTailRecursion.hs b/example/OptimizeTailRecursion.hs new file mode 100644 --- /dev/null +++ b/example/OptimizeTailRecursion.hs @@ -0,0 +1,149 @@ +{-# language OverloadedStrings #-} +{-# language DataKinds #-} +{-# language BangPatterns #-} +module OptimizeTailRecursion where + +import Control.Applicative ((<|>)) +import Control.Lens.Cons (_last, _init) +import Control.Lens.Fold ((^..), (^?), (^?!), allOf, anyOf, folded, foldrOf) +import Control.Lens.Getter ((^.), to) +import Control.Lens.Plated (cosmos, transform, transformOn) +import Control.Lens.Prism (_Just) +import Control.Lens.Review ((#)) +import Control.Lens.Setter ((%~), (.~)) +import Control.Lens.Tuple (_2, _3) +import Data.Foldable (toList) +import Data.Function ((&)) +import Data.Semigroup ((<>)) + +import Language.Python.Optics +import Language.Python.DSL +import Language.Python.Syntax.Expr (Expr (..), _Exprs, argExpr, paramName) +import Language.Python.Syntax.Statement (CompoundStatement (..), Statement (..), SmallStatement (..), SimpleStatement (..), _Statements) + +optimizeTailRecursion :: Raw Statement -> Maybe (Raw Statement) +optimizeTailRecursion st = do + function <- st ^? _Fundef + let functionBody = function ^. body_ + bodyLast <- lastStatement functionBody + + let + functionName = function ^. fdName.identValue + bodyInit = functionBody ^?! _init + paramNames = function ^.. fdParameters.folded.paramName.identValue + + if not $ hasTC functionName bodyLast + then Nothing + else + Just $ + _Fundef # + (function & + body_ .~ + (zipWith + (\a b -> line_ (var_ (a <> "__tr") .= var_ b)) + paramNames + paramNames <> + + [ line_ ("__res__tr" .= none_) + , line_ . while_ true_ . + transformOn (traverse._Exprs) (renameIn paramNames "__tr") $ + bodyInit <> + looped functionName paramNames bodyLast + , line_ $ return_ "__res__tr" + ])) + + where + lastStatement :: [Raw Line] -> Maybe (Raw Statement) + lastStatement = go Nothing + where + go !res [] = res + go !res (a:as) = go (a ^? _Statements <|> res) as + + isTailCall :: String -> Raw Expr -> Bool + isTailCall name e + | anyOf (cosmos._Call.callFunction._Ident.identValue) (== name) e + = (e ^? _Call.callFunction._Ident.identValue) == Just name + | otherwise = False + + hasTC :: String -> Raw Statement -> Bool + hasTC name st = + case st of + CompoundStatement (If _ _ _ _ sts [] sts') -> + allOf _last (hasTC name) (sts ^.. _Statements) || + allOf _last (hasTC name) (sts' ^.. _Just._3._Statements) + SmallStatement _ (MkSmallStatement s ss _ _ _) -> + case last (s : fmap (^. _2) ss) of + Return _ _ (Just e) -> isTailCall name e + -- Return _ _ Nothing -> True + Expr _ e -> isTailCall name e + _ -> False + _ -> False + + renameIn :: [String] -> String -> Raw Expr -> Raw Expr + renameIn params suffix = + transform + (_Ident.identValue %~ (\a -> if a `elem` params then a <> suffix else a)) + + looped :: String -> [String] -> Raw Statement -> [Raw Line] + looped name params st + | Just ifSt <- st ^? _If + , hasTC name st = + let + ifBodyLines = toList $ ifSt ^. body_ + in + case ifSt ^? to getElse._Just.body_ of + Nothing -> + [ line_ $ + if_ (ifSt ^. ifCond) + ((ifBodyLines ^?! _init) <> + looped name params (ifBodyLines ^?! _last._Statements)) + ] + Just sts'' -> + [ line_ $ + if_ (ifSt ^. ifCond) + ((ifSt ^?! body_.to toList._init) <> + looped name params (ifBodyLines ^?! _last._Statements)) & + else_ + ((toList sts'' ^?! _init) <> + looped name params (toList sts'' ^?! _last._Statements)) + ] + | otherwise = + case st of + CompoundStatement{} -> [line_ st] + SmallStatement idnts (MkSmallStatement s ss sc cmt nl) -> + let + initExps = foldr (\_ _ -> init ss) [] ss + lastExp = foldrOf (folded._2) (\_ _ -> last ss ^. _2) s ss + newSts = + case initExps of + [] -> [] + first : rest -> + [ line_ $ + SmallStatement idnts + (MkSmallStatement (first ^. _2) rest sc cmt nl) + ] + in + case lastExp of + Return _ _ e -> + case e ^? _Just._Call of + Just call + | Just name' <- call ^? callFunction._Ident.identValue + , name' == name -> + newSts <> + fmap + (\a -> line_ (var_ (a <> "__tr__old") .= var_ (a <> "__tr"))) + params <> + zipWith + (\a b -> line_ (var_ (a <> "__tr") .= b)) + params + (transformOn + traverse + (renameIn params "__tr__old") + (call ^.. callArguments.folded.folded.argExpr)) + _ -> + newSts <> + maybe [] (\e' -> [ line_ ("__res__tr" .= e') ]) e <> + [ line_ break_ ] + Expr _ e + | isTailCall name e -> newSts <> [line_ pass_] + _ -> [line_ st] diff --git a/example/Programs.hs b/example/Programs.hs new file mode 100644 --- /dev/null +++ b/example/Programs.hs @@ -0,0 +1,208 @@ +{-# language OverloadedStrings #-} +{-# language FlexibleContexts #-} +module Programs where + +import Control.Lens.Getter ((^.)) +import Control.Lens.Iso (from) +import Data.Function ((&)) +import Data.List.NonEmpty (NonEmpty(..)) + +import Language.Python.DSL +import Language.Python.Syntax + + +-- | +-- @ +-- def append_to(element, to=[]): +-- to.append(element) +-- return to +-- @ +-- +-- Written without the DSL (not recommended!) +append_to :: Raw Statement +append_to = + CompoundStatement $ + Fundef () [] (Indents [] ()) + Nothing + (Space :| []) + "append_to" + [] + ( CommaSepMany (PositionalParam () "element" Nothing) (MkComma [Space]) $ + CommaSepOne (KeywordParam () "to" Nothing [] (List () [] Nothing [])) + ) + [] + Nothing + (SuiteMany () (MkColon []) Nothing LF $ + Block [] + ( SmallStatement + (Indents [replicate 4 Space ^. from indentWhitespaces] ()) + (MkSmallStatement + (Expr () $ + Call () + (Deref () (Ident "to") [] "append") + [] + (Just $ CommaSepOne1' (PositionalArg () (Ident "element")) Nothing) + []) + [] + Nothing + Nothing + (Just LF)) + ) + [ Right $ + SmallStatement + (Indents [replicate 4 Space ^. from indentWhitespaces] ()) + (MkSmallStatement + (Return () [Space] (Just $ Ident "to")) + [] + Nothing + Nothing + (Just LF)) + ]) + +-- | +-- @ +-- def append_to(element, to=[]): +-- to.append(element) +-- return to +-- @ +-- +-- Written with the DSL +append_to' :: Raw Fundef +append_to' = + def_ "append_to" [ p_ "element", k_ "to" (list_ []) ] + [ line_ $ call_ ("to" /> "append") [ "element" ] + , line_ $ return_ "to" + ] + +-- | +-- @ +-- def fact(n) +-- def go(n, acc) +-- if n == 0: +-- return acc +-- else: +-- go(n-1, n*acc) +-- return go(n, 1) +-- @ +fact_tr :: Raw Fundef +fact_tr = + def_ "fact" [p_ "n"] + [ line_ $ + def_ "go" [p_ "n", p_ "acc"] + [ line_ $ + if_ ("n" .== 0) + [line_ $ return_ (var_ "acc")] & + else_ + [line_ . return_ $ call_ "go" [p_ $ "n" .- 1, p_ $ "n" .* "acc"]] + ] + , line_ . return_ $ call_ "go" [p_ "n", p_ 1] + ] + +-- | +-- @ +-- def spin(): +-- spin() +-- @ +spin :: Raw Fundef +spin = def_ "spin" [] [line_ $ call_ "spin" []] + +-- | +-- @ +-- def yes() +-- print("yes") +-- yes() +-- @ +yes :: Raw Fundef +yes = + def_ "yes" [] + [ line_ $ call_ "print" [p_ $ str_ "yes"] + , line_ $ call_ "yes" [] + ] + +counter :: Raw ClassDef +counter = + class_ "Counter" [] + [ line_ $ + def_ "__init__" ["self"] + [line_ ("self" /> "x" .= 0)] + + , blank_ + + , line_ $ + def_ "incr" ["self"] + [line_ ("self" /> "x" .+= 1)] + + , blank_ + + , line_ $ + def_ "reset" ["self"] + [line_ ("self" /> "x" .= 0)] + + , blank_ + + , line_ $ + def_ "get" ["self"] + [line_ $ return_ ("self" /> "x")] + ] + +exceptions :: Raw Fundef +exceptions = + def_ "exceptions" [] + [ line_ $ + tryE_ [line_ pass_] & + except_ [line_ pass_] + , blank_ + + , line_ $ + tryE_ [line_ pass_] & + exceptAs_ (var_ "a" `as_` id_ "b") [line_ pass_] + , blank_ + + , line_ $ + tryE_ [line_ pass_] & + exceptAs_ (var_ "a" `as_` id_ "b") [line_ pass_] & + finally_ [line_ pass_] + , blank_ + + , line_ $ + tryE_ [line_ pass_] & + exceptAs_ (var_ "a" `as_` id_ "b") [line_ pass_] & + else_ [line_ pass_] & + finally_ [line_ pass_] + , blank_ + + , line_ $ tryF_ [line_ pass_] [line_ pass_] + , blank_ + + , line_ $ tryF_ [line_ pass_] & finally_ [line_ pass_] + , blank_ + + , line_ $ + tryF_ [line_ pass_] [line_ pass_] & + exceptAs_ (var_ "a" `as_` id_ "b") [line_ pass_] & + else_ [line_ pass_] + ] + +everything :: Raw Module +everything = + module_ + [ line_ append_to + , blank_ + + , line_ append_to' + , blank_ + + , line_ fact_tr + , blank_ + + , line_ spin + , blank_ + + , line_ yes + , blank_ + + , line_ counter + , blank_ + + , line_ exceptions + ] diff --git a/example/Validation.hs b/example/Validation.hs new file mode 100644 --- /dev/null +++ b/example/Validation.hs @@ -0,0 +1,63 @@ +{-# language OverloadedStrings #-} +{-# language TypeApplications #-} +module Validation where + +import qualified Data.Text.IO as Text + +import Language.Python.DSL +import Language.Python.Render +import Language.Python.Validate + +good_program :: Raw Module +good_program = + module_ + [ line_ $ + def_ "a" [p_ "b", p_ "c"] + [ line_ $ return_ ("b" .+ "c") + ] + ] + +bad_program :: Raw Module +bad_program = + module_ + [ line_ $ + def_ "a" [p_ "b", p_ "c"] + [ line_ $ return_ ("b" .+ "d") + ] + ] + +doValidating :: IO () +doValidating = do + putStrLn "Validating good program:\n" + + {- + + We can render unvalidated programs + + -} + Text.putStrLn $ showModule good_program + + {- + + Validate the module for indentation, syntax, and scope correctness + + We use the type application specify the error type so that we can Show the + result + + On success, we get back the same program we put in, but it has a slightly + different type to indicate that it has been validated + + -} + print $ validateModuleAll @(ValidationError ()) good_program + + putStrLn "" + + putStrLn "Validating bad program:\n" + Text.putStrLn $ showModule bad_program + + {- + + On failure, we get back a non-empty list of errors that occurred + + -} + print $ validateModuleAll @(ValidationError ()) bad_program diff --git a/hpython.cabal b/hpython.cabal new file mode 100644 --- /dev/null +++ b/hpython.cabal @@ -0,0 +1,209 @@ +-- Initial hpython.cabal generated by cabal init. For further +-- documentation, see http://haskell.org/cabal/users-guide/ + +name: hpython +version: 0.1 +synopsis: Python language tools +description: + `hpython` provides an abstract syntax tree for Python 3.5, along with a parser, + printer, and syntax checker. It also contains optics for working with the AST, + and a DSL for writing Python programs directly in Haskell. + . + For a high-level overview of the library, see the @Language.Python@ module. + . + For code examples, see the [examples directory on GitHub](https://github.com/qfpl/hpython/tree/master/example). + . + For general information about the project, see the [project readme](https://github.com/qfpl/hpython/blob/master/README.md). +license: BSD3 +license-file: LICENCE +author: Isaac Elliott +maintainer: isaace71295@gmail.com +copyright: Copyright (c) 2017-2018, Commonwealth Scientific and Industrial Research Organisation (CSIRO) ABN 41 687 119 230. +category: Language +build-type: Simple +extra-source-files: ChangeLog.md + benchmarks/pypy.py + test/files/ansible.py + test/files/asyncstatements.py + test/files/comments.py + test/files/decorators.py + test/files/dictcomp.py + test/files/django.py + test/files/django2.py + test/files/imaginary.py + test/files/indent_optics_in.py + test/files/indent_optics_out.py + test/files/indent_optics_in2.py + test/files/indent_optics_out2.py + test/files/joblib.py + test/files/joblib2.py + test/files/mypy.py + test/files/mypy2.py + test/files/numpy.py + test/files/numpy2.py + test/files/pandas.py + test/files/pandas2.py + test/files/pypy.py + test/files/pypy2.py + test/files/regex.py + test/files/requests.py + test/files/requests2.py + test/files/set.py + test/files/string.py + test/files/sqlalchemy.py + test/files/test.py + test/files/typeann.py + test/files/weird.py + test/files/weird2.py + +cabal-version: >=1.10 +tested-with: GHC == 8.0.2 + , GHC == 8.2.2 + , GHC == 8.4.4 + , GHC == 8.6.1 + + +source-repository head + type: git + location: git@github.com/qfpl/hpython.git + +flag development + default: False + manual: True + +library + exposed-modules: Data.Type.Set + , Data.Validate.Monadic + , Language.Python + , Language.Python.DSL + , Language.Python.Internal.Lexer + , Language.Python.Internal.Parse + , Language.Python.Internal.Render + , Language.Python.Internal.Render.Correction + , Language.Python.Internal.Token + , Language.Python.Internal.Syntax.IR + , Language.Python.Optics + , Language.Python.Optics.Indents + , Language.Python.Optics.Newlines + , Language.Python.Optics.Validated + , Language.Python.Parse + , Language.Python.Parse.Error + , Language.Python.Render + , Language.Python.Syntax + , Language.Python.Syntax.AugAssign + , Language.Python.Syntax.CommaSep + , Language.Python.Syntax.Comment + , Language.Python.Syntax.Expr + , Language.Python.Syntax.Ident + , Language.Python.Syntax.Import + , Language.Python.Syntax.Module + , Language.Python.Syntax.Operator.Binary + , Language.Python.Syntax.Operator.Unary + , Language.Python.Syntax.ModuleNames + , Language.Python.Syntax.Numbers + , Language.Python.Syntax.Punctuation + , Language.Python.Syntax.Raw + , Language.Python.Syntax.Statement + , Language.Python.Syntax.Strings + , Language.Python.Syntax.Types + , Language.Python.Syntax.Whitespace + , Language.Python.Validate + , Language.Python.Validate.Error + , Language.Python.Validate.Scope + , Language.Python.Validate.Scope.Error + , Language.Python.Validate.Syntax + , Language.Python.Validate.Syntax.Error + , Language.Python.Validate.Indentation + , Language.Python.Validate.Indentation.Error + build-depends: base >=4.9 && <5 + , bifunctors >= 0.1 && < 5.6 + , bytestring >= 0.10 && < 0.11 + , digit >=0.7 && < 0.8 + , dlist >=0.8 && <0.9 + , lens >= 4 && < 4.18 + , parsers >= 0.10 && < 0.13 + , megaparsec >=6.3 && <7 + , fingertree >=0.1 && <0.2 + , mtl >= 2.1 && < 2.3 + , containers >=0.5.7.1 && <0.7 + , deriving-compat >=0.4 && <0.6 + , semigroupoids >=5.2.2 && <5.4 + , text >=1.2 && <1.3 + , these >=0.7.4 && <0.8 + , validation >= 1 && < 1.1 + , parsers-megaparsec >=0.1 && <0.2 + hs-source-dirs: src + default-language: Haskell2010 + ghc-options: -Wall + -Wincomplete-patterns + -Wincomplete-record-updates + -Wunused-imports + -fno-warn-name-shadowing + if flag(development) + ghc-options: -Werror + +executable example + main-is: Main.hs + other-modules: Indentation + , FixMutableDefaultArguments + , OptimizeTailRecursion + , Programs + , Validation + hs-source-dirs: example + build-depends: base >=4.9 && <5, lens, hpython, text + default-language: Haskell2010 + ghc-options: -Wincomplete-patterns + -Wincomplete-record-updates + -Wunused-imports + if flag(development) + ghc-options: -Werror + + +benchmark bench + main-is: Main.hs + type: exitcode-stdio-1.0 + hs-source-dirs: benchmarks + build-depends: base >=4.9 && <5 + , hpython + , megaparsec >=6.3 && < 7 + , criterion >= 1 && < 1.6 + , deepseq + , text + , validation >= 1 && < 1.1 + default-language: Haskell2010 + ghc-options: -Wall + -Wincomplete-patterns + -Wincomplete-record-updates + -Wunused-imports + if flag(development) + ghc-options: -Werror + + +test-suite hpython-tests + main-is: Main.hs + type: exitcode-stdio-1.0 + other-modules: DSL + , Helpers + , LexerParser + , Optics + , Parser + , Roundtrip + , Scope + , Syntax + hs-source-dirs: test + build-depends: base >=4.9 && <5 + , filepath + , hpython + , hedgehog >= 0.5 && < 0.7 + , lens >= 4 && < 4.18 + , text >=1.2 && <1.3 + , megaparsec >=6.3 && < 7 + , validation >= 1 && < 1.1 + default-language: Haskell2010 + ghc-options: -Wall + -Wincomplete-patterns + -Wincomplete-record-updates + -Wunused-imports + -fno-warn-name-shadowing + if flag(development) + ghc-options: -Werror diff --git a/src/Data/Type/Set.hs b/src/Data/Type/Set.hs new file mode 100644 --- /dev/null +++ b/src/Data/Type/Set.hs @@ -0,0 +1,27 @@ +{-# language DataKinds, TypeFamilies, TypeOperators #-} +{-# language FlexibleInstances, MultiParamTypeClasses, PolyKinds #-} + +{-| +Module : Data.Type.Set +Copyright : (C) CSIRO 2017-2018 +License : BSD3 +Maintainer : Isaac Elliott +Stability : experimental +Portability : non-portable + +This module defines some helpful set-like functions for working with type-level lists. +-} + +module Data.Type.Set (Nub, Member) where + +-- | Remove adjacent equal elements from a type-level list +type family Nub t where + Nub '[] = '[] + Nub '[e] = '[e] + Nub (e ': e ': s) = Nub (e ': s) + Nub (e ': f ': s) = e ': Nub (f ': s) + +-- | Determine whether type @a@ is a member of type-level list @s@ +class Member a s where +instance {-# OVERLAPS #-} Member a (a ': s) where +instance {-# OVERLAPPABLE #-} Member a s => Member a (b ': s) where diff --git a/src/Data/Validate/Monadic.hs b/src/Data/Validate/Monadic.hs new file mode 100644 --- /dev/null +++ b/src/Data/Validate/Monadic.hs @@ -0,0 +1,60 @@ +{-# language GeneralizedNewtypeDeriving #-} +{-# language RankNTypes #-} + +{-| +Module : Data.Validate.Monadic +Copyright : (C) CSIRO 2017-2018 +License : BSD3 +Maintainer : Isaac Elliott +Stability : experimental +Portability : non-portable +-} + +module Data.Validate.Monadic + ( ValidateM (ValidateM, unValidateM) + , runValidateM + , bindVM + , liftVM0 + , liftVM1 + , errorVM + , errorVM1 + ) +where + +import Data.Functor.Compose (Compose(..)) +import Data.Semigroup (Semigroup) +import Data.Validation (Validation(..)) + +-- | The composition of 'Data.Validation.Validation' with another 'Applicative' functor. +-- +-- 'Data.Validation.Validation' is not a 'Monad', and 'ValidateM' is not a monad transformer. +-- It is equipped with a useful bind function, but that function does not have +-- the right type to make 'ValidateM' a 'Monad' (besides which it would break +-- the laws) +newtype ValidateM e m a = ValidateM { unValidateM :: Compose m (Validation e) a } + deriving (Functor, Applicative) + +-- | Unwrap a 'ValidateM' +runValidateM :: ValidateM e m a -> m (Validation e a) +runValidateM = getCompose . unValidateM + +-- | Bind into a 'ValidateM'. Note that the first parameter is @m a@, not @ValidateM e m a@. +bindVM :: Monad m => m a -> (a -> ValidateM e m b) -> ValidateM e m b +bindVM m f = ValidateM . Compose $ m >>= getCompose . unValidateM . f + +-- | Lift into a succeeding validation +liftVM0 :: (Functor m, Semigroup e) => m a -> ValidateM e m a +liftVM0 m = ValidateM . Compose $ pure <$> m + +-- | Run a natural transformation across 'ValidateM' to alter @m@ +liftVM1 :: (forall x. m x -> m x) -> ValidateM e m a -> ValidateM e m a +liftVM1 f = ValidateM . Compose . f . getCompose . unValidateM + +-- | Lift an error into 'ValidateM' +errorVM :: Applicative m => e -> ValidateM e m a +errorVM = ValidateM . Compose . pure . Failure + +-- | Lift an error in an 'Applicative' into 'ValidateM'. This is especially +-- useful if you're using list or 'Data.List.NonEmpty.NonEmpty' to collect errors. +errorVM1 :: (Applicative f, Applicative m) => e -> ValidateM (f e) m a +errorVM1 = errorVM . pure diff --git a/src/Language/Python.hs b/src/Language/Python.hs new file mode 100644 --- /dev/null +++ b/src/Language/Python.hs @@ -0,0 +1,40 @@ +{-| +Module : Language.Python +Copyright : (C) CSIRO 2017-2018 +License : BSD3 +Maintainer : Isaac Elliott +Stability : experimental +Portability : non-portable + +@hpython@ provides tools for working with Python source code. + +"Language.Python.DSL": A DSL for writing Python programs + +"Language.Python.Optics": Optics for working with Python syntax trees + +"Language.Python.Parse": Parse Python source into a syntax tree + +"Language.Python.Render": Pretty print Python syntax trees + +"Language.Python.Syntax": The data structures that represent Python programs, like 'Statement' and 'Expr' + +"Language.Python.Validate": Validate aspects of Python syntax trees, like indentation, syntax, or scope + +-} + +module Language.Python + ( module Language.Python.DSL + , module Language.Python.Optics + , module Language.Python.Parse + , module Language.Python.Render + , module Language.Python.Syntax + , module Language.Python.Validate + ) +where + +import Language.Python.DSL +import Language.Python.Optics +import Language.Python.Parse +import Language.Python.Render +import Language.Python.Syntax +import Language.Python.Validate diff --git a/src/Language/Python/DSL.hs b/src/Language/Python/DSL.hs new file mode 100644 --- /dev/null +++ b/src/Language/Python/DSL.hs @@ -0,0 +1,2293 @@ +{-| +Module : Language.Python.DSL +Copyright : (C) CSIRO 2017-2018 +License : BSD3 +Maintainer : Isaac Elliott +Stability : experimental +Portability : non-portable + +Passing @[]@ to a function which expects a @['Raw' 'Line']@ is the same as +passing @['line_' 'pass_']@ +-} + + +{-# language DataKinds #-} +{-# language FlexibleContexts #-} +{-# language MultiParamTypeClasses, FunctionalDependencies, FlexibleInstances #-} +{-# language LambdaCase #-} +{-# language RankNTypes #-} +{-# language RecordWildCards #-} +{-# language TemplateHaskell #-} +{-# language TypeApplications #-} +{-# language TypeFamilies #-} +{-# language UndecidableInstances #-} +module Language.Python.DSL + ( (&) + , Raw + , Module + , Statement + , Expr + -- * Modules + , module_ + -- * Lines of code + , blank_ + , AsLine(..) + , Line(..) + -- * Identifiers + , id_ + , Ident(..) + -- ** Lenses + , identAnn + , identValue + , identWhitespace + -- * Starred values + , StarSyntax(..) + , star_ + -- * Double-starred values + , DoubleStarSyntax(..) + -- * @as@ syntax + , As(..) + -- * @if@ syntax + , IfSyntax(..) + -- * @for@ syntax + , ForSyntax(..) + -- * @in@ syntax + , InSyntax(..), In(..), InList(..) + -- * @:@ syntax + , ColonSyntax(..) + -- * Comprehensions + , comp_ + , Guard(..) + -- * Parameters and arguments + -- ** Parameters + , Param(..) + , ParametersSyntax(..) + -- ** Arguments + , Arg(..) + , ArgumentsSyntax(..) + -- ** Positional + , PositionalSyntax(..) + , PositionalParam(..) + , _PositionalParam + -- *** Lenses + , ppAnn + , ppName + , ppType + -- ** Keyword + , KeywordSyntax(..) + , KeywordParam(..) + , _KeywordParam + -- *** Lenses + , kpAnn + , kpName + , kpType + , kpEquals + , kpExpr + -- * Decorators + , decorated_ + , DecoratorsSyntax(..) + -- * Statements + -- ** @async@ + , AsyncSyntax(..) + -- ** Block bodies + , BodySyntax(..) + -- ** Function definitions + , def_ + , Fundef(..) + , mkFundef + -- *** Lenses + , fdAnn + , fdDecorators + , fdIndents + , fdAsync + , fdDefSpaces + , fdName + , fdLeftParenSpaces + , fdParameters + , fdRightParenSpaces + , fdReturnType + , fdBody + -- ** Class definitions + , class_ + , ClassDef(..) + , mkClassDef + -- *** Lenses + , cdAnn + , cdDecorators + , cdIndents + , cdClass + , cdName + , cdArguments + , cdBody + -- ** Assignment + , chainEq + , (.=) + , (.+=) + , (.-=) + , (.*=) + , (.@=) + , (./=) + , (.%=) + , (.&=) + , (.|=) + , (.^=) + , (.<<=) + , (.>>=) + , (.**=) + , (.//=) + -- ** Exceptions + , tryE_ + , tryF_ + , ExceptSyntax(..) + , FinallySyntax(..) + , TryExcept(..) + , mkTryExcept + , TryFinally(..) + , mkTryFinally + , ExceptAs(..) + , AsExceptAs(..) + , Except(..) + , mkExcept + , Finally(..) + , mkFinally + -- *** Lenses + , teAnn + , teIndents + , teTry + , teBody + , teExcepts + , teElse + , teFinally + , exceptIndents + , exceptExcept + , exceptExceptAs + , exceptBody + , finallyIndents + , finallyFinally + , finallyBody + -- ** With statements + , with_ + , withItem_ + , With(..) + , mkWith + , AsWithItem(..) + , WithItem(..) + -- *** Lenses + , withAnn + , withIndents + , withAsync + , withWith + , withItems + , withBody + -- ** Flow control + -- *** 'Else' clauses + -- | 'If', 'While', 'For', and 'TryExcept' statements can have an 'Else' + -- component. + -- + -- 'else_' is considered to be a modifier on these structures. + -- + -- \-\-\- + -- + -- 'If' ... 'Else': + -- + -- >>> if_ false_ [line_ pass_] & else_ [line_ pass_] + -- if False: + -- pass + -- else: + -- pass + -- + -- \-\-\- + -- + -- 'While' ... 'Else': + -- + -- >>> while_ false_ [line_ pass_] & else_ [line_ pass_] + -- while False: + -- pass + -- else: + -- pass + -- + -- \-\-\- + -- + -- 'For' ... 'Else': + -- + -- >>> for_ (var_ "a" `in_` [var_ b]) [line_ pass_] & else_ [line_ pass_] + -- for a in b: + -- pass + -- else: + -- pass + -- + -- \-\-\- + -- + -- 'TryExcept' ... 'Else': + -- + -- >>> tryE_ [line_ pass_] & except_ [line_ pass_] & else_ [line_ pass_] + -- try: + -- pass + -- except: + -- pass + -- else: + -- pass + , else_ + , ElseSyntax(..) + -- *** Break + , break_ + -- *** For loops + -- | 'For' loops are built using 'for_' syntax: + -- + -- >>> for_ (var_ "a" `in_` [1, 2, 3]) [line_ (call_ "print" [var_ "a"])] + -- for a in 1, 2, 3: + -- print(a) + -- + -- See also: 'ForSyntax' + , forSt_ + , For(..) + , _For + , mkFor + -- *** If statements + , ifThen_ + , elif_ + , If(..) + , mkIf + , Elif(..) + , mkElif + , Else(..) + , mkElse + -- **** Lenses + , ifAnn + , ifIndents + , ifIf + , ifCond + , ifBody + , ifElifs + , ifElse + , elifIndents + , elifElif + , elifCond + , elifBody + , elseIndents + , elseElse + , elseBody + -- *** Pass + , pass_ + -- *** Return + , return_ + -- *** While loops + , while_ + , While(..) + , mkWhile + -- **** Lenses + , whileAnn + , whileIndents + , whileWhile + , whileCond + , whileBody + -- * Expressions + , expr_ + , var_ + -- ** @await@ + , await_ + -- ** @... if ... else ...@ + , ifThenElse_ + -- ** Generators + , gen_ + -- ** @yield@ + , yield_ + -- ** @yield from ...@ + , yieldFrom_ + -- ** Tuples + , tuple_ + , Tuple(..) + , AsTupleItem(..) + , TupleItem() + -- ** Function calls + , call_ + , Call(..) + , mkCall + -- *** Lenses + , callAnn + , callFunction + , callLeftParen + , callArguments + , callRightParen + -- ** Literals + -- *** @None@ + , none_ + , None(..) + , _None + -- **** Lenses + , noneAnn + , noneWhitespace + -- *** Strings + , str_ + , str'_ + , longStr_ + , longStr'_ + -- *** Integers + , int_ + -- *** Booleans + , true_ + , false_ + -- *** Ellipses + , ellipsis_ + -- ** Lists + , AsList(..) + , AsListItem(..) + , ListItem() + -- ** Dictionaries + , AsDict(..) + , DictItem() + -- ** Sets + , AsSet(..) + , AsSetItem(..) + , SetItem() + -- ** Lambdas + , lambda_ + -- ** Subscripting + , subs_ + -- *** Slicing + , sliceF_ + , sliceFS_ + , sliceT_ + , sliceTS_ + , sliceFT_ + , sliceFTS_ + , sliceS_ + , fullSlice_ + , slice_ + -- ** Dereferencing + , (/>) + -- ** Unary operators + , not_ + , neg_ + , pos_ + , compl_ + -- ** Binary operators + -- | Comparison, bitwise, and arithmetic operators have precedences that are + -- consistent with their Python counterparts. This meansPython expressions can + -- be translated to kellSyntax with minimal parentheses. + -- + -- Note: this doesn't apply to unary operators (because kellSyntax doesn't have + -- unary operators), or the boolean operations 'and_' and 'or_' (because we ran + -- out of precedence levels) + + -- *** Boolean operations + , or_ + , and_ + + -- *** Comparison operations + , is_ + , isNot_ + , notIn_ + , (.==) + , (.>) + , (.>=) + , (.<) + , (.<=) + , (.!=) + -- *** Bitwise operations + , (.|) + , (.^) + , (.&) + , (.<<) + , (.>>) + -- *** Arithmetic operations + , (.-) + , (.+) + , (.*) + , (.@) + , (./) + , (.//) + , (.%) + , (.**) + -- * Miscellaneous + , linesToBlock + , blockToLines + ) +where + +import Control.Applicative ((<|>)) +import Control.Lens.Fold ((^..), (^?), folded, lengthOf) +import Control.Lens.Getter ((^.), to) +import Control.Lens.Iso (from) +import Control.Lens.Lens (Lens') +import Control.Lens.Prism (_Right, _Just) +import Control.Lens.Review ((#)) +import Control.Lens.Setter ((.~), (<>~), (?~), (%~), Setter', set, over, mapped) +import Control.Lens.TH (makeWrapped) +import Control.Lens.Traversal (Traversal', traverseOf) +import Control.Lens.Tuple (_2) +import Control.Lens.Wrapped (_Wrapped) +import Data.Foldable (toList) +import Data.Function ((&)) +import Data.String (fromString) +import Data.List.NonEmpty (NonEmpty(..)) +import Data.Maybe (fromMaybe) +import Data.Semigroup ((<>)) + +import Language.Python.Optics +import Language.Python.Syntax.AugAssign +import Language.Python.Syntax.CommaSep +import Language.Python.Syntax.Expr +import Language.Python.Syntax.Ident +import Language.Python.Syntax.Module +import Language.Python.Syntax.Operator.Binary +import Language.Python.Syntax.Operator.Unary +import Language.Python.Syntax.Punctuation +import Language.Python.Syntax.Raw +import Language.Python.Syntax.Statement +import Language.Python.Syntax.Strings +import Language.Python.Syntax.Types +import Language.Python.Syntax.Whitespace + +-- | 'Ident' has an 'Data.String.IsString' instance, but when a type class dispatches on +-- an 'Ident' we will run into ambiguity if we try to use @OverloadedStrings@. In these +-- cases we can use 'id_' to provide the extra type information +id_ :: String -> Raw Ident +id_ = fromString + +-- | Create a 'Module' +-- +-- >>> module_ +-- >>> [ line_ $ def_ "a" [] [line_ pass_] +-- >>> , blank_ +-- >>> , line_ $ def_ "b" [] [line_ pass_] +-- >>> ] +-- def a(): +-- pass +-- +-- def b(): +-- pass +module_ :: [Raw Line] -> Raw Module +module_ [] = ModuleEmpty +module_ (a:as) = + case unLine a of + Left (bl, nl) -> ModuleBlank bl nl $ module_ as + Right a -> ModuleStatement a $ module_ as + +-- | One or more lines of Python code +newtype Line v a + = Line + { unLine :: Either (Blank a, Newline) (Statement v a) + } deriving (Eq, Show) +makeWrapped ''Line + +-- | Create a blank 'Line' +blank_ :: Raw Line +blank_ = Line $ Left (Blank () [] Nothing, LF) + +-- | Convert some data to a 'Line' +class AsLine s where + line_ :: Raw s -> Raw Line + +instance AsLine SmallStatement where + line_ ss = + Line . Right $ SmallStatement (Indents [] ()) ss + +instance AsLine SimpleStatement where + line_ ss = + Line . Right . SmallStatement (Indents [] ()) $ + MkSmallStatement ss [] Nothing Nothing (Just LF) + +instance AsLine CompoundStatement where + line_ = Line . Right . CompoundStatement + +instance AsLine ClassDef where + line_ = line_ @Statement . (_ClassDef #) + +instance AsLine Fundef where + line_ = line_ @Statement . (_Fundef #) + +instance AsLine If where + line_ = line_ @Statement . (_If #) + +instance AsLine While where + line_ = line_ @Statement . (_While #) + +instance AsLine With where + line_ = line_ @Statement . (_With #) + +instance AsLine Statement where + line_ = Line . Right + +instance AsLine Expr where + line_ e = line_ $ Expr (e ^. exprAnn) e + +instance HasExprs Line where + _Exprs f (Line a) = Line <$> (_Right._Exprs) f a + +instance HasStatements Line where + _Statements f (Line a) = Line <$> _Right f a + +class BodySyntax s where + -- | A faux-Lens that targets lines in the body of some statement-piece, but + -- does so \'around\' indentation. + -- + -- >>> def_ "a" [] [ line_ pass_, line_ pass_ ] + -- def a (): + -- pass + -- pass + -- + -- >>> def_ "a" [] [ line_ pass_, line_ pass_ ] ^. body_ + -- pass + -- pass + -- + -- >>> def_ "a" [] [ line_ pass_, line_ pass_ ] & body_ .~ [ line_ $ var_ "b" += 1 ] + -- def a(): + -- b += 1 + -- + -- >>> def_ "a" [] [ line_ pass_, line_ pass_ ] & body_ <>~ [ line_ $ var_ "b" += 1 ] + -- def a(): + -- pass + -- pass + -- b += 1 + -- + -- >>> def_ "a" [] [ line_ pass_, line_ pass_ ] & body_ .~ [] + -- def a(): + -- pass + -- + -- \-\-\- + -- + -- It's a fake 'Lens' because it violates some of the laws. The most obvious violation is + -- that setting the 'body_' to the empty list actually sets it to a singleton list containing + -- 'pass_'. (This is because blocks must contain one or more statements) + body_ :: Functor f => ([Raw Line] -> f [Raw Line]) -> Raw s -> f (Raw s) + body :: Lens' (Raw s) (Raw Suite) + +class ColonSyntax s t | s -> t, t -> s where + (.:) :: Raw s -> Raw Expr -> Raw t + +infix 0 .: + +-- | Constructing dictionary items +-- +-- @('.:') :: 'Raw' 'Expr' -> 'Raw' 'Expr' -> 'Raw' 'DictItem'@ +instance ColonSyntax Expr DictItem where + (.:) a = DictItem () a (MkColon [Space]) + +-- | Function parameter type annotations +-- +-- @('.:') :: 'Raw' 'Param' -> 'Raw' 'Expr' -> 'Raw' 'Param'@ +-- +-- 'star_' can be annotated using '.:', but it will have no effect on the output program, +-- as unnamed starred parameters cannot have type annotations. +-- +-- See 'def_' +instance ColonSyntax Param Param where + (.:) p t = p & paramType_ ?~ (MkColon [Space], t) + +-- | Positional parameters/arguments +-- +-- @ +-- p_ :: 'Raw' 'Expr' -> 'Raw' 'Arg' +-- @ +-- +-- @ +-- p_ :: 'Raw' 'Ident' -> 'Raw' 'Param' +-- @ +class PositionalSyntax p v | p -> v, v -> p where + p_ :: Raw v -> Raw p + +-- | See 'def_' +instance StarSyntax Ident Param where + s_ i = StarParam () [] i Nothing + +-- | See 'def_' +instance DoubleStarSyntax Ident Param where + ss_ i = DoubleStarParam () [] i Nothing + +class StarSyntax s t | t -> s where + s_ :: Raw s -> Raw t + +-- | See 'call_' +instance StarSyntax Expr Arg where + s_ = StarArg () [] + +-- | See 'call_' +instance DoubleStarSyntax Expr Arg where + ss_ = DoubleStarArg () [] + +-- | Keyword parameters/arguments +-- +-- @ +-- p_ :: 'Raw' 'Expr' -> 'Raw' 'Expr' -> 'Raw' 'Arg' +-- @ +-- +-- @ +-- p_ :: 'Raw' 'Ident' -> 'Raw' 'Expr' -> 'Raw' 'Param' +-- @ +class KeywordSyntax p where + k_ :: Raw Ident -> Raw Expr -> Raw p + +-- | Unnamed starred parameter +-- +-- >>> def_ "a" [ p_ "b", star_ ] [ line_ pass_ ] +-- def a(b, *): +-- pass +star_ :: Raw Param +star_ = UnnamedStarParam () [] + +class DoubleStarSyntax s t | t -> s where + ss_ :: Raw s -> Raw t + +-- | See 'dict_' +instance DoubleStarSyntax Expr DictItem where + ss_ = DictUnpack () [] + +-- | See 'def_' +instance PositionalSyntax Param Ident where + p_ i = PositionalParam () i Nothing + +-- | See 'def_' +instance KeywordSyntax Param where + k_ a = KeywordParam () a Nothing [] + +-- | See 'call_' +instance PositionalSyntax Arg Expr where; p_ = PositionalArg () + +-- | See 'call_' +instance KeywordSyntax Arg where; k_ a = KeywordArg () a [] + +class ParametersSyntax s where + -- | A faux-Lens that allows targeting 'Param's in-between existing formatting, + -- and adding appropriate formatting when extra parameters are introduced. + -- + -- >>> showStatement myStatement + -- "def a(b , c ):\n pass" + -- + -- >>> showStatement (myStatement & _Fundef.parameters_ .~ [p_ "d", p_ "e"] + -- "def a(d , e ):\n pass" + -- + -- >>> showStatement (myStatement & _Fundef.parameters_ .~ [p_ "d", p_ "e", p_ "f"] + -- "def a(d , e , f):\n pass" + -- + -- \-\-\- + -- + -- It's not a 'Lens' because repeated 'set's can drop trailing commas, violating + -- the 'Lens' laws. For example: + -- + -- >>> someFunction + -- def a(b, c,): + -- pass + -- + -- >>> set parameters_ [var_ "d", var_ "e"] someFunction + -- def a(d, e,): + -- pass + -- + -- >>> set parameters_ [] someFunction + -- def a(): + -- pass + -- + -- >>> set parameters_ [var_ "d", var_ "e"] (set parameters_ [] someFunction) + -- def a(d, e): + -- pass + parameters_ :: Functor f => ([Raw Param] -> f [Raw Param]) -> Raw s -> f (Raw s) + parameters :: Lens' (Raw s) (CommaSep (Raw Param)) + +class ArgumentsSyntax s where + setArguments :: [Raw Arg] -> Raw s -> Raw s + getArguments :: Raw s -> [Raw Arg] + +class DecoratorsSyntax s where + setDecorators :: [Raw Expr] -> Raw s -> Raw s + getDecorators :: Raw s -> [Raw Expr] + decorators :: Lens' (Raw s) [Raw Decorator] + +decorated_ :: DecoratorsSyntax s => [Raw Expr] -> Raw s -> Raw s +decorated_ = setDecorators + +exprsToDecorators :: Indents () -> [Raw Expr] -> [Raw Decorator] +exprsToDecorators is = fmap (\e -> Decorator () is (MkAt []) e Nothing LF []) + +instance DecoratorsSyntax Fundef where + decorators = fdDecorators + + setDecorators new code = + code + { _fdDecorators = exprsToDecorators (_fdIndents code) new + } + + getDecorators code = code ^.. fdDecorators.folded._Exprs + +blockToLines :: Raw Block -> [Raw Line] +blockToLines (Block x y z) = fmap (Line . Left) x <> (Line (Right y) : fmap Line z) + +mkBody_ + :: Traversal' (Raw s) (Indents ()) + -> Lens' (Raw s) (Raw Suite) + -> forall f. Functor f => ([Raw Line] -> f [Raw Line]) -> Raw s -> f (Raw s) +mkBody_ gIndents gBody f e = + (\ls -> e & gBody._Blocks .~ mkNewBlock allIndents ls id) <$> blLines' + where + -- | The default indent amount is the indentation level of the first statement + -- in a block. If the first statement has no indentation, it defaults to 4 + -- spaces. + defaultIndent = + fromMaybe + (Indents [replicate 4 Space ^. from indentWhitespaces] ()) + (e ^? gIndents) + + -- | The number of indentation chunks that precede the lines we're focusing on. + -- + -- It's one more than @defaultIndent@. + -- + -- For example, if we're looking at this code, which is inside some larger + -- context: + -- + -- @ + -- def a(): + -- pass + -- @ + -- + -- @defaultIndent@ refers to this part: + -- + -- @ + -- def a(): + -- ^^^^ + -- @ + -- + -- It's a single chunk. The code body has 2 (= one + 1) chunks: + -- + -- @ + -- def a(): + -- pass + -- ^^^^ + -- @ + -- + -- and + -- + -- @ + -- def a(): + -- pass + -- ^^ + -- @ + -- + -- So we will need to drop/take two chunks from the beginning of each line in + -- the body. + numChunks = lengthOf (indentsValue.folded) defaultIndent + 1 + + -- | The lines of the block + blLines = e ^.. gBody._Blocks.to blockToLines.folded + + -- | The lines of the block, with leading indentation chopped off appropriately + -- + -- For example: + -- + -- @ + -- def a(): + -- pass + -- pass + -- @ + -- + -- the unprocessed lines are: + -- + -- @ + -- pass + -- pass + -- @ + -- + -- so the processed lines should be: + -- + -- @ + -- pass + -- pass + -- @ + blLines' = + f $ + over + (mapped._Wrapped._Right._Indents.indentsValue) + (drop numChunks) + blLines + + -- | @defaultNewIndent@ is the amount of indentation that 'new' lines should get. + -- 'New' lines are only introduced when we set the @[Raw Line]@ to a list longer + -- than its original value. + -- + -- @allIndents@ is a list of indentation corresponding to the indents of the old + -- @[Raw Line]@ + defaultNewIndent :: Indents (); allIndents :: [Indents ()] + (defaultNewIndent, allIndents) = + foldr + (\a (di, as) -> + maybe + (di, di : as) + (\x -> (x, x : as)) + (a ^? to unLine._Right._Indents.to (indentsValue %~ take numChunks))) + (defaultIndent, []) + blLines + + -- | @mkNewBlock@ zips the old indentation with the new lines, but if the new + -- list of lines is longer than the old one then the extra lines at the end + -- are indented by @defaultNewIndent@ + mkNewBlock + :: [Indents ()] + -> [Raw Line] + -> (Raw Block -> Raw Block) + -> Raw Block + mkNewBlock [] [] k = + k $ Block [] (pass_ & _Indents %~ (defaultNewIndent <>)) [] + mkNewBlock (a:_) [] k = + k $ Block [] (pass_ & _Indents %~ (a <>)) [] + mkNewBlock [] [b] k = + k $ + either + (\w -> Block [w] (pass_ & _Indents %~ (defaultNewIndent <>)) []) + (\w -> Block [] (w & _Indents %~ (defaultNewIndent <>)) []) + (unLine b) + mkNewBlock (a:_) [b] k = + k $ + either + (\w -> Block [w] (pass_ & _Indents %~ (a <>)) []) + (\w -> Block [] (w & _Indents %~ (a <>)) []) + (unLine b) + mkNewBlock [] (b:bs) k = + mkNewBlock [] bs $ + \(Block x y z) -> + k $ + either + (\w -> Block (w:x) y z) + (\w -> + Block [] + (w & _Indents %~ (defaultNewIndent <>)) + ((Left <$> x) <> (Right y:z))) + (unLine b) + mkNewBlock (a:as) (b:bs) k = + mkNewBlock as bs $ + \(Block x y z) -> + k $ + either + (\w -> Block (w:x) y z) + (\w -> + Block [] + (w & _Indents %~ (a <>)) + ((Left <$> x) <> (Right y:z))) + (unLine b) + +instance BodySyntax Fundef where + body = fdBody + body_ = mkBody_ fdIndents fdBody + +instance ParametersSyntax Fundef where + parameters_ f e = flip (set fdParameters) e . go ps <$> ps' + where + ps = e ^. fdParameters + ps' = f $ toList ps + + go :: CommaSep (Raw Param) -> [Raw Param] -> CommaSep (Raw Param) + go CommaSepNone [] = CommaSepNone + go CommaSepNone (x:xs) = listToCommaSep $ x:xs + go CommaSepOne{} [] = CommaSepNone + go (CommaSepOne a) [x] = + CommaSepOne $ x & trailingWhitespace .~ (a ^. trailingWhitespace) + go (CommaSepOne a) (x:xs) = + listToCommaSep $ (x & trailingWhitespace .~ (a ^. trailingWhitespace)) :xs + go CommaSepMany{} [] = CommaSepNone + go (CommaSepMany a b c) (x:xs) = + CommaSepMany (x & trailingWhitespace .~ (a ^. trailingWhitespace)) b $ go c xs + + parameters = fdParameters + +-- | Create a minimal valid function definition +mkFundef :: Raw Ident -> [Raw Line] -> Raw Fundef +mkFundef name body = + MkFundef + { _fdAnn = () + , _fdDecorators = [] + , _fdIndents = Indents [] () + , _fdAsync = Nothing + , _fdDefSpaces = pure Space + , _fdName = name + , _fdLeftParenSpaces = [] + , _fdParameters = CommaSepNone + , _fdRightParenSpaces = [] + , _fdReturnType = Nothing + , _fdBody = SuiteMany () (MkColon []) Nothing LF $ linesToBlockIndented body + } + +-- | +-- >>> def_ "f" [p_ "x"] [line_ $ return_ "x"] +-- def f(x): +-- return x +-- +-- >>> def_ "f" [p_ "x", k_ "y" 2] [line_ $ return_ "x"] +-- def f(x, y=2): +-- return x +-- +-- >>> def_ "f" [p_ "x", k_ "y" 2, s_ "z"] [line_ $ return_ "x"] +-- def f(x, y=2, *z): +-- return x +-- +-- >>> def_ "f" [p_ "x", k_ "y" 2, s_ "z", ss_ "w"] [line_ $ return_ "x"] +-- def f(x, y=2, *z, **w) +-- return x +-- +-- >>> def_ "f" [p_ "x" .: "String"] [line_ $ return_ "x"] +-- def f(x: String): +-- return x +def_ :: Raw Ident -> [Raw Param] -> [Raw Line] -> Raw Fundef +def_ name params body = (mkFundef name body) { _fdParameters = listToCommaSep params } + +-- | Create a minimal valid 'Call' +mkCall :: Raw Expr -> Raw Call +mkCall e = + MkCall + { _callAnn = () + , _callFunction = e + , _callLeftParen = [] + , _callArguments = Nothing + , _callRightParen = [] + } + +instance ArgumentsSyntax Call where + setArguments args code = + code + { _callArguments = + case args of + [] -> Nothing + a:as -> Just $ (a, zip (repeat (MkComma [Space])) as, Nothing) ^. _CommaSep1' + } + + getArguments code = _callArguments code ^.. folded.folded + +-- | +-- >>> call_ "f" [p_ $ var_ "x"] +-- f(x) +-- +-- >>> call_ "f" [p_ $ var_ "x", k_ "y" 2] +-- f(x, y=2) +-- +-- >>> call_ "f" [p_ $ var_ "x", k_ "y" 2, s_ "z"] +-- f(x, y=2, *z) +-- +-- >>> call_ "f" [p_ $ var_ "x", k_ "y" 2, s_ "z", ss_ "w"] +-- f(x, y=2, *z, **w) +call_ :: Raw Expr -> [Raw Arg] -> Raw Expr +call_ expr args = + _Call # + (mkCall expr) + { _callArguments = + case args of + [] -> Nothing + a:as -> Just $ (a, zip (repeat (MkComma [Space])) as, Nothing) ^. _CommaSep1' + } + +-- | +-- >>> return_ (var_ "a") +-- return a +return_ :: Raw Expr -> Raw Statement +return_ e = + SmallStatement + (Indents [] ()) + (MkSmallStatement (Return () [Space] $ Just e) [] Nothing Nothing (Just LF)) + +-- | Turns an 'Expr' into a 'Statement' +-- +-- >>> expr_ (int_ 3) +-- 3 +expr_ :: Raw Expr -> Raw Statement +expr_ e = + SmallStatement + (Indents [] ()) + (MkSmallStatement (Expr () e) [] Nothing Nothing (Just LF)) + +-- | +-- >>> list_ [li_ $ var_ "a"] +-- [a] +-- +-- >>> list_ [s_ $ var_ "a"] +-- [*a] +-- +-- >>> list_ [li_ $ var_ "a", s_ $ var_ "b"] +-- [a, *b] +-- +-- >>> list_ $ comp_ (var_ "a") (for_ $ var_ "a" `in_` list_ [li_ $ int_ 1, li_ $ int_ 2, li_ $ int_ 3]) [if_ $ var_ "a" .== 2] +-- [a for a in [1, 2, 3] if a == 2] +class AsList s where + list_ :: s -> Raw Expr + +class AsListItem s where + -- | Create a 'ListItem' + li_ :: Raw s -> Raw ListItem + +instance AsListItem ListItem where + li_ = id + +instance AsListItem Expr where + li_ = ListItem () + +-- | See 'list_' +instance StarSyntax Expr ListItem where + s_ = ListUnpack () [] [] + +instance e ~ Raw ListItem => AsList [e] where + list_ es = List () [] (listToCommaSep1' es) [] + +instance e ~ Comprehension Expr => AsList (Raw e) where + list_ c = ListComp () [] c [] + +newtype Guard v a = MkGuard { unGuard :: Either (CompFor v a) (CompIf v a) } + +class ForSyntax a x | a -> x where + for_ :: Raw x -> a + +-- | +-- @'for_' :: 'Raw' 'In' -> 'Raw' 'CompFor'@ +-- +-- >>> comp_ (var_ "a") (for_ $ var_ "a" `in_` var_ "b") [] +-- a for a in b +instance ForSyntax (Raw CompFor) In where + for_ (MkIn a b) = CompFor () [Space] a [Space] b + +-- | +-- @'for_' :: 'Raw' 'In' -> 'Raw' 'Guard'@ +-- +-- >>> comp_ (var_ "a") (for_ $ var_ "a" `in_` var_ "b") [for_ $ var_ "c" `in_` var_ "d"] +-- a for a in b for c in d +instance ForSyntax (Raw Guard) In where + for_ (MkIn a b) = MkGuard . Left $ CompFor () [Space] a [Space] b + +class IfSyntax a where + if_ :: Raw Expr -> a + +-- | +-- @'if_' :: 'Raw' 'Expr' -> 'Raw' 'Guard'@ +-- +-- >>> comp_ (var_ "a") (for_ $ var_ "a" `in_` var_ "b") [if_ $ var_ "c" .== var_ "d"] +-- a for a in b if c == d +instance IfSyntax (Raw Guard) where + if_ = MkGuard . Right . CompIf () [Space] + +-- | +-- >>> set_ [] +-- set() +-- +-- >>> set_ [si_ $ var_ "a"] +-- {a} +-- +-- >>> set_ [s_ $ var_ "a"] +-- {*a} +-- +-- >>> set_ [si_ $ var_ "a", s_ $ var_ "b"] +-- {a, *b} +-- +-- >>> set_ $ comp_ (var_ "a") (for_ $ var_ "a" `in_` set_ [si_ $ int_ 1, si_ $ int_ 2, si_ $ int_ 3]) [if_ $ var_ "a" .== 2] +-- {a for a in [1, 2, 3] if a == 2} +class AsSet s where + set_ :: s -> Raw Expr + +class AsSetItem s where + -- | Create a 'SetItem' + si_ :: Raw s -> Raw SetItem + +instance AsSetItem SetItem where + si_ = id + +instance AsSetItem Expr where + si_ = SetItem () + +-- | See 'set_' +instance StarSyntax Expr SetItem where + s_ = SetUnpack () [] [] + +instance e ~ Raw SetItem => AsSet [e] where + set_ es = + case es of + [] -> call_ (var_ "set") [] + a:as -> Set () [] ((a, zip (repeat (MkComma [Space])) as, Nothing) ^. _CommaSep1') [] + +instance e ~ Comprehension SetItem => AsSet (Raw e) where + set_ c = SetComp () [] c [] + +comp_ :: Raw e -> Raw CompFor -> [Raw Guard] -> Raw (Comprehension e) +comp_ val cfor guards = + Comprehension () + val + (if null guards + then cfor + else cfor & trailingWhitespace .~ [Space]) + (unGuard <$> guards) + +-- | +-- >>> gen_ $ comp_ (var_ "a") (for_ $ var_ "a" `in_` list_ [li_ $ int_ 1, li_ $ int_ 2, li_ $ int_ 3]) [if_ $ var_ "a" .== 2] +-- (a for a in [1, 2, 3] if a == 2) +gen_ :: Raw (Comprehension Expr) -> Raw Expr +gen_ = Generator () + +-- | +-- >>> dict_ [var_ "a" .: 1] +-- {a: 1} +-- +-- >>> dict_ [ss_ $ var_ "a"] +-- {**a} +-- +-- >>> dict_ [var_ "a" .: 1, ss_ $ var_ "b"] +-- {a: 1, **b} +-- +-- >>> dict_ $ comp_ (var_ "a" .: 1) (for_ $ var_ "a" `in_` list_ [li_ $ int_ 1, li_ $ int_ 2, li_ $ int_ 3]) [if_ $ var_ "a" .== 2] +-- {a: 1 for a in [1, 2, 3] if a == 2} +class AsDict s where + dict_ :: s -> Raw Expr + +-- | +-- @'dict_' :: ['Raw' 'DictItem'] -> 'Raw' 'Expr'@ +instance e ~ Raw DictItem => AsDict [e] where + dict_ ds = + Dict () + [] + (case ds of + [] -> Nothing + a:as -> Just $ (a, zip (repeat (MkComma [Space])) as, Nothing) ^. _CommaSep1') + [] + +-- | +-- @'dict_' :: 'Raw' ('Comprehension' 'DictItem') -> 'Raw' 'Expr'@ +instance e ~ Comprehension DictItem => AsDict (Raw e) where + dict_ comp = DictComp () [] comp [] + +mkBinOp :: ([Whitespace] -> BinOp ()) -> Raw Expr -> Raw Expr -> Raw Expr +mkBinOp bop a = BinOp () (a & trailingWhitespace .~ [Space]) (bop [Space]) + +-- | @a is b@ +is_ :: Raw Expr -> Raw Expr -> Raw Expr +is_ = mkBinOp $ Is () +infixl 1 `is_` + +-- | +-- >>> var_ "a" `in_` var_ "b" +-- a in b +data In v a = MkIn (Expr v a) (Expr v a) + +-- | +-- >>> var_ "a" `in_` [var_ "b", var_ "c"] +-- a in b, c +data InList v a = MkInList (Expr v a) [Expr v a] + +class InSyntax a x | a -> x, x -> a where + in_ :: Raw Expr -> x -> Raw a +infixl 1 `in_` + +-- | @a and b@ +-- +-- Does not have a precedence +and_ :: Raw Expr -> Raw Expr -> Raw Expr +and_ a = BinOp () (a & trailingWhitespace .~ [Space]) (BoolAnd () [Space]) + +-- | @a or b@ +-- +-- Does not have a precedence +or_ :: Raw Expr -> Raw Expr -> Raw Expr +or_ a = BinOp () (a & trailingWhitespace .~ [Space]) (BoolOr () [Space]) + +-- | +-- >>> var_ "a" `in_` var_ "b" +-- a in b +instance InSyntax Expr (Raw Expr) where + in_ = mkBinOp $ In () + +-- | See 'for_' +instance e ~ Raw Expr => InSyntax InList [e] where + in_ = MkInList + +-- | @a not in b@ +notIn_ :: Raw Expr -> Raw Expr -> Raw Expr +notIn_ = mkBinOp $ NotIn () [Space] +infixl 1 `notIn_` + +-- | @a is not b@ +isNot_ :: Raw Expr -> Raw Expr -> Raw Expr +isNot_ = mkBinOp $ IsNot () [Space] +infixl 1 `isNot_` + +-- | @not a@ +not_ :: Raw Expr -> Raw Expr +not_ = Not () [Space] + +-- | @a == b@ +(.==) :: Raw Expr -> Raw Expr -> Raw Expr +(.==) = mkBinOp $ Eq () +infixl 1 .== + +-- | @a < b@ +(.<) :: Raw Expr -> Raw Expr -> Raw Expr +(.<) = mkBinOp $ Lt () +infixl 1 .< + +-- | @a <= b@ +(.<=) :: Raw Expr -> Raw Expr -> Raw Expr +(.<=) = mkBinOp $ LtEq () +infixl 1 .<= + +-- | @a > b@ +(.>) :: Raw Expr -> Raw Expr -> Raw Expr +(.>) = mkBinOp $ Gt () +infixl 1 .> + +-- | @a >= b@ +(.>=) :: Raw Expr -> Raw Expr -> Raw Expr +(.>=) = mkBinOp $ GtEq () +infixl 1 .>= + +-- | @a != b@ +(.!=) :: Raw Expr -> Raw Expr -> Raw Expr +(.!=) = mkBinOp $ NotEq () +infixl 1 .!= + +-- | @a | b@ +(.|) :: Raw Expr -> Raw Expr -> Raw Expr +(.|) = mkBinOp $ BitOr () +infixl 2 .| + +-- | @a ^ b@ +(.^) :: Raw Expr -> Raw Expr -> Raw Expr +(.^) = mkBinOp $ BitXor () +infixl 3 .^ + +-- | @a & b@ +(.&) :: Raw Expr -> Raw Expr -> Raw Expr +(.&) = mkBinOp $ BitAnd () +infixl 4 .& + +-- | @a << b@ +(.<<) :: Raw Expr -> Raw Expr -> Raw Expr +(.<<) = mkBinOp $ ShiftLeft () +infixl 5 .<< + +-- | @a >> b@ +(.>>) :: Raw Expr -> Raw Expr -> Raw Expr +(.>>) = mkBinOp $ ShiftRight () +infixl 5 .>> + +-- | @a + b@ +(.+) :: Raw Expr -> Raw Expr -> Raw Expr +(.+) = (+) +infixl 6 .+ + +-- | @a - b@ +(.-) :: Raw Expr -> Raw Expr -> Raw Expr +(.-) = (-) +infixl 6 .- + +-- | @a * b@ +(.*) :: Raw Expr -> Raw Expr -> Raw Expr +(.*) = (*) +infixl 7 .* + +-- | @a \@ b@ +(.@) :: Raw Expr -> Raw Expr -> Raw Expr +(.@) = mkBinOp $ At () +infixl 7 .@ + +-- | @a / b@ +(./) :: Raw Expr -> Raw Expr -> Raw Expr +(./) = mkBinOp $ Divide () +infixl 7 ./ + +-- | @a // b@ +(.//) :: Raw Expr -> Raw Expr -> Raw Expr +(.//) = mkBinOp $ FloorDivide () +infixl 7 .// + +-- | @a % b@ +(.%) :: Raw Expr -> Raw Expr -> Raw Expr +(.%) = mkBinOp $ Percent () +infixl 7 .% + +-- | @a ** b@ +(.**) :: Raw Expr -> Raw Expr -> Raw Expr +(.**) = mkBinOp $ Exp () +infixr 8 .** + +-- | +-- >>> var_ "a" /> var_ "b" +-- a.b +(/>) :: Raw Expr -> Raw Ident -> Raw Expr +(/>) a = Deref () a [] +infixl 9 /> + +-- | @-a@ +neg_ :: Raw Expr -> Raw Expr +neg_ = negate + +-- | @+a@ +pos_ :: Raw Expr -> Raw Expr +pos_ = UnOp () (Positive () []) + +-- | @~a@ +compl_ :: Raw Expr -> Raw Expr +compl_ = UnOp () (Complement () []) + +-- | Convert a list of 'Line's to a 'Block', giving them 4 spaces of indentation +linesToBlockIndented :: [Raw Line] -> Raw Block +linesToBlockIndented = over _Indents (indentIt $ replicate 4 Space) . linesToBlock + +-- | Convert a list of 'Line's to a 'Block', without indenting them +linesToBlock :: [Raw Line] -> Raw Block +linesToBlock = go + where + go [] = Block [] pass_ [] + go [y] = + case unLine y of + Left l -> Block [l] pass_ [] + Right st -> Block [] st [] + go (y:ys) = + case unLine y of + Left l -> + case go ys of + Block a b c -> Block (l:a) b c + Right st -> Block [] st (unLine <$> ys) + +instance BodySyntax While where + body = whileBody + body_ = mkBody_ whileIndents whileBody + +instance ElseSyntax While where + getElse = mkGetElse _whileIndents _whileElse + setElse = mkSetElse _whileIndents whileElse + +-- | Create a minimal valid 'While' +mkWhile :: Raw Expr -> [Raw Line] -> Raw While +mkWhile cond body = + MkWhile + { _whileAnn = () + , _whileIndents = Indents [] () + , _whileWhile = [Space] + , _whileCond = cond + , _whileBody = SuiteMany () (MkColon []) Nothing LF $ linesToBlockIndented body + , _whileElse = Nothing + } + +while_ :: Raw Expr -> [Raw Line] -> Raw While +while_ = mkWhile + +-- | Create a minimal valid 'If' +mkIf :: Raw Expr -> [Raw Line] -> Raw If +mkIf cond body = + MkIf + { _ifAnn = () + , _ifIndents = Indents [] () + , _ifIf = [Space] + , _ifCond = cond + , _ifBody = SuiteMany () (MkColon []) Nothing LF $ linesToBlockIndented body + , _ifElifs = [] + , _ifElse = Nothing + } + +instance BodySyntax Elif where + body = elifBody + body_ = mkBody_ elifIndents elifBody + +instance BodySyntax Else where + body = elseBody + body_ = mkBody_ elseIndents elseBody + +instance BodySyntax If where + body = ifBody + body_ = mkBody_ ifIndents ifBody + +-- | +-- @'if_' :: 'Raw' 'Expr' -> ['Raw' 'Line'] -> 'Raw' 'If'@ +-- +-- >>> if_ (var_ "a" .< 10) [var_ "a" .+= 1] +-- if a < 10: +-- a += 1 +instance (l ~ Raw Line, s ~ Raw If) => IfSyntax ([l] -> s) where + if_ = mkIf + +ifThen_ :: Raw Expr -> [Raw Line] -> Raw If +ifThen_ = mkIf + +var_ :: String -> Raw Expr +var_ s = Ident $ MkIdent () s [] + +-- | +-- >>> none_ +-- None +none_ :: Raw Expr +none_ = None () [] + +-- | @'Raw' 'Expr'@ has a 'Num' instance, but sometimes we need to name integers +-- explicitly +-- +-- >>> int_ 10 +-- 10 +int_ :: Integer -> Raw Expr +int_ = fromInteger + +-- | +-- >>> pass_ +-- pass +pass_ :: Raw Statement +pass_ = + SmallStatement + (Indents [] ()) + (MkSmallStatement (Pass () []) [] Nothing Nothing (Just LF)) + +-- | Create a minimal valid 'Elif' +mkElif :: Raw Expr -> [Raw Line] -> Raw Elif +mkElif cond body = + MkElif + { _elifIndents = Indents [] () + , _elifElif = [Space] + , _elifCond = cond + , _elifBody = SuiteMany () (MkColon []) Nothing LF $ linesToBlockIndented body + } + +elif_ :: Raw Expr -> [Raw Line] -> Raw If -> Raw If +elif_ cond body code = code & ifElifs <>~ [mkElif cond body] + +-- | Create a minimal valid 'Else' +mkElse :: [Raw Line] -> Raw Else +mkElse body = + MkElse + { _elseIndents = Indents [] () + , _elseElse = [] + , _elseBody = SuiteMany () (MkColon []) Nothing LF $ linesToBlockIndented body + } + +class ElseSyntax s where + getElse :: Raw s -> Maybe (Raw Else) + setElse :: [Whitespace] -> Maybe (Raw Else) -> Raw s -> Raw s + +else_ :: ElseSyntax s => [Raw Line] -> Raw s -> Raw s +else_ body = setElse (replicate 4 Space) $ Just (mkElse body) + +mkGetElse + :: (Raw s -> Indents ()) + -> (Raw s -> Maybe (Raw Else)) + -> Raw s + -> Maybe (Raw Else) +mkGetElse indentLevel elseField code = + fromMaybe + (error "malformed indentation in else block") + (traverseOf + (traverse._Indents) + (subtractStart (indentLevel code)) + (elseField code)) + +mkSetElse + :: (Raw s -> Indents ()) + -> Setter' (Raw s) (Maybe (Raw Else)) + -> [Whitespace] + -> Maybe (Raw Else) + -> Raw s + -> Raw s +mkSetElse indentLevel elseField _ new code = + code & + elseField .~ + fmap (elseIndents .~ indentLevel code) + (over + (traverse._Indents.indentsValue) + (indentLevel code ^. indentsValue <>) + new) + +instance ElseSyntax For where + getElse = mkGetElse _forIndents _forElse + setElse = mkSetElse _forIndents forElse + +instance ElseSyntax If where + getElse = mkGetElse _ifIndents _ifElse + setElse = mkSetElse _ifIndents ifElse + +instance ElseSyntax TryExcept where + getElse = mkGetElse _teIndents _teElse + setElse = mkSetElse _teIndents teElse + +break_ :: Raw Statement +break_ = + SmallStatement + (Indents [] ()) + (MkSmallStatement (Break () []) [] Nothing Nothing (Just LF)) + +-- | +-- >>> true_ +-- True +true_ :: Raw Expr +true_ = Bool () True [] + +-- | +-- >>> false_ +-- False +false_ :: Raw Expr +false_ = Bool () False [] + +-- | Double-quoted string +-- +-- >>> str_ "asdf" +-- "asdf" +str_ :: String -> Raw Expr +str_ s = + String () . pure $ + StringLiteral () Nothing ShortString DoubleQuote (Char_lit <$> s) [] + +-- | Single-quoted string +-- +-- >>> str_ "asdf" +-- 'asdf' +str'_ :: String -> Raw Expr +str'_ s = + String () . pure $ + StringLiteral () Nothing ShortString SingleQuote (Char_lit <$> s) [] + +-- | Long double-quoted string +-- +-- >>> longStr_ "asdf" +-- """asdf""" +longStr_ :: String -> Raw Expr +longStr_ s = + String () . pure $ + StringLiteral () Nothing LongString DoubleQuote (Char_lit <$> s) [] + +-- | Long single-quoted string +-- +-- >>> longStr'_ "asdf" +-- '''asdf''' +longStr'_ :: String -> Raw Expr +longStr'_ s = + String () . pure $ + StringLiteral () Nothing LongString SingleQuote (Char_lit <$> s) [] + +mkAugAssign :: AugAssignOp -> Raw Expr -> Raw Expr -> Raw Statement +mkAugAssign at a b = + SmallStatement + (Indents [] ()) + (MkSmallStatement + (AugAssign () (a & trailingWhitespace .~ [Space]) (MkAugAssign at () [Space]) b) + [] + Nothing + Nothing + (Just LF)) + +-- | Chained assignment +-- +-- >>> chainEq (var_ "a") [] +-- a +-- +-- >>> chainEq (var_ "a") [var_ "b", var_ "c"] +-- a = b = c +chainEq :: Raw Expr -> [Raw Expr] -> Raw Statement +chainEq t [] = expr_ t +chainEq t (a:as) = + SmallStatement + (Indents [] ()) + (MkSmallStatement + (Assign () t $ (,) (MkEquals [Space]) <$> (a :| as)) + [] + Nothing + Nothing + (Just LF)) + +-- | @a = b@ +(.=) :: Raw Expr -> Raw Expr -> Raw Statement +(.=) a b = + SmallStatement + (Indents [] ()) + (MkSmallStatement + (Assign () (a & trailingWhitespace .~ [Space]) $ pure (MkEquals [Space], b)) + [] + Nothing + Nothing + (Just LF)) +infix 0 .= + +-- | @a += b@ +(.+=) :: Raw Expr -> Raw Expr -> Raw Statement +(.+=) = mkAugAssign PlusEq +infix 0 .+= + +-- | @a -= b@ +(.-=) :: Raw Expr -> Raw Expr -> Raw Statement +(.-=) = mkAugAssign MinusEq +infix 0 .-= + +-- | @a *= b@ +(.*=) :: Raw Expr -> Raw Expr -> Raw Statement +(.*=) = mkAugAssign StarEq +infix 0 .*= + +-- | @a @= b@ +(.@=) :: Raw Expr -> Raw Expr -> Raw Statement +(.@=) = mkAugAssign AtEq +infix 0 .@= + +-- | @a /= b@ +(./=) :: Raw Expr -> Raw Expr -> Raw Statement +(./=) = mkAugAssign SlashEq +infix 0 ./= + +-- | @a %= b@ +(.%=) :: Raw Expr -> Raw Expr -> Raw Statement +(.%=) = mkAugAssign PercentEq +infix 0 .%= + +-- | @a &= b@ +(.&=) :: Raw Expr -> Raw Expr -> Raw Statement +(.&=) = mkAugAssign AmpersandEq +infix 0 .&= + +-- | @a |= b@ +(.|=) :: Raw Expr -> Raw Expr -> Raw Statement +(.|=) = mkAugAssign PipeEq +infix 0 .|= + +-- | @a ^= b@ +(.^=) :: Raw Expr -> Raw Expr -> Raw Statement +(.^=) = mkAugAssign CaretEq +infix 0 .^= + +-- | @a <<= b@ +(.<<=) :: Raw Expr -> Raw Expr -> Raw Statement +(.<<=) = mkAugAssign ShiftLeftEq +infix 0 .<<= + +-- | @a >>= b@ +(.>>=) :: Raw Expr -> Raw Expr -> Raw Statement +(.>>=) = mkAugAssign ShiftRightEq +infix 0 .>>= + +-- | @a **= b@ +(.**=) :: Raw Expr -> Raw Expr -> Raw Statement +(.**=) = mkAugAssign DoubleStarEq +infix 0 .**= + +-- | @a //= b@ +(.//=) :: Raw Expr -> Raw Expr -> Raw Statement +(.//=) = mkAugAssign DoubleSlashEq +infix 0 .//= + +mkFor :: Raw Expr -> [Raw Expr] -> [Raw Line] -> Raw For +mkFor binder collection body = + MkFor + { _forAnn = () + , _forIndents = Indents [] () + , _forAsync = Nothing + , _forFor = [Space] + , _forBinder = binder & trailingWhitespace .~ [Space] + , _forIn = [Space] + , _forCollection = + fromMaybe + (CommaSepOne1' (Unit () [] []) Nothing) + (listToCommaSep1' collection) + , _forBody = SuiteMany () (MkColon []) Nothing LF $ linesToBlockIndented body + , _forElse = Nothing + } + +-- | +-- @'for_' :: 'Raw' 'InList' -> ['Raw' 'Line'] -> 'Raw' 'Statement'@ +-- +-- >>> for_ (var_ "a" `in_` [var_ "b"]) [line_ (var_ "c" .+= var_ "a")] +-- for a in b: +-- c += a +instance (l ~ [Raw Line], s ~ Raw For) => ForSyntax (l -> s) InList where + for_ (MkInList a b) = mkFor a b + +forSt_ :: Raw Expr -> [Raw Expr] -> [Raw Line] -> Raw For +forSt_ = mkFor + +instance BodySyntax For where + body = forBody + body_ = mkBody_ forIndents forBody + +instance AsLine For where + line_ = line_ @Statement . (_For #) + +class AsyncSyntax s where + async_ :: Raw s -> Raw s + +instance AsyncSyntax Fundef where + async_ = fdAsync ?~ pure Space + +instance AsyncSyntax For where + async_ = forAsync ?~ pure Space + +-- | Create a minimal valid 'Finally' +mkFinally :: [Raw Line] -> Raw Finally +mkFinally body = + MkFinally + { _finallyIndents = Indents [] () + , _finallyFinally = [] + , _finallyBody = SuiteMany () (MkColon []) Nothing LF $ linesToBlockIndented body + } + +-- | Create a minimal valid 'Except' +mkExcept :: [Raw Line] -> Raw Except +mkExcept body = + MkExcept + { _exceptIndents = Indents [] () + , _exceptExcept = [] + , _exceptExceptAs = Nothing + , _exceptBody = SuiteMany () (MkColon []) Nothing LF $ linesToBlockIndented body + } + +-- | Create a minimal valid 'TryExcept' +mkTryExcept :: [Raw Line] -> Raw Except -> Raw TryExcept +mkTryExcept body except = + MkTryExcept + { _teAnn = () + , _teIndents = Indents [] () + , _teTry = [Space] + , _teBody = SuiteMany () (MkColon []) Nothing LF $ linesToBlockIndented body + , _teExcepts = pure except + , _teElse = Nothing + , _teFinally = Nothing + } + +-- | Create a minimal valid 'TryFinally' +mkTryFinally :: [Raw Line] -> [Raw Line] -> Raw TryFinally +mkTryFinally body fBody = + MkTryFinally + { _tfAnn = () + , _tfIndents = Indents [] () + , _tfTry = [Space] + , _tfBody = SuiteMany () (MkColon []) Nothing LF $ linesToBlockIndented body + , _tfFinally = mkFinally fBody + } + +class FinallySyntax s t | s -> t where + finally_ :: [Raw Line] -> s -> Raw t + +-- | +-- >>> tryE_ [line_ pass_] & finally_ [line_ pass_] +-- try: +-- pass +-- finally: +-- pass +-- +-- >>> tryF_ [line_ pass_] [line_ (a .+= 1)] & finally_ [line_ pass_] +-- try: +-- pass +-- finally: +-- pass +-- +-- >>> tryF_ [line_ pass_] & finally_ [line_ pass_] +-- try: +-- pass +-- finally: +-- pass +instance FinallySyntax (Raw TryExcept) TryExcept where + finally_ body = teFinally ?~ mkFinally body + +instance FinallySyntax (Raw TryFinally) TryFinally where + finally_ body = tfFinally .~ mkFinally body + +instance (a ~ [Raw Line], b ~ Raw TryFinally) => FinallySyntax (a -> b) TryFinally where + finally_ body f = f body + +instance BodySyntax TryExcept where + body = teBody + body_ = mkBody_ teIndents teBody + +-- | @try ... except@ with optional @else@ and optional @finally@ +-- +-- >>> tryE_ [line_ pass_] [line_ ("a" .+= 1)] +-- try: +-- pass +-- except +-- a += 1 +tryE_ :: [Raw Line] -> Raw Except -> Raw TryExcept +tryE_ = mkTryExcept + +instance BodySyntax TryFinally where + body = tfBody + body_ = mkBody_ tfIndents tfBody + +-- | +-- @try ... finally@ +-- +-- >>> tryF_ [line_ pass_] [line_ ("a" .+= 1)] +-- try: +-- pass +-- finally: +-- a += 1 +tryF_ :: [Raw Line] -> [Raw Line] -> Raw TryFinally +tryF_ = mkTryFinally + +class AsExceptAs s where + toExceptAs :: Raw s -> Raw ExceptAs + +instance AsExceptAs ExceptAs where + toExceptAs = id + +instance AsExceptAs Expr where + toExceptAs e = ExceptAs () e Nothing + +class ExceptSyntax s where + except_ :: [Raw Line] -> s -> Raw TryExcept + -- | You can use 'exceptAs_' without a binder: + -- + -- @'exceptAs_' :: 'Raw' 'Expr' -> ['Raw' 'Line'] -> 'Raw' s -> 'Raw' 'TryExcept'@ + -- + -- @ + -- 'exceptAs_' ('var_' \"Exception\") body + -- @ + -- + -- or with a binder: + -- + -- @'exceptAs_' :: 'Raw' 'ExceptAs' -> ['Raw' 'Line'] -> 'Raw' s -> 'Raw' 'TryExcept'@ + -- + -- @ + -- 'exceptAs_' ('var_' \"Exception\" \``as_`\` 'id_' "a") body + -- @ + exceptAs_ :: AsExceptAs e => Raw e -> [Raw Line] -> s -> Raw TryExcept + +-- | +-- @'except_' :: ['Raw' 'Line'] -> ('Raw' 'Except' -> 'Raw' 'TryExcept') -> 'Raw' 'TryExcept'@ +-- +-- @'exceptAs_' :: 'AsExceptAs' e => 'Raw' e -> ['Raw' 'Line'] -> ('Raw' 'Except' -> 'Raw' 'TryExcept') -> 'Raw' 'TryExcept'@ +-- +-- >>> tryE_ [var_ "a" .= 2] & except_ [var_ "a" .= 3] +-- try: +-- a = 2 +-- except: +-- a = 3 +-- +-- >>> tryE_ [var_ "a" .= 2] & exceptAs_ (var_ "Exception" `as_` id_ "b") [var_ "a" .= 3] +-- try: +-- a = 2 +-- except Exception as b: +-- a = 3 +instance (e ~ Raw Except, s ~ Raw TryExcept) => ExceptSyntax (e -> s) where + except_ body f = f $ mkExcept body + exceptAs_ ea body f = f $ mkExcept body & exceptExceptAs ?~ toExceptAs ea + +-- | +-- @'except_' :: ['Raw' 'Line'] -> 'Raw' 'TryExcept' -> 'Raw' 'TryExcept'@ +-- +-- @'exceptAs_' :: AsExceptAs => 'Raw' e -> ['Raw' 'Line'] -> 'Raw' 'TryExcept' -> 'Raw' 'TryExcept'@ +-- +-- @ +-- (someTryStatement :: 'Raw' 'TryExcept') '&' +-- 'except_' ['line_' 'pass_'] +-- @ +-- +-- @ +-- (someTryStatement :: 'Raw' 'TryExcept') '&' +-- 'exceptAs_' ('var_' \"Exception\" \``as_`\` 'id_' "b") ['line_' 'pass_'] +-- @ +instance ExceptSyntax (Raw TryExcept) where + except_ body = teExcepts %~ (<> pure (mkExcept body)) + exceptAs_ ea body = + teExcepts %~ (<> pure (mkExcept body & exceptExceptAs ?~ toExceptAs ea)) + +-- | +-- @'except_' :: ['Raw' 'Line'] -> 'Raw' 'TryFinally' -> 'Raw' 'TryExcept'@ +-- +-- @'exceptAs_' :: AsExceptAs => 'Raw' e -> ['Raw' 'Line'] -> 'Raw' 'TryFinally' -> 'Raw' 'TryExcept'@ +-- +-- @ +-- (someTryStatement :: 'Raw' 'TryFinally') '&' +-- 'except_' ['line_' 'pass_'] +-- @ +-- +-- @ +-- (someTryStatement :: 'Raw' 'TryFinally') '&' +-- 'exceptAs_' ('var_' \"Exception\" \``as_`\` 'id_' "b") ['line_' 'pass_'] +-- @ +instance ExceptSyntax (Raw TryFinally) where + except_ body MkTryFinally{..} = + MkTryExcept + { _teAnn = _tfAnn + , _teIndents = _tfIndents + , _teTry = _tfTry + , _teBody = _tfBody + , _teExcepts = pure $ mkExcept body + , _teElse = Nothing + , _teFinally = Just _tfFinally + } + + exceptAs_ ea body MkTryFinally{..} = + MkTryExcept + { _teAnn = _tfAnn + , _teIndents = _tfIndents + , _teTry = _tfTry + , _teBody = _tfBody + , _teExcepts = pure $ mkExcept body & exceptExceptAs ?~ toExceptAs ea + , _teElse = Nothing + , _teFinally = Just _tfFinally + } + +instance AsLine TryExcept where + line_ = line_ @Statement . (_TryExcept #) + +instance AsLine TryFinally where + line_ = line_ @Statement . (_TryFinally #) + +class As s t u | s t -> u, u -> s t where + as_ :: Raw s -> Raw t -> Raw u + +-- | See 'exceptAs_' +instance As Expr Ident ExceptAs where + as_ e name = ExceptAs () e $ Just ([Space], name) + +-- | +-- >>> class_ "A" [] [line_ pass_] +-- class A: +-- pass +class_ :: Raw Ident -> [Raw Arg] -> [Raw Line] -> Raw ClassDef +class_ name args body = + (mkClassDef name body) { + _cdArguments = + case args of + [] -> Nothing + a:as -> Just ([], Just $ (a, zip (repeat (MkComma [Space])) as, Nothing) ^. _CommaSep1', []) + } + +-- | Create a minimal 'ClassDef' +mkClassDef :: Raw Ident -> [Raw Line] -> Raw ClassDef +mkClassDef name body = + MkClassDef + { _cdAnn = () + , _cdDecorators = [] + , _cdIndents = Indents [] () + , _cdClass = Space :| [] + , _cdName = name + , _cdArguments = Nothing + , _cdBody = SuiteMany () (MkColon []) Nothing LF $ linesToBlockIndented body + } + +instance BodySyntax ClassDef where + body = cdBody + body_ = mkBody_ cdIndents cdBody + +instance DecoratorsSyntax ClassDef where + decorators = cdDecorators + + setDecorators new code = + code + { _cdDecorators = exprsToDecorators (_cdIndents code) new + } + + getDecorators code = code ^.. cdDecorators.folded._Exprs + +instance ArgumentsSyntax ClassDef where + setArguments args code = + code + { _cdArguments = + case args of + [] -> Nothing + a:as -> Just ([], Just $ (a, zip (repeat (MkComma [Space])) as, Nothing) ^. _CommaSep1', []) + } + + getArguments code = _cdArguments code ^.. folded._2.folded.folded + +-- | Create a minimal valid 'With' +mkWith :: NonEmpty (Raw WithItem) -> [Raw Line] -> Raw With +mkWith items body = + MkWith + { _withAnn = () + , _withIndents = Indents [] () + , _withAsync = Nothing + , _withWith = [Space] + , _withItems = listToCommaSep1 items + , _withBody = SuiteMany () (MkColon []) Nothing LF $ linesToBlockIndented body + } + +-- | +-- +-- @ +-- with_ :: 'NonEmpty' ('Raw' 'Expr') -> ['Raw' 'Line'] -> 'Raw' 'Statement' +-- with_ :: 'NonEmpty' ('Raw' 'WithItem') -> ['Raw' 'Line'] -> 'Raw' 'Statement' +-- @ +-- +-- >>> with_ [var_ "a"] [line_ $ var_ "b"] +-- with a: +-- b +-- +-- >>> with_ [var_ "a" `as_` id_ "name"] [line_ $ var_ "b"] +-- with a as name: +-- b +-- +-- >>> with_ [withItem_ e Nothing] [line_ $ var_ "b"] +-- with a: +-- b +with_ :: AsWithItem e => NonEmpty (Raw e) -> [Raw Line] -> Raw With +with_ items = mkWith (toWithItem <$> items) + +withItem_ :: Raw Expr -> Maybe (Raw Expr) -> Raw WithItem +withItem_ a b = WithItem () a ((,) [Space] <$> b) + +-- | See 'with_' +instance As Expr Expr WithItem where + as_ a b = WithItem () a $ Just ([Space], b) + +class AsWithItem s where + toWithItem :: Raw s -> Raw WithItem + +instance AsWithItem Expr where + toWithItem e = WithItem () e Nothing + +instance AsWithItem WithItem where + toWithItem = id + +instance BodySyntax With where + body = withBody + body_ = mkBody_ withIndents withBody + +instance AsyncSyntax With where + async_ = withAsync ?~ pure Space + +-- | +-- >>> ellipsis_ +-- ... +ellipsis_ :: Raw Expr +ellipsis_ = Ellipsis () [] + +class AsTupleItem e where + -- | Create a 'TupleItem' + ti_ :: Raw e -> Raw TupleItem + +-- | See 'tuple_' +instance StarSyntax Expr TupleItem where + s_ = TupleUnpack () [] [] + +instance AsTupleItem Expr where + ti_ = TupleItem () + +instance AsTupleItem TupleItem where + ti_ = id + +-- | +-- >>> tuple_ [] +-- () +-- +-- >>> tuple_ [ti_ $ var_ "a"] +-- a, +-- +-- >>> tuple_ [s_ $ var_ "a"] +-- (*a), +-- +-- >>> tuple_ [ti_ $ var_ "a", ti_ $ var_ "b"] +-- a, b +-- +-- >>> tuple_ [ti_ $ var_ "a", s_ $ var_ "b"] +-- a, *b +tuple_ :: [Raw TupleItem] -> Raw Expr +tuple_ [] = Unit () [] [] +tuple_ (a:as) = + case as of + [] -> Tuple () (ti_ a) (MkComma []) Nothing + b:bs -> + Tuple () a (MkComma [Space]) . Just $ + (b, zip (repeat (MkComma [Space])) bs, Nothing) ^. _CommaSep1' + +-- | +-- >>> await (var_ "a") +-- await a +await_ :: Raw Expr -> Raw Expr +await_ = Await () [Space] + +-- | +-- >>> ifThenElse_ (var_ "a") (var_ "b") (var_ "c") +-- a if b else c +ifThenElse_ :: Raw Expr -> Raw Expr -> Raw Expr -> Raw Expr +ifThenElse_ a b = Ternary () a [Space] b [Space] + +-- | +-- >>> lambda_ [p_ "x"] "x" +-- lambda x: x +-- +-- >>> lambda_ [p_ "x", k_ "y" 2] ("x" .+ "y") +-- lambda x, y=2: x + y +-- +-- >>> lambda_ [p_ "x", k_ "y" 2, s_ "z"] "a" +-- lambda x, y=2, *z: a +-- +-- >>> lambda_ [p_ "x", k_ "y" 2, s_ "z", ss_ "w"] "a" +-- lambda x, y=2, *z, **w: a +lambda_ :: [Raw Param] -> Raw Expr -> Raw Expr +lambda_ params = + Lambda () + (if null params then [] else [Space]) + (listToCommaSep params) + (MkColon [Space]) + +-- | +-- >>> yield_ [] +-- yield +-- +-- >>> yield_ [var_ "a"] +-- yield a +-- +-- >>> yield_ [var_ "a", var_ "b"] +-- yield a, b +yield_ :: [Raw Expr] -> Raw Expr +yield_ as = Yield () (foldr (\_ _ -> [Space]) [] as) (listToCommaSep as) + +-- | +-- >>> yieldFrom_ (var_ "a") +-- yield from a +yieldFrom_ :: Raw Expr -> Raw Expr +yieldFrom_ = YieldFrom () [Space] [Space] + +-- | The slice with no bounds +-- +-- >>> subs_ (var_ "a") fullSlice_ +-- a[:] +-- +-- >>> fullSlice_ +-- slice(None, None, None) +fullSlice_ :: Raw Expr +fullSlice_ = slice_ Nothing Nothing Nothing + +-- | Slice with *step* @x@ +-- +-- >>> subs_ (var_ "a") (sliceS_ $ int_ (-1)) +-- a[::-1] +-- +-- >>> sliceS_ $ int_ (-1) +-- slice(None, None, -1) +sliceS_ :: Raw Expr -> Raw Expr +sliceS_ x = slice_ Nothing Nothing (Just x) + +-- | Slice *from* @x@ +-- +-- >>> subs_ (var_ "a") (sliceF_ $ int_ 0) +-- a[1:] +-- +-- >>> sliceF_ $ int_ 0 +-- slice(1, None, None) +sliceF_ :: Raw Expr -> Raw Expr +sliceF_ x = slice_ (Just x) Nothing Nothing + +-- | Slice *from* @x@, with *step* @y@ +-- +-- >>> subs_ (var_ "a") (sliceFS_ (int_ 0) (int_ 2)) +-- a[1::2] +-- +-- >>> sliceFS_ (int_ 0) (int_ 2) +-- slice(1, None, 2) +sliceFS_ :: Raw Expr -> Raw Expr -> Raw Expr +sliceFS_ x y = slice_ (Just x) Nothing (Just y) + +-- | Slice /To/ @x@ +-- +-- >>> subs_ (var_ "a") (sliceT_ $ int_ 10) +-- a[:10] +-- +-- >>> sliceT_ $ int_ 10 +-- slice(None, 10, None) +sliceT_ :: Raw Expr -> Raw Expr +sliceT_ x = slice_ Nothing (Just x) Nothing + +-- | Slice /To/ @x@, with /Step/ @y@ +-- +-- >>> subs_ (var_ "a") (sliceTS_ (int_ 10) (int_ 2)) +-- a[:10:2] +-- +-- >>> sliceTS_ (int_ 10) (int_ 2) +-- slice(None, 10, 2) +sliceTS_ :: Raw Expr -> Raw Expr -> Raw Expr +sliceTS_ x y = slice_ Nothing (Just x) (Just y) + +-- | Slice /From/ @x@ /To/ @y@ +-- +-- >>> subs_ (var_ "a") (sliceFT_ (int_ 1) (int_ 10)) +-- a[1:10] +-- +-- >>> sliceFT_ (int_ 1) (int_ 10) +-- slice(1, 10, None) +sliceFT_ :: Raw Expr -> Raw Expr -> Raw Expr +sliceFT_ x y = slice_ (Just x) (Just y) Nothing + +-- | Slice /From/ @x@ /To/ @y@, with /Step/ @z@ +-- +-- >>> subs_ (var_ "a") (sliceFTS_ (int_ 1) (int_ 10) (int_ 2)) +-- a[1:10:2] +-- +-- >>> sliceFTS_ (int_ 1) (int_ 10) (int_ 2) +-- slice(1, 10, 2) +sliceFTS_ :: Raw Expr -> Raw Expr -> Raw Expr -> Raw Expr +sliceFTS_ x y z = slice_ (Just x) (Just y) (Just z) + +-- | A slice object +-- +-- Represents a call to a function named @slice@, with 3 arguments. +-- If an argument is a 'Nothing' then it becomes 'None', and if the argument is a +-- 'Just' then the contents are extracted. +slice_ :: Maybe (Raw Expr) -> Maybe (Raw Expr) -> Maybe (Raw Expr) -> Raw Expr +slice_ a b c = + call_ (var_ "slice") + [ p_ $ fromMaybe none_ a + , p_ $ fromMaybe none_ b + , p_ $ fromMaybe none_ c + ] + +-- | +-- >>> subs_ (var_ "a") (int_ 1) +-- a[1] +-- +-- >>> subs_ (var_ "a") (tuple_ [ti_ $ int_ 1]) +-- a[1,] +-- +-- >>> subs_ (var_ "a") (tuple_ [ti_ $ int_ 1, ti_ $ int_ 2]) +-- a[1, 2] +-- +-- >>> subs_ (var_ "a") (tuple_ [s_ $ var_ "b"]) +-- a[((*b),)] +-- +-- >>> subs_ (var_ "a") (tuple_ [ti_ $ int_ 1, s_ $ var_ "b"]) +-- a[(1, *b)] +subs_ :: Raw Expr -> Raw Expr -> Raw Expr +subs_ a e = + Subscript () a + [] + (exprToSubscript e ^. _CommaSep1') + [] + where + exprToSubscript + :: Raw Expr + -> (Raw Subscript, [(Comma, Raw Subscript)], Maybe Comma) + exprToSubscript e = + let + notSlice :: (Raw Subscript, [(Comma, Raw Subscript)], Maybe Comma) + notSlice = + case e ^? _Tuple of + Nothing -> (SubscriptExpr e, [], Nothing) + Just tup -> + let + h = tup ^. tupleHead + comma = tup ^. tupleComma + t = tup ^? tupleTail._Just.from _CommaSep1' + res = + case t of + Just (a, bs, c) -> + (,,) <$> + fromTupleItem h <*> + traverseOf (traverse._2) fromTupleItem ((comma, a) : bs) <*> + pure c + Nothing -> (\a -> (a, [], Just comma)) <$> fromTupleItem h + in + fromMaybe (SubscriptExpr e, [], Nothing) res + in + maybe notSlice (\a -> (a, [], Nothing)) $ mkSlice e + where + mkSlice + :: Raw Expr + -> Maybe (Raw Subscript) + mkSlice e = do + c <- e ^? _Call + case c ^? callFunction._Ident.identValue of + Just "slice" -> + pure $ case c ^.. callArguments.folded.folded of + [PositionalArg _ x] -> + SubscriptSlice Nothing (MkColon []) (Just x) Nothing + [PositionalArg _ x, PositionalArg _ y] -> + SubscriptSlice + (noneToMaybe x) + (MkColon []) + (noneToMaybe y) + Nothing + [PositionalArg _ x, PositionalArg _ y, PositionalArg _ z] -> + SubscriptSlice + (noneToMaybe x) + (MkColon []) + (noneToMaybe y) + ((,) (MkColon []) . Just <$> noneToMaybe z) + _ -> SubscriptExpr e + _ -> Nothing + + noneToMaybe x = fromMaybe (Just x) $ Nothing <$ (x ^? _None) + + fromTupleItem + :: Raw TupleItem + -> Maybe (Raw Subscript) + fromTupleItem (TupleItem _ a) = mkSlice a <|> pure (SubscriptExpr a) + fromTupleItem _ = Nothing diff --git a/src/Language/Python/Internal/Lexer.hs b/src/Language/Python/Internal/Lexer.hs new file mode 100644 --- /dev/null +++ b/src/Language/Python/Internal/Lexer.hs @@ -0,0 +1,763 @@ +{-# language BangPatterns #-} +{-# language TypeApplications #-} +{-# language FunctionalDependencies, MultiParamTypeClasses #-} +{-# language GeneralizedNewtypeDeriving #-} +{-# language FlexibleContexts #-} +{-# language TypeFamilies #-} +{-# language OverloadedStrings #-} +{-# language LambdaCase #-} + +{-| +Module : Language.Python.Internal.Lexer +Copyright : (C) CSIRO 2017-2018 +License : BSD3 +Maintainer : Isaac Elliott +Stability : experimental +Portability : non-portable +-} + +module Language.Python.Internal.Lexer + ( tokenizeWithTabs + -- * Source Information + , SrcInfo(..), initialSrcInfo, withSrcInfo + -- * Errors + , AsLexicalError(..), unsafeFromLexicalError + , AsTabError(..), AsIncorrectDedent(..), fromTabError, TabError(..) + -- * Miscellaneous + , tokenize + , insertTabs + -- * Megaparsec re-exports + , Parsec.ParseError(..) + ) +where + +import Control.Applicative ((<|>), many, optional) +import Control.Lens.Getter ((^.)) +import Control.Lens.Iso (from) +import Control.Lens.Prism (Prism') +import Control.Lens.Review ((#)) +import Control.Monad ((<=<), when, replicateM) +import Control.Monad.Except (throwError) +import Control.Monad.State (StateT, evalStateT, get, modify, put) +import Data.Bifunctor (first) +import Data.Digit.Binary (parseBinary) +import Data.Digit.Class.D0 (parse0) +import Data.Digit.Decimal (parseDecimal, parseDecimalNoZero) +import Data.Digit.Hexadecimal.MixedCase (parseHeXaDeCiMaL) +import Data.Digit.Octal (parseOctal) +import Data.FingerTree (FingerTree, Measured(..)) +import Data.Foldable (asum) +import Data.Functor.Identity (Identity) +import Data.List.NonEmpty (NonEmpty(..), some1) +import Data.Monoid (Sum(..)) +import Data.Set (Set) +import Data.Semigroup (Semigroup, (<>)) +import Data.Semigroup.Foldable (foldMap1) +import Data.These (These(..)) +import Data.Void (Void) +import GHC.Stack (HasCallStack) +import Text.Megaparsec (MonadParsec, ParseError, parse, unPos) +import Text.Megaparsec.Parsers + ( ParsecT, CharParsing, LookAheadParsing, lookAhead, unParsecT, satisfy, text + , char, manyTill, try + , notFollowedBy, anyChar, digit, oneOf + ) + +import qualified Data.FingerTree as FingerTree +import qualified Data.List.NonEmpty as NonEmpty +import qualified Data.Text as Text +import qualified Text.Megaparsec as Parsec + +import Language.Python.Internal.Token (PyToken(..), pyTokenAnn) +import Language.Python.Syntax.Comment +import Language.Python.Syntax.Ident +import Language.Python.Syntax.Numbers +import Language.Python.Syntax.Strings +import Language.Python.Syntax.Whitespace + +data SrcInfo + = SrcInfo + { _srcInfoName :: FilePath + , _srcInfoLineStart :: !Int + , _srcInfoLineEnd :: !Int + , _srcInfoColStart :: !Int + , _srcInfoColEnd :: !Int + , _srcInfoOffsetStart :: !Int + , _srcInfoOffsetEnd :: !Int + } + deriving (Eq, Show) + +instance Semigroup SrcInfo where + SrcInfo _ ls le cs ce os oe <> SrcInfo n' ls' le' cs' ce' os' oe' = + SrcInfo n' (min ls ls') (max le le') (min cs cs') (max ce ce') (min os os') (max oe oe') + +initialSrcInfo :: FilePath -> SrcInfo +initialSrcInfo fp = SrcInfo fp 0 0 0 0 0 0 + +{-# inline withSrcInfo #-} +withSrcInfo :: MonadParsec e s m => m (SrcInfo -> a) -> m a +withSrcInfo m = + (\(Parsec.SourcePos name l c) o f (Parsec.SourcePos _ l' c') o' -> + f $ SrcInfo name (unPos l) (unPos l') (unPos c) (unPos c') o o') <$> + Parsec.getPosition <*> + Parsec.getTokensProcessed <*> + m <*> + Parsec.getPosition <*> + Parsec.getTokensProcessed + +newline :: CharParsing m => m Newline +newline = LF <$ char '\n' <|> char '\r' *> (CRLF <$ char '\n' <|> pure CR) + +parseNewline :: (CharParsing m, Monad m) => m (SrcInfo -> PyToken SrcInfo) +parseNewline = TkNewline <$> newline + +parseComment :: (CharParsing m, Monad m) => m (SrcInfo -> PyToken SrcInfo) +parseComment = + (\a b -> TkComment (MkComment b a)) <$ char '#' <*> many (satisfy (`notElem` ['\r', '\n'])) + +stringOrBytesPrefix + :: CharParsing m + => m (Either + (Either RawStringPrefix StringPrefix) + (Either RawBytesPrefix BytesPrefix)) +stringOrBytesPrefix = + (char 'r' *> + (Right (Left Prefix_rb) <$ char 'b' <|> + Right (Left Prefix_rB) <$ char 'B' <|> + pure (Left $ Left Prefix_r))) <|> + (char 'R' *> + (Right (Left Prefix_Rb) <$ char 'b' <|> + Right (Left Prefix_RB) <$ char 'B' <|> + pure (Left $ Left Prefix_R))) <|> + (char 'b' *> + (Right (Left Prefix_br) <$ char 'r' <|> + Right (Left Prefix_bR) <$ char 'R' <|> + pure (Right $ Right Prefix_b))) <|> + (char 'B' *> + (Right (Left Prefix_Br) <$ char 'r' <|> + Right (Left Prefix_BR) <$ char 'R' <|> + pure (Right $ Right Prefix_B))) <|> + (Left (Right Prefix_u) <$ char 'u') <|> + (Left (Right Prefix_U) <$ char 'U') + +rawStringChar :: CharParsing m => m [PyChar] +rawStringChar = + (\a -> [Char_lit '\\', Char_lit a]) <$ char '\\' <*> anyChar <|> + pure . Char_lit <$> anyChar + +stringChar :: (CharParsing m, LookAheadParsing m) => m PyChar +stringChar = + (try (char '\\' <* lookAhead (oneOf "\"'U\\abfntuvx01234567")) *> + (escapeChar <|> unicodeChar <|> octChar <|> hexChar)) <|> + other + where + other = Char_lit <$> anyChar + escapeChar = + asum @[] + [ Char_esc_bslash <$ char '\\' + , Char_esc_singlequote <$ char '\'' + , Char_esc_doublequote <$ char '"' + , Char_esc_a <$ char 'a' + , Char_esc_b <$ char 'b' + , Char_esc_f <$ char 'f' + , char 'n' *> (Char_newline <$ text "ewline" <|> pure Char_esc_n) + , Char_esc_r <$ char 'r' + , Char_esc_t <$ char 't' + , Char_esc_v <$ char 'v' + ] + + unicodeChar = + char 'U' *> + ((\[a, b, c, d, e, f, g, h] -> Char_uni32 a b c d e f g h) <$> + replicateM 8 parseHeXaDeCiMaL) + <|> + char 'u' *> + ((\[a, b, c, d] -> Char_uni16 a b c d) <$> + replicateM 4 parseHeXaDeCiMaL) + + hexChar = Char_hex <$ char 'x' <*> parseHeXaDeCiMaL <*> parseHeXaDeCiMaL + octChar = + (\a b c -> + maybe + (Char_octal1 a) + (\b' -> maybe (Char_octal2 a b') (Char_octal3 a b') c) + b) <$> + parseOctal <*> + optional parseOctal <*> + optional parseOctal + +number :: (CharParsing m, Monad m) => m (a -> PyToken a) +number = do + zero <- optional parse0 + case zero of + Nothing -> do + nn <- optional $ (:|) <$> parseDecimalNoZero <*> many parseDecimal + case nn of + Just n -> + (\x j ann -> + case x of + Nothing -> + maybe (TkInt $ IntLiteralDec ann n) (TkImag . ImagLiteralInt ann n) j + Just (Right e) -> + let + f = FloatLiteralWhole ann n e + in + maybe (TkFloat f) (TkImag . ImagLiteralFloat ann f) j + Just (Left (Left e)) -> + let + f = FloatLiteralFull ann n (Just (That e)) + in + maybe (TkFloat f) (TkImag . ImagLiteralFloat ann f) j + Just (Left (Right (a, b))) -> + let + f = FloatLiteralFull ann n $ + case (a, b) of + (Nothing, Nothing) -> Nothing + (Just x, Nothing) -> Just $ This x + (Nothing, Just x) -> Just $ That x + (Just x, Just y) -> Just $ These x y + in + maybe (TkFloat f) (TkImag . ImagLiteralFloat ann f) j) <$> + optional + (Left <$ char '.' <*> + (Left <$> floatExp <|> + Right <$> ((,) <$> optional (some1 parseDecimal) <*> optional floatExp)) <|> + Right <$> floatExp) <*> + optional jJ + Nothing -> + (\a b j ann -> + let + f = FloatLiteralPoint ann a b + in + maybe (TkFloat f) (TkImag . ImagLiteralFloat ann f) j) <$> + -- try is necessary here to prevent the intercepting of dereference tokens + try (char '.' *> some1 parseDecimal) <*> + optional floatExp <*> + optional jJ + Just z -> + (\xX a b -> TkInt (IntLiteralHex b xX a)) <$> + (True <$ char 'X' <|> False <$ char 'x') <*> + some1 parseHeXaDeCiMaL + <|> + (\bB a b -> TkInt (IntLiteralBin b bB a)) <$> + (True <$ char 'B' <|> False <$ char 'b') <*> + some1 parseBinary + <|> + (\oO a b -> TkInt (IntLiteralOct b oO a)) <$> + (True <$ char 'O' <|> False <$ char 'o') <*> + some1 parseOctal + <|> + (\n j a -> + maybe (TkInt $ IntLiteralDec a (z :| n)) (TkImag . ImagLiteralInt a (z :| n)) j) <$> + try (many parse0 <* notFollowedBy (char '.' <|> char 'e' <|> char 'E' <|> digit)) <*> + optional jJ + <|> + (\n' a ann -> + case a of + Left (Left (b, c, j)) -> + let + f = FloatLiteralFull ann (z :| n') $ + case (b, c) of + (Nothing, Nothing) -> Nothing + (Just x, Nothing) -> Just $ This x + (Nothing, Just x) -> Just $ That x + (Just x, Just y) -> Just $ These x y + in + maybe (TkFloat f) (TkImag . ImagLiteralFloat ann f) j + Left (Right (x, j)) -> + let + f = FloatLiteralWhole ann (z :| n') x + in + maybe (TkFloat f) (TkImag . ImagLiteralFloat ann f) j + Right j -> TkImag $ ImagLiteralInt ann (z :| n') j) <$> + many parseDecimal <*> + (Left <$> + (Left <$> + ((,,) <$ char '.' <*> + optional (some1 parseDecimal) <*> + optional floatExp <*> + optional jJ) <|> + Right <$> + ((,) <$> floatExp <*> optional jJ)) <|> + Right <$> jJ) + where + jJ = False <$ char 'j' <|> True <$ char 'J' + floatExp = + FloatExponent <$> + (EE <$ char 'E' <|> Ee <$ char 'e') <*> + optional (Pos <$ char '+' <|> Neg <$ char '-') <*> + some1 parseDecimal + +{-# inline parseToken #-} +parseToken + :: (Monad m, CharParsing m, LookAheadParsing m, MonadParsec e s m) + => m (PyToken SrcInfo) +parseToken = + withSrcInfo $ + try + (asum + [ TkIf <$ text "if" + , TkElse <$ text "else" + , TkElif <$ text "elif" + , TkWhile <$ text "while" + , TkAssert <$ text "assert" + , TkDef <$ text "def" + , TkReturn <$ text "return" + , TkPass <$ text "pass" + , TkBreak <$ text "break" + , TkContinue <$ text "continue" + , TkTrue <$ text "True" + , TkFalse <$ text "False" + , TkNone <$ text "None" + , TkOr <$ text "or" + , TkAnd <$ text "and" + , TkIs <$ text "is" + , TkNot <$ text "not" + , TkGlobal <$ text "global" + , TkNonlocal <$ text "nonlocal" + , TkDel <$ text "del" + , TkLambda <$ text "lambda" + , TkImport <$ text "import" + , TkFrom <$ text "from" + , TkAs <$ text "as" + , TkRaise <$ text "raise" + , TkTry <$ text "try" + , TkExcept <$ text "except" + , TkFinally <$ text "finally" + , TkClass <$ text "class" + , TkWith <$ text "with" + , TkFor <$ text "for" + , TkIn <$ text "in" + , TkYield <$ text "yield" + ] <* notFollowedBy (satisfy isIdentifierChar)) + + <|> + + asum + [ number + , TkRightArrow <$ text "->" + , TkEllipsis <$ text "..." + , TkSpace <$ char ' ' + , TkTab <$ char '\t' + , TkLeftBracket <$ char '[' + , TkRightBracket <$ char ']' + , TkLeftParen <$ char '(' + , TkRightParen <$ char ')' + , TkLeftBrace <$ char '{' + , TkRightBrace <$ char '}' + , char '<' *> + (TkLte <$ char '=' <|> + char '<' *> (TkShiftLeftEq <$ char '=' <|> pure TkShiftLeft) <|> + pure TkLt) + , char '=' *> (TkDoubleEq <$ char '=' <|> pure TkEq) + , char '>' *> + (TkGte <$ char '=' <|> + char '>' *> (TkShiftRightEq <$ char '=' <|> pure TkShiftRight) <|> + pure TkGt) + , char '*' *> + (char '*' *> (TkDoubleStarEq <$ char '=' <|> pure TkDoubleStar) <|> + TkStarEq <$ char '=' <|> + pure TkStar) + , char '/' *> + (char '/' *> (TkDoubleSlashEq <$ char '=' <|> pure TkDoubleSlash) <|> + TkSlashEq <$ char '=' <|> + pure TkSlash) + , TkBangEq <$ text "!=" + , char '^' *> (TkCaretEq <$ char '=' <|> pure TkCaret) + , char '|' *> (TkPipeEq <$ char '=' <|> pure TkPipe) + , char '&' *> (TkAmpersandEq <$ char '=' <|> pure TkAmpersand) + , char '@' *> (TkAtEq <$ char '=' <|> pure TkAt) + , char '+' *> (TkPlusEq <$ char '=' <|> pure TkPlus) + , char '-' *> (TkMinusEq <$ char '=' <|> pure TkMinus) + , char '%' *> (TkPercentEq <$ char '=' <|> pure TkPercent) + , TkTilde <$ char '~' + , TkContinued <$ char '\\' <*> newline + , TkColon <$ char ':' + , TkSemicolon <$ char ';' + , parseComment + , parseNewline + , TkComma <$ char ',' + , TkDot <$ char '.' + , do + sp <- try $ optional stringOrBytesPrefix <* char '"' + case sp of + Nothing -> + TkString Nothing LongString DoubleQuote <$ + text "\"\"" <*> + manyTill stringChar (text "\"\"\"") + <|> + TkString Nothing ShortString DoubleQuote <$> manyTill stringChar (char '"') + Just (Left (Left prefix)) -> + TkRawString prefix LongString DoubleQuote . concat <$ + text "\"\"" <*> + manyTill rawStringChar (text "\"\"\"") + <|> + TkRawString prefix ShortString DoubleQuote . concat <$> + manyTill rawStringChar (char '"') + Just (Left (Right prefix)) -> + TkString (Just prefix) LongString DoubleQuote <$ + text "\"\"" <*> + manyTill stringChar (text "\"\"\"") + <|> + TkString (Just prefix) ShortString DoubleQuote <$> manyTill stringChar (char '"') + Just (Right (Left prefix)) -> + TkRawBytes prefix LongString DoubleQuote . concat <$ + text "\"\"" <*> + manyTill rawStringChar (text "\"\"\"") + <|> + TkRawBytes prefix ShortString DoubleQuote . concat <$> + manyTill rawStringChar (char '"') + Just (Right (Right prefix)) -> + TkBytes prefix LongString DoubleQuote <$ + text "\"\"" <*> + manyTill stringChar (text "\"\"\"") + <|> + TkBytes prefix ShortString DoubleQuote <$> manyTill stringChar (char '"') + , do + sp <- try $ optional stringOrBytesPrefix <* char '\'' + case sp of + Nothing -> + TkString Nothing LongString SingleQuote <$ + text "''" <*> + manyTill stringChar (text "'''") + <|> + TkString Nothing ShortString SingleQuote <$> manyTill stringChar (char '\'') + Just (Left (Left prefix)) -> + TkRawString prefix LongString SingleQuote . concat <$ + text "''" <*> + manyTill rawStringChar (text "'''") + <|> + TkRawString prefix ShortString SingleQuote . concat <$> + manyTill rawStringChar (char '\'') + Just (Left (Right prefix)) -> + TkString (Just prefix) LongString SingleQuote <$ + text "''" <*> + manyTill stringChar (text "'''") + <|> + TkString (Just prefix) ShortString SingleQuote <$> manyTill stringChar (char '\'') + Just (Right (Left prefix)) -> + TkRawBytes prefix LongString SingleQuote . concat <$ + text "''" <*> + manyTill rawStringChar (text "'''") + <|> + TkRawBytes prefix ShortString SingleQuote . concat <$> + manyTill rawStringChar (char '\'') + Just (Right (Right prefix)) -> + TkBytes prefix LongString SingleQuote <$ + text "''" <*> + manyTill stringChar (text "'''") + <|> + TkBytes prefix ShortString SingleQuote <$> manyTill stringChar (char '\'') + , fmap TkIdent $ + (:) <$> + satisfy isIdentifierStart <*> + many (satisfy isIdentifierChar) + ] + +class AsLexicalError s t | s -> t where + _LexicalError + :: Prism' + s + ( NonEmpty Parsec.SourcePos + , Maybe (Parsec.ErrorItem t) + , Set (Parsec.ErrorItem t) + ) + +-- | Convert a concrete 'ParseError' to a value that has an instance of 'AsLexicalError' +-- +-- This function is partial, because our parser will never use 'Parsec.FancyError' +unsafeFromLexicalError + :: ( HasCallStack + , AsLexicalError s t + ) + => ParseError t Void + -> s +unsafeFromLexicalError (Parsec.TrivialError a b c) = _LexicalError # (a, b, c) +unsafeFromLexicalError Parsec.FancyError{} = error "'fancy error' used in lexer" + +{-# noinline tokenize #-} +-- | Convert some input to a sequence of tokens. Indent and dedent tokens are not added +-- (see 'insertTabs') +tokenize + :: AsLexicalError e Char + => FilePath -- ^ File name + -> Text.Text -- ^ Input to tokenize + -> Either e [PyToken SrcInfo] +tokenize fp = first unsafeFromLexicalError . parse (unParsecT tokens) fp + where + tokens :: ParsecT Void Text.Text Identity [PyToken SrcInfo] + tokens = many parseToken <* Parsec.eof + +data LogicalLine a + = LogicalLine + a -- annotation + ([PyToken a], Indent) -- spaces + [PyToken a] -- line + (Maybe (PyToken a)) -- end + | BlankLine + [PyToken a] -- line + (Maybe (PyToken a)) -- end + deriving (Eq, Show) + +logicalLineToTokens :: LogicalLine a -> [PyToken a] +logicalLineToTokens (LogicalLine _ _ ts m) = ts <> maybe [] pure m +logicalLineToTokens (BlankLine ts m) = ts <> maybe [] pure m + +spaceToken :: PyToken a -> Maybe Whitespace +spaceToken TkSpace{} = Just Space +spaceToken TkTab{} = Just Tab +spaceToken (TkContinued nl _) = Just $ Continued nl [] +spaceToken _ = Nothing + +collapseContinue :: [(PyToken a, Whitespace)] -> [([PyToken a], Whitespace)] +collapseContinue [] = [] +collapseContinue ((tk@TkSpace{}, Space) : xs) = + ([tk], Space) : collapseContinue xs +collapseContinue ((tk@TkTab{}, Tab) : xs) = + ([tk], Tab) : collapseContinue xs +collapseContinue ((tk@TkNewline{}, Newline nl) : xs) = + ([tk], Newline nl) : collapseContinue xs +collapseContinue ((tk@TkContinued{}, Continued nl ws) : xs) = + let + xs' = collapseContinue xs + in + [(tk : (xs' >>= fst), Continued nl $ ws <> fmap snd xs')] +collapseContinue _ = error "invalid token/whitespace pair in collapseContinue" + +spanMaybe :: (a -> Maybe b) -> [a] -> ([b], [a]) +spanMaybe f as = + case as of + [] -> ([], []) + x : xs -> + case f x of + Nothing -> ([], as) + Just b -> first (b :) $ spanMaybe f xs + +-- | Acts like break, but encodes the "insignificant whitespace" rule for parens, braces +-- and brackets +breakOnNewline :: [PyToken a] -> ([PyToken a], Maybe (PyToken a, [PyToken a])) +breakOnNewline = go 0 + where + go :: Int -> [PyToken a] -> ([PyToken a], Maybe (PyToken a, [PyToken a])) + go _ [] = ([], Nothing) + go !careWhen0 (tk : tks) = + case tk of + TkLeftParen{} -> first (tk :) $ go (careWhen0 + 1) tks + TkLeftBracket{} -> first (tk :) $ go (careWhen0 + 1) tks + TkLeftBrace{} -> first (tk :) $ go (careWhen0 + 1) tks + TkRightParen{} -> first (tk :) $ go (max 0 $ careWhen0 - 1) tks + TkRightBracket{} -> first (tk :) $ go (max 0 $ careWhen0 - 1) tks + TkRightBrace{} -> first (tk :) $ go (max 0 $ careWhen0 - 1) tks + TkNewline{} + | careWhen0 == 0 -> ([], Just (tk, tks)) + | otherwise -> first (tk :) $ go careWhen0 tks + _ -> first (tk :) $ go careWhen0 tks + +logicalLines :: [PyToken a] -> [LogicalLine a] +logicalLines [] = [] +logicalLines tks = + let + (spaces, rest) = spanMaybe (\a -> (,) a <$> spaceToken a) tks + (line, rest') = breakOnNewline rest + spaces' = collapseContinue spaces + in + (if + not (any (\case; Continued{} -> True; _ -> False) $ snd <$> spaces) && + all isBlankToken line + then + BlankLine (fmap fst spaces <> line) (fst <$> rest') + else + LogicalLine + (case tks of + [] -> error "couldn't generate annotation for logical line" + tk : _ -> pyTokenAnn tk) + (spaces' >>= fst, fmap snd spaces' ^. from indentWhitespaces) + line + (fst <$> rest')) : + logicalLines (maybe [] snd rest') + +data IndentedLine a + = Indent Int Indent a + | Level (NonEmpty Whitespace) a + | Dedent a + | IndentedLine (LogicalLine a) + deriving (Eq, Show) + +isBlankToken :: PyToken a -> Bool +isBlankToken TkSpace{} = True +isBlankToken TkTab{} = True +isBlankToken TkComment{} = True +isBlankToken TkNewline{} = True +isBlankToken _ = False + +data TabError a + -- | Tabs and spaces were used inconsistently + = TabError a + -- | The dedent at the end of a block doesn't match and preceding indents + -- + -- e.g. + -- + -- @ + -- def a(): + -- if b: + -- pass + -- else: + -- pass + -- pass + -- @ + -- + -- The final line will cause an 'IncorrectDedent' error + | IncorrectDedent a + deriving (Eq, Show) + +class AsTabError s a | s -> a where + _TabError :: Prism' s a + +class AsIncorrectDedent s a | s -> a where + _IncorrectDedent :: Prism' s a + +-- | Convert a concrete 'TabError' to a value that has an instance of 'AsTabError' +fromTabError + :: ( AsTabError s a + , AsIncorrectDedent s a + ) + => TabError a -> s +fromTabError (TabError a) = _TabError # a +fromTabError (IncorrectDedent a) = _IncorrectDedent # a + +indentation :: Semigroup a => a -> [LogicalLine a] -> Either (TabError a) [IndentedLine a] +indentation ann lls = + flip evalStateT (pure (ann, mempty)) $ + (<>) <$> (concat <$> traverse go lls) <*> finalDedents + where + finalDedents :: StateT (NonEmpty (a, Indent)) (Either (TabError a)) [IndentedLine a] + finalDedents = do + (ann, _) :| is <- get + case is of + [] -> pure [] + i' : is' -> do + put $ i' :| is' + (Dedent ann :) <$> finalDedents + + dedents + :: a + -> Int + -> StateT (NonEmpty (a, Indent)) (Either (TabError a)) [IndentedLine a] + dedents ann n = do + is <- get + let (popped, remainder) = NonEmpty.span ((> n) . indentLevel . snd) is + when (n `notElem` fmap (indentLevel . snd) (NonEmpty.toList is)) . + throwError $ IncorrectDedent ann + put $ case remainder of + [] -> error "I don't know whether this can happen" + x : xs -> x :| xs + pure $ replicate (length popped) (Dedent ann) + + go + :: Semigroup a + => LogicalLine a + -> StateT (NonEmpty (a, Indent)) (Either (TabError a)) [IndentedLine a] + go ll@BlankLine{} = pure [IndentedLine ll] + go ll@(LogicalLine ann (spTks, spcs) _ _) = do + (_, i) :| _ <- get + let + et8 = absoluteIndentLevel 8 spcs + et1 = absoluteIndentLevel 1 spcs + et8i = absoluteIndentLevel 8 i + et1i = absoluteIndentLevel 1 i + when + (not (et8 < et8i && et1 < et1i) && + not (et8 > et8i && et1 > et1i) && + not (et8 == et8i && et1 == et1i)) + (throwError $ TabError ann) + let + ilSpcs = indentLevel spcs + ili = indentLevel i + levelIndent = + case (spTks, spcs ^. indentWhitespaces) of + ([], []) -> [] + (x:xs, y:ys) -> [ Level (y:|ys) (foldMap1 pyTokenAnn $ x:|xs) ] + _ -> error "impossible" + case compare ilSpcs ili of + LT -> (<> (levelIndent <> [IndentedLine ll])) <$> dedents ann ilSpcs + EQ -> + pure $ levelIndent <> [ IndentedLine ll ] + GT -> do + modify $ NonEmpty.cons (ann, spcs) + pure [Indent (ilSpcs - ili) spcs ann, IndentedLine ll] + +newtype Summed a = Summed a + deriving (Eq, Show, Ord, Num) + +instance Num a => Measured (Sum a) (Summed a) where + measure (Summed a) = Sum a + +-- | Given a list of indentation jumps (first to last) and some whitespace, +-- divide the whitespace up into "blocks" which correspond to each jump +splitIndents :: FingerTree (Sum Int) (Summed Int) -> Indent -> [Indent] +splitIndents ns ws = go ns ws [] + where + go :: FingerTree (Sum Int) (Summed Int) -> Indent -> [Indent] -> [Indent] + go ns ws = + case FingerTree.viewr ns of + FingerTree.EmptyR -> (ws :) + ns' FingerTree.:> n + | FingerTree.null ns' -> (ws :) + | otherwise -> + let + (befores, afters) = + FingerTree.split ((> getSum (measure ns')) . getIndentLevel) $ unIndent ws + in + if FingerTree.null afters + then error $ "could not carve out " <> show n <> " from " <> show ws + else go ns' (MkIndent befores) . (MkIndent afters :) + +chunked :: [IndentedLine a] -> [PyToken a] +chunked = go FingerTree.empty + where + go + :: FingerTree (Sum Int) (Summed Int) + -> [IndentedLine a] + -> [PyToken a] + go _ [] = [] + go leaps (Indent n i a : is) = + let + leaps' = leaps FingerTree.|> Summed n + in + TkIndent a (Indents (splitIndents leaps' i) a) : go leaps' is + go leaps (Dedent a : is) = + case FingerTree.viewr leaps of + FingerTree.EmptyR -> error "impossible" + leaps' FingerTree.:> _ -> TkDedent a : go leaps' is + go leaps (IndentedLine ll : is) = logicalLineToTokens ll <> go leaps is + go leaps (Level i a : is) = + TkLevel a (Indents (splitIndents leaps $ NonEmpty.toList i ^. from indentWhitespaces) a) : go leaps is + +-- | Insert indent and dedent tokens +-- +-- https://docs.python.org/3.5/reference/lexical_analysis.html#indentation +insertTabs + :: ( Semigroup a + , AsTabError s a + , AsIncorrectDedent s a + ) + => a -- ^ Initial source annotation + -> [PyToken a] -- ^ Token stream + -> Either s [PyToken a] +insertTabs a = + first fromTabError . + fmap chunked . + indentation a . + logicalLines + +-- | Tokenize an input file, inserting indent\/level\/dedent tokens in appropriate +-- positions according to the block structure. +tokenizeWithTabs + :: ( AsLexicalError s Char + , AsTabError s SrcInfo + , AsIncorrectDedent s SrcInfo + ) + => FilePath -- ^ File name + -> Text.Text -- ^ Input to tokenize + -> Either s [PyToken SrcInfo] +tokenizeWithTabs fp = insertTabs (initialSrcInfo fp) <=< tokenize fp diff --git a/src/Language/Python/Internal/Parse.hs b/src/Language/Python/Internal/Parse.hs new file mode 100644 --- /dev/null +++ b/src/Language/Python/Internal/Parse.hs @@ -0,0 +1,1575 @@ +{-# language DataKinds #-} +{-# language FlexibleContexts #-} +{-# language LambdaCase #-} +{-# language RankNTypes #-} +{-# language FunctionalDependencies, MultiParamTypeClasses #-} +{-# language TypeFamilies #-} + +{-| +Module : Language.Python.Internal.Parse +Copyright : (C) CSIRO 2017-2018 +License : BSD3 +Maintainer : Isaac Elliott +Stability : experimental +Portability : non-portable +-} + +module Language.Python.Internal.Parse + ( Parser + , runParser + -- * Stream type + , PyTokens(..) + -- * Errors + , AsParseError(..) + , unsafeFromParseError + -- * Parsers + , token + -- ** Symbols + , at + , colon + , comma + , dot + , doubleStar + , equals + , rightParen + , semicolon + , star + -- ** Atomic forms + , identifier + , bool + , none + , ellipsis + , integer + , float + , imag + , stringOrBytes + -- ** Compound forms + , arg + , binOp + , commaSep + , commaSep1 + , commaSep1' + , commaSepRest + , compIf + , compFor + , compoundStatement + , decorator + , decoratorValue + , decorators + , expr + , exprList + , exprListComp + , exprNoCond + , exprComp + , exprOrStarList + , lambda + , lambdaNoCond + , module_ + , orExpr + , orExprList + , orTest + , smallStatement + , someParams + , simpleStatement + , starExpr + , statement + , suite + , tpPositional + , tpStar + , tpDoubleStar + , tyAnn + , typedParams + , untypedParams + , upPositional + , upStar + , upDoubleStar + , yieldExpr + -- ** Formatting + , anySpace + , space + , eol + , continued + , newline + , indent + , dedent + , level + , blank + , comment + -- ** Miscellaneous combinators + , sepBy1' + ) +where + +import Control.Applicative (Alternative, (<|>), optional, many, some) +import Control.Lens.Cons (snoc) +import Control.Lens.Getter ((^.), view) +import Control.Lens.Prism (Prism') +import Control.Lens.Review ((#)) +import Control.Monad (void) +import Data.Bifunctor (first, second) +import Data.Coerce (coerce) +import Data.Function ((&)) +import Data.List (foldl') +import Data.List.NonEmpty (NonEmpty, some1) +import Data.Proxy (Proxy(..)) +import Data.Set (Set) +import Data.Void (Void) +import GHC.Stack (HasCallStack) +import Text.Megaparsec + ( (), MonadParsec, Parsec, Stream(..), SourcePos(..), eof, try, lookAhead + , notFollowedBy + ) +import Text.Megaparsec.Char (satisfy) + + +import qualified Data.List.NonEmpty as NonEmpty +import qualified Text.Megaparsec as Megaparsec + +import Language.Python.Internal.Lexer (SrcInfo(..), withSrcInfo) +import Language.Python.Internal.Syntax.IR +import Language.Python.Internal.Token +import Language.Python.Syntax.AugAssign +import Language.Python.Syntax.CommaSep +import Language.Python.Syntax.Comment +import Language.Python.Syntax.Ident +import Language.Python.Syntax.Import +import Language.Python.Syntax.ModuleNames +import Language.Python.Syntax.Numbers +import Language.Python.Syntax.Operator.Binary +import Language.Python.Syntax.Operator.Unary +import Language.Python.Syntax.Punctuation +import Language.Python.Syntax.Strings +import Language.Python.Syntax.Whitespace + +newtype PyTokens = PyTokens { unPyTokens :: [PyToken SrcInfo] } + deriving (Eq, Ord) + +instance Stream PyTokens where + type Token PyTokens = PyToken SrcInfo + type Tokens PyTokens = PyTokens + tokenToChunk Proxy = PyTokens . pure + tokensToChunk Proxy = PyTokens + chunkToTokens Proxy = unPyTokens + chunkLength Proxy = length . unPyTokens + chunkEmpty Proxy = null . unPyTokens + positionAt1 Proxy _ tk = + let + ann = pyTokenAnn tk + in + SourcePos + (_srcInfoName ann) + (Megaparsec.mkPos $ _srcInfoLineStart ann) + (Megaparsec.mkPos $ _srcInfoColStart ann) + positionAtN Proxy spos (PyTokens tks) = + case tks of + [] -> spos + _ -> + let + ann = pyTokenAnn $ last tks + in + SourcePos + (_srcInfoName ann) + (Megaparsec.mkPos $ _srcInfoLineStart ann) + (Megaparsec.mkPos $ _srcInfoColStart ann) + advance1 Proxy _ _ tk = + let + ann = pyTokenAnn tk + in + SourcePos + (_srcInfoName ann) + (Megaparsec.mkPos $ _srcInfoLineEnd ann) + (Megaparsec.mkPos $ _srcInfoColEnd ann) + advanceN Proxy _ spos (PyTokens tks) = + case tks of + [] -> spos + _ -> + let + ann = pyTokenAnn $ last tks + in + SourcePos + (_srcInfoName ann) + (Megaparsec.mkPos $ _srcInfoLineEnd ann) + (Megaparsec.mkPos $ _srcInfoColEnd ann) + + take1_ (PyTokens p) = + case p of + [] -> Nothing + t:ts -> Just (t, PyTokens ts) + + takeN_ n (PyTokens s) + | n <= 0 = Just (PyTokens [], PyTokens s) + | null s = Nothing + | otherwise = Just (coerce (splitAt n s)) + + takeWhile_ f = coerce (span f) + +class AsParseError s t | s -> t where + _ParseError + :: Prism' + s + ( NonEmpty SourcePos + , Maybe (Megaparsec.ErrorItem t) + , Set (Megaparsec.ErrorItem t) + ) + +-- | Convert a concrete 'Megaparsec.ParseError' to a value that has an instance of 'AsParseError' +-- +-- This function is partial because our parser will never use 'Megaparsec.FancyError' +unsafeFromParseError + :: (HasCallStack, AsParseError s t) + => Megaparsec.ParseError t e + -> s +unsafeFromParseError Megaparsec.FancyError{} = error "there are none of these" +unsafeFromParseError (Megaparsec.TrivialError pos a b) = _ParseError # (pos, a, b) + +type Parser = Parsec Void PyTokens + +-- | Run a parser on some input +{-# inline runParser #-} +runParser + :: AsParseError e (PyToken SrcInfo) + => FilePath -- ^ File name + -> Parser a -- ^ Parser + -> [PyToken SrcInfo] -- ^ Input to parse + -> Either e a +runParser file p input = + first unsafeFromParseError $ Megaparsec.parse p file (PyTokens input) + +eol :: MonadParsec e PyTokens m => m Newline +eol = + (\(TkNewline nl _) -> nl) <$> + satisfy (\case; TkNewline{} -> True; _ -> False) "newline" + +dedent :: MonadParsec e PyTokens m => m () +dedent = () <$ satisfy (\case; TkDedent{} -> True; _ -> False) "dedent" + +space :: MonadParsec e PyTokens m => m Whitespace +space = + Space <$ satisfy (\case; TkSpace{} -> True; _ -> False) <|> + Tab <$ satisfy (\case; TkTab{} -> True; _ -> False) <|> + continued + +continued :: MonadParsec e PyTokens m => m Whitespace +continued = + (\(TkContinued nl _) -> Continued nl) <$> + satisfy (\case; TkContinued{} -> True; _ -> False) <*> + many space + +newline :: MonadParsec e PyTokens m => m Newline +newline = (\(TkNewline nl _) -> nl) <$> satisfy (\case; TkNewline{} -> True; _ -> False) + +anySpace :: MonadParsec e PyTokens m => m Whitespace +anySpace = + Space <$ satisfy (\case; TkSpace{} -> True; _ -> False) <|> + Tab <$ satisfy (\case; TkTab{} -> True; _ -> False) <|> + continued <|> + Newline <$> newline <|> + Comment . void <$> comment + +token + :: MonadParsec e PyTokens m + => m Whitespace + -> (PyToken SrcInfo -> Bool) + -> String + -> m (PyToken SrcInfo, [Whitespace]) +token ws f label = (,) <$> satisfy f <*> many ws label + +identifier :: MonadParsec e PyTokens m => m Whitespace -> m (Ident '[] SrcInfo) +identifier ws = + (\(TkIdent n ann) -> MkIdent ann n) <$> + satisfy (\case; TkIdent{} -> True; _ -> False) <*> + many ws + +bool :: MonadParsec e PyTokens m => m Whitespace -> m (Expr SrcInfo) +bool ws = + (\(tk, s) -> + Bool + (pyTokenAnn tk) + (case tk of + TkTrue{} -> True + TkFalse{} -> False + _ -> error "impossible") + s) <$> + (token ws (\case; TkTrue{} -> True; _ -> False) "True" <|> + token ws (\case; TkFalse{} -> True; _ -> False) "False") + +none :: MonadParsec e PyTokens m => m Whitespace -> m (Expr SrcInfo) +none ws = + (\(tk, s) -> None (pyTokenAnn tk) s) <$> + token ws (\case; TkNone{} -> True; _ -> False) "None" + +ellipsis :: MonadParsec e PyTokens m => m Whitespace -> m (Expr SrcInfo) +ellipsis ws = + (\(tk, s) -> Ellipsis (pyTokenAnn tk) s) <$> + token ws (\case; TkEllipsis{} -> True; _ -> False) "..." + +integer :: MonadParsec e PyTokens m => m Whitespace -> m (Expr SrcInfo) +integer ws = + (\(TkInt n) -> Int (_intLiteralAnn n) n) <$> + satisfy (\case; TkInt{} -> True; _ -> False) <*> + many ws + +float :: MonadParsec e PyTokens m => m Whitespace -> m (Expr SrcInfo) +float ws = + (\(TkFloat n) -> Float (_floatLiteralAnn n) n) <$> + satisfy (\case; TkFloat{} -> True; _ -> False) <*> + many ws + +imag :: MonadParsec e PyTokens m => m Whitespace -> m (Expr SrcInfo) +imag ws = + (\(TkImag n) -> Imag (_imagLiteralAnn n) n) <$> + satisfy (\case; TkImag{} -> True; _ -> False) <*> + many ws + +stringOrBytes :: MonadParsec e PyTokens m => m Whitespace -> m (Expr SrcInfo) +stringOrBytes ws = + fmap (\vs -> String (_stringLiteralAnn $ NonEmpty.head vs) vs) . some1 $ + (\case + TkString sp qt st val ann -> StringLiteral ann sp qt st val + TkBytes sp qt st val ann -> BytesLiteral ann sp qt st val + TkRawString sp st qt val ann -> RawStringLiteral ann sp st qt val + TkRawBytes sp st qt val ann -> RawBytesLiteral ann sp st qt val + _ -> error "impossible") <$> + satisfy + (\case + TkString{} -> True + TkBytes{} -> True + TkRawString{} -> True + TkRawBytes{} -> True + _ -> False) <*> + many ws + +comment :: MonadParsec e PyTokens m => m (Comment SrcInfo) +comment = + (\(TkComment c) -> c) <$> + satisfy (\case; TkComment{} -> True; _ -> False) "comment" + +indent :: MonadParsec e PyTokens m => m (Indents SrcInfo) +indent = + (\(TkIndent _ i) -> i) <$> satisfy (\case; TkIndent{} -> True; _ -> False) "indent" + +level :: MonadParsec s PyTokens m => m (Indents SrcInfo) +level = + (\(TkLevel _ i) -> i) <$> satisfy (\case; TkLevel{} -> True; _ -> False) "level indentation" + +comma :: MonadParsec e PyTokens m => m Whitespace -> m (PyToken SrcInfo, Comma) +comma ws = second MkComma <$> token ws (\case; TkComma{} -> True; _ -> False) "," + +dot :: MonadParsec e PyTokens m => m Whitespace -> m (PyToken SrcInfo, Dot) +dot ws = second MkDot <$> token ws (\case; TkDot{} -> True; _ -> False) "." + +at :: MonadParsec e PyTokens m => m Whitespace -> m (PyToken SrcInfo, At) +at ws = second MkAt <$> token ws (\case; TkAt{} -> True; _ -> False) "@" + +colon :: MonadParsec e PyTokens m => m Whitespace -> m (PyToken SrcInfo, Colon) +colon ws = second MkColon <$> token ws (\case; TkColon{} -> True; _ -> False) ":" + +equals :: MonadParsec e PyTokens m => m Whitespace -> m (PyToken SrcInfo, Equals) +equals ws = second MkEquals <$> token ws (\case; TkEq{} -> True; _ -> False) "=" + +semicolon :: MonadParsec e PyTokens m => m Whitespace -> m (PyToken SrcInfo, Semicolon SrcInfo) +semicolon ws = + (\(a, b) -> (a, MkSemicolon (pyTokenAnn a) b)) <$> + token ws (\case; TkSemicolon{} -> True; _ -> False) ";" + +exprList :: MonadParsec e PyTokens m => m Whitespace -> m (Expr SrcInfo) +exprList ws = + (\e -> maybe e (uncurry $ Tuple (e ^. exprAnn) e)) <$> + expr ws <*> + optional + ((,) <$> + (snd <$> comma ws) <*> + optional (commaSep1' ws $ expr ws)) + +exprOrStarList :: MonadParsec e PyTokens m => m Whitespace -> m (Expr SrcInfo) +exprOrStarList ws = + (\e -> maybe e (uncurry $ Tuple (e ^. exprAnn) e)) <$> + (expr ws <|> starExpr ws) <*> + optional + ((,) <$> + (snd <$> comma ws) <*> + optional (commaSep1' ws $ expr ws <|> starExpr ws)) + +compIf :: MonadParsec e PyTokens m => m (CompIf SrcInfo) +compIf = + (\(tk, s) -> CompIf (pyTokenAnn tk) s) <$> + token anySpace (\case; TkIf{} -> True; _ -> False) "if" <*> + exprNoCond anySpace + +compFor :: MonadParsec e PyTokens m => m (CompFor SrcInfo) +compFor = + (\(tk, s) -> CompFor (pyTokenAnn tk) s) <$> + token anySpace (\case; TkFor{} -> True; _ -> False) "for" <*> + orExprList anySpace <*> + (snd <$> token anySpace (\case; TkIn{} -> True; _ -> False) "in") <*> + orTest anySpace + +commaSepRest :: MonadParsec e PyTokens m => m b -> m ([(Comma, b)], Maybe Comma) +commaSepRest x = do + c <- optional $ snd <$> comma anySpace + case c of + Nothing -> pure ([], Nothing) + Just c' -> do + e <- optional x + case e of + Nothing -> pure ([], Just c') + Just e' -> first ((c', e') :) <$> commaSepRest x + +exprComp :: MonadParsec e PyTokens m => m Whitespace -> m (Expr SrcInfo) +exprComp ws = + (\ex a -> + case a of + Nothing -> ex + Just (cf, rest) -> + Generator (ex ^. exprAnn) $ + Comprehension (ex ^. exprAnn) ex cf rest) <$> + expr ws <*> + optional ((,) <$> compFor <*> many (Left <$> compFor <|> Right <$> compIf)) + +star :: MonadParsec e PyTokens m => m Whitespace -> m (PyToken SrcInfo, [Whitespace]) +star sp = token sp (\case; TkStar{} -> True; _ -> False) "*" + +starExpr :: MonadParsec e PyTokens m => m Whitespace -> m (Expr SrcInfo) +starExpr ws = + (\(tk, sp) -> StarExpr (pyTokenAnn tk) sp) <$> + star ws <*> + orExpr ws + +exprListComp :: MonadParsec e PyTokens m => m Whitespace -> m (Expr SrcInfo) +exprListComp ws = + (\e a -> + case a of + Left (cf, cfs) -> + let + ann = e ^. exprAnn + in + Generator ann $ Comprehension ann e cf cfs + Right (Just (c, cs)) -> Tuple (e ^. exprAnn) e c cs + Right Nothing -> e) <$> + (expr ws <|> starExpr ws) <*> + (Left <$> + ((,) <$> + compFor <*> + many (Left <$> compFor <|> Right <$> compIf)) <|> + Right <$> + optional + ((,) <$> + (snd <$> comma ws) <*> + optional (commaSep1' ws $ expr ws <|> starExpr ws))) + +orExprList :: MonadParsec e PyTokens m => m Whitespace -> m (Expr SrcInfo) +orExprList ws = + (\e -> maybe e (uncurry $ Tuple (e ^. exprAnn) e)) <$> + (orExpr ws <|> starExpr ws) <*> + optional + ((,) <$> + (snd <$> comma ws) <*> + optional (commaSep1' ws $ orExpr ws <|> starExpr ws)) + +binOp :: MonadParsec e PyTokens m => m (BinOp SrcInfo) -> m (Expr SrcInfo) -> m (Expr SrcInfo) +binOp op tm = + (\t ts -> + case ts of + [] -> t + _ -> foldl (\tm (o, val) -> BinOp (tm ^. exprAnn) tm o val) t ts) <$> + tm <*> + many ((,) <$> op <*> tm) + +orTest :: MonadParsec e PyTokens m => m Whitespace -> m (Expr SrcInfo) +orTest ws = binOp orOp andTest + where + orOp = + (\(tk, ws) -> BoolOr (pyTokenAnn tk) ws) <$> + token ws (\case; TkOr{} -> True; _ -> False) "or" + + andOp = + (\(tk, ws) -> BoolAnd (pyTokenAnn tk) ws) <$> + token ws (\case; TkAnd{} -> True; _ -> False) "and" + andTest = binOp andOp notTest + + notTest = + (\(tk, s) -> Not (pyTokenAnn tk) s) <$> + token ws (\case; TkNot{} -> True; _ -> False) "not" <*> notTest <|> + comparison + + compOp = + (\(tk, ws) -> maybe (Is (pyTokenAnn tk) ws) (IsNot (pyTokenAnn tk) ws)) <$> + token ws (\case; TkIs{} -> True; _ -> False) "is" <*> + optional (snd <$> token ws (\case; TkNot{} -> True; _ -> False) "not") + + <|> + + (\(tk, ws) -> NotIn (pyTokenAnn tk) ws) <$> + token ws (\case; TkNot{} -> True; _ -> False) "not" <*> + (snd <$> token ws (\case; TkIn{} -> True; _ -> False) "in") + + <|> + + (\(tk, ws) -> In (pyTokenAnn tk) ws) <$> + token ws (\case; TkIn{} -> True; _ -> False) "in" + + <|> + + (\(tk, ws) -> Eq (pyTokenAnn tk) ws) <$> + token ws (\case; TkDoubleEq{} -> True; _ -> False) "==" + + <|> + + (\(tk, ws) -> Lt (pyTokenAnn tk) ws) <$> + token ws (\case; TkLt{} -> True; _ -> False) "<" + + <|> + + (\(tk, ws) -> LtEq (pyTokenAnn tk) ws) <$> + token ws (\case; TkLte{} -> True; _ -> False) "<=" + + <|> + + (\(tk, ws) -> Gt (pyTokenAnn tk) ws) <$> + token ws (\case; TkGt{} -> True; _ -> False) ">" + + <|> + + (\(tk, ws) -> GtEq (pyTokenAnn tk) ws) <$> + token ws (\case; TkGte{} -> True; _ -> False) ">=" + + <|> + + (\(tk, ws) -> NotEq (pyTokenAnn tk) ws) <$> + token ws (\case; TkBangEq{} -> True; _ -> False) "!=" + + comparison = binOp compOp $ orExpr ws + +yieldExpr :: MonadParsec e PyTokens m => m Whitespace -> m (Expr SrcInfo) +yieldExpr ws = + (\(tk, s) -> either (uncurry $ YieldFrom (pyTokenAnn tk) s) (Yield (pyTokenAnn tk) s)) <$> + token ws (\case; TkYield{} -> True; _ -> False) "yield" <*> + (fmap Left + ((,) <$> + (snd <$> token ws (\case; TkFrom{} -> True; _ -> False) "from") <*> + expr ws) + <|> + Right <$> commaSep ws (expr ws)) + +lambda :: MonadParsec e PyTokens m => m Whitespace -> m (Expr SrcInfo) +lambda ws = + (\(tk, s) -> Lambda (pyTokenAnn tk) s) <$> + token ws (\case; TkLambda{} -> True; _ -> False) "lambda" <*> + untypedParams ws <*> + (MkColon . snd <$> token ws (\case; TkColon{} -> True; _ -> False) ":") <*> + expr ws + +lambdaNoCond :: MonadParsec e PyTokens m => m Whitespace -> m (Expr SrcInfo) +lambdaNoCond ws = + (\(tk, s) -> Lambda (pyTokenAnn tk) s) <$> + token ws (\case; TkLambda{} -> True; _ -> False) "lambda" <*> + untypedParams ws <*> + (MkColon . snd <$> token ws (\case; TkColon{} -> True; _ -> False) ":") <*> + exprNoCond ws + +exprNoCond :: MonadParsec e PyTokens m => m Whitespace -> m (Expr SrcInfo) +exprNoCond ws = orTest ws <|> lambdaNoCond ws + +expr :: MonadParsec e PyTokens m => m Whitespace -> m (Expr SrcInfo) +expr ws = + (\a -> maybe a (\(b, c, d, e) -> Ternary (a ^. exprAnn) a b c d e)) <$> + orTest ws <*> + optional + ((,,,) <$> + (snd <$> token ws (\case; TkIf{} -> True; _ -> False) "if") <*> + orTest ws <*> + (snd <$> token ws (\case; TkElse{} -> True; _ -> False) "else") <*> + expr ws) + <|> + lambda ws + +rightParen + :: MonadParsec e PyTokens m + => m Whitespace + -> m (PyToken SrcInfo, [Whitespace]) +rightParen sp = token sp (\case; TkRightParen{} -> True; _ -> False) ")" + +doubleStar + :: MonadParsec e PyTokens m + => m Whitespace + -> m (PyToken SrcInfo, [Whitespace]) +doubleStar sp = token sp (\case; TkDoubleStar{} -> True; _ -> False) "**" + +orExpr :: MonadParsec e PyTokens m => m Whitespace -> m (Expr SrcInfo) +orExpr ws = + binOp + ((\(tk, ws) -> BitOr (pyTokenAnn tk) ws) <$> + token ws (\case; TkPipe{} -> True; _ -> False) "|") + xorExpr + where + xorExpr = + binOp + ((\(tk, ws) -> BitXor (pyTokenAnn tk) ws) <$> + token ws (\case; TkCaret{} -> True; _ -> False) "^") + andExpr + + andExpr = + binOp + ((\(tk, ws) -> BitAnd (pyTokenAnn tk) ws) <$> + token ws (\case; TkAmpersand{} -> True; _ -> False) "&") + shiftExpr + + shiftExpr = + binOp + ((\(tk, ws) -> ShiftLeft (pyTokenAnn tk) ws) <$> + token ws (\case; TkShiftLeft{} -> True; _ -> False) "<<" + + <|> + + (\(tk, ws) -> ShiftRight (pyTokenAnn tk) ws) <$> + token ws (\case; TkShiftRight{} -> True; _ -> False) ">>") + arithExpr + + arithOp = + (\(tk, ws) -> Plus (pyTokenAnn tk) ws) <$> + token ws (\case; TkPlus{} -> True; _ -> False) "+" + + <|> + + (\(tk, ws) -> Minus (pyTokenAnn tk) ws) <$> + token ws (\case; TkMinus{} -> True; _ -> False) "-" + + arithExpr = binOp arithOp term + + termOp = + (\(tk, ws) -> Multiply (pyTokenAnn tk) ws) <$> + star ws + + <|> + + (\(tk, ws) -> At (pyTokenAnn tk) ws) <$> + token ws (\case; TkAt{} -> True; _ -> False) "@" + + <|> + + (\(tk, ws) -> Divide (pyTokenAnn tk) ws) <$> + token ws (\case; TkSlash{} -> True; _ -> False) "/" + + <|> + + (\(tk, ws) -> FloorDivide (pyTokenAnn tk) ws) <$> + token ws (\case; TkDoubleSlash{} -> True; _ -> False) "//" + + <|> + + (\(tk, ws) -> Percent (pyTokenAnn tk) ws) <$> + token ws (\case; TkPercent{} -> True; _ -> False) "%" + + term = binOp termOp factor + + factor = + ((\(tk, s) -> let ann = pyTokenAnn tk in UnOp ann (Negate ann s)) <$> + token ws (\case; TkMinus{} -> True; _ -> False) "-" + <|> + (\(tk, s) -> let ann = pyTokenAnn tk in UnOp ann (Positive ann s)) <$> + token ws (\case; TkPlus{} -> True; _ -> False) "+" + <|> + (\(tk, s) -> let ann = pyTokenAnn tk in UnOp ann (Complement ann s)) <$> + token ws (\case; TkTilde{} -> True; _ -> False) "~") <*> factor + <|> + power + + powerOp = + (\(tk, ws) -> Exp (pyTokenAnn tk) ws) <$> + doubleStar ws + + power = + (\a -> maybe a (uncurry $ BinOp (a ^. exprAnn) a)) <$> + atomExpr <*> + optional ((,) <$> powerOp <*> factor) + + subscript = do + mex <- optional $ expr anySpace + case mex of + Nothing -> + SubscriptSlice Nothing <$> + (snd <$> colon anySpace) <*> + optional (expr anySpace) <*> + optional ((,) <$> (snd <$> colon anySpace) <*> optional (expr anySpace)) + Just ex -> do + mws <- optional $ snd <$> colon anySpace + case mws of + Nothing -> pure $ SubscriptExpr ex + Just ws -> + SubscriptSlice (Just ex) ws <$> + optional (expr anySpace) <*> + optional ((,) <$> (snd <$> colon anySpace) <*> optional (expr anySpace)) + + trailer = + (\a b c -> Deref (c ^. exprAnn) c a b) <$> + (snd <$> token ws (\case; TkDot{} -> True; _ -> False) ".") <*> + identifier ws + + <|> + + (\a b c d -> Call (d ^. exprAnn) d a b c) <$> + (snd <$> token anySpace (\case; TkLeftParen{} -> True; _ -> False) "(") <*> + optional (commaSep1' anySpace arg) <*> + (snd <$> rightParen ws) + + <|> + + (\a b c d -> Subscript (d ^. exprAnn) d a b c) <$> + (snd <$> token anySpace (\case; TkLeftBracket{} -> True; _ -> False) "[") <*> + commaSep1' anySpace subscript <*> + (snd <$> token ws (\case; TkRightBracket{} -> True; _ -> False) "]") + + atomExpr = + (\(mAwait, a) b -> + let e = foldl' (&) a b + in maybe e (\(tk, sp) -> Await (pyTokenAnn tk) sp e) mAwait) <$> + try + ((,) <$> + optional (token ws (\case; TkIdent "await" _ -> True; _ -> False) "await") <*> + atom) <*> + many trailer + <|> + foldl' (&) <$> atom <*> many trailer + + parensOrUnit = + (\(tk, s) maybeEx sps -> + case maybeEx of + Nothing -> Unit (pyTokenAnn tk) s sps + Just ex -> Parens (pyTokenAnn tk) s ex sps) <$> + token anySpace (\case; TkLeftParen{} -> True; _ -> False) "(" <*> + optional (yieldExpr anySpace <|> exprListComp anySpace) <*> + (snd <$> rightParen ws) + + list = + (\(tk, sp1) -> + maybe (List (pyTokenAnn tk) sp1 Nothing) (\f -> f (pyTokenAnn tk) sp1)) <$> + token anySpace (\case; TkLeftBracket{} -> True; _ -> False) "[" <*> + optional + ((\e a ann ws1 -> + case a of + Left (cf, cfs) -> ListComp ann ws1 (Comprehension (e ^. exprAnn) e cf cfs) + Right Nothing -> List ann ws1 (Just $ CommaSepOne1' e Nothing) + Right (Just (c, Nothing)) -> List ann ws1 (Just $ CommaSepOne1' e $ Just c) + Right (Just (c, Just cs)) -> List ann ws1 (Just $ CommaSepMany1' e c cs)) <$> + (expr anySpace <|> starExpr anySpace) <*> + (Left <$> + ((,) <$> + compFor <*> + many (Left <$> compFor <|> Right <$> compIf)) <|> + Right <$> + optional + ((,) <$> + (snd <$> comma anySpace) <*> + optional (commaSep1' anySpace (expr anySpace <|> starExpr anySpace))))) <*> + (snd <$> token ws (\case; TkRightBracket{} -> True; _ -> False) "]") + + doubleStarExpr ws = + (\(tk, sp) -> DictUnpack (pyTokenAnn tk) sp) <$> + doubleStar ws <*> + orExpr ws + + dictItem = + (\a -> DictItem (a ^. exprAnn) a) <$> + expr anySpace <*> + (snd <$> colon anySpace) <*> + expr anySpace + <|> + doubleStarExpr anySpace + + compRHS = (,) <$> compFor <*> many (Left <$> compFor <|> Right <$> compIf) + + dictOrSet = do + (a, ws1) <- token anySpace (\case; TkLeftBrace{} -> True; _ -> False) "{" + let ann = pyTokenAnn a + maybeExpr <- + optional $ + Left . Left <$> expr anySpace <|> + Left . Right <$> starExpr anySpace <|> + Right <$> doubleStarExpr anySpace + (case maybeExpr of + Nothing -> pure $ Dict ann ws1 Nothing + Just (Left (Left ex)) -> do + maybeColon <- + optional $ MkColon . snd <$> token anySpace (\case; TkColon{} -> True; _ -> False) ":" + case maybeColon of + Nothing -> + -- The order of this choice matters because commaSepRest is implemented + -- in a slightly odd way + (\(c, d) -> SetComp ann ws1 (Comprehension (ex ^. exprAnn) ex c d)) <$> + compRHS + <|> + + (\(rest, final) -> Set ann ws1 ((ex, rest, final) ^. _CommaSep1')) <$> + commaSepRest (expr anySpace <|> starExpr anySpace) + Just clws -> + (\ex2 a -> + let + dictItemAnn = ex ^. exprAnn + firstDictItem = DictItem dictItemAnn ex clws ex2 + in + case a of + Left (c, d) -> + DictComp ann ws1 (Comprehension dictItemAnn firstDictItem c d) + Right (rest, final) -> + Dict ann ws1 (Just $ (firstDictItem, rest, final) ^. _CommaSep1')) <$> + expr anySpace <*> + (Left <$> compRHS <|> Right <$> commaSepRest dictItem) + Just (Left (Right ex)) -> + ((\(c, d) -> SetComp ann ws1 (Comprehension (ex ^. exprAnn) ex c d)) <$> + compRHS + + <|> + + (\(rest, final) -> Set ann ws1 ((ex, rest, final) ^. _CommaSep1')) <$> + commaSepRest (expr anySpace <|> starExpr anySpace)) + Just (Right ex) -> + ((\(c, d) -> DictComp ann ws1 (Comprehension (_dictItemAnn ex) ex c d)) <$> + compRHS + + <|> + + (\(rest, final) -> Dict ann ws1 (Just $ (ex, rest, final) ^. _CommaSep1')) <$> + commaSepRest dictItem)) <*> + + (snd <$> token ws (\case; TkRightBrace{} -> True; _ -> False) "}") + + atom = + dictOrSet <|> + list <|> + none ws <|> + bool ws <|> + ellipsis ws <|> + integer ws <|> + float ws <|> + imag ws <|> + stringOrBytes ws <|> + Ident <$> identifier ws <|> + parensOrUnit + +simpleStatement :: MonadParsec e PyTokens m => m (SimpleStatement SrcInfo) +simpleStatement = + returnSt <|> + passSt <|> + breakSt <|> + continueSt <|> + globalSt <|> + nonlocalSt <|> + delSt <|> + importSt <|> + raiseSt <|> + exprOrAssignSt <|> + yieldSt <|> + assertSt + where + assertSt = + (\(tk, s) -> Assert (pyTokenAnn tk) s) <$> + token space (\case; TkAssert{} -> True; _ -> False) "assert" <*> + expr space <*> + optional ((,) <$> (snd <$> comma space) <*> expr space) + + yieldSt = (\a -> Expr (a ^. exprAnn) a) <$> yieldExpr space + + returnSt = + (\(tkReturn, retSpaces) -> Return (pyTokenAnn tkReturn) retSpaces) <$> + token space (\case; TkReturn{} -> True; _ -> False) "return" <*> + optional (exprList space) + + passSt = + uncurry (Pass . pyTokenAnn) <$> + token space (\case; TkPass{} -> True; _ -> False) "pass" + + breakSt = + uncurry (Break . pyTokenAnn) <$> + token space (\case; TkBreak{} -> True; _ -> False) "break" + + continueSt = + uncurry (Continue . pyTokenAnn) <$> + token space (\case; TkContinue{} -> True; _ -> False) "continue" + + augAssign = + (\(tk, s) -> MkAugAssign PlusEq (pyTokenAnn tk) s) <$> + token space (\case; TkPlusEq{} -> True; _ -> False) "+=" + + <|> + + (\(tk, s) -> MkAugAssign MinusEq (pyTokenAnn tk) s) <$> + token space (\case; TkMinusEq{} -> True; _ -> False) "-=" + + <|> + + (\(tk, s) -> MkAugAssign AtEq (pyTokenAnn tk) s) <$> + token space (\case; TkAtEq{} -> True; _ -> False) "@=" + + <|> + + (\(tk, s) -> MkAugAssign StarEq (pyTokenAnn tk) s) <$> + token space (\case; TkStarEq{} -> True; _ -> False) "*=" + + <|> + + (\(tk, s) -> MkAugAssign SlashEq (pyTokenAnn tk) s) <$> + token space (\case; TkSlashEq{} -> True; _ -> False) "/=" + + <|> + + (\(tk, s) -> MkAugAssign PercentEq (pyTokenAnn tk) s) <$> + token space (\case; TkPercentEq{} -> True; _ -> False) "%=" + + <|> + + (\(tk, s) -> MkAugAssign AmpersandEq (pyTokenAnn tk) s) <$> + token space (\case; TkAmpersandEq{} -> True; _ -> False) "&=" + + <|> + + (\(tk, s) -> MkAugAssign PipeEq (pyTokenAnn tk) s) <$> + token space (\case; TkPipeEq{} -> True; _ -> False) "|=" + + <|> + + (\(tk, s) -> MkAugAssign CaretEq (pyTokenAnn tk) s) <$> + token space (\case; TkCaretEq{} -> True; _ -> False) "^=" + + <|> + + (\(tk, s) -> MkAugAssign ShiftLeftEq (pyTokenAnn tk) s) <$> + token space (\case; TkShiftLeftEq{} -> True; _ -> False) "<<=" + + <|> + + (\(tk, s) -> MkAugAssign ShiftRightEq (pyTokenAnn tk) s) <$> + token space (\case; TkShiftRightEq{} -> True; _ -> False) ">>=" + + <|> + + (\(tk, s) -> MkAugAssign DoubleStarEq (pyTokenAnn tk) s) <$> + token space (\case; TkDoubleStarEq{} -> True; _ -> False) "**=" + + <|> + + (\(tk, s) -> MkAugAssign DoubleSlashEq (pyTokenAnn tk) s) <$> + token space (\case; TkDoubleSlashEq{} -> True; _ -> False) "//=" + + exprOrAssignSt = + (\a -> + maybe + (Expr (a ^. exprAnn) a) + (either + (Assign (a ^. exprAnn) a) + (uncurry $ AugAssign (a ^. exprAnn) a))) <$> + exprOrStarList space <*> + optional + (Left <$> + some1 + ((,) <$> + (snd <$> equals space) <*> + (yieldExpr space <|> exprOrStarList space)) + + <|> + + Right <$> ((,) <$> augAssign <*> (yieldExpr space <|> exprList space))) + + globalSt = + (\(tk, s) -> Global (pyTokenAnn tk) $ NonEmpty.fromList s) <$> + token space (\case; TkGlobal{} -> True; _ -> False) "global" <*> + commaSep1 space (identifier space) + + nonlocalSt = + (\(tk, s) -> Nonlocal (pyTokenAnn tk) $ NonEmpty.fromList s) <$> + token space (\case; TkNonlocal{} -> True; _ -> False) "nonlocal" <*> + commaSep1 space (identifier space) + + delSt = + (\(tk, s) -> Del (pyTokenAnn tk) s) <$> + token space (\case; TkDel{} -> True; _ -> False) "del" <*> + commaSep1' space (orExpr space) + + raiseSt = + (\(tk, s) -> Raise (pyTokenAnn tk) s) <$> + token space (\case; TkRaise{} -> True; _ -> False) "raise" <*> + optional + ((,) <$> + expr space <*> + optional + ((,) <$> + (snd <$> token space (\case; TkFrom{} -> True; _ -> False) "from") <*> + expr space)) + + importSt = importName <|> importFrom + where + moduleName = + makeModuleName <$> + identifier space <*> + many + ((,) <$> + (snd <$> token space (\case; TkDot{} -> True; _ -> False) ".") <*> + identifier space) + + importAs ws getAnn p = + (\a -> ImportAs (getAnn a) a) <$> + p <*> + optional + ((,) <$> + (NonEmpty.fromList . snd <$> token ws (\case; TkAs{} -> True; _ -> False) "as") <*> + identifier ws) + + importName = + (\(tk, s) -> Import (pyTokenAnn tk) $ NonEmpty.fromList s) <$> + token space (\case; TkImport{} -> True; _ -> False) "import" <*> + commaSep1 space (importAs space _moduleNameAnn moduleName) + + dots = + fmap concat . some $ + pure . snd <$> dot space + + <|> + + (\(_, ws) -> [MkDot [], MkDot [], MkDot ws]) <$> + token space (\case; TkEllipsis{} -> True; _ -> False) "..." + + relativeModuleName = + RelativeWithName [] <$> moduleName + + <|> + + (\a -> maybe (Relative $ NonEmpty.fromList a) (RelativeWithName a)) <$> + dots <*> + optional moduleName + + importTargets = + (\(tk, s) -> ImportAll (pyTokenAnn tk) s) <$> + star space + + <|> + + (\(tk, s) -> ImportSomeParens (pyTokenAnn tk) s) <$> + token anySpace (\case; TkLeftParen{} -> True; _ -> False) "(" <*> + commaSep1' anySpace (importAs anySpace _identAnn (identifier anySpace)) <*> + (snd <$> rightParen space) + + <|> + + (\a -> ImportSome (commaSep1Head a ^. importAsAnn) a) <$> + commaSep1 space (importAs space _identAnn (identifier space)) + + importFrom = + (\(tk, s) -> From (pyTokenAnn tk) s) <$> + token space (\case; TkFrom{} -> True; _ -> False) "from" <*> + relativeModuleName <*> + (snd <$> token space (\case; TkImport{} -> True; _ -> False) "import") <*> + importTargets + +sepBy1' :: MonadParsec e PyTokens m => m a -> m sep -> m (a, [(sep, a)], Maybe sep) +sepBy1' val sep = go + where + go = + (\a b -> + case b of + Nothing -> (a, [], Nothing) + Just (sc, b') -> + case b' of + Nothing -> (a, [], Just sc) + Just (a', ls, sc') -> (a, (sc, a') : ls, sc')) <$> + val <*> + optional ((,) <$> sep <*> optional go) + +smallStatement + :: MonadParsec e PyTokens m + => m (SmallStatement SrcInfo) +smallStatement = + (\(a, b, c) d -> MkSmallStatement a b c d) <$> + sepBy1' simpleStatement (snd <$> semicolon space) <*> + optional comment <*> + optional eol + +statement + :: (Alternative m, MonadParsec e PyTokens m) + => m (Indents SrcInfo) + -> Indents SrcInfo + -> m (Statement SrcInfo) +statement pIndent indentBefore = + -- It's important to parse compound statements first, because the 'async' keyword + -- is actually an identifier and we'll have to backtrack + CompoundStatement <$> compoundStatement pIndent indentBefore <|> + SmallStatement indentBefore <$> smallStatement + +blank :: MonadParsec e PyTokens m => m (Blank SrcInfo) +blank = + withSrcInfo $ + (\b c a -> Blank a b c) <$> + some space <*> + optional comment + + <|> + + (\b a -> Blank a [] b) <$> optional comment + +suite :: MonadParsec e PyTokens m => m (Suite SrcInfo) +suite = + (\(tk, s) -> + either + (SuiteOne (pyTokenAnn tk) s) + (\(a, b,c ) -> SuiteMany (pyTokenAnn tk) s a b c)) <$> + colon space <*> + (Left <$> smallStatement + + <|> + + (fmap Right $ + (,,) <$> + optional comment <*> + eol <*> + (Block <$> + many ((,) <$> blank <*> eol) <*> + (statement level =<< indent) <*> + many (line level)) <* + dedent)) + where + + line i = + Left <$> ((,) <$> blank <*> eol) <|> + Right <$> (statement level =<< i) + +commaSep :: MonadParsec e PyTokens m => m Whitespace -> m a -> m (CommaSep a) +commaSep ws pa = + (\a -> maybe (CommaSepOne a) (uncurry $ CommaSepMany a)) <$> + pa <*> + optional ((,) <$> (snd <$> comma ws) <*> commaSep ws pa) + + <|> + + pure CommaSepNone + +commaSep1 :: MonadParsec e PyTokens m => m Whitespace -> m a -> m (CommaSep1 a) +commaSep1 ws val = go + where + go = + (\a -> maybe (CommaSepOne1 a) (uncurry $ CommaSepMany1 a)) <$> + val <*> + optional ((,) <$> (snd <$> comma ws) <*> go) + +commaSep1' :: MonadParsec e PyTokens m => m Whitespace -> m a -> m (CommaSep1' a) +commaSep1' ws pa = + (\(a, b, c) -> from a b c) <$> sepBy1' pa (snd <$> comma ws) + where + from a [] b = CommaSepOne1' a b + from a ((b, c) : bs) d = CommaSepMany1' a b $ from c bs d + +someParams + :: MonadParsec e PyTokens m + => m (Param SrcInfo) + -> m (Param SrcInfo) + -> m (Param SrcInfo) + -> m (CommaSep (Param SrcInfo)) +someParams paramPositional paramStar paramDoubleStar = + fmap (view _CommaSep) . optional $ + + (\a b c -> + case c of + Just (d, e) -> + case e of + Nothing -> (a, b, Just d) + Just f -> + case f of + Left (g, h, i) -> (a, b ++ (d, g) : maybe h (snoc h) i, Nothing) + Right g -> (a, snoc b (d, g), Nothing) + Nothing -> (a, b, Nothing)) <$> + + paramPositional <*> + + many commaPositional <*> + + optional + ((,) <$> + (snd <$> comma anySpace) <*> + optional + (Left <$> + ((,,) <$> paramStar <*> many commaPositional <*> optional commaDoubleStar) + + <|> + + Right <$> paramDoubleStar)) + + <|> + + (\a b -> (a, b, Nothing)) <$> + paramStar <*> + ((\a -> maybe a (a `snoc`)) <$> + many commaPositional <*> + optional commaDoubleStar) + + <|> + + (\a -> (a, [], Nothing)) <$> paramDoubleStar + + where + commaPositional = + try + ((,) <$> + fmap snd (comma anySpace) <* + notFollowedBy + (star anySpace <|> + doubleStar anySpace <|> + rightParen space)) <*> + paramPositional + + commaDoubleStar = + (,) <$> (snd <$> comma anySpace) <*> paramDoubleStar + +upPositional :: MonadParsec e PyTokens m => m Whitespace -> m (Param SrcInfo) +upPositional ws = + (\a -> + maybe + (PositionalParam (_identAnn a) a Nothing) + (uncurry $ KeywordParam (_identAnn a) a Nothing)) <$> + identifier ws <*> + optional + ((,) <$> + (snd <$> token ws (\case; TkEq{} -> True; _ -> False) "=") <*> + expr ws) + +upStar :: MonadParsec e PyTokens m => m Whitespace -> m (Param SrcInfo) +upStar ws = + (\(a, b) -> + maybe + (UnnamedStarParam (pyTokenAnn a) b) + (uncurry $ StarParam (pyTokenAnn a) b)) <$> + star ws <*> + optional ((\a -> (a, Nothing)) <$> identifier ws) + +upDoubleStar :: MonadParsec e PyTokens m => m Whitespace -> m (Param SrcInfo) +upDoubleStar ws = + (\(a, b) c -> DoubleStarParam (pyTokenAnn a) b c Nothing) <$> + doubleStar ws <*> + identifier ws + +untypedParams + :: MonadParsec e PyTokens m + => m Whitespace + -> m (CommaSep (Param SrcInfo)) +untypedParams ws = someParams (upPositional ws) (upStar ws) (upDoubleStar ws) + +tyAnn :: MonadParsec e PyTokens m => m (Colon, Expr SrcInfo) +tyAnn = + (,) <$> + (MkColon . snd <$> token anySpace (\case; TkColon{} -> True; _ -> False) ":") <*> + expr anySpace + +tpPositional :: MonadParsec e PyTokens m => m (Param SrcInfo) +tpPositional = + (\a b -> + maybe + (PositionalParam (_identAnn a) a b) + (uncurry $ KeywordParam (_identAnn a) a b)) <$> + identifier anySpace <*> + optional tyAnn <*> + optional + ((,) <$> + (snd <$> token anySpace (\case; TkEq{} -> True; _ -> False) "=") <*> + expr anySpace) + +tpStar :: MonadParsec e PyTokens m => m (Param SrcInfo) +tpStar = + (\(a, b) -> + maybe + (UnnamedStarParam (pyTokenAnn a) b) + (uncurry $ StarParam (pyTokenAnn a) b)) <$> + star anySpace <*> + optional ((,) <$> identifier anySpace <*> optional tyAnn) + +tpDoubleStar :: MonadParsec e PyTokens m => m (Param SrcInfo) +tpDoubleStar = + (\(a, b) -> DoubleStarParam (pyTokenAnn a) b) <$> + doubleStar anySpace <*> + identifier anySpace <*> + optional tyAnn + +typedParams :: MonadParsec e PyTokens m => m (CommaSep (Param SrcInfo)) +typedParams = someParams tpPositional tpStar tpDoubleStar + +arg :: MonadParsec e PyTokens m => m (Arg SrcInfo) +arg = + (do + e <- exprComp anySpace + case e of + Ident ident -> do + eqSpaces <- + optional $ snd <$> token anySpace (\case; TkEq{} -> True; _ -> False) "=" + case eqSpaces of + Nothing -> pure $ PositionalArg (e ^. exprAnn) e + Just s -> KeywordArg (e ^. exprAnn) ident s <$> expr anySpace + _ -> pure $ PositionalArg (e ^. exprAnn) e) + + <|> + + (\a -> PositionalArg (a ^. exprAnn) a) <$> expr anySpace + + <|> + + (\(a, b) -> StarArg (pyTokenAnn a) b) <$> + star anySpace <*> + expr anySpace + + <|> + + (\(a, b) -> DoubleStarArg (pyTokenAnn a) b) <$> + doubleStar anySpace <*> + expr anySpace + +decoratorValue :: MonadParsec e PyTokens m => m (Expr SrcInfo) +decoratorValue = do + id1 <- identifier space + ids <- + many + ((,) <$> + (snd <$> token space (\case; TkDot{} -> True; _ -> False) ".") <*> + identifier space) + args <- + optional $ + (,,) <$> + (snd <$> token anySpace (\case; TkLeftParen{} -> True; _ -> False) "(") <*> + optional (commaSep1' anySpace arg) <*> + (snd <$> rightParen space) + let + derefs = + foldl + (\b (ws, a) -> Deref (b ^. exprAnn) b ws a) + (Ident id1) + ids + pure $ + case args of + Nothing -> derefs + Just (l, x, r) -> Call (derefs ^. exprAnn) derefs l x r + +decorator + :: MonadParsec e PyTokens m + => Indents SrcInfo + -> m (Decorator SrcInfo) +decorator indentBefore = + (\(tk, spcs) a b -> Decorator (pyTokenAnn tk) indentBefore spcs a b) <$> + at space <*> + decoratorValue <*> + optional comment <*> + eol <*> + many ((,) <$> blank <*> eol) + +decorators + :: MonadParsec e PyTokens m + => m (Indents SrcInfo) + -> Indents SrcInfo + -> m [Decorator SrcInfo] +decorators pIndent indentBefore = + (:) <$> + decorator indentBefore <*> + many (try i >>= decorator) + where + i = + pIndent <* + lookAhead (token space (\case; TkAt{} -> True; _ -> False) "@") + +compoundStatement + :: MonadParsec e PyTokens m + => m (Indents SrcInfo) + -> Indents SrcInfo + -> m (CompoundStatement SrcInfo) +compoundStatement pIndent indentBefore = + ifSt <|> + whileSt <|> + trySt <|> + decorated <|> + asyncSt <|> + classSt indentBefore [] <|> + fundef indentBefore Nothing [] <|> + withSt Nothing <|> + forSt Nothing + where + decorated = do + ds <- decorators pIndent indentBefore + i <- pIndent + (do; a <- doAsync; fundef i (Just a) ds) <|> + fundef i Nothing ds <|> + classSt i ds + + classSt ib decs = + (\(tk, s) a b c -> + ClassDef + (pyTokenAnn tk) + decs + ib + (NonEmpty.fromList s) a b c) <$> + token space (\case; TkClass{} -> True; _ -> False) "class" <*> + identifier space <*> + optional + ((,,) <$> + (snd <$> token anySpace (\case; TkLeftParen{} -> True; _ -> False) "(") <*> + optional (commaSep1' anySpace arg) <*> + (snd <$> rightParen space)) <*> + suite + + ifSt = + (\(tk, s) a b c d -> If (pyTokenAnn tk) indentBefore s a b c d) <$> + token space (\case; TkIf{} -> True; _ -> False) "if" <*> + expr space <*> + suite <*> + many + (try + ((,,,) <$> + pIndent <*> + (snd <$> token space (\case; TkElif{} -> True; _ -> False) "elif")) <*> + expr space <*> + suite) <*> + optional + (try + ((,,) <$> + pIndent <*> + (snd <$> token space (\case; TkElse{} -> True; _ -> False) "else")) <*> + suite) + + whileSt = + (\(tk, s) a b -> While (pyTokenAnn tk) indentBefore s a b) <$> + token space (\case; TkWhile{} -> True; _ -> False) "while" <*> + expr space <*> + suite <*> + optional + (try + ((,,) <$> + pIndent <*> + (snd <$> token space (\case; TkElse{} -> True; _ -> False) "else")) <*> + suite) + + exceptAs = + (\a -> ExceptAs (a ^. exprAnn) a) <$> + expr space <*> + optional + ((,) <$> + (snd <$> token space (\case; TkAs{} -> True; _ -> False) "as") <*> + identifier space) + + trySt = + (\(tk, s) a d -> + case d of + Left (e, f, g) -> TryFinally (pyTokenAnn tk) indentBefore s a e f g + Right (e, f, g) -> TryExcept (pyTokenAnn tk) indentBefore s a e f g) <$> + token space (\case; TkTry{} -> True; _ -> False) "try" <*> + suite <*> + (fmap Left + (try + ((,,) <$> + pIndent <*> + (snd <$> token space (\case; TkFinally{} -> True; _ -> False) "finally")) <*> + suite) + + <|> + + fmap Right + ((,,) <$> + some1 + (try + ((,,,) <$> + pIndent <*> + (snd <$> token space (\case; TkExcept{} -> True; _ -> False) "except")) <*> + optional exceptAs <*> + suite) <*> + optional + (try + ((,,) <$> + pIndent <*> + (snd <$> token space (\case; TkElse{} -> True; _ -> False) "else")) <*> + suite) <*> + optional + (try + ((,,) <$> + pIndent <*> + (snd <$> token space (\case; TkFinally{} -> True; _ -> False) "finally")) <*> + suite))) + + doAsync = token space (\case; TkIdent "async" _ -> True; _ -> False) "async" + + asyncSt = do + a <- + try $ + doAsync <* + lookAhead + (token space (\case; TkDef{} -> True; _ -> False) "def" <|> + token space (\case; TkWith{} -> True; _ -> False) "with" <|> + token space (\case; TkFor{} -> True; _ -> False) "for") + fundef indentBefore (Just a) [] <|> + withSt (Just a) <|> + forSt (Just a) + + fundef ib async decs = + (\(tkDef, defSpaces) a b c d e f -> + Fundef + (maybe (pyTokenAnn tkDef) (pyTokenAnn . fst) async) + decs + ib + (NonEmpty.fromList . snd <$> async) + (NonEmpty.fromList defSpaces) + a b c d e f) <$> + token space (\case; TkDef{} -> True; _ -> False) "def" <*> + identifier space <*> + fmap snd (token anySpace (\case; TkLeftParen{} -> True; _ -> False) "(") <*> + typedParams <*> + fmap snd (rightParen space) <*> + optional + ((,) <$> + (snd <$> token space (\case; TkRightArrow{} -> True; _ -> False) "->") <*> + expr space) <*> + suite + + withSt async = + (\(tk, s) a b -> + With + (maybe (pyTokenAnn tk) (pyTokenAnn . fst) async) + indentBefore + (NonEmpty.fromList . snd <$> async) + s a b) <$> + token space (\case; TkWith{} -> True; _ -> False) "with" <*> + commaSep1 + space + ((\a -> WithItem (a ^. exprAnn) a) <$> + expr space <*> + optional + ((,) <$> + (snd <$> token space (\case; TkAs{} -> True; _ -> False) "as") <*> + orExpr space)) <*> + suite + + forSt async = + (\(tk, s) a b c d e -> + For + (maybe (pyTokenAnn tk) (pyTokenAnn . fst) async) + indentBefore + (NonEmpty.fromList . snd <$> async) + s a b c d e) <$> + token space (\case; TkFor{} -> True; _ -> False) "for" <*> + orExprList space <*> + (snd <$> token space (\case; TkIn{} -> True; _ -> False) "in") <*> + commaSep1' space (expr space) <*> + suite <*> + optional + (try + ((,,) <$> + pIndent <*> + (snd <$> token space (\case; TkElse{} -> True; _ -> False) "else")) <*> + suite) + +module_ :: MonadParsec e PyTokens m => m (Module SrcInfo) +module_ = + ModuleStatement <$> (statement tlIndent =<< tlIndent) <*> module_ + + <|> + + (\bl rest -> + case rest of + Left (nl, md) -> ModuleBlank bl nl md + Right{} -> ModuleBlankFinal bl) <$> + blank <*> + (Left <$> ((,) <$> newline <*> module_) <|> Right <$> eof) + + <|> + + ModuleEmpty <$ eof + + where + tlIndent = level <|> withSrcInfo (pure $ Indents []) diff --git a/src/Language/Python/Internal/Render.hs b/src/Language/Python/Internal/Render.hs new file mode 100644 --- /dev/null +++ b/src/Language/Python/Internal/Render.hs @@ -0,0 +1,1639 @@ +{-# language GeneralizedNewtypeDeriving #-} +{-# language FlexibleInstances, MultiParamTypeClasses #-} +{-# language OverloadedStrings #-} + +{-| +Module : Language.Python.Internal.Render +Copyright : (C) CSIRO 2017-2018 +License : BSD3 +Maintainer : Isaac Elliott +Stability : experimental +Portability : non-portable +-} + +module Language.Python.Internal.Render + ( -- * Common Functions + showModule, showStatement, showExpr + -- * Rendering + , RenderOutput, showRenderOutput, singleton + , renderModule, renderStatement, renderExpr + -- * Miscellany + , showQuoteType, showStringPrefix, showBytesPrefix, showToken, showTokens + , expandIndents, whitespaceTokens, commentTokens + , parens, braces, brackets + , renderWhitespace, renderCommaSep, renderCommaSep1, renderCommaSep1' + , renderIdent, renderComment, renderModuleName, renderDot, renderRelativeModuleName + , renderImportAs, renderImportTargets, renderSimpleStatement, renderCompoundStatement + , renderBlock, renderIndent, renderIndents, renderExceptAs, renderArg, renderParam + , renderParams, renderCompFor, renderCompIf, renderComprehension, renderBinOp, renderUnOp + , renderSubscript, renderPyChars, escapeChars, intToHex + ) +where + +import Control.Lens.Cons (_init, _last) +import Control.Lens.Fold ((^..), folded, traverseOf_) +import Control.Lens.Getter ((^.)) +import Control.Lens.Review ((#)) +import Control.Lens.Setter ((.~)) +import Control.Monad.Writer.Strict (Writer, execWriter, writer) +import Control.Monad.Reader (ReaderT, runReaderT, local, ask) +import Data.Bifoldable (bitraverse_, bitraverse_) +import Data.Char (ord) +import Data.Digit.Char (charHeXaDeCiMaL, charOctal) +import Data.Digit.Hexadecimal.MixedCase (HeXDigit(..)) +import Data.DList (DList) +import Data.Function ((&)) +import Data.Foldable (toList, traverse_) +import Data.Maybe (isNothing) +import Data.Semigroup (Semigroup(..)) +import Data.Text (Text) + +import qualified Data.DList as DList +import qualified Data.List.NonEmpty as NonEmpty +import qualified Data.Text as Text +import qualified Data.Text.Lazy as Lazy +import qualified Data.Text.Lazy.Builder as Builder + +import Language.Python.Internal.Render.Correction +import Language.Python.Internal.Token (PyToken(..)) +import Language.Python.Syntax.AugAssign +import Language.Python.Syntax.CommaSep +import Language.Python.Syntax.Comment +import Language.Python.Syntax.Expr +import Language.Python.Syntax.Ident +import Language.Python.Syntax.Import +import Language.Python.Syntax.Module +import Language.Python.Syntax.ModuleNames +import Language.Python.Syntax.Numbers +import Language.Python.Syntax.Operator.Binary +import Language.Python.Syntax.Operator.Unary +import Language.Python.Syntax.Punctuation +import Language.Python.Syntax.Statement +import Language.Python.Syntax.Strings +import Language.Python.Syntax.Whitespace + +-- | A 'RenderOutput' is an intermediate form used during rendering +-- with efficient concatenation +newtype RenderOutput a + = RenderOutput + { unRenderOutput + :: ReaderT + -- Is the thing we're rendering followed by an optional + -- newline? (as opposed to mandatory newline) + -- + -- This is because the AST may be missing critical newlines + -- and we supply them during rendering + Bool + (Writer (DList (PyToken ()))) + a + } deriving (Functor, Applicative, Monad) + +-- | Treats the input as a terminating statement (does not cause additional newlines to +-- be inserted) +final :: RenderOutput a -> RenderOutput a +final = id + +-- | Treats the input as a non-terminating statement (causes additional newlines to be +-- inserted) +notFinal :: RenderOutput a -> RenderOutput a +notFinal (RenderOutput a) = RenderOutput $ local (const False) a + +-- | Are we inside a terminating or non-terminating context? +isFinal :: RenderOutput Bool +isFinal = RenderOutput ask + +-- | Render a single token as a 'RenderOutput' +singleton :: PyToken () -> RenderOutput () +singleton a = RenderOutput $ writer ((), DList.singleton a) + +-- | Run a 'RenderOutput' to produce a final 'Text'. +-- +-- These 'Text's should then not be appended any more. All appending should +-- be done during the 'RenderOutput' phase. +showRenderOutput :: RenderOutput a -> Text +showRenderOutput = + Lazy.toStrict . + Builder.toLazyText . + foldMap (Builder.fromText . showToken) . + correctSpaces showToken . + correctNewlines . + DList.toList . + execWriter . + flip runReaderT True . + unRenderOutput + +renderComment :: Comment a -> RenderOutput () +renderComment = traverse_ singleton . commentTokens + +commentTokens :: Comment a -> [PyToken ()] +commentTokens c = [TkComment $ () <$ c] + +showComment :: Comment a -> Text +showComment (MkComment _ s) = Text.pack $ "#" <> s + +between :: RenderOutput l -> RenderOutput r -> RenderOutput a -> RenderOutput a +between l r m = l *> m <* r + +parens :: RenderOutput a -> RenderOutput a +parens = between (singleton $ TkLeftParen ()) (singleton $ TkRightParen ()) + +brackets :: RenderOutput a -> RenderOutput a +brackets = between (singleton $ TkLeftBracket ()) (singleton $ TkRightBracket ()) + +braces :: RenderOutput a -> RenderOutput a +braces = between (singleton $ TkLeftBrace ()) (singleton $ TkRightBrace ()) + +-- | Parenthesise a term, but put its trailing whitespace *outside* the parens +parensDistTWS + :: HasTrailingWhitespace s + => (s -> RenderOutput ()) + -> s -> RenderOutput () +parensDistTWS f a = do + parens $ f (a & trailingWhitespace .~ []) + traverse_ renderWhitespace (a ^. trailingWhitespace) + +parensTuple :: Expr v a -> RenderOutput () +parensTuple e = + case e of + Tuple{} -> parensDistTWS renderExpr e + _ -> renderExpr e + +parensGenerator :: Expr v a -> RenderOutput () +parensGenerator e = + case e of + Generator{} -> parensDistTWS renderExpr e + _ -> renderExpr e + +parensTupleGenerator :: Expr v a -> RenderOutput () +parensTupleGenerator e = + case e of + Tuple{} -> parensDistTWS renderExpr e + Generator{} -> parensDistTWS renderExpr e + _ -> renderExpr e + +escapeChars :: [(Char, Char)] +escapeChars = + [ ('\\', '\\') + , ('"', '"') + , ('\a', 'a') + , ('\b', 'b') + , ('\f', 'f') + , ('\n', 'n') + , ('\r', 'r') + , ('\t', 't') + , ('\v', 'v') + ] + +intToHex :: Int -> Text +intToHex n = Text.pack $ go n [] + where + go 0 = (++"0") + go 1 = (++"1") + go 2 = (++"2") + go 3 = (++"3") + go 4 = (++"4") + go 5 = (++"5") + go 6 = (++"6") + go 7 = (++"7") + go 8 = (++"8") + go 9 = (++"9") + go 10 = (++"A") + go 11 = (++"B") + go 12 = (++"C") + go 13 = (++"D") + go 14 = (++"E") + go 15 = (++"F") + go b = let (q, r) = quotRem b 16 in go r . go q + +intToHexH :: Int -> [HeXDigit] +intToHexH n = go n [] + where + go 0 = (++[HeXDigit0]) + go 1 = (++[HeXDigit1]) + go 2 = (++[HeXDigit2]) + go 3 = (++[HeXDigit3]) + go 4 = (++[HeXDigit4]) + go 5 = (++[HeXDigit5]) + go 6 = (++[HeXDigit6]) + go 7 = (++[HeXDigit7]) + go 8 = (++[HeXDigit8]) + go 9 = (++[HeXDigit9]) + go 10 = (++[HeXDigitA]) + go 11 = (++[HeXDigitB]) + go 12 = (++[HeXDigitC]) + go 13 = (++[HeXDigitD]) + go 14 = (++[HeXDigitE]) + go 15 = (++[HeXDigitF]) + go b = let (q, r) = quotRem b 16 in go r . go q + +renderPyCharsWithCorrection + :: (QuoteType -> StringType -> [PyChar] -> [PyChar]) + -> QuoteType + -> StringType + -> [PyChar] -> Text +renderPyCharsWithCorrection c qt st = Text.pack . go . c qt st + where + go s = + case s of + [] -> "" + Char_newline : cs -> "\\newline" <> go cs + Char_octal1 a : cs -> + "\\" <> + [charOctal # a] <> + go cs + Char_octal2 a b : cs -> + "\\" <> + [charOctal # a, charOctal # b] <> + go cs + Char_octal3 a b c : cs -> + "\\" <> + [charOctal # a, charOctal # b, charOctal # c] <> + go cs + Char_hex a b : cs -> + "\\x" <> [charHeXaDeCiMaL # a, charHeXaDeCiMaL # b] <> go cs + Char_uni16 a b c d : cs -> + "\\u" <> + [ charHeXaDeCiMaL # a + , charHeXaDeCiMaL # b + , charHeXaDeCiMaL # c + , charHeXaDeCiMaL # d + ] <> + go cs + Char_uni32 a b c d e f g h : cs -> + "\\u" <> + [ charHeXaDeCiMaL # a + , charHeXaDeCiMaL # b + , charHeXaDeCiMaL # c + , charHeXaDeCiMaL # d + , charHeXaDeCiMaL # e + , charHeXaDeCiMaL # f + , charHeXaDeCiMaL # g + , charHeXaDeCiMaL # h + ] <> + go cs + Char_esc_bslash : cs -> '\\' : '\\' : go cs + Char_esc_singlequote : cs -> '\\' : '\'' : go cs + Char_esc_doublequote : cs -> '\\' : '"' : go cs + Char_esc_a : cs -> '\\' : 'a' : go cs + Char_esc_b : cs -> '\\' : 'b' : go cs + Char_esc_f : cs -> '\\' : 'f' : go cs + Char_esc_n : cs -> '\\' : 'n' : go cs + Char_esc_r : cs -> '\\' : 'r' : go cs + Char_esc_t : cs -> '\\' : 't' : go cs + Char_esc_v : cs -> '\\' : 'v' : go cs + Char_lit c : cs -> + case st of + LongString -> c : go cs + ShortString -> + case c of + '\r' -> go $ Char_esc_r : cs + '\n' -> go $ Char_esc_n : cs + _ -> c : go cs + +renderPyChars :: QuoteType -> StringType -> [PyChar] -> Text +renderPyChars = + renderPyCharsWithCorrection $ + \qt st -> + case st of + LongString -> + correctBackslashes . correctBackslashEscapes . + correctInitialFinalQuotesLong qt + ShortString -> + correctBackslashes . correctBackslashEscapes . + correctQuotes qt + +renderRawPyChars :: QuoteType -> StringType -> [PyChar] -> Text +renderRawPyChars = + renderPyCharsWithCorrection $ + \qt st -> + case st of + LongString -> + correctInitialFinalQuotesLongRaw qt . + correctBackslashEscapesRaw . + correctBackslashesRaw + ShortString -> + correctBackslashEscapesRaw . correctBackslashesRaw . + correctQuotesRaw qt + +renderPyCharsBytesWithCorrection + :: (QuoteType -> StringType -> [PyChar] -> [PyChar]) + -> QuoteType + -> StringType + -> [PyChar] -> Text +renderPyCharsBytesWithCorrection c qt st = Text.pack . go . c qt st + where + go s = + case s of + [] -> "" + Char_newline : cs -> "\\newline" <> go cs + Char_octal1 a : cs -> + "\\" <> + [charOctal # a] <> + go cs + Char_octal2 a b : cs -> + "\\" <> + [charOctal # a, charOctal # b] <> + go cs + Char_octal3 a b c : cs -> + "\\" <> + [charOctal # a, charOctal # b, charOctal # c] <> + go cs + Char_hex a b : cs -> + "\\x" <> [charHeXaDeCiMaL # a, charHeXaDeCiMaL # b] <> go cs + Char_uni16 a b c d : cs -> + "\\u" <> + [ charHeXaDeCiMaL # a + , charHeXaDeCiMaL # b + , charHeXaDeCiMaL # c + , charHeXaDeCiMaL # d + ] <> + go cs + Char_uni32 a b c d e f g h : cs -> + "\\u" <> + [ charHeXaDeCiMaL # a + , charHeXaDeCiMaL # b + , charHeXaDeCiMaL # c + , charHeXaDeCiMaL # d + , charHeXaDeCiMaL # e + , charHeXaDeCiMaL # f + , charHeXaDeCiMaL # g + , charHeXaDeCiMaL # h + ] <> + go cs + Char_esc_bslash : cs -> '\\' : '\\' : go cs + Char_esc_singlequote : cs -> '\\' : '\'' : go cs + Char_esc_doublequote : cs -> '\\' : '"' : go cs + Char_esc_a : cs -> '\\' : 'a' : go cs + Char_esc_b : cs -> '\\' : 'b' : go cs + Char_esc_f : cs -> '\\' : 'f' : go cs + Char_esc_n : cs -> '\\' : 'n' : go cs + Char_esc_r : cs -> '\\' : 'r' : go cs + Char_esc_t : cs -> '\\' : 't' : go cs + Char_esc_v : cs -> '\\' : 'v' : go cs + Char_lit c : cs + | o <- ord c, o > 127 -> + let + h = intToHexH o + in + case replicate (8 - length h) HeXDigit0 <> h of + [a, b, c, d, e, f, g, h] -> go $ Char_uni32 a b c d e f g h : cs + _ -> error $ "character " <> show c <> " out of unicode range" + | otherwise -> + case st of + LongString -> c : go cs + ShortString -> + case c of + '\r' -> go $ Char_esc_r : cs + '\n' -> go $ Char_esc_n : cs + _ -> c : go cs + +renderPyCharsBytes :: QuoteType -> StringType -> [PyChar] -> Text +renderPyCharsBytes = + renderPyCharsBytesWithCorrection $ + \qt st -> + case st of + LongString -> + correctBackslashes . correctBackslashEscapes . correctInitialFinalQuotesLong qt + ShortString -> + correctBackslashes . correctBackslashEscapes . correctQuotes qt + +renderRawPyCharsBytes :: QuoteType -> StringType -> [PyChar] -> Text +renderRawPyCharsBytes = + renderPyCharsBytesWithCorrection $ + \qt st -> + case st of + LongString -> + correctInitialFinalQuotesLongRaw qt . + correctBackslashEscapesRaw . + correctBackslashesRaw + ShortString -> + correctBackslashEscapesRaw . correctBackslashesRaw . + correctQuotesRaw qt + +showTokens :: [PyToken a] -> Text +showTokens = + Lazy.toStrict . + Builder.toLazyText . + foldMap (Builder.fromText . showToken . (() <$)) . + (expandIndents =<<) + +expandIndents :: PyToken a -> [PyToken ()] +expandIndents (TkIndent _ i) = + (i ^.. indentsValue.folded.indentWhitespaces.folded) >>= + whitespaceTokens +expandIndents (TkLevel _ i) = + (i ^.. indentsValue.folded.indentWhitespaces.folded) >>= + whitespaceTokens +expandIndents TkDedent{} = [] +expandIndents a = pure $ () <$ a + +showToken :: PyToken a -> Text +showToken t = + case t of + TkIndent{} -> error "trying to show indent token" + TkLevel{} -> error "trying to show level token" + TkDedent{} -> error "trying to show dedent token" + TkIf{} -> "if" + TkElse{} -> "else" + TkElif{} -> "elif" + TkWhile{} -> "while" + TkAssert{} -> "assert" + TkDef{} -> "def" + TkReturn{} -> "return" + TkPass{} -> "pass" + TkBreak{} -> "break" + TkContinue{} -> "continue" + TkTrue{} -> "True" + TkFalse{} -> "False" + TkNone{} -> "None" + TkEllipsis{} -> "..." + TkOr{} -> "or" + TkAnd{} -> "and" + TkIs{} -> "is" + TkNot{} -> "not" + TkGlobal{} -> "global" + TkNonlocal{} -> "nonlocal" + TkDel{} -> "del" + TkLambda{} -> "lambda" + TkImport{} -> "import" + TkFrom{} -> "from" + TkAs{} -> "as" + TkRaise{} -> "raise" + TkTry{} -> "try" + TkExcept{} -> "except" + TkFinally{} -> "finally" + TkClass{} -> "class" + TkRightArrow{} -> "->" + TkWith{} -> "with" + TkFor{} -> "for" + TkIn{} -> "in" + TkYield{} -> "yield" + TkInt i -> showIntLiteral i + TkFloat i -> showFloatLiteral i + TkImag i -> showImagLiteral i + TkIdent s _ -> Text.pack s + TkString sp st qt s _ -> + let + quote = + Text.pack $ + (case st of; LongString -> replicate 3; ShortString -> pure) (showQuoteType qt) + in + foldMap showStringPrefix sp <> + quote <> + renderPyChars qt st s <> + quote + TkBytes sp st qt s _ -> + let + quote = + Text.pack $ + (case st of; LongString -> replicate 3; ShortString -> pure) (showQuoteType qt) + in + showBytesPrefix sp <> + quote <> + renderPyCharsBytes qt st s <> + quote + TkRawString sp st qt s _ -> + let + quote = + case st of + LongString -> Text.pack . replicate 3 $ showQuoteType qt + ShortString -> Text.singleton $ showQuoteType qt + in + showRawStringPrefix sp <> + quote <> + renderRawPyChars qt st s <> + quote + TkRawBytes sp st qt s _ -> + let + quote = + case st of + LongString -> Text.pack . replicate 3 $ showQuoteType qt + ShortString -> Text.singleton $ showQuoteType qt + in + showRawBytesPrefix sp <> + quote <> + renderRawPyCharsBytes qt st s <> + quote + TkSpace{} -> " " + TkTab{} -> "\t" + TkNewline nl _ -> + case nl of + CR -> "\r" + LF -> "\n" + CRLF -> "\r\n" + TkLeftBracket{} -> "[" + TkRightBracket{} -> "]" + TkLeftParen{} -> "(" + TkRightParen{} -> ")" + TkLeftBrace{} -> "{" + TkRightBrace{} -> "}" + TkLt{} -> "<" + TkLte{} -> "<=" + TkEq{} -> "=" + TkDoubleEq{}-> "==" + TkBangEq{}-> "!=" + TkGt{} -> ">" + TkGte{} -> ">=" + TkContinued nl _ -> + "\\" <> + case nl of + CR -> "\r" + LF -> "\n" + CRLF -> "\r\n" + TkColon{} -> ":" + TkSemicolon{} -> ";" + TkComma{} -> "," + TkDot{} -> "." + TkPlus{} -> "+" + TkMinus{} -> "-" + TkTilde{} -> "~" + TkComment c -> showComment c + TkStar{} -> "*" + TkDoubleStar{} -> "**" + TkSlash{} -> "/" + TkDoubleSlash{} -> "//" + TkPercent{} -> "%" + TkShiftLeft{} -> "<<" + TkShiftRight{} -> ">>" + TkPlusEq{} -> "+=" + TkMinusEq{} -> "-=" + TkStarEq{} -> "*=" + TkAtEq{} -> "@=" + TkAt{} -> "@" + TkSlashEq{} -> "/=" + TkPercentEq{} -> "%=" + TkAmpersandEq{} -> "&=" + TkPipeEq{} -> "|=" + TkCaretEq{} -> "^=" + TkAmpersand{} -> "&" + TkPipe{} -> "|" + TkCaret{} -> "^" + TkShiftLeftEq{} -> "<<=" + TkShiftRightEq{} -> ">>=" + TkDoubleStarEq{} -> "**=" + TkDoubleSlashEq{} -> "//=" + +whitespaceTokens :: Whitespace -> [PyToken ()] +whitespaceTokens Space = [TkSpace ()] +whitespaceTokens Tab = [TkTab ()] +whitespaceTokens (Continued nl ws) = TkContinued nl () : (ws >>= whitespaceTokens) +whitespaceTokens (Newline nl) = [TkNewline nl ()] +whitespaceTokens (Comment cmt) = commentTokens cmt + +renderWhitespace :: Whitespace -> RenderOutput () +renderWhitespace = traverse_ singleton . whitespaceTokens + +renderNewline :: Newline -> PyToken () +renderNewline nl = TkNewline nl () + +renderComma :: Comma -> RenderOutput () +renderComma (MkComma ws) = do + singleton $ TkComma () + traverse_ renderWhitespace ws + +renderAt :: At -> RenderOutput () +renderAt (MkAt ws) = do + singleton $ TkAt () + traverse_ renderWhitespace ws + +renderCommaSep :: (a -> RenderOutput ()) -> CommaSep a -> RenderOutput () +renderCommaSep _ CommaSepNone = pure () +renderCommaSep f (CommaSepOne a) = f a +renderCommaSep f (CommaSepMany a c cs) = do + f a + renderComma c + renderCommaSep f cs + +renderCommaSep1 :: (a -> RenderOutput ()) -> CommaSep1 a -> RenderOutput () +renderCommaSep1 f (CommaSepOne1 a) = f a +renderCommaSep1 f (CommaSepMany1 a comma c) = do + f a + renderComma comma + renderCommaSep1 f c + +renderCommaSep1' :: (a -> RenderOutput ()) -> CommaSep1' a -> RenderOutput () +renderCommaSep1' f (CommaSepOne1' a b) = do + f a + traverse_ + renderComma + b +renderCommaSep1' f (CommaSepMany1' a comma c) = do + f a + renderComma comma + renderCommaSep1' f c + +renderIdent :: Ident v a -> RenderOutput () +renderIdent (MkIdent _ a b) = do + singleton $ TkIdent a () + traverse_ renderWhitespace b + +parensTernaryLambda :: (Expr v a -> RenderOutput ()) -> Expr v a -> RenderOutput () +parensTernaryLambda _ e@Ternary{} = parensDistTWS renderExpr e +parensTernaryLambda _ e@Lambda{} = parensDistTWS renderExpr e +parensTernaryLambda f e = f e + +renderCompFor :: CompFor v a -> RenderOutput () +renderCompFor (CompFor _ ws1 ex1 ws2 ex2) = do + singleton $ TkFor () + traverse_ renderWhitespace ws1 + (case ex1 of + Not{} -> parensDistTWS renderExpr ex1 + _ -> parensGenerator ex1) + singleton $ TkIn () + traverse_ renderWhitespace ws2 + parensTernaryLambda parensTupleGenerator ex2 + +renderCompIf :: CompIf v a -> RenderOutput () +renderCompIf (CompIf _ ws ex) = do + singleton $ TkIf () + traverse_ renderWhitespace ws + parensTernaryLambda parensTupleGenerator ex + +renderComprehension + :: (e v a -> RenderOutput ()) + -> Comprehension e v a + -> RenderOutput () +renderComprehension f (Comprehension _ expr cf cs) = do + f expr + renderCompFor cf + traverse_ (bitraverse_ renderCompFor renderCompIf) cs + +renderDictItem :: DictItem v a -> RenderOutput () +renderDictItem (DictItem _ a b c) = do + parensTupleGenerator a + renderColon b + parensTupleGenerator c +renderDictItem (DictUnpack _ a b) = do + singleton $ TkDoubleStar () + traverse_ renderWhitespace a + case b of + BinOp _ _ BoolAnd{} _ -> parensDistTWS renderExpr b + BinOp _ _ BoolOr{} _ -> parensDistTWS renderExpr b + BinOp _ _ op _ | isComparison op -> parensDistTWS renderExpr b + Not{} -> parensDistTWS renderExpr b + _ -> parensTernaryLambda parensTupleGenerator b + +renderStringLiteral :: StringLiteral a -> RenderOutput () +renderStringLiteral (StringLiteral _ a b c d e) = do + singleton $ TkString a b c d () + traverse_ renderWhitespace e +renderStringLiteral (BytesLiteral _ a b c d e) = do + singleton $ TkBytes a b c d () + traverse_ renderWhitespace e +renderStringLiteral (RawStringLiteral _ a b c d e) = do + singleton $ TkRawString a b c d () + traverse_ renderWhitespace e +renderStringLiteral (RawBytesLiteral _ a b c d e) = do + singleton $ TkRawBytes a b c d () + traverse_ renderWhitespace e + +renderSubscript :: Subscript v a -> RenderOutput () +renderSubscript (SubscriptExpr a) = + case a of + Await{} -> parensDistTWS renderExpr a + _ -> parensTupleGenerator a +renderSubscript (SubscriptSlice a b c d) = do + traverse_ parensTupleGenerator a + renderColon b + traverse_ parensTupleGenerator c + traverse_ + (bitraverse_ + renderColon + (traverse_ parensTupleGenerator)) + d + +renderYield :: (Expr v a -> RenderOutput ()) -> Expr v a -> RenderOutput () +renderYield _ (Yield _ a b) = do + singleton $ TkYield () + traverse_ renderWhitespace a + renderCommaSep parensTupleGenerator b +renderYield _ (YieldFrom _ a b c) = do + singleton $ TkYield () + traverse_ renderWhitespace a + singleton $ TkFrom () + traverse_ renderWhitespace b + parensTupleGenerator c +renderYield re e = re e + +renderUnpackTarget :: Expr v a -> RenderOutput () +renderUnpackTarget e = + case e of + BinOp _ _ BoolAnd{} _ -> parensDistTWS renderExpr e + BinOp _ _ BoolOr{} _ -> parensDistTWS renderExpr e + BinOp _ _ op _ | isComparison op -> parensDistTWS renderExpr e + Not{} -> parensDistTWS renderExpr e + _ -> parensTernaryLambda parensTupleGenerator e + +renderNestedParens + :: RenderOutput () + -> [([Whitespace], [Whitespace])] + -> RenderOutput () +renderNestedParens = + foldr + (\(ws1, ws2) y -> do + singleton $ TkLeftParen () + traverse_ renderWhitespace ws1 + y + singleton $ TkRightParen () + traverse_ renderWhitespace ws2) + +renderTupleItems + :: CommaSep1' (TupleItem v a) + -> RenderOutput () +renderTupleItems (CommaSepOne1' a Nothing) = + case a of + TupleItem _ b -> parensTupleGenerator b + TupleUnpack _ b c d -> + renderNestedParens + (do + singleton $ TkStar () + traverse_ renderWhitespace c + renderUnpackTarget d) + b +renderTupleItems (CommaSepOne1' a (Just comma)) = do + (case a of + TupleItem _ b -> parensTupleGenerator b + TupleUnpack _ [] b c -> + parens $ do + singleton $ TkStar () + traverse_ renderWhitespace b + renderUnpackTarget c + TupleUnpack _ b c d -> + renderNestedParens + (do + singleton $ TkStar () + traverse_ renderWhitespace c + renderUnpackTarget d) + b) + renderComma comma +renderTupleItems (CommaSepMany1' a comma rest) = do + (case a of + TupleItem _ b -> parensTupleGenerator b + TupleUnpack _ [] b c -> + parens $ do + singleton $ TkStar () + traverse_ renderWhitespace b + renderUnpackTarget c + TupleUnpack _ b c d -> + renderNestedParens + (do + singleton $ TkStar () + traverse_ renderWhitespace c + renderUnpackTarget d) + b) + renderComma comma + renderTupleItems rest + +renderSetItem :: SetItem v a -> RenderOutput () +renderSetItem a = + case a of + SetItem _ b -> parensTupleGenerator b + SetUnpack _ b c d -> + renderNestedParens + (do + singleton $ TkStar () + traverse_ renderWhitespace c + renderUnpackTarget d) + b + +renderSetItems :: CommaSep1' (SetItem v a) -> RenderOutput () +renderSetItems (CommaSepOne1' a Nothing) = + case a of + SetItem _ b -> parensTupleGenerator b + SetUnpack _ b c d -> + renderNestedParens + (do + singleton $ TkStar () + traverse_ renderWhitespace c + renderUnpackTarget d) + b +renderSetItems (CommaSepOne1' a (Just comma)) = do + (case a of + SetItem _ b -> parensTupleGenerator b + SetUnpack _ [] b c -> do + singleton $ TkStar () + traverse_ renderWhitespace b + renderUnpackTarget c + SetUnpack _ b c d -> + renderNestedParens + (do + singleton $ TkStar () + traverse_ renderWhitespace c + renderUnpackTarget d) + b) + renderComma comma +renderSetItems (CommaSepMany1' a comma rest) = do + (case a of + SetItem _ b -> parensTupleGenerator b + SetUnpack _ [] b c -> do + singleton $ TkStar () + traverse_ renderWhitespace b + renderUnpackTarget c + SetUnpack _ b c d -> + renderNestedParens + (do + singleton $ TkStar () + traverse_ renderWhitespace c + renderUnpackTarget d) + b) + renderComma comma + renderSetItems rest + +renderListItems :: CommaSep1' (ListItem v a) -> RenderOutput () +renderListItems (CommaSepOne1' a Nothing) = + case a of + ListItem _ b -> parensTupleGenerator b + ListUnpack _ b c d -> + renderNestedParens + (do + singleton $ TkStar () + traverse_ renderWhitespace c + renderUnpackTarget d) + b +renderListItems (CommaSepOne1' a (Just comma)) = do + (case a of + ListItem _ b -> parensTupleGenerator b + ListUnpack _ [] b c -> do + singleton $ TkStar () + traverse_ renderWhitespace b + renderUnpackTarget c + ListUnpack _ b c d -> + renderNestedParens + (do + singleton $ TkStar () + traverse_ renderWhitespace c + renderUnpackTarget d) + b) + renderComma comma +renderListItems (CommaSepMany1' a comma rest) = do + (case a of + ListItem _ b -> parensTupleGenerator b + ListUnpack _ [] b c -> do + singleton $ TkStar () + traverse_ renderWhitespace b + renderUnpackTarget c + ListUnpack _ b c d -> + renderNestedParens + (do + singleton $ TkStar () + traverse_ renderWhitespace c + renderUnpackTarget d) + b) + renderComma comma + renderListItems rest + +renderExpr :: Expr v a -> RenderOutput () +renderExpr (Unit _ a b) = do + singleton $ TkLeftParen () + traverse_ renderWhitespace a + singleton $ TkRightParen () + traverse_ renderWhitespace b +renderExpr (Lambda _ a b c d) = do + singleton $ TkLambda () + traverse_ renderWhitespace a + renderParams b + renderColon c + parensTupleGenerator d +renderExpr e@Yield{} = parensDistTWS (renderYield parensTupleGenerator) e +renderExpr e@YieldFrom{} = parensDistTWS (renderYield parensTupleGenerator) e +renderExpr (Ternary _ a b c d e) = do + (case a of + Generator{} -> parensDistTWS renderExpr a + _ -> parensTupleGenerator a) + singleton $ TkIf () + traverse_ renderWhitespace b + parensTernaryLambda parensTupleGenerator c + singleton $ TkElse () + traverse_ renderWhitespace d + parensTupleGenerator e +renderExpr (Subscript _ a b c d) = do + (case a of + BinOp{} -> parensDistTWS renderExpr a + UnOp{} -> parensDistTWS renderExpr a + Not{} -> parensDistTWS renderExpr a + Ternary{} -> parensDistTWS renderExpr a + Lambda{} -> parensDistTWS renderExpr a + Await{} -> parensDistTWS renderExpr a + _ -> parensTupleGenerator a) + brackets $ do + traverse_ renderWhitespace b + renderCommaSep1' renderSubscript c + traverse_ renderWhitespace d +renderExpr (Not _ ws e) = do + singleton $ TkNot () + traverse_ renderWhitespace ws + case e of + BinOp _ _ BoolAnd{} _ -> parensDistTWS renderExpr e + BinOp _ _ BoolOr{} _ -> parensDistTWS renderExpr e + Ternary{} -> parensDistTWS renderExpr e + Lambda{} -> parensDistTWS renderExpr e + _ -> parensTupleGenerator e +renderExpr (Parens _ ws1 e ws2) = do + parens $ do + traverse_ renderWhitespace ws1 + renderYield renderExpr e + traverse_ renderWhitespace ws2 +renderExpr (Bool _ b ws) = do + singleton $ if b then TkTrue () else TkFalse () + traverse_ renderWhitespace ws +renderExpr (UnOp _ op expr) = do + renderUnOp op + case expr of + BinOp _ _ Exp{} _ -> parensTupleGenerator expr + BinOp{} -> parensDistTWS renderExpr expr + Deref _ Int{} _ _ -> parensDistTWS renderExpr expr + Not{} -> parensDistTWS renderExpr expr + Ternary{} -> parensDistTWS renderExpr expr + Lambda{} -> parensDistTWS renderExpr expr + _ -> parensTupleGenerator expr +renderExpr (String _ vs) = + traverse_ renderStringLiteral $ correctAdjacentStrings vs +renderExpr (Int _ n ws) = do + singleton $ TkInt (() <$ n) + traverse_ renderWhitespace ws +renderExpr (Float _ n ws) = do + singleton $ TkFloat (() <$ n) + traverse_ renderWhitespace ws +renderExpr (Imag _ n ws) = do + singleton $ TkImag (() <$ n) + traverse_ renderWhitespace ws +renderExpr (Ident name) = renderIdent name +renderExpr (List _ ws1 exprs ws2) = do + brackets $ do + traverse_ renderWhitespace ws1 + traverse_ renderListItems exprs + traverse_ renderWhitespace ws2 +renderExpr (ListComp _ ws1 comp ws2) = do + brackets $ do + traverse_ renderWhitespace ws1 + renderComprehension + (\e -> case e of + Yield{} -> parensDistTWS renderExpr e + YieldFrom{} -> parensDistTWS renderExpr e + _ -> parensTupleGenerator e) + comp + traverse_ renderWhitespace ws2 +renderExpr (Call _ expr ws args ws2) = do + (case expr of + UnOp{} -> parensDistTWS renderExpr expr + BinOp{} -> parensDistTWS renderExpr expr + Tuple{} -> parensDistTWS renderExpr expr + Not{} -> parensDistTWS renderExpr expr + Ternary{} -> parensDistTWS renderExpr expr + Lambda{} -> parensDistTWS renderExpr expr + _ -> parensGenerator expr) + parens $ do + traverse_ renderWhitespace ws + traverse_ renderArgs args + traverse_ renderWhitespace ws2 +renderExpr (Deref _ expr ws name) = do + (case expr of + Int{} -> parensDistTWS renderExpr expr + BinOp{} -> parensDistTWS renderExpr expr + Tuple{} -> parensDistTWS renderExpr expr + Not{} -> parensDistTWS renderExpr expr + UnOp{} -> parensDistTWS renderExpr expr + Ternary{} -> parensDistTWS renderExpr expr + Lambda{} -> parensDistTWS renderExpr expr + Await{} -> parensDistTWS renderExpr expr + _ -> parensGenerator expr) + singleton $ TkDot () + traverse_ renderWhitespace ws + renderIdent name +renderExpr (None _ ws) = do + singleton $ TkNone () + traverse_ renderWhitespace ws +renderExpr (Ellipsis _ ws) = do + singleton $ TkEllipsis () + traverse_ renderWhitespace ws +renderExpr (BinOp _ e1 op e2) = do + if shouldGroupLeft op e1 + then parensDistTWS renderExpr e1 + else parensTernaryLambda parensGenerator e1 + + renderBinOp op + + if shouldGroupRight op e2 + then parensDistTWS renderExpr e2 + else parensTernaryLambda parensGenerator e2 +renderExpr (Tuple _ a ws c) = + renderTupleItems $ + case c of + Nothing -> CommaSepOne1' a (Just ws) + Just c' -> CommaSepMany1' a ws c' +renderExpr (DictComp _ ws1 comp ws2) = do + braces $ do + traverse_ renderWhitespace ws1 + renderComprehension renderDictItem comp + traverse_ renderWhitespace ws2 +renderExpr (Dict _ a b c) = do + braces $ do + traverse_ renderWhitespace a + traverse_ (renderCommaSep1' renderDictItem) b + traverse_ renderWhitespace c +renderExpr (SetComp _ ws1 comp ws2) = do + braces $ do + traverse_ renderWhitespace ws1 + renderComprehension renderSetItem comp + traverse_ renderWhitespace ws2 +renderExpr (Set _ a b c) = do + braces $ do + traverse_ renderWhitespace a + renderSetItems b + traverse_ renderWhitespace c +renderExpr (Generator _ a) = + renderComprehension + (\e -> case e of + Yield{} -> parensDistTWS renderExpr e + YieldFrom{} -> parensDistTWS renderExpr e + _ -> parensTupleGenerator e) + a +renderExpr (Await _ ws expr) = do + singleton $ TkIdent "await" () + traverse_ renderWhitespace ws + (case expr of + UnOp{} -> parensDistTWS renderExpr expr + BinOp{} -> parensDistTWS renderExpr expr + Tuple{} -> parensDistTWS renderExpr expr + Not{} -> parensDistTWS renderExpr expr + Ternary{} -> parensDistTWS renderExpr expr + Lambda{} -> parensDistTWS renderExpr expr + Await{} -> parensDistTWS renderExpr expr + _ -> parensGenerator expr) + +renderModuleName :: ModuleName v a -> RenderOutput () +renderModuleName (ModuleNameOne _ s) = renderIdent s +renderModuleName (ModuleNameMany _ n dot rest) = do + renderIdent n + renderDot dot + renderModuleName rest + +renderDot :: Dot -> RenderOutput () +renderDot (MkDot ws) = do + singleton $ TkDot () + traverse_ renderWhitespace ws + +renderRelativeModuleName :: RelativeModuleName v a -> RenderOutput () +renderRelativeModuleName (RelativeWithName ds mn) = do + traverse_ renderDot ds + renderModuleName mn +renderRelativeModuleName (Relative ds) = + traverse_ renderDot ds + +renderImportAs :: (e a -> RenderOutput ()) -> ImportAs e v a -> RenderOutput () +renderImportAs f (ImportAs _ ea m) = do + f ea + traverse_ + (\(a, b) -> do + singleton $ TkAs () + traverse_ renderWhitespace a + renderIdent b) + m + +renderImportTargets :: ImportTargets v a -> RenderOutput () +renderImportTargets (ImportAll _ ws) = do + singleton $ TkStar () + traverse_ renderWhitespace ws +renderImportTargets (ImportSome _ ts) = + renderCommaSep1 (renderImportAs renderIdent) ts +renderImportTargets (ImportSomeParens _ ws1 ts ws2) = do + parens $ do + traverse_ renderWhitespace ws1 + renderCommaSep1' (renderImportAs renderIdent) ts + traverse_ renderWhitespace ws2 + +renderAugAssign :: AugAssign a -> RenderOutput () +renderAugAssign aa = do + singleton $ case _augAssignType aa of + PlusEq -> TkPlusEq () + MinusEq -> TkMinusEq () + StarEq -> TkStarEq () + AtEq -> TkAtEq () + SlashEq -> TkSlashEq () + PercentEq -> TkPercentEq () + AmpersandEq -> TkAmpersandEq () + PipeEq -> TkPipeEq () + CaretEq -> TkCaretEq () + ShiftLeftEq -> TkShiftLeftEq () + ShiftRightEq -> TkShiftRightEq () + DoubleStarEq -> TkDoubleStarEq () + DoubleSlashEq -> TkDoubleSlashEq () + traverse_ renderWhitespace (_augAssignWhitespace aa) + +renderSimpleStatement :: SimpleStatement v a -> RenderOutput () +renderSimpleStatement (Assert _ b c d) = do + singleton $ TkAssert () + traverse_ renderWhitespace b + parensTupleGenerator c + traverse_ + (\(a, b) -> do + renderComma a + parensTupleGenerator b) + d +renderSimpleStatement (Raise _ ws x) = do + singleton $ TkRaise () + traverse_ renderWhitespace ws + traverse_ + (\(b, c) -> do + parensTupleGenerator b + traverse_ + (\(d, e) -> do + singleton $ TkFrom () + traverse_ renderWhitespace d + parensTupleGenerator e) + c) + x +renderSimpleStatement (Return _ ws expr) = do + singleton $ TkReturn () + traverse_ renderWhitespace ws + traverse_ parensGenerator expr +renderSimpleStatement (Expr _ expr) = renderYield parensGenerator expr +renderSimpleStatement (Assign _ lvalue rvalues) = do + renderExpr lvalue + traverse_ + (\(ws2, rvalue) -> do + renderEquals ws2 + renderYield parensGenerator rvalue) + rvalues +renderSimpleStatement (AugAssign _ lvalue as rvalue) = do + renderExpr lvalue + renderAugAssign as + parensTupleGenerator rvalue +renderSimpleStatement (Pass _ ws) = do + singleton $ TkPass () + traverse_ renderWhitespace ws +renderSimpleStatement (Continue _ ws) = do + singleton $ TkContinue () + traverse_ renderWhitespace ws +renderSimpleStatement (Break _ ws) = do + singleton $ TkBreak () + traverse_ renderWhitespace ws +renderSimpleStatement (Global _ ws ids) = do + singleton $ TkGlobal () + traverse_ renderWhitespace ws + renderCommaSep1 renderIdent ids +renderSimpleStatement (Nonlocal _ ws ids) = do + singleton $ TkNonlocal () + traverse_ renderWhitespace ws + renderCommaSep1 renderIdent ids +renderSimpleStatement (Del _ ws vals) = do + singleton $ TkDel () + traverse_ renderWhitespace ws + renderCommaSep1' + (\a -> case a of + BinOp{} -> parensDistTWS renderExpr a + Not{} -> parensDistTWS renderExpr a + Ternary{} -> parensDistTWS renderExpr a + Lambda{} -> parensDistTWS renderExpr a + _ -> parensTupleGenerator a) + vals +renderSimpleStatement (Import _ ws ns) = do + singleton $ TkImport () + traverse_ renderWhitespace ws + renderCommaSep1 (renderImportAs renderModuleName) ns +renderSimpleStatement (From _ ws1 name ws3 ns) = do + singleton $ TkFrom () + traverse_ renderWhitespace ws1 + renderRelativeModuleName name + singleton $ TkImport () + traverse_ renderWhitespace ws3 + renderImportTargets ns + +renderBlank :: Blank a -> RenderOutput () +renderBlank (Blank _ a b) = do + traverse_ renderWhitespace a + traverse_ renderComment b + +renderBlock :: Block v a -> RenderOutput () +renderBlock (Block a b c) = do + traverse_ (bitraverse_ renderBlank (singleton . renderNewline)) a + (if null c then final else notFinal) $ renderStatement b + traverseOf_ + (_init.traverse) + (bitraverse_ + (bitraverse_ renderBlank (singleton . renderNewline)) + (notFinal . renderStatement)) + c + traverseOf_ + _last + (bitraverse_ + (bitraverse_ renderBlank (singleton . renderNewline)) + (final . renderStatement)) + c + +renderSemicolon :: Semicolon a -> RenderOutput () +renderSemicolon (MkSemicolon _ ws) = do + singleton $ TkSemicolon () + traverse_ renderWhitespace ws + +renderEquals :: Equals -> RenderOutput () +renderEquals (MkEquals ws) = do + singleton $ TkEq () + traverse_ renderWhitespace ws + +renderColon :: Colon -> RenderOutput () +renderColon (MkColon ws) = do + singleton $ TkColon () + traverse_ renderWhitespace ws + +renderSuite + :: Suite v a + -> RenderOutput () +renderSuite (SuiteMany _ a b c d) = do + renderColon a + traverse_ renderComment b + singleton (renderNewline c) + renderBlock d +renderSuite (SuiteOne _ a b) = do + renderColon a + fin <- isFinal + renderSmallStatement $ correctTrailingNewline fin b + +renderDecorator :: Decorator v a -> RenderOutput () +renderDecorator (Decorator _ a b c d e f) = do + renderIndents a + renderAt b + renderExpr c + traverse_ renderComment d + singleton (renderNewline e) + traverse_ (bitraverse_ renderBlank (singleton . renderNewline)) f + +renderCompoundStatement :: CompoundStatement v a -> RenderOutput () +renderCompoundStatement (Fundef _ decos idnt asyncWs ws1 name ws2 params ws3 mty s) = do + traverse_ renderDecorator decos + renderIndents idnt + traverse_ + (\ws -> do + singleton $ TkIdent "async" () + traverse_ renderWhitespace ws) + asyncWs + singleton (TkDef ()) + traverse_ renderWhitespace ws1 + renderIdent name + parens $ do + traverse_ renderWhitespace ws2 + renderParams params + traverse_ renderWhitespace ws3 + traverse_ + (\(ws, ty) -> do + singleton $ TkRightArrow () + traverse_ renderWhitespace ws + parensTupleGenerator ty) + mty + final $ renderSuite s +renderCompoundStatement (If _ idnt ws1 expr s elifs body') = do + renderIndents idnt + singleton $ TkIf () + traverse_ renderWhitespace ws1 + parensTupleGenerator expr + notFinal $ renderSuite s + traverseOf_ + (_init.traverse) + (\(idnt, ws4, ex, s) -> do + renderIndents idnt + singleton $ TkElif () + traverse_ renderWhitespace ws4 + parensTupleGenerator ex + notFinal $ renderSuite s) + elifs + traverseOf_ + _last + (\(idnt, ws4, ex, s) -> do + renderIndents idnt + singleton $ TkElif () + traverse_ renderWhitespace ws4 + parensTupleGenerator ex + (if isNothing body' then final else notFinal) $ renderSuite s) + elifs + traverse_ + (\(idnt, ws4, s) -> do + renderIndents idnt + singleton $ TkElse () + traverse_ renderWhitespace ws4 + final $ renderSuite s) + body' +renderCompoundStatement (While _ idnt ws1 expr s els) = do + renderIndents idnt + singleton $ TkWhile () + traverse_ renderWhitespace ws1 + parensTupleGenerator expr + (if isNothing els then final else notFinal) $ renderSuite s + traverse_ + (\(idnt, ws4, s) -> do + renderIndents idnt + singleton $ TkElse () + traverse_ renderWhitespace ws4 + final $ renderSuite s) + els +renderCompoundStatement (TryExcept _ idnt a s e f g) = do + renderIndents idnt + singleton $ TkTry () + traverse_ renderWhitespace a + notFinal $ renderSuite s + traverse_ + (\(idnt, ws1, eas, s) -> do + renderIndents idnt + singleton $ TkExcept () + traverse_ renderWhitespace ws1 + traverse_ renderExceptAs eas + notFinal $ renderSuite s) + (NonEmpty.init e) + (case NonEmpty.last e of + (idnt, ws1, eas, s) -> do + renderIndents idnt + singleton $ TkExcept () + traverse_ renderWhitespace ws1 + traverse_ renderExceptAs eas + (if isNothing f && isNothing g then final else notFinal) $ renderSuite s) + traverse_ + (\(idnt, ws1, s) -> do + renderIndents idnt + singleton $ TkElse () + traverse_ renderWhitespace ws1 + (if isNothing g then final else notFinal) $ renderSuite s) + f + traverse_ + (\(idnt, ws1, s) -> do + renderIndents idnt + singleton $ TkFinally () + traverse_ renderWhitespace ws1 + final $ renderSuite s) + g +renderCompoundStatement (TryFinally _ idnt a s idnt2 e s') = do + renderIndents idnt + singleton $ TkTry () + traverse_ renderWhitespace a + notFinal $ renderSuite s + renderIndents idnt2 + singleton $ TkFinally () + traverse_ renderWhitespace e + final $ renderSuite s' +renderCompoundStatement (For _ idnt asyncWs a b c d s h) = do + renderIndents idnt + traverse_ + (\ws -> do + singleton $ TkIdent "async" () + traverse_ renderWhitespace ws) + asyncWs + singleton $ TkFor () + traverse_ renderWhitespace a + parensGenerator b + singleton $ TkIn () + traverse_ renderWhitespace c + renderCommaSep1' parensTupleGenerator d + (if isNothing h then final else notFinal) $ renderSuite s + traverse_ + (\(idnt, x, s) -> do + renderIndents idnt + singleton $ TkElse () + traverse_ renderWhitespace x + final $ renderSuite s) + h +renderCompoundStatement (ClassDef _ decos idnt a b c s) = do + traverse_ renderDecorator decos + renderIndents idnt + singleton $ TkClass () + traverse_ renderWhitespace a + renderIdent b + traverse_ + (\(x, y, z) -> do + parens $ do + traverse_ renderWhitespace x + traverse_ renderArgs y + traverse_ renderWhitespace z) + c + final $ renderSuite s +renderCompoundStatement (With _ idnt asyncWs a b s) = do + renderIndents idnt + traverse_ + (\ws -> do + singleton $ TkIdent "async" () + traverse_ renderWhitespace ws) + asyncWs + singleton $ TkWith () + traverse_ renderWhitespace a + renderCommaSep1 renderWithItem b + final $ renderSuite s + +renderWithItem :: WithItem v a -> RenderOutput () +renderWithItem (WithItem _ a b) = do + parensTupleGenerator a + traverse_ + (\(c, d) -> do + singleton $ TkAs () + traverse_ renderWhitespace c + parensTupleGenerator d) + b + +renderIndent :: Indent -> RenderOutput () +renderIndent (MkIndent ws) = traverse_ renderWhitespace $ toList ws + +renderSmallStatement :: SmallStatement v a -> RenderOutput () +renderSmallStatement (MkSmallStatement s ss sc cmt nl) = do + renderSimpleStatement s + traverse_ + (\(b, c) -> do + renderSemicolon b + renderSimpleStatement c) + ss + traverse_ renderSemicolon sc + traverse_ renderComment cmt + traverse_ (singleton . renderNewline) nl + +renderStatement :: Statement v a -> RenderOutput () +renderStatement (CompoundStatement c) = renderCompoundStatement c +renderStatement (SmallStatement idnts s) = do + renderIndents idnts + fin <- isFinal + renderSmallStatement $ correctTrailingNewline fin s + +renderExceptAs :: ExceptAs v a -> RenderOutput () +renderExceptAs (ExceptAs _ e f) = do + parensTupleGenerator e + traverse_ + (\(a, b) -> do + singleton $ TkAs () + traverse_ renderWhitespace a + renderIdent b) + f + +renderArgs :: CommaSep1' (Arg v a) -> RenderOutput () +renderArgs (CommaSepOne1' a Nothing) = renderArg parensTuple a +renderArgs e = renderCommaSep1' (renderArg parensTupleGenerator) e + +renderArg :: (Expr v a -> RenderOutput ()) -> Arg v a -> RenderOutput () +renderArg re (PositionalArg _ expr) = re expr +renderArg _ (KeywordArg _ name ws2 expr) = do + renderIdent name + singleton $ TkEq () + traverse_ renderWhitespace ws2 + parensTupleGenerator expr +renderArg _ (StarArg _ ws expr) = do + singleton $ TkStar () + traverse_ renderWhitespace ws + parensTupleGenerator expr +renderArg _ (DoubleStarArg _ ws expr) = do + singleton $ TkDoubleStar () + traverse_ renderWhitespace ws + parensTupleGenerator expr + +renderParams :: CommaSep (Param v a) -> RenderOutput () +renderParams = renderCommaSep renderParam . correctParams + +renderParam :: Param v a -> RenderOutput () +renderParam (PositionalParam _ name mty) = do + renderIdent name + traverse_ + (\(c, ty) -> do + renderColon c + parensTupleGenerator ty) + mty +renderParam (StarParam _ ws name mty) = do + singleton $ TkStar () + traverse_ renderWhitespace ws + renderIdent name + traverse_ + (\(c, ty) -> do + renderColon c + parensTupleGenerator ty) + mty +renderParam (UnnamedStarParam _ ws) = do + singleton $ TkStar () + traverse_ renderWhitespace ws +renderParam (DoubleStarParam _ ws name mty) = do + singleton $ TkDoubleStar () + traverse_ renderWhitespace ws + renderIdent name + traverse_ + (\(c, ty) -> do + renderColon c + parensTupleGenerator ty) + mty +renderParam (KeywordParam _ name mty ws2 expr) = do + renderIdent name + traverse_ + (\(c, ty) -> do + renderColon c + parensTupleGenerator ty) + mty + singleton $ TkEq () + traverse_ renderWhitespace ws2 + parensTupleGenerator expr + +renderUnOp :: UnOp a -> RenderOutput () +renderUnOp (Negate _ ws) = do + singleton $ TkMinus () + traverse_ renderWhitespace ws +renderUnOp (Positive _ ws) = do + singleton $ TkPlus () + traverse_ renderWhitespace ws +renderUnOp (Complement _ ws) = do + singleton $ TkTilde () + traverse_ renderWhitespace ws + +renderBinOp :: BinOp a -> RenderOutput () +renderBinOp (Is _ ws) = do + singleton $ TkIs () + traverse_ renderWhitespace ws +renderBinOp (IsNot _ ws1 ws2) = do + singleton $ TkIs () + traverse_ renderWhitespace ws1 + singleton $ TkNot () + traverse_ renderWhitespace ws2 +renderBinOp (In _ ws) = do + singleton $ TkIn () + traverse_ renderWhitespace ws +renderBinOp (NotIn _ ws1 ws2) = do + singleton $ TkNot () + traverse_ renderWhitespace ws1 + singleton $ TkIn () + traverse_ renderWhitespace ws2 +renderBinOp (Plus _ ws) = do + singleton $ TkPlus () + traverse_ renderWhitespace ws +renderBinOp (Minus _ ws) = do + singleton $ TkMinus () + traverse_ renderWhitespace ws +renderBinOp (Multiply _ ws) = do + singleton $ TkStar () + traverse_ renderWhitespace ws +renderBinOp (At _ ws) = do + singleton $ TkAt () + traverse_ renderWhitespace ws +renderBinOp (Divide _ ws) = do + singleton $ TkSlash () + traverse_ renderWhitespace ws +renderBinOp (FloorDivide _ ws) = do + singleton $ TkDoubleSlash () + traverse_ renderWhitespace ws +renderBinOp (Exp _ ws) = do + singleton $ TkDoubleStar () + traverse_ renderWhitespace ws +renderBinOp (BoolAnd _ ws) = do + singleton $ TkAnd () + traverse_ renderWhitespace ws +renderBinOp (BoolOr _ ws) = do + singleton $ TkOr () + traverse_ renderWhitespace ws +renderBinOp (Eq _ ws) = do + singleton $ TkDoubleEq () + traverse_ renderWhitespace ws +renderBinOp (Lt _ ws) = do + singleton $ TkLt () + traverse_ renderWhitespace ws +renderBinOp (LtEq _ ws) = do + singleton $ TkLte () + traverse_ renderWhitespace ws +renderBinOp (Gt _ ws) = do + singleton $ TkGt () + traverse_ renderWhitespace ws +renderBinOp (GtEq _ ws) = do + singleton $ TkGte () + traverse_ renderWhitespace ws +renderBinOp (NotEq _ ws) = do + singleton $ TkBangEq () + traverse_ renderWhitespace ws +renderBinOp (Percent _ ws) = do + singleton $ TkPercent () + traverse_ renderWhitespace ws +renderBinOp (BitOr _ ws) = do + singleton $ TkPipe () + traverse_ renderWhitespace ws +renderBinOp (BitXor _ ws) = do + singleton $ TkCaret () + traverse_ renderWhitespace ws +renderBinOp (BitAnd _ ws) = do + singleton $ TkAmpersand () + traverse_ renderWhitespace ws +renderBinOp (ShiftLeft _ ws) = do + singleton $ TkShiftLeft () + traverse_ renderWhitespace ws +renderBinOp (ShiftRight _ ws) = do + singleton $ TkShiftRight () + traverse_ renderWhitespace ws + +renderIndents :: Indents a -> RenderOutput () +renderIndents (Indents is _) = traverse_ renderIndent is + +renderModule :: Module v a -> RenderOutput () +renderModule ModuleEmpty = pure () +renderModule (ModuleBlankFinal a) = renderBlank a +renderModule (ModuleBlank a b c) = do + renderBlank a + singleton $ renderNewline b + renderModule c +renderModule (ModuleStatement a b) = do + renderStatement a + renderModule b + +-- | Render an entire Python module to 'Text' +showModule :: Module v a -> Text +showModule = showRenderOutput . renderModule + +-- | Render a single Python statement to 'Text' +showStatement :: Statement v a -> Text +showStatement = showRenderOutput . renderStatement + +-- | Render a single Python expression to 'Text' +showExpr :: Expr v a -> Text +showExpr = showRenderOutput . parensGenerator diff --git a/src/Language/Python/Internal/Render/Correction.hs b/src/Language/Python/Internal/Render/Correction.hs new file mode 100644 --- /dev/null +++ b/src/Language/Python/Internal/Render/Correction.hs @@ -0,0 +1,326 @@ +{-# language BangPatterns #-} +{-# language LambdaCase #-} + +{-| +Module : Language.Python.Internal.Render.Correction +Copyright : (C) CSIRO 2017-2018 +License : BSD3 +Maintainer : Isaac Elliott +Stability : experimental +Portability : non-portable + +There are configurations of the core syntax tree which won't print to valid Python +if we printed them naively. Many of these we catch in the +'Language.Python.Validation.Syntax' phase, because those mistakes correspond to +some Python syntax error. In other cases, the mistakes are more benign and have +a "resonable correction" which doesn't break the "print-parse idempotence" law. + +This module is where such corrections are defined +-} + +module Language.Python.Internal.Render.Correction where + +import Control.Lens.Fold (hasn't) +import Control.Lens.Getter ((^.)) +import Control.Lens.Plated (transform) +import Control.Lens.Setter ((.~), (<>~)) +import Data.Function ((&)) +import Data.List.NonEmpty (NonEmpty(..)) +import Data.Semigroup ((<>)) +import Data.Text (Text) + +import qualified Data.List.NonEmpty as NonEmpty +import qualified Data.Text as Text + +import Language.Python.Internal.Token +import Language.Python.Syntax.CommaSep +import Language.Python.Syntax.Expr +import Language.Python.Syntax.Ident +import Language.Python.Syntax.Numbers +import Language.Python.Syntax.Strings +import Language.Python.Syntax.Whitespace + +-- | Trailing commas can only be present in a parameter list of entirely +-- positional arguments. This removes the bad trailing comma, and appends +-- the comma's trailing whitespace to the previous token +correctParams :: CommaSep (Param v a) -> CommaSep (Param v a) +correctParams CommaSepNone = CommaSepNone +correctParams (CommaSepOne a) = CommaSepOne a +correctParams (CommaSepMany a (MkComma b) c) = + case c of + CommaSepNone -> + case a of + PositionalParam{} -> CommaSepMany a (MkComma b) c + _ -> CommaSepOne (a & trailingWhitespace <>~ b) + _ -> CommaSepMany a (MkComma b) (correctParams c) + +correctSpaces :: (PyToken () -> Text) -> [PyToken ()] -> [PyToken ()] +correctSpaces f = + transform $ + \case + a : b : rest + | isIdentifierChar (Text.last $ f a) + , isIdentifierChar (Text.head $ f b) + -> a : TkSpace () : b : rest + a@(TkFloat (FloatLiteralFull _ _ Nothing)) : b : rest + | isIdentifierChar (Text.head $ f b) -> a : TkSpace () : b : rest + a -> a + +correctNewlines :: [PyToken ()] -> [PyToken ()] +correctNewlines = + transform $ + \case + TkNewline CR () : TkNewline LF () : rest -> + TkNewline CRLF () : TkNewline LF () : rest + TkContinued CR () : TkNewline LF () : rest -> + TkContinued CRLF () : TkNewline LF () : rest + a -> a + +-- | +-- Two non-typed single-quoted strings cannot be lexically +-- adjacent, because this would be a parse error +-- +-- eg. '''' or """" +-- +-- we correct for this by inserting a single space where required +-- '' '' or "" "" +correctAdjacentStrings :: NonEmpty (StringLiteral a) -> NonEmpty (StringLiteral a) +correctAdjacentStrings (a :| []) = a :| [] +correctAdjacentStrings (a:|b:cs) = + if + _stringLiteralQuoteType a == _stringLiteralQuoteType b && + _stringLiteralStringType a == _stringLiteralStringType b && + null (a ^. trailingWhitespace) && + not (hasPrefix b) + then + NonEmpty.cons (a & trailingWhitespace .~ [Space]) (correctAdjacentStrings $ b :| cs) + else + NonEmpty.cons a $ correctAdjacentStrings (b :| cs) + +quoteChar :: QuoteType -> PyChar +quoteChar qt = + case qt of + SingleQuote -> Char_esc_singlequote + DoubleQuote -> Char_esc_doublequote + +quote :: QuoteType -> Char +quote qt = + case qt of + DoubleQuote -> '\"' + SingleQuote -> '\'' + +-- | When a backslash character, precedes an escape sequence it needs to be escaped +-- so that it doesn't interfere with the backslash that begins the escape sequence. +-- +-- For example: +-- +-- @['Char_lit' \'\\\\\', Char_esc_n]@ would naively render to \'\\\\n\', which +-- would parse to @['Char_esc_bslash', 'Char_lit' \'n\']@, breaking the +-- @parse . print@ identity +correctBackslashEscapes :: [PyChar] -> [PyChar] +correctBackslashEscapes [] = [] +correctBackslashEscapes [x] = [x] +correctBackslashEscapes (x:y:ys) = + case x of + Char_lit '\\' + -- if the next character is an escape sequence, then the current backslash + -- must be escaped + | isEscape y -> Char_esc_bslash : y : correctBackslashEscapes ys + | Char_lit c <- y -> + case c of + '\\' -> Char_esc_bslash : correctBackslashEscapes ys + '\'' -> Char_esc_bslash : correctBackslashEscapes ys + '\"' -> Char_esc_bslash : correctBackslashEscapes ys + -- if we print out ['\', 'u'] then the parser will think it's beginning a + -- unicode point + 'u' -> Char_esc_bslash : y : correctBackslashEscapes ys + 'U' -> Char_esc_bslash : y : correctBackslashEscapes ys + -- same for 'x' and hex values + 'x' -> Char_esc_bslash : y : correctBackslashEscapes ys + _ -> x : correctBackslashEscapes (y : ys) + _ -> x : correctBackslashEscapes (y : ys) + +correctBackslashes :: [PyChar] -> [PyChar] +correctBackslashes [] = [] +correctBackslashes [x] = + case x of + Char_lit '\\' -> [Char_esc_bslash] + _ -> [x] +correctBackslashes (x:y:ys) = + case x of + Char_lit '\\' + -- if the next character is an escape sequence, then the current backslash + -- must be escaped + | Char_esc_bslash <- y -> Char_esc_bslash : y : correctBackslashes ys + _ -> x : correctBackslashes (y : ys) + +-- | @(as, bs) = span p xs@ +-- @bs@ is the longest suffix that satisfies the predicate, and @as@ is the +-- prefix up to that point +-- +-- It's like the reverse of 'span' +naps :: (a -> Maybe b) -> [a] -> ([a], [b]) +naps p = go (,) (,) + where + go _ r [] = r [] [] + go l r (x:xs) = + go + (\res res' -> l (x:res) res') + (\res res' -> + case p x of + Just x' -> r res (x':res') + Nothing -> l (x:res) res') + xs + +-- | Sometimes strings need to be corrected when certain characters follow a literal +-- backslash. For example, a literal backslash followed by an escape sequence means +-- that the literal backslash actually needs to be escaped, so that it doesn't get +-- 'combined' with the backslash in the escape sequence. +correctBackslashEscapesRaw :: [PyChar] -> [PyChar] +correctBackslashEscapesRaw [] = [] +correctBackslashEscapesRaw [x] = [x] +correctBackslashEscapesRaw(x:y:ys) = + case x of + Char_lit '\\' -> + case y of + Char_esc_doublequote -> Char_esc_bslash : y : correctBackslashEscapesRaw ys + Char_esc_singlequote -> Char_esc_bslash : y : correctBackslashEscapesRaw ys + Char_esc_bslash -> Char_esc_bslash : correctBackslashEscapesRaw (Char_lit '\\' : ys) + _ -> x : correctBackslashEscapesRaw (y : ys) + _ -> x : correctBackslashEscapesRaw (y : ys) + +-- | It turns out that raw strings can only ever be constructed with an even number +-- of trailing backslash characters. This functon corrects raw strings with an +-- odd number of trailing backslash characters +correctBackslashesRaw :: [PyChar] -> [PyChar] +correctBackslashesRaw ps = + let + (as, bs) = + naps + (\a -> + case a of + Char_lit '\\' -> Just a + Char_esc_bslash -> Just a + _ -> Nothing) + ps + in + if even (numSlashes bs) + then ps + else + as <> (Char_lit '\\' : bs) + where + numSlashes :: [PyChar] -> Int + numSlashes [] = 0 + numSlashes (Char_lit '\\' : xs) = 1 + numSlashes xs + numSlashes (Char_esc_bslash : xs) = 2 + numSlashes xs + numSlashes _ = undefined + +-- | Every quote in a string of a particular quote type should be escaped +correctQuotes :: QuoteType -> [PyChar] -> [PyChar] +correctQuotes qt = + fmap + (case qt of + DoubleQuote -> \case; Char_lit '"' -> Char_esc_doublequote; c -> c + SingleQuote -> \case; Char_lit '\'' -> Char_esc_singlequote; c -> c) + +-- | Every quote in short raw string that isn't preceded by +-- a backslash should be escaped +correctQuotesRaw :: QuoteType -> [PyChar] -> [PyChar] +correctQuotesRaw _ [] = [] +correctQuotesRaw qt [x] = + case x of + Char_lit c | quote qt == c -> [quoteChar qt] + _ -> [x] +correctQuotesRaw qt (x:y:ys) = + case x of + Char_lit c | q == c -> go (qc:y:ys) + _ -> go (x:y:ys) + where + qc = quoteChar qt + q = quote qt + + go [] = [] + go [x] = [x] + go (x:y:ys) = + case x of + Char_lit '\\' -> x : go (y:ys) + _ -> + case y of + Char_lit c | q == c -> x : go (qc:ys) + _ -> x : go (y:ys) + +-- | Every third literal quote at the beginning of a long (non-raw) string should +-- be escaped +correctInitialQuotes :: QuoteType -> [PyChar] -> [PyChar] +correctInitialQuotes qt = go (0::Int) + where + qc = quoteChar qt + q = quote qt + + go !_ [] = [] + go !n (c:cs) = + if c == Char_lit q + then + if n == 2 + then qc : go (n+1 `mod` 3) cs + else c : go (n+1 `mod` 3) cs + else c : cs + +-- | Literal quotes at the beginning and end of a long raw string should be escaped +correctInitialFinalQuotesLongRaw :: QuoteType -> [PyChar] -> [PyChar] +correctInitialFinalQuotesLongRaw qt = correctFinalQuotes . correctInitialQuotes qt + where + qc = quoteChar qt + q = quote qt + + -- | Literal quotes at the end of a long raw string should be escaped + correctFinalQuotes :: [PyChar] -> [PyChar] + correctFinalQuotes = snd . go + where + go [] = (True, []) + go (c:cs) = + if c /= Char_lit '\\' + then + case go cs of + (b, cs') -> + if b && c == Char_lit q + then (True, qc : cs') + else (False, c : cs') + else + let + (ds, es) = span (== Char_lit '\\') cs + in + case es of + [] -> (False, c : ds) + e':es' -> + case go es' of + (_, es'') -> (False, c : ds <> (e' : es'')) + +-- | Literal quotes at the beginning and end of a long (non-raw) string should be escaped +correctInitialFinalQuotesLong :: QuoteType -> [PyChar] -> [PyChar] +correctInitialFinalQuotesLong qt = correctFinalQuotes . correctInitialQuotes qt + where + qc = quoteChar qt + q = quote qt + + -- | Literal quotes at the end of a long (non-raw) string should be escaped + correctFinalQuotes :: [PyChar] -> [PyChar] + correctFinalQuotes = snd . go + where + go [] = (True, []) + go (c:cs) = + case go cs of + (b, cs') -> + if b && c == Char_lit q + then (True, qc : cs') + else (False, c : cs') + +-- | It's possible that successive statements have no newlines in between +-- them. This would cause them to be displayed on the same line. In every line where +-- this would be the case, we explicitly insert a line-feed character. +correctTrailingNewline :: HasTrailingNewline s => Bool -> s v a -> s v a +correctTrailingNewline False s = + if hasn't trailingNewline s + then setTrailingNewline s LF + else s +correctTrailingNewline True s = s diff --git a/src/Language/Python/Internal/Syntax/IR.hs b/src/Language/Python/Internal/Syntax/IR.hs new file mode 100644 --- /dev/null +++ b/src/Language/Python/Internal/Syntax/IR.hs @@ -0,0 +1,1005 @@ +{-# language DataKinds #-} +{-# language DeriveFunctor, DeriveFoldable, DeriveTraversable #-} +{-# language FunctionalDependencies, MultiParamTypeClasses #-} +{-# language LambdaCase #-} +{-# language TemplateHaskell #-} + +{-| +Module : Language.Python.Internal.Syntax.IR +Copyright : (C) CSIRO 2017-2018 +License : BSD3 +Maintainer : Isaac Elliott +Stability : experimental +Portability : non-portable + +This module only exists as our current best solution to decoupling parts of the +concrete syntax from abstract syntax. You won't need to care about its existence +and hopefully it will be deleted soon. + +-} + +module Language.Python.Internal.Syntax.IR where + +import Control.Lens.Fold (foldMapOf, folded) +import Control.Lens.Getter ((^.)) +import Control.Lens.Lens (Lens', lens) +import Control.Lens.Prism (Prism') +import Control.Lens.Review ((#)) +import Control.Lens.Setter ((.~), over, mapped) +import Control.Lens.TH (makeLenses) +import Control.Lens.Traversal (traverseOf) +import Control.Lens.Tuple (_1, _2, _3) +import Data.Bifoldable (bifoldMap) +import Data.Bifunctor (bimap) +import Data.Bitraversable (bitraverse) +import Data.Function ((&)) +import Data.List.NonEmpty (NonEmpty) +import Data.Monoid ((<>)) +import Data.Validation (Validation(..)) + +import Language.Python.Syntax.AugAssign +import Language.Python.Syntax.CommaSep +import Language.Python.Syntax.Comment +import Language.Python.Syntax.Ident +import Language.Python.Syntax.Import +import Language.Python.Syntax.ModuleNames +import Language.Python.Syntax.Numbers +import Language.Python.Syntax.Operator.Binary +import Language.Python.Syntax.Operator.Unary +import Language.Python.Syntax.Punctuation +import Language.Python.Syntax.Strings +import Language.Python.Syntax.Whitespace + +import qualified Language.Python.Syntax.Module as Syntax +import qualified Language.Python.Syntax.Expr as Syntax +import qualified Language.Python.Syntax.Statement as Syntax + +class AsIRError s a | s -> a where + _InvalidUnpacking :: Prism' s a + +data IRError a + -- | Unpacking ( @*value@ ) was used in an invalid position + = InvalidUnpacking a + deriving (Eq, Show) + +fromIRError :: AsIRError s a => IRError a -> s +fromIRError (InvalidUnpacking a) = _InvalidUnpacking # a + +data SmallStatement a + = MkSmallStatement + (SimpleStatement a) + [(Semicolon a, SimpleStatement a)] + (Maybe (Semicolon a)) + (Maybe (Comment a)) + (Maybe Newline) + deriving (Eq, Show, Functor, Foldable, Traversable) + +data Statement a + = SmallStatement (Indents a) (SmallStatement a) + | CompoundStatement (CompoundStatement a) + deriving (Eq, Show, Functor, Foldable, Traversable) + +data CompoundStatement a + = Fundef + { _csAnn :: a + , _unsafeCsFundefDecorators :: [Decorator a] + , _csIndents :: Indents a + , _unsafeCsFundefAsync :: Maybe (NonEmpty Whitespace) -- ^ @[\'async\' \]@ + , _unsafeCsFundefDef :: NonEmpty Whitespace -- ^ @\'def\' \@ + , _unsafeCsFundefName :: Ident '[] a -- ^ @\@ + , _unsafeCsFundefLeftParen :: [Whitespace] -- ^ @\'(\' \@ + , _unsafeCsFundefParameters :: CommaSep (Param a) -- ^ @\@ + , _unsafeCsFundefRightParen :: [Whitespace] -- ^ @\')\' \@ + , _unsafeCsFundefReturnType :: Maybe ([Whitespace], Expr a) -- ^ @[\'->\' \ \]@ + , _unsafeCsFundefBody :: Suite a -- ^ @\@ + } + | If + { _csAnn :: a + , _csIndents :: Indents a + , _unsafeCsIfIf :: [Whitespace] -- ^ @\'if\' \@ + , _unsafeCsIfCond :: Expr a -- ^ @\@ + , _unsafeCsIfBody :: Suite a -- ^ @\@ + , _unsafeCsIfElifs :: [(Indents a, [Whitespace], Expr a, Suite a)] -- ^ @(\'elif\' \ \ \)*@ + , _unsafeCsIfElse :: Maybe (Indents a, [Whitespace], Suite a) -- ^ @[\'else\' \ \]@ + } + | While + { _csAnn :: a + , _csIndents :: Indents a + , _unsafeCsWhileWhile :: [Whitespace] -- ^ @\'while\' \@ + , _unsafeCsWhileCond :: Expr a -- ^ @\@ + , _unsafeCsWhileBody :: Suite a -- ^ @\@ + , _unsafeCsWhileElse + :: Maybe (Indents a, [Whitespace], Suite a) -- ^ @[\'else\' \ \]@ + } + | TryExcept + { _csAnn :: a + , _csIndents :: Indents a + , _unsafeCsTryExceptTry :: [Whitespace] -- ^ @\'try\' \@ + , _unsafeCsTryExceptBody :: Suite a -- ^ @\@ + , _unsafeCsTryExceptExcepts :: NonEmpty (Indents a, [Whitespace], Maybe (ExceptAs a), Suite a) -- ^ @(\'except\' \ \ \)+@ + , _unsafeCsTryExceptElse :: Maybe (Indents a, [Whitespace], Suite a) -- ^ @[\'else\' \ \]@ + , _unsafeCsTryExceptFinally :: Maybe (Indents a, [Whitespace], Suite a) -- ^ @[\'finally\' \ \]@ + } + | TryFinally + { _csAnn :: a + , _csIndents :: Indents a + , _unsafeCsTryFinallyTry :: [Whitespace] -- ^ @\'try\' \@ + , _unsafeCsTryFinallyTryBody :: Suite a -- ^ @\@ + , _unsafeCsTryFinallyFinallyIndents :: Indents a + , _unsafeCsTryFinallyFinally :: [Whitespace] -- ^ @\'finally\' \@ + , _unsafeCsTryFinallyFinallyBody :: Suite a -- ^ @\@ + } + | For + { _csAnn :: a + , _csIndents :: Indents a + , _unsafeCsForAsync :: Maybe (NonEmpty Whitespace) -- ^ @[\'async\' \]@ + , _unsafeCsForFor :: [Whitespace] -- ^ @\'for\' \@ + , _unsafeCsForBinder :: Expr a -- ^ @\@ + , _unsafeCsForIn :: [Whitespace] -- ^ @\'in\' \@ + , _unsafeCsForCollection :: CommaSep1' (Expr a) -- ^ @\@ + , _unsafeCsForBody :: Suite a -- ^ @\@ + , _unsafeCsForElse :: Maybe (Indents a, [Whitespace], Suite a) -- ^ @[\'else\' \ \]@ + } + | ClassDef + { _csAnn :: a + , _unsafeCsClassDefDecorators :: [Decorator a] + , _csIndents :: Indents a + , _unsafeCsClassDefClass :: NonEmpty Whitespace -- ^ @\'class\' \@ + , _unsafeCsClassDefName :: Ident '[] a -- ^ @\@ + , _unsafeCsClassDefArguments :: Maybe ([Whitespace], Maybe (CommaSep1' (Arg a)), [Whitespace]) -- ^ @[\'(\' \ [\] \')\' \]@ + , _unsafeCsClassDefBody :: Suite a -- ^ @\@ + } + | With + { _csAnn :: a + , _csIndents :: Indents a + , _unsafeCsWithAsync :: Maybe (NonEmpty Whitespace) -- ^ @[\'async\' \]@ + , _unsafeCsWithWith :: [Whitespace] -- ^ @\'with\' \@ + , _unsafeCsWithItems :: CommaSep1 (WithItem a) -- ^ @\@ + , _unsafeCsWithBody :: Suite a -- ^ @\@ + } + deriving (Eq, Show, Functor, Foldable, Traversable) + +data SimpleStatement a + = Return a [Whitespace] (Maybe (Expr a)) + | Expr a (Expr a) + | Assign a (Expr a) (NonEmpty (Equals, Expr a)) + | AugAssign a (Expr a) (AugAssign a) (Expr a) + | Pass a [Whitespace] + | Break a [Whitespace] + | Continue a [Whitespace] + | Global a (NonEmpty Whitespace) (CommaSep1 (Ident '[] a)) + | Nonlocal a (NonEmpty Whitespace) (CommaSep1 (Ident '[] a)) + | Del a [Whitespace] (CommaSep1' (Expr a)) + | Import + a + (NonEmpty Whitespace) + (CommaSep1 (ImportAs (ModuleName '[]) '[] a)) + | From + a + [Whitespace] + (RelativeModuleName '[] a) + [Whitespace] + (ImportTargets '[] a) + | Raise a + [Whitespace] + (Maybe (Expr a, Maybe ([Whitespace], Expr a))) + | Assert a + [Whitespace] + (Expr a) + (Maybe (Comma, Expr a)) + deriving (Eq, Show, Functor, Foldable, Traversable) + +data Param a + = PositionalParam + { _paramAnn :: a + , _paramName :: Ident '[] a + , _paramType :: Maybe (Colon, Expr a) + } + | KeywordParam + { _paramAnn :: a + , _paramName :: Ident '[] a + -- ':' spaces + , _paramType :: Maybe (Colon, Expr a) + -- = spaces + , _unsafeKeywordParamWhitespaceRight :: [Whitespace] + , _unsafeKeywordParamExpr :: Expr a + } + | StarParam + { _paramAnn :: a + -- '*' spaces + , _unsafeStarParamWhitespace :: [Whitespace] + , _unsafeStarParamName :: Ident '[] a + , _paramType :: Maybe (Colon, Expr a) + } + | UnnamedStarParam + { _paramAnn :: a + -- '*' spaces + , _unsafeUnnamedStarParamWhitespace :: [Whitespace] + } + | DoubleStarParam + { _paramAnn :: a + -- '**' spaces + , _unsafeDoubleStarParamWhitespace :: [Whitespace] + , _paramName :: Ident '[] a + , _paramType :: Maybe (Colon, Expr a) + } + deriving (Eq, Show, Functor, Foldable, Traversable) + +data CompIf a + = CompIf a [Whitespace] (Expr a) -- ^ 'if' + deriving (Eq, Show, Functor, Foldable, Traversable) + +data CompFor a + = CompFor a [Whitespace] (Expr a) [Whitespace] (Expr a) -- ^ 'for' 'in' + deriving (Eq, Show, Functor, Foldable, Traversable) + +data Comprehension e a + = Comprehension a (e a) (CompFor a) [Either (CompFor a) (CompIf a)] -- ^ (comp_for | comp_if)* + deriving (Eq, Show) + +instance Functor e => Functor (Comprehension e) where + fmap f (Comprehension a b c d) = + Comprehension (f a) (fmap f b) (fmap f c) (fmap (bimap (fmap f) (fmap f)) d) + +instance Foldable e => Foldable (Comprehension e) where + foldMap f (Comprehension a b c d) = + f a <> foldMap f b <> foldMap f c <> foldMap (bifoldMap (foldMap f) (foldMap f)) d + +instance Traversable e => Traversable (Comprehension e) where + traverse f (Comprehension a b c d) = + Comprehension <$> + f a <*> + traverse f b <*> + traverse f c <*> + traverse (bitraverse (traverse f) (traverse f)) d + +data Subscript a + = SubscriptExpr (Expr a) + | SubscriptSlice + -- [expr] + (Maybe (Expr a)) + -- ':' + Colon + -- [expr] + (Maybe (Expr a)) + -- [':' [expr]] + (Maybe (Colon, Maybe (Expr a))) + deriving (Eq, Show, Functor, Foldable, Traversable) + +data DictItem a + = DictItem + { _dictItemAnn :: a + , _unsafeDictItemKey :: Expr a + , _unsafeDictItemColon :: Colon + , _unsafeDictItemvalue :: Expr a + } + | DictUnpack + { _dictItemAnn :: a + , _unsafeDictItemUnpackWhitespace :: [Whitespace] + , _unsafeDictItemUnpackValue :: Expr a + } deriving (Eq, Show, Functor, Foldable, Traversable) + +data Arg a + = PositionalArg + { _argAnn :: a + , _argExpr :: Expr a + } + | KeywordArg + { _argAnn :: a + , _unsafeKeywordArgName :: Ident '[] a + , _unsafeKeywordArgWhitespaceRight :: [Whitespace] + , _argExpr :: Expr a + } + | StarArg + { _argAnn :: a + , _unsafeStarArgWhitespace :: [Whitespace] + , _argExpr :: Expr a + } + | DoubleStarArg + { _argAnn :: a + , _unsafeDoubleStarArgWhitespace :: [Whitespace] + , _argExpr :: Expr a + } + deriving (Eq, Show, Functor, Foldable, Traversable) + +data Expr a + = StarExpr + { _unsafeExprAnn :: a + , _unsafeStarExprWhitespace :: [Whitespace] + , _unsafeStarExprValue :: Expr a + } + | Unit + { _unsafeExprAnn :: a + , _unsafeUnitWhitespaceInner :: [Whitespace] + , _unsafeUnitWhitespaceRight :: [Whitespace] + } + | Lambda + { _unsafeExprAnn :: a + , _unsafeLambdaWhitespace :: [Whitespace] + , _unsafeLambdaArgs :: CommaSep (Param a) + , _unsafeLambdaColon :: Colon + , _unsafeLambdaBody :: Expr a + } + | Yield + { _unsafeExprAnn :: a + , _unsafeYieldWhitespace :: [Whitespace] + , _unsafeYieldValue :: CommaSep (Expr a) + } + | YieldFrom + { _unsafeExprAnn :: a + , _unsafeYieldWhitespace :: [Whitespace] + , _unsafeFromWhitespace :: [Whitespace] + , _unsafeYieldFromValue :: Expr a + } + | Ternary + { _unsafeExprAnn :: a + -- expr + , _unsafeTernaryValue :: Expr a + -- 'if' spaces + , _unsafeTernaryWhitespaceIf :: [Whitespace] + -- expr + , _unsafeTernaryCond :: Expr a + -- 'else' spaces + , _unsafeTernaryWhitespaceElse :: [Whitespace] + -- expr + , _unsafeTernaryElse :: Expr a + } + | ListComp + { _unsafeExprAnn :: a + -- [ spaces + , _unsafeListCompWhitespaceLeft :: [Whitespace] + -- comprehension + , _unsafeListCompValue :: Comprehension Expr a + -- ] spaces + , _unsafeListCompWhitespaceRight :: [Whitespace] + } + | List + { _unsafeExprAnn :: a + -- [ spaces + , _unsafeListWhitespaceLeft :: [Whitespace] + -- exprs + , _unsafeListValues :: Maybe (CommaSep1' (Expr a)) + -- ] spaces + , _unsafeListWhitespaceRight :: [Whitespace] + } + | DictComp + { _unsafeExprAnn :: a + -- { spaces + , _unsafeDictCompWhitespaceLeft :: [Whitespace] + -- comprehension + , _unsafeDictCompValue :: Comprehension DictItem a + -- } spaces + , _unsafeDictCompWhitespaceRight :: [Whitespace] + } + | Dict + { _unsafeExprAnn :: a + , _unsafeDictWhitespaceLeft :: [Whitespace] + , _unsafeDictValues :: Maybe (CommaSep1' (DictItem a)) + , _unsafeDictWhitespaceRight :: [Whitespace] + } + | SetComp + { _unsafeExprAnn :: a + -- { spaces + , _unsafeSetCompWhitespaceLeft :: [Whitespace] + -- comprehension + , _unsafeSetCompValue :: Comprehension Expr a + -- } spaces + , _unsafeSetCompWhitespaceRight :: [Whitespace] + } + | Set + { _unsafeExprAnn :: a + , _unsafeSetWhitespaceLeft :: [Whitespace] + , _unsafeSetValues :: CommaSep1' (Expr a) + , _unsafeSetWhitespaceRight :: [Whitespace] + } + | Deref + { _unsafeExprAnn :: a + -- expr + , _unsafeDerefValueLeft :: Expr a + -- . spaces + , _unsafeDerefWhitespaceLeft :: [Whitespace] + -- ident + , _unsafeDerefValueRight :: Ident '[] a + } + | Subscript + { _unsafeExprAnn :: a + -- expr + , _unsafeSubscriptValueLeft :: Expr a + -- [ spaces + , _unsafeSubscriptWhitespaceLeft :: [Whitespace] + -- expr + , _unsafeSubscriptValueRight :: CommaSep1' (Subscript a) + -- ] spaces + , _unsafeSubscriptWhitespaceRight :: [Whitespace] + } + | Call + { _unsafeExprAnn :: a + -- expr + , _unsafeCallFunction :: Expr a + -- ( spaces + , _unsafeCallWhitespaceLeft :: [Whitespace] + -- exprs + , _unsafeCallArguments :: Maybe (CommaSep1' (Arg a)) + -- ) spaces + , _unsafeCallWhitespaceRight :: [Whitespace] + } + | None + { _unsafeExprAnn :: a + , _unsafeNoneWhitespace :: [Whitespace] + } + | Ellipsis + { _unsafeExprAnn :: a + , _unsafeEllipsisWhitespace :: [Whitespace] + } + | BinOp + { _unsafeExprAnn :: a + , _unsafeBinOpExprLeft :: Expr a + , _unsafeBinOpOp :: BinOp a + , _unsafeBinOpExprRight :: Expr a + } + | UnOp + { _unsafeExprAnn :: a + , _unsafeUnOpOp :: UnOp a + , _unsafeUnOpValue :: Expr a + } + | Parens + { _unsafeExprAnn :: a + -- ( spaces + , _unsafeParensWhitespaceLeft :: [Whitespace] + -- expr + , _unsafeParensValue :: Expr a + -- ) spaces + , _unsafeParensWhitespaceAfter :: [Whitespace] + } + | Ident + { _unsafeIdentValue :: Ident '[] a + } + | Int + { _unsafeExprAnn :: a + , _unsafeIntValue :: IntLiteral a + , _unsafeIntWhitespace :: [Whitespace] + } + | Float + { _unsafeExprAnn :: a + , _unsafeFloatValue :: FloatLiteral a + , _unsafeFloatWhitespace :: [Whitespace] + } + | Imag + { _unsafeExprAnn :: a + , _unsafeImagValue :: ImagLiteral a + , _unsafeImagWhitespace :: [Whitespace] + } + | Bool + { _unsafeExprAnn :: a + , _unsafeBoolValue :: Bool + , _unsafeBoolWhitespace :: [Whitespace] + } + | String + { _unsafeExprAnn :: a + , _unsafeStringLiteralValue :: NonEmpty (StringLiteral a) + } + | Tuple + { _unsafeExprAnn :: a + -- expr + , _unsafeTupleHead :: Expr a + -- , spaces + , _unsafeTupleWhitespace :: Comma + -- [exprs] + , _unsafeTupleTail :: Maybe (CommaSep1' (Expr a)) + } + | Not + { _unsafeExprAnn :: a + , _unsafeNotWhitespace :: [Whitespace] + , _unsafeNotValue :: Expr a + } + | Generator + { _unsafeExprAnn :: a + , _generatorValue :: Comprehension Expr a + } + | Await + { _unsafeExprAnn :: a + , _unsafeAwaitWhitespace :: [Whitespace] + , _unsafeAwaitValue :: Expr a + } + deriving (Eq, Show, Functor, Foldable, Traversable) + +exprAnn :: Lens' (Expr a) a +exprAnn = + lens + (\case + Unit a _ _ -> a + StarExpr a _ _ -> a + Lambda a _ _ _ _ -> a + Yield a _ _ -> a + YieldFrom a _ _ _ -> a + Ternary a _ _ _ _ _ -> a + None a _ -> a + Ellipsis a _ -> a + List a _ _ _ -> a + ListComp a _ _ _ -> a + Deref a _ _ _ -> a + Subscript a _ _ _ _ -> a + Call a _ _ _ _ -> a + BinOp a _ _ _ -> a + UnOp a _ _ -> a + Parens a _ _ _ -> a + Ident a -> a ^. identAnn + Int a _ _ -> a + Float a _ _ -> a + Imag a _ _ -> a + Bool a _ _ -> a + String a _ -> a + Not a _ _ -> a + Tuple a _ _ _ -> a + DictComp a _ _ _ -> a + Dict a _ _ _ -> a + SetComp a _ _ _ -> a + Set a _ _ _ -> a + Generator a _ -> a + Await a _ _ -> a) + (\e ann -> + case e of + Unit _ a b -> Unit ann a b + StarExpr _ a b -> StarExpr ann a b + Lambda _ a b c d -> Lambda ann a b c d + Yield _ a b -> Yield ann a b + YieldFrom ann a b c -> YieldFrom ann a b c + Ternary ann a b c d e -> Ternary ann a b c d e + None _ a -> None ann a + Ellipsis _ a -> Ellipsis ann a + List _ a b c -> List ann a b c + ListComp _ a b c -> ListComp ann a b c + Deref _ a b c -> Deref ann a b c + Subscript _ a b c d -> Subscript ann a b c d + Call _ a b c d -> Call ann a b c d + BinOp _ a b c -> BinOp ann a b c + UnOp _ a b -> UnOp ann a b + Parens _ a b c -> Parens ann a b c + Ident a -> Ident $ a & identAnn .~ ann + Int _ a b -> Int ann a b + Float _ a b -> Float ann a b + Imag _ a b -> Imag ann a b + Bool _ a b -> Bool ann a b + String _ a -> String ann a + Not _ a b -> Not ann a b + Tuple _ a b c -> Tuple ann a b c + DictComp _ a b c -> DictComp ann a b c + Dict _ a b c -> Dict ann a b c + SetComp _ a b c -> SetComp ann a b c + Set _ a b c -> Set ann a b c + Generator _ a -> Generator ann a + Await _ a b -> Not ann a b) + +data Suite a + -- ':' smallStatement + = SuiteOne a Colon (SmallStatement a) + | SuiteMany a + -- ':' [comment] + Colon (Maybe (Comment a)) Newline + -- + (Block a) + deriving (Eq, Show, Functor, Foldable, Traversable) + +data Block a + = Block + { _blockBlankLines :: [(Blank a, Newline)] + , _blockHead :: Statement a + , _blockTail :: [Either (Blank a, Newline) (Statement a)] + } deriving (Eq, Show) + +instance Functor Block where + fmap f (Block a b c) = + Block + (over (mapped._1.mapped) f a) + (fmap f b) + (bimap (over (_1.mapped) f) (fmap f) <$> c) + +instance Foldable Block where + foldMap f (Block a b c) = + foldMapOf (folded._1.folded) f a <> + foldMap f b <> + foldMap (bifoldMap (foldMapOf (_1.folded) f) (foldMap f)) c + +instance Traversable Block where + traverse f (Block a b c) = + Block <$> + traverseOf (traverse._1.traverse) f a <*> + traverse f b <*> + traverse (bitraverse (traverseOf (_1.traverse) f) (traverse f)) c + +data WithItem a + = WithItem + { _withItemAnn :: a + , _withItemValue :: Expr a + , _withItemBinder :: Maybe ([Whitespace], Expr a) + } + deriving (Eq, Show, Functor, Foldable, Traversable) + +data Decorator a + = Decorator + { _decoratorAnn :: a + , _decoratorIndents :: Indents a + , _decoratorAt :: At + , _decoratorExpr :: Expr a + , _decoratorComment :: Maybe (Comment a) + , _decoratorNewline :: Newline + , _decoratorBlankLines :: [(Blank a, Newline)] + } + deriving (Eq, Show, Functor, Foldable, Traversable) + +data ExceptAs a + = ExceptAs + { _exceptAsAnn :: a + , _exceptAsExpr :: Expr a + , _exceptAsName :: Maybe ([Whitespace], Ident '[] a) + } + deriving (Eq, Show, Functor, Foldable, Traversable) + +data Module a + = ModuleEmpty + | ModuleBlankFinal (Blank a) + | ModuleBlank (Blank a) Newline (Module a) + | ModuleStatement (Statement a) (Module a) + deriving (Eq, Show, Functor, Foldable, Traversable) + +data FromIRContext + = FromIRContext + { _allowStarred :: Bool + } + +makeLenses ''FromIRContext + +fromIR_expr + :: AsIRError e a + => Expr a + -> Validation (NonEmpty e) (Syntax.Expr '[] a) +fromIR_expr ex = + case ex of + StarExpr{} -> Failure $ pure (_InvalidUnpacking # (ex ^. exprAnn)) + Unit a b c -> pure $ Syntax.Unit a b c + Lambda a b c d e -> + (\c' -> Syntax.Lambda a b c' d) <$> + traverse fromIR_param c <*> + fromIR_expr e + Yield a b c -> Syntax.Yield a b <$> traverse fromIR_expr c + YieldFrom a b c d -> Syntax.YieldFrom a b c <$> fromIR_expr d + Ternary a b c d e f -> + (\b' d' -> Syntax.Ternary a b' c d' e) <$> + fromIR_expr b <*> + fromIR_expr d <*> + fromIR_expr f + ListComp a b c d -> + (\c' -> Syntax.ListComp a b c' d) <$> + fromIR_comprehension fromIR_expr c + List a b c d -> + (\c' -> Syntax.List a b c' d) <$> + traverseOf (traverse.traverse) fromIR_listItem c + DictComp a b c d -> + (\c' -> Syntax.DictComp a b c' d) <$> + fromIR_comprehension fromIR_dictItem c + Dict a b c d -> + (\c' -> Syntax.Dict a b c' d) <$> + traverseOf (traverse.traverse) fromIR_dictItem c + SetComp a b c d -> + (\c' -> Syntax.SetComp a b c' d) <$> + fromIR_comprehension fromIR_setItem c + Set a b c d -> + (\c' -> Syntax.Set a b c' d) <$> + traverse fromIR_setItem c + Deref a b c d -> + (\b' -> Syntax.Deref a b' c d) <$> + fromIR_expr b + Subscript a b c d e -> + (\b' d' -> Syntax.Subscript a b' c d' e) <$> + fromIR_expr b <*> + traverse fromIR_subscript d + Call a b c d e -> + (\b' d' -> Syntax.Call a b' c d' e) <$> + fromIR_expr b <*> + traverseOf (traverse.traverse) fromIR_arg d + None a b -> pure $ Syntax.None a b + Ellipsis a b -> pure $ Syntax.Ellipsis a b + BinOp a b c d -> + (\b' d' -> Syntax.BinOp a b' c d') <$> + fromIR_expr b <*> + fromIR_expr d + UnOp a b c -> + Syntax.UnOp a b <$> fromIR_expr c + Parens a b c d -> + (\c' -> Syntax.Parens a b c' d) <$> + fromIR_expr c + Ident a -> pure $ Syntax.Ident a + Int a b c -> pure $ Syntax.Int a b c + Float a b c -> pure $ Syntax.Float a b c + Imag a b c -> pure $ Syntax.Imag a b c + Bool a b c -> pure $ Syntax.Bool a b c + String a b -> pure $ Syntax.String a b + Tuple a b c d -> + (\b' -> Syntax.Tuple a b' c) <$> + fromIR_tupleItem b <*> + traverseOf (traverse.traverse) fromIR_tupleItem d + Not a b c -> Syntax.Not a b <$> fromIR_expr c + Generator a b -> Syntax.Generator a <$> fromIR_comprehension fromIR_expr b + Await a b c -> Syntax.Await a b <$> fromIR_expr c + +fromIR_suite + :: AsIRError e a + => Suite a + -> Validation (NonEmpty e) (Syntax.Suite '[] a) +fromIR_suite s = + case s of + SuiteOne a b c -> + Syntax.SuiteOne a b <$> fromIR_smallStatement c + SuiteMany a b c d e -> + Syntax.SuiteMany a b c d <$> fromIR_block e + +fromIR_param + :: AsIRError e a + => Param a + -> Validation (NonEmpty e) (Syntax.Param '[] a) +fromIR_param p = + case p of + PositionalParam a b c -> + Syntax.PositionalParam a b <$> traverseOf (traverse._2) fromIR_expr c + KeywordParam a b c d e -> + Syntax.KeywordParam a b <$> + traverseOf (traverse._2) fromIR_expr c <*> + pure d <*> + fromIR_expr e + StarParam a b c d -> + Syntax.StarParam a b c <$> traverseOf (traverse._2) fromIR_expr d + UnnamedStarParam a b -> pure $ Syntax.UnnamedStarParam a b + DoubleStarParam a b c d -> + Syntax.DoubleStarParam a b c <$> traverseOf (traverse._2) fromIR_expr d + +fromIR_arg + :: AsIRError e a + => Arg a + -> Validation (NonEmpty e) (Syntax.Arg '[] a) +fromIR_arg a = + case a of + PositionalArg a b -> Syntax.PositionalArg a <$> fromIR_expr b + KeywordArg a b c d -> Syntax.KeywordArg a b c <$> fromIR_expr d + StarArg a b c -> Syntax.StarArg a b <$> fromIR_expr c + DoubleStarArg a b c -> Syntax.DoubleStarArg a b <$> fromIR_expr c + +fromIR_decorator + :: AsIRError e a + => Decorator a + -> Validation (NonEmpty e) (Syntax.Decorator '[] a) +fromIR_decorator (Decorator a b c d e f g) = + (\d' -> Syntax.Decorator a b c d' e f g) <$> + fromIR_expr d + +fromIR_exceptAs + :: AsIRError e a + => ExceptAs a + -> Validation (NonEmpty e) (Syntax.ExceptAs '[] a) +fromIR_exceptAs (ExceptAs a b c) = + (\b' -> Syntax.ExceptAs a b' c) <$> + fromIR_expr b + +fromIR_withItem + :: AsIRError e a + => WithItem a + -> Validation (NonEmpty e) (Syntax.WithItem '[] a) +fromIR_withItem (WithItem a b c) = + Syntax.WithItem a <$> + fromIR_expr b <*> + traverseOf (traverse._2) fromIR_expr c + +fromIR_comprehension + :: AsIRError e a + => (ex a -> Validation (NonEmpty e) (ex' '[] a)) + -> Comprehension ex a + -> Validation (NonEmpty e) (Syntax.Comprehension ex' '[] a) +fromIR_comprehension f (Comprehension a b c d) = + Syntax.Comprehension a <$> + f b <*> + fromIR_compFor c <*> + traverse (bitraverse fromIR_compFor fromIR_compIf) d + +fromIR_dictItem + :: AsIRError e a + => DictItem a + -> Validation (NonEmpty e) (Syntax.DictItem '[] a) +fromIR_dictItem di = + case di of + DictItem a b c d -> + (\b' -> Syntax.DictItem a b' c) <$> + fromIR_expr b <*> + fromIR_expr d + DictUnpack a b c -> + Syntax.DictUnpack a b <$> fromIR_expr c + +fromIR_subscript + :: AsIRError e a + => Subscript a + -> Validation (NonEmpty e) (Syntax.Subscript '[] a) +fromIR_subscript s = + case s of + SubscriptExpr a -> Syntax.SubscriptExpr <$> fromIR_expr a + SubscriptSlice a b c d -> + (\a' -> Syntax.SubscriptSlice a' b) <$> + traverse fromIR_expr a <*> + traverse fromIR_expr c <*> + traverseOf (traverse._2.traverse) fromIR_expr d + +fromIR_block + :: AsIRError e a + => Block a + -> Validation (NonEmpty e) (Syntax.Block '[] a) +fromIR_block (Block a b c) = + Syntax.Block a <$> + fromIR_statement b <*> + traverseOf (traverse.traverse) fromIR_statement c + +fromIR_compFor + :: AsIRError e a + => CompFor a + -> Validation (NonEmpty e) (Syntax.CompFor '[] a) +fromIR_compFor (CompFor a b c d e) = + (\c' -> Syntax.CompFor a b c' d) <$> + fromIR_expr c <*> + fromIR_expr e + +fromIR_compIf + :: AsIRError e a + => CompIf a + -> Validation (NonEmpty e) (Syntax.CompIf '[] a) +fromIR_compIf (CompIf a b c) = + Syntax.CompIf a b <$> fromIR_expr c + +fromIR_smallStatement + :: AsIRError e a + => SmallStatement a + -> Validation (NonEmpty e) (Syntax.SmallStatement '[] a) +fromIR_smallStatement (MkSmallStatement b c d e f) = + (\b' c' -> Syntax.MkSmallStatement b' c' d e f) <$> + fromIR_SimpleStatement b <*> + traverseOf (traverse._2) fromIR_SimpleStatement c + +fromIR_statement + :: AsIRError e a + => Statement a + -> Validation (NonEmpty e) (Syntax.Statement '[] a) +fromIR_statement ex = + case ex of + SmallStatement i a -> + Syntax.SmallStatement i <$> fromIR_smallStatement a + CompoundStatement a -> + Syntax.CompoundStatement <$> fromIR_compoundStatement a + +fromIR_SimpleStatement + :: AsIRError e a + => SimpleStatement a + -> Validation (NonEmpty e) (Syntax.SimpleStatement '[] a) +fromIR_SimpleStatement ex = + case ex of + Assign a b c -> + Syntax.Assign a <$> + fromIR_expr b <*> + traverseOf (traverse._2) fromIR_expr c + Return a b c -> Syntax.Return a b <$> traverse fromIR_expr c + Expr a b -> Syntax.Expr a <$> fromIR_expr b + AugAssign a b c d -> + (\b' d' -> Syntax.AugAssign a b' c d') <$> + fromIR_expr b <*> + fromIR_expr d + Pass a ws -> pure $ Syntax.Pass a ws + Break a ws -> pure $ Syntax.Break a ws + Continue a ws -> pure $ Syntax.Continue a ws + Global a b c -> pure $ Syntax.Global a b c + Nonlocal a b c -> pure $ Syntax.Nonlocal a b c + Del a b c -> Syntax.Del a b <$> traverse fromIR_expr c + Import a b c -> pure $ Syntax.Import a b c + From a b c d e -> pure $ Syntax.From a b c d e + Raise a b c -> + Syntax.Raise a b <$> + traverse + (\(a, b) -> (,) <$> + fromIR_expr a <*> + traverseOf (traverse._2) fromIR_expr b) + c + Assert a b c d -> + Syntax.Assert a b <$> + fromIR_expr c <*> + traverseOf (traverse._2) fromIR_expr d + +fromIR_compoundStatement + :: AsIRError e a + => CompoundStatement a + -> Validation (NonEmpty e) (Syntax.CompoundStatement '[] a) +fromIR_compoundStatement st = + case st of + Fundef a b asyncWs c d e f g h i j -> + (\b' g' i' -> Syntax.Fundef a b' asyncWs c d e f g' h i') <$> + traverse fromIR_decorator b <*> + traverse fromIR_param g <*> + traverseOf (traverse._2) fromIR_expr i <*> + fromIR_suite j + If a b c d e f g -> + Syntax.If a b c <$> + fromIR_expr d <*> + fromIR_suite e <*> + traverse (\(a, b, c, d) -> (,,,) a b <$> fromIR_expr c <*> fromIR_suite d) f <*> + traverseOf (traverse._3) fromIR_suite g + While a b c d e f -> + Syntax.While a b c <$> + fromIR_expr d <*> + fromIR_suite e <*> + traverseOf (traverse._3) fromIR_suite f + TryExcept a b c d e f g -> + Syntax.TryExcept a b c <$> + fromIR_suite d <*> + traverse + (\(a, b, c, d) -> (,,,) a b <$> traverse fromIR_exceptAs c <*> fromIR_suite d) + e <*> + traverseOf (traverse._3) fromIR_suite f <*> + traverseOf (traverse._3) fromIR_suite g + TryFinally a b c d e f g -> + (\d' -> Syntax.TryFinally a b c d' e f) <$> fromIR_suite d <*> fromIR_suite g + For a b asyncWs c d e f g h -> + (\d' -> Syntax.For a b asyncWs c d' e) <$> + fromIR_expr d <*> + traverse fromIR_expr f <*> + fromIR_suite g <*> + traverseOf (traverse._3) fromIR_suite h + ClassDef a b c d e f g -> + (\b' -> Syntax.ClassDef a b' c d e) <$> + traverse fromIR_decorator b <*> + traverseOf (traverse._2.traverse.traverse) fromIR_arg f <*> + fromIR_suite g + With a b asyncWs c d e -> + Syntax.With a b asyncWs c <$> + traverse fromIR_withItem d <*> + fromIR_suite e + +fromIR_listItem + :: AsIRError e a + => Expr a + -> Validation (NonEmpty e) (Syntax.ListItem '[] a) +fromIR_listItem (StarExpr a b c) = + Syntax.ListUnpack a [] b <$> fromIR_expr c +fromIR_listItem (Parens a b c d) = + (\case + Syntax.ListUnpack w x y z -> Syntax.ListUnpack w ((b, d) : x) y z + Syntax.ListItem x y -> Syntax.ListItem a (Syntax.Parens x b y d)) <$> + fromIR_listItem c +fromIR_listItem e = (\x -> Syntax.ListItem (x ^. Syntax.exprAnn) x) <$> fromIR_expr e + +fromIR_tupleItem + :: AsIRError e a + => Expr a + -> Validation (NonEmpty e) (Syntax.TupleItem '[] a) +fromIR_tupleItem (StarExpr a b c) = + Syntax.TupleUnpack a [] b <$> fromIR_expr c +fromIR_tupleItem (Parens a b c d) = + (\case + Syntax.TupleUnpack w x y z -> Syntax.TupleUnpack w ((b, d) : x) y z + Syntax.TupleItem x y -> Syntax.TupleItem a (Syntax.Parens x b y d)) <$> + fromIR_tupleItem c +fromIR_tupleItem e = + (\x -> Syntax.TupleItem (x ^. Syntax.exprAnn) x) <$> fromIR_expr e + +fromIR_setItem + :: AsIRError e a + => Expr a + -> Validation (NonEmpty e) (Syntax.SetItem '[] a) +fromIR_setItem (StarExpr a b c) = + Syntax.SetUnpack a [] b <$> fromIR_expr c +fromIR_setItem (Parens a b c d) = + (\case + Syntax.SetUnpack w x y z -> Syntax.SetUnpack w ((b, d) : x) y z + Syntax.SetItem x y -> Syntax.SetItem a (Syntax.Parens x b y d)) <$> + fromIR_setItem c +fromIR_setItem e = (\x -> Syntax.SetItem (x ^. Syntax.exprAnn) x) <$> fromIR_expr e + +fromIR + :: AsIRError e a + => Module a + -> Validation (NonEmpty e) (Syntax.Module '[] a) +fromIR ModuleEmpty = pure Syntax.ModuleEmpty +fromIR (ModuleBlankFinal a) = pure $ Syntax.ModuleBlankFinal a +fromIR (ModuleBlank a b c) = Syntax.ModuleBlank a b <$> fromIR c +fromIR (ModuleStatement a b) = Syntax.ModuleStatement <$> fromIR_statement a <*> fromIR b diff --git a/src/Language/Python/Internal/Token.hs b/src/Language/Python/Internal/Token.hs new file mode 100644 --- /dev/null +++ b/src/Language/Python/Internal/Token.hs @@ -0,0 +1,230 @@ +{-# language DeriveFunctor #-} +{-# language OverloadedStrings #-} +{-# language TemplateHaskell #-} + +{-| +Module : Language.Python.Internal.Token +Copyright : (C) CSIRO 2017-2018 +License : BSD3 +Maintainer : Isaac Elliott +Stability : experimental +Portability : non-portable +-} + +module Language.Python.Internal.Token where + +import Data.Deriving (deriveEq1, deriveOrd1) +import Data.Functor.Classes (liftCompare, liftEq) + +import Language.Python.Syntax.Comment (Comment(..)) +import Language.Python.Syntax.Numbers +import Language.Python.Syntax.Strings +import Language.Python.Syntax.Whitespace (Newline(..), Indents) + +-- | A 'PyToken' is a single lexical token of Python source. A 'PyToken' has an +-- optional annotation, which can be '()' when no annotation is desired. +data PyToken a + = TkIf a + | TkElse a + | TkElif a + | TkWhile a + | TkAssert a + | TkDef a + | TkReturn a + | TkPass a + | TkBreak a + | TkContinue a + | TkTrue a + | TkFalse a + | TkNone a + | TkEllipsis a + | TkOr a + | TkAnd a + | TkIs a + | TkNot a + | TkGlobal a + | TkNonlocal a + | TkDel a + | TkLambda a + | TkImport a + | TkFrom a + | TkAs a + | TkRaise a + | TkTry a + | TkExcept a + | TkFinally a + | TkClass a + | TkRightArrow a + | TkWith a + | TkFor a + | TkIn a + | TkYield a + | TkInt (IntLiteral a) + | TkFloat (FloatLiteral a) + | TkImag (ImagLiteral a) + | TkIdent String a + | TkString (Maybe StringPrefix) StringType QuoteType [PyChar] a + | TkBytes BytesPrefix StringType QuoteType [PyChar] a + | TkRawString RawStringPrefix StringType QuoteType [PyChar] a + | TkRawBytes RawBytesPrefix StringType QuoteType [PyChar] a + | TkSpace a + | TkTab a + | TkNewline Newline a + | TkLeftBracket a + | TkRightBracket a + | TkLeftParen a + | TkRightParen a + | TkLeftBrace a + | TkRightBrace a + | TkLt a + | TkLte a + | TkEq a + | TkDoubleEq a + | TkBangEq a + | TkGt a + | TkGte a + | TkContinued Newline a + | TkColon a + | TkSemicolon a + | TkComma a + | TkDot a + | TkPlus a + | TkMinus a + | TkTilde a + | TkComment (Comment a) + | TkStar a + | TkDoubleStar a + | TkSlash a + | TkDoubleSlash a + | TkPercent a + | TkShiftLeft a + | TkShiftRight a + | TkPlusEq a + | TkMinusEq a + | TkStarEq a + | TkAtEq a + | TkAt a + | TkSlashEq a + | TkPercentEq a + | TkAmpersandEq a + | TkPipeEq a + | TkCaretEq a + | TkShiftLeftEq a + | TkShiftRightEq a + | TkDoubleStarEq a + | TkDoubleSlashEq a + | TkPipe a + | TkCaret a + | TkAmpersand a + | TkIndent a (Indents a) + | TkLevel a (Indents a) + | TkDedent a + deriving (Show, Functor) +deriveEq1 ''PyToken +deriveOrd1 ''PyToken + +instance Eq (PyToken a) where + (==) = liftEq (\_ _ -> True) + +instance Ord (PyToken a) where + compare = liftCompare (\_ _ -> EQ) + +-- | Get the annotation from a 'PyToken'. +pyTokenAnn :: PyToken a -> a +pyTokenAnn tk = + case tk of + TkPipe a -> a + TkCaret a -> a + TkAmpersand a -> a + TkIndent a _ -> a + TkLevel a _ -> a + TkDedent a -> a + TkDef a -> a + TkReturn a -> a + TkPass a -> a + TkBreak a -> a + TkContinue a -> a + TkTrue a -> a + TkFalse a -> a + TkNone a -> a + TkEllipsis a -> a + TkOr a -> a + TkAnd a -> a + TkIs a -> a + TkNot a -> a + TkGlobal a -> a + TkNonlocal a -> a + TkDel a -> a + TkLambda a -> a + TkImport a -> a + TkFrom a -> a + TkAs a -> a + TkRaise a -> a + TkTry a -> a + TkExcept a -> a + TkFinally a -> a + TkClass a -> a + TkRightArrow a -> a + TkWith a -> a + TkFor a -> a + TkIn a -> a + TkYield a -> a + TkPlus a -> a + TkMinus a -> a + TkTilde a -> a + TkIf a -> a + TkElse a -> a + TkElif a -> a + TkWhile a -> a + TkAssert a -> a + TkInt a -> _intLiteralAnn a + TkFloat a -> _floatLiteralAnn a + TkImag a -> _imagLiteralAnn a + TkIdent _ a -> a + TkString _ _ _ _ a -> a + TkBytes _ _ _ _ a -> a + TkRawString _ _ _ _ a -> a + TkRawBytes _ _ _ _ a -> a + TkSpace a -> a + TkTab a -> a + TkNewline _ a -> a + TkLeftBracket a -> a + TkRightBracket a -> a + TkLeftParen a -> a + TkRightParen a -> a + TkLeftBrace a -> a + TkRightBrace a -> a + TkLt a -> a + TkLte a -> a + TkEq a -> a + TkDoubleEq a -> a + TkBangEq a -> a + TkGt a -> a + TkGte a -> a + TkContinued _ a -> a + TkColon a -> a + TkSemicolon a -> a + TkComma a -> a + TkDot a -> a + TkComment a -> _commentAnn a + TkStar a -> a + TkDoubleStar a -> a + TkSlash a -> a + TkDoubleSlash a -> a + TkPercent a -> a + TkShiftLeft a -> a + TkShiftRight a -> a + TkPlusEq a -> a + TkMinusEq a -> a + TkStarEq a -> a + TkAtEq a -> a + TkAt a -> a + TkSlashEq a -> a + TkPercentEq a -> a + TkAmpersandEq a -> a + TkPipeEq a -> a + TkCaretEq a -> a + TkShiftLeftEq a -> a + TkShiftRightEq a -> a + TkDoubleStarEq a -> a + TkDoubleSlashEq a -> a diff --git a/src/Language/Python/Optics.hs b/src/Language/Python/Optics.hs new file mode 100644 --- /dev/null +++ b/src/Language/Python/Optics.hs @@ -0,0 +1,453 @@ +{-# language DataKinds #-} +{-# language FlexibleContexts #-} +{-# language MultiParamTypeClasses #-} +{-# language FlexibleInstances #-} +{-# language InstanceSigs, ScopedTypeVariables #-} +{-# language PolyKinds #-} +{-# language LambdaCase #-} + +{-| +Module : Language.Python.Optics +Copyright : (C) CSIRO 2017-2018 +License : BSD3 +Maintainer : Isaac Elliott +Stability : experimental +Portability : non-portable + +Optics for manipulating Python syntax trees + +-} + +module Language.Python.Optics + ( module Language.Python.Optics.Validated + -- * Indentation + , module Language.Python.Optics.Indents + -- * Newlines + , module Language.Python.Optics.Newlines + -- * Simple statements + -- ** Assignment + , assignTargets + -- * Compound statements + , HasCompoundStatement(..) + -- ** Function definitions + , HasFundef(..) + -- ** Class defintions + , HasClassDef(..) + -- ** @while@ statements + , HasWhile(..) + -- ** @for@ statements + , HasFor(..) + -- ** @with@ statements + , HasWith(..) + -- ** @if@ statements + , HasIf(..) + , _Elif + -- ** @try@ statements + , HasTryExcept(..) + , HasTryFinally(..) + , _Finally + , _Except + -- ** @else@ + , _Else + -- * Parameters + , _PositionalParam + , _KeywordParam + , _UnnamedStarParam + , _StarParam + -- * Expressions + -- ** Identifiers + , _Ident + -- ** @None@ + , _None + -- ** Function calls + , _Call + -- ** Tuples + , _Tuple + , _TupleUnpack + , tupleItems + -- ** Lists + , _List + , _ListUnpack + , listItems + ) +where + +import Control.Lens.Getter ((^.), view) +import Control.Lens.Iso (Iso', iso, from) +import Control.Lens.Traversal (Traversal) +import Control.Lens.Prism (Prism, prism) + +import Language.Python.Optics.Indents +import Language.Python.Optics.Newlines +import Language.Python.Optics.Validated +import Language.Python.Syntax.Expr +import Language.Python.Syntax.Ident +import Language.Python.Syntax.Statement +import Language.Python.Syntax.Types +import Language.Python.Syntax.Whitespace + +_TupleUnpack :: Prism (TupleItem v a) (TupleItem '[] a) (TupleUnpack v a) (TupleUnpack '[] a) +_TupleUnpack = + prism + (\(MkTupleUnpack a b c d) -> TupleUnpack a b c d) + (\case + TupleUnpack a b c d -> Right $ MkTupleUnpack a b c d + a -> Left $ a ^. unvalidated) + +_Tuple :: Prism (Expr v a) (Expr '[] a) (Tuple v a) (Tuple '[] a) +_Tuple = + prism + (\(MkTuple a b c d) -> Tuple a b c d) + (\case + Tuple a b c d -> Right (MkTuple a b c d) + a -> Left $ a ^. unvalidated) + +tupleItems :: Traversal (Tuple v a) (Tuple '[] a) (TupleItem v a) (TupleItem '[] a) +tupleItems f (MkTuple a b c d) = + (\b' d' -> MkTuple a b' c d') <$> + f b <*> + (traverse.traverse) f d + +_ListUnpack :: Prism (ListItem v a) (ListItem '[] a) (ListUnpack v a) (ListUnpack '[] a) +_ListUnpack = + prism + (\(MkListUnpack a b c d) -> ListUnpack a b c d) + (\case + ListUnpack a b c d -> Right $ MkListUnpack a b c d + a -> Left $ a ^. unvalidated) + +_List :: Prism (Expr v a) (Expr '[] a) (List v a) (List '[] a) +_List = + prism + (\(MkList a b c d) -> List a b c d) + (\case + List a b c d -> Right (MkList a b c d) + a -> Left $ a ^. unvalidated) + +listItems :: Traversal (List v a) (List '[] a) (ListItem v a) (ListItem '[] a) +listItems f (MkList a b c d) = + (\c' -> MkList a b c' d) <$> + (traverse.traverse) f c + +_None :: Prism (Expr v a) (Expr '[] a) (None v a) (None '[] a) +_None = + prism + (\(MkNone a b) -> None a b) + (\case + None a b -> Right (MkNone a b) + a -> Left $ a ^. unvalidated) + +_KeywordParam + :: Prism + (Param v a) + (Param '[] a) + (KeywordParam v a) + (KeywordParam '[] a) +_KeywordParam = + prism + (\(MkKeywordParam a b c d e) -> KeywordParam a b c d e) + (\case + KeywordParam a b c d e -> Right (MkKeywordParam a b c d e) + a -> Left $ a ^. unvalidated) + +_PositionalParam + :: Prism + (Param v a) + (Param '[] a) + (PositionalParam v a) + (PositionalParam '[] a) +_PositionalParam = + prism + (\(MkPositionalParam a b c) -> PositionalParam a b c) + (\case + PositionalParam a b c -> Right (MkPositionalParam a b c) + a -> Left $ a ^. unvalidated) + +_StarParam + :: Prism + (Param v a) + (Param '[] a) + (StarParam v a) + (StarParam '[] a) +_StarParam = + prism + (\(MkStarParam a b c d) -> StarParam a b c d) + (\case + StarParam a b c d -> Right (MkStarParam a b c d) + a -> Left $ a ^. unvalidated) + +_UnnamedStarParam + :: Prism + (Param v a) + (Param '[] a) + (UnnamedStarParam v a) + (UnnamedStarParam '[] a) +_UnnamedStarParam = + prism + (\(MkUnnamedStarParam a b) -> UnnamedStarParam a b) + (\case + UnnamedStarParam a b -> Right (MkUnnamedStarParam a b) + a -> Left $ a ^. unvalidated) + +class HasCompoundStatement s where + _CompoundStatement :: Prism (s v a) (s '[] a) (CompoundStatement v a) (CompoundStatement '[] a) + +instance HasCompoundStatement CompoundStatement where + _CompoundStatement = id + +instance HasCompoundStatement Statement where + _CompoundStatement = + prism + CompoundStatement + (\case + CompoundStatement a -> Right a + a -> Left (a ^. unvalidated)) + +class HasFundef s where + _Fundef :: Prism (s v a) (s '[] a) (Fundef v a) (Fundef '[] a) + +instance HasFundef Fundef where + _Fundef = id + +instance HasFundef CompoundStatement where + _Fundef = + prism + (\(MkFundef idnt a b c d e f g h i j) -> + Fundef idnt a b c d e f g h i j) + (\case + Fundef idnt a b c d e f g h i j -> + Right $ MkFundef idnt a b c d e f g h i j + a -> Left $ a ^. unvalidated) + +instance HasFundef Statement where + _Fundef = _CompoundStatement._Fundef + +class HasWhile s where + _While :: Prism (s v a) (s '[] a) (While v a) (While '[] a) + +instance HasWhile While where + _While = id + +instance HasWhile CompoundStatement where + _While = + prism + (\(MkWhile a b c d e f) -> + While a b c d e $ view _Else <$> f) + (\case + While a b c d e f -> + Right . MkWhile a b c d e $ view (from _Else) <$> f + a -> Left $ a ^. unvalidated) + +instance HasWhile Statement where + _While = _CompoundStatement._While + +_Else :: Iso' (Else v a) (Indents a, [Whitespace], Suite v a) +_Else = iso (\(MkElse a b c) -> (a, b, c)) (\(a, b, c) -> MkElse a b c) + +_Elif :: Iso' (Elif v a) (Indents a, [Whitespace], Expr v a, Suite v a) +_Elif = iso (\(MkElif a b c d) -> (a, b, c, d)) (\(a, b, c, d) -> MkElif a b c d) + +_Finally :: Iso' (Finally v a) (Indents a, [Whitespace], Suite v a) +_Finally = iso (\(MkFinally a b c) -> (a, b, c)) (\(a, b, c) -> MkFinally a b c) + +_Except :: Iso' (Except v a) (Indents a, [Whitespace], Maybe (ExceptAs v a), Suite v a) +_Except = iso (\(MkExcept a b c d) -> (a, b, c, d)) (\(a, b, c, d) -> MkExcept a b c d) + +class HasIf s where + _If :: Prism (s v a) (s '[] a) (If v a) (If '[] a) + +instance HasIf If where + _If = id + +instance HasIf CompoundStatement where + _If = + prism + (\(MkIf a b c d e f g) -> + If a b c d e (view _Elif <$> f) (view _Else <$> g)) + (\case + If a b c d e f g -> + Right $ MkIf a b c d e (view (from _Elif) <$> f) (view (from _Else) <$> g) + a -> Left $ a ^. unvalidated) + +instance HasIf Statement where + _If = _CompoundStatement._If + +class HasTryExcept s where + _TryExcept :: Prism (s v a) (s '[] a) (TryExcept v a) (TryExcept '[] a) + +instance HasTryExcept TryExcept where + _TryExcept = id + +instance HasTryExcept CompoundStatement where + _TryExcept = + prism + (\(MkTryExcept a b c d e f g) -> + TryExcept a b c d (view _Except <$> e) (view _Else <$> f) (view _Finally <$> g)) + (\case + TryExcept a b c d e f g -> + Right $ + MkTryExcept a b c d + (view (from _Except) <$> e) + (view (from _Else) <$> f) + (view (from _Finally) <$> g) + a -> Left $ a ^. unvalidated) + +instance HasTryExcept Statement where + _TryExcept = _CompoundStatement._TryExcept + +class HasTryFinally s where + _TryFinally :: Prism (s v a) (s '[] a) (TryFinally v a) (TryFinally '[] a) + +instance HasTryFinally TryFinally where + _TryFinally = id + +instance HasTryFinally CompoundStatement where + _TryFinally = + prism + (\(MkTryFinally a b c d e) -> + (\(x, y, z) -> TryFinally a b c d x y z) (e ^. _Finally)) + (\case + TryFinally a b c d e f g -> + Right $ MkTryFinally a b c d ((e, f, g) ^. from _Finally) + a -> Left $ a ^. unvalidated) + +instance HasTryFinally Statement where + _TryFinally = _CompoundStatement._TryFinally + +class HasFor s where + _For :: Prism (s v a) (s '[] a) (For v a) (For '[] a) + +instance HasFor For where + _For = id + +instance HasFor CompoundStatement where + _For = + prism + (\(MkFor a b c d e f g h i) -> + For a b c d e f g h (view _Else <$> i)) + (\case + For a b c d e f g h i -> + Right $ MkFor a b c d e f g h (view (from _Else) <$> i) + a -> Left $ a ^. unvalidated) + +instance HasFor Statement where + _For = _CompoundStatement._For + +_Call :: Prism (Expr v a) (Expr '[] a) (Call v a) (Call '[] a) +_Call = + prism + (\(MkCall a b c d e) -> Call a b c d e) + (\case + Call a b c d e -> Right $ MkCall a b c d e + a -> Left $ a ^. unvalidated) + +class HasClassDef s where + _ClassDef :: Prism (s v a) (s '[] a) (ClassDef v a) (ClassDef '[] a) + +instance HasClassDef ClassDef where + _ClassDef = id + +instance HasClassDef CompoundStatement where + _ClassDef = + prism + (\(MkClassDef a b c d e f g) -> ClassDef a b c d e f g) + (\case + ClassDef a b c d e f g -> Right $ MkClassDef a b c d e f g + a -> Left $ a ^. unvalidated) + +instance HasClassDef Statement where + _ClassDef = _CompoundStatement._ClassDef + +class HasWith s where + _With :: Prism (s v a) (s '[] a) (With v a) (With '[] a) + +instance HasWith With where + _With = id + +instance HasWith CompoundStatement where + _With = + prism + (\(MkWith a b c d e f) -> With a b c d e f) + (\case + With a b c d e f -> Right $ MkWith a b c d e f + a -> Left $ a ^. unvalidated) + +instance HasWith Statement where + _With = _CompoundStatement._With + +_Ident :: Prism (Expr v a) (Expr '[] a) (Ident v a) (Ident '[] a) +_Ident = + prism + Ident + (\case + Ident a -> Right a + a -> Left $ a ^. unvalidated) + +-- | 'Traversal' targeting the variables that would modified as a result of an assignment +-- +-- Here are some examples of assignment targets: +-- +-- @ +-- a = b +-- ^ +-- @ +-- +-- @ +-- (a, b, c) = d +-- ^ ^ ^ +-- @ +-- +-- @ +-- [a, b, *c] = d +-- ^ ^ ^ +-- @ +-- +-- These expressions have variables on the left hand side of the @=@, but those variables +-- don't count as assignment targets: +-- +-- @ +-- a[b] = c +-- @ +-- +-- @ +-- a(b) = c +-- @ +-- +-- @ +-- {a: b} = c +-- @ +assignTargets :: Traversal (Expr v a) (Expr '[] a) (Ident v a) (Ident '[] a) +assignTargets f e = + case e of + List a b c d -> (\c' -> List a b c' d) <$> (traverse.traverse._Exprs.assignTargets) f c + Parens a b c d -> (\c' -> Parens a b c' d) <$> assignTargets f c + Ident a -> Ident <$> f a + Tuple a b c d -> + (\b' d' -> Tuple a b' c d') <$> + (_Exprs.assignTargets) f b <*> + (traverse.traverse._Exprs.assignTargets) f d + Unit{} -> pure $ e ^. unvalidated + Lambda{} -> pure $ e ^. unvalidated + Yield{} -> pure $ e ^. unvalidated + YieldFrom{} -> pure $ e ^. unvalidated + Ternary{} -> pure $ e ^. unvalidated + ListComp{} -> pure $ e ^. unvalidated + Deref{} -> pure $ e ^. unvalidated + Subscript{} -> pure $ e ^. unvalidated + Call{} -> pure $ e ^. unvalidated + None{} -> pure $ e ^. unvalidated + Ellipsis{} -> pure $ e ^. unvalidated + BinOp{} -> pure $ e ^. unvalidated + UnOp{} -> pure $ e ^. unvalidated + Int{} -> pure $ e ^. unvalidated + Float{} -> pure $ e ^. unvalidated + Imag{} -> pure $ e ^. unvalidated + Bool{} -> pure $ e ^. unvalidated + String{} -> pure $ e ^. unvalidated + Not{} -> pure $ e ^. unvalidated + DictComp{} -> pure $ e ^. unvalidated + Dict{} -> pure $ e ^. unvalidated + SetComp{} -> pure $ e ^. unvalidated + Set{} -> pure $ e ^. unvalidated + Generator{} -> pure $ e ^. unvalidated + Await{} -> pure $ e ^. unvalidated diff --git a/src/Language/Python/Optics/Indents.hs b/src/Language/Python/Optics/Indents.hs new file mode 100644 --- /dev/null +++ b/src/Language/Python/Optics/Indents.hs @@ -0,0 +1,262 @@ +{-# language DataKinds #-} +{-# language FlexibleInstances, FunctionalDependencies, MultiParamTypeClasses #-} + +{-| +Module : Language.Python.Optics.Indents +Copyright : (C) CSIRO 2017-2018 +License : BSD3 +Maintainer : Isaac Elliott +Stability : experimental +Portability : non-portable +-} + +module Language.Python.Optics.Indents where + +import Control.Lens.Traversal (Traversal') + +import Language.Python.Syntax +import Language.Python.Internal.Token + +-- | 'Traversal'' targeting the indent-chunks in a structure +-- +-- e.g. +-- +-- This is one indent chunk: +-- +-- @ +-- def a(): +-- pass +-- if b: +-- pass +-- ^^^^ +-- @ +-- +-- and this is another +-- +-- @ +-- def a(): +-- pass +-- if b: +-- pass +-- ^^^^ +-- @ +_Indent :: HasIndents s a => Traversal' s [Whitespace] +_Indent = _Indents.indentsValue.traverse.indentWhitespaces + +class HasIndents s a | s -> a where + -- | 'Traversal'' targeting the indentation inside a structure + -- + -- Note: whitespace inside \'enclosed forms\' (such as lists or tuples) is not + -- considered indentation. + -- + -- e.g. + -- + -- In the following code, there is only one chunk of indentation: + -- + -- @ + -- def a(): + -- [ b + -- , c + -- , d + -- ] + -- @ + -- + -- it's here: + -- + -- @ + -- def a(): + -- [ b + -- ^^^^ + -- , c + -- , d + -- ] + -- @ + -- + -- The rest is whitespace that is internal to the list. + _Indents :: Traversal' s (Indents a) + +instance HasIndents (PyToken a) a where + _Indents f (TkIndent a i) = TkIndent a <$> f i + _Indents f (TkLevel a i) = TkLevel a <$> f i + _Indents _ a = pure a + +instance HasIndents (Fundef '[] a) a where + _Indents fun (MkFundef a b c d e f g h i j k) = + (\b' c' -> MkFundef a b' c' d e f g h i j) <$> + (traverse._Indents) fun b <*> + fun c <*> + _Indents fun k + +instance HasIndents (For '[] a) a where + _Indents fun (MkFor a b c d e f g h i) = + (\b' -> MkFor a b' c d e f g) <$> + fun b <*> + _Indents fun h <*> + (traverse._Indents) fun i + +instance HasIndents (TryFinally '[] a) a where + _Indents fun (MkTryFinally a b c d e) = + (\b' -> MkTryFinally a b' c) <$> + fun b <*> + _Indents fun d <*> + _Indents fun e + +instance HasIndents (TryExcept '[] a) a where + _Indents fun (MkTryExcept a b c d e f g) = + (\b' -> MkTryExcept a b' c) <$> + fun b <*> + _Indents fun d <*> + (traverse._Indents) fun e <*> + (traverse._Indents) fun f <*> + (traverse._Indents) fun g + +instance HasIndents (Except '[] a) a where + _Indents fun (MkExcept a b c d) = + (\a' -> MkExcept a' b c) <$> + fun a <*> + _Indents fun d + +instance HasIndents (Finally '[] a) a where + _Indents fun (MkFinally a b c) = + (\a' -> MkFinally a' b) <$> + fun a <*> + _Indents fun c + +instance HasIndents (If '[] a) a where + _Indents fun (MkIf a b c d e f g) = + (\b' -> MkIf a b' c d) <$> + fun b <*> + _Indents fun e <*> + (traverse._Indents) fun f <*> + (traverse._Indents) fun g + +instance HasIndents (While '[] a) a where + _Indents fun (MkWhile a b c d e f) = + (\b' -> MkWhile a b' c d) <$> + fun b <*> + _Indents fun e <*> + (traverse._Indents) fun f + +instance HasIndents (Elif '[] a) a where + _Indents fun (MkElif a b c d) = + (\a' -> MkElif a' b c) <$> + fun a <*> + _Indents fun d + +instance HasIndents (Else '[] a) a where + _Indents f (MkElse a b c) = MkElse <$> f a <*> pure b <*> _Indents f c + +instance HasIndents (SmallStatement '[] a) a where + _Indents _ (MkSmallStatement a b c d e) = + pure $ MkSmallStatement a b c d e + +instance HasIndents (Statement '[] a) a where + _Indents f (SmallStatement idnt a) = SmallStatement <$> f idnt <*> _Indents f a + _Indents f (CompoundStatement c) = CompoundStatement <$> _Indents f c + +instance HasIndents (Block '[] a) a where + _Indents = _Statements._Indents + +instance HasIndents (Suite '[] a) a where + _Indents _ (SuiteOne a b c) = pure $ SuiteOne a b c + _Indents f (SuiteMany a b c d e) = SuiteMany a b c d <$> _Indents f e + +instance HasIndents (Decorator '[] a) a where + _Indents fun (Decorator a b c d e f g) = + (\b' -> Decorator a b' c d e f g) <$> + fun b + +instance HasIndents (ClassDef '[] a) a where + _Indents fun (MkClassDef a b c d e f g) = + (\b' c' -> MkClassDef a b' c' d e f) <$> + (traverse._Indents) fun b <*> + fun c <*> + _Indents fun g + +instance HasIndents (With '[] a) a where + _Indents fun (MkWith a b c d e f) = + (\b' -> MkWith a b' c d e) <$> + fun b <*> + _Indents fun f + +instance HasIndents (CompoundStatement '[] a) a where + _Indents fun s = + case s of + Fundef a decos idnt asyncWs b c d e f g h -> + (\decos' idnt' -> Fundef a decos' idnt' asyncWs b c d e f g) <$> + (traverse._Indents) fun decos <*> + fun idnt <*> + _Indents fun h + If a idnt b c d elifs e -> + (\idnt' -> If a idnt' b c) <$> + fun idnt <*> + _Indents fun d <*> + traverse + (\(idnt, a, b, c) -> + (\idnt' -> (,,,) idnt' a b) <$> + fun idnt <*> + _Indents fun c) + elifs <*> + traverse + (\(idnt, a, b) -> + (\idnt' -> (,,) idnt' a) <$> + fun idnt <*> + _Indents fun b) + e + While a idnt b c d e -> + (\idnt' -> While a idnt' b c) <$> + fun idnt <*> + _Indents fun d <*> + traverse + (\(idnt, a, b) -> + (\idnt' -> (,,) idnt' a) <$> + fun idnt <*> + _Indents fun b) + e + TryExcept a idnt b c d e f -> + (\idnt' -> TryExcept a idnt' b) <$> + fun idnt <*> + _Indents fun c <*> + traverse + (\(idnt, a, b, c) -> + (\idnt' -> (,,,) idnt' a b) <$> + fun idnt <*> + _Indents fun c) + d <*> + traverse + (\(idnt, a, b) -> + (\idnt' -> (,,) idnt' a) <$> + fun idnt <*> + _Indents fun b) + e <*> + traverse + (\(idnt, a, b) -> + (\idnt' -> (,,) idnt' a) <$> + fun idnt <*> + _Indents fun b) + f + TryFinally a idnt b c idnt2 d e -> + (\idnt' c' idnt2' -> TryFinally a idnt' b c' idnt2' d) <$> + fun idnt <*> + _Indents fun c <*> + fun idnt2 <*> + _Indents fun e + For a idnt asyncWs b c d e f g -> + (\idnt' -> For a idnt' asyncWs b c d e) <$> + fun idnt <*> + _Indents fun f <*> + traverse + (\(idnt, a, b) -> + (\idnt' -> (,,) idnt' a) <$> + fun idnt <*> + _Indents fun b) + g + ClassDef a decos idnt b c d e -> + (\decos' idnt' -> ClassDef a decos' idnt' b c d) <$> + traverse (_Indents fun) decos <*> + fun idnt <*> + _Indents fun e + With a b asyncWs c d e -> + (\b' -> With a b' asyncWs c d) <$> + fun b <*> + _Indents fun e diff --git a/src/Language/Python/Optics/Newlines.hs b/src/Language/Python/Optics/Newlines.hs new file mode 100644 --- /dev/null +++ b/src/Language/Python/Optics/Newlines.hs @@ -0,0 +1,576 @@ +{-| +Module : Language.Python.Optics.Newlines +Copyright : (C) CSIRO 2017-2018 +License : BSD3 +Maintainer : Isaac Elliott +Stability : experimental +Portability : non-portable +-} +module Language.Python.Optics.Newlines where + +import Control.Lens.Traversal (Traversal') +import Data.List.NonEmpty (NonEmpty(..)) + +import qualified Data.FingerTree as FingerTree + +import Language.Python.Syntax + +{- + +I can't derive this with generic-lens :( it wants generic instances for certain +things that can't have those instance (and don't contain newlines anyways) + +-} +class HasNewlines s where + -- | 'Traversal'' targeting all of thie 'Newline's in a structure + -- + -- This only targets places that contain the 'Newline' datatype; it doesn't target + -- newline characters in string literals, for example. + _Newlines :: Traversal' s Newline + +instance (HasNewlines a, HasNewlines b) => HasNewlines (a, b) where + _Newlines f (a, b) = (,) <$> _Newlines f a <*> _Newlines f b + +instance (HasNewlines a, HasNewlines b, HasNewlines c) => HasNewlines (a, b, c) where + _Newlines f (a, b, c) = + (,,) <$> + _Newlines f a <*> + _Newlines f b <*> + _Newlines f c + +instance (HasNewlines a, HasNewlines b, HasNewlines c, HasNewlines d) => HasNewlines (a, b, c, d) where + _Newlines f (a, b, c, d) = + (,,,) <$> + _Newlines f a <*> + _Newlines f b <*> + _Newlines f c <*> + _Newlines f d + +instance HasNewlines (e a) => HasNewlines (ImportAs e v a) where + _Newlines f (ImportAs a b c) = + ImportAs a <$> + _Newlines f b <*> + _Newlines f c + +instance HasNewlines (RelativeModuleName v a) where + _Newlines f (RelativeWithName a b) = + RelativeWithName <$> + _Newlines f a <*> + _Newlines f b + _Newlines f (Relative a) = Relative <$> _Newlines f a + +instance (HasNewlines a, HasNewlines b) => HasNewlines (Either a b) where + _Newlines f (Left a) = Left <$> _Newlines f a + _Newlines f (Right a) = Right <$> _Newlines f a + +instance HasNewlines Newline where + _Newlines = id + +instance HasNewlines a => HasNewlines [a] where + _Newlines = traverse._Newlines + +instance HasNewlines Whitespace where + _Newlines _ Space = pure Space + _Newlines _ Tab = pure Tab + _Newlines f (Continued nl ws) = Continued <$> f nl <*> _Newlines f ws + _Newlines _ (Comment c) = pure $ Comment c + _Newlines f (Newline nl) = Newline <$> f nl + +instance HasNewlines (Blank a) where + _Newlines f (Blank a b c) = (\b' -> Blank a b' c) <$> _Newlines f b + +instance HasNewlines (Block v a) where + _Newlines f (Block a b c) = + Block <$> + _Newlines f a <*> + _Newlines f b <*> + _Newlines f c + +instance HasNewlines Colon where + _Newlines f (MkColon a) = MkColon <$> _Newlines f a + +instance HasNewlines Dot where + _Newlines f (MkDot a) = MkDot <$> _Newlines f a + +instance HasNewlines Comma where + _Newlines f (MkComma a) = MkComma <$> _Newlines f a + +instance HasNewlines At where + _Newlines f (MkAt a) = MkAt <$> _Newlines f a + +instance HasNewlines (Semicolon a) where + _Newlines f (MkSemicolon a b) = MkSemicolon a <$> _Newlines f b + +instance HasNewlines Equals where + _Newlines f (MkEquals a) = MkEquals <$> _Newlines f a + +instance HasNewlines (Suite v a) where + _Newlines f (SuiteOne a b c) = SuiteOne a b <$> _Newlines f c + _Newlines f (SuiteMany a b c d e) = + (\b' d' e' -> SuiteMany a b' c d' e') <$> + _Newlines f b <*> + f d <*> + _Newlines f e + +instance HasNewlines Indent where + _Newlines f (MkIndent a) = MkIndent <$> (FingerTree.traverse'._Newlines) f a + +instance HasNewlines (Indents a) where + _Newlines f (Indents a b) = (\a' -> Indents a' b) <$> _Newlines f a + +instance HasNewlines (UnOp a) where + _Newlines f (Negate a b) = Negate a <$> _Newlines f b + _Newlines f (Positive a b) = Positive a <$> _Newlines f b + _Newlines f (Complement a b) = Complement a <$> _Newlines f b + +instance HasNewlines (BinOp a) where + _Newlines f x = + case x of + Is a b -> Is a <$> _Newlines f b + IsNot a b c -> IsNot a <$> _Newlines f b <*> _Newlines f c + In a b -> In a <$> _Newlines f b + NotIn a b c -> NotIn a <$> _Newlines f b <*> _Newlines f c + Minus a b -> Minus a <$> _Newlines f b + Exp a b -> Exp a <$> _Newlines f b + BoolAnd a b -> BoolAnd a <$> _Newlines f b + BoolOr a b -> BoolOr a <$> _Newlines f b + Eq a b -> Eq a <$> _Newlines f b + Lt a b -> Lt a <$> _Newlines f b + LtEq a b -> LtEq a <$> _Newlines f b + Gt a b -> Gt a <$> _Newlines f b + GtEq a b -> GtEq a <$> _Newlines f b + NotEq a b -> NotEq a <$> _Newlines f b + Multiply a b -> Multiply a <$> _Newlines f b + Divide a b -> Divide a <$> _Newlines f b + FloorDivide a b -> FloorDivide a <$> _Newlines f b + Percent a b -> Percent a <$> _Newlines f b + Plus a b -> Plus a <$> _Newlines f b + BitOr a b -> BitOr a <$> _Newlines f b + BitXor a b -> BitXor a <$> _Newlines f b + BitAnd a b -> BitAnd a <$> _Newlines f b + ShiftLeft a b -> ShiftLeft a <$> _Newlines f b + ShiftRight a b -> ShiftRight a <$> _Newlines f b + At a b -> At a <$> _Newlines f b + +instance HasNewlines a => HasNewlines (CommaSep a) where + _Newlines f = go + where + go CommaSepNone = pure CommaSepNone + go (CommaSepOne a) = CommaSepOne <$> _Newlines f a + go (CommaSepMany a b c) = + CommaSepMany <$> + _Newlines f a <*> + _Newlines f b <*> + go c + +instance HasNewlines a => HasNewlines (CommaSep1 a) where + _Newlines f = go + where + go (CommaSepOne1 a) = CommaSepOne1 <$> _Newlines f a + go (CommaSepMany1 a b c) = + CommaSepMany1 <$> + _Newlines f a <*> + _Newlines f b <*> + go c + +instance HasNewlines a => HasNewlines (CommaSep1' a) where + _Newlines f = go + where + go (CommaSepOne1' a b) = CommaSepOne1' <$> _Newlines f a <*> _Newlines f b + go (CommaSepMany1' a b c) = + CommaSepMany1' <$> + _Newlines f a <*> + _Newlines f b <*> + go c + +instance HasNewlines (Ident v a) where + _Newlines f (MkIdent a b c) = MkIdent a b <$> _Newlines f c + +instance HasNewlines a => HasNewlines (Maybe a) where + _Newlines = traverse._Newlines + +instance HasNewlines (Param v a) where + _Newlines f (PositionalParam a b c) = + PositionalParam a <$> + _Newlines f b <*> + _Newlines f c + _Newlines f (KeywordParam a b c d e) = + KeywordParam a <$> + _Newlines f b <*> + _Newlines f c <*> + _Newlines f d <*> + _Newlines f e + _Newlines f (StarParam a b c d) = + StarParam a <$> + _Newlines f b <*> + _Newlines f c <*> + _Newlines f d + _Newlines f (UnnamedStarParam a b) = + UnnamedStarParam a <$> + _Newlines f b + _Newlines f (DoubleStarParam a b c d) = + DoubleStarParam a <$> + _Newlines f b <*> + _Newlines f c <*> + _Newlines f d + +instance HasNewlines (Arg v a) where + _Newlines f (PositionalArg a b) = + PositionalArg a <$> + _Newlines f b + _Newlines f (KeywordArg a b c d) = + KeywordArg a <$> + _Newlines f b <*> + _Newlines f c <*> + _Newlines f d + _Newlines f (StarArg a b c) = + StarArg a <$> + _Newlines f b <*> + _Newlines f c + _Newlines f (DoubleStarArg a b c) = + DoubleStarArg a <$> + _Newlines f b <*> + _Newlines f c + +instance HasNewlines (CompFor v a) where + _Newlines f (CompFor a b c d e) = + CompFor a <$> + _Newlines f b <*> + _Newlines f c <*> + _Newlines f d <*> + _Newlines f e + +instance HasNewlines (CompIf v a) where + _Newlines f (CompIf a b c) = + CompIf a <$> + _Newlines f b <*> + _Newlines f c + +instance HasNewlines (e v a) => HasNewlines (Comprehension e v a) where + _Newlines f (Comprehension a b c d) = + Comprehension a <$> + _Newlines f b <*> + _Newlines f c <*> + _Newlines f d + +instance HasNewlines (TupleItem v a) where + _Newlines f (TupleItem a b) = TupleItem a <$> _Newlines f b + _Newlines f (TupleUnpack a b c d) = + TupleUnpack a <$> + _Newlines f b <*> + _Newlines f c <*> + _Newlines f d + +instance HasNewlines (ListItem v a) where + _Newlines f (ListItem a b) = ListItem a <$> _Newlines f b + _Newlines f (ListUnpack a b c d) = + ListUnpack a <$> + _Newlines f b <*> + _Newlines f c <*> + _Newlines f d + +instance HasNewlines (SetItem v a) where + _Newlines f (SetItem a b) = SetItem a <$> _Newlines f b + _Newlines f (SetUnpack a b c d) = + SetUnpack a <$> + _Newlines f b <*> + _Newlines f c <*> + _Newlines f d + +instance HasNewlines (DictItem v a) where + _Newlines f (DictItem a b c d) = + DictItem a <$> + _Newlines f b <*> + _Newlines f c <*> + _Newlines f d + _Newlines f (DictUnpack a b c) = + DictUnpack a <$> + _Newlines f b <*> + _Newlines f c + +instance HasNewlines (Subscript v a) where + _Newlines f (SubscriptExpr a) = SubscriptExpr <$> _Newlines f a + _Newlines f (SubscriptSlice a b c d) = + SubscriptSlice <$> + _Newlines f a <*> + _Newlines f b <*> + _Newlines f c <*> + _Newlines f d + +instance HasNewlines a => HasNewlines (NonEmpty a) where + _Newlines f (a :| as) = (:|) <$> _Newlines f a <*> _Newlines f as + +instance HasNewlines (StringLiteral a) where + _Newlines = stringLiteralWhitespace.traverse._Newlines + +instance HasNewlines (Expr v a) where + _Newlines fun = go + where + go e = + case e of + Unit a b c -> Unit a <$> _Newlines fun b <*> _Newlines fun c + Lambda a b c d e -> + Lambda a <$> + _Newlines fun b <*> + _Newlines fun c <*> + _Newlines fun d <*> + go e + Yield a b c -> + Yield a <$> _Newlines fun b <*> _Newlines fun c + YieldFrom a b c d -> + YieldFrom a <$> _Newlines fun b <*> _Newlines fun c <*> go d + Ternary a b c d e f -> + Ternary a <$> + go b <*> + _Newlines fun c <*> + go d <*> + _Newlines fun e <*> + go f + ListComp a b c d -> + ListComp a <$> + _Newlines fun b <*> + _Newlines fun c <*> + _Newlines fun d + List a b c d -> + List a <$> + _Newlines fun b <*> + _Newlines fun c <*> + _Newlines fun d + DictComp a b c d -> + DictComp a <$> + _Newlines fun b <*> + _Newlines fun c <*> + _Newlines fun d + Dict a b c d -> + Dict a <$> + _Newlines fun b <*> + _Newlines fun c <*> + _Newlines fun d + SetComp a b c d -> + SetComp a <$> + _Newlines fun b <*> + _Newlines fun c <*> + _Newlines fun d + Set a b c d -> + Set a <$> + _Newlines fun b <*> + _Newlines fun c <*> + _Newlines fun d + Deref a b c d -> + Deref a <$> + go b <*> + _Newlines fun c <*> + _Newlines fun d + Subscript a b c d e -> + Subscript a <$> + go b <*> + _Newlines fun c <*> + _Newlines fun d <*> + _Newlines fun e + Call a b c d e -> + Call a <$> + go b <*> + _Newlines fun c <*> + _Newlines fun d <*> + _Newlines fun e + None a b -> None a <$> _Newlines fun b + Ellipsis a b -> Ellipsis a <$> _Newlines fun b + BinOp a b c d -> + BinOp a <$> + go b <*> + _Newlines fun c <*> + go d + UnOp a b c -> + UnOp a <$> + _Newlines fun b <*> + go c + Parens a b c d -> + Parens a <$> + _Newlines fun b <*> + go c <*> + _Newlines fun d + Ident a -> Ident <$> _Newlines fun a + Int a b c -> Int a b <$> _Newlines fun c + Float a b c -> Float a b <$> _Newlines fun c + Imag a b c -> Imag a b <$> _Newlines fun c + Bool a b c -> Bool a b <$> _Newlines fun c + String a b -> String a <$> _Newlines fun b + Tuple a b c d -> + Tuple a <$> + _Newlines fun b <*> + _Newlines fun c <*> + _Newlines fun d + Not a b c -> Not a <$> _Newlines fun b <*> go c + Generator a b -> Generator a <$> _Newlines fun b + Await a b c -> Await a <$> _Newlines fun b <*> _Newlines fun c + +instance HasNewlines (Decorator v a) where + _Newlines fun (Decorator a b c d e f g) = + Decorator a <$> + _Newlines fun b <*> + _Newlines fun c <*> + _Newlines fun d <*> + pure e <*> + fun f <*> + _Newlines fun g + +instance HasNewlines (ExceptAs v a) where + _Newlines f (ExceptAs a b c) = ExceptAs a <$> _Newlines f b <*> _Newlines f c + +instance HasNewlines (WithItem v a) where + _Newlines f (WithItem a b c) = WithItem a <$> _Newlines f b <*> _Newlines f c + +instance HasNewlines (CompoundStatement v a) where + _Newlines fun s = + case s of + Fundef ann decos idnt asyncWs ws1 name ws2 params ws3 mty s -> + Fundef ann <$> + _Newlines fun decos <*> + _Newlines fun idnt <*> + _Newlines fun asyncWs <*> + _Newlines fun ws1 <*> + _Newlines fun name <*> + _Newlines fun ws2 <*> + _Newlines fun params <*> + _Newlines fun ws3 <*> + _Newlines fun mty <*> + _Newlines fun s + If ann idnt ws1 cond s elifs els -> + If ann <$> + _Newlines fun idnt <*> + _Newlines fun ws1 <*> + _Newlines fun cond <*> + _Newlines fun s <*> + _Newlines fun elifs <*> + _Newlines fun els + While ann idnt ws1 cond s els -> + While ann <$> + _Newlines fun idnt <*> + _Newlines fun ws1 <*> + _Newlines fun cond <*> + _Newlines fun s <*> + _Newlines fun els + TryExcept ann idnt b c f k l -> + TryExcept ann <$> + _Newlines fun idnt <*> + _Newlines fun b <*> + _Newlines fun c <*> + _Newlines fun f <*> + _Newlines fun k <*> + _Newlines fun l + TryFinally ann idnt b c idnt2 f g -> + TryFinally ann <$> + _Newlines fun idnt <*> + _Newlines fun b <*> + _Newlines fun c <*> + _Newlines fun idnt2 <*> + _Newlines fun f <*> + _Newlines fun g + For ann idnt asyncWs b c d e f g -> + For ann <$> + _Newlines fun idnt <*> + _Newlines fun asyncWs <*> + _Newlines fun b <*> + _Newlines fun c <*> + _Newlines fun d <*> + _Newlines fun e <*> + _Newlines fun f <*> + _Newlines fun g + ClassDef a decos idnt b c d e -> + ClassDef a <$> + _Newlines fun decos <*> + _Newlines fun idnt <*> + _Newlines fun b <*> + _Newlines fun c <*> + _Newlines fun d <*> + _Newlines fun e + With a b asyncWs c d e -> + With a <$> + _Newlines fun b <*> + _Newlines fun asyncWs <*> + _Newlines fun c <*> + _Newlines fun d <*> + _Newlines fun e + +instance HasNewlines (ModuleName v a) where + _Newlines f = go + where + go (ModuleNameOne a b) = + ModuleNameOne a <$> _Newlines f b + go (ModuleNameMany a b c d) = + ModuleNameMany a <$> _Newlines f b <*> _Newlines f c <*> go d + +instance HasNewlines (ImportTargets v a) where + _Newlines f (ImportAll a b) = + ImportAll a <$> _Newlines f b + _Newlines f (ImportSome a b) = + ImportSome a <$> _Newlines f b + _Newlines f (ImportSomeParens a b c d) = + ImportSomeParens a <$> + _Newlines f b <*> + _Newlines f c <*> + _Newlines f d + +instance HasNewlines (SimpleStatement v a) where + _Newlines fun s = + case s of + Return a b c -> Return a <$> _Newlines fun b <*> _Newlines fun c + Expr a b -> Expr a <$> _Newlines fun b + Assign a b c -> Assign a <$> _Newlines fun b <*> _Newlines fun c + AugAssign a b c d -> + AugAssign a <$> + _Newlines fun b <*> + pure c <*> + _Newlines fun d + Pass a b -> Pass a <$> _Newlines fun b + Break a b -> Break a <$> _Newlines fun b + Continue a b -> Continue a <$> _Newlines fun b + Global a b c -> Global a <$> _Newlines fun b <*> _Newlines fun c + Nonlocal a b c -> Nonlocal a <$> _Newlines fun b <*> _Newlines fun c + Del a b c -> Del a <$> _Newlines fun b <*> _Newlines fun c + Import a b c -> + Import a <$> + _Newlines fun b <*> + _Newlines fun c + From a b c d e -> + From a <$> + _Newlines fun b <*> + _Newlines fun c <*> + _Newlines fun d <*> + _Newlines fun e + Raise a b c -> + Raise a <$> + _Newlines fun b <*> + _Newlines fun c + Assert a b c d -> + Assert a <$> + _Newlines fun b <*> + _Newlines fun c <*> + _Newlines fun d + +instance HasNewlines (SmallStatement v a) where + _Newlines f (MkSmallStatement s ss sc cmt nl) = + MkSmallStatement <$> + _Newlines f s <*> + _Newlines f ss <*> + _Newlines f sc <*> + pure cmt <*> + _Newlines f nl + +instance HasNewlines (Statement v a) where + _Newlines f (CompoundStatement c) = + CompoundStatement <$> _Newlines f c + _Newlines f (SmallStatement i a) = + SmallStatement <$> + _Newlines f i <*> + _Newlines f a + +instance HasNewlines (Module v a) where + _Newlines f = go + where + go ModuleEmpty = pure ModuleEmpty + go (ModuleBlankFinal a) = pure $ ModuleBlankFinal a + go (ModuleBlank a b c) = + ModuleBlank a <$> f b <*> go c + go (ModuleStatement a b) = + ModuleStatement <$> _Newlines f a <*> go b diff --git a/src/Language/Python/Optics/Validated.hs b/src/Language/Python/Optics/Validated.hs new file mode 100644 --- /dev/null +++ b/src/Language/Python/Optics/Validated.hs @@ -0,0 +1,25 @@ +{-# language DataKinds, PolyKinds, DefaultSignatures #-} + +{-| +Module : Language.Python.Optics.Validated +Copyright : (C) CSIRO 2017-2018 +License : BSD3 +Maintainer : Isaac Elliott +Stability : experimental +Portability : non-portable +-} + +module Language.Python.Optics.Validated ( + Validated (unvalidated) +) where + +import Control.Lens.Getter (Getter, to) +import Data.Coerce (Coercible, coerce) + +-- | A type class for things for which we can strip the validation information. +-- This can help types line up when they need to, for example to put many +-- things of various validation statuses together in a list. +class Validated (s :: [*] -> * -> *) where + unvalidated :: Getter (s v a) (s '[] a) + default unvalidated :: Coercible (s v a) (s '[] a) => Getter (s v a) (s '[] a) + unvalidated = to coerce diff --git a/src/Language/Python/Parse.hs b/src/Language/Python/Parse.hs new file mode 100644 --- /dev/null +++ b/src/Language/Python/Parse.hs @@ -0,0 +1,143 @@ +{-# language DataKinds #-} +{-# language FlexibleContexts #-} +{-# language MultiParamTypeClasses, FlexibleInstances #-} + +{-| +Module : Language.Python.Parse +Copyright : (C) CSIRO 2017-2018 +License : BSD3 +Maintainer : Isaac Elliott +Stability : experimental +Portability : non-portable +-} + +module Language.Python.Parse + ( module Language.Python.Parse.Error + , Parser + , parseModule + , parseStatement + , parseExpr + , parseExprList + -- * Source Information + , SrcInfo(..), initialSrcInfo + ) +where + +import Control.Applicative ((<|>)) +import Data.Bifunctor (first) +import Data.List.NonEmpty (NonEmpty) +import Data.Text (Text) +import Data.Validation (Validation, bindValidation, fromEither) +import Text.Megaparsec (eof) + +import Language.Python.Internal.Lexer + ( SrcInfo(..), initialSrcInfo, withSrcInfo + , tokenize, insertTabs + ) +import Language.Python.Internal.Token (PyToken) +import Language.Python.Internal.Parse + ( Parser, runParser, level, module_, statement, exprOrStarList + , expr, space + ) +import Language.Python.Internal.Syntax.IR (AsIRError) +import Language.Python.Parse.Error +import Language.Python.Syntax.Expr (Expr) +import Language.Python.Syntax.Module (Module) +import Language.Python.Syntax.Statement (Statement) +import Language.Python.Syntax.Whitespace (Indents (..)) + +import qualified Language.Python.Internal.Syntax.IR as IR + +-- | Parse a module +-- +-- https://docs.python.org/3/reference/toplevel_components.html#file-input +parseModule + :: ( AsLexicalError e Char + , AsTabError e SrcInfo + , AsIncorrectDedent e SrcInfo + , AsParseError e (PyToken SrcInfo) + , AsIRError e SrcInfo + ) + => FilePath -- ^ File name + -> Text -- ^ Input to parse + -> Validation (NonEmpty e) (Module '[] SrcInfo) +parseModule fp input = + let + si = initialSrcInfo fp + ir = do + tokens <- tokenize fp input + tabbed <- insertTabs si tokens + runParser fp module_ tabbed + in + fromEither (first pure ir) `bindValidation` IR.fromIR + +-- | Parse a statement +-- +-- https://docs.python.org/3/reference/compound_stmts.html#grammar-token-statement +parseStatement + :: ( AsLexicalError e Char + , AsTabError e SrcInfo + , AsIncorrectDedent e SrcInfo + , AsParseError e (PyToken SrcInfo) + , AsIRError e SrcInfo + ) + => FilePath -- ^ File name + -> Text -- ^ Input to parse + -> Validation (NonEmpty e) (Statement '[] SrcInfo) +parseStatement fp input = + let + si = initialSrcInfo fp + ir = do + tokens <- tokenize fp input + tabbed <- insertTabs si tokens + runParser fp ((statement tlIndent =<< tlIndent) <* eof) tabbed + in + fromEither (first pure ir) `bindValidation` IR.fromIR_statement + where + tlIndent = level <|> withSrcInfo (pure $ Indents []) + +-- | Parse an expression list (unparenthesised tuple) +-- +-- https://docs.python.org/3.5/reference/expressions.html#grammar-token-expression_list +parseExprList + :: ( AsLexicalError e Char + , AsTabError e SrcInfo + , AsIncorrectDedent e SrcInfo + , AsParseError e (PyToken SrcInfo) + , AsIRError e SrcInfo + ) + => FilePath -- ^ File name + -> Text -- ^ Input to parse + -> Validation (NonEmpty e) (Expr '[] SrcInfo) +parseExprList fp input = + let + si = initialSrcInfo fp + ir = do + tokens <- tokenize fp input + tabbed <- insertTabs si tokens + runParser fp (exprOrStarList space <* eof) tabbed + in + fromEither (first pure ir) `bindValidation` IR.fromIR_expr + +-- | Parse an expression +-- +-- https://docs.python.org/3.5/reference/expressions.html#grammar-token-expression +parseExpr + :: ( AsLexicalError e Char + , AsTabError e SrcInfo + , AsIncorrectDedent e SrcInfo + , AsParseError e (PyToken SrcInfo) + , AsIRError e SrcInfo + ) + => FilePath -- ^ File name + -> Text -- ^ Input to parse + -> Validation (NonEmpty e) (Expr '[] SrcInfo) +parseExpr fp input = + let + si = initialSrcInfo fp + ir = do + tokens <- tokenize fp input + tabbed <- insertTabs si tokens + runParser fp (expr space <* eof) tabbed + in + fromEither (first pure ir) `bindValidation` IR.fromIR_expr diff --git a/src/Language/Python/Parse/Error.hs b/src/Language/Python/Parse/Error.hs new file mode 100644 --- /dev/null +++ b/src/Language/Python/Parse/Error.hs @@ -0,0 +1,96 @@ +{-# language FlexibleInstances, MultiParamTypeClasses #-} +{-# language LambdaCase #-} +module Language.Python.Parse.Error + ( ParseError(..) + -- * Classy Prisms + , AsLexicalError(..), AsTabError(..), AsIncorrectDedent(..) + , AsIRError(..), AsParseError(..) + -- * Megaparsec re-exports + , ErrorItem(..) + , SourcePos(..) + ) +where + +import Control.Lens.Prism (prism') +import Data.Set (Set) +import Data.List.NonEmpty (NonEmpty) +import Text.Megaparsec.Error (ErrorItem(..)) +import Text.Megaparsec.Pos (SourcePos(..)) + +import Language.Python.Internal.Lexer + (AsLexicalError(..), AsTabError(..), AsIncorrectDedent(..)) +import Language.Python.Internal.Parse (AsParseError(..)) +import Language.Python.Internal.Syntax.IR (AsIRError(..)) +import Language.Python.Internal.Token (PyToken) + +data ParseError a + -- | An error occured during tokenization (this is a re-packed megaparsec error) + = LexicalError + (NonEmpty SourcePos) + (Maybe (ErrorItem Char)) + (Set (ErrorItem Char)) + -- | An error occured during parsing (this is a re-packed megaparsec error) + | ParseError + (NonEmpty SourcePos) + (Maybe (ErrorItem (PyToken a))) + (Set (ErrorItem (PyToken a))) + -- | Tabs and spaces were used inconsistently + | TabError a + -- | The dedent at the end of a block doesn't match and preceding indents + -- + -- e.g. + -- + -- @ + -- def a(): + -- if b: + -- pass + -- else: + -- pass + -- pass + -- @ + -- + -- The final line will cause an 'IncorrectDedent' error + | IncorrectDedent a + -- | Unpacking ( @*value@ ) was used in an invalid position + | InvalidUnpacking a + deriving (Eq, Show) + +instance AsLexicalError (ParseError a) Char where + _LexicalError = + prism' + (\(a, b, c) -> LexicalError a b c) + (\case + LexicalError a b c -> Just (a, b ,c) + _ -> Nothing) + +instance AsTabError (ParseError a) a where + _TabError = + prism' + TabError + (\case + TabError a -> Just a + _ -> Nothing) + +instance AsIncorrectDedent (ParseError a) a where + _IncorrectDedent = + prism' + IncorrectDedent + (\case + IncorrectDedent a -> Just a + _ -> Nothing) + +instance AsParseError (ParseError a) (PyToken a) where + _ParseError = + prism' + (\(a, b, c) -> ParseError a b c) + (\case + ParseError a b c -> Just (a, b ,c) + _ -> Nothing) + +instance AsIRError (ParseError a) a where + _InvalidUnpacking = + prism' + InvalidUnpacking + (\case + InvalidUnpacking a -> Just a + _ -> Nothing) diff --git a/src/Language/Python/Render.hs b/src/Language/Python/Render.hs new file mode 100644 --- /dev/null +++ b/src/Language/Python/Render.hs @@ -0,0 +1,15 @@ +{-| +Module : Language.Python.Render +Copyright : (C) CSIRO 2017-2018 +License : BSD3 +Maintainer : Isaac Elliott +Stability : experimental +Portability : non-portable +-} + +module Language.Python.Render + ( showModule, showStatement, showExpr + ) +where + +import Language.Python.Internal.Render diff --git a/src/Language/Python/Syntax.hs b/src/Language/Python/Syntax.hs new file mode 100644 --- /dev/null +++ b/src/Language/Python/Syntax.hs @@ -0,0 +1,49 @@ +{-| +Module : Language.Python.Syntax +Copyright : (C) CSIRO 2017-2018 +License : BSD3 +Maintainer : Isaac Elliott +Stability : experimental +Portability : non-portable + +The abstract syntax tree for Python. + +Key datatypes include 'Module', 'Statement', and 'Expr'. +-} + +module Language.Python.Syntax + ( module Language.Python.Syntax.AugAssign + , module Language.Python.Syntax.CommaSep + , module Language.Python.Syntax.Comment + , module Language.Python.Syntax.Expr + , module Language.Python.Syntax.Ident + , module Language.Python.Syntax.Import + , module Language.Python.Syntax.Module + , module Language.Python.Syntax.ModuleNames + , module Language.Python.Syntax.Numbers + , module Language.Python.Syntax.Operator.Binary + , module Language.Python.Syntax.Operator.Unary + , module Language.Python.Syntax.Punctuation + , module Language.Python.Syntax.Statement + , module Language.Python.Syntax.Strings + , module Language.Python.Syntax.Types + , module Language.Python.Syntax.Whitespace + ) +where + +import Language.Python.Syntax.AugAssign +import Language.Python.Syntax.CommaSep +import Language.Python.Syntax.Comment +import Language.Python.Syntax.Expr +import Language.Python.Syntax.Ident +import Language.Python.Syntax.Import +import Language.Python.Syntax.Module +import Language.Python.Syntax.ModuleNames +import Language.Python.Syntax.Numbers +import Language.Python.Syntax.Operator.Binary +import Language.Python.Syntax.Operator.Unary +import Language.Python.Syntax.Punctuation +import Language.Python.Syntax.Statement +import Language.Python.Syntax.Strings +import Language.Python.Syntax.Types +import Language.Python.Syntax.Whitespace diff --git a/src/Language/Python/Syntax/AugAssign.hs b/src/Language/Python/Syntax/AugAssign.hs new file mode 100644 --- /dev/null +++ b/src/Language/Python/Syntax/AugAssign.hs @@ -0,0 +1,72 @@ +{-# language DeriveFunctor, DeriveFoldable, DeriveTraversable #-} + +{-| +Module : Language.Python.Syntax.AugAssign +Copyright : (C) CSIRO 2017-2018 +License : BSD3 +Maintainer : Isaac Elliott +Stability : experimental +Portability : non-portable +-} + +module Language.Python.Syntax.AugAssign where + +import Control.Lens.Lens (lens) + +import Language.Python.Syntax.Whitespace + +-- | Augmented assignments (PEP 203), such as: +-- +-- @ +-- x += y +-- @ +-- +-- or +-- +-- @ +-- x <<= 8 +-- @ +-- +-- An 'AugAssign' has an 'AugAssignOp' and trailing whitespace. There is an +-- optional annotation, which can simply be @()@ if no annotation is desired. +data AugAssign a + = MkAugAssign + { _augAssignType :: AugAssignOp + , _augAssignAnn :: a + , _augAssignWhitespace :: [Whitespace] + } + deriving (Eq, Show, Functor, Foldable, Traversable) + +instance HasTrailingWhitespace (AugAssign a) where + trailingWhitespace = + lens _augAssignWhitespace (\a b -> a { _augAssignWhitespace = b }) + +-- | Augmented assignment operators +data AugAssignOp + -- | @+=@ + = PlusEq + -- | @-=@ + | MinusEq + -- | @*=@ + | StarEq + -- | @\@=@ + | AtEq + -- | @/=@ + | SlashEq + -- | @%=@ + | PercentEq + -- | @&=@ + | AmpersandEq + -- | @|=@ + | PipeEq + -- | @^=@ + | CaretEq + -- | @<<=@ + | ShiftLeftEq + -- | @>>=@ + | ShiftRightEq + -- | @**=@ + | DoubleStarEq + -- | @//=@ + | DoubleSlashEq + deriving (Eq, Show) diff --git a/src/Language/Python/Syntax/CommaSep.hs b/src/Language/Python/Syntax/CommaSep.hs new file mode 100644 --- /dev/null +++ b/src/Language/Python/Syntax/CommaSep.hs @@ -0,0 +1,210 @@ +{-# language LambdaCase #-} +{-# language DeriveFunctor, DeriveFoldable, DeriveTraversable #-} + +{-| +Module : Language.Python.Syntax.CommaSep +Copyright : (C) CSIRO 2017-2018 +License : BSD3 +Maintainer : Isaac Elliott +Stability : experimental +Portability : non-portable +-} + +module Language.Python.Syntax.CommaSep + ( Comma(..) + , CommaSep(..), _CommaSep, csTrailingWhitespace + , appendCommaSep, maybeToCommaSep, listToCommaSep + , CommaSep1(..) + , commaSep1Head, appendCommaSep1, listToCommaSep1, listToCommaSep1' + , CommaSep1'(..) + , _CommaSep1' + ) +where + +import Control.Lens.Getter ((^.)) +import Control.Lens.Iso (Iso, iso) +import Control.Lens.Lens (lens) +import Control.Lens.Setter ((.~)) +import Control.Lens.Traversal (Traversal') +import Data.Coerce (coerce) +import Data.Function ((&)) +import Data.Functor (($>)) +import Data.List.NonEmpty (NonEmpty(..)) +import Data.Maybe (fromMaybe) +import Data.Semigroup (Semigroup(..)) + +import Language.Python.Syntax.Punctuation +import Language.Python.Syntax.Whitespace (Whitespace (Space), HasTrailingWhitespace (..)) + +-- | Items separated by commas, with optional whitespace following each comma +data CommaSep a + = CommaSepNone + | CommaSepOne a + | CommaSepMany a Comma (CommaSep a) + deriving (Eq, Show, Functor, Foldable, Traversable) + +-- | 'Traversal' targeting the trailing whitespace in a comma separated list. +-- +-- This can't be an instance of 'HasTrailingWhitespace' because 'CommaSepNone' never +-- has trailing whitespace. +csTrailingWhitespace + :: HasTrailingWhitespace a + => Traversal' (CommaSep a) [Whitespace] +csTrailingWhitespace _ CommaSepNone = pure CommaSepNone +csTrailingWhitespace f (CommaSepOne a) = CommaSepOne <$> trailingWhitespace f a +csTrailingWhitespace f (CommaSepMany a (MkComma b) CommaSepNone) = + (\b' -> CommaSepMany a (MkComma b') CommaSepNone) <$> f b +csTrailingWhitespace f (CommaSepMany a b c) = + CommaSepMany a b <$> csTrailingWhitespace f c + + +-- | Convert a maybe to a singleton or nullary 'CommaSep' +maybeToCommaSep :: Maybe a -> CommaSep a +maybeToCommaSep = maybe CommaSepNone CommaSepOne + +-- | Convert a list to a 'CommaSep' +-- +-- Anywhere where whitespace is ambiguous, this function puts a single space +listToCommaSep :: [a] -> CommaSep a +listToCommaSep [] = CommaSepNone +listToCommaSep [a] = CommaSepOne a +listToCommaSep (a:as) = CommaSepMany a (MkComma [Space]) $ listToCommaSep as + +-- | Appends two comma separated values together. +-- +-- The provided whitespace is to follow the joining comma which is added +appendCommaSep :: [Whitespace] -> CommaSep a -> CommaSep a -> CommaSep a +appendCommaSep _ CommaSepNone b = b +appendCommaSep _ (CommaSepOne a) CommaSepNone = CommaSepOne a +appendCommaSep ws (CommaSepOne a) (CommaSepOne b) = CommaSepMany a (MkComma ws) (CommaSepOne b) +appendCommaSep ws (CommaSepOne a) (CommaSepMany b c cs) = CommaSepMany a (MkComma ws) (CommaSepMany b c cs) +appendCommaSep ws (CommaSepMany a c cs) b = CommaSepMany a c (appendCommaSep ws cs b) + +instance Semigroup (CommaSep a) where + (<>) = appendCommaSep [Space] + +instance Monoid (CommaSep a) where + mempty = CommaSepNone + mappend = (<>) + +-- | Non-empty 'CommaSep' +data CommaSep1 a + = CommaSepOne1 a + | CommaSepMany1 a Comma (CommaSep1 a) + deriving (Eq, Show, Functor, Foldable, Traversable) + +-- | Get the first element of a 'CommaSep1' +commaSep1Head :: CommaSep1 a -> a +commaSep1Head (CommaSepOne1 a) = a +commaSep1Head (CommaSepMany1 a _ _) = a + +-- | Appends two non-empty comma separated values together. +-- +-- The provided whitespace is to follow the joining comma which is added +appendCommaSep1 :: [Whitespace] -> CommaSep1 a -> CommaSep1 a -> CommaSep1 a +appendCommaSep1 ws a b = + CommaSepMany1 + (case a of; CommaSepOne1 x -> x; CommaSepMany1 x _ _ -> x) + (case a of; CommaSepOne1 _ -> MkComma ws; CommaSepMany1 _ ws' _ -> ws') + (case a of; CommaSepOne1 _ -> b; CommaSepMany1 _ _ x -> x <> b) + +instance Semigroup (CommaSep1 a) where + (<>) = appendCommaSep1 [Space] + +instance HasTrailingWhitespace s => HasTrailingWhitespace (CommaSep1 s) where + trailingWhitespace = + lens + (\case + CommaSepOne1 a -> a ^. trailingWhitespace + CommaSepMany1 _ _ a -> a ^. trailingWhitespace) + (\cs ws -> + case cs of + CommaSepOne1 a -> + CommaSepOne1 (a & trailingWhitespace .~ ws) + CommaSepMany1 a b c -> CommaSepMany1 (coerce a) b (c & trailingWhitespace .~ ws)) + +-- | Convert a 'NonEmpty' to a 'CommaSep1' +-- +-- Anywhere where whitespace is ambiguous, this function puts a single space +listToCommaSep1 :: NonEmpty a -> CommaSep1 a +listToCommaSep1 (a :| as) = go (a:as) + where + go [] = error "impossible" + go [x] = CommaSepOne1 x + go (x:xs) = CommaSepMany1 x (MkComma [Space]) $ go xs + +-- | Non-empty 'CommaSep', optionally terminated by a comma +-- +-- Assumes that the contents consumes trailing whitespace +data CommaSep1' a + = CommaSepOne1' a (Maybe Comma) + | CommaSepMany1' a Comma (CommaSep1' a) + deriving (Eq, Show, Functor, Foldable, Traversable) + +-- | Iso to unpack a 'CommaSep' +_CommaSep + :: Iso + (Maybe (a, [(Comma, a)], Maybe Comma)) + (Maybe (b, [(Comma, b)], Maybe Comma)) + (CommaSep a) + (CommaSep b) +_CommaSep = iso toCs fromCs + where + toCs :: Maybe (a, [(Comma, a)], Maybe Comma) -> CommaSep a + toCs Nothing = CommaSepNone + toCs (Just (a, b, c)) = + case b of + [] -> maybe (CommaSepOne a) (\c' -> CommaSepMany a c' CommaSepNone) c + (d, e):ds -> CommaSepMany a d $ toCs (Just (e, ds, c)) + + fromCs :: CommaSep a -> Maybe (a, [(Comma, a)], Maybe Comma) + fromCs CommaSepNone = Nothing + fromCs (CommaSepOne a) = Just (a, [], Nothing) + fromCs (CommaSepMany a b c) = + case fromCs c of + Nothing -> Just (a, [], Just b) + Just (x, y, z) -> Just (a, (b, x) : y, z) + +-- | Iso to unpack a 'CommaSep1'' +_CommaSep1' + :: Iso + (a, [(Comma, a)], Maybe Comma) + (b, [(Comma, b)], Maybe Comma) + (CommaSep1' a) + (CommaSep1' b) +_CommaSep1' = iso toCs fromCs + where + toCs (a, [], b) = CommaSepOne1' a b + toCs (a, (b, c) : bs, d) = CommaSepMany1' a b $ toCs (c, bs, d) + + fromCs (CommaSepOne1' a b) = (a, [], b) + fromCs (CommaSepMany1' a b c) = + let + (d, e, f) = fromCs c + in + (a, (b, d) : e, f) + +-- | Attempt to insert comma separators into a list, which will not be +-- terminated by a comma. +-- +-- If the list is empty, 'Nothing' is returned. +listToCommaSep1' :: [a] -> Maybe (CommaSep1' a) +listToCommaSep1' [] = Nothing +listToCommaSep1' [a] = Just (CommaSepOne1' a Nothing) +listToCommaSep1' (a:as) = + CommaSepMany1' a (MkComma [Space]) <$> listToCommaSep1' as + +instance HasTrailingWhitespace s => HasTrailingWhitespace (CommaSep1' s) where + trailingWhitespace = + lens + (\case + CommaSepOne1' a b -> maybe (a ^. trailingWhitespace) (^. trailingWhitespace) b + CommaSepMany1' _ _ a -> a ^. trailingWhitespace) + (\cs ws -> + case cs of + CommaSepOne1' a b -> + CommaSepOne1' + (fromMaybe (a & trailingWhitespace .~ ws) $ b $> coerce a) + (b $> MkComma ws) + CommaSepMany1' a b c -> + CommaSepMany1' (coerce a) b (c & trailingWhitespace .~ ws)) diff --git a/src/Language/Python/Syntax/Comment.hs b/src/Language/Python/Syntax/Comment.hs new file mode 100644 --- /dev/null +++ b/src/Language/Python/Syntax/Comment.hs @@ -0,0 +1,44 @@ +{-# language DeriveFunctor, DeriveFoldable, DeriveTraversable #-} +{-# language TemplateHaskell #-} + +{-| +Module : Language.Python.Syntax.Comment +Copyright : (C) CSIRO 2017-2018 +License : BSD3 +Maintainer : Isaac Elliott +Stability : experimental +Portability : non-portable +-} + +module Language.Python.Syntax.Comment where + +import Data.Deriving (deriveEq1, deriveOrd1) + +-- | A Python single-line comment, such as on the following line: +-- +-- @ +-- y = x + 4 # add four to the value of x +-- @ +-- +-- In this case, the structure parsed would be +-- +-- @ +-- MkComment () " add four to the value of x" +-- @ +-- +-- with the hash being inferred, and the space after the hash being preserved. +-- +-- Python does not have multi-line comments. There is a common convention of +-- using a multi-line string expression as a multi-line comment, since a +-- string expression is a no-op statement. Such multi-line comments are +-- __NOT__ represented with this data type, but rather as normal +-- string expressions (since that's what they are). +data Comment a + = MkComment + { _commentAnn :: a + , _commentValue :: String + } + deriving (Eq, Ord, Show, Functor, Foldable, Traversable) + +deriveEq1 ''Comment +deriveOrd1 ''Comment diff --git a/src/Language/Python/Syntax/Expr.hs b/src/Language/Python/Syntax/Expr.hs new file mode 100644 --- /dev/null +++ b/src/Language/Python/Syntax/Expr.hs @@ -0,0 +1,1096 @@ +{-# language LambdaCase #-} +{-# language DataKinds, KindSignatures #-} +{-# language ScopedTypeVariables #-} +{-# language MultiParamTypeClasses, FlexibleInstances #-} +{-# language DeriveFunctor, DeriveFoldable, DeriveTraversable, DeriveGeneric #-} +{-# language ExistentialQuantification #-} + +{-| +Module : Language.Python.Syntax.Expr +Copyright : (C) CSIRO 2017-2018 +License : BSD3 +Maintainer : Isaac Elliott +Stability : experimental +Portability : non-portable +-} + +module Language.Python.Syntax.Expr + ( -- * Expressions + Expr(..), HasExprs(..), exprAnn, shouldGroupLeft, shouldGroupRight + -- * Parameters and arguments + , Param(..), paramAnn, paramType_, paramType, paramName + , Arg(..), argExpr + -- * Comprehension expressions + -- | https://docs.python.org/3/reference/expressions.html#grammar-token-comprehension + , Comprehension(..), CompIf(..), CompFor(..) + -- * Collection items + , DictItem(..), ListItem(..), SetItem(..), TupleItem(..) + -- * Subscripts + , Subscript(..) + ) +where + +import Control.Lens.Cons (_last) +import Control.Lens.Fold ((^?), (^?!)) +import Control.Lens.Getter ((^.), getting, to, view) +import Control.Lens.Lens (Lens, Lens', lens) +import Control.Lens.Plated (Plated(..), gplate) +import Control.Lens.Prism (_Just, _Left, _Right) +import Control.Lens.Setter ((.~), mapped, over) +import Control.Lens.Traversal (Traversal, failing, traverseOf) +import Control.Lens.Tuple (_2) +import Data.Bifunctor (bimap) +import Data.Bifoldable (bifoldMap) +import Data.Bitraversable (bitraverse) +import Data.Coerce (coerce) +import Data.Digit.Integral (integralDecDigits) +import Data.Function ((&)) +import Data.List.NonEmpty (NonEmpty) +import Data.Maybe (isNothing) +import Data.Monoid ((<>)) +import Data.String (IsString(..)) +import GHC.Generics (Generic) +import Unsafe.Coerce (unsafeCoerce) + +import Language.Python.Optics.Validated (Validated(..)) +import Language.Python.Syntax.CommaSep +import Language.Python.Syntax.Ident +import Language.Python.Syntax.Numbers +import Language.Python.Syntax.Operator.Binary +import Language.Python.Syntax.Operator.Unary +import Language.Python.Syntax.Punctuation +import Language.Python.Syntax.Strings +import Language.Python.Syntax.Whitespace + +{- + +[unsafeCoerce Validation] + +We can't 'coerce' 'Expr's because the @v@ parameter is considered to have a +nominal role, due to datatypes like 'Comprehension'. We only ever use @v@ in +as a phantom in 'Expr', so 'unsafeCoerce :: Expr v a -> Expr '[] a' is safe. + +-} +instance Validated Expr where; unvalidated = to unsafeCoerce +instance Validated Param where; unvalidated = to unsafeCoerce +instance Validated Arg where; unvalidated = to unsafeCoerce +instance Validated DictItem where; unvalidated = to unsafeCoerce +instance Validated SetItem where; unvalidated = to unsafeCoerce +instance Validated TupleItem where; unvalidated = to unsafeCoerce +instance Validated ListItem where; unvalidated = to unsafeCoerce + +-- | 'Control.Lens.Traversal.Traversal' over all the expressions in a term +class HasExprs s where + _Exprs :: Traversal (s v a) (s '[] a) (Expr v a) (Expr '[] a) + +-- | Formal parameters for functions +-- +-- See +data Param (v :: [*]) a + -- | @def foo(a):@ + = PositionalParam + { _paramAnn :: a + , _paramName :: Ident v a + , _paramType :: Maybe (Colon, Expr v a) + } + -- | @def foo(bar=None):@ + | KeywordParam + { _paramAnn :: a + , _paramName :: Ident v a + -- ':' spaces + , _paramType :: Maybe (Colon, Expr v a) + -- = spaces + , _unsafeKeywordParamWhitespaceRight :: [Whitespace] + , _unsafeKeywordParamExpr :: Expr v a + } + -- | @def foo(*xs):@ + | StarParam + { _paramAnn :: a + -- '*' spaces + , _unsafeStarParamWhitespace :: [Whitespace] + , _unsafeStarParamName :: Ident v a + -- ':' spaces + , _paramType :: Maybe (Colon, Expr v a) + } + -- | @def foo(*):@ + | UnnamedStarParam + { _paramAnn :: a + -- '*' spaces + , _unsafeUnnamedStarParamWhitespace :: [Whitespace] + } + -- | @def foo(**dict):@ + | DoubleStarParam + { _paramAnn :: a + -- '**' spaces + , _unsafeDoubleStarParamWhitespace :: [Whitespace] + , _paramName :: Ident v a + -- ':' spaces + , _paramType :: Maybe (Colon, Expr v a) + } + deriving (Eq, Show, Functor, Foldable, Traversable) + +instance IsString (Param '[] ()) where + fromString a = PositionalParam () (fromString a) Nothing + +instance HasTrailingWhitespace (Param v a) where + trailingWhitespace = + lens + (\case + PositionalParam _ a b -> + maybe (a ^. trailingWhitespace) (^. _2.trailingWhitespace) b + KeywordParam _ _ _ _ a -> a ^. trailingWhitespace + UnnamedStarParam _ a -> a + StarParam _ _ b c -> + maybe + (b ^. trailingWhitespace) + (^. _2.trailingWhitespace) + c + DoubleStarParam _ _ a b -> + maybe + (a ^. trailingWhitespace) + (^. _2.trailingWhitespace) + b) + (\p ws -> case p of + PositionalParam a b c -> + PositionalParam a + (if isNothing c then b & trailingWhitespace .~ ws else b) + (c & _Just._2.trailingWhitespace .~ ws) + KeywordParam a b c d e -> + KeywordParam a b c d $ e & trailingWhitespace .~ ws + UnnamedStarParam a _ -> UnnamedStarParam a ws + StarParam a b c d -> + StarParam a + b + (if isNothing d then c & trailingWhitespace .~ ws else c) + (d & _Just._2.trailingWhitespace .~ ws) + DoubleStarParam a b c d -> + DoubleStarParam a b + (if isNothing d then c & trailingWhitespace .~ ws else c) + (d & _Just._2.trailingWhitespace .~ ws)) + +-- | Lens on the syntrax tree annotation on a parameter +paramAnn :: Lens' (Param v a) a +paramAnn = lens _paramAnn (\s a -> s { _paramAnn = a}) + +-- | A faux-lens on the optional Python type annotation which may follow a parameter +-- +-- This is not a lawful 'Lens' because setting an 'UnnamedStarParam''s type won't +-- have any effect. +-- +-- This optic, like many others in hpython, loses validation information +-- (the @v@ type parameter) +-- +-- The following is an example, where @int@ is the paramtype: +-- +-- @ +-- def foo(x: int): +-- @ +paramType_ + :: Functor f + => (Maybe (Colon, Expr v a) -> f (Maybe (Colon, Expr '[] a))) + -> Param v a -> f (Param '[] a) +paramType_ = + lens + (\case + UnnamedStarParam{} -> Nothing + a -> _paramType a) + (\s ty -> case s ^. unvalidated of + PositionalParam a b _ -> PositionalParam a b ty + KeywordParam a b _ c d -> KeywordParam a b ty c d + StarParam a b c _ -> StarParam a b c ty + UnnamedStarParam a b -> UnnamedStarParam a b + DoubleStarParam a b c _ -> DoubleStarParam a b c ty) + +-- | 'Traversal' targeting the Python type annotations which may follow a parameter +paramType :: Traversal (Param v a) (Param '[] a) (Colon, Expr v a) (Colon, Expr '[] a) +paramType = paramType_._Just + +-- | (affine) 'Control.Lens.Traversal.Traversal' on the name of a parameter +-- +-- The name is @x@ in the following examples: +-- +-- @ +-- def foo(x): +-- def foo(x=None): +-- def foo(*x): +-- def foo(**x): +-- @ +-- +-- But the following example does not have a 'paramName': +-- +-- @ +-- def foo(*): +-- @ +paramName :: Traversal (Param v a) (Param '[] a) (Ident v a) (Ident '[] a) +paramName f (PositionalParam a b c) = + PositionalParam a <$> f b <*> pure (over (mapped._2) (view unvalidated) c) +paramName f (KeywordParam a b c d e) = + (\b' -> KeywordParam a b' (over (mapped._2) (view unvalidated) c) d (e ^. unvalidated)) <$> + f b +paramName f (StarParam a b c d) = + (\c' -> StarParam a b c' (over (mapped._2) (view unvalidated) d)) <$> + f c +paramName _ (UnnamedStarParam a b) = pure $ UnnamedStarParam a b +paramName f (DoubleStarParam a b c d) = + (\c' -> DoubleStarParam a b c' (over (mapped._2) (view unvalidated) d)) <$> + f c + +instance HasExprs Param where + _Exprs f (KeywordParam a name ty ws2 expr) = + KeywordParam a (coerce name) <$> + traverseOf (traverse._2) f ty <*> + pure ws2 <*> + f expr + _Exprs f (PositionalParam a b c) = + PositionalParam a (coerce b) <$> traverseOf (traverse._2) f c + _Exprs f (StarParam a b c d) = + StarParam a b (coerce c) <$> traverseOf (traverse._2) f d + _Exprs _ (UnnamedStarParam a b) = pure $ UnnamedStarParam a b + _Exprs f (DoubleStarParam a b c d) = + DoubleStarParam a b (coerce c) <$> traverseOf (traverse._2) f d + +-- | Actual parameters for functions +-- +-- In the following examples, @x@ is an actual parameter. +-- +-- @ +-- y = foo(x) +-- y = bar(quux=x) +-- y = baz(*x) +-- y = flux(**x) +-- @ +data Arg (v :: [*]) a + = PositionalArg + { _argAnn :: a + , _argExpr :: Expr v a + } + | KeywordArg + { _argAnn :: a + , _unsafeKeywordArgName :: Ident v a + , _unsafeKeywordArgWhitespaceRight :: [Whitespace] + , _argExpr :: Expr v a + } + | StarArg + { _argAnn :: a + , _unsafeStarArgWhitespace :: [Whitespace] + , _argExpr :: Expr v a + } + | DoubleStarArg + { _argAnn :: a + , _unsafeDoubleStarArgWhitespace :: [Whitespace] + , _argExpr :: Expr v a + } + deriving (Eq, Show, Functor, Foldable, Traversable) + +instance IsString (Arg '[] ()) where; fromString = PositionalArg () . fromString + +-- | Lens on the Python expression which is passed as the argument +argExpr :: Lens (Arg v a) (Arg '[] a) (Expr v a) (Expr '[] a) +argExpr = lens _argExpr (\s a -> (s ^. unvalidated) { _argExpr = a }) + +instance HasExprs Arg where + _Exprs f (KeywordArg a name ws2 expr) = KeywordArg a (coerce name) ws2 <$> f expr + _Exprs f (PositionalArg a expr) = PositionalArg a <$> f expr + _Exprs f (StarArg a ws expr) = StarArg a ws <$> f expr + _Exprs f (DoubleStarArg a ws expr) = StarArg a ws <$> f expr + +-- | A Python for comprehension, such as +-- +-- @ +-- x for y in z +-- @ +data Comprehension e (v :: [*]) a + = Comprehension a (e v a) (CompFor v a) [Either (CompFor v a) (CompIf v a)] -- ^ (comp_for | comp_if)* + deriving (Eq, Show) + +instance HasTrailingWhitespace (Comprehension e v a) where + trailingWhitespace = + lens + (\(Comprehension _ _ a b) -> + case b of + [] -> a ^. trailingWhitespace + _ -> b ^?! _last.failing (_Left.trailingWhitespace) (_Right.trailingWhitespace)) + (\(Comprehension a b c d) ws -> + case d of + [] -> Comprehension a b (c & trailingWhitespace .~ ws) d + _ -> + Comprehension a b c + (d & + _last.failing (_Left.trailingWhitespace) (_Right.trailingWhitespace) .~ ws)) + +instance Functor (e v) => Functor (Comprehension e v) where + fmap f (Comprehension a b c d) = + Comprehension (f a) (fmap f b) (fmap f c) (fmap (bimap (fmap f) (fmap f)) d) + +instance Foldable (e v) => Foldable (Comprehension e v) where + foldMap f (Comprehension a b c d) = + f a <> foldMap f b <> foldMap f c <> foldMap (bifoldMap (foldMap f) (foldMap f)) d + +instance Traversable (e v) => Traversable (Comprehension e v) where + traverse f (Comprehension a b c d) = + Comprehension <$> + f a <*> + traverse f b <*> + traverse f c <*> + traverse (bitraverse (traverse f) (traverse f)) d + +-- | A condition inside a comprehension, e.g. @[x for x in xs if even(x)]@ +data CompIf (v :: [*]) a + = CompIf a [Whitespace] (Expr v a) + deriving (Eq, Show, Functor, Foldable, Traversable) + +instance HasTrailingWhitespace (CompIf v a) where + trailingWhitespace = + lens + (\(CompIf _ _ a) -> a ^. trailingWhitespace) + (\(CompIf a b c) ws -> CompIf a b $ c & trailingWhitespace .~ ws) + +-- | A nested comprehesion, e.g. @[(x, y) for x in xs for y in ys]@ +data CompFor (v :: [*]) a + = CompFor a [Whitespace] (Expr v a) [Whitespace] (Expr v a) + deriving (Eq, Show, Functor, Foldable, Traversable) + +instance HasTrailingWhitespace (CompFor v a) where + trailingWhitespace = + lens + (\(CompFor _ _ _ _ a) -> a ^. trailingWhitespace) + (\(CompFor a b c d e) ws -> CompFor a b c d $ e & trailingWhitespace .~ ws) + +-- | @a : b@ or @**a@ +-- +-- Used to construct dictionaries, e.g. @{ 1: a, 2: b, **c }@ +-- +-- https://docs.python.org/3/reference/expressions.html#dictionary-displays +data DictItem (v :: [*]) a + = DictItem + { _dictItemAnn :: a + , _unsafeDictItemKey :: Expr v a + , _unsafeDictItemColon :: Colon + , _unsafeDictItemValue :: Expr v a + } + | DictUnpack + { _dictItemAnn :: a + , _unsafeDictItemUnpackWhitespace :: [Whitespace] + , _unsafeDictItemUnpackValue :: Expr v a + } deriving (Eq, Show, Functor, Foldable, Traversable) + +instance HasTrailingWhitespace (DictItem v a) where + trailingWhitespace = + lens + (\(DictItem _ _ _ a) -> a ^. trailingWhitespace) + (\(DictItem a b c d) ws -> DictItem a b c (d & trailingWhitespace .~ ws)) + +-- | Syntax for things that can be used as subscripts (inside the square brackets) +-- +-- e.g. +-- +-- @a[b]@ +-- +-- @a[:]@ +-- +-- @a[b:]@ +-- +-- @a[:b]@ +-- +-- @a[b:c]@ +-- +-- @a[b:c:d]@ +-- +-- https://docs.python.org/3/reference/expressions.html#subscriptions +data Subscript (v :: [*]) a + = SubscriptExpr (Expr v a) + | SubscriptSlice + -- [expr] + (Maybe (Expr v a)) + -- ':' + Colon + -- [expr] + (Maybe (Expr v a)) + -- [':' [expr]] + (Maybe (Colon, Maybe (Expr v a))) + deriving (Eq, Show, Functor, Foldable, Traversable) + +instance HasTrailingWhitespace (Subscript v a) where + trailingWhitespace = + lens + (\case + SubscriptExpr e -> e ^. trailingWhitespace + SubscriptSlice _ b c d -> + case d of + Nothing -> + case c of + Nothing -> b ^. trailingWhitespace + Just e -> e ^. trailingWhitespace + Just (e, f) -> + case f of + Nothing -> e ^. trailingWhitespace + Just g -> g ^. trailingWhitespace) + (\x ws -> + case x of + SubscriptExpr e -> SubscriptExpr $ e & trailingWhitespace .~ ws + SubscriptSlice a b c d -> + (\(b', c', d') -> SubscriptSlice a b' c' d') $ + case d of + Nothing -> + case c of + Nothing -> (MkColon ws, c, d) + Just e -> (b, Just $ e & trailingWhitespace .~ ws, d) + Just (e, f) -> + case f of + Nothing -> (b, c, Just (MkColon ws, f)) + Just g -> (b, c, Just (e, Just $ g & trailingWhitespace .~ ws))) + +-- | @a@ or @*a@ +-- +-- Used to construct lists, e.g. @[ 1, 'x', **c ]@ +-- +-- https://docs.python.org/3/reference/expressions.html#list-displays +data ListItem (v :: [*]) a + = ListItem + { _listItemAnn :: a + , _unsafeListItemValue :: Expr v a + } + | ListUnpack + { _listItemAnn :: a + , _unsafeListUnpackParens :: [([Whitespace], [Whitespace])] + , _unsafeListUnpackWhitespace :: [Whitespace] + , _unsafeListUnpackValue :: Expr v a + } deriving (Eq, Show, Functor, Foldable, Traversable) + +instance HasExprs ListItem where + _Exprs f (ListItem a b) = ListItem a <$> f b + _Exprs f (ListUnpack a b c d) = ListUnpack a b c <$> f d + +instance HasTrailingWhitespace (ListItem v a) where + trailingWhitespace = + lens + (\case + ListItem _ a -> a ^. trailingWhitespace + ListUnpack _ [] _ a -> a ^. trailingWhitespace + ListUnpack _ ((_, ws) : _) _ _ -> ws) + (\a ws -> + case a of + ListItem b c -> ListItem b $ c & trailingWhitespace .~ ws + ListUnpack b [] d e -> ListUnpack b [] d $ e & trailingWhitespace .~ ws + ListUnpack b ((c, _) : rest) e f -> ListUnpack b ((c, ws) : rest) e f) + +-- | @a@ or @*a@ +-- +-- Used to construct sets, e.g. @{ 1, 'x', **c }@ +-- +-- https://docs.python.org/3/reference/expressions.html#set-displays +data SetItem (v :: [*]) a + = SetItem + { _setItemAnn :: a + , _unsafeSetItemValue :: Expr v a + } + | SetUnpack + { _setItemAnn :: a + , _unsafeSetUnpackParens :: [([Whitespace], [Whitespace])] + , _unsafeSetUnpackWhitespace :: [Whitespace] + , _unsafeSetUnpackValue :: Expr v a + } deriving (Eq, Show, Functor, Foldable, Traversable) + +instance HasExprs SetItem where + _Exprs f (SetItem a b) = SetItem a <$> f b + _Exprs f (SetUnpack a b c d) = SetUnpack a b c <$> f d + +instance HasTrailingWhitespace (SetItem v a) where + trailingWhitespace = + lens + (\case + SetItem _ a -> a ^. trailingWhitespace + SetUnpack _ [] _ a -> a ^. trailingWhitespace + SetUnpack _ ((_, ws) : _) _ _ -> ws) + (\a ws -> + case a of + SetItem b c -> SetItem b $ c & trailingWhitespace .~ ws + SetUnpack b [] d e -> SetUnpack b [] d $ e & trailingWhitespace .~ ws + SetUnpack b ((c, _) : rest) e f -> SetUnpack b ((c, ws) : rest) e f) + +-- | @a@ or @*a@ +-- +-- Used to construct tuples, e.g. @(1, 'x', **c)@ +data TupleItem (v :: [*]) a + = TupleItem + { _tupleItemAnn :: a + , _unsafeTupleItemValue :: Expr v a + } + | TupleUnpack + { _tupleItemAnn :: a + , _unsafeTupleUnpackParens :: [([Whitespace], [Whitespace])] + , _unsafeTupleUnpackWhitespace :: [Whitespace] + , _unsafeTupleUnpackValue :: Expr v a + } deriving (Eq, Show, Functor, Foldable, Traversable) + +instance HasExprs TupleItem where + _Exprs f (TupleItem a b) = TupleItem a <$> f b + _Exprs f (TupleUnpack a b c d) = TupleUnpack a b c <$> f d + +instance HasTrailingWhitespace (TupleItem v a) where + trailingWhitespace = + lens + (\case + TupleItem _ a -> a ^. trailingWhitespace + TupleUnpack _ [] _ a -> a ^. trailingWhitespace + TupleUnpack _ ((_, ws) : _) _ _ -> ws) + (\a ws -> + case a of + TupleItem b c -> TupleItem b $ c & trailingWhitespace .~ ws + TupleUnpack b [] d e -> TupleUnpack b [] d $ e & trailingWhitespace .~ ws + TupleUnpack b ((c, _) : rest) e f -> TupleUnpack b ((c, ws) : rest) e f) + +-- | This large sum type covers all valid Python /expressions/ +data Expr (v :: [*]) a + -- | @()@ + -- + -- https://docs.python.org/3/reference/expressions.html#parenthesized-forms + = Unit + { _unsafeExprAnn :: a + , _unsafeUnitWhitespaceInner :: [Whitespace] + , _unsafeUnitWhitespaceRight :: [Whitespace] + } + -- | @lambda x, y: x@ + -- + -- https://docs.python.org/3/reference/expressions.html#lambda + | Lambda + { _unsafeExprAnn :: a + , _unsafeLambdaWhitespace :: [Whitespace] + , _unsafeLambdaArgs :: CommaSep (Param v a) + , _unsafeLambdaColon :: Colon + , _unsafeLambdaBody :: Expr v a + } + -- | @yield@ + -- + -- @yield a@ + -- + -- @yield a, b@ + -- + -- https://docs.python.org/3/reference/expressions.html#yield-expressions + | Yield + { _unsafeExprAnn :: a + , _unsafeYieldWhitespace :: [Whitespace] + , _unsafeYieldValue :: CommaSep (Expr v a) + } + -- | @yield from a@ + -- + -- https://docs.python.org/3/reference/expressions.html#yield-expressions + | YieldFrom + { _unsafeExprAnn :: a + , _unsafeYieldWhitespace :: [Whitespace] + , _unsafeFromWhitespace :: [Whitespace] + , _unsafeYieldFromValue :: Expr v a + } + -- | @a if b else c@ + -- + -- https://docs.python.org/3/reference/expressions.html#conditional-expressions + | Ternary + { _unsafeExprAnn :: a + -- expr + , _unsafeTernaryValue :: Expr v a + -- 'if' spaces + , _unsafeTernaryWhitespaceIf :: [Whitespace] + -- expr + , _unsafeTernaryCond :: Expr v a + -- 'else' spaces + , _unsafeTernaryWhitespaceElse :: [Whitespace] + -- expr + , _unsafeTernaryElse :: Expr v a + } + -- | @[a for b in c if d]@ + -- + -- https://docs.python.org/3/reference/expressions.html#list-displays + | ListComp + { _unsafeExprAnn :: a + -- [ spaces + , _unsafeListCompWhitespaceLeft :: [Whitespace] + -- comprehension + , _unsafeListCompValue :: Comprehension Expr v a + -- ] spaces + , _unsafeListCompWhitespaceRight :: [Whitespace] + } + -- | @[a, b, c]@ + -- + -- https://docs.python.org/3/reference/expressions.html#list-displays + | List + { _unsafeExprAnn :: a + -- [ spaces + , _unsafeListWhitespaceLeft :: [Whitespace] + -- exprs + , _unsafeListValues :: Maybe (CommaSep1' (ListItem v a)) + -- ] spaces + , _unsafeListWhitespaceRight :: [Whitespace] + } + -- | @{a: b for c in d if e}@ + -- + -- https://docs.python.org/3/reference/expressions.html#dictionary-displays + | DictComp + { _unsafeExprAnn :: a + -- { spaces + , _unsafeDictCompWhitespaceLeft :: [Whitespace] + -- comprehension + , _unsafeDictCompValue :: Comprehension DictItem v a + -- } spaces + , _unsafeDictCompWhitespaceRight :: [Whitespace] + } + -- | @{}@ + -- + -- @{a: 1, b: 2, c: 3}@ + -- + -- https://docs.python.org/3/reference/expressions.html#dictionary-displays + | Dict + { _unsafeExprAnn :: a + , _unsafeDictWhitespaceLeft :: [Whitespace] + , _unsafeDictValues :: Maybe (CommaSep1' (DictItem v a)) + , _unsafeDictWhitespaceRight :: [Whitespace] + } + -- | @{a for b in c if d}@ + -- + -- https://docs.python.org/3/reference/expressions.html#set-displays + | SetComp + { _unsafeExprAnn :: a + -- { spaces + , _unsafeSetCompWhitespaceLeft :: [Whitespace] + -- comprehension + , _unsafeSetCompValue :: Comprehension SetItem v a + -- } spaces + , _unsafeSetCompWhitespaceRight :: [Whitespace] + } + -- | @{a, b, c}@ + -- + -- https://docs.python.org/3/reference/expressions.html#set-displays + | Set + { _unsafeExprAnn :: a + , _unsafeSetWhitespaceLeft :: [Whitespace] + , _unsafeSetValues :: CommaSep1' (SetItem v a) + , _unsafeSetWhitespaceRight :: [Whitespace] + } + -- | @a.b@ + -- + -- https://docs.python.org/3/reference/expressions.html#attribute-references + | Deref + { _unsafeExprAnn :: a + -- expr + , _unsafeDerefValueLeft :: Expr v a + -- . spaces + , _unsafeDerefWhitespaceLeft :: [Whitespace] + -- ident + , _unsafeDerefValueRight :: Ident v a + } + -- | @a[b]@ + -- + -- @a[:]@ + -- + -- @a[:, b:]@ + -- + -- etc. + -- + -- https://docs.python.org/3/reference/expressions.html#subscriptions + | Subscript + { _unsafeExprAnn :: a + -- expr + , _unsafeSubscriptValueLeft :: Expr v a + -- [ spaces + , _unsafeSubscriptWhitespaceLeft :: [Whitespace] + -- expr + , _unsafeSubscriptValueRight :: CommaSep1' (Subscript v a) + -- ] spaces + , _unsafeSubscriptWhitespaceRight :: [Whitespace] + } + -- | @f(x)@ + -- + -- https://docs.python.org/3/reference/expressions.html#calls + | Call + { _unsafeExprAnn :: a + -- expr + , _unsafeCallFunction :: Expr v a + -- ( spaces + , _unsafeCallWhitespaceLeft :: [Whitespace] + -- exprs + , _unsafeCallArguments :: Maybe (CommaSep1' (Arg v a)) + -- ) spaces + , _unsafeCallWhitespaceRight :: [Whitespace] + } + -- | @None@ + -- + -- https://docs.python.org/3/library/constants.html#None + | None + { _unsafeExprAnn :: a + , _unsafeNoneWhitespace :: [Whitespace] + } + -- | @...@ + -- + -- https://docs.python.org/3/library/constants.html#Ellipsis + | Ellipsis + { _unsafeExprAnn :: a + , _unsafeEllipsisWhitespace :: [Whitespace] + } + -- | @a + b@ + -- + -- https://docs.python.org/3/reference/expressions.html#the-power-operator + -- + -- https://docs.python.org/3/reference/expressions.html#binary-arithmetic-operations + -- + -- https://docs.python.org/3/reference/expressions.html#shifting-operations + -- + -- https://docs.python.org/3/reference/expressions.html#binary-bitwise-operations + -- + -- https://docs.python.org/3/reference/expressions.html#comparisons + -- + -- https://docs.python.org/3/reference/expressions.html#membership-test-operations + -- + -- https://docs.python.org/3/reference/expressions.html#is-not + -- + -- https://docs.python.org/3/reference/expressions.html#boolean-operations + | BinOp + { _unsafeExprAnn :: a + , _unsafeBinOpExprLeft :: Expr v a + , _unsafeBinOpOp :: BinOp a + , _unsafeBinOpExprRight :: Expr v a + } + -- | @-a@ + -- + -- @~a@ + -- + -- @+a@ + -- + -- https://docs.python.org/3/reference/expressions.html#unary-arithmetic-and-bitwise-operations + | UnOp + { _unsafeExprAnn :: a + , _unsafeUnOpOp :: UnOp a + , _unsafeUnOpValue :: Expr v a + } + | Parens + { _unsafeExprAnn :: a + -- ( spaces + , _unsafeParensWhitespaceLeft :: [Whitespace] + -- expr + , _unsafeParensValue :: Expr v a + -- ) spaces + , _unsafeParensWhitespaceAfter :: [Whitespace] + } + -- | @a@ + -- + -- https://docs.python.org/3/reference/expressions.html#atom-identifiers + | Ident + { _unsafeIdentValue :: Ident v a + } + -- | @1@ + -- + -- @0xF3A + -- + -- @0o177 + -- + -- @0b1011@ + -- + -- https://docs.python.org/3/reference/lexical_analysis.html#grammar-token-integer + | Int + { _unsafeExprAnn :: a + , _unsafeIntValue :: IntLiteral a + , _unsafeIntWhitespace :: [Whitespace] + } + -- | @1.@ + -- + -- @3.14@ + -- + -- @10e100@ + -- + -- https://docs.python.org/3/reference/lexical_analysis.html#floating-point-literals + | Float + { _unsafeExprAnn :: a + , _unsafeFloatValue :: FloatLiteral a + , _unsafeFloatWhitespace :: [Whitespace] + } + -- | @10j@ + -- + -- @5.j@ + -- + -- https://docs.python.org/3/reference/lexical_analysis.html#floating-point-literals + | Imag + { _unsafeExprAnn :: a + , _unsafeImagValue :: ImagLiteral a + , _unsafeImagWhitespace :: [Whitespace] + } + -- | @True@ + -- + -- @False@ + -- + -- https://docs.python.org/3/library/constants.html#True + -- + -- https://docs.python.org/3/library/constants.html#False + | Bool + { _unsafeExprAnn :: a + , _unsafeBoolValue :: Bool + , _unsafeBoolWhitespace :: [Whitespace] + } + -- | @\"asdf\"@ + -- + -- @b\"asdf\"@ + -- + -- @\"asdf\" \'asdf\'@ + -- + -- @\'\'\'asdf\'\'\'@ + -- + -- https://docs.python.org/3/reference/lexical_analysis.html#grammar-token-stringliteral + | String + { _unsafeExprAnn :: a + , _unsafeStringValue :: NonEmpty (StringLiteral a) + } + -- | @a, b, c@ + -- + -- @(a, b)@ + -- + -- @(a,)@ + -- + -- https://docs.python.org/3/reference/expressions.html#expression-lists + | Tuple + { _unsafeExprAnn :: a + -- expr + , _unsafeTupleHead :: TupleItem v a + -- , spaces + , _unsafeTupleWhitespace :: Comma + -- [exprs] + , _unsafeTupleTail :: Maybe (CommaSep1' (TupleItem v a)) + } + -- | @not a@ + -- + -- https://docs.python.org/3/reference/expressions.html#boolean-operations + | Not + { _unsafeExprAnn :: a + , _unsafeNotWhitespace :: [Whitespace] + , _unsafeNotValue :: Expr v a + } + -- | @(a for b in c)@ + -- + -- https://docs.python.org/3/reference/expressions.html#generator-expressions + | Generator + { _unsafeExprAnn :: a + , _generatorValue :: Comprehension Expr v a + } + -- | @await a@ + -- + -- https://docs.python.org/3/reference/expressions.html#await + | Await + { _unsafeExprAnn :: a + , _unsafeAwaitWhitespace :: [Whitespace] + , _unsafeAwaitValue :: Expr v a + } + deriving (Eq, Show, Functor, Foldable, Traversable, Generic) + +-- | Lens on the top-level annotation in an expression +exprAnn :: Lens' (Expr v a) a +exprAnn = + lens + (\case + Unit a _ _ -> a + Lambda a _ _ _ _ -> a + Yield a _ _ -> a + YieldFrom a _ _ _ -> a + Ternary a _ _ _ _ _ -> a + None a _ -> a + Ellipsis a _ -> a + List a _ _ _ -> a + ListComp a _ _ _ -> a + Deref a _ _ _ -> a + Subscript a _ _ _ _ -> a + Call a _ _ _ _ -> a + BinOp a _ _ _ -> a + UnOp a _ _ -> a + Parens a _ _ _ -> a + Ident a -> a ^. identAnn + Int a _ _ -> a + Float a _ _ -> a + Imag a _ _ -> a + Bool a _ _ -> a + String a _ -> a + Not a _ _ -> a + Tuple a _ _ _ -> a + DictComp a _ _ _ -> a + Dict a _ _ _ -> a + SetComp a _ _ _ -> a + Set a _ _ _ -> a + Generator a _ -> a + Await a _ _ -> a) + (\e ann -> + case e of + Unit _ a b -> Unit ann a b + Lambda _ a b c d -> Lambda ann a b c d + Yield _ a b -> Yield ann a b + YieldFrom ann a b c -> YieldFrom ann a b c + Ternary ann a b c d e -> Ternary ann a b c d e + None _ a -> None ann a + Ellipsis _ a -> Ellipsis ann a + List _ a b c -> List ann a b c + ListComp _ a b c -> ListComp ann a b c + Deref _ a b c -> Deref ann a b c + Subscript _ a b c d -> Subscript ann a b c d + Call _ a b c d -> Call ann a b c d + BinOp _ a b c -> BinOp ann a b c + UnOp _ a b -> UnOp ann a b + Parens _ a b c -> Parens ann a b c + Ident a -> Ident $ a & identAnn .~ ann + Int _ a b -> Int ann a b + Float _ a b -> Float ann a b + Imag _ a b -> Imag ann a b + Bool _ a b -> Bool ann a b + String _ a -> String ann a + Not _ a b -> Not ann a b + Tuple _ a b c -> Tuple ann a b c + DictComp _ a b c -> DictComp ann a b c + Dict _ a b c -> Dict ann a b c + SetComp _ a b c -> SetComp ann a b c + Set _ a b c -> Set ann a b c + Generator _ a -> Generator ann a + Await _ a b -> Not ann a b) + +instance HasTrailingWhitespace (Expr v a) where + trailingWhitespace = + lens + (\case + Unit _ _ a -> a + Lambda _ _ _ _ a -> a ^. trailingWhitespace + Yield _ ws CommaSepNone -> ws + Yield _ _ e -> e ^?! csTrailingWhitespace + YieldFrom _ _ _ e -> e ^. trailingWhitespace + Ternary _ _ _ _ _ e -> e ^. trailingWhitespace + None _ ws -> ws + Ellipsis _ ws -> ws + List _ _ _ ws -> ws + ListComp _ _ _ ws -> ws + Deref _ _ _ a -> a ^. trailingWhitespace + Subscript _ _ _ _ ws -> ws + Call _ _ _ _ ws -> ws + BinOp _ _ _ e -> e ^. trailingWhitespace + UnOp _ _ e -> e ^. trailingWhitespace + Parens _ _ _ ws -> ws + Ident a -> a ^. getting trailingWhitespace + Int _ _ ws -> ws + Float _ _ ws -> ws + Imag _ _ ws -> ws + Bool _ _ ws -> ws + String _ v -> v ^. trailingWhitespace + Not _ _ e -> e ^. trailingWhitespace + Tuple _ _ (MkComma ws) Nothing -> ws + Tuple _ _ _ (Just cs) -> cs ^. trailingWhitespace + DictComp _ _ _ ws -> ws + Dict _ _ _ ws -> ws + SetComp _ _ _ ws -> ws + Set _ _ _ ws -> ws + Generator _ a -> a ^. trailingWhitespace + Await _ _ e -> e ^. trailingWhitespace) + (\e ws -> + case e of + Unit a b _ -> Unit a b ws + Lambda a b c d f -> Lambda a b c d (f & trailingWhitespace .~ ws) + Yield a _ CommaSepNone -> Yield a ws CommaSepNone + Yield a b c -> Yield a b (c & csTrailingWhitespace .~ ws) + YieldFrom a b c d -> YieldFrom a b c (d & trailingWhitespace .~ ws) + Ternary a b c d e f -> Ternary a b c d e (f & trailingWhitespace .~ ws) + None a _ -> None a ws + Ellipsis a _ -> Ellipsis a ws + List a b c _ -> List a b (coerce c) ws + ListComp a b c _ -> ListComp a b (coerce c) ws + Deref a b c d -> Deref a (coerce b) c (d & trailingWhitespace .~ ws) + Subscript a b c d _ -> Subscript a (coerce b) c d ws + Call a b c d _ -> Call a (coerce b) c (coerce d) ws + BinOp a b c e -> BinOp a (coerce b) c (e & trailingWhitespace .~ ws) + UnOp a b c -> UnOp a b (c & trailingWhitespace .~ ws) + Parens a b c _ -> Parens a b (coerce c) ws + Ident a -> Ident $ a & trailingWhitespace .~ ws + Int a b _ -> Int a b ws + Float a b _ -> Float a b ws + Imag a b _ -> Imag a b ws + Bool a b _ -> Bool a b ws + String a v -> String a (v & trailingWhitespace .~ ws) + Not a b c -> Not a b (c & trailingWhitespace .~ ws) + Tuple a b _ Nothing -> Tuple a (coerce b) (MkComma ws) Nothing + Tuple a b c (Just cs) -> + Tuple a (coerce b) c (Just $ cs & trailingWhitespace .~ ws) + DictComp a b c _ -> DictComp a b c ws + Dict a b c _ -> Dict a b c ws + SetComp a b c _ -> SetComp a b c ws + Set a b c _ -> Set a b c ws + Generator a b -> Generator a $ b & trailingWhitespace .~ ws + Await a b c -> Await a b (c & trailingWhitespace .~ ws)) + +instance IsString (Expr '[] ()) where + fromString s = Ident $ MkIdent () s [] + +instance Num (Expr '[] ()) where + fromInteger n + | n >= 0 = Int () (IntLiteralDec () $ integralDecDigits n ^?! _Right) [] + | otherwise = + UnOp + () + (Negate () []) + (Int () (IntLiteralDec () $ integralDecDigits (-n) ^?! _Right) []) + + negate = UnOp () (Negate () []) + + (+) a = BinOp () (a & trailingWhitespace .~ [Space]) (Plus () [Space]) + (*) a = BinOp () (a & trailingWhitespace .~ [Space]) (Multiply () [Space]) + (-) a = BinOp () (a & trailingWhitespace .~ [Space]) (Minus () [Space]) + signum = undefined + abs = undefined + +instance Plated (Expr '[] a) where; plate = gplate + +instance HasExprs Expr where + _Exprs = id + +-- | +-- @shouldGroupLeft op left@ returns true if @left@ needs to be parenthesised +-- when it is the left argument of @op@ +shouldGroupLeft :: BinOp a -> Expr v a -> Bool +shouldGroupLeft op left = + let + entry = lookupOpEntry op operatorTable + + lEntry = + case left of + BinOp _ _ lOp _ -> Just $ lookupOpEntry lOp operatorTable + _ -> Nothing + + leftf = + case entry ^. opAssoc of + R | Just (OpEntry _ prec R) <- lEntry -> prec <= entry ^. opPrec + _ -> False + + leftf' = + case (left, op) of + (UnOp{}, Exp{}) -> True + (Tuple{}, _) -> True + (Not{}, BoolAnd{}) -> False + (Not{}, BoolOr{}) -> False + (Not{}, _) -> True + _ -> maybe False (\p -> p < entry ^. opPrec) (lEntry ^? _Just.opPrec) + in + leftf || leftf' + +-- | +-- @shouldGroupRight op right@ returns true if @right@ needs to be parenthesised +-- when it is the right argument of @op@ +shouldGroupRight :: BinOp a -> Expr v a -> Bool +shouldGroupRight op right = + let + entry = lookupOpEntry op operatorTable + + rEntry = + case right of + BinOp _ _ rOp _ -> Just $ lookupOpEntry rOp operatorTable + _ -> Nothing + + rightf = + case entry ^. opAssoc of + L | Just (OpEntry _ prec L) <- rEntry -> prec <= entry ^. opPrec + _ -> False + + rightf' = + case (op, right) of + (_, Tuple{}) -> True + (BoolAnd{}, Not{}) -> False + (BoolOr{}, Not{}) -> False + (_, Not{}) -> True + _ -> maybe False (\p -> p < entry ^. opPrec) (rEntry ^? _Just.opPrec) + in + rightf || rightf' diff --git a/src/Language/Python/Syntax/Ident.hs b/src/Language/Python/Syntax/Ident.hs new file mode 100644 --- /dev/null +++ b/src/Language/Python/Syntax/Ident.hs @@ -0,0 +1,76 @@ +{-# language DataKinds, KindSignatures #-} +{-# language FlexibleInstances #-} +{-# language DeriveFunctor, DeriveFoldable, DeriveTraversable #-} + +{-| +Module : Language.Python.Syntax.Ident +Copyright : (C) CSIRO 2017-2018 +License : BSD3 +Maintainer : Isaac Elliott +Stability : experimental +Portability : non-portable +-} + +module Language.Python.Syntax.Ident + ( Ident(..) + -- * Lenses + , identAnn + , identValue + , identWhitespace + -- * Extra functions + , isIdentifierStart + , isIdentifierChar + ) +where + +import Control.Lens.Lens (Lens, lens) +import Data.Char (isDigit, isLetter) +import Data.String (IsString(..)) + +import Language.Python.Optics.Validated (Validated) +import Language.Python.Syntax.Raw +import Language.Python.Syntax.Whitespace + +-- | An identifier. Like many types in hpython, it has an optional annotation +-- and tracks its trailing whitespace. +-- +-- 'Raw' 'Ident's have an 'IsString' instance. +-- +-- See +data Ident (v :: [*]) a + = MkIdent + { _identAnn :: a + , _identValue :: String + , _identWhitespace :: [Whitespace] + } deriving (Eq, Show, Functor, Foldable, Traversable) + +-- | Determine whether this character could start a valid identifier +isIdentifierStart :: Char -> Bool +isIdentifierStart = do + a <- isLetter + b <- (=='_') + pure $ a || b + +-- | Determine whether this character could be part of a valid identifier +isIdentifierChar :: Char -> Bool +isIdentifierChar = do + a <- isIdentifierStart + b <- isDigit + pure $ a || b + +instance IsString (Raw Ident) where + fromString s = MkIdent () s [] + +identValue :: Lens (Ident v a) (Ident '[] a) String String +identValue = lens _identValue (\s a -> s { _identValue = a }) + +identAnn :: Lens (Ident v a) (Ident v a) a a +identAnn = lens _identAnn (\s a -> s { _identAnn = a }) + +identWhitespace :: Lens (Ident v a) (Ident v a) [Whitespace] [Whitespace] +identWhitespace = lens _identWhitespace (\s ws -> s { _identWhitespace = ws }) + +instance HasTrailingWhitespace (Ident v a) where + trailingWhitespace = identWhitespace + +instance Validated Ident where diff --git a/src/Language/Python/Syntax/Import.hs b/src/Language/Python/Syntax/Import.hs new file mode 100644 --- /dev/null +++ b/src/Language/Python/Syntax/Import.hs @@ -0,0 +1,113 @@ +{-# language DeriveFunctor, DeriveFoldable, DeriveTraversable #-} +{-# language DataKinds #-} +{-# language LambdaCase #-} + +{-| +Module : Language.Python.Syntax.Import +Copyright : (C) CSIRO 2017-2018 +License : BSD3 +Maintainer : Isaac Elliott +Stability : experimental +Portability : non-portable + +Syntax used in import statements + +https://docs.python.org/3.5/reference/simple_stmts.html#the-import-statement +-} + +module Language.Python.Syntax.Import + ( ImportAs(..) + , ImportTargets(..) + -- * Lenses + , importAsAnn + , importAsName + , importAsQual + ) +where + +import Control.Lens.Getter ((^.), getting) +import Control.Lens.Lens (Lens, Lens', lens) +import Control.Lens.Prism (_Just) +import Control.Lens.Setter ((.~)) +import Control.Lens.Tuple (_2) +import Data.Function ((&)) +import Data.List.NonEmpty (NonEmpty) + +import Language.Python.Optics.Validated +import Language.Python.Syntax.CommaSep +import Language.Python.Syntax.Ident +import Language.Python.Syntax.Whitespace + +-- | Some data optionally followed by @as @ +-- +-- Used in: +-- +-- @import a as b@ +-- +-- @from a import b as c, d as e@ +-- +-- @from a import (b as c, d as e)@ +data ImportAs e v a + = ImportAs + { _importAsAnn :: a + , _importAsName :: e a + , _importAsQual :: Maybe (NonEmpty Whitespace, Ident v a) + } + deriving (Eq, Show, Functor, Foldable, Traversable) + +instance Validated (ImportAs e) + +importAsAnn :: Lens' (ImportAs e v a) a +importAsAnn = lens _importAsAnn (\s a -> s { _importAsAnn = a }) + +importAsName :: Lens (ImportAs e v a) (ImportAs e' '[] a) (e a) (e' a) +importAsName = lens _importAsName (\s a -> (s ^. unvalidated) { _importAsName = a }) + +importAsQual + :: Lens + (ImportAs e v a) + (ImportAs e '[] a) + (Maybe (NonEmpty Whitespace, Ident v a)) + (Maybe (NonEmpty Whitespace, Ident '[] a)) +importAsQual = lens _importAsQual (\s a -> (s ^. unvalidated) { _importAsQual = a }) + +instance HasTrailingWhitespace (e a) => HasTrailingWhitespace (ImportAs e v a) where + trailingWhitespace = + lens + (\(ImportAs _ a b) -> + maybe (a ^. getting trailingWhitespace) (^. _2.trailingWhitespace) b) + (\(ImportAs x a b) ws -> + ImportAs + x + (maybe (a & trailingWhitespace .~ ws) (const a) b) + (b & _Just._2.trailingWhitespace .~ ws)) + +-- | The targets of a @from ... import ...@ statement +data ImportTargets v a + -- | @from x import *@ + = ImportAll a [Whitespace] + -- | @from x import a, b, c@ + | ImportSome a (CommaSep1 (ImportAs (Ident v) v a)) + -- | @from x import (a, b, c)@ + | ImportSomeParens + a + -- ( spaces + [Whitespace] + -- imports as + (CommaSep1' (ImportAs (Ident v) v a)) + -- ) spaces + [Whitespace] + deriving (Eq, Show, Functor, Foldable, Traversable) + +instance HasTrailingWhitespace (ImportTargets v a) where + trailingWhitespace = + lens + (\case + ImportAll _ ws -> ws + ImportSome _ cs -> cs ^. trailingWhitespace + ImportSomeParens _ _ _ ws -> ws) + (\ts ws -> + case ts of + ImportAll a _ -> ImportAll a ws + ImportSome a cs -> ImportSome a (cs & trailingWhitespace .~ ws) + ImportSomeParens x a b _ -> ImportSomeParens x a b ws) diff --git a/src/Language/Python/Syntax/Module.hs b/src/Language/Python/Syntax/Module.hs new file mode 100644 --- /dev/null +++ b/src/Language/Python/Syntax/Module.hs @@ -0,0 +1,35 @@ +{-# language DeriveFunctor, DeriveFoldable, DeriveTraversable #-} + +{-| +Module : Language.Python.Syntax.Module +Copyright : (C) CSIRO 2017-2018 +License : BSD3 +Maintainer : Isaac Elliott +Stability : experimental +Portability : non-portable +-} + +module Language.Python.Syntax.Module + ( Module (ModuleEmpty, ModuleBlankFinal, ModuleBlank, ModuleStatement) + ) +where + +import Language.Python.Syntax.Statement +import Language.Python.Syntax.Whitespace + +-- | A Python 'Module', which is stored as a sequence of statements. +-- A module corresponds to one source file of Python code. +data Module v a + = ModuleEmpty + | ModuleBlankFinal (Blank a) + | ModuleBlank (Blank a) Newline (Module v a) + | ModuleStatement (Statement v a) (Module v a) + deriving (Eq, Show, Functor, Foldable, Traversable) + +instance HasStatements Module where + _Statements f = go + where + go ModuleEmpty = pure ModuleEmpty + go (ModuleBlankFinal a) = pure $ ModuleBlankFinal a + go (ModuleBlank a b c) = ModuleBlank a b <$> go c + go (ModuleStatement a b) = ModuleStatement <$> f a <*> go b diff --git a/src/Language/Python/Syntax/ModuleNames.hs b/src/Language/Python/Syntax/ModuleNames.hs new file mode 100644 --- /dev/null +++ b/src/Language/Python/Syntax/ModuleNames.hs @@ -0,0 +1,97 @@ +{-# language DeriveFunctor, DeriveFoldable, DeriveTraversable #-} +{-# language DataKinds, FlexibleInstances, MultiParamTypeClasses #-} +{-# language LambdaCase #-} + +{-| +Module : Language.Python.Syntax.ModuleNames +Copyright : (C) CSIRO 2017-2018 +License : BSD3 +Maintainer : Isaac Elliott +Stability : experimental +Portability : non-portable + +Module names, including those qualified by packages. + +See +-} + +module Language.Python.Syntax.ModuleNames + ( ModuleName (..) + , RelativeModuleName (..) + , makeModuleName + , _moduleNameAnn + ) +where + +import Control.Lens.Cons (_last) +import Control.Lens.Fold ((^?!)) +import Control.Lens.Getter ((^.)) +import Control.Lens.Lens (lens) +import Control.Lens.Setter ((.~)) +import Data.Coerce (coerce) +import Data.Function ((&)) +import Data.List.NonEmpty (NonEmpty(..)) +import qualified Data.List.NonEmpty as NonEmpty + +import Language.Python.Syntax.Ident +import Language.Python.Syntax.Punctuation +import Language.Python.Syntax.Whitespace + +-- | @.a.b@ +-- +-- @.@ +-- +-- @...@ +-- +--See +data RelativeModuleName v a + = RelativeWithName [Dot] (ModuleName v a) + | Relative (NonEmpty Dot) + deriving (Eq, Show, Functor, Foldable, Traversable) + +instance HasTrailingWhitespace (RelativeModuleName v a) where + trailingWhitespace = + lens + (\case + RelativeWithName _ mn -> mn ^. trailingWhitespace + Relative (a :| as) -> (a : as) ^?! _last.trailingWhitespace) + (\a ws -> case a of + RelativeWithName x mn -> RelativeWithName x (mn & trailingWhitespace .~ ws) + Relative (a :| as) -> + Relative . + NonEmpty.fromList $ + (a : as) & _last.trailingWhitespace .~ ws) + +-- | A module name. It can be a single segment, or a sequence of them which +-- are implicitly separated by period character. +-- +-- @a@ +-- +-- @a.b@ +data ModuleName v a + = ModuleNameOne a (Ident v a) + | ModuleNameMany a (Ident v a) Dot (ModuleName v a) + deriving (Eq, Show, Functor, Foldable, Traversable) + +-- | Get the annotation from a 'ModuleName' +_moduleNameAnn :: ModuleName v a -> a +_moduleNameAnn (ModuleNameOne a _) = a +_moduleNameAnn (ModuleNameMany a _ _ _) = a + +-- | Convenience constructor for 'ModuleName' +makeModuleName :: Ident v a -> [([Whitespace], Ident v a)] -> ModuleName v a +makeModuleName i [] = ModuleNameOne (_identAnn i) i +makeModuleName i ((a, b) : as) = + ModuleNameMany (_identAnn i) i (MkDot a) $ + makeModuleName b as + +instance HasTrailingWhitespace (ModuleName v a) where + trailingWhitespace = + lens + (\case + ModuleNameOne _ i -> i ^. trailingWhitespace + ModuleNameMany _ _ _ mn -> mn ^. trailingWhitespace) + (\mn ws -> case mn of + ModuleNameOne a b -> ModuleNameOne a (b & trailingWhitespace .~ ws) + ModuleNameMany a b d mn -> + ModuleNameMany a (coerce b) d (mn & trailingWhitespace .~ ws)) diff --git a/src/Language/Python/Syntax/Numbers.hs b/src/Language/Python/Syntax/Numbers.hs new file mode 100644 --- /dev/null +++ b/src/Language/Python/Syntax/Numbers.hs @@ -0,0 +1,224 @@ +{-# language DeriveFunctor, DeriveFoldable, DeriveTraversable #-} +{-# language LambdaCase #-} +{-# language TemplateHaskell #-} + +{-| +Module : Language.Python.Syntax.Numbers +Copyright : (C) CSIRO 2017-2018 +License : BSD3 +Maintainer : Isaac Elliott +Stability : experimental +Portability : non-portable + +Numerical literal values in Python +-} + +module Language.Python.Syntax.Numbers + ( -- * Datatypes + IntLiteral(..) + , Sign(..) + , E(..) + , FloatExponent(..) + , FloatLiteral(..) + , ImagLiteral(..) + -- * Rendering + -- | The output of these functions is guaranteed to be valid Python code + , showIntLiteral + , showFloatLiteral + , showFloatExponent + , showImagLiteral + ) +where + +import Control.Lens.Review ((#)) +import Data.Deriving (deriveEq1, deriveOrd1) +import Data.Digit.Binary (BinDigit) +import Data.Digit.Char (charHeXaDeCiMaL, charOctal, charBinary, charDecimal) +import Data.Digit.Octal (OctDigit) +import Data.Digit.Decimal (DecDigit) +import Data.Digit.Hexadecimal.MixedCase (HeXDigit) +import Data.List.NonEmpty (NonEmpty) +import Data.Semigroup ((<>)) +import Data.Text (Text) +import Data.These (These(..)) + +import qualified Data.List.NonEmpty as NonEmpty +import qualified Data.Text as Text + +-- | An integer literal value. +-- +-- @5@ is an integer literal. +-- +-- @6.2@ is a literal but is not an integer +-- +-- @x@ might be an integer, but is not a literal +-- +-- See +data IntLiteral a + -- | Decimal + -- + -- @1234@ + = IntLiteralDec + { _intLiteralAnn :: a + , _unsafeIntLiteralDecValue :: NonEmpty DecDigit + } + -- | Binary + -- + -- @0b10110@ + | IntLiteralBin + { _intLiteralAnn :: a + , _unsafeIntLiteralBinUppercase :: Bool + , _unsafeIntLiteralBinValue :: NonEmpty BinDigit + } + -- | Octal + -- + -- @0o1367@ + | IntLiteralOct + { _intLiteralAnn :: a + , _unsafeIntLiteralOctUppercase :: Bool + , _unsafeIntLiteralOctValue :: NonEmpty OctDigit + } + -- | Mixed-case hexadecimal + -- + -- @0x18B4f@ + | IntLiteralHex + { _intLiteralAnn :: a + , _unsafeIntLiteralHexUppercase :: Bool + , _unsafeIntLiteralHexValue :: NonEmpty HeXDigit + } + deriving (Eq, Show, Functor, Foldable, Traversable) +deriveEq1 ''IntLiteral +deriveOrd1 ''IntLiteral + +-- | Positive or negative, as in @-7@ +data Sign = Pos | Neg deriving (Eq, Ord, Show) + +-- | When a floating point literal is in scientific notation, it includes the character +-- @e@, which can be lower or upper case. +data E = Ee | EE deriving (Eq, Ord, Show) + +-- | The exponent of a floating point literal. +-- +-- An @e@, followed by an optional 'Sign', followed by at least one digit. +data FloatExponent = FloatExponent E (Maybe Sign) (NonEmpty DecDigit) + deriving (Eq, Ord, Show) + +-- | A literal floating point value. +-- +-- Eg. @7.63@ +-- +-- See +data FloatLiteral a + -- | \'Complete\' floats + -- + -- @12.@ + -- + -- @12.34@ + -- + -- @12.e34@ + -- + -- @12.34e56@ + = FloatLiteralFull + { _floatLiteralAnn :: a + , _floatLiteralFullLeft :: NonEmpty DecDigit + , _floatLiteralFullRight + :: Maybe (These (NonEmpty DecDigit) FloatExponent) + } + -- | Floats that begin with a decimal point + -- + -- @.12@ + -- + -- @.12e34@ + | FloatLiteralPoint + { _floatLiteralAnn :: a + -- . [0-9]+ + , _floatLiteralPointRight :: NonEmpty DecDigit + -- [ 'e' ['-' | '+'] [0-9]+ ] + , _floatLiteralPointExponent :: Maybe FloatExponent + } + -- | Floats with no decimal points + -- + -- @12e34@ + | FloatLiteralWhole + { _floatLiteralAnn :: a + -- [0-9]+ + , _floatLiteralWholeRight :: NonEmpty DecDigit + -- [ 'e' ['-' | '+'] [0-9]+ ] + , _floatLiteralWholeExponent :: FloatExponent + } + deriving (Eq, Show, Functor, Foldable, Traversable) +deriveEq1 ''FloatLiteral +deriveOrd1 ''FloatLiteral + +-- | Imaginary number literals +-- +-- See +data ImagLiteral a + -- | A decimal integer followed by a \'j\' + -- + -- @12j@ + = ImagLiteralInt + { _imagLiteralAnn :: a + , _unsafeImagLiteralIntValue :: NonEmpty DecDigit + , _imagLiteralUppercase :: Bool + } + -- | A float followed by a \'j\' + -- + -- @12.j@ + -- + -- @12.3j@ + -- + -- @.3j@ + | ImagLiteralFloat + { _imagLiteralAnn :: a + , _unsafeImagLiteralFloatValue :: FloatLiteral a + , _imagLiteralUppercase :: Bool + } + deriving (Eq, Show, Functor, Foldable, Traversable) +deriveEq1 ''ImagLiteral +deriveOrd1 ''ImagLiteral + +showIntLiteral :: IntLiteral a -> Text +showIntLiteral (IntLiteralDec _ n) = + Text.pack $ + (charDecimal #) <$> NonEmpty.toList n +showIntLiteral (IntLiteralBin _ b n) = + Text.pack $ + '0' : (if b then 'B' else 'b') : fmap (charBinary #) (NonEmpty.toList n) +showIntLiteral (IntLiteralOct _ b n) = + Text.pack $ + '0' : (if b then 'O' else 'o') : fmap (charOctal #) (NonEmpty.toList n) +showIntLiteral (IntLiteralHex _ b n) = + Text.pack $ + '0' : (if b then 'X' else 'x') : fmap (charHeXaDeCiMaL #) (NonEmpty.toList n) + +showFloatExponent :: FloatExponent -> Text +showFloatExponent (FloatExponent e s ds) = + Text.pack $ + (case e of; EE -> 'E'; Ee -> 'e') : + foldMap (\case; Pos -> "+"; Neg -> "-") s <> + fmap (charDecimal #) (NonEmpty.toList ds) + +showFloatLiteral :: FloatLiteral a -> Text +showFloatLiteral (FloatLiteralFull _ a b) = + Text.pack (fmap (charDecimal #) (NonEmpty.toList a) <> ".") <> + foldMap + (\case + This x -> Text.pack $ fmap (charDecimal #) (NonEmpty.toList x) + That x -> showFloatExponent x + These x y -> + Text.pack (fmap (charDecimal #) (NonEmpty.toList x)) <> + showFloatExponent y) + b +showFloatLiteral (FloatLiteralPoint _ a b) = + Text.pack ('.' : fmap (charDecimal #) (NonEmpty.toList a)) <> + foldMap showFloatExponent b +showFloatLiteral (FloatLiteralWhole _ a b) = + Text.pack (fmap (charDecimal #) (NonEmpty.toList a)) <> + showFloatExponent b + +showImagLiteral :: ImagLiteral a -> Text +showImagLiteral (ImagLiteralInt _ ds b) = + Text.pack $ fmap (charDecimal #) (NonEmpty.toList ds) ++ [if b then 'J' else 'j'] +showImagLiteral (ImagLiteralFloat _ f b) = + showFloatLiteral f <> Text.singleton (if b then 'J' else 'j') diff --git a/src/Language/Python/Syntax/Operator/Binary.hs b/src/Language/Python/Syntax/Operator/Binary.hs new file mode 100644 --- /dev/null +++ b/src/Language/Python/Syntax/Operator/Binary.hs @@ -0,0 +1,254 @@ +{-# language LambdaCase #-} +{-# language MultiParamTypeClasses, FlexibleInstances #-} +{-# language DeriveFunctor, DeriveFoldable, DeriveTraversable #-} +{-# language TemplateHaskell #-} + +{-| +Module : Language.Python.Syntax.Operator.Binary +Copyright : (C) CSIRO 2017-2018 +License : BSD3 +Maintainer : Isaac Elliott +Stability : experimental +Portability : non-portable + +This module contains a datatype for binary operators and a precedence table +with associated operations. This presentation of operators is simpler and more +flexible than hard-coding them into the syntax tree. +-} + +module Language.Python.Syntax.Operator.Binary where + +import Control.Lens.Getter ((^.)) +import Control.Lens.Lens (lens) +import Control.Lens.TH (makeLenses) +import Data.Functor (($>)) +import Data.Semigroup ((<>)) + +import Language.Python.Syntax.Whitespace + +-- | A Python binary operator, such as @+@, along with its trailing 'Whitespace' +-- +-- The type variable allows annotations, but it can simply be made @()@ for an unannotated @BinOp@. +data BinOp a + -- | @a is b@ + = Is a [Whitespace] + -- | @a is not b@ + | IsNot a [Whitespace] [Whitespace] + -- | @a in b@ + | In a [Whitespace] + -- | @a not in b@ + | NotIn a [Whitespace] [Whitespace] + -- | @a - b@ + | Minus a [Whitespace] + -- | @a ** b@ + | Exp a [Whitespace] + -- | @a and b@ + | BoolAnd a [Whitespace] + -- | @a or b@ + | BoolOr a [Whitespace] + -- | @a == b@ + | Eq a [Whitespace] + -- | @a < b@ + | Lt a [Whitespace] + -- | @a <= b@ + | LtEq a [Whitespace] + -- | @a > b@ + | Gt a [Whitespace] + -- | @a >= b@ + | GtEq a [Whitespace] + -- | @a != b@ + | NotEq a [Whitespace] + -- | @a * b@ + | Multiply a [Whitespace] + -- | @a / b@ + | Divide a [Whitespace] + -- | @a // b@ + | FloorDivide a [Whitespace] + -- | @a % b@ + | Percent a [Whitespace] + -- | @a + b@ + | Plus a [Whitespace] + -- | @a | b@ + | BitOr a [Whitespace] + -- | @a ^ b@ + | BitXor a [Whitespace] + -- | @a & b@ + | BitAnd a [Whitespace] + -- | @a << b@ + | ShiftLeft a [Whitespace] + -- | @a >> b@ + | ShiftRight a [Whitespace] + -- | @a @ b@ + | At a [Whitespace] + deriving (Eq, Show, Functor, Foldable, Traversable) + +instance HasTrailingWhitespace (BinOp a) where + trailingWhitespace = + lens + (\case + Is _ a -> a + IsNot _ _ a -> a + In _ a -> a + NotIn _ _ a -> a + Minus _ a -> a + Exp _ a -> a + BoolAnd _ a -> a + BoolOr _ a -> a + Multiply _ a -> a + Divide _ a -> a + FloorDivide _ a -> a + Plus _ a -> a + Eq _ a -> a + Lt _ a -> a + LtEq _ a -> a + Gt _ a -> a + GtEq _ a -> a + NotEq _ a -> a + BitOr _ a -> a + BitXor _ a -> a + BitAnd _ a -> a + ShiftLeft _ a -> a + ShiftRight _ a -> a + Percent _ a -> a + At _ a -> a) + (\op ws -> + case op of + Is a _ -> Is a ws + IsNot a b _ -> IsNot a b ws + In a _ -> In a ws + NotIn a b _ -> NotIn a b ws + Minus a _ -> Minus a ws + Exp a _ -> Exp a ws + BoolAnd a _ -> BoolAnd a ws + BoolOr a _ -> BoolOr a ws + Multiply a _ -> Multiply a ws + Divide a _ -> Divide a ws + FloorDivide a _ -> FloorDivide a ws + Plus a _ -> Plus a ws + Eq a _ -> Eq a ws + Lt a _ -> Lt a ws + LtEq a _ -> LtEq a ws + Gt a _ -> Gt a ws + GtEq a _ -> GtEq a ws + NotEq a _ -> NotEq a ws + BitOr a _ -> BitOr a ws + BitAnd a _ -> BitAnd a ws + BitXor a _ -> BitXor a ws + ShiftLeft a _ -> ShiftLeft a ws + ShiftRight a _ -> ShiftRight a ws + Percent a _ -> Eq a ws + At a _ -> At a ws) + +-- | The associativity of an operator. Each operator is either left-associative or right associative. +-- +-- Left associative: +-- +-- @ +-- x + y + z = (x + y) + z +-- @ +-- +-- Right associative: +-- +-- @ +-- x + y + z = x + (y + z) +-- @ +data Assoc = L | R deriving (Eq, Show) + +-- | An operator along with its precedence and associativity. +data OpEntry + = OpEntry + { _opOperator :: BinOp () + , _opPrec :: Int + , _opAssoc :: Assoc + } +makeLenses ''OpEntry + +-- | 'operatorTable' is a list of all operators in ascending order of precedence. +operatorTable :: [OpEntry] +operatorTable = + [ entry BoolOr 4 L + , entry BoolAnd 5 L + , entry Is 10 L + , entry1 IsNot 10 L + , entry In 10 L + , entry1 NotIn 10 L + , entry Eq 10 L + , entry Lt 10 L + , entry LtEq 10 L + , entry Gt 10 L + , entry GtEq 10 L + , entry NotEq 10 L + , entry BitOr 14 L + , entry BitXor 15 L + , entry BitAnd 16 L + , entry ShiftLeft 17 L + , entry ShiftRight 17 L + , entry Minus 20 L + , entry Plus 20 L + , entry Multiply 25 L + , entry At 25 L + , entry Divide 25 L + , entry FloorDivide 25 L + , entry Percent 25 L + , entry Exp 30 R + ] + where + entry a = OpEntry (a () []) + entry1 a = OpEntry (a () [] []) + +-- | Compare two 'BinOp's to determine whether they represent the same operator, ignoring annotations and trailing whitespace. +sameOperator :: BinOp a -> BinOp a' -> Bool +sameOperator op op' = + case (op, op') of + (BoolOr{}, BoolOr{}) -> True + (BoolAnd{}, BoolAnd{}) -> True + (Is{}, Is{}) -> True + (IsNot{}, IsNot{}) -> True + (In{}, In{}) -> True + (NotIn{}, NotIn{}) -> True + (Eq{}, Eq{}) -> True + (Lt{}, Lt{}) -> True + (LtEq{}, LtEq{}) -> True + (Gt{}, Gt{}) -> True + (GtEq{}, GtEq{}) -> True + (NotEq{}, NotEq{}) -> True + (Minus{}, Minus{}) -> True + (Plus{}, Plus{}) -> True + (Multiply{}, Multiply{}) -> True + (Divide{}, Divide{}) -> True + (FloorDivide{}, FloorDivide{}) -> True + (Exp{}, Exp{}) -> True + (Percent{}, Percent{}) -> True + (BitOr{}, BitOr{}) -> True + (BitXor{}, BitXor{}) -> True + (BitAnd{}, BitAnd{}) -> True + (ShiftLeft{}, ShiftLeft{}) -> True + (ShiftRight{}, ShiftRight{}) -> True + (At{}, At{}) -> True + _ -> False + +-- | Is a 'BinOp' a comparison, such as @<=@ +isComparison :: BinOp a -> Bool +isComparison a = + case a of + Is{} -> True + IsNot{} -> True + In{} -> True + NotIn{} -> True + Eq{} -> True + Lt{} -> True + LtEq{} -> True + Gt{} -> True + GtEq{} -> True + NotEq{} -> True + _ -> False + +-- | Retrieve the information for a given operator from the operator table. +lookupOpEntry :: BinOp a -> [OpEntry] -> OpEntry +lookupOpEntry op = + go (op $> ()) + where + go op [] = error $ show op <> " not found in operator table" + go op (x:xs) + | sameOperator (x ^. opOperator) op = x + | otherwise = go op xs diff --git a/src/Language/Python/Syntax/Operator/Unary.hs b/src/Language/Python/Syntax/Operator/Unary.hs new file mode 100644 --- /dev/null +++ b/src/Language/Python/Syntax/Operator/Unary.hs @@ -0,0 +1,42 @@ +{-# language DeriveFunctor, DeriveFoldable, DeriveTraversable #-} +{-# language LambdaCase #-} + +{-| +Module : Language.Python.Syntax.Operator.Unary +Copyright : (C) CSIRO 2017-2018 +License : BSD3 +Maintainer : Isaac Elliott +Stability : experimental +Portability : non-portable + +Unary operators +-} + +module Language.Python.Syntax.Operator.Unary where + +import Control.Lens.Lens (lens) +import Language.Python.Syntax.Whitespace + +-- | An 'UnOp' is a unary operator in Python, such as @-@ for negation. +-- An operator is stored with an annotation and its trailing whitespace. +data UnOp a + -- | @-a@ + = Negate a [Whitespace] + -- | @+a@ + | Positive a [Whitespace] + -- | @~a@ + | Complement a [Whitespace] + deriving (Eq, Show, Functor, Foldable, Traversable) + +instance HasTrailingWhitespace (UnOp a) where + trailingWhitespace = + lens + (\case + Negate _ a -> a + Positive _ a -> a + Complement _ a -> a) + (\op ws -> + case op of + Negate a _ -> Negate a ws + Positive a _ -> Positive a ws + Complement a _ -> Complement a ws) diff --git a/src/Language/Python/Syntax/Punctuation.hs b/src/Language/Python/Syntax/Punctuation.hs new file mode 100644 --- /dev/null +++ b/src/Language/Python/Syntax/Punctuation.hs @@ -0,0 +1,63 @@ +{-# language DeriveFunctor, DeriveFoldable, DeriveTraversable #-} +{-| +Module : Language.Python.Syntax.Punctuation +Copyright : (C) CSIRO 2017-2018 +License : BSD3 +Maintainer : Isaac Elliott +Stability : experimental +Portability : non-portable + +These types are used throughout the syntax tree to help preserve formatting. +-} + +module Language.Python.Syntax.Punctuation where + +import Control.Lens.Lens (lens) + +import Language.Python.Syntax.Whitespace + +-- | A period character, possibly followed by some whitespace. +data Dot = MkDot [Whitespace] + deriving (Eq, Show) + +instance HasTrailingWhitespace Dot where + trailingWhitespace = + lens (\(MkDot ws) -> ws) (\_ ws -> MkDot ws) + +-- | The venerable comma separator +newtype Comma = MkComma [Whitespace] + deriving (Eq, Show) + +instance HasTrailingWhitespace Comma where + trailingWhitespace = + lens (\(MkComma ws) -> ws) (\_ ws -> MkComma ws) + +newtype Colon = MkColon [Whitespace] + deriving (Eq, Show) + +instance HasTrailingWhitespace Colon where + trailingWhitespace = + lens (\(MkColon ws) -> ws) (\_ ws -> MkColon ws) + +data Semicolon a = MkSemicolon a [Whitespace] + deriving (Eq, Show, Functor, Foldable, Traversable) + +instance HasTrailingWhitespace (Semicolon a) where + trailingWhitespace = + lens (\(MkSemicolon _ ws) -> ws) (\(MkSemicolon a _) ws -> MkSemicolon a ws) + +newtype Equals + = MkEquals [Whitespace] + deriving (Eq, Show) + +instance HasTrailingWhitespace Equals where + trailingWhitespace = + lens (\(MkEquals ws) -> ws) (\_ ws -> MkEquals ws) + +newtype At + = MkAt [Whitespace] + deriving (Eq, Show) + +instance HasTrailingWhitespace At where + trailingWhitespace = + lens (\(MkAt ws) -> ws) (\_ ws -> MkAt ws) diff --git a/src/Language/Python/Syntax/Raw.hs b/src/Language/Python/Syntax/Raw.hs new file mode 100644 --- /dev/null +++ b/src/Language/Python/Syntax/Raw.hs @@ -0,0 +1,5 @@ +{-# language DataKinds #-} +module Language.Python.Syntax.Raw where + +-- | 'Raw' represents unvalidated, un-annotated terms. +type Raw f = f '[] () diff --git a/src/Language/Python/Syntax/Statement.hs b/src/Language/Python/Syntax/Statement.hs new file mode 100644 --- /dev/null +++ b/src/Language/Python/Syntax/Statement.hs @@ -0,0 +1,702 @@ +{-# language TemplateHaskell #-} +{-# language DataKinds, KindSignatures #-} +{-# language MultiParamTypeClasses, FlexibleInstances #-} +{-# language DeriveFunctor, DeriveFoldable, DeriveTraversable, DeriveGeneric #-} +{-# language TypeFamilies #-} +{-# language LambdaCase #-} +{-# language UndecidableInstances #-} + +{-| +Module : Language.Python.Syntax.Statement +Copyright : (C) CSIRO 2017-2018 +License : BSD3 +Maintainer : Isaac Elliott +Stability : experimental +Portability : non-portable +-} + +module Language.Python.Syntax.Statement + ( -- * Statements + Statement(..) + -- ** Traversals + , HasStatements(..) + -- * Decorators + , Decorator(..) + -- ** Compound statements + , CompoundStatement(..) + -- ** Small statements + , SmallStatement(..) + -- ** Simple statements + , SimpleStatement(..) + -- *** @with ... as ...@ + , WithItem(..) + -- **** Lenses + , withItemAnn + , withItemValue + , withItemBinder + -- *** @except ... as ...@ + , ExceptAs (..) + -- **** Lenses + , exceptAsAnn + , exceptAsExpr + , exceptAsName + -- * Suites + , Suite(..) + -- * Blocks + , Block(..) + -- ** Lenses + , blockBlankLines + , blockHead + , blockTail + -- ** Traversals + , HasBlocks(..) + ) +where + +import Control.Lens.Cons (_last) +import Control.Lens.Fold (foldMapOf, folded) +import Control.Lens.Getter ((^.), to, view) +import Control.Lens.Plated (Plated(..), gplate) +import Control.Lens.Prism (_Right) +import Control.Lens.Setter ((.~), over, mapped) +import Control.Lens.TH (makeLenses) +import Control.Lens.Traversal (Traversal, traverseOf) +import Control.Lens.Tuple (_1, _2, _3, _4) +import Data.Bifoldable (bifoldMap) +import Data.Bifunctor (bimap) +import Data.Bitraversable (bitraverse) +import Data.Coerce (coerce) +import Data.List.NonEmpty (NonEmpty) +import Data.Maybe (isNothing) +import Data.Monoid ((<>)) +import GHC.Generics (Generic) +import Unsafe.Coerce (unsafeCoerce) + +import qualified Data.List.NonEmpty as NonEmpty + +import Language.Python.Optics.Validated +import Language.Python.Syntax.AugAssign +import Language.Python.Syntax.CommaSep +import Language.Python.Syntax.Comment +import Language.Python.Syntax.Expr +import Language.Python.Syntax.Ident +import Language.Python.Syntax.Import +import Language.Python.Syntax.ModuleNames +import Language.Python.Syntax.Punctuation +import Language.Python.Syntax.Whitespace + +-- See note [unsafeCoerce Validation] in Language.Python.Internal.Syntax.Expr +instance Validated Statement where; unvalidated = to unsafeCoerce +instance Validated SmallStatement where; unvalidated = to unsafeCoerce +instance Validated SimpleStatement where; unvalidated = to unsafeCoerce +instance Validated CompoundStatement where; unvalidated = to unsafeCoerce +instance Validated Block where; unvalidated = to unsafeCoerce +instance Validated Suite where; unvalidated = to unsafeCoerce +instance Validated WithItem where; unvalidated = to unsafeCoerce +instance Validated ExceptAs where; unvalidated = to unsafeCoerce +instance Validated Decorator where; unvalidated = to unsafeCoerce + +-- | 'Traversal' over all the 'Statement's in a term +class HasStatements s where + _Statements :: Traversal (s v a) (s '[] a) (Statement v a) (Statement '[] a) + +-- | A 'Block' is an indented multi-line chunk of code, forming part of a +-- 'Suite'. +data Block (v :: [*]) a + = Block + { _blockBlankLines :: [(Blank a, Newline)] -- ^ Blank lines at the beginning of the block + , _blockHead :: Statement v a -- ^ The first statement of the block + , _blockTail :: [Either (Blank a, Newline) (Statement v a)] -- ^ The remaining items of the block, which may be statements or blank lines + } deriving (Eq, Show) + +instance Functor (Block v) where + fmap f (Block a b c) = + Block + (over (mapped._1.mapped) f a) + (fmap f b) + (bimap (over (_1.mapped) f) (fmap f) <$> c) + +instance Foldable (Block v) where + foldMap f (Block a b c) = + foldMapOf (folded._1.folded) f a <> + foldMap f b <> + foldMap (bifoldMap (foldMapOf (_1.folded) f) (foldMap f)) c + +instance Traversable (Block v) where + traverse f (Block a b c) = + Block <$> + traverseOf (traverse._1.traverse) f a <*> + traverse f b <*> + traverse (bitraverse (traverseOf (_1.traverse) f) (traverse f)) c + +class HasBlocks s where + -- | 'Traversal' targeting all the 'Block's in a structure + _Blocks :: Traversal (s v a) (s '[] a) (Block v a) (Block '[] a) + +instance HasBlocks Suite where + _Blocks _ (SuiteOne a b c) = pure $ SuiteOne a b (c ^. unvalidated) + _Blocks f (SuiteMany a b c d e) = SuiteMany a b c d <$> f e + +instance HasBlocks CompoundStatement where + _Blocks f (Fundef a decos idnt asyncWs ws1 name ws2 params ws3 mty s) = + Fundef a + (view unvalidated <$> decos) idnt asyncWs ws1 (coerce name) ws2 + (view unvalidated <$> params) ws3 (over (mapped._2) (view unvalidated) mty) <$> + _Blocks f s + _Blocks f (If idnt a ws1 e1 s elifs b') = + If idnt a ws1 (e1 ^. unvalidated) <$> + _Blocks f s <*> + traverse (\(a, b, c, d) -> (,,,) a b (c ^. unvalidated) <$> _Blocks f d) elifs <*> + traverseOf (traverse._3._Blocks) f b' + _Blocks f (While idnt a ws1 e1 s els) = + While idnt a ws1 (e1 ^. unvalidated) <$> + _Blocks f s <*> + traverseOf (traverse._3._Blocks) f els + _Blocks fun (TryExcept idnt a b c d e f) = + TryExcept idnt a (coerce b) <$> + _Blocks fun c <*> + traverse + (\(a, b, c, d) -> (,,,) a b (view unvalidated <$> c) <$> _Blocks fun d) + d <*> + traverseOf (traverse._3._Blocks) fun e <*> + traverseOf (traverse._3._Blocks) fun f + _Blocks fun (TryFinally idnt a b c d e f) = + TryFinally idnt a b <$> + _Blocks fun c <*> + pure d <*> + pure e <*> + _Blocks fun f + _Blocks fun (For idnt a asyncWs b c d e f g) = + For idnt a asyncWs b (c ^. unvalidated) d (view unvalidated <$> e) <$> + _Blocks fun f <*> + (traverse._3._Blocks) fun g + _Blocks fun (ClassDef a decos idnt b c d e) = + ClassDef a + (view unvalidated <$> decos) idnt b + (coerce c) (over (mapped._2.mapped.mapped) (view unvalidated) d) <$> + _Blocks fun e + _Blocks fun (With a b asyncWs c d e) = + With a b asyncWs c (view unvalidated <$> d) <$> _Blocks fun e + +instance HasStatements Block where + _Statements f (Block a b c) = + Block a <$> f b <*> (traverse._Right) f c + +instance HasStatements Suite where + _Statements _ (SuiteOne a b c) = pure $ SuiteOne a b (c ^. unvalidated) + _Statements f (SuiteMany a b c d e) = SuiteMany a b c d <$> _Statements f e + +-- | See @simpl_stmt@ at . The grammar +-- has the terminology mixed up - it should really be called @small_stmt@ there. +data SmallStatement (v :: [*]) a + = MkSmallStatement + (SimpleStatement v a) + [(Semicolon a, SimpleStatement v a)] + (Maybe (Semicolon a)) + (Maybe (Comment a)) + (Maybe Newline) + deriving (Eq, Show, Functor, Foldable, Traversable) + +-- | A 'Statement' is either a 'SmallStatement' or a 'CompoundStatement' +-- +-- https://docs.python.org/3.5/reference/compound_stmts.html#compound-statements +data Statement (v :: [*]) a + = SmallStatement (Indents a) (SmallStatement v a) + | CompoundStatement (CompoundStatement v a) + deriving (Eq, Show, Functor, Foldable, Traversable) + +instance HasExprs SmallStatement where + _Exprs f (MkSmallStatement s ss a b c) = + MkSmallStatement <$> + _Exprs f s <*> + (traverse._2._Exprs) f ss <*> + pure a <*> + pure b <*> + pure c + +instance HasExprs Statement where + _Exprs f (SmallStatement idnt a) = SmallStatement idnt <$> _Exprs f a + _Exprs f (CompoundStatement c) = CompoundStatement <$> _Exprs f c + +instance HasBlocks SmallStatement where + _Blocks _ (MkSmallStatement a b c d e) = + pure $ + MkSmallStatement + (a ^. unvalidated) + (over (mapped._2) (view unvalidated) b) + c d e + +instance HasBlocks Statement where + _Blocks f (CompoundStatement c) = CompoundStatement <$> _Blocks f c + _Blocks f (SmallStatement idnt a) = SmallStatement idnt <$> _Blocks f a + +instance Plated (Statement '[] a) where + plate _ (SmallStatement idnt s) = pure $ SmallStatement idnt s + plate fun (CompoundStatement s) = + CompoundStatement <$> + case s of + Fundef idnt a decos asyncWs ws1 b ws2 c ws3 mty s -> + Fundef idnt a decos asyncWs ws1 b ws2 c ws3 mty <$> _Statements fun s + If idnt a ws1 b s elifs sts' -> + If idnt a ws1 b <$> + _Statements fun s <*> + (traverse._4._Statements) fun elifs <*> + (traverse._3._Statements) fun sts' + While idnt a ws1 b s els -> + While idnt a ws1 b <$> + _Statements fun s <*> + (traverse._3._Statements) fun els + TryExcept idnt a b c d e f -> + TryExcept idnt a b <$> _Statements fun c <*> + (traverse._4._Statements) fun d <*> + (traverse._3._Statements) fun e <*> + (traverse._3._Statements) fun f + TryFinally idnt a b c d e f -> + TryFinally idnt a b <$> _Statements fun c <*> pure d <*> + pure e <*> _Statements fun f + For idnt a asyncWs b c d e f g -> + For idnt a asyncWs b c d e <$> + _Statements fun f <*> + (traverse._3._Statements) fun g + ClassDef idnt a decos b c d e -> + ClassDef idnt a decos b c d <$> _Statements fun e + With a b asyncWs c d e -> With a b asyncWs c (coerce d) <$> _Statements fun e + +-- | https://docs.python.org/3.5/reference/simple_stmts.html +data SimpleStatement (v :: [*]) a + -- | @\'return\' \ [\]@ + -- + -- https://docs.python.org/3.5/reference/simple_stmts.html#the-return-statement + = Return a [Whitespace] (Maybe (Expr v a)) + -- | @\@ + -- + -- https://docs.python.org/3.5/reference/simple_stmts.html#expression-statements + | Expr a (Expr v a) + -- | @\ (\'=\' \ \)+@ + -- + -- https://docs.python.org/3.5/reference/simple_stmts.html#assignment-statements + | Assign a (Expr v a) (NonEmpty (Equals, Expr v a)) + -- | @\ \ \@ + -- + -- https://docs.python.org/3.5/reference/simple_stmts.html#augmented-assignment-statements + | AugAssign a (Expr v a) (AugAssign a) (Expr v a) + -- | @\'pass\' \@ + -- + -- https://docs.python.org/3.5/reference/simple_stmts.html#the-pass-statement + | Pass a [Whitespace] + -- | @\'break\' \@ + -- + -- https://docs.python.org/3.5/reference/simple_stmts.html#the-break-statement + | Break a [Whitespace] + -- | @\'continue\' \@ + -- + -- https://docs.python.org/3.5/reference/simple_stmts.html#the-continue-statement + | Continue a [Whitespace] + -- | @\'global\' \ \@ + -- + -- https://docs.python.org/3.5/reference/simple_stmts.html#the-global-statement + | Global a (NonEmpty Whitespace) (CommaSep1 (Ident v a)) + -- | @\'nonlocal\' \ \@ + -- + -- https://docs.python.org/3.5/reference/simple_stmts.html#the-nonlocal-statement + | Nonlocal a (NonEmpty Whitespace) (CommaSep1 (Ident v a)) + -- | @\'del\' \ \@ + -- + -- https://docs.python.org/3.5/reference/simple_stmts.html#the-del-statement + | Del a [Whitespace] (CommaSep1' (Expr v a)) + -- | @\'import\' \ \@ + -- + -- https://docs.python.org/3.5/reference/simple_stmts.html#the-import-statement + | Import + a + (NonEmpty Whitespace) + (CommaSep1 (ImportAs (ModuleName v) v a)) + -- | @\'from\' \ \ \'import\' \ \@ + -- + -- https://docs.python.org/3.5/reference/simple_stmts.html#the-import-statement + | From + a + [Whitespace] + (RelativeModuleName v a) + [Whitespace] + (ImportTargets v a) + -- | @\'raise\' \ [\ [\'as\' \ \]]@ + -- + -- https://docs.python.org/3.5/reference/simple_stmts.html#the-raise-statement + | Raise a + [Whitespace] + (Maybe (Expr v a, Maybe ([Whitespace], Expr v a))) + -- | @\'assert\' \ \ [\',\' \ \]@ + -- + -- https://docs.python.org/3.5/reference/simple_stmts.html#the-assert-statement + | Assert a + [Whitespace] + (Expr v a) + (Maybe (Comma, Expr v a)) + deriving (Eq, Show, Functor, Foldable, Traversable, Generic) + +instance Plated (SimpleStatement '[] a) where; plate = gplate + +instance HasExprs SimpleStatement where + _Exprs f (Assert a b c d) = Assert a b <$> f c <*> traverseOf (traverse._2) f d + _Exprs f (Raise a ws x) = + Raise a ws <$> + traverse + (\(b, c) -> (,) <$> f b <*> traverseOf (traverse._2) f c) + x + _Exprs f (Return a ws e) = Return a ws <$> traverse f e + _Exprs f (Expr a e) = Expr a <$> f e + _Exprs f (Assign a e1 es) = Assign a <$> f e1 <*> traverseOf (traverse._2) f es + _Exprs f (AugAssign a e1 as e2) = AugAssign a <$> f e1 <*> pure as <*> f e2 + _Exprs _ p@Pass{} = pure $ p ^. unvalidated + _Exprs _ p@Break{} = pure $ p ^. unvalidated + _Exprs _ p@Continue{} = pure $ p ^. unvalidated + _Exprs _ p@Global{} = pure $ p ^. unvalidated + _Exprs _ p@Nonlocal{} = pure $ p ^. unvalidated + _Exprs _ p@Del{} = pure $ p ^. unvalidated + _Exprs _ p@Import{} = pure $ p ^. unvalidated + _Exprs _ p@From{} = pure $ p ^. unvalidated + +-- | See +data ExceptAs (v :: [*]) a + = ExceptAs + { _exceptAsAnn :: a + , _exceptAsExpr :: Expr v a -- ^ @\@ + , _exceptAsName :: Maybe ([Whitespace], Ident v a) -- ^ @[\'as\' \]@ + } + deriving (Eq, Show, Functor, Foldable, Traversable) + +-- | A compound statement consists of one or more clauses. +-- A clause consists of a header and a suite. +data Suite (v :: [*]) a + -- ':' smallStatement + = SuiteOne a Colon (SmallStatement v a) + | SuiteMany a + -- ':' [comment] + Colon (Maybe (Comment a)) Newline + -- + (Block v a) + deriving (Eq, Show, Functor, Foldable, Traversable) + +-- | See +data WithItem (v :: [*]) a + = WithItem + { _withItemAnn :: a + , _withItemValue :: Expr v a -- ^ @\@ + , _withItemBinder :: Maybe ([Whitespace], Expr v a) -- ^ @[\'as\' \ \]@ + } + deriving (Eq, Show, Functor, Foldable, Traversable) + +-- | Decorators on function definitions +-- +-- +-- +-- +data Decorator (v :: [*]) a + = Decorator + { _decoratorAnn :: a + , _decoratorIndents :: Indents a -- ^ Preceding indentation + , _decoratorAt :: At -- ^ @\'\@\' \@ + , _decoratorExpr :: Expr v a -- ^ @\@ + , _decoratorComment :: Maybe (Comment a) -- ^ Trailing comment + , _decoratorNewline :: Newline -- ^ Trailing newline + , _decoratorBlankLines :: [(Blank a, Newline)] -- ^ Trailing blank lines + } + deriving (Eq, Show, Functor, Foldable, Traversable) + +-- | See +data CompoundStatement (v :: [*]) a + -- | https://docs.python.org/3.5/reference/compound_stmts.html#function-definitions + -- + -- https://docs.python.org/3.5/reference/compound_stmts.html#coroutine-function-definition + = Fundef + { _csAnn :: a + , _unsafeCsFundefDecorators :: [Decorator v a] -- ^ Preceding 'Decorator's + , _csIndents :: Indents a -- ^ Preceding indentation + , _unsafeCsFundefAsync :: Maybe (NonEmpty Whitespace) -- ^ @[\'async\' \]@ + , _unsafeCsFundefDef :: NonEmpty Whitespace -- ^ @\'def\' \@ + , _unsafeCsFundefName :: Ident v a -- ^ @\@ + , _unsafeCsFundefLeftParen :: [Whitespace] -- ^ @\'(\' \@ + , _unsafeCsFundefParameters :: CommaSep (Param v a) -- ^ @\@ + , _unsafeCsFundefRightParen :: [Whitespace] -- ^ @\')\' \@ + , _unsafeCsFundefReturnType :: Maybe ([Whitespace], Expr v a) -- ^ @[\'->\' \ \]@ + , _unsafeCsFundefBody :: Suite v a -- ^ @\@ + } + -- | https://docs.python.org/3.5/reference/compound_stmts.html#the-if-statement + | If + { _csAnn :: a + , _csIndents :: Indents a -- ^ Preceding indentation + , _unsafeCsIfIf :: [Whitespace] -- ^ @\'if\' \@ + , _unsafeCsIfCond :: Expr v a -- ^ @\@ + , _unsafeCsIfBody :: Suite v a -- ^ @\@ + , _unsafeCsIfElifs :: [(Indents a, [Whitespace], Expr v a, Suite v a)] -- ^ @(\'elif\' \ \ \)*@ + , _unsafeCsIfElse :: Maybe (Indents a, [Whitespace], Suite v a) -- ^ @[\'else\' \ \]@ + } + -- | https://docs.python.org/3.5/reference/compound_stmts.html#the-while-statement + | While + { _csAnn :: a + , _csIndents :: Indents a -- ^ Preceding indentation + , _unsafeCsWhileWhile :: [Whitespace] -- ^ @\'while\' \@ + , _unsafeCsWhileCond :: Expr v a -- ^ @\@ + , _unsafeCsWhileBody :: Suite v a -- ^ @\@ + , _unsafeCsWhileElse + :: Maybe (Indents a, [Whitespace], Suite v a) -- ^ @[\'else\' \ \]@ + } + -- | https://docs.python.org/3.5/reference/compound_stmts.html#the-try-statement + | TryExcept + { _csAnn :: a + , _csIndents :: Indents a -- ^ Preceding indentation + , _unsafeCsTryExceptTry :: [Whitespace] -- ^ @\'try\' \@ + , _unsafeCsTryExceptBody :: Suite v a -- ^ @\@ + , _unsafeCsTryExceptExcepts :: NonEmpty (Indents a, [Whitespace], Maybe (ExceptAs v a), Suite v a) -- ^ @(\'except\' \ \ \)+@ + , _unsafeCsTryExceptElse :: Maybe (Indents a, [Whitespace], Suite v a) -- ^ @[\'else\' \ \]@ + , _unsafeCsTryExceptFinally :: Maybe (Indents a, [Whitespace], Suite v a) -- ^ @[\'finally\' \ \]@ + } + -- | https://docs.python.org/3.5/reference/compound_stmts.html#the-try-statement + | TryFinally + { _csAnn :: a + , _csIndents :: Indents a -- ^ Preceding indentation + , _unsafeCsTryFinallyTry :: [Whitespace] -- ^ @\'try\' \@ + , _unsafeCsTryFinallyTryBody :: Suite v a -- ^ @\@ + , _unsafeCsTryFinallyFinallyIndents :: Indents a + , _unsafeCsTryFinallyFinally :: [Whitespace] -- ^ @\'finally\' \@ + , _unsafeCsTryFinallyFinallyBody :: Suite v a -- ^ @\@ + } + -- | https://docs.python.org/3.5/reference/compound_stmts.html#the-for-statement + -- + -- https://docs.python.org/3.5/reference/compound_stmts.html#the-async-for-statement + | For + { _csAnn :: a + , _csIndents :: Indents a -- ^ Preceding indentation + , _unsafeCsForAsync :: Maybe (NonEmpty Whitespace) -- ^ @[\'async\' \]@ + , _unsafeCsForFor :: [Whitespace] -- ^ @\'for\' \@ + , _unsafeCsForBinder :: Expr v a -- ^ @\@ + , _unsafeCsForIn :: [Whitespace] -- ^ @\'in\' \@ + , _unsafeCsForCollection :: CommaSep1' (Expr v a) -- ^ @\@ + , _unsafeCsForBody :: Suite v a -- ^ @\@ + , _unsafeCsForElse :: Maybe (Indents a, [Whitespace], Suite v a) -- ^ @[\'else\' \ \]@ + } + -- | https://docs.python.org/3.5/reference/compound_stmts.html#class-definitions + | ClassDef + { _csAnn :: a + , _unsafeCsClassDefDecorators :: [Decorator v a] -- ^ Preceding 'Decorator's + , _csIndents :: Indents a -- ^ Preceding indentation + , _unsafeCsClassDefClass :: NonEmpty Whitespace -- ^ @\'class\' \@ + , _unsafeCsClassDefName :: Ident v a -- ^ @\@ + , _unsafeCsClassDefArguments :: Maybe ([Whitespace], Maybe (CommaSep1' (Arg v a)), [Whitespace]) -- ^ @[\'(\' \ [\] \')\' \]@ + , _unsafeCsClassDefBody :: Suite v a -- ^ @\@ + } + -- | https://docs.python.org/3.5/reference/compound_stmts.html#the-with-statement + -- + -- https://docs.python.org/3.5/reference/compound_stmts.html#the-async-with-statement + | With + { _csAnn :: a + , _csIndents :: Indents a -- ^ Preceding indentation + , _unsafeCsWithAsync :: Maybe (NonEmpty Whitespace) -- ^ @[\'async\' \]@ + , _unsafeCsWithWith :: [Whitespace] -- ^ @\'with\' \@ + , _unsafeCsWithItems :: CommaSep1 (WithItem v a) -- ^ @\@ + , _unsafeCsWithBody :: Suite v a -- ^ @\@ + } + deriving (Eq, Show, Functor, Foldable, Traversable) + +instance HasExprs ExceptAs where + _Exprs f (ExceptAs ann e a) = ExceptAs ann <$> f e <*> pure (coerce a) + +instance HasExprs Block where + _Exprs f (Block a b c) = + Block a <$> _Exprs f b <*> (traverse._Right._Exprs) f c + +instance HasExprs Suite where + _Exprs f (SuiteOne a b c) = (\c' -> SuiteOne a b c') <$> _Exprs f c + _Exprs f (SuiteMany a b c d e) = SuiteMany a b c d <$> _Exprs f e + +instance HasExprs WithItem where + _Exprs f (WithItem a b c) = WithItem a <$> f b <*> traverseOf (traverse._2) f c + +instance HasExprs Decorator where + _Exprs fun (Decorator a b c d e f g) = (\d' -> Decorator a b c d' e f g) <$> _Exprs fun d + +instance HasExprs CompoundStatement where + _Exprs f (Fundef a decos idnt asyncWs ws1 name ws2 params ws3 mty s) = + Fundef a <$> + (traverse._Exprs) f decos <*> + pure idnt <*> + pure asyncWs <*> + pure ws1 <*> + pure (coerce name) <*> + pure ws2 <*> + (traverse._Exprs) f params <*> + pure ws3 <*> + traverseOf (traverse._2) f mty <*> + _Exprs f s + _Exprs fun (If idnt a ws1 e s elifs sts') = + If idnt a ws1 <$> + fun e <*> + _Exprs fun s <*> + traverse + (\(a, b, c, d) -> (,,,) a b <$> fun c <*> _Exprs fun d) + elifs <*> + (traverse._3._Exprs) fun sts' + _Exprs f (While idnt a ws1 e s els) = + While idnt a ws1 <$> + f e <*> + _Exprs f s <*> + (traverse._3._Exprs) f els + _Exprs fun (TryExcept idnt a b c d e f) = + TryExcept idnt a b <$> _Exprs fun c <*> + traverse + (\(a, b, c, d) -> (,,,) a b <$> traverse (_Exprs fun) c <*> _Exprs fun d) + d <*> + (traverse._3._Exprs) fun e <*> + (traverse._3._Exprs) fun f + _Exprs fun (TryFinally idnt a b c d e f) = + TryFinally idnt a b <$> _Exprs fun c <*> pure d <*> + pure e <*> _Exprs fun f + _Exprs fun (For idnt a asyncWs b c d e f g) = + For idnt a asyncWs b <$> fun c <*> pure d <*> traverse fun e <*> + _Exprs fun f <*> + (traverse._3._Exprs) fun g + _Exprs fun (ClassDef a decos idnt b c d e) = + ClassDef a <$> + traverse (_Exprs fun) decos <*> + pure idnt <*> + pure b <*> + pure (coerce c) <*> + (traverse._2.traverse.traverse._Exprs) fun d <*> + _Exprs fun e + _Exprs fun (With a b asyncWs c d e) = + With a b asyncWs c <$> traverseOf (traverse._Exprs) fun d <*> _Exprs fun e + +instance HasTrailingNewline Statement where + trailingNewline f x = + case x of + SmallStatement a b -> SmallStatement a <$> trailingNewline f b + CompoundStatement c -> CompoundStatement <$> trailingNewline f c + + setTrailingNewline s n = + case s of + SmallStatement i a -> SmallStatement i $ setTrailingNewline a n + CompoundStatement c -> CompoundStatement $ setTrailingNewline c n + +instance HasTrailingNewline SmallStatement where + trailingNewline f (MkSmallStatement a b c d e) = + MkSmallStatement a b c d <$> traverse f e + setTrailingNewline (MkSmallStatement a b c d _) n = + MkSmallStatement a b c d (Just n) + +instance HasTrailingNewline Suite where + trailingNewline f x = + case x of + SuiteOne a b c -> SuiteOne a b <$> trailingNewline f c + SuiteMany a b c d e -> SuiteMany a b c d <$> trailingNewline f e + setTrailingNewline x n = + case x of + SuiteOne a b c -> SuiteOne a b $ setTrailingNewline c n + SuiteMany a b c d e -> SuiteMany a b c d $ setTrailingNewline e n + +instance HasTrailingNewline Block where + trailingNewline f (Block a b []) = Block a <$> trailingNewline f b <*> pure [] + trailingNewline f (Block a b (c:cs)) = + Block a b <$> + traverseOf + _last + (bitraverse (traverseOf _2 f) (trailingNewline f)) + (c:cs) + + setTrailingNewline (Block a b []) n = + Block a (setTrailingNewline b n) [] + setTrailingNewline (Block a b (c:cs)) n = + Block a b (over _last (bimap (_2 .~ n) (flip setTrailingNewline n)) $ c:cs) + +instance HasTrailingNewline CompoundStatement where + trailingNewline fun s = + case s of + Fundef a b c d e f g h i j k -> + Fundef a b c d e f g h i j <$> trailingNewline fun k + If a b c d e f g -> + If a b c d <$> + (if null f && isNothing g + then trailingNewline fun e + else pure e) <*> + (if isNothing g + then (_last._4.trailingNewline) fun f + else pure f)<*> + (traverse._3.trailingNewline) fun g + While a b c d e f -> + While a b c d <$> + (if isNothing f + then trailingNewline fun e + else pure e) <*> + (traverse._3.trailingNewline) fun f + TryExcept a b c d e f g -> + TryExcept a b c d <$> + (if isNothing f && isNothing g + then + fmap + NonEmpty.fromList + ((_last._4.trailingNewline) fun $ NonEmpty.toList e) + else pure e) <*> + (if isNothing g + then (traverse._3.trailingNewline) fun f + else pure f) <*> + (traverse._3.trailingNewline) fun g + TryFinally a b c d e f g -> + TryFinally a b c d e f <$> + trailingNewline fun g + For a b c d e f g h i -> + For a b c d e f g <$> + (if isNothing i + then trailingNewline fun h + else pure h) <*> + (traverse._3.trailingNewline) fun i + ClassDef a b c d e f g -> + ClassDef a b c d e f <$> trailingNewline fun g + With a b c d e f -> + With a b c d e <$> trailingNewline fun f + setTrailingNewline s n = + case s of + Fundef a b c d e f g h i j k -> + Fundef a b c d e f g h i j $ setTrailingNewline k n + If a b c d e f g -> + If a b c d + (if null f && isNothing g + then setTrailingNewline e n + else e) + (if isNothing g + then over (_last._4) (flip setTrailingNewline n) f + else f) + (over (mapped._3) (flip setTrailingNewline n) g) + While a b c d e f -> + While a b c d + (if isNothing f + then setTrailingNewline e n + else e) + (over (mapped._3) (flip setTrailingNewline n) f) + TryExcept a b c d e f g -> + TryExcept a b c d + (if isNothing f && isNothing g + then + NonEmpty.fromList + (over (_last._4) (flip setTrailingNewline n) $ NonEmpty.toList e) + else e) + (if isNothing g + then over (mapped._3) (flip setTrailingNewline n) f + else f) + (over (mapped._3) (flip setTrailingNewline n) g) + TryFinally a b c d e f g -> + TryFinally a b c d e f $ + setTrailingNewline g n + For a b c d e f g h i -> + For a b c d e f g + (if isNothing i + then setTrailingNewline h n + else h) + (over (mapped._3) (flip setTrailingNewline n) i) + ClassDef a b c d e f g -> + ClassDef a b c d e f $ setTrailingNewline g n + With a b c d e f -> + With a b c d e $ setTrailingNewline f n + +makeLenses ''WithItem +makeLenses ''ExceptAs +makeLenses ''Block diff --git a/src/Language/Python/Syntax/Strings.hs b/src/Language/Python/Syntax/Strings.hs new file mode 100644 --- /dev/null +++ b/src/Language/Python/Syntax/Strings.hs @@ -0,0 +1,318 @@ +{-# language DeriveFunctor, DeriveFoldable, DeriveTraversable #-} +{-# language OverloadedStrings #-} +{-# language LambdaCase #-} +{-# language TemplateHaskell #-} + +{-| +Module : Language.Python.Syntax.Strings +Copyright : (C) CSIRO 2017-2018 +License : BSD3 +Maintainer : Isaac Elliott +Stability : experimental +Portability : non-portable + +Python string literals. + +See +-} + +module Language.Python.Syntax.Strings + ( -- * Datatypes + -- ** Characters + PyChar(..) + , fromHaskellString + -- ** String information + , QuoteType(..) + , StringType(..) + -- ** String prefixes + , StringPrefix(..) + , RawStringPrefix(..) + , BytesPrefix(..) + , RawBytesPrefix(..) + , hasPrefix + -- ** String literals + , StringLiteral(..) + -- *** Lenses + , stringLiteralStringType + , stringLiteralQuoteType + , stringLiteralValue + , stringLiteralWhitespace + -- * Rendering + , showQuoteType + , showStringPrefix + , showRawStringPrefix + , showBytesPrefix + , showRawBytesPrefix + -- * Extra functions + , isEscape + ) +where + +import Control.Lens.Lens (lens) +import Control.Lens.TH (makeLensesFor) +import Data.Digit.Octal (OctDigit) +import Data.Digit.Hexadecimal.MixedCase (HeXDigit(..)) +import Data.Maybe (isJust) +import Data.Text (Text) + +import Language.Python.Syntax.Whitespace + +-- | Double or single quotation marks? +-- +-- @ +-- "Double quotes" +-- """Double quotes""" +-- 'Single quotes' +-- '''Single quotes''' +-- @ +data QuoteType + = SingleQuote + | DoubleQuote + deriving (Eq, Ord, Show) + +-- | Three pairs of quotations or one? +-- +-- @ +-- """Long string""" +-- '''Also long string''' +-- "Short string" +-- 'Also short string' +-- @ +data StringType + = ShortString + | LongString + deriving (Eq, Ord, Show) + +-- | In Python 3.5, a prefix of @u@ or @U@ is allowed, but doesn't have any +-- meaning. They exist for backwards compatibility with Python 2. +-- +-- See +data StringPrefix + = Prefix_u + | Prefix_U + deriving (Eq, Ord, Show) + +-- | Raw strings are prefixed with either @r@ or @R@. +data RawStringPrefix + = Prefix_r + | Prefix_R + deriving (Eq, Ord, Show) + +-- | This prefix indicates it's a bytes literal rather than a string literal. +data BytesPrefix + = Prefix_b + | Prefix_B + deriving (Eq, Ord, Show) + +-- | A string of raw bytes can be indicated by a number of prefixes +data RawBytesPrefix + = Prefix_br + | Prefix_Br + | Prefix_bR + | Prefix_BR + | Prefix_rb + | Prefix_rB + | Prefix_Rb + | Prefix_RB + deriving (Eq, Ord, Show) + +-- | Most types of 'StringLiteral' have prefixes. Plain old strings may have +-- an optional prefix, but it is meaningless. +hasPrefix :: StringLiteral a -> Bool +hasPrefix RawStringLiteral{} = True +hasPrefix RawBytesLiteral{} = True +hasPrefix (StringLiteral _ a _ _ _ _) = isJust a +hasPrefix BytesLiteral{} = True + +-- | A 'StringLiteral', complete with a prefix, information about +-- quote type and number, and a list of 'PyChar's. +-- +-- Like many other data types in hpython, it has an annotation and +-- trailing whitespace. +data StringLiteral a + = RawStringLiteral + { _stringLiteralAnn :: a + , _unsafeRawStringLiteralPrefix :: RawStringPrefix + , _stringLiteralStringType :: StringType + , _stringLiteralQuoteType :: QuoteType + , _stringLiteralValue :: [PyChar] + , _stringLiteralWhitespace :: [Whitespace] + } + | StringLiteral + { _stringLiteralAnn :: a + , _unsafeStringLiteralPrefix :: Maybe StringPrefix + , _stringLiteralStringType :: StringType + , _stringLiteralQuoteType :: QuoteType + , _stringLiteralValue :: [PyChar] + , _stringLiteralWhitespace :: [Whitespace] + } + | RawBytesLiteral + { _stringLiteralAnn :: a + , _unsafeRawBytesLiteralPrefix :: RawBytesPrefix + , _stringLiteralStringType :: StringType + , _stringLiteralQuoteType :: QuoteType + , _stringLiteralValue :: [PyChar] + , _stringLiteralWhitespace :: [Whitespace] + } + | BytesLiteral + { _stringLiteralAnn :: a + , _unsafeBytesLiteralPrefix :: BytesPrefix + , _stringLiteralStringType :: StringType + , _stringLiteralQuoteType :: QuoteType + , _stringLiteralValue :: [PyChar] + , _stringLiteralWhitespace :: [Whitespace] + } + deriving (Eq, Ord, Show, Functor, Foldable, Traversable) + +instance HasTrailingWhitespace (StringLiteral a) where + trailingWhitespace = + lens + (\case + RawStringLiteral _ _ _ _ _ ws -> ws + StringLiteral _ _ _ _ _ ws -> ws + RawBytesLiteral _ _ _ _ _ ws -> ws + BytesLiteral _ _ _ _ _ ws -> ws) + (\s ws -> case s of + StringLiteral a b c d e _ -> StringLiteral a b c d e ws + RawStringLiteral a b c d e _ -> RawStringLiteral a b c d e ws + BytesLiteral a b c d e _ -> BytesLiteral a b c d e ws + RawBytesLiteral a b c d e _ -> RawBytesLiteral a b c d e ws) + +-- | A character in a string literal. This is a large sum type, with a +-- catch-all of a Haskell 'Char'. +data PyChar + -- | @\\newline@ + = Char_newline + -- | @\\1@ + | Char_octal1 OctDigit + -- | @\\12@ + | Char_octal2 OctDigit OctDigit + -- | @\\123@ + | Char_octal3 OctDigit OctDigit OctDigit + -- | @\\xFb@ + | Char_hex HeXDigit HeXDigit + -- | @\\u12aD@ + | Char_uni16 + HeXDigit + HeXDigit + HeXDigit + HeXDigit + -- | @\\Udeadbeef@ + | Char_uni32 + HeXDigit + HeXDigit + HeXDigit + HeXDigit + HeXDigit + HeXDigit + HeXDigit + HeXDigit + -- | @\\\\@ + | Char_esc_bslash + -- | @\\\'@ + | Char_esc_singlequote + -- | @\\\"@ + | Char_esc_doublequote + -- | @\\a@ + | Char_esc_a + -- | @\\b@ + | Char_esc_b + -- | @\\f@ + | Char_esc_f + -- | @\\n@ + | Char_esc_n + -- | @\\r@ + | Char_esc_r + -- | @\\t@ + | Char_esc_t + -- | @\\v@ + | Char_esc_v + -- | Any character + | Char_lit Char + deriving (Eq, Ord, Show) + +-- | Determine whether a 'PyChar' is an escape character or not. +isEscape :: PyChar -> Bool +isEscape c = + case c of + Char_newline -> True + Char_octal1{} -> True + Char_octal2{} -> True + Char_octal3{} -> True + Char_hex{} -> True + Char_uni16{} -> True + Char_uni32{} -> True + Char_esc_bslash -> True + Char_esc_singlequote -> True + Char_esc_doublequote -> True + Char_esc_a -> True + Char_esc_b -> True + Char_esc_f -> True + Char_esc_n -> True + Char_esc_r -> True + Char_esc_t -> True + Char_esc_v -> True + Char_lit{} -> False + +-- | Convert a Haskell string to a list of 'PyChar'. This is useful when +-- writing Python in Haskell. +fromHaskellString :: String -> [PyChar] +fromHaskellString = + fmap + (\c -> case c of + '\\' -> Char_esc_bslash + '\'' -> Char_esc_singlequote + '\"' -> Char_esc_doublequote + '\a' -> Char_esc_a + '\b' -> Char_esc_b + '\f' -> Char_esc_f + '\n' -> Char_esc_n + '\r' -> Char_esc_r + '\t' -> Char_esc_t + '\v' -> Char_esc_v + '\0' -> Char_hex HeXDigit0 HeXDigit0 + _ -> Char_lit c) + +showStringPrefix :: StringPrefix -> Text +showStringPrefix sp = + case sp of + Prefix_u -> "u" + Prefix_U -> "U" + +showRawStringPrefix :: RawStringPrefix -> Text +showRawStringPrefix sp = + case sp of + Prefix_r -> "r" + Prefix_R -> "R" + +showBytesPrefix :: BytesPrefix -> Text +showBytesPrefix sp = + case sp of + Prefix_b -> "b" + Prefix_B -> "B" + +showRawBytesPrefix :: RawBytesPrefix -> Text +showRawBytesPrefix sp = + case sp of + Prefix_br -> "br" + Prefix_Br -> "Br" + Prefix_bR -> "bR" + Prefix_BR -> "BR" + Prefix_rb -> "rb" + Prefix_rB -> "rB" + Prefix_Rb -> "Rb" + Prefix_RB -> "RB" + +showQuoteType :: QuoteType -> Char +showQuoteType qt = + case qt of + DoubleQuote -> '\"' + SingleQuote -> '\'' + +makeLensesFor + [ ("_stringLiteralValue", "stringLiteralValue") + , ("_stringLiteralStringType", "stringLiteralStringType") + , ("_stringLiteralQuoteType", "stringLiteralQuoteType") + , ("_stringLiteralWhitespace", "stringLiteralWhitespace") + ] + ''StringLiteral diff --git a/src/Language/Python/Syntax/Types.hs b/src/Language/Python/Syntax/Types.hs new file mode 100644 --- /dev/null +++ b/src/Language/Python/Syntax/Types.hs @@ -0,0 +1,427 @@ +{-# language DataKinds #-} +{-# language KindSignatures #-} +{-# language TemplateHaskell #-} + +{-| +Module : Language.Python.Syntax.Types +Copyright : (C) CSIRO 2017-2018 +License : BSD3 +Maintainer : Isaac Elliott +Stability : experimental +Portability : non-portable + +Datatypes for different parts of Python syntax +-} + +module Language.Python.Syntax.Types + ( -- * Parameters + -- ** Positional parameters + PositionalParam(..) + -- *** Lenses + , ppAnn + , ppName + , ppType + -- ** Starred Parameters + , StarParam(..) + -- *** Lenses + , spAnn + , spWhitespace + , spName + , spType + -- ** Unnamed Starred Parameters + , UnnamedStarParam(..) + -- *** Lenses + , uspAnn + , uspWhitespace + -- ** Keyword parameters + , KeywordParam(..) + -- *** Lenses + , kpAnn + , kpName + , kpType + , kpEquals + , kpExpr + -- * Compound statements + -- ** Function definitions + , Fundef(..) + -- *** Lenses + , fdAnn + , fdDecorators + , fdIndents + , fdAsync + , fdDefSpaces + , fdName + , fdLeftParenSpaces + , fdParameters + , fdRightParenSpaces + , fdReturnType + , fdBody + -- ** Class definitions + , ClassDef(..) + -- *** Lenses + , cdAnn + , cdDecorators + , cdIndents + , cdClass + , cdName + , cdArguments + , cdBody + -- ** @if@ statements + , If(..) + -- *** Lenses + , ifAnn + , ifIndents + , ifIf + , ifCond + , ifBody + , ifElifs + , ifElse + -- ** @elif@ + , Elif(..) + -- *** Lenses + , elifIndents + , elifElif + , elifCond + , elifBody + -- ** @for@ statements + , For(..) + -- *** Lenses + , forAnn + , forIndents + , forAsync + , forFor + , forBinder + , forIn + , forCollection + , forBody + , forElse + -- ** @while@ statements + , While(..) + -- *** Lenses + , whileAnn + , whileIndents + , whileWhile + , whileCond + , whileBody + , whileElse + -- ** @try ... except ... else ... finally@ + , TryExcept(..) + -- *** Lenses + , teAnn + , teIndents + , teTry + , teBody + , teExcepts + , teElse + , teFinally + -- *** @except@ + , Except(..) + -- **** Lenses + , exceptIndents + , exceptExcept + , exceptExceptAs + , exceptBody + -- ** @try ... finally@ + , TryFinally(..) + -- *** Lenses + , tfAnn + , tfIndents + , tfTry + , tfBody + , tfFinally + -- ** @finally@ + , Finally(..) + -- *** Lenses + , finallyIndents + , finallyFinally + , finallyBody + -- ** @with@ statements + , With(..) + -- *** Lenses + , withAnn + , withIndents + , withAsync + , withWith + , withItems + , withBody + -- ** @else@ + , Else(..) + -- *** Lenses + , elseIndents + , elseElse + , elseBody + -- * Expressions + -- ** @None@ + , None(..) + -- *** Lenses + , noneAnn + , noneWhitespace + -- ** Function calls + , Call(..) + -- *** Lenses + , callAnn + , callFunction + , callLeftParen + , callArguments + , callRightParen + -- ** Tuples + , Tuple(..) + -- *** Lenses + , tupleAnn + , tupleHead + , tupleComma + , tupleTail + -- *** Tuple items + -- **** Unpacking + , TupleUnpack(..) + -- ***** Lenses + , tupleUnpackAnn + , tupleUnpackParens + , tupleUnpackWhitespace + , tupleUnpackValue + -- ** Lists + , List(..) + -- *** Lenses + , listAnn + , listWhitespaceLeft + , listBody + , listWhitespaceRight + -- *** List items + -- **** Unpacking + , ListUnpack(..) + -- ***** Lenses + , listUnpackAnn + , listUnpackParens + , listUnpackWhitespace + , listUnpackValue + ) +where + +import Control.Lens.TH (makeLenses) +import Data.List.NonEmpty (NonEmpty) + +import Language.Python.Syntax.CommaSep (Comma, CommaSep, CommaSep1, CommaSep1') +import Language.Python.Syntax.Expr (Arg, Expr, ListItem, Param, TupleItem) +import Language.Python.Syntax.Ident (Ident) +import Language.Python.Syntax.Punctuation (Colon) +import Language.Python.Syntax.Statement (Decorator, ExceptAs, Suite, WithItem) +import Language.Python.Syntax.Whitespace + +data Fundef v a + = MkFundef + { _fdAnn :: a + , _fdDecorators :: [Decorator v a] + , _fdIndents :: Indents a + , _fdAsync :: Maybe (NonEmpty Whitespace) + , _fdDefSpaces :: NonEmpty Whitespace + , _fdName :: Ident v a + , _fdLeftParenSpaces :: [Whitespace] + , _fdParameters :: CommaSep (Param v a) + , _fdRightParenSpaces :: [Whitespace] + , _fdReturnType :: Maybe ([Whitespace], Expr v a) + , _fdBody :: Suite v a + } deriving (Eq, Show) +makeLenses ''Fundef + +data Else v a + = MkElse + { _elseIndents :: Indents a + , _elseElse :: [Whitespace] + , _elseBody :: Suite v a + } deriving (Eq, Show) +makeLenses ''Else + +data While v a + = MkWhile + { _whileAnn :: a + , _whileIndents :: Indents a + , _whileWhile :: [Whitespace] + , _whileCond :: Expr v a + , _whileBody :: Suite v a + , _whileElse :: Maybe (Else v a) + } deriving (Eq, Show) +makeLenses ''While + +data KeywordParam v a + = MkKeywordParam + { _kpAnn :: a + , _kpName :: Ident v a + , _kpType :: Maybe (Colon, Expr v a) + , _kpEquals :: [Whitespace] + , _kpExpr :: Expr v a + } deriving (Eq, Show) +makeLenses ''KeywordParam + +data PositionalParam v a + = MkPositionalParam + { _ppAnn :: a + , _ppName :: Ident v a + , _ppType :: Maybe (Colon, Expr v a) + } deriving (Eq, Show) +makeLenses ''PositionalParam + +data StarParam v a + = MkStarParam + { _spAnn :: a + , _spWhitespace :: [Whitespace] + , _spName :: Ident v a + , _spType :: Maybe (Colon, Expr v a) + } deriving (Eq, Show) +makeLenses ''StarParam + +data UnnamedStarParam (v :: [*]) a + = MkUnnamedStarParam + { _uspAnn :: a + , _uspWhitespace :: [Whitespace] + } deriving (Eq, Show) +makeLenses ''UnnamedStarParam + +data Call v a + = MkCall + { _callAnn :: a + , _callFunction :: Expr v a + , _callLeftParen :: [Whitespace] + , _callArguments :: Maybe (CommaSep1' (Arg v a)) + , _callRightParen :: [Whitespace] + } deriving (Eq, Show) +makeLenses ''Call + +data Elif v a + = MkElif + { _elifIndents :: Indents a + , _elifElif :: [Whitespace] + , _elifCond :: Expr v a + , _elifBody :: Suite v a + } deriving (Eq, Show) +makeLenses ''Elif + +data If v a + = MkIf + { _ifAnn :: a + , _ifIndents :: Indents a + , _ifIf :: [Whitespace] + , _ifCond :: Expr v a + , _ifBody :: Suite v a + , _ifElifs :: [Elif v a] + , _ifElse :: Maybe (Else v a) + } deriving (Eq, Show) +makeLenses ''If + +data For v a + = MkFor + { _forAnn :: a + , _forIndents :: Indents a + , _forAsync :: Maybe (NonEmpty Whitespace) + , _forFor :: [Whitespace] + , _forBinder :: Expr v a + , _forIn :: [Whitespace] + , _forCollection :: CommaSep1' (Expr v a) + , _forBody :: Suite v a + , _forElse :: Maybe (Else v a) + } deriving (Eq, Show) +makeLenses ''For + +data Finally v a + = MkFinally + { _finallyIndents :: Indents a + , _finallyFinally :: [Whitespace] + , _finallyBody :: Suite v a + } deriving (Eq, Show) +makeLenses ''Finally + +data Except v a + = MkExcept + { _exceptIndents :: Indents a + , _exceptExcept :: [Whitespace] + , _exceptExceptAs :: Maybe (ExceptAs v a) + , _exceptBody :: Suite v a + } deriving (Eq, Show) +makeLenses ''Except + +data TryExcept v a + = MkTryExcept + { _teAnn :: a + , _teIndents :: Indents a + , _teTry :: [Whitespace] + , _teBody :: Suite v a + , _teExcepts :: NonEmpty (Except v a) + , _teElse :: Maybe (Else v a) + , _teFinally :: Maybe (Finally v a) + } deriving (Eq, Show) +makeLenses ''TryExcept + +data TryFinally v a + = MkTryFinally + { _tfAnn :: a + , _tfIndents :: Indents a + , _tfTry :: [Whitespace] + , _tfBody :: Suite v a + , _tfFinally :: Finally v a + } deriving (Eq, Show) +makeLenses ''TryFinally + +data ClassDef v a + = MkClassDef + { _cdAnn :: a + , _cdDecorators :: [Decorator v a] + , _cdIndents :: Indents a + , _cdClass :: NonEmpty Whitespace + , _cdName :: Ident v a + , _cdArguments :: Maybe ([Whitespace], Maybe (CommaSep1' (Arg v a)), [Whitespace]) + , _cdBody :: Suite v a + } deriving (Eq, Show) +makeLenses ''ClassDef + +data With v a + = MkWith + { _withAnn :: a + , _withIndents :: Indents a + , _withAsync :: Maybe (NonEmpty Whitespace) + , _withWith :: [Whitespace] + , _withItems :: CommaSep1 (WithItem v a) + , _withBody :: Suite v a + } deriving (Eq, Show) +makeLenses ''With + +data Tuple v a + = MkTuple + { _tupleAnn :: a + , _tupleHead :: TupleItem v a + , _tupleComma :: Comma + , _tupleTail :: Maybe (CommaSep1' (TupleItem v a)) + } deriving (Eq, Show) +makeLenses ''Tuple + +data List v a + = MkList + { _listAnn :: a + , _listWhitespaceLeft :: [Whitespace] + , _listBody :: Maybe (CommaSep1' (ListItem v a)) + , _listWhitespaceRight :: [Whitespace] + } deriving (Eq, Show) +makeLenses ''List + +data ListUnpack v a + = MkListUnpack + { _listUnpackAnn :: a + , _listUnpackParens :: [([Whitespace], [Whitespace])] + , _listUnpackWhitespace :: [Whitespace] + , _listUnpackValue :: Expr v a + } deriving (Eq, Show) +makeLenses ''ListUnpack + +data None (v :: [*]) a + = MkNone + { _noneAnn :: a + , _noneWhitespace :: [Whitespace] + } deriving (Eq, Show) +makeLenses ''None + +data TupleUnpack v a + = MkTupleUnpack + { _tupleUnpackAnn :: a + , _tupleUnpackParens :: [([Whitespace], [Whitespace])] + , _tupleUnpackWhitespace :: [Whitespace] + , _tupleUnpackValue :: Expr v a + } deriving (Eq, Show) +makeLenses ''TupleUnpack diff --git a/src/Language/Python/Syntax/Whitespace.hs b/src/Language/Python/Syntax/Whitespace.hs new file mode 100644 --- /dev/null +++ b/src/Language/Python/Syntax/Whitespace.hs @@ -0,0 +1,205 @@ +{-# language DataKinds #-} +{-# language GeneralizedNewtypeDeriving, MultiParamTypeClasses, BangPatterns #-} +{-# language TypeFamilies #-} +{-# language DeriveFunctor, DeriveFoldable, DeriveTraversable #-} +{-# language TemplateHaskell #-} + +{-| +Module : Language.Python.Syntax.Whitespace +Copyright : (C) CSIRO 2017-2018 +License : BSD3 +Maintainer : Isaac Elliott +Stability : experimental +Portability : non-portable +-} + +module Language.Python.Syntax.Whitespace + ( -- * Whitespace + Newline(..) + , Whitespace(..) + , Blank(..) + , HasTrailingWhitespace(..) + , HasTrailingNewline(..) + -- * Indentation + , IndentLevel, getIndentLevel, indentLevel, absoluteIndentLevel + , Indent(..), indentWhitespaces, indentIt, dedentIt + , Indents(..), indentsValue, indentsAnn, subtractStart + ) +where + +import Control.Lens.Iso (Iso', iso, from) +import Control.Lens.Getter ((^.), view) +import Control.Lens.Lens (Lens', lens) +import Control.Lens.Setter ((.~)) +import Control.Lens.TH (makeLenses) +import Control.Lens.Traversal (Traversal') +import Data.Deriving (deriveEq1, deriveOrd1) +import Data.Foldable (toList) +import Data.Function ((&)) +import Data.FingerTree (FingerTree, Measured(..), fromList) +import Data.List (stripPrefix) +import Data.List.NonEmpty (NonEmpty(..)) +import Data.Monoid (Monoid, Endo(..), Dual(..)) +import Data.Semigroup (Semigroup, (<>)) +import GHC.Exts (IsList(..)) + +import qualified Data.List.NonEmpty as NonEmpty + +import Language.Python.Syntax.Comment (Comment) + +-- | A newline is either a carriage return, a line feed, or a carriage return +-- followed by a line feed. +data Newline = CR | LF | CRLF deriving (Eq, Ord, Show) + +-- | Whitespace is either a space, a tab, a newline that continues the +-- logical line ('Continued'), a newline that ends the logical line ('Newline'), +-- or a 'Comment'. +-- +-- Despite not literally being whitespace, comments inside enclosed forms +-- are treated as whitespace. See +-- +-- Example and counterexample of comments as whitespace +-- +-- @ +--( 1 + +-- # here's a comment +-- 2 + +-- 3 # another comment +--) +-- @ +-- +-- @ +-- x = 5 + 5 +-- # this line is not considered whitespace +-- y = x * 2 +-- @ +-- +-- @ +-- [ 1 +-- , 2 # I'm whitespace +-- , 3 +-- # also whitespace +-- ] +-- @ +data Whitespace + = Space + | Tab + | Continued Newline [Whitespace] + | Newline Newline + | Comment (Comment ()) + deriving (Eq, Ord, Show) + +-- | Every syntactic element contains the whitespace that immediately follows it. +-- +-- This type class lets us access this trailing whitespace in many different +-- types throughout hpython. +class HasTrailingWhitespace s where + trailingWhitespace :: Lens' s [Whitespace] + +instance HasTrailingWhitespace a => HasTrailingWhitespace (NonEmpty a) where + trailingWhitespace = + lens + (view trailingWhitespace . NonEmpty.last) + (\(x :| xs) ws -> + case xs of + [] -> (x & trailingWhitespace .~ ws) :| xs + x' : xs' -> NonEmpty.cons x $ (x' :| xs') & trailingWhitespace .~ ws) + +-- | A statement-containing thing may have a trailing newline +-- +-- Some forms /always/ have a trailing newline, which is why this class isn't just +-- @trailingNewline :: 'Lens'' (s v a) ('Maybe' 'Newline')@ +class HasTrailingNewline (s :: [*] -> * -> *) where + trailingNewline :: Traversal' (s v a) Newline + setTrailingNewline :: s v a -> Newline -> s v a + +-- | Lines which are "blank", meaning that they contain, if anything, only +-- whitespace and/or a comment. +data Blank a + = Blank + { _blankAnn :: a + , _blankWhitespaces :: [Whitespace] + , _blankComment :: Maybe (Comment a) + } deriving (Eq, Show, Functor, Foldable, Traversable) + +-- | Python has rules regarding the expansion of tabs into spaces and how to +-- go about computing indentation after this is done. +-- +-- See +-- +-- This data structure implements those rules as a monoid. +newtype IndentLevel + = IndentLevel + { appIndentLevel + :: Maybe Int -> Dual (Endo (Bool, Int)) + } + deriving (Semigroup, Monoid) + +indentLevel :: Indent -> Int +indentLevel = getIndentLevel . measure . unIndent + +getIndentLevel :: IndentLevel -> Int +getIndentLevel il = + snd $ + appEndo (getDual (appIndentLevel il Nothing)) (False, 0) + +absoluteIndentLevel :: Int -> Indent -> Int +absoluteIndentLevel n il = + snd $ + appEndo (getDual (appIndentLevel (measure $ unIndent il) $ Just n)) (False, 0) + +instance Measured IndentLevel Whitespace where + measure e = + IndentLevel $ + \absolute -> Dual . Endo $ + \(b, !i) -> + case e of + Space -> (b, if b then i else i+1) + Tab -> (b, if b then i else maybe (i + 8 - rem i 8) (+i) absolute) + Continued{} -> (True, i) + Newline{} -> error "Newline does not have an IndentLevel" + Comment{} -> error "Comment does not have an IndentLevel" + +newtype Indent + = MkIndent + { unIndent :: FingerTree IndentLevel Whitespace + } deriving (Eq, Ord, Show, Semigroup, Monoid) + +instance IsList Indent where + type Item Indent = Whitespace + toList = view indentWhitespaces + fromList = view $ from indentWhitespaces + +-- | Indent some indentation by a chunk +indentIt :: [Whitespace] -> Indents a -> Indents a +indentIt ws (Indents a b) = Indents (ws ^. from indentWhitespaces : a) b + +-- | Deent some indentation by a chunk +dedentIt :: Indents a -> Indents a +dedentIt i@(Indents [] _) = i +dedentIt (Indents (_:b) c) = Indents b c + +-- | An 'Indent' is isomorphic to a list of 'Whitespace' +indentWhitespaces :: Iso' Indent [Whitespace] +indentWhitespaces = + iso (Data.Foldable.toList . unIndent) (MkIndent . Data.FingerTree.fromList) + +-- | Subtract the first argument from the beginning of the second +-- +-- Returns 'Nothing' if the first list is not a prefix of the second. +subtractStart :: Indents a -> Indents a -> Maybe (Indents a) +subtractStart (Indents a _) (Indents b c) = Indents <$> stripPrefix a b <*> pure c + +-- | A possibly annotated list of 'Indent's. +data Indents a + = Indents + { _indentsValue :: [Indent] + , _indentsAnn :: a + } deriving (Eq, Show, Functor, Foldable, Traversable) + +instance Semigroup a => Semigroup (Indents a) where + Indents a b <> Indents c d = Indents (a <> c) (b <> d) + +makeLenses ''Indents +deriveEq1 ''Indents +deriveOrd1 ''Indents diff --git a/src/Language/Python/Validate.hs b/src/Language/Python/Validate.hs new file mode 100644 --- /dev/null +++ b/src/Language/Python/Validate.hs @@ -0,0 +1,88 @@ +{-# language DataKinds, TypeOperators #-} +{-# language FlexibleContexts #-} +{-| +Module : Language.Python.Validate +Copyright : (C) CSIRO 2017-2018 +License : BSD3 +Maintainer : Isaac Elliott +Stability : experimental +Portability : non-portable +-} + +module Language.Python.Validate + ( module Data.Validation + , module Language.Python.Validate.Error + , module Language.Python.Validate.Indentation + , module Language.Python.Validate.Scope + , module Language.Python.Validate.Syntax + , validateModuleAll + , validateStatementAll + , validateExprAll + , validateAll + ) +where + +import Data.List.NonEmpty (NonEmpty) +import Data.Validation + +import Language.Python.Syntax +import Language.Python.Validate.Error +import Language.Python.Validate.Indentation +import Language.Python.Validate.Scope +import Language.Python.Validate.Syntax + +validateModuleAll + :: ( AsIndentationError e a + , AsSyntaxError e a + , AsScopeError e a + ) + => Module '[] a -- ^ 'Module' to validate + -> Validation (NonEmpty e) (Module '[Scope, Syntax, Indentation] a) +validateModuleAll = + validateAll validateModuleIndentation validateModuleSyntax validateModuleScope + +validateStatementAll + :: ( AsIndentationError e a + , AsSyntaxError e a + , AsScopeError e a + ) + => Statement '[] a -- ^ 'Statement' to validate + -> Validation (NonEmpty e) (Statement '[Scope, Syntax, Indentation] a) +validateStatementAll = + validateAll validateStatementIndentation validateStatementSyntax validateStatementScope + +validateExprAll + :: ( AsIndentationError e a + , AsSyntaxError e a + , AsScopeError e a + ) + => Expr '[] a -- ^ 'Expr' to validate + -> Validation (NonEmpty e) (Expr '[Scope, Syntax, Indentation] a) +validateExprAll = + validateAll validateExprIndentation validateExprSyntax validateExprScope + +-- | Validate a datatype for indentation, syntax, and scope correctness +-- +-- e.g. +-- +-- @ +-- 'validateModuleAll' = +-- 'validateAll' +-- 'validateModuleIndentation' +-- 'validateModuleSyntax' +-- 'validateModuleScope' +-- @ +validateAll + :: ( AsIndentationError e a + , AsSyntaxError e a + , AsScopeError e a + ) + => (s '[] a -> ValidateIndentation e (s '[Indentation] a)) -- ^ Indentation validator + -> (s '[Indentation] a -> ValidateSyntax e (s '[Syntax, Indentation] a)) -- ^ Syntax validator + -> (s '[Syntax, Indentation] a -> ValidateScope a e (s '[Scope, Syntax, Indentation] a)) -- ^ Scope validator + -> s '[] a + -> Validation (NonEmpty e) (s '[Scope, Syntax, Indentation] a) +validateAll vi vsyn vsco m = + runValidateIndentation (vi m) `bindValidation` \m' -> + runValidateSyntax (vsyn m') `bindValidation` \m'' -> + runValidateScope (vsco m'') diff --git a/src/Language/Python/Validate/Error.hs b/src/Language/Python/Validate/Error.hs new file mode 100644 --- /dev/null +++ b/src/Language/Python/Validate/Error.hs @@ -0,0 +1,41 @@ +{-# language FlexibleInstances, MultiParamTypeClasses #-} +{-# language LambdaCase #-} +module Language.Python.Validate.Error + ( module Language.Python.Validate.Indentation.Error + , module Language.Python.Validate.Scope.Error + , module Language.Python.Validate.Syntax.Error + , ValidationError(..) + ) +where + +import Control.Lens.Prism (prism') +import Language.Python.Validate.Indentation.Error +import Language.Python.Validate.Scope.Error +import Language.Python.Validate.Syntax.Error + +data ValidationError a + = IndentationError (IndentationError a) + | ScopeError (ScopeError a) + | SyntaxError (SyntaxError a) + deriving (Eq, Show) + +instance AsTabError (ValidationError a) a where + _TabError = _IndentationError._TabError + +instance AsIndentationError (ValidationError a) a where + _IndentationError = + prism' + IndentationError + (\case; IndentationError a -> Just a; _ -> Nothing) + +instance AsScopeError (ValidationError a) a where + _ScopeError = + prism' + ScopeError + (\case; ScopeError a -> Just a; _ -> Nothing) + +instance AsSyntaxError (ValidationError a) a where + _SyntaxError = + prism' + SyntaxError + (\case; SyntaxError a -> Just a; _ -> Nothing) diff --git a/src/Language/Python/Validate/Indentation.hs b/src/Language/Python/Validate/Indentation.hs new file mode 100644 --- /dev/null +++ b/src/Language/Python/Validate/Indentation.hs @@ -0,0 +1,397 @@ +{-# language DataKinds, TypeOperators #-} +{-# language ScopedTypeVariables, TypeApplications #-} +{-# language LambdaCase #-} + +{-| +Module : Language.Python.Validate.Indentation +Copyright : (C) CSIRO 2017-2018 +License : BSD3 +Maintainer : Isaac Elliott +Stability : experimental +Portability : non-portable +-} + +module Language.Python.Validate.Indentation + ( module Data.Validation + , module Language.Python.Validate.Indentation.Error + -- * Main validation functions + , Indentation, ValidateIndentation, runValidateIndentation + , validateModuleIndentation + , validateStatementIndentation + , validateExprIndentation + -- * Miscellany + -- ** Extra types + , NextIndent(..) + -- ** Extra functions + , equivalentIndentation + , runValidateIndentation' + -- ** Validation functions + , validateArgsIndentation + , validateBlockIndentation + , validateCompoundStatementIndentation + , validateDecoratorIndentation + , validateExceptAsIndentation + , validateParamsIndentation + , validateSuiteIndentation + ) +where + +import Data.Validation + +import Control.Lens.Fold ((^?!), folded) +import Control.Lens.Getter ((^.)) +import Control.Lens.Prism (_Right) +import Control.Lens.Review ((#)) +import Control.Lens.Setter (over, mapped) +import Control.Lens.Traversal (traverseOf) +import Control.Lens.Tuple (_1, _2) +import Control.Monad.State (State, evalState, get, put) +import Data.Coerce (coerce) +import Data.Foldable (fold) +import Data.Functor.Compose (Compose(..)) +import Data.List.NonEmpty (NonEmpty(..)) +import Data.Type.Set +import Unsafe.Coerce (unsafeCoerce) +import Data.Validate.Monadic (ValidateM(..), liftVM0, errorVM, errorVM1) +import qualified Data.List.NonEmpty as NonEmpty + +import Language.Python.Optics +import Language.Python.Optics.Validated (unvalidated) +import Language.Python.Syntax.CommaSep +import Language.Python.Syntax.Module +import Language.Python.Syntax.Expr +import Language.Python.Syntax.Statement +import Language.Python.Syntax.Whitespace +import Language.Python.Validate.Indentation.Error + +data Indentation + +-- | "The next line must be..." +data NextIndent + = GreaterThan + | EqualTo + deriving (Eq, Show) + +type ValidateIndentation e = ValidateM (NonEmpty e) (State (NextIndent, [Indent])) + +runValidateIndentation :: ValidateIndentation e a -> Validation (NonEmpty e) a +runValidateIndentation = runValidateIndentation' EqualTo [] + +runValidateIndentation' :: NextIndent -> [Indent] -> ValidateIndentation e a -> Validation (NonEmpty e) a +runValidateIndentation' ni is = + flip evalState (ni, is) . + getCompose . + unValidateM + +withNextIndent + :: (NextIndent -> [Indent] -> ValidateIndentation e a) + -> ValidateIndentation e a +withNextIndent f = + ValidateM . Compose $ + get >>= getCompose . unValidateM . uncurry f + +checkIndent :: AsIndentationError e a => Indents a -> ValidateIndentation e (Indents a) +checkIndent i = + withNextIndent $ \ni i' -> + let + a = i ^. indentsAnn + ii = fold (i ^. indentsValue) + ii' = fold i' + absolute1Comparison = compare (absoluteIndentLevel 1 ii) (absoluteIndentLevel 1 ii') + absolute8Comparison = compare (absoluteIndentLevel 8 ii) (absoluteIndentLevel 8 ii') + in + case ni of + GreaterThan -> + case (absolute1Comparison, absolute8Comparison) of + (GT, GT) -> pure i + (GT, _) -> errorVM $ pure (_TabError # a) + (_, GT) -> errorVM $ pure (_TabError # a) + (EQ, EQ) -> errorVM $ pure (_ExpectedGreaterThan # (i', i)) + (_, EQ) -> errorVM $ pure (_TabError # a) + (EQ, _) -> errorVM $ pure (_TabError # a) + (LT, LT) -> errorVM $ pure (_ExpectedGreaterThan # (i', i)) + EqualTo -> + case (absolute1Comparison, absolute8Comparison) of + (EQ, EQ) -> pure i + (EQ, _) -> errorVM $ pure (_TabError # a) + (_, EQ) -> errorVM $ pure (_TabError # a) + (GT, GT) -> errorVM $ pure (_ExpectedEqualTo # (i', i)) + (_, GT) -> errorVM $ pure (_TabError # a) + (GT, _) -> errorVM $ pure (_TabError # a) + (LT, LT) -> errorVM $ pure (_ExpectedEqualTo # (i', i)) + +setNextIndent :: NextIndent -> [Indent] -> ValidateIndentation e () +setNextIndent ni is = liftVM0 $ put (ni, is) + +equivalentIndentation :: [Whitespace] -> [Whitespace] -> Bool +equivalentIndentation [] [] = True +equivalentIndentation (x:_) [] = + case x of + Continued _ _ -> True + _ -> False +equivalentIndentation [] (y:_) = + case y of + Continued _ _ -> True + _ -> False +equivalentIndentation (x:xs) (y:ys) = + case (x, y) of + (Space, Space) -> equivalentIndentation xs ys + (Tab, Tab) -> equivalentIndentation xs ys + (Continued _ _, Continued _ _) -> True + _ -> False + +validateBlankIndentation + :: forall e a. + AsIndentationError e a + => Blank a + -> ValidateIndentation e (Blank a) +validateBlankIndentation (Blank a ws cmt) = + if any (\case; Continued{} -> True; _ -> False) ws + then errorVM1 $ _EmptyContinuedLine # a + else pure $ Blank a ws cmt + +validateBlockIndentation + :: forall e v a. + AsIndentationError e a + => Block v a + -> ValidateIndentation e (Block (Nub (Indentation ': v)) a) +validateBlockIndentation (Block x b bs) = + (\x' (b' :| bs') -> + case b' of + Right b'' -> Block x' b'' bs' + _ -> error "impossible") <$> + traverseOf (traverse._1) validateBlankIndentation x <*> + go False (Right b) bs + where + is = (Right b:|bs) ^?! folded._Right.unvalidated._Indents.indentsValue + + go + :: Bool + -> Either + (Blank a, Newline) + (Statement v a) + -> [Either (Blank a, Newline) (Statement v a)] + -> ValidateIndentation e + (NonEmpty + (Either + (Blank a, Newline) + (Statement (Nub (Indentation ': v)) a))) + go flag (Left e) rest = + case rest of + [] -> + pure . Left <$> + traverseOf _1 validateBlankIndentation e + r : rs -> + NonEmpty.cons . Left <$> + traverseOf _1 validateBlankIndentation e <*> + go flag r rs + go flag (Right st) rest = + let + validated = + Right <$ + (if flag then setNextIndent EqualTo is else pure ()) <*> + validateStatementIndentation st + in + case rest of + [] -> (:| []) <$> validated + r : rs -> NonEmpty.cons <$> validated <*> go True r rs + +validateSuiteIndentation + :: AsIndentationError e a + => Indents a + -> Suite v a + -> ValidateIndentation e (Suite (Nub (Indentation ': v)) a) +validateSuiteIndentation idnt (SuiteMany ann a b c d) = + SuiteMany ann a b c <$ + setNextIndent GreaterThan (idnt ^. indentsValue) <*> + validateBlockIndentation d +validateSuiteIndentation _ (SuiteOne ann a b) = + SuiteOne ann a <$> validateSmallStatementIndentation b + +validateExprIndentation + :: AsIndentationError e a + => Expr v a + -> ValidateIndentation e (Expr (Nub (Indentation ': v)) a) +validateExprIndentation e = pure $ unsafeCoerce e + +validateParamsIndentation + :: AsIndentationError e a + => CommaSep (Param v a) + -> ValidateIndentation e (CommaSep (Param (Nub (Indentation ': v)) a)) +validateParamsIndentation e = pure $ unsafeCoerce e + +validateArgsIndentation + :: AsIndentationError e a + => CommaSep (Arg v a) + -> ValidateIndentation e (CommaSep (Arg (Nub (Indentation ': v)) a)) +validateArgsIndentation e = pure $ unsafeCoerce e + +validateExceptAsIndentation + :: AsIndentationError e a + => ExceptAs v a + -> ValidateIndentation e (ExceptAs (Nub (Indentation ': v)) a) +validateExceptAsIndentation (ExceptAs ann e f) = + ExceptAs ann <$> + validateExprIndentation e <*> + pure (over (traverse._2) coerce f) + +validateDecoratorIndentation + :: AsIndentationError e a + => Decorator v a + -> ValidateIndentation e (Decorator (Nub (Indentation ': v)) a) +validateDecoratorIndentation (Decorator a b c d e f g) = + (\b' -> Decorator a b' c (unsafeCoerce d) e f) <$> + checkIndent b <*> + traverseOf (traverse._1) validateBlankIndentation g + +validateCompoundStatementIndentation + :: forall e v a + . AsIndentationError e a + => CompoundStatement v a + -> ValidateIndentation e (CompoundStatement (Nub (Indentation ': v)) a) +validateCompoundStatementIndentation (Fundef a decos idnt asyncWs ws1 name ws2 params ws3 mty s) = + (\decos' idnt' params' -> + Fundef a decos' idnt' asyncWs ws1 (coerce name) ws2 params' ws3 (unsafeCoerce mty)) <$> + traverse validateDecoratorIndentation decos <*> + checkIndent idnt <*> + validateParamsIndentation params <*> + validateSuiteIndentation idnt s +validateCompoundStatementIndentation (If a idnt ws1 expr s elifs body1) = + (\idnt' -> If a idnt' ws1) <$> + checkIndent idnt <*> + validateExprIndentation expr <*> + validateSuiteIndentation idnt s <*> + traverse + (\(idnt2, a, b, c) -> + (,,,) <$ + setNextIndent EqualTo (idnt ^. indentsValue) <*> + checkIndent idnt2 <*> + pure a <*> + validateExprIndentation b <*> + validateSuiteIndentation idnt c) + elifs <*> + traverse + (\(idnt2, a, b) -> + (,,) <$ + setNextIndent EqualTo (idnt ^. indentsValue) <*> + checkIndent idnt2 <*> + pure a <*> + validateSuiteIndentation idnt b) + body1 +validateCompoundStatementIndentation (While a idnt ws1 expr s els) = + (\idnt' expr' -> While a idnt' ws1 expr') <$> + checkIndent idnt <*> + validateExprIndentation expr <*> + validateSuiteIndentation idnt s <*> + traverse + (\(idnt2, a, b) -> + (,,) <$ + setNextIndent EqualTo (idnt ^. indentsValue) <*> + checkIndent idnt2 <*> + pure a <*> + validateSuiteIndentation idnt b) + els +validateCompoundStatementIndentation (TryExcept a idnt b c d e f) = + (\idnt' -> TryExcept a idnt' b) <$> + checkIndent idnt <*> + validateSuiteIndentation idnt c <*> + traverse + (\(a, b, c, d) -> + (\a' -> (,,,) a' b) <$ + setNextIndent EqualTo (idnt ^. indentsValue) <*> + checkIndent a <*> + traverse validateExceptAsIndentation c <*> + validateSuiteIndentation idnt d) + d <* + setNextIndent EqualTo (idnt ^. indentsValue) <*> + traverse + (\(idnt2, a, b) -> + (\idnt2' -> (,,) idnt2' a) <$ + setNextIndent EqualTo (idnt ^. indentsValue) <*> + checkIndent idnt2 <*> + validateSuiteIndentation idnt b) + e <* + setNextIndent EqualTo (idnt ^. indentsValue) <*> + traverse + (\(idnt2, a, b) -> + (\idnt2' -> (,,) idnt2' a) <$ + setNextIndent EqualTo (idnt ^. indentsValue) <*> + checkIndent idnt2 <*> + validateSuiteIndentation idnt b) + f +validateCompoundStatementIndentation (TryFinally a idnt b c idnt2 d e) = + (\idnt' c' idnt2' -> TryFinally a idnt' b c' idnt2' d) <$> + checkIndent idnt <*> + validateSuiteIndentation idnt c <* + setNextIndent EqualTo (idnt ^. indentsValue) <*> + checkIndent idnt2 <*> + validateSuiteIndentation idnt e +validateCompoundStatementIndentation (For a idnt asyncWs b c d e h i) = + (\idnt' c' -> For a idnt' asyncWs b c' d) <$> + checkIndent idnt <*> + validateExprIndentation c <*> + traverse validateExprIndentation e <*> + validateSuiteIndentation idnt h <* + setNextIndent EqualTo (idnt ^. indentsValue) <*> + traverse + (\(idnt2, a, b) -> + (\idnt2' -> (,,) idnt2' a) <$ + setNextIndent EqualTo (idnt ^. indentsValue) <*> + checkIndent idnt2 <*> + validateSuiteIndentation idnt b) + i +validateCompoundStatementIndentation (ClassDef a decos idnt b c d e) = + (\decos' idnt' -> + ClassDef @(Nub (Indentation ': v)) a decos' idnt' b (coerce c) (unsafeCoerce d)) <$> + traverse validateDecoratorIndentation decos <*> + checkIndent idnt <*> + validateSuiteIndentation idnt e +validateCompoundStatementIndentation (With a idnt asyncWs b c d) = + (\idnt' -> With @(Nub (Indentation ': v)) a idnt' asyncWs b) <$> + checkIndent idnt <*> + traverse validateWithItemIndentation c <*> + validateSuiteIndentation idnt d + +validateWithItemIndentation + :: AsIndentationError e a + => WithItem v a + -> ValidateIndentation e (WithItem (Nub (Indentation ': v)) a) +validateWithItemIndentation a = pure $ unsafeCoerce a + +validateSmallStatementIndentation + :: AsIndentationError e a + => SmallStatement v a + -> ValidateIndentation e (SmallStatement (Nub (Indentation ': v)) a) +validateSmallStatementIndentation (MkSmallStatement a b c d e) = + pure $ MkSmallStatement (unsafeCoerce a) (over (mapped._2) unsafeCoerce b) c d e + +validateStatementIndentation + :: AsIndentationError e a + => Statement v a + -> ValidateIndentation e (Statement (Nub (Indentation ': v)) a) +validateStatementIndentation (CompoundStatement c) = + CompoundStatement <$> validateCompoundStatementIndentation c +validateStatementIndentation (SmallStatement idnt a) = + SmallStatement <$> + checkIndent idnt <*> + validateSmallStatementIndentation a + +validateModuleIndentation + :: AsIndentationError e a + => Module v a + -> ValidateIndentation e (Module (Nub (Indentation ': v)) a) +validateModuleIndentation m = + case m of + ModuleEmpty -> pure ModuleEmpty + ModuleBlankFinal a -> + ModuleBlankFinal <$> + validateBlankIndentation a + ModuleBlank a b c -> + (\a' -> ModuleBlank a' b) <$> + validateBlankIndentation a <*> + validateModuleIndentation c + ModuleStatement a b -> + ModuleStatement <$ + setNextIndent EqualTo [] <*> + validateStatementIndentation a <*> + validateModuleIndentation b diff --git a/src/Language/Python/Validate/Indentation/Error.hs b/src/Language/Python/Validate/Indentation/Error.hs new file mode 100644 --- /dev/null +++ b/src/Language/Python/Validate/Indentation/Error.hs @@ -0,0 +1,74 @@ +{-# language LambdaCase #-} +{-# language TemplateHaskell, MultiParamTypeClasses, FunctionalDependencies, + FlexibleInstances, DataKinds, KindSignatures #-} + +{-| +Module : Language.Python.Validate.Indentation.Error +Copyright : (C) CSIRO 2017-2018 +License : BSD3 +Maintainer : Isaac Elliott +Stability : experimental +Portability : non-portable +-} + +module Language.Python.Validate.Indentation.Error + ( IndentationError(..) + , AsTabError(..) + , AsIndentationError(..) + ) +where + +import Control.Lens.Prism (Prism', prism') + +import Language.Python.Internal.Lexer (AsTabError(..)) +import Language.Python.Syntax.Whitespace + + +data IndentationError a + = IndentationTabError a + | ExpectedGreaterThan [Indent] (Indents a) + | ExpectedEqualTo [Indent] (Indents a) + | EmptyContinuedLine a + deriving (Eq, Show) + +class AsTabError s a => AsIndentationError s a | s -> a where + _IndentationError :: Prism' s (IndentationError a) + + _ExpectedGreaterThan :: Prism' s ([Indent], Indents a) + _ExpectedGreaterThan = _IndentationError._ExpectedGreaterThan + + _ExpectedEqualTo :: Prism' s ([Indent], Indents a) + _ExpectedEqualTo = _IndentationError._ExpectedEqualTo + + _EmptyContinuedLine :: Prism' s a + _EmptyContinuedLine = _IndentationError._EmptyContinuedLine + +instance AsTabError (IndentationError a) a where + _TabError = + prism' + IndentationTabError + (\case + IndentationTabError a -> Just a + _ -> Nothing) + +instance AsIndentationError (IndentationError a) a where + _IndentationError = id + + _ExpectedGreaterThan = + prism' + (uncurry ExpectedGreaterThan) + (\case + ExpectedGreaterThan a b -> Just (a, b) + _ -> Nothing) + _ExpectedEqualTo = + prism' + (uncurry ExpectedEqualTo) + (\case + ExpectedEqualTo a b -> Just (a, b) + _ -> Nothing) + _EmptyContinuedLine = + prism' + EmptyContinuedLine + (\case + EmptyContinuedLine a -> Just a + _ -> Nothing) diff --git a/src/Language/Python/Validate/Scope.hs b/src/Language/Python/Validate/Scope.hs new file mode 100644 --- /dev/null +++ b/src/Language/Python/Validate/Scope.hs @@ -0,0 +1,675 @@ +{-# language DataKinds, TypeOperators #-} +{-# language GeneralizedNewtypeDeriving #-} +{-# language TemplateHaskell, TypeFamilies, FlexibleInstances, MultiParamTypeClasses #-} +{-# language FlexibleContexts #-} +{-# language RankNTypes #-} +{-# language LambdaCase #-} +{-# language ScopedTypeVariables, TypeApplications #-} + +{-| +Module : Language.Python.Validate.Scope +Copyright : (C) CSIRO 2017-2018 +License : BSD3 +Maintainer : Isaac Elliott +Stability : experimental +Portability : non-portable +-} + +module Language.Python.Validate.Scope + ( module Data.Validation + , module Language.Python.Validate.Scope.Error + -- * Main validation functions + , Scope, ValidateScope, runValidateScope + , validateModuleScope + , validateStatementScope + , validateExprScope + -- * Miscellany + -- ** Extra types + , ScopeContext(..), scGlobalScope, scLocalScope, scImmediateScope + , runValidateScope' + , initialScopeContext + , Binding(..) + -- ** Extra functions + , inScope + , extendScope + , locallyOver + , locallyExtendOver + -- ** Validation functions + , validateArgScope + , validateAssignExprScope + , validateBlockScope + , validateCompoundStatementScope + , validateComprehensionScope + , validateDecoratorScope + , validateDictItemScope + , validateExceptAsScope + , validateIdentScope + , validateListItemScope + , validateParamScope + , validateSetItemScope + , validateSimpleStatementScope + , validateSubscriptScope + , validateSuiteScope + , validateTupleItemScope + ) +where + +import Data.Validation + +import Control.Arrow ((&&&)) +import Control.Applicative ((<|>)) +import Control.Lens.Cons (snoc) +import Control.Lens.Fold ((^..), toListOf, folded) +import Control.Lens.Getter ((^.), to, getting, use) +import Control.Lens.Lens (Lens') +import Control.Lens.Plated (cosmos) +import Control.Lens.Prism (_Right, _Just) +import Control.Lens.Review ((#)) +import Control.Lens.Setter ((%~), (.~), Setter', mapped, over) +import Control.Lens.TH (makeLenses) +import Control.Lens.Tuple (_2, _3) +import Control.Lens.Traversal (traverseOf) +import Control.Monad.State (MonadState, State, evalState, modify) +import Data.Bitraversable (bitraverse) +import Data.ByteString (ByteString) +import Data.Coerce (coerce) +import Data.Foldable (traverse_) +import Data.Functor.Compose (Compose(..)) +import Data.List.NonEmpty (NonEmpty(..)) +import Data.Map.Strict (Map) +import Data.String (fromString) +import Data.Type.Set (Nub) +import Data.Validate.Monadic (ValidateM(..), runValidateM, bindVM, liftVM0, errorVM1) +import Unsafe.Coerce (unsafeCoerce) + +import qualified Data.List.NonEmpty as NonEmpty +import qualified Data.Map.Strict as Map + +import Language.Python.Optics +import Language.Python.Optics.Validated (unvalidated) +import Language.Python.Syntax.Statement +import Language.Python.Syntax.Expr +import Language.Python.Syntax.Ident +import Language.Python.Syntax.Module +import Language.Python.Validate.Scope.Error + +data Scope + +data Binding = Clean | Dirty + deriving (Eq, Ord, Show) + +data ScopeContext a + = ScopeContext + { _scGlobalScope :: !(Map ByteString a) + , _scLocalScope :: !(Map ByteString a) + , _scImmediateScope :: !(Map ByteString a) + } + deriving (Eq, Show) +makeLenses ''ScopeContext + +initialScopeContext :: ScopeContext a +initialScopeContext = ScopeContext Map.empty Map.empty Map.empty + +type ValidateScope ann e = ValidateM (NonEmpty e) (State (ScopeContext ann)) + +runValidateScope :: ValidateScope ann e a -> Validation (NonEmpty e) a +runValidateScope = runValidateScope' initialScopeContext + +runValidateScope' :: ScopeContext ann -> ValidateScope ann e a -> Validation (NonEmpty e) a +runValidateScope' s = flip evalState s . runValidateM + +extendScope + :: Setter' (ScopeContext ann) (Map ByteString ann) + -> [(ann, String)] + -> ValidateScope ann e () +extendScope l s = + liftVM0 $ do + gs <- use scGlobalScope + let t = buildMap gs Map.empty + modify (over l (t `unionL`)) + where + buildMap gs t = + foldr + (\(ann, a) b -> + let + a' = fromString a + in + if Map.member a' gs + then b + else Map.insert a' ann b) + t + s + +locallyOver + :: Lens' (ScopeContext ann) b + -> (b -> b) + -> ValidateScope ann e a + -> ValidateScope ann e a +locallyOver l f m = + ValidateM . Compose $ do + before <- use l + modify (l %~ f) + getCompose (unValidateM m) <* modify (l .~ before) + +locallyExtendOver + :: Lens' (ScopeContext ann) (Map ByteString ann) + -> [(ann, String)] + -> ValidateScope ann e a + -> ValidateScope ann e a +locallyExtendOver l s m = locallyOver l id $ extendScope l s *> m + +inScope + :: MonadState (ScopeContext ann) m + => String + -> m (Maybe (Binding, ann)) +inScope s = do + gs <- use scGlobalScope + ls <- use scLocalScope + is <- use scImmediateScope + let + s' = fromString s + inls = Map.lookup s' ls + ings = Map.lookup s' gs + pure $ + ((,) Clean <$> Map.lookup s' is) <|> + (ings *> ((,) Clean <$> inls)) <|> + ((,) Clean <$> ings) <|> + ((,) Dirty <$> inls) + +validateExceptAsScope + :: AsScopeError e a + => ExceptAs v a + -> ValidateScope a e (ExceptAs (Nub (Scope ': v)) a) +validateExceptAsScope (ExceptAs ann e f) = + ExceptAs ann <$> + validateExprScope e <*> + pure (over (mapped._2) coerce f) + +validateSuiteScope + :: AsScopeError e a + => Suite v a + -> ValidateScope a e (Suite (Nub (Scope ': v)) a) +validateSuiteScope (SuiteMany ann a b c d) = SuiteMany ann a b c <$> validateBlockScope d +validateSuiteScope (SuiteOne ann a b) = + SuiteOne ann a <$> validateSmallStatementScope b + +validateDecoratorScope + :: AsScopeError e a + => Decorator v a + -> ValidateScope a e (Decorator (Nub (Scope ': v)) a) +validateDecoratorScope (Decorator a b c d e f g) = + (\d' -> Decorator a b c d' e f g) <$> + validateExprScope d + +validateCompoundStatementScope + :: forall e v a + . AsScopeError e a + => CompoundStatement v a + -> ValidateScope a e (CompoundStatement (Nub (Scope ': v)) a) +validateCompoundStatementScope (Fundef a decos idnts asyncWs ws1 name ws2 params ws3 mty s) = + (locallyOver scLocalScope (const Map.empty) $ + locallyOver scImmediateScope (const Map.empty) $ + (\decos' -> Fundef a decos' idnts asyncWs ws1 (coerce name) ws2) <$> + traverse validateDecoratorScope decos <*> + traverse validateParamScope params <*> + pure ws3 <*> + traverseOf (traverse._2) validateExprScope mty <*> + locallyExtendOver + scGlobalScope + ((_identAnn &&& _identValue) name : + toListOf (folded.getting paramName.to (_identAnn &&& _identValue)) params) + (validateSuiteScope s)) <* + extendScope scLocalScope [(_identAnn &&& _identValue) name] <* + extendScope scImmediateScope [(_identAnn &&& _identValue) name] +validateCompoundStatementScope (If idnts a ws1 e b elifs melse) = + use scLocalScope `bindVM` (\ls -> + use scImmediateScope `bindVM` (\is -> + locallyOver scGlobalScope (`unionR` unionR ls is) $ + locallyOver scImmediateScope (const Map.empty) + (If idnts a ws1 <$> + validateExprScope e <*> + validateSuiteScope b <*> + traverse + (\(a, b, c, d) -> + (\c' -> (,,,) a b c') <$> + validateExprScope c <*> + validateSuiteScope d) + elifs <*> + traverseOf (traverse._3) validateSuiteScope melse))) +validateCompoundStatementScope (While idnts a ws1 e b els) = + use scLocalScope `bindVM` (\ls -> + use scImmediateScope `bindVM` (\is -> + locallyOver scGlobalScope (`unionR` unionR ls is) $ + locallyOver scImmediateScope (const Map.empty) + (While idnts a ws1 <$> + validateExprScope e <*> + validateSuiteScope b <*> + traverseOf (traverse._3) validateSuiteScope els))) +validateCompoundStatementScope (TryExcept idnts a b e f k l) = + use scLocalScope `bindVM` (\ls -> + use scImmediateScope `bindVM` (\is -> + locallyOver scGlobalScope (`unionR` unionR ls is) $ + locallyOver scImmediateScope (const Map.empty) + (TryExcept idnts a b <$> + validateSuiteScope e <*> + traverse + (\(idnts, ws, g, h) -> + (,,,) idnts ws <$> + traverse validateExceptAsScope g <*> + locallyExtendOver + scGlobalScope + (toListOf (folded.exceptAsName._Just._2.to (_identAnn &&& _identValue)) g) + (validateSuiteScope h)) + f <*> + traverseOf (traverse._3) validateSuiteScope k <*> + traverseOf (traverse._3) validateSuiteScope l))) +validateCompoundStatementScope (TryFinally idnts a b e idnts2 f i) = + use scLocalScope `bindVM` (\ls -> + use scImmediateScope `bindVM` (\is -> + locallyOver scGlobalScope (`unionR` unionR ls is) $ + locallyOver scImmediateScope (const Map.empty) + (TryFinally idnts a b <$> + validateSuiteScope e <*> + pure idnts2 <*> + pure f <*> + validateSuiteScope i))) +validateCompoundStatementScope (For idnts a asyncWs b c d e h i) = + use scLocalScope `bindVM` (\ls -> + use scImmediateScope `bindVM` (\is -> + locallyOver scGlobalScope (`unionR` unionR ls is) $ + locallyOver scImmediateScope (const Map.empty) $ + For @(Nub (Scope ': v)) idnts a asyncWs b <$> + (unsafeCoerce c <$ + traverse + (\s -> + inScope (s ^. identValue) `bindVM` \res -> + maybe (pure ()) (\_ -> errorVM1 (_BadShadowing # coerce s)) res) + (c ^.. unvalidated.cosmos._Ident)) <*> + pure d <*> + traverse validateExprScope e <*> + (let + ls = c ^.. unvalidated.cosmos._Ident.to (_identAnn &&& _identValue) + in + extendScope scLocalScope ls *> + extendScope scImmediateScope ls *> + validateSuiteScope h) <*> + traverseOf (traverse._3) validateSuiteScope i)) +validateCompoundStatementScope (ClassDef a decos idnts b c d g) = + (\decos' -> ClassDef @(Nub (Scope ': v)) a decos' idnts b (coerce c)) <$> + traverse validateDecoratorScope decos <*> + traverseOf (traverse._2.traverse.traverse) validateArgScope d <*> + validateSuiteScope g <* + extendScope scImmediateScope [c ^. to (_identAnn &&& _identValue)] +validateCompoundStatementScope (With a b asyncWs c d e) = + let + names = + d ^.. + folded.unvalidated.to _withItemBinder.folded._2. + assignTargets.to (_identAnn &&& _identValue) + in + With @(Nub (Scope ': v)) a b asyncWs c <$> + traverse + (\(WithItem a b c) -> + WithItem @(Nub (Scope ': v)) a <$> + validateExprScope b <*> + traverseOf (traverse._2) validateAssignExprScope c) + d <* + extendScope scLocalScope names <* + extendScope scImmediateScope names <*> + validateSuiteScope e + +validateSimpleStatementScope + :: AsScopeError e a + => SimpleStatement v a + -> ValidateScope a e (SimpleStatement (Nub (Scope ': v)) a) +validateSimpleStatementScope (Assert a b c d) = + Assert a b <$> + validateExprScope c <*> + traverseOf (traverse._2) validateExprScope d +validateSimpleStatementScope (Raise a ws f) = + Raise a ws <$> + traverse + (\(b, c) -> + (,) <$> + validateExprScope b <*> + traverseOf (traverse._2) validateExprScope c) + f +validateSimpleStatementScope (Return a ws e) = Return a ws <$> traverse validateExprScope e +validateSimpleStatementScope (Expr a e) = Expr a <$> validateExprScope e +validateSimpleStatementScope (Assign a l rs) = + let + ls = + (l : (snd <$> NonEmpty.init rs)) ^.. + folded.unvalidated.assignTargets.to (_identAnn &&& _identValue) + in + Assign a <$> + validateAssignExprScope l <*> + ((\a b -> case a of; [] -> b :| []; a : as -> a :| snoc as b) <$> + traverseOf (traverse._2) validateAssignExprScope (NonEmpty.init rs) <*> + (\(ws, b) -> (,) ws <$> validateExprScope b) (NonEmpty.last rs)) <* + extendScope scLocalScope ls <* + extendScope scImmediateScope ls +validateSimpleStatementScope (AugAssign a l aa r) = + (\l' -> AugAssign a l' aa) <$> + validateExprScope l <*> + validateExprScope r +validateSimpleStatementScope (Global a _ _) = errorVM1 (_FoundGlobal # a) +validateSimpleStatementScope (Nonlocal a _ _) = errorVM1 (_FoundNonlocal # a) +validateSimpleStatementScope (Del a ws cs) = + Del a ws <$ + traverse_ + (\case; Ident a -> errorVM1 (_DeletedIdent # (a ^. identAnn)); _ -> pure ()) + cs <*> + traverse validateExprScope cs +validateSimpleStatementScope s@Pass{} = pure $ unsafeCoerce s +validateSimpleStatementScope s@Break{} = pure $ unsafeCoerce s +validateSimpleStatementScope s@Continue{} = pure $ unsafeCoerce s +validateSimpleStatementScope s@Import{} = pure $ unsafeCoerce s +validateSimpleStatementScope s@From{} = pure $ unsafeCoerce s + +validateSmallStatementScope + :: AsScopeError e a + => SmallStatement v a + -> ValidateScope a e (SmallStatement (Nub (Scope ': v)) a) +validateSmallStatementScope (MkSmallStatement s ss sc cmt nl) = + (\s' ss' -> MkSmallStatement s' ss' sc cmt nl) <$> + validateSimpleStatementScope s <*> + traverseOf (traverse._2) validateSimpleStatementScope ss + +validateStatementScope + :: AsScopeError e a + => Statement v a + -> ValidateScope a e (Statement (Nub (Scope ': v)) a) +validateStatementScope (CompoundStatement c) = + CompoundStatement <$> validateCompoundStatementScope c +validateStatementScope (SmallStatement idnts a) = + SmallStatement idnts <$> validateSmallStatementScope a + +validateIdentScope + :: AsScopeError e a + => Ident v a + -> ValidateScope a e (Ident (Nub (Scope ': v)) a) +validateIdentScope i = + inScope (_identValue i) `bindVM` + \context -> + case context of + Just (Clean, _) -> pure $ coerce i + Just (Dirty, ann)-> errorVM1 (_FoundDynamic # (ann, i ^. unvalidated)) + Nothing -> errorVM1 (_NotInScope # (i ^. unvalidated)) + +validateArgScope + :: AsScopeError e a + => Arg v a + -> ValidateScope a e (Arg (Nub (Scope ': v)) a) +validateArgScope (PositionalArg a e) = + PositionalArg a <$> validateExprScope e +validateArgScope (KeywordArg a ident ws2 expr) = + KeywordArg a (coerce ident) ws2 <$> validateExprScope expr +validateArgScope (StarArg a ws e) = + StarArg a ws <$> validateExprScope e +validateArgScope (DoubleStarArg a ws e) = + DoubleStarArg a ws <$> validateExprScope e + +validateParamScope + :: AsScopeError e a + => Param v a + -> ValidateScope a e (Param (Nub (Scope ': v)) a) +validateParamScope (PositionalParam a ident mty) = + PositionalParam a (coerce ident) <$> + traverseOf (traverse._2) validateExprScope mty +validateParamScope (KeywordParam a ident mty ws2 expr) = + KeywordParam a (coerce ident) <$> + traverseOf (traverse._2) validateExprScope mty <*> + pure ws2 <*> + validateExprScope expr +validateParamScope (StarParam a b c d) = + StarParam a b (coerce c) <$> + traverseOf (traverse._2) validateExprScope d +validateParamScope (UnnamedStarParam a b) = pure $ UnnamedStarParam a b +validateParamScope (DoubleStarParam a b c d) = + DoubleStarParam a b (coerce c) <$> + traverseOf (traverse._2) validateExprScope d + +validateBlockScope + :: AsScopeError e a + => Block v a + -> ValidateScope a e (Block (Nub (Scope ': v)) a) +validateBlockScope (Block x b bs) = + Block x <$> + validateStatementScope b <*> + traverseOf (traverse._Right) validateStatementScope bs + +validateComprehensionScope + :: AsScopeError e a + => (ex v a -> ValidateScope a e (ex (Nub (Scope ': v)) a)) + -> Comprehension ex v a + -> ValidateScope a e (Comprehension ex (Nub (Scope ': v)) a) +validateComprehensionScope f (Comprehension a b c d) = + locallyOver scGlobalScope id $ + (\c' d' b' -> Comprehension a b' c' d') <$> + validateCompForScope c <*> + traverse (bitraverse validateCompForScope validateCompIfScope) d <*> + f b + where + validateCompForScope + :: AsScopeError e a + => CompFor v a + -> ValidateScope a e (CompFor (Nub (Scope ': v)) a) + validateCompForScope (CompFor a b c d e) = + (\c' -> CompFor a b c' d) <$> + validateAssignExprScope c <*> + validateExprScope e <* + extendScope scGlobalScope + (c ^.. unvalidated.assignTargets.to (_identAnn &&& _identValue)) + + validateCompIfScope + :: AsScopeError e a + => CompIf v a + -> ValidateScope a e (CompIf (Nub (Scope ': v)) a) + validateCompIfScope (CompIf a b c) = + CompIf a b <$> validateExprScope c + +validateAssignExprScope + :: AsScopeError e a + => Expr v a + -> ValidateScope a e (Expr (Nub (Scope ': v)) a) +validateAssignExprScope (Subscript a e1 ws1 e2 ws2) = + (\e1' e2' -> Subscript a e1' ws1 e2' ws2) <$> + validateAssignExprScope e1 <*> + traverse validateSubscriptScope e2 +validateAssignExprScope (List a ws1 es ws2) = + List a ws1 <$> + traverseOf (traverse.traverse) listItem es <*> + pure ws2 + where + listItem (ListItem a b) = ListItem a <$> validateAssignExprScope b + listItem (ListUnpack a b c d) = ListUnpack a b c <$> validateAssignExprScope d +validateAssignExprScope (Deref a e ws1 r) = + Deref a <$> + validateExprScope e <*> + pure ws1 <*> + validateIdentScope r +validateAssignExprScope (Parens a ws1 e ws2) = + Parens a ws1 <$> + validateAssignExprScope e <*> + pure ws2 +validateAssignExprScope (Tuple a b ws d) = + Tuple a <$> + tupleItem b <*> + pure ws <*> + traverseOf (traverse.traverse) tupleItem d + where + tupleItem (TupleItem a b) = TupleItem a <$> validateAssignExprScope b + tupleItem (TupleUnpack a b c d) = TupleUnpack a b c <$> validateAssignExprScope d +validateAssignExprScope e@Unit{} = pure $ unsafeCoerce e +validateAssignExprScope e@Lambda{} = pure $ unsafeCoerce e +validateAssignExprScope e@Yield{} = pure $ unsafeCoerce e +validateAssignExprScope e@YieldFrom{} = pure $ unsafeCoerce e +validateAssignExprScope e@Not{} = pure $ unsafeCoerce e +validateAssignExprScope e@ListComp{} = pure $ unsafeCoerce e +validateAssignExprScope e@Call{} = pure $ unsafeCoerce e +validateAssignExprScope e@UnOp{} = pure $ unsafeCoerce e +validateAssignExprScope e@BinOp{} = pure $ unsafeCoerce e +validateAssignExprScope e@Ident{} = pure $ unsafeCoerce e +validateAssignExprScope e@None{} = pure $ unsafeCoerce e +validateAssignExprScope e@Ellipsis{} = pure $ unsafeCoerce e +validateAssignExprScope e@Int{} = pure $ unsafeCoerce e +validateAssignExprScope e@Float{} = pure $ unsafeCoerce e +validateAssignExprScope e@Imag{} = pure $ unsafeCoerce e +validateAssignExprScope e@Bool{} = pure $ unsafeCoerce e +validateAssignExprScope e@String{} = pure $ unsafeCoerce e +validateAssignExprScope e@DictComp{} = pure $ unsafeCoerce e +validateAssignExprScope e@Dict{} = pure $ unsafeCoerce e +validateAssignExprScope e@SetComp{} = pure $ unsafeCoerce e +validateAssignExprScope e@Set{} = pure $ unsafeCoerce e +validateAssignExprScope e@Generator{} = pure $ unsafeCoerce e +validateAssignExprScope e@Await{} = pure $ unsafeCoerce e +validateAssignExprScope e@Ternary{} = pure $ unsafeCoerce e + +validateDictItemScope + :: AsScopeError e a + => DictItem v a + -> ValidateScope a e (DictItem (Nub (Scope ': v)) a) +validateDictItemScope (DictItem a b c d) = + (\b' -> DictItem a b' c) <$> + validateExprScope b <*> + validateExprScope d +validateDictItemScope (DictUnpack a b c) = + DictUnpack a b <$> validateExprScope c + +validateSubscriptScope + :: AsScopeError e a + => Subscript v a + -> ValidateScope a e (Subscript (Nub (Scope ': v)) a) +validateSubscriptScope (SubscriptExpr e) = SubscriptExpr <$> validateExprScope e +validateSubscriptScope (SubscriptSlice a b c d) = + (\a' -> SubscriptSlice a' b) <$> + traverse validateExprScope a <*> + traverse validateExprScope c <*> + traverseOf (traverse._2.traverse) validateExprScope d + +validateListItemScope + :: AsScopeError e a + => ListItem v a + -> ValidateScope a e (ListItem (Nub (Scope ': v)) a) +validateListItemScope (ListItem a b) = ListItem a <$> validateExprScope b +validateListItemScope (ListUnpack a b c d) = ListUnpack a b c <$> validateExprScope d + +validateSetItemScope + :: AsScopeError e a + => SetItem v a + -> ValidateScope a e (SetItem (Nub (Scope ': v)) a) +validateSetItemScope (SetItem a b) = SetItem a <$> validateExprScope b +validateSetItemScope (SetUnpack a b c d) = SetUnpack a b c <$> validateExprScope d + +validateTupleItemScope + :: AsScopeError e a + => TupleItem v a + -> ValidateScope a e (TupleItem (Nub (Scope ': v)) a) +validateTupleItemScope (TupleItem a b) = TupleItem a <$> validateExprScope b +validateTupleItemScope (TupleUnpack a b c d) = TupleUnpack a b c <$> validateExprScope d + +validateExprScope + :: AsScopeError e a + => Expr v a + -> ValidateScope a e (Expr (Nub (Scope ': v)) a) +validateExprScope (Lambda a b c d e) = + Lambda a b <$> + traverse validateParamScope c <*> + pure d <*> + validateExprScope e +validateExprScope (Yield a b c) = + Yield a b <$> traverse validateExprScope c +validateExprScope (YieldFrom a b c d) = + YieldFrom a b c <$> validateExprScope d +validateExprScope (Ternary a b c d e f) = + (\b' d' f' -> Ternary a b' c d' e f') <$> + validateExprScope b <*> + validateExprScope d <*> + validateExprScope f +validateExprScope (Subscript a b c d e) = + (\b' d' -> Subscript a b' c d' e) <$> + validateExprScope b <*> + traverse validateSubscriptScope d +validateExprScope (Not a ws e) = Not a ws <$> validateExprScope e +validateExprScope (List a ws1 es ws2) = + List a ws1 <$> + traverseOf (traverse.traverse) validateListItemScope es <*> + pure ws2 +validateExprScope (ListComp a ws1 comp ws2) = + ListComp a ws1 <$> + validateComprehensionScope validateExprScope comp <*> + pure ws2 +validateExprScope (Generator a comp) = + Generator a <$> + validateComprehensionScope validateExprScope comp +validateExprScope (Await a ws expr) = Await a ws <$> validateExprScope expr +validateExprScope (Deref a e ws1 r) = + Deref a <$> + validateExprScope e <*> + pure ws1 <*> + validateIdentScope r +validateExprScope (Call a e ws1 as ws2) = + Call a <$> + validateExprScope e <*> + pure ws1 <*> + traverseOf (traverse.traverse) validateArgScope as <*> + pure ws2 +validateExprScope (BinOp a l op r) = + BinOp a <$> + validateExprScope l <*> + pure op <*> + validateExprScope r +validateExprScope (UnOp a op e) = + UnOp a op <$> + validateExprScope e +validateExprScope (Parens a ws1 e ws2) = + Parens a ws1 <$> + validateExprScope e <*> + pure ws2 +validateExprScope (Ident i) = Ident <$> validateIdentScope i +validateExprScope (Tuple a b ws d) = + Tuple a <$> + validateTupleItemScope b <*> + pure ws <*> + traverseOf (traverse.traverse) validateTupleItemScope d +validateExprScope e@None{} = pure $ unsafeCoerce e +validateExprScope e@Ellipsis{} = pure $ unsafeCoerce e +validateExprScope e@Int{} = pure $ unsafeCoerce e +validateExprScope e@Float{} = pure $ unsafeCoerce e +validateExprScope e@Imag{} = pure $ unsafeCoerce e +validateExprScope e@Bool{} = pure $ unsafeCoerce e +validateExprScope e@String{} = pure $ unsafeCoerce e +validateExprScope e@Unit{} = pure $ unsafeCoerce e +validateExprScope (DictComp a ws1 comp ws2) = + DictComp a ws1 <$> + validateComprehensionScope validateDictItemScope comp <*> + pure ws2 +validateExprScope (Dict a b c d) = + (\c' -> Dict a b c' d) <$> traverseOf (traverse.traverse) validateDictItemScope c +validateExprScope (SetComp a ws1 comp ws2) = + SetComp a ws1 <$> + validateComprehensionScope validateSetItemScope comp <*> + pure ws2 +validateExprScope (Set a b c d) = + (\c' -> Set a b c' d) <$> traverse validateSetItemScope c + +validateModuleScope + :: AsScopeError e a + => Module v a + -> ValidateScope a e (Module (Nub (Scope ': v)) a) +validateModuleScope m = + case m of + ModuleEmpty -> pure ModuleEmpty + ModuleBlankFinal a -> pure $ ModuleBlankFinal a + ModuleBlank a b c -> ModuleBlank a b <$> validateModuleScope c + ModuleStatement a b -> + ModuleStatement <$> + validateStatementScope a <*> + validateModuleScope b + +unionL :: Ord k => Map k v -> Map k v -> Map k v +unionL = Map.unionWith const + +unionR :: Ord k => Map k v -> Map k v -> Map k v +unionR = Map.unionWith (const id) diff --git a/src/Language/Python/Validate/Scope/Error.hs b/src/Language/Python/Validate/Scope/Error.hs new file mode 100644 --- /dev/null +++ b/src/Language/Python/Validate/Scope/Error.hs @@ -0,0 +1,68 @@ +{-# language DataKinds, KindSignatures #-} +{-# language TemplateHaskell #-} +{-# language MultiParamTypeClasses, FunctionalDependencies, FlexibleInstances #-} + +{-| +Module : Language.Python.Validate.Scope.Error +Copyright : (C) CSIRO 2017-2018 +License : BSD3 +Maintainer : Isaac Elliott +Stability : experimental +Portability : non-portable +-} + +module Language.Python.Validate.Scope.Error where + +import Control.Lens.TH +import Language.Python.Syntax.Ident + +data ScopeError a + -- | + -- Using @nonlocal@ to modify function scopes makes scope checking intractible + = FoundNonlocal a + -- | + -- Using @global@ to add identifiers to the global scope makes scope checking + -- intractible + | FoundGlobal a + -- | + -- Using @del@ to remove identifiers from scope makes scope checking intractible + | DeletedIdent a + -- | + -- Variable assignments deep in control flow can modify the scope outside + -- the control flow. For example: + -- + -- @ + -- if a: + -- x = 0 + -- else: + -- pass + -- + -- print(x) + -- @ + -- + -- @x@ will be in scope if the @True@ branch was entered, but not if the @False@ + -- branch was entered. This kind of behaviour makes scope checking intractible, so + -- programs like this are considered scope errors. + | FoundDynamic a (Ident '[] a) + -- | An identifier is not in scope + | NotInScope (Ident '[] a) + -- | + -- For loops don't execute in a fresh scope, so if the 'counter' of the loop + -- shadows a variable, then that variable will be mutated. + -- + -- e.g. + -- + -- @ + -- x = 0 + -- for x in 1, 2, 3: + -- pass + -- print(x) + -- @ + -- + -- outputs @3@ + -- + -- This error occurs when we spot this pattern. + | BadShadowing (Ident '[] a) + deriving (Eq, Show) + +makeClassyPrisms ''ScopeError diff --git a/src/Language/Python/Validate/Syntax.hs b/src/Language/Python/Validate/Syntax.hs new file mode 100644 --- /dev/null +++ b/src/Language/Python/Validate/Syntax.hs @@ -0,0 +1,1330 @@ +{-# language DataKinds #-} +{-# language GeneralizedNewtypeDeriving #-} +{-# language FlexibleContexts #-} +{-# language PolyKinds #-} +{-# language TypeOperators #-} +{-# language TypeSynonymInstances, FlexibleInstances #-} +{-# language TemplateHaskell, TypeFamilies, MultiParamTypeClasses #-} +{-# language RankNTypes #-} +{-# language LambdaCase #-} +{-# language ScopedTypeVariables #-} + +{-| +Module : Language.Python.Validate.Syntax +Copyright : (C) CSIRO 2017-2018 +License : BSD3 +Maintainer : Isaac Elliott +Stability : experimental +Portability : non-portable +-} + +module Language.Python.Validate.Syntax + ( module Data.Validation + , module Language.Python.Validate.Syntax.Error + -- * Main validation functions + , Syntax, ValidateSyntax, runValidateSyntax + , validateModuleSyntax + , validateStatementSyntax + , validateExprSyntax + -- * Miscellany + -- ** Extra types + , SyntaxContext(..), FunctionInfo(..), inLoop, inFunction, inGenerator, inParens + , runValidateSyntax' + , initialSyntaxContext + -- ** Extra functions + , reservedWords + , canAssignTo + , deleteBy' + , deleteFirstsBy' + , localNonlocals + -- ** Validation functions + , validateArgsSyntax + , validateBlockSyntax + , validateCompoundStatementSyntax + , validateComprehensionSyntax + , validateDecoratorSyntax + , validateDictItemSyntax + , validateExceptAsSyntax + , validateIdentSyntax + , validateImportAsSyntax + , validateImportTargetsSyntax + , validateListItemSyntax + , validateParamsSyntax + , validateSetItemSyntax + , validateSimpleStatementSyntax + , validateStringLiteralSyntax + , validateSubscriptSyntax + , validateSuiteSyntax + , validateTupleItemSyntax + , validateWhitespace + ) +where + +import Data.Validation + +import Control.Applicative ((<|>), liftA2) +import Control.Lens.Cons (snoc, _init) +import Control.Lens.Fold + ((^..), (^?), (^?!), folded, allOf, toListOf, anyOf, lengthOf, has) +import Control.Lens.Getter ((^.), getting, view) +import Control.Lens.Prism (_Right, _Just) +import Control.Lens.Review ((#)) +import Control.Lens.Setter ((.~), (%~)) +import Control.Lens.TH (makeLenses) +import Control.Lens.Tuple (_1, _2, _3) +import Control.Lens.Traversal (traverseOf) +import Control.Monad (when) +import Control.Monad.State (State, put, modify, get, evalState) +import Control.Monad.Reader (ReaderT, local, ask, runReaderT) +import Data.Char (isAscii, ord) +import Data.Coerce (coerce) +import Data.Foldable (toList, traverse_) +import Data.Bitraversable (bitraverse) +import Data.Functor.Compose (Compose(..)) +import Data.List (intersect, union) +import Data.List.NonEmpty (NonEmpty(..), (<|)) +import Data.Maybe (isJust, isNothing, fromMaybe) +import Data.Semigroup (Semigroup(..)) +import Data.Type.Set (Nub, Member) +import Data.Validate.Monadic (ValidateM(..), bindVM, liftVM0, liftVM1, errorVM, errorVM1) +import Unsafe.Coerce (unsafeCoerce) + +import qualified Data.List.NonEmpty as NonEmpty + +import Language.Python.Optics +import Language.Python.Optics.Validated (unvalidated) +import Language.Python.Syntax.CommaSep +import Language.Python.Syntax.Expr +import Language.Python.Syntax.Ident +import Language.Python.Syntax.Import +import Language.Python.Syntax.Module +import Language.Python.Syntax.Punctuation +import Language.Python.Syntax.Statement +import Language.Python.Syntax.Strings +import Language.Python.Syntax.Whitespace +import Language.Python.Validate.Indentation +import Language.Python.Validate.Syntax.Error + +deleteBy' :: (a -> b -> Bool) -> a -> [b] -> [b] +deleteBy' _ _ [] = [] +deleteBy' eq a (b:bs) = if a `eq` b then bs else b : deleteBy' eq a bs + +deleteFirstsBy' :: (a -> b -> Bool) -> [a] -> [b] -> [a] +deleteFirstsBy' eq = foldl (flip (deleteBy' (flip eq))) + +reservedWords :: [String] +reservedWords = + [ "False" + , "class" + , "finally" + , "is" + , "return" + , "None" + , "continue" + , "for" + , "lambda" + , "try" + , "True" + , "def" + , "from" + , "nonlocal" + , "while" + , "and" + , "del" + , "global" + , "not" + , "with" + , "as" + , "elif" + , "if" + , "or" + , "yield" + , "assert" + , "else" + , "import" + , "pass" + , "break" + , "except" + , "in" + , "raise" + ] + +data Syntax + +data FunctionInfo + = FunctionInfo + { _functionParams :: [String] + , _asyncFunction :: Bool + } +makeLenses ''FunctionInfo + +data SyntaxContext + = SyntaxContext + { _inLoop :: Bool + , _inFinally :: Bool + , _inFunction :: Maybe FunctionInfo + , _inGenerator :: Bool + , _inClass :: Bool + , _inParens :: Bool + } +makeLenses ''SyntaxContext + +type ValidateSyntax e = ValidateM (NonEmpty e) (ReaderT SyntaxContext (State [String])) + +runValidateSyntax :: ValidateSyntax e a -> Validation (NonEmpty e) a +runValidateSyntax = runValidateSyntax' initialSyntaxContext [] + +runValidateSyntax' :: SyntaxContext -> [String] -> ValidateSyntax e a -> Validation (NonEmpty e) a +runValidateSyntax' ctxt nlscope = + flip evalState nlscope . + flip runReaderT ctxt . getCompose . + unValidateM + +localNonlocals :: ([String] -> [String]) -> ValidateSyntax e a -> ValidateSyntax e a +localNonlocals f v = + ValidateM . Compose $ do + before <- get + modify f + res <- getCompose $ unValidateM v + put before + pure res + +initialSyntaxContext :: SyntaxContext +initialSyntaxContext = + SyntaxContext + { _inLoop = False + , _inFinally = False + , _inFunction = Nothing + , _inGenerator = False + , _inClass = False + , _inParens = False + } + +validateIdentSyntax + :: ( AsSyntaxError e a + , Member Indentation v + ) + => Ident v a + -> ValidateSyntax e (Ident (Nub (Syntax ': v)) a) +validateIdentSyntax (MkIdent a name ws) + | not (all isAscii name) = errorVM1 (_BadCharacter # (a, name)) + | null name = errorVM1 (_EmptyIdentifier # a) + | otherwise = + bindVM (view inFunction) $ \fi -> + let + reserved = + reservedWords <> + if fromMaybe False (fi ^? _Just.asyncFunction) + then ["async", "await"] + else [] + in + if (name `elem` reserved) + then errorVM1 (_IdentifierReservedWord # (a, name)) + else pure $ MkIdent a name ws + +validateWhitespace + :: (AsSyntaxError e a, Foldable f) + => a + -> f Whitespace + -> ValidateSyntax e (f Whitespace) +validateWhitespace ann ws = + ask `bindVM` \ctxt -> + if _inParens ctxt + then pure ws + else if + any + (\case + Newline{} -> True + Comment{} -> False + Continued{} -> False + Tab -> False + Space -> False) + ws + then errorVM1 (_UnexpectedNewline # ann) + else if + any + (\case + Newline{} -> False + Comment{} -> True + Continued{} -> False + Tab -> False + Space -> False) + ws + then errorVM1 (_UnexpectedComment # ann) + else pure ws + +validateAt + :: (AsSyntaxError e a) + => a + -> At + -> ValidateSyntax e At +validateAt a (MkAt ws) = MkAt <$> validateWhitespace a ws + +validateComma + :: (AsSyntaxError e a) + => a + -> Comma + -> ValidateSyntax e Comma +validateComma a (MkComma ws) = MkComma <$> validateWhitespace a ws + +validateColon + :: (AsSyntaxError e a) + => a + -> Colon + -> ValidateSyntax e Colon +validateColon a (MkColon ws) = MkColon <$> validateWhitespace a ws + +validateSemicolon + :: AsSyntaxError e a + => Semicolon a + -> ValidateSyntax e (Semicolon a) +validateSemicolon (MkSemicolon a ws) = MkSemicolon a <$> validateWhitespace a ws + +validateEquals + :: AsSyntaxError e a + => a + -> Equals + -> ValidateSyntax e Equals +validateEquals a (MkEquals ws) = MkEquals <$> validateWhitespace a ws + +validateAssignmentSyntax + :: ( AsSyntaxError e a + , Member Indentation v + ) + => a + -> Expr v a + -> ValidateSyntax e (Expr (Nub (Syntax ': v)) a) +validateAssignmentSyntax a ex = + (if + lengthOf (getting $ _Tuple.tupleItems._TupleUnpack) ex > 1 || + lengthOf (getting $ _List.listItems._ListUnpack) ex > 1 + then errorVM1 $ _ManyStarredTargets # a + else pure ()) *> + (if canAssignTo ex + then validateExprSyntax ex + else errorVM1 $ _CannotAssignTo # (a, ex ^. unvalidated)) + +validateCompForSyntax + :: ( AsSyntaxError e a + , Member Indentation v + ) + => CompFor v a + -> ValidateSyntax e (CompFor (Nub (Syntax ': v)) a) +validateCompForSyntax (CompFor a b c d e) = + (\c' -> CompFor a b c' d) <$> + liftVM1 (local $ inGenerator .~ True) (validateAssignmentSyntax a c) <*> + validateExprSyntax e + +validateCompIfSyntax + :: ( AsSyntaxError e a + , Member Indentation v + ) + => CompIf v a + -> ValidateSyntax e (CompIf (Nub (Syntax ': v)) a) +validateCompIfSyntax (CompIf a b c) = + CompIf a b <$> liftVM1 (local $ inGenerator .~ True) (validateExprSyntax c) + +validateComprehensionSyntax + :: ( AsSyntaxError e a + , Member Indentation v + ) + => (ex v a -> ValidateSyntax e (ex (Nub (Syntax ': v)) a)) + -> Comprehension ex v a + -> ValidateSyntax e (Comprehension ex (Nub (Syntax ': v)) a) +validateComprehensionSyntax f (Comprehension a b c d) = + Comprehension a <$> + liftVM1 (local $ inGenerator .~ True) (f b) <*> + validateCompForSyntax c <*> + liftVM1 + (local $ inGenerator .~ True) + (traverse + (bitraverse validateCompForSyntax validateCompIfSyntax) + d) + +validateStringPyChar + :: ( AsSyntaxError e a + ) + => a + -> PyChar + -> ValidateSyntax e PyChar +validateStringPyChar a (Char_lit '\0') = + errorVM1 $ _NullByte # a +validateStringPyChar _ a = pure a + +validateBytesPyChar + :: ( AsSyntaxError e a + ) + => a + -> PyChar + -> ValidateSyntax e PyChar +validateBytesPyChar a (Char_lit '\0') = + errorVM1 $ _NullByte # a +validateBytesPyChar a (Char_lit c) | ord c >= 128 = + errorVM1 $ _NonAsciiInBytes # (a, c) +validateBytesPyChar _ a = pure a + +validateStringLiteralSyntax + :: AsSyntaxError e a + => StringLiteral a + -> ValidateSyntax e (StringLiteral a) +validateStringLiteralSyntax (StringLiteral a b c d e f) = + StringLiteral a b c d <$> + traverse (validateStringPyChar a) e <*> + validateWhitespace a f +validateStringLiteralSyntax (BytesLiteral a b c d e f) = + BytesLiteral a b c d <$> + traverse (validateBytesPyChar a) e <*> + validateWhitespace a f +validateStringLiteralSyntax (RawStringLiteral a b c d e f) = + RawStringLiteral a b c d e <$> + validateWhitespace a f +validateStringLiteralSyntax (RawBytesLiteral a b c d e f) = + RawBytesLiteral a b c d e <$> + validateWhitespace a f + +validateDictItemSyntax + :: ( AsSyntaxError e a + , Member Indentation v + ) + => DictItem v a + -> ValidateSyntax e (DictItem (Nub (Syntax ': v)) a) +validateDictItemSyntax (DictItem a b c d) = + (\b' -> DictItem a b' c) <$> + validateExprSyntax b <*> + validateExprSyntax d +validateDictItemSyntax (DictUnpack a b c) = + DictUnpack a <$> + validateWhitespace a b <*> + validateExprSyntax c + +validateSubscriptSyntax + :: ( AsSyntaxError e a + , Member Indentation v + ) + => Subscript v a + -> ValidateSyntax e (Subscript (Nub (Syntax ': v)) a) +validateSubscriptSyntax (SubscriptExpr e) = SubscriptExpr <$> validateExprSyntax e +validateSubscriptSyntax (SubscriptSlice a b c d) = + (\a' -> SubscriptSlice a' b) <$> + traverse validateExprSyntax a <*> + traverse validateExprSyntax c <*> + traverseOf (traverse._2.traverse) validateExprSyntax d + +validateListItemSyntax + :: ( AsSyntaxError e a + , Member Indentation v + ) + => ListItem v a + -> ValidateSyntax e (ListItem (Nub (Syntax ': v)) a) +validateListItemSyntax (ListItem a b) = ListItem a <$> validateExprSyntax b +validateListItemSyntax (ListUnpack a b c d) = + ListUnpack a <$> + traverseOf (traverse._2) (validateWhitespace a) b <*> + validateWhitespace a c <*> + validateExprSyntax d + +validateSetItemSyntax + :: ( AsSyntaxError e a + , Member Indentation v + ) + => SetItem v a + -> ValidateSyntax e (SetItem (Nub (Syntax ': v)) a) +validateSetItemSyntax (SetItem a b) = SetItem a <$> validateExprSyntax b +validateSetItemSyntax (SetUnpack a b c d) = + SetUnpack a <$> + traverseOf (traverse._2) (validateWhitespace a) b <*> + validateWhitespace a c <*> + validateExprSyntax d + +validateTupleItemSyntax + :: ( AsSyntaxError e a + , Member Indentation v + ) + => TupleItem v a + -> ValidateSyntax e (TupleItem (Nub (Syntax ': v)) a) +validateTupleItemSyntax (TupleItem a b) = TupleItem a <$> validateExprSyntax b +validateTupleItemSyntax (TupleUnpack a b c d) = + TupleUnpack a <$> + traverseOf (traverse._2) (validateWhitespace a) b <*> + validateWhitespace a c <*> + validateExprSyntax d + +validateExprSyntax + :: ( AsSyntaxError e a + , Member Indentation v + ) + => Expr v a + -> ValidateSyntax e (Expr (Nub (Syntax ': v)) a) +validateExprSyntax (Unit a b c) = + Unit a <$> + liftVM1 (local $ inParens .~ True) (validateWhitespace a b) <*> + validateWhitespace a c +validateExprSyntax (Lambda a b c d e) = + let + paramIdents = c ^.. folded.unvalidated.paramName.identValue + in + Lambda a <$> + validateWhitespace a b <*> + validateParamsSyntax True c <*> + validateColon a d <*> + liftVM1 + (local $ + \ctxt -> + ctxt + { _inLoop = False + , _inFunction = + fmap + ((functionParams %~ (`union` paramIdents)) . (asyncFunction .~ False)) + (_inFunction ctxt) <|> + Just (FunctionInfo paramIdents False) + }) + (validateExprSyntax e) +validateExprSyntax (Yield a b c) = + Yield a <$> + validateWhitespace a b <* + (ask `bindVM` \ctxt -> + case _inFunction ctxt of + Nothing + | _inGenerator ctxt -> pure () + | otherwise -> errorVM1 (_YieldOutsideGenerator # a) + Just info -> + if info^.asyncFunction + then errorVM1 $ _YieldInsideCoroutine # a + else pure ()) <*> + traverse validateExprSyntax c +validateExprSyntax (YieldFrom a b c d) = + YieldFrom a <$> + validateWhitespace a b <*> + validateWhitespace a c <* + (ask `bindVM` \ctxt -> + case _inFunction ctxt of + Nothing + | _inGenerator ctxt -> pure () + | otherwise -> errorVM1 (_YieldOutsideGenerator # a) + Just fi -> + if fi ^. asyncFunction + then errorVM1 (_YieldFromInsideCoroutine # a) + else pure ()) <*> + validateExprSyntax d +validateExprSyntax (Ternary a b c d e f) = + (\b' d' f' -> Ternary a b' c d' e f') <$> + validateExprSyntax b <*> + validateExprSyntax d <*> + validateExprSyntax f +validateExprSyntax (Subscript a b c d e) = + (\b' d' -> Subscript a b' c d' e) <$> + validateExprSyntax b <*> + traverse validateSubscriptSyntax d +validateExprSyntax (Not a ws e) = + Not a <$> + validateWhitespace a ws <*> + validateExprSyntax e +validateExprSyntax (Parens a ws1 e ws2) = + Parens a ws1 <$> + liftVM1 (local $ inParens .~ True) (validateExprSyntax e) <*> + validateWhitespace a ws2 +validateExprSyntax (Bool a b ws) = pure $ Bool a b ws +validateExprSyntax (UnOp a op expr) = + UnOp a op <$> validateExprSyntax expr +validateExprSyntax (String a strLits) = + if + all + (\case + StringLiteral{} -> True + RawStringLiteral{} -> True + _ -> False) + strLits + || + all + (\case + BytesLiteral{} -> True + RawBytesLiteral{} -> True + _ -> False) + strLits + then + String a <$> traverse validateStringLiteralSyntax strLits + else + errorVM1 (_Can'tJoinStringAndBytes # a) +validateExprSyntax (Int a n ws) = pure $ Int a n ws +validateExprSyntax (Float a n ws) = pure $ Float a n ws +validateExprSyntax (Imag a n ws) = pure $ Imag a n ws +validateExprSyntax (Ident name) = Ident <$> validateIdentSyntax name +validateExprSyntax (List a ws1 exprs ws2) = + List a ws1 <$> + liftVM1 + (local $ inParens .~ True) + (traverseOf (traverse.traverse) validateListItemSyntax exprs) <*> + validateWhitespace a ws2 +validateExprSyntax (ListComp a ws1 comp ws2) = + liftVM1 + (local $ inParens .~ True) + (ListComp a ws1 <$> + validateComprehensionSyntax validateExprSyntax comp) <*> + validateWhitespace a ws2 +validateExprSyntax (Generator a comp) = + Generator a <$> validateComprehensionSyntax validateExprSyntax comp +validateExprSyntax (Await a ws expr) = + bindVM ask $ \ctxt -> + Await a <$> + validateWhitespace a ws <* + (if not $ fromMaybe False (ctxt ^? inFunction._Just.asyncFunction) + then errorVM1 $ _AwaitOutsideCoroutine # a + else pure () *> + if ctxt^.inGenerator + then errorVM1 $ _AwaitInsideComprehension # a + else pure ()) <*> + validateExprSyntax expr +validateExprSyntax (Deref a expr ws1 name) = + Deref a <$> + validateExprSyntax expr <*> + validateWhitespace a ws1 <*> + validateIdentSyntax name +validateExprSyntax (Call a expr ws args ws2) = + Call a <$> + validateExprSyntax expr <*> + liftVM1 (local $ inParens .~ True) (validateWhitespace a ws) <*> + liftVM1 (local $ inParens .~ True) (traverse validateArgsSyntax args) <*> + validateWhitespace a ws2 +validateExprSyntax (None a ws) = None a <$> validateWhitespace a ws +validateExprSyntax (Ellipsis a ws) = Ellipsis a <$> validateWhitespace a ws +validateExprSyntax (BinOp a e1 op e2) = + BinOp a <$> + validateExprSyntax e1 <*> + pure op <*> + validateExprSyntax e2 +validateExprSyntax (Tuple a b comma d) = + Tuple a <$> + validateTupleItemSyntax b <*> + validateComma a comma <*> + traverseOf (traverse.traverse) validateTupleItemSyntax d +validateExprSyntax (DictComp a ws1 comp ws2) = + liftVM1 + (local $ inParens .~ True) + (DictComp a ws1 <$> + validateComprehensionSyntax dictItem comp) <*> + validateWhitespace a ws2 + where + dictItem (DictUnpack a _ _) = errorVM1 (_InvalidDictUnpacking # a) + dictItem a = validateDictItemSyntax a +validateExprSyntax (Dict a b c d) = + Dict a b <$> + liftVM1 + (local $ inParens .~ True) + (traverseOf (traverse.traverse) validateDictItemSyntax c) <*> + validateWhitespace a d +validateExprSyntax (SetComp a ws1 comp ws2) = + liftVM1 + (local $ inParens .~ True) + (SetComp a ws1 <$> + validateComprehensionSyntax setItem comp) <*> + validateWhitespace a ws2 + where + setItem (SetUnpack a _ _ _) = errorVM1 (_InvalidSetUnpacking # a) + setItem a = validateSetItemSyntax a +validateExprSyntax (Set a b c d) = + Set a b <$> + liftVM1 + (local $ inParens .~ True) + (traverse validateSetItemSyntax c) <*> + validateWhitespace a d + +validateBlockSyntax + :: ( AsSyntaxError e a + , Member Indentation v + ) + => Block v a + -> ValidateSyntax e (Block (Nub (Syntax ': v)) a) +validateBlockSyntax (Block x b bs) = + Block x <$> + validateStatementSyntax b <*> + traverseOf (traverse._Right) validateStatementSyntax bs + +validateSuiteSyntax + :: ( AsSyntaxError e a + , Member Indentation v + ) + => Suite v a + -> ValidateSyntax e (Suite (Nub (Syntax ': v)) a) +validateSuiteSyntax (SuiteMany a b c d e) = + (\b' -> SuiteMany a b' c d) <$> + validateColon a b <*> + validateBlockSyntax e +validateSuiteSyntax (SuiteOne a b c) = + SuiteOne a <$> + validateColon a b <*> + validateSmallStatementSyntax c + +validateDecoratorSyntax + :: ( AsSyntaxError e a + , Member Indentation v + ) + => Decorator v a + -> ValidateSyntax e (Decorator (Nub (Syntax ': v)) a) +validateDecoratorSyntax (Decorator a b c d e f g) = + (\c' d' -> Decorator a b c' d' e f) <$> + validateAt a c <*> + isDecoratorValue d <*> + traverseOf (traverse._1) validateBlankSyntax g + where + someDerefs Ident{} = True + someDerefs (Deref _ a _ _) = someDerefs a + someDerefs _ = False + + isDecoratorValue e@(Call _ a _ _ _) | someDerefs a = pure $ unsafeCoerce e + isDecoratorValue e | someDerefs e = pure $ unsafeCoerce e + isDecoratorValue _ = errorVM1 (_MalformedDecorator # a) + +validateBlankSyntax :: AsSyntaxError e a => Blank a -> ValidateSyntax e (Blank a) +validateBlankSyntax (Blank a ws cmt) = + (\ws' -> Blank a ws' cmt) <$> + validateWhitespace a ws + +validateCompoundStatementSyntax + :: forall e v a + . ( AsSyntaxError e a + , Member Indentation v + ) + => CompoundStatement v a + -> ValidateSyntax e (CompoundStatement (Nub (Syntax ': v)) a) +validateCompoundStatementSyntax (Fundef a decos idnts asyncWs ws1 name ws2 params ws3 mty body) = + let + paramIdents = params ^.. folded.unvalidated.paramName.identValue + in + (\decos' -> Fundef a decos' idnts) <$> + traverse validateDecoratorSyntax decos <*> + traverse (validateWhitespace a) asyncWs <*> + validateWhitespace a ws1 <*> + validateIdentSyntax name <*> + pure ws2 <*> + liftVM1 (local $ inParens .~ True) (validateParamsSyntax False params) <*> + pure ws3 <*> + traverse (bitraverse (validateWhitespace a) validateExprSyntax) mty <*> + localNonlocals id + (liftVM1 + (local $ + \ctxt -> + ctxt + { _inLoop = False + , _inFunction = + fmap + ((functionParams %~ (`union` paramIdents)) . + (asyncFunction %~ (|| isJust asyncWs))) + (_inFunction ctxt) <|> + Just (FunctionInfo paramIdents $ isJust asyncWs) + }) + (validateSuiteSyntax body)) +validateCompoundStatementSyntax (If a idnts ws1 expr body elifs body') = + If a idnts <$> + validateWhitespace a ws1 <*> + validateExprSyntax expr <*> + validateSuiteSyntax body <*> + traverse + (\(a, b, c, d) -> + (\c' -> (,,,) a b c') <$> + validateExprSyntax c <*> + validateSuiteSyntax d) + elifs <*> + traverseOf (traverse._3) validateSuiteSyntax body' +validateCompoundStatementSyntax (While a idnts ws1 expr body els) = + While a idnts <$> + validateWhitespace a ws1 <*> + validateExprSyntax expr <*> + liftVM1 (local $ (inFinally .~ False) . (inLoop .~ True)) (validateSuiteSyntax body) <*> + traverseOf (traverse._3) validateSuiteSyntax els +validateCompoundStatementSyntax (TryExcept a idnts b e f k l) = + TryExcept a idnts <$> + validateWhitespace a b <*> + validateSuiteSyntax e <*> + traverse + (\(idnts, f, g, j) -> + (,,,) idnts <$> + validateWhitespace a f <*> + traverse validateExceptAsSyntax g <*> + validateSuiteSyntax j) + f <* + (if anyOf (_init.folded._3) isNothing $ NonEmpty.toList f + then errorVM1 $ _DefaultExceptMustBeLast # a + else pure ()) <*> + traverse + (\(idnts, x, w) -> + (,,) idnts <$> + validateWhitespace a x <*> + validateSuiteSyntax w) + k <*> + traverse + (\(idnts, x, w) -> + (,,) idnts <$> + validateWhitespace a x <*> + liftVM1 (local $ inFinally .~ True) (validateSuiteSyntax w)) + l +validateCompoundStatementSyntax (TryFinally a idnts b e idnts2 f i) = + TryFinally a idnts <$> + validateWhitespace a b <*> + validateSuiteSyntax e <*> pure idnts2 <*> + validateWhitespace a f <*> + liftVM1 (local $ inFinally .~ True) (validateSuiteSyntax i) +validateCompoundStatementSyntax (ClassDef a decos idnts b c d g) = + liftVM1 (local $ inLoop .~ False) $ + (\decos' -> ClassDef a decos' idnts) <$> + traverse validateDecoratorSyntax decos <*> + validateWhitespace a b <*> + validateIdentSyntax c <*> + traverse + (\(x, y, z) -> + (,,) <$> + validateWhitespace a x <*> + traverse + (liftVM1 (local $ inParens .~ True) . validateArgsSyntax) + y <*> + validateWhitespace a z) + d <*> + liftVM1 + (local $ (inClass .~ True) . (inFunction .~ Nothing)) + (validateSuiteSyntax g) +validateCompoundStatementSyntax (For a idnts asyncWs b c d e h i) = + bindVM ask $ \ctxt -> + For a idnts <$ + (if isJust asyncWs && not (fromMaybe False $ ctxt ^? inFunction._Just.asyncFunction) + then errorVM1 (_AsyncForOutsideCoroutine # a) + else pure ()) <*> + traverse (validateWhitespace a) asyncWs <*> + validateWhitespace a b <*> + validateAssignmentSyntax a c <*> + validateWhitespace a d <*> + traverse validateExprSyntax e <*> + liftVM1 + (local $ (inFinally .~ False) . (inLoop .~ True)) + (validateSuiteSyntax h) <*> + traverse + (\(idnts, x, w) -> + (,,) idnts <$> + validateWhitespace a x <*> + validateSuiteSyntax w) + i +validateCompoundStatementSyntax (With a b asyncWs c d e) = + bindVM ask $ \ctxt -> + With a b <$ + (if isJust asyncWs && not (fromMaybe False $ ctxt ^? inFunction._Just.asyncFunction) + then errorVM1 (_AsyncWithOutsideCoroutine # a) + else pure ()) <*> + traverse (validateWhitespace a) asyncWs <*> + validateWhitespace a c <*> + traverse + (\(WithItem a b c) -> + WithItem a <$> + validateExprSyntax b <*> + traverse + (\(ws, b) -> (,) <$> validateWhitespace a ws <*> validateAssignmentSyntax a b) + c) + d <*> + validateSuiteSyntax e + +validateExceptAsSyntax + :: ( AsSyntaxError e a + , Member Indentation v + ) + => ExceptAs v a + -> ValidateSyntax e (ExceptAs (Nub (Syntax ': v)) a) +validateExceptAsSyntax (ExceptAs ann e f) = + ExceptAs ann <$> + validateExprSyntax e <*> + traverse (\(a, b) -> (,) <$> validateWhitespace ann a <*> validateIdentSyntax b) f + +validateImportAsSyntax + :: ( AsSyntaxError e a + , Member Indentation v + ) + => (t a -> ValidateSyntax e (t' a)) + -> ImportAs t v a + -> ValidateSyntax e (ImportAs t' (Nub (Syntax ': v)) a) +validateImportAsSyntax v (ImportAs x a b) = + ImportAs x <$> + v a <*> + traverse + (\(c, d) -> + (,) <$> + (c <$ validateWhitespace x (NonEmpty.toList c)) <*> + validateIdentSyntax d) + b + +validateImportTargetsSyntax + :: ( AsSyntaxError e a + , Member Indentation v + ) + => ImportTargets v a + -> ValidateSyntax e (ImportTargets (Nub (Syntax ': v)) a) +validateImportTargetsSyntax (ImportAll a ws) = + bindVM ask $ \ctxt -> + if ctxt ^. inClass || has (inFunction._Just) ctxt + then errorVM1 $ _WildcardImportInDefinition # a + else ImportAll a <$> validateWhitespace a ws +validateImportTargetsSyntax (ImportSome a cs) = + ImportSome a <$> traverse (validateImportAsSyntax validateIdentSyntax) cs +validateImportTargetsSyntax (ImportSomeParens a ws1 cs ws2) = + liftVM1 + (local $ inParens .~ True) + (ImportSomeParens a <$> + validateWhitespace a ws1 <*> + traverse (validateImportAsSyntax validateIdentSyntax) cs) <*> + validateWhitespace a ws2 + +validateSimpleStatementSyntax + :: ( AsSyntaxError e a + , Member Indentation v + ) + => SimpleStatement v a + -> ValidateSyntax e (SimpleStatement (Nub (Syntax ': v)) a) +validateSimpleStatementSyntax (Assert a b c d) = + Assert a <$> + validateWhitespace a b <*> + validateExprSyntax c <*> + traverseOf (traverse._2) validateExprSyntax d +validateSimpleStatementSyntax (Raise a ws f) = + Raise a <$> + validateWhitespace a ws <*> + traverse + (\(b, c) -> + (,) <$> + validateExprSyntax b <*> + traverse + (\(d, e) -> + (,) <$> + validateWhitespace a d <*> + validateExprSyntax e) + c) + f +validateSimpleStatementSyntax (Return a ws expr) = + ask `bindVM` \sctxt -> + case _inFunction sctxt of + Just{} -> + Return a <$> + validateWhitespace a ws <*> + traverse validateExprSyntax expr + _ -> errorVM1 (_ReturnOutsideFunction # a) +validateSimpleStatementSyntax (Expr a expr) = + Expr a <$> + validateExprSyntax expr +validateSimpleStatementSyntax (Assign a lvalue rs) = + ask `bindVM` \sctxt -> + let + assigns = + if isJust (_inFunction sctxt) + then + (lvalue : (snd <$> NonEmpty.init rs)) ^.. + folded.unvalidated.assignTargets.identValue + else [] + in + Assign a <$> + validateAssignmentSyntax a lvalue <*> + ((\a b -> case a of; [] -> pure b; a : as -> a :| (snoc as b)) <$> + traverse + (\(ws, b) -> + (,) <$> + validateEquals a ws <*> + validateAssignmentSyntax a b) + (NonEmpty.init rs) <*> + (\(ws, b) -> (,) <$> validateEquals a ws <*> validateExprSyntax b) + (NonEmpty.last rs)) <* + liftVM0 (modify (assigns ++)) +validateSimpleStatementSyntax (AugAssign a lvalue aa rvalue) = + AugAssign a <$> + (if canAssignTo lvalue + then case lvalue of + Ident{} -> validateExprSyntax lvalue + Deref{} -> validateExprSyntax lvalue + Subscript{} -> validateExprSyntax lvalue + _ -> errorVM1 (_CannotAugAssignTo # (a, lvalue ^. unvalidated)) + else errorVM1 (_CannotAssignTo # (a, lvalue ^. unvalidated))) <*> + pure aa <*> + validateExprSyntax rvalue +validateSimpleStatementSyntax (Pass a ws) = + Pass a <$> validateWhitespace a ws +validateSimpleStatementSyntax (Break a ws) = + Break a <$ + (ask `bindVM` \sctxt -> + if _inLoop sctxt + then pure () + else errorVM1 (_BreakOutsideLoop # a)) <*> + validateWhitespace a ws +validateSimpleStatementSyntax (Continue a ws) = + Continue a <$ + (ask `bindVM` \sctxt -> + (if _inLoop sctxt + then pure () + else errorVM1 (_ContinueOutsideLoop # a)) *> + (if _inFinally sctxt + then errorVM1 (_ContinueInsideFinally # a) + else pure ())) <*> + validateWhitespace a ws +validateSimpleStatementSyntax (Global a ws ids) = + ask `bindVM` \ctx -> + let + params = ctx ^.. inFunction.folded.functionParams.folded + in + Global a ws <$> + traverse + (\i -> + let + ival = i ^. getting identValue + in + (if ival `elem` params + then errorVM1 $ _ParameterMarkedGlobal # (a, ival) + else pure ()) *> + validateIdentSyntax i) + ids +validateSimpleStatementSyntax (Nonlocal a ws ids) = + ask `bindVM` \sctxt -> + get `bindVM` \nls -> + (case deleteFirstsBy' (\a -> (==) (a ^. unvalidated.identValue)) (ids ^.. folded.unvalidated) nls of + [] -> pure () + ids -> traverse_ (\e -> errorVM1 (_NoBindingNonlocal # e)) ids) *> + case sctxt ^? inFunction._Just.functionParams of + Nothing -> errorVM1 (_NonlocalOutsideFunction # a) + Just params -> + case intersect params (ids ^.. folded.unvalidated.identValue) of + [] -> Nonlocal a ws <$> traverse validateIdentSyntax ids + bad -> errorVM1 (_ParametersNonlocal # (a, bad)) +validateSimpleStatementSyntax (Del a ws ids) = + Del a ws <$> + traverse + (\x -> + validateExprSyntax x <* + if canDelete x + then pure () + else errorVM1 $ _CannotDelete # (a, x ^. unvalidated)) + ids +validateSimpleStatementSyntax (Import a ws mns) = + Import a ws <$> traverse (pure . coerce) mns +validateSimpleStatementSyntax (From a ws1 mn ws2 ts) = + From a ws1 (coerce mn) <$> + validateWhitespace a ws2 <*> + validateImportTargetsSyntax ts + +canDelete :: Expr v a -> Bool +canDelete None{} = False +canDelete Ellipsis{} = False +canDelete UnOp{} = False +canDelete Int{} = False +canDelete Call{} = False +canDelete BinOp{} = False +canDelete Bool{} = False +canDelete Unit{} = False +canDelete Yield{} = False +canDelete YieldFrom{} = False +canDelete Ternary{} = False +canDelete ListComp{} = False +canDelete DictComp{} = False +canDelete Dict{} = False +canDelete SetComp{} = False +canDelete Set{} = False +canDelete Lambda{} = False +canDelete Float{} = False +canDelete Imag{} = False +canDelete Not{} = False +canDelete Generator{} = False +canDelete Await{} = False +canDelete String{} = False +canDelete (Parens _ _ a _) = canDelete a +canDelete (List _ _ a _) = + all (allOf (folded.getting _Exprs) canDelete) a && + not (any (\case; ListUnpack{} -> True; _ -> False) $ a ^.. folded.folded) +canDelete (Tuple _ a _ b) = + all + canDelete + ((a ^?! getting _Exprs) : toListOf (folded.folded.getting _Exprs) b) && + not (any (\case; TupleUnpack{} -> True; _ -> False) $ a : toListOf (folded.folded) b) +canDelete Deref{} = True +canDelete Subscript{} = True +canDelete Ident{} = True + +validateSmallStatementSyntax + :: ( AsSyntaxError e a + , Member Indentation v + ) + => SmallStatement v a + -> ValidateSyntax e (SmallStatement (Nub (Syntax ': v)) a) +validateSmallStatementSyntax (MkSmallStatement s ss sc cmt nl) = + (\s' ss' sc' -> MkSmallStatement s' ss' sc' cmt nl) <$> + validateSimpleStatementSyntax s <*> + traverse (bitraverse validateSemicolon validateSimpleStatementSyntax) ss <*> + traverse validateSemicolon sc + +validateStatementSyntax + :: ( AsSyntaxError e a + , Member Indentation v + ) + => Statement v a + -> ValidateSyntax e (Statement (Nub (Syntax ': v)) a) +validateStatementSyntax (CompoundStatement c) = + CompoundStatement <$> validateCompoundStatementSyntax c +validateStatementSyntax (SmallStatement idnts a) = + SmallStatement idnts <$> validateSmallStatementSyntax a + +canAssignTo :: Expr v a -> Bool +canAssignTo None{} = False +canAssignTo Ellipsis{} = False +canAssignTo UnOp{} = False +canAssignTo Int{} = False +canAssignTo Call{} = False +canAssignTo BinOp{} = False +canAssignTo Bool{} = False +canAssignTo Unit{} = False +canAssignTo Yield{} = False +canAssignTo YieldFrom{} = False +canAssignTo Ternary{} = False +canAssignTo ListComp{} = False +canAssignTo DictComp{} = False +canAssignTo Dict{} = False +canAssignTo SetComp{} = False +canAssignTo Set{} = False +canAssignTo Lambda{} = False +canAssignTo Float{} = False +canAssignTo Imag{} = False +canAssignTo Not{} = False +canAssignTo Generator{} = False +canAssignTo Await{} = False +canAssignTo String{} = False +canAssignTo (Parens _ _ a _) = canAssignTo a +canAssignTo (List _ _ a _) = + all (allOf (folded.getting _Exprs) canAssignTo) a +canAssignTo (Tuple _ a _ b) = + all canAssignTo ((a ^?! getting _Exprs) : toListOf (folded.folded.getting _Exprs) b) +canAssignTo Deref{} = True +canAssignTo Subscript{} = True +canAssignTo Ident{} = True + +validateArgsSyntax + :: ( AsSyntaxError e a + , Member Indentation v + ) + => CommaSep1' (Arg v a) + -> ValidateSyntax e (CommaSep1' (Arg (Nub (Syntax ': v)) a)) +validateArgsSyntax e = unsafeCoerce e <$ go [] False False (toList e) + where + go + :: (AsSyntaxError e a, Member Indentation v) + => [String] + -- ^ Have we seen a keyword argument? + -> Bool + -- ^ Have we seen a **argument? + -> Bool + -> [Arg v a] + -> ValidateSyntax e [Arg (Nub (Syntax ': v)) a] + go _ _ _ [] = pure [] + go names False False (PositionalArg a expr : args) = + liftA2 (:) + (PositionalArg a <$> validateExprSyntax expr) + (go names False False args) + go names seenKeyword seenUnpack (PositionalArg a expr : args) = + when seenKeyword (errorVM1 (_PositionalAfterKeywordArg # (a, expr ^. unvalidated))) *> + when seenUnpack (errorVM1 (_PositionalAfterKeywordUnpacking # (a, expr ^. unvalidated))) *> + go names seenKeyword seenUnpack args + go names seenKeyword False (StarArg a ws expr : args) = + liftA2 (:) + (StarArg a <$> validateWhitespace a ws <*> validateExprSyntax expr) + (go names seenKeyword False args) + go names seenKeyword seenUnpack (StarArg a _ expr : args) = + when seenKeyword (errorVM1 (_PositionalAfterKeywordArg # (a, expr ^. unvalidated))) *> + when seenUnpack (errorVM1 (_PositionalAfterKeywordUnpacking # (a, expr ^. unvalidated))) *> + go names seenKeyword seenUnpack args + go names _ seenUnpack (KeywordArg a name ws2 expr : args) + | _identValue name `elem` names = + errorVM1 (_DuplicateArgument # (a, _identValue name)) <*> + validateIdentSyntax name <*> + go names True seenUnpack args + | otherwise = + liftA2 (:) + (KeywordArg a <$> + validateIdentSyntax name <*> + pure ws2 <*> + validateExprSyntax expr) + (go (_identValue name:names) True seenUnpack args) + go names seenKeyword _ (DoubleStarArg a ws expr : args) = + liftA2 (:) + (DoubleStarArg a <$> + validateWhitespace a ws <*> + validateExprSyntax expr) + (go names seenKeyword True args) + +newtype HaveSeenStarArg = HaveSeenStarArg Bool +newtype HaveSeenKeywordArg = HaveSeenKeywordArg Bool +newtype HaveSeenEmptyStarArg a = HaveSeenEmptyStarArg (Maybe a) + +validateParamsSyntax + :: forall e v a + . ( AsSyntaxError e a + , Member Indentation v + ) + => Bool -- ^ These are the parameters to a lambda + -> CommaSep (Param v a) + -> ValidateSyntax e (CommaSep (Param (Nub (Syntax ': v)) a)) +validateParamsSyntax isLambda e = + unsafeCoerce e <$ + go + [] + (HaveSeenStarArg False) + (HaveSeenEmptyStarArg Nothing) + (HaveSeenKeywordArg False) + (toList e) + where + checkTy + :: a + -> Maybe (Colon, Expr v a) + -> ValidateSyntax e (Maybe (Colon, Expr (Nub (Syntax ': v)) a)) + checkTy a mty = + if isLambda + then traverse (\_ -> errorVM1 (_TypedParamInLambda # a)) mty + else traverseOf (traverse._2) validateExprSyntax mty + + go + :: [String] -- identifiers that we've seen + -> HaveSeenStarArg -- have we seen a star argument? + -> HaveSeenEmptyStarArg a -- have we seen an empty star argument? + -> HaveSeenKeywordArg -- have we seen a keyword parameter? + -> [Param v a] + -> ValidateSyntax e [Param (Nub (Syntax ': v)) a] + go _ _ (HaveSeenEmptyStarArg b) _ [] = + case b of + Nothing -> pure [] + Just b' -> errorVM1 $ _NoKeywordsAfterEmptyStarArg # b' + go names bsa besa bkw@(HaveSeenKeywordArg False) (PositionalParam a name mty : params) + | _identValue name `elem` names = + errorVM1 (_DuplicateArgument # (a, _identValue name)) <*> + validateIdentSyntax name <*> + checkTy a mty <*> + go (_identValue name:names) bsa besa bkw params + | otherwise = + liftA2 + (:) + (PositionalParam a <$> + validateIdentSyntax name <*> + checkTy a mty) + (go (_identValue name:names) bsa besa bkw params) + go names (HaveSeenStarArg b) besa bkw (StarParam a _ name mty : params) + | _identValue name `elem` names = + if b + then + errorVM1 (_ManyStarredParams # a) <*> + errorVM1 (_DuplicateArgument # (a, _identValue name)) <*> + validateIdentSyntax name <*> + checkTy a mty <*> + go + (_identValue name:names) + (HaveSeenStarArg True) + besa + bkw + params + else + errorVM1 (_DuplicateArgument # (a, _identValue name)) <*> + validateIdentSyntax name <*> + checkTy a mty <*> + go + (_identValue name:names) + (HaveSeenStarArg True) + besa + bkw + params + | otherwise = + if b + then + errorVM1 (_ManyStarredParams # a) <*> + validateIdentSyntax name *> + checkTy a mty *> + go + (_identValue name:names) + (HaveSeenStarArg True) + besa + bkw + params + else + validateIdentSyntax name *> + checkTy a mty *> + go + (_identValue name:names) + (HaveSeenStarArg True) + besa + bkw + params + go names (HaveSeenStarArg b) _ bkw (UnnamedStarParam a _ : params) = + if b + then + errorVM1 (_ManyStarredParams # a) <*> + go + names + (HaveSeenStarArg True) + (HaveSeenEmptyStarArg $ Just a) + bkw + params + else + go + names + (HaveSeenStarArg True) + (HaveSeenEmptyStarArg $ Just a) + bkw + params + go names bsa besa bkw@(HaveSeenKeywordArg True) (PositionalParam a name mty : params) = + let + name' = _identValue name + errs = + foldr (<|) + (_PositionalAfterKeywordParam # (a, name') :| []) + [_DuplicateArgument # (a, name') | name' `elem` names] + in + errorVM errs <*> + checkTy a mty <*> + go (name':names) bsa besa bkw params + go names bsa _ _ (KeywordParam a name mty ws2 expr : params) + | _identValue name `elem` names = + errorVM1 (_DuplicateArgument # (a, _identValue name)) <*> + checkTy a mty <*> + go names bsa (HaveSeenEmptyStarArg Nothing) (HaveSeenKeywordArg True) params + | otherwise = + liftA2 (:) + (KeywordParam a <$> + validateIdentSyntax name <*> + checkTy a mty <*> + pure ws2 <*> + validateExprSyntax expr) + (go + (_identValue name:names) + bsa + (HaveSeenEmptyStarArg Nothing) + (HaveSeenKeywordArg True) + params) + go names bsa besa bkw [DoubleStarParam a ws name mty] + | _identValue name `elem` names = + errorVM1 (_DuplicateArgument # (a, _identValue name)) <*> + checkTy a mty <* + go names bsa besa bkw [] + | otherwise = + fmap pure $ + DoubleStarParam a ws <$> + validateIdentSyntax name <*> + checkTy a mty <* + go names bsa besa bkw [] + go names bsa besa bkw (DoubleStarParam a _ name mty : _) = + (if _identValue name `elem` names + then errorVM1 (_DuplicateArgument # (a, _identValue name)) + else pure ()) *> + errorVM1 (_UnexpectedDoubleStarParam # (a, _identValue name)) <*> + checkTy a mty <* + go names bsa besa bkw [] + +validateModuleSyntax + :: ( AsSyntaxError e a + , Member Indentation v + ) + => Module v a + -> ValidateSyntax e (Module (Nub (Syntax ': v)) a) +validateModuleSyntax m = + case m of + ModuleEmpty -> pure ModuleEmpty + ModuleBlankFinal a -> ModuleBlankFinal <$> validateBlankSyntax a + ModuleBlank a b c -> + (\a' -> ModuleBlank a' b) <$> + validateBlankSyntax a <*> + validateModuleSyntax c + ModuleStatement a b -> + ModuleStatement <$> + validateStatementSyntax a <*> + validateModuleSyntax b diff --git a/src/Language/Python/Validate/Syntax/Error.hs b/src/Language/Python/Validate/Syntax/Error.hs new file mode 100644 --- /dev/null +++ b/src/Language/Python/Validate/Syntax/Error.hs @@ -0,0 +1,63 @@ +{-# language DataKinds, KindSignatures #-} +{-# language MultiParamTypeClasses, TemplateHaskell, FunctionalDependencies, + FlexibleInstances #-} + +{-| +Module : Language.Python.Validate.Syntax.Error +Copyright : (C) CSIRO 2017-2018 +License : BSD3 +Maintainer : Isaac Elliott +Stability : experimental +Portability : non-portable +-} + +module Language.Python.Validate.Syntax.Error where + +import Control.Lens.TH +import Language.Python.Syntax.Expr (Expr) +import Language.Python.Syntax.Ident (Ident) + +data SyntaxError a + = PositionalAfterKeywordArg a (Expr '[] a) + | PositionalAfterKeywordUnpacking a (Expr '[] a) + | CannotAssignTo a (Expr '[] a) + | CannotDelete a (Expr '[] a) + | CannotAugAssignTo a (Expr '[] a) + | NoBindingNonlocal (Ident '[] a) + | PositionalAfterKeywordParam a String + | UnexpectedDoubleStarParam a String + | DuplicateArgument a String + | UnexpectedNewline a + | UnexpectedComment a + | IdentifierReservedWord a String + | EmptyIdentifier a + | BadCharacter a String + | BreakOutsideLoop a + | ContinueOutsideLoop a + | ReturnOutsideFunction a + | NonlocalOutsideFunction a + | ParametersNonlocal a [String] + | Can'tJoinStringAndBytes a + | YieldOutsideGenerator a + | MalformedDecorator a + | InvalidDictUnpacking a + | InvalidSetUnpacking a + | TypedParamInLambda a + | AsyncWithOutsideCoroutine a + | AsyncForOutsideCoroutine a + | YieldFromInsideCoroutine a + | YieldInsideCoroutine a + | AwaitOutsideCoroutine a + | AwaitInsideComprehension a + | NullByte a + | NonAsciiInBytes a Char + | DefaultExceptMustBeLast a + | WildcardImportInDefinition a + | NoKeywordsAfterEmptyStarArg a + | ManyStarredTargets a + | ManyStarredParams a + | ContinueInsideFinally a + | ParameterMarkedGlobal a String + deriving (Eq, Show) + +makeClassyPrisms ''SyntaxError diff --git a/test/DSL.hs b/test/DSL.hs new file mode 100644 --- /dev/null +++ b/test/DSL.hs @@ -0,0 +1,312 @@ +{-# language OverloadedStrings, TemplateHaskell #-} +module DSL (dslTests) where + +import Hedgehog + +import Control.Lens.Fold ((^?)) +import Control.Lens.Setter ((.~), over) +import Data.Function ((&)) + +import Language.Python.DSL +import Language.Python.Optics +import Language.Python.Render (showExpr) +import Language.Python.Syntax.CommaSep (CommaSep(..)) +import Language.Python.Syntax.Punctuation (Comma(..)) +import Language.Python.Syntax.Whitespace (Whitespace(..), Indents(..)) + +dslTests :: Group +dslTests = $$discover + +prop_subscript_1 :: Property +prop_subscript_1 = + withTests 1 . property $ do + let + expr = subs_ (var_ "a") (int_ 1) + showExpr expr === "a[1]" + +prop_subscript_2 :: Property +prop_subscript_2 = + withTests 1 . property $ do + let + expr = subs_ (var_ "a") (tuple_ [ti_ $ int_ 1, ti_ $ int_ 2]) + showExpr expr === "a[1, 2]" + +prop_subscript_3 :: Property +prop_subscript_3 = + withTests 1 . property $ do + let + expr = subs_ (var_ "a") (tuple_ [ti_ $ int_ 1, s_ $ var_ "b"]) + showExpr expr === "a[(1, *b)]" + +prop_subscript_4 :: Property +prop_subscript_4 = + withTests 1 . property $ do + let + expr = subs_ (var_ "a") (tuple_ [ti_ $ int_ 1]) + showExpr expr === "a[1,]" + +prop_subscript_5 :: Property +prop_subscript_5 = + withTests 1 . property $ do + let + expr = subs_ (var_ "a") (tuple_ [s_ $ var_ "b"]) + showExpr expr === "a[((*b),)]" + +prop_subscript_6 :: Property +prop_subscript_6 = + withTests 1 . property $ do + let + expr = subs_ (var_ "a") (slice_ Nothing Nothing Nothing) + showExpr expr === "a[:]" + +prop_subscript_7 :: Property +prop_subscript_7 = + withTests 1 . property $ do + let + expr = subs_ (var_ "a") (tuple_ [ti_ $ slice_ Nothing Nothing Nothing]) + showExpr expr === "a[:,]" + +prop_subscript_8 :: Property +prop_subscript_8 = + withTests 1 . property $ do + let + expr = subs_ (var_ "a") (tuple_ [ti_ $ tuple_ [ti_ $ slice_ Nothing Nothing Nothing]]) + showExpr expr === "a[(slice(None, None, None),),]" + +prop_subscript_9 :: Property +prop_subscript_9 = + withTests 1 . property $ do + let + expr = + subs_ (var_ "a") $ + tuple_ [ti_ $ slice_ Nothing Nothing Nothing, ti_ $ slice_ Nothing Nothing Nothing] + showExpr expr === "a[:, :]" + +prop_subscript_10 :: Property +prop_subscript_10 = + withTests 1 . property $ do + let + expr = subs_ (var_ "a") (slice_ (Just $ int_ 1) Nothing Nothing) + showExpr expr === "a[1:]" + +prop_subscript_11 :: Property +prop_subscript_11 = + withTests 1 . property $ do + let + expr = subs_ (var_ "a") (slice_ (Just $ int_ 1) (Just $ int_ 1) Nothing) + showExpr expr === "a[1:1]" + +prop_subscript_12 :: Property +prop_subscript_12 = + withTests 1 . property $ do + let + expr = subs_ (var_ "a") (slice_ (Just $ int_ 1) (Just $ int_ 1) (Just $ int_ 1)) + showExpr expr === "a[1:1:1]" + +prop_subscript_13 :: Property +prop_subscript_13 = + withTests 1 . property $ do + let + expr = + subs_ (var_ "a") $ + tuple_ + [ ti_ $ slice_ (Just $ int_ 1) Nothing Nothing + , ti_ $ slice_ (Just $ int_ 2) Nothing Nothing + ] + showExpr expr === "a[1:, 2:]" + +prop_subscript_14 :: Property +prop_subscript_14 = + withTests 1 . property $ do + let + expr = + subs_ (var_ "a") $ + tuple_ + [ ti_ $ slice_ (Just $ int_ 1) (Just $ int_ 1) Nothing + , ti_ $ slice_ (Just $ int_ 2) Nothing Nothing + ] + showExpr expr === "a[1:1, 2:]" + +prop_subscript_15 :: Property +prop_subscript_15 = + withTests 1 . property $ do + let + expr = + subs_ (var_ "a") $ + tuple_ + [ ti_ $ slice_ (Just $ int_ 1) (Just $ int_ 1) (Just $ int_ 1) + , ti_ $ slice_ (Just $ int_ 2) Nothing Nothing + ] + showExpr expr === "a[1:1:1, 2:]" + +prop_subscript_16 :: Property +prop_subscript_16 = + withTests 1 . property $ do + let + expr = + subs_ (var_ "a") $ + tuple_ + [ ti_ $ + tuple_ + [ ti_ $ slice_ (Just $ int_ 1) Nothing Nothing + , ti_ $ int_ 2 + ] + ] + showExpr expr === "a[(slice(1, None, None), 2),]" + +prop_parameters_1 :: Property +prop_parameters_1 = + withTests 1 . property $ do + let + st = def_ "a" [] [line_ pass_] + + params1 = + CommaSepMany (p_ "test1") (MkComma $ replicate 5 Space) $ + CommaSepMany (p_ "test2") (MkComma $ replicate 3 Space) $ + CommaSepNone + st1 = st & _Fundef.fdParameters .~ params1 + + params2 = + CommaSepMany (p_ "test3") (MkComma $ replicate 5 Space) $ + CommaSepMany (p_ "test4") (MkComma $ replicate 3 Space) $ + CommaSepNone + st2 = st & _Fundef.fdParameters .~ params2 + + (st1 & _Fundef.parameters_ .~ [p_ "test3", p_ "test4"]) === st2 + +prop_parameters_2 :: Property +prop_parameters_2 = + withTests 1 . property $ do + let + st = def_ "a" [] [line_ pass_] + + params1 = + CommaSepMany (p_ "test1") (MkComma $ replicate 5 Space) $ + CommaSepMany (p_ "test2") (MkComma $ replicate 3 Space) $ + CommaSepNone + st1 = st & _Fundef.fdParameters .~ params1 + + params2 = + CommaSepMany (p_ "test3") (MkComma $ replicate 5 Space) $ + CommaSepMany (p_ "test4") (MkComma $ replicate 3 Space) $ + CommaSepOne (p_ "test5") + st2 = st & _Fundef.fdParameters .~ params2 + + (st1 & _Fundef.parameters_ .~ [p_ "test3", p_ "test4", p_ "test5"]) === st2 + +prop_parameters_3 :: Property +prop_parameters_3 = + withTests 1 . property $ do + let + st = def_ "a" [] [line_ pass_] + + params1 = + CommaSepMany (p_ "test1") (MkComma $ replicate 5 Space) $ + CommaSepMany (p_ "test2") (MkComma $ replicate 3 Space) $ + CommaSepNone + st1 = st & _Fundef.fdParameters .~ params1 + + params2 = CommaSepMany (p_ "test3") (MkComma $ replicate 5 Space) CommaSepNone + st2 = st & _Fundef.fdParameters .~ params2 + + (st1 & _Fundef.parameters_ .~ [p_ "test3"]) === st2 + +prop_parameters_4 :: Property +prop_parameters_4 = + withTests 1 . property $ do + let + st = def_ "a" [] [line_ pass_] + + params1 = + CommaSepMany (p_ "test1") (MkComma $ replicate 5 Space) $ + CommaSepOne (p_ "test2") + st1 = st & _Fundef.fdParameters .~ params1 + + params2 = + CommaSepMany (p_ "test3") (MkComma $ replicate 5 Space) $ + CommaSepOne (p_ "test4") + st2 = st & _Fundef.fdParameters .~ params2 + + (st1 & _Fundef.parameters_ .~ [p_ "test3", p_ "test4"]) === st2 + +prop_parameters_5 :: Property +prop_parameters_5 = + withTests 1 . property $ do + let + st = def_ "a" [] [line_ pass_] + + params1 = + CommaSepMany (p_ "test1") (MkComma $ replicate 5 Space) $ + CommaSepOne (p_ "test2") + st1 = st & _Fundef.fdParameters .~ params1 + + params2 = + CommaSepMany (p_ "test3") (MkComma $ replicate 5 Space) $ + CommaSepMany (p_ "test4") (MkComma [Space]) $ + CommaSepOne (p_ "test5") + st2 = st & _Fundef.fdParameters .~ params2 + + (st1 & _Fundef.parameters_ .~ [p_ "test3", p_ "test4", p_ "test5"]) === st2 + +prop_body_1 :: Property +prop_body_1 = + withTests 1 . property $ do + let + st = def_ "a" [] [line_ pass_] + + st ^? _Fundef.fdIndents === Just (Indents [] ()) + over (_Fundef.body_) id st === st + +prop_body_2 :: Property +prop_body_2 = + withTests 1 . property $ do + let + st = def_ "a" [] [line_ pass_] + + st ^? _Fundef.body_ === Just [line_ pass_] + +prop_body_3 :: Property +prop_body_3 = + withTests 1 . property $ do + let + stInner = def_ "b" [] [line_ pass_] + stOuter = def_ "a" [] [line_ pass_, line_ stInner] + + newIndent = replicate 10 Space + + (stOuter & _Indent .~ newIndent) ^? _Fundef.body_ === + Just + [ line_ $ pass_ & _Indent .~ newIndent + , line_ $ stInner & _Indent .~ newIndent + ] + +prop_body_4 :: Property +prop_body_4 = + withTests 1 . property $ do + let + newIndent = replicate 10 Space + + stInner = def_ "b" [] [line_ pass_] + + outerBody = + [ line_ pass_ + , line_ stInner + ] + + outerBody' = + [ line_ $ pass_ & _Indent .~ newIndent + , line_ $ stInner & _Indent .~ newIndent + ] + + stOuter = def_ "a" [] outerBody & _Indent .~ newIndent + + finalBody = + [ line_ pass_ + , line_ $ stInner & _Indent .~ newIndent + , line_ pass_ + ] + + stFinal' = def_ "a" [] finalBody & _Indent .~ newIndent + + stOuter ^? _Fundef.body_ === Just outerBody' + (stOuter & _Fundef.body_ .~ finalBody) === stFinal' diff --git a/test/Helpers.hs b/test/Helpers.hs new file mode 100644 --- /dev/null +++ b/test/Helpers.hs @@ -0,0 +1,150 @@ +{-# language DataKinds #-} +{-# language RankNTypes #-} +{-# language ScopedTypeVariables #-} +module Helpers where + +import Hedgehog + +import Control.Lens.Fold ((^?), folded) +import Control.Monad (void) +import Data.List.NonEmpty (NonEmpty(..)) +import Data.Semigroup (Semigroup) +import Data.Text (Text) +import Data.Validation (Validation(..), _Failure) +import Text.Megaparsec.Pos (SourcePos(..), mkPos) + +import Language.Python.Internal.Lexer + (SrcInfo, insertTabs, tokenize + ) +import Language.Python.Internal.Token (PyToken) +import Language.Python.Parse (Parser) +import Language.Python.Parse.Error (ParseError, ErrorItem(..), _ParseError) +import Language.Python.Internal.Parse (runParser) +import Language.Python.Syntax.Expr (Expr) +import Language.Python.Syntax.Module (Module) +import Language.Python.Syntax.Statement (Statement) +import Language.Python.Validate + +doTokenize :: Monad m => Text -> PropertyT m [PyToken SrcInfo] +doTokenize input = + case tokenize "test" input of + Left err -> annotateShow (err :: ParseError SrcInfo) *> failure + Right a -> pure a + +doTabs + :: forall ann m + . (Semigroup ann, Show ann, Monad m) + => ann + -> [PyToken ann] + -> PropertyT m [PyToken ann] +doTabs ann input = + case insertTabs ann input of + Left err -> annotateShow (err :: ParseError ann) *> failure + Right a -> pure a + +doParse :: Monad m => Parser a -> [PyToken SrcInfo] -> PropertyT m a +doParse pa input = do + let res = runParser "test" pa input + case res of + Left err -> do + annotateShow (err :: ParseError SrcInfo) + failure + Right a -> pure a + +syntaxValidateModule + :: Module '[] () + -> PropertyT IO + (Validation + (NonEmpty (SyntaxError ())) + (Module '[Syntax, Indentation] ())) +syntaxValidateModule x = + case runValidateIndentation $ validateModuleIndentation x of + Failure errs -> do + annotateShow (errs :: NonEmpty (IndentationError ())) + failure + Success a -> + pure $ runValidateSyntax (validateModuleSyntax a) + +syntaxValidateStatement + :: Statement '[] () + -> PropertyT IO + (Validation + (NonEmpty (SyntaxError ())) + (Statement '[Syntax, Indentation] ())) +syntaxValidateStatement x = + case runValidateIndentation $ validateStatementIndentation x of + Failure errs -> do + annotateShow (errs :: NonEmpty (IndentationError ())) + failure + Success a -> + pure $ runValidateSyntax (validateStatementSyntax a) + +syntaxValidateExpr + :: Expr '[] () + -> PropertyT IO + (Validation + (NonEmpty (SyntaxError ())) + (Expr '[Syntax, Indentation] ())) +syntaxValidateExpr x = + case runValidateIndentation $ validateExprIndentation x of + Failure errs -> do + annotateShow (errs :: NonEmpty (IndentationError ())) + failure + Success a -> + pure $ runValidateSyntax (validateExprSyntax a) + +shouldBeFailure :: MonadTest m => Validation e a -> m () +shouldBeFailure res = + case res of + Success{} -> failure + Failure{} -> success + +shouldBeSuccess :: (MonadTest m, Show e) => Validation e a -> m a +shouldBeSuccess res = + case res of + Success a -> pure a + Failure err -> do + annotateShow err + failure + +shouldBeParseSuccess + :: MonadTest m + => (FilePath -> Text -> Validation (NonEmpty (ParseError SrcInfo)) a) + -> Text -> m a +shouldBeParseSuccess p = shouldBeSuccess . p "test" + +shouldBeParseFailure + :: MonadTest m + => (FilePath -> Text -> Validation (NonEmpty (ParseError SrcInfo)) a) + -> Text -> m () +shouldBeParseFailure p = shouldBeFailure . p "test" + +shouldBeParseError + :: (MonadTest m, Show e, Show a) + => Int + -> Int + -> PyToken () + -> Validation (NonEmpty (ParseError e)) a + -> m () +shouldBeParseError line col tk res = + case res ^? _Failure.folded._ParseError of + Just (srcPos :| _, Just (Tokens (errorItem :| [])), _) -> do + sourceLine srcPos === mkPos line + sourceColumn srcPos === mkPos col + + void errorItem === tk + _ -> do + annotateShow res + failure + +shouldBeSyntaxError + :: (MonadTest m, Show a) + => SyntaxError () + -> Validation (NonEmpty (SyntaxError ())) a + -> m () +shouldBeSyntaxError err res = + case res ^? _Failure.folded of + Just err' -> err === err' + _ -> do + annotateShow res + failure diff --git a/test/LexerParser.hs b/test/LexerParser.hs new file mode 100644 --- /dev/null +++ b/test/LexerParser.hs @@ -0,0 +1,543 @@ +{-# language OverloadedStrings, OverloadedLists, TemplateHaskell #-} +module LexerParser (lexerParserTests) where + +import Hedgehog +import Control.Monad (void) +import Data.List.NonEmpty (NonEmpty(..)) +import qualified Data.Text as Text + +import Language.Python.DSL +import Language.Python.Render +import Language.Python.Parse (parseModule, parseStatement, parseExpr, parseExprList) +import Language.Python.Syntax.CommaSep (CommaSep(..), Comma(..)) +import Language.Python.Syntax.Expr (Expr(..)) +import Language.Python.Syntax.Strings + ( StringLiteral(..), StringType(..), QuoteType(..), PyChar(..) + , RawBytesPrefix(..), RawStringPrefix(..) + ) +import Language.Python.Syntax.Whitespace (Whitespace(..)) + +import Helpers (shouldBeParseSuccess, shouldBeParseFailure) + +lexerParserTests :: Group +lexerParserTests = $$discover + +prop_fulltrip_1 :: Property +prop_fulltrip_1 = + withTests 1 . property $ do + let str = "def a(x, y=2, *z, **w):\n return 2 + 3" + + tree <- shouldBeParseSuccess parseStatement str + + showStatement tree === str + +prop_fulltrip_2 :: Property +prop_fulltrip_2 = + withTests 1 . property $ do + let str = "( 1\n *\n 3\n )" + + tree <- shouldBeParseSuccess parseExpr str + + showExpr tree === str + +prop_fulltrip_3 :: Property +prop_fulltrip_3 = + withTests 1 . property $ do + let str = "pass;" + + tree <- shouldBeParseSuccess parseStatement str + + showStatement tree === str + +prop_fulltrip_4 :: Property +prop_fulltrip_4 = + withTests 1 . property $ do + let str = "def a():\n pass\n #\n pass\n" + + tree <- shouldBeParseSuccess parseStatement str + + showStatement tree === str + +prop_fulltrip_5 :: Property +prop_fulltrip_5 = + withTests 1 . property $ do + let str = "if False:\n pass\n pass\nelse:\n pass\n pass\n" + + tree <- shouldBeParseSuccess parseStatement str + + showStatement tree === str + +prop_fulltrip_6 :: Property +prop_fulltrip_6 = + withTests 1 . property $ do + let str = "# blah\ndef boo():\n pass\n #bing\n # bop\n" + + tree <- shouldBeParseSuccess parseModule str + + showModule tree === str + +prop_fulltrip_7 :: Property +prop_fulltrip_7 = + withTests 1 . property $ do + let str = "if False:\n pass\nelse \\\n \\\r\n:\n pass\n" + + tree <- shouldBeParseSuccess parseModule str + + showModule tree === str + +prop_fulltrip_8 :: Property +prop_fulltrip_8 = + withTests 1 . property $ do + let str = "def a():\n \n pass\n pass\n" + + tree <- shouldBeParseSuccess parseModule str + + showModule tree === str + +prop_fulltrip_9 :: Property +prop_fulltrip_9 = + withTests 1 . property $ do + let + str = + "try:\n pass\nexcept False:\n pass\nelse:\n pass\nfinally:\n pass\n def a():\n pass\n pass\n" + + tree <- shouldBeParseSuccess parseModule str + + showModule tree === str + +prop_fulltrip_10 :: Property +prop_fulltrip_10 = + withTests 1 . property $ do + let + str = + Text.unlines + [ "from blah import boo" + , "import baz as wop" + , "" + , "def thing():" + , " pass" + , "" + , "def hello():" + , " what; up;" + , "" + , "def boo(a, *b, c=1, **d):" + , " pass" + ] + + tree <- shouldBeParseSuccess parseModule str + + showModule tree === str + +prop_fulltrip_11 :: Property +prop_fulltrip_11 = + withTests 1 . property $ do + let + str = + Text.unlines + [ "if False:" + , " pass" + , " pass" + , "else:" + , " \tpass" + , " \tpass" + ] + + tree <- shouldBeParseSuccess parseModule str + + showModule tree === str + +prop_fulltrip_12 :: Property +prop_fulltrip_12 = + withTests 1 . property $ do + let + str = + Text.unlines + [ "try:" + , " \tpass" + , " \tdef a():" + , " \t pass" + , " \tpass" + , "finally:" + , " pass" + ] + + tree <- shouldBeParseSuccess parseModule str + + showModule tree === str + +prop_fulltrip_13 :: Property +prop_fulltrip_13 = + withTests 1 . property $ do + let + str = + Text.unlines + [ "if []:" + , " False" + , " def a():" + , " pass" + , " pass" + , "" + , "else:" + , " pass" + , " pass" + ] + + tree <- shouldBeParseSuccess parseModule str + + showModule tree === str + +prop_fulltrip_14 :: Property +prop_fulltrip_14 = + withTests 1 . property $ do + let + str = "not ((False for a in False) if False else False or False)" + + tree <- shouldBeParseSuccess parseModule str + + showModule tree === str + +prop_fulltrip_15 :: Property +prop_fulltrip_15 = + withTests 1 . property $ do + let + str = "01." + + tree <- shouldBeParseSuccess parseModule str + + showModule tree === str + +prop_fulltrip_16 :: Property +prop_fulltrip_16 = + withTests 1 . property $ do + let + str = "def a():\n return ~i" + + tree <- shouldBeParseSuccess parseModule str + + showModule tree === str + +prop_fulltrip_17 :: Property +prop_fulltrip_17 = + withTests 1 . property $ do + let str = "r\"\\\"\"" + + tree <- shouldBeParseSuccess parseModule str + + showModule tree === str + +prop_fulltrip_18 :: Property +prop_fulltrip_18 = + withTests 1 . property $ do + let str = "\"\0\"" + + tree <- shouldBeParseSuccess parseModule str + + showModule tree === str + +prop_fulltrip_19 :: Property +prop_fulltrip_19 = + withTests 1 . property $ do + let str = " \\\n" + + shouldBeParseFailure parseModule str + +prop_fulltrip_20 :: Property +prop_fulltrip_20 = + withTests 1 . property $ do + let str = " pass" + + shouldBeParseFailure parseModule str + +prop_fulltrip_21 :: Property +prop_fulltrip_21 = + withTests 1 . property $ do + let str = "if a:\n \\\n\n pass" + + shouldBeParseFailure parseModule str + +prop_fulltrip_22 :: Property +prop_fulltrip_22 = + withTests 1 . property $ do + let str = "for a in (b, *c): pass" + + void $ shouldBeParseSuccess parseModule str + +prop_fulltrip_23 :: Property +prop_fulltrip_23 = + withTests 1 . property $ do + let str = "None,*None" + + void $ shouldBeParseSuccess parseModule str + +prop_fulltrip_24 :: Property +prop_fulltrip_24 = + withTests 1 . property $ do + let str = "'\1'" + + void $ shouldBeParseSuccess parseModule str + +prop_fulltrip_25 :: Property +prop_fulltrip_25 = + withTests 1 . property $ do + let str = "'\11'" + + void $ shouldBeParseSuccess parseModule str + +prop_fulltrip_26 :: Property +prop_fulltrip_26 = + withTests 1 . property $ do + let str = + showExpr $ + String () + (pure $ + RawBytesLiteral () + Prefix_br + LongString + SingleQuote + [ Char_esc_bslash ] + []) + annotateShow str + + res <- shouldBeParseSuccess parseExpr str + str === showExpr (() <$ res) + +prop_fulltrip_27 :: Property +prop_fulltrip_27 = + withTests 1 . property $ do + let str = + showExpr $ + String () + (pure $ + RawStringLiteral () + Prefix_r + LongString + SingleQuote + [ Char_lit '\\', Char_lit '\\', Char_lit '\\', Char_lit '\'' ] + []) + annotateShow str + + res <- shouldBeParseSuccess parseExpr str + str === showExpr (() <$ res) + +prop_fulltrip_28 :: Property +prop_fulltrip_28 = + withTests 1 . property $ do + let str = + showExpr $ + String () + (pure $ + RawStringLiteral () + Prefix_r + ShortString + DoubleQuote + [ Char_lit '\\' ] + []) + annotateShow str + + res <- shouldBeParseSuccess parseExpr str + str === showExpr (() <$ res) + +prop_fulltrip_29 :: Property +prop_fulltrip_29 = + withTests 1 . property $ do + let str = + showExpr $ + String () + (pure $ + RawStringLiteral () + Prefix_r + ShortString + DoubleQuote + [ Char_lit '\\', Char_lit '\\' ] + []) + annotateShow str + + res <- shouldBeParseSuccess parseExpr str + str === showExpr (() <$ res) + +prop_fulltrip_30 :: Property +prop_fulltrip_30 = + withTests 1 . property $ do + let str = + showExpr $ + String () + (pure $ + RawStringLiteral () + Prefix_r + ShortString + DoubleQuote + [ Char_lit '\\', Char_lit '\\', Char_lit '\\' ] + []) + annotateShow str + + res <- shouldBeParseSuccess parseExpr str + str === showExpr (() <$ res) + +prop_fulltrip_31 :: Property +prop_fulltrip_31 = + withTests 1 . property $ do + let str = "del(a)" + + void $ shouldBeParseSuccess parseModule str + +prop_fulltrip_32 :: Property +prop_fulltrip_32 = + withTests 1 . property $ do + let str = + showExpr $ + String () + (pure $ + RawStringLiteral () + Prefix_r + LongString + DoubleQuote + [ Char_lit ' ', Char_lit '"' ] + []) + annotateShow str + + res <- shouldBeParseSuccess parseExpr str + str === showExpr (() <$ res) + +prop_fulltrip_33 :: Property +prop_fulltrip_33 = + withTests 1 . property $ do + let str = + showExpr $ + String () + (pure $ + RawStringLiteral () + Prefix_r + LongString + DoubleQuote + [ Char_lit '"', Char_lit ' ' ] + []) + annotateShow str + + res <- shouldBeParseSuccess parseExpr str + str === showExpr (() <$ res) + +prop_fulltrip_34 :: Property +prop_fulltrip_34 = + withTests 1 . property $ do + let str = + showExpr $ + String () + (pure $ + RawStringLiteral () + Prefix_r + LongString + DoubleQuote + [ Char_lit '"' ] + []) + annotateShow str + + res <- shouldBeParseSuccess parseExpr str + str === showExpr (() <$ res) + +prop_fulltrip_35 :: Property +prop_fulltrip_35 = + withTests 1 . property $ do + let str = + showExpr $ + String () + (pure $ + RawStringLiteral () + Prefix_r + LongString + DoubleQuote + [ Char_lit '\\' + , Char_esc_bslash + , Char_esc_doublequote + ] + []) + annotateShow str + + res <- shouldBeParseSuccess parseExpr str + str === showExpr (() <$ res) + +prop_fulltrip_36 :: Property +prop_fulltrip_36 = + withTests 1 . property $ do + let str = + showExpr $ + String () + (RawStringLiteral () + Prefix_r + LongString + SingleQuote + [Char_lit '\\', Char_esc_bslash] [] :| + []) + annotateShow str + + res <- shouldBeParseSuccess parseExpr str + str === showExpr (() <$ res) + +prop_fulltrip_37 :: Property +prop_fulltrip_37 = + withTests 1 . property $ do + let str = "None,*None" + + void $ shouldBeParseSuccess parseExprList str + +prop_fulltrip_38 :: Property +prop_fulltrip_38 = + withTests 1 . property $ do + let str = + showExpr $ + String () + (RawStringLiteral () + Prefix_r + LongString + SingleQuote + [Char_esc_bslash, Char_lit '\\'] [] :| + []) + annotateShow str + + res <- shouldBeParseSuccess parseExpr str + str === showExpr (() <$ res) + +prop_fulltrip_39 :: Property +prop_fulltrip_39 = + withTests 1 . property $ do + let str = "def a(*b, *): pass" + + shouldBeParseFailure parseStatement str + +prop_fulltrip_40 :: Property +prop_fulltrip_40 = + withTests 1 . property $ do + let str = "def a():\n yield op, oparg" + res <- shouldBeParseSuccess parseStatement str + str === showStatement (() <$ res) + +prop_fulltrip_41 :: Property +prop_fulltrip_41 = + withTests 1 . property $ do + let + s = "def a(*a, *b): pass" + shouldBeParseFailure parseModule s + +prop_fulltrip_42 :: Property +prop_fulltrip_42 = + withTests 1 . property $ do + let + s = "lambda *a, *b: pass" + shouldBeParseFailure parseModule s + +prop_fulltrip_43 :: Property +prop_fulltrip_43 = + withTests 1 . property $ do + let + e = + Yield + { _unsafeExprAnn = () + , _unsafeYieldWhitespace = [Space] + , _unsafeYieldValue = + CommaSepMany (Ident (MkIdent () "a" [])) (MkComma [Space]) $ + CommaSepMany (tuple_ [ti_ $ var_ "b"]) (MkComma []) $ + CommaSepNone + } + -- yield a, (b,), + str = showExpr e + res <- shouldBeParseSuccess parseExpr str + str === showExpr (() <$ res) diff --git a/test/Main.hs b/test/Main.hs new file mode 100644 --- /dev/null +++ b/test/Main.hs @@ -0,0 +1,25 @@ +{-# options_ghc -fno-warn-unused-do-bind #-} +{-# language DataKinds, TypeOperators, FlexibleContexts #-} +{-# language OverloadedStrings #-} +module Main where + +import DSL +import LexerParser +import Optics +import Parser +import Roundtrip +import Scope +import Syntax + +import Control.Monad (when) +import System.Exit + +import Hedgehog + +main :: IO () +main = do + results <- traverse checkParallel groups + when (not (and results)) + exitFailure + where + groups = [lexerParserTests, dslTests, parserTests, opticsTests, scopeTests, syntaxTests, roundtripTests] diff --git a/test/Optics.hs b/test/Optics.hs new file mode 100644 --- /dev/null +++ b/test/Optics.hs @@ -0,0 +1,43 @@ +{-# language OverloadedStrings, TemplateHaskell #-} +module Optics (opticsTests) where + +import Hedgehog + +import Control.Lens.Plated (transformOn) +import Control.Lens.Setter ((.~)) +import Control.Monad.IO.Class (liftIO) +import qualified Data.Text.IO as Text + +import Language.Python.Parse (parseModule) +import Language.Python.Render (showModule) +import Language.Python.Syntax.Statement (_Statements) +import Language.Python.Syntax.Whitespace (Whitespace (..)) +import Language.Python.Optics (_Indent) + +import Helpers (shouldBeParseSuccess) + +opticsTests :: Group +opticsTests = $$discover + +prop_optics_1 :: Property +prop_optics_1 = + withTests 1 . property $ do + str <- liftIO $ Text.readFile "test/files/indent_optics_in.py" + + tree <- shouldBeParseSuccess parseModule str + + str' <- liftIO $ Text.readFile "test/files/indent_optics_out.py" + showModule + (transformOn _Statements (_Indent .~ [Space, Space, Space, Space]) tree) === str' + +prop_optics_2 :: Property +prop_optics_2 = + withTests 1 . property $ do + str <- liftIO $ Text.readFile "test/files/indent_optics_in2.py" + + tree <- shouldBeParseSuccess parseModule str + -- annotateShow $! tree + + str' <- liftIO $ Text.readFile "test/files/indent_optics_out2.py" + showModule + (transformOn _Statements (_Indent .~ [Space, Space, Space, Space]) tree) === str' diff --git a/test/Parser.hs b/test/Parser.hs new file mode 100644 --- /dev/null +++ b/test/Parser.hs @@ -0,0 +1,21 @@ +{-# language OverloadedStrings, TemplateHaskell #-} +module Parser (parserTests) where + +import Hedgehog + +import Language.Python.Internal.Token (PyToken(..)) +import Language.Python.Parse (parseStatement) + +import Helpers (shouldBeParseError) + +parserTests :: Group +parserTests = $$discover + +prop_parser_1 :: Property +prop_parser_1 = + withTests 1 . property $ do + let + e = "for x in a, *b: pass" + res = parseStatement "test" e + + shouldBeParseError 1 13 (TkStar ()) res diff --git a/test/Roundtrip.hs b/test/Roundtrip.hs new file mode 100644 --- /dev/null +++ b/test/Roundtrip.hs @@ -0,0 +1,83 @@ +{-# language OverloadedStrings #-} +{-# language DataKinds #-} +module Roundtrip (roundtripTests) where + +import Control.Monad.IO.Class (liftIO) +import Data.List.NonEmpty (NonEmpty) +import Data.String (fromString) +import Data.Text (Text) +import Data.Validation (Validation(..)) +import Hedgehog + ( (===), Group(..), Property, PropertyT, annotateShow, failure, property + , withTests, withShrinks + ) +import System.FilePath (()) + +import qualified Data.Text.IO as StrictText + +import Language.Python.Internal.Lexer (SrcInfo) +import Language.Python.Render (showModule) +import Language.Python.Parse (parseModule) +import Language.Python.Validate + ( IndentationError, SyntaxError + , runValidateIndentation, validateModuleIndentation, runValidateSyntax + , validateModuleSyntax + ) + +import Helpers (shouldBeParseSuccess) + +roundtripTests :: Group +roundtripTests = + Group "Roundtrip tests" $ + (\name -> (fromString name, withTests 1 . withShrinks 0 $ doRoundtripFile name)) <$> + [ "decorators.py" + , "string.py" + , "set.py" + , "regex.py" + , "asyncstatements.py" + , "typeann.py" + , "dictcomp.py" + , "imaginary.py" + , "weird.py" + , "weird2.py" + , "django.py" + , "django2.py" + , "test.py" + , "ansible.py" + , "comments.py" + , "pypy.py" + , "pypy2.py" + , "sqlalchemy.py" + , "numpy.py" + , "numpy2.py" + , "mypy.py" + , "mypy2.py" + , "requests.py" + , "requests2.py" + , "joblib.py" + , "joblib2.py" + , "pandas.py" + , "pandas2.py" + ] + +doRoundtripFile :: FilePath -> Property +doRoundtripFile name = + property $ do + file <- liftIO . StrictText.readFile $ "test/files" name + doRoundtrip file + +doRoundtrip :: Text -> PropertyT IO () +doRoundtrip file = do + py <- shouldBeParseSuccess parseModule file + case runValidateIndentation $ validateModuleIndentation py of + Failure errs -> do + annotateShow (errs :: NonEmpty (IndentationError SrcInfo)) + failure + Success res -> + case runValidateSyntax (validateModuleSyntax res) of + Failure errs' -> do + annotateShow (errs' :: NonEmpty (SyntaxError SrcInfo)) + failure + Success _ -> do + annotateShow py + showModule py === file diff --git a/test/Scope.hs b/test/Scope.hs new file mode 100644 --- /dev/null +++ b/test/Scope.hs @@ -0,0 +1,192 @@ +{-# language OverloadedStrings, DataKinds, TemplateHaskell #-} +module Scope (scopeTests) where + +import Hedgehog + +import Control.Lens ((#), has) +import Data.Function ((&)) +import Data.Functor (($>)) +import Data.List.NonEmpty (NonEmpty ((:|))) +import Data.Validation (Validation(..), _Success) + +import Language.Python.Validate +import Language.Python.DSL +import Language.Python.Optics +import Language.Python.Syntax.Whitespace + +scopeTests :: Group +scopeTests = $$discover + +fullyValidate + :: Statement '[] () + -> PropertyT IO + (Validation + (NonEmpty (ScopeError ())) + (Statement '[Scope, Syntax, Indentation] ())) +fullyValidate x = + case runValidateIndentation $ validateStatementIndentation x of + Failure errs -> do + annotateShow (errs :: NonEmpty (IndentationError ())) + failure + Success a -> + case runValidateSyntax (validateStatementSyntax a) of + Failure errs -> do + annotateShow (errs :: NonEmpty (SyntaxError ())) + failure + Success a' -> pure $ runValidateScope (validateStatementScope a') + +prop_scope_1 :: Property +prop_scope_1 = + withTests 1 . property $ do + let + expr = + _Fundef # + def_ "test" [p_ "a", p_ "b"] + [ line_ $ if_ true_ [ line_ (var_ "c" .= 2) ] + , line_ . return_ $ var_ "a" .+ var_ "b" .+ var_ "c" + ] + res <- fullyValidate expr + res === Failure (FoundDynamic () (MkIdent () "c" []) :| []) + +prop_scope_2 :: Property +prop_scope_2 = + withTests 1 . property $ do + let + expr = + _Fundef # + def_ "test" [p_ "a", p_ "b"] + [ line_ (var_ "c" .= 0) + , line_ $ if_ true_ [ line_ (var_ "c" .= 2) ] + , line_ . return_ $ var_ "a" .+ var_ "b" .+ var_ "c" + ] + res <- fullyValidate expr + annotateShow res + assert $ has _Success res + +prop_scope_3 :: Property +prop_scope_3 = + withTests 1 . property $ do + let + expr = + _Fundef # + def_ "test" [p_ "a", p_ "b"] + [ line_ . return_ $ var_ "a" .+ var_ "b" .+ var_ "c" ] + res <- fullyValidate expr + annotateShow res + res === Failure (NotInScope (MkIdent () "c" []) :| []) + +prop_scope_4 :: Property +prop_scope_4 = + withTests 1 . property $ do + let + expr = + _Fundef # + def_ "test" [p_ "a", p_ "b"] + [ line_ $ def_ "f" [] [ line_ $ def_ "g" [] [ line_ pass_ ] ] + , line_ $ call_ (var_ "g") [] + ] + res <- fullyValidate expr + res === Failure (NotInScope (MkIdent () "g" []) :| []) + +prop_scope_5 :: Property +prop_scope_5 = + withTests 1 . property $ do + let + expr = + _Fundef # + def_ "test" [p_ "a"] + [ line_ $ def_ "f" [k_ "b" (var_ "c")] [ line_ pass_ ] + ] + res <- fullyValidate expr + annotateShow res + res === Failure (NotInScope (MkIdent () "c" []) :| []) + +prop_scope_6 :: Property +prop_scope_6 = + withTests 1 . property $ do + let + expr = + _Fundef # + def_ "test" [] + [ line_ $ + if_ true_ [ line_ (var_ "x" .= 2) ] & + else_ [ line_ pass_ ] + , line_ $ var_ "x" + ] + res <- fullyValidate expr + annotateShow res + res === Failure (FoundDynamic () (MkIdent () "x" []) :| []) + +prop_scope_7 :: Property +prop_scope_7 = + withTests 1 . property $ do + let + expr = + _Fundef # + def_ "test" [] + [ line_ $ + if_ true_ [ line_ pass_ ] & + else_ [ line_ (var_ "x" .= 3) ] + , line_ $ var_ "x" + ] + res <- fullyValidate expr + annotateShow res + res === Failure (FoundDynamic () (MkIdent () "x" []) :| []) + +prop_scope_8 :: Property +prop_scope_8 = + withTests 1 . property $ do + let + expr = + _Fundef # + def_ "test" [] + [ line_ $ + if_ true_ [ line_ pass_ ] & + else_ [ line_ (var_ "x" .= 3) ] + , line_ (var_ "x" .= 1) + , line_ $ var_ "x" + ] + res <- fullyValidate expr + annotateShow res + (res $> ()) === Success () + +prop_scope_9 :: Property +prop_scope_9 = + withTests 1 . property $ do + let + expr = + _Fundef # + def_ "test" [] + [ line_ $ for_ ("x" `in_` [ list_ [li_ $ int_ 1] ]) [ line_ pass_ ] + , line_ $ var_ "x" + ] + res <- fullyValidate expr + annotateShow res + res === Failure (FoundDynamic () (MkIdent () "x" []) :| []) + +prop_scope_10 :: Property +prop_scope_10 = + withTests 1 . property $ do + let + expr = + _Fundef # + def_ "test" [] + [ line_ $ for_ ("x" `in_` [ list_ [li_ $ int_ 1] ]) [ line_ $ var_ "x" ] + ] + res <- fullyValidate expr + annotateShow res + (res $> ()) === Success () + +prop_scope_11 :: Property +prop_scope_11 = + withTests 1 . property $ do + let + st = + _Fundef # + def_ "test" [] + [ line_ ("x" .= 2) + , line_ $ for_ ("x" `in_` [ list_ [li_ $ int_ 1] ]) [ line_ pass_ ] + ] + res <- fullyValidate st + annotateShow res + res === Failure (BadShadowing (MkIdent () "x" [Space]) :| []) diff --git a/test/Syntax.hs b/test/Syntax.hs new file mode 100644 --- /dev/null +++ b/test/Syntax.hs @@ -0,0 +1,180 @@ +{-# language OverloadedStrings, TemplateHaskell #-} +{-# language DataKinds #-} +module Syntax (syntaxTests) where + +import Hedgehog + +import Control.Lens.Iso (from) +import Control.Lens.Getter ((^.)) +import Control.Lens.Review ((#)) +import Control.Monad (void) + +import Language.Python.DSL +import Language.Python.Optics +import Language.Python.Parse (parseModule, parseStatement, parseExpr) +import Language.Python.Render (showStatement, showExpr) +import Language.Python.Syntax.CommaSep +import Language.Python.Syntax.Expr +import Language.Python.Syntax.Punctuation +import Language.Python.Syntax.Statement +import Language.Python.Syntax.Strings +import Language.Python.Syntax.Whitespace + +import Helpers + ( shouldBeParseSuccess, shouldBeFailure, shouldBeSuccess + , syntaxValidateExpr, syntaxValidateStatement, syntaxValidateModule + ) + +syntaxTests :: Group +syntaxTests = $$discover + +prop_syntax_1 :: Property +prop_syntax_1 = + withTests 1 . property $ do + let + e = + -- lambda *: None + Lambda () + [Space] + (CommaSepMany (UnnamedStarParam () []) (MkComma []) CommaSepNone) + (MkColon [Space]) + (None () []) + res <- syntaxValidateExpr e + shouldBeFailure res + +prop_syntax_2 :: Property +prop_syntax_2 = + withTests 1 . property $ do + let + i = replicate 4 Space ^. from indentWhitespaces + e :: Statement '[] () + e = + CompoundStatement . + Fundef () [] + (Indents mempty ()) + Nothing + (pure Space) + "test" + [] CommaSepNone [] Nothing . + SuiteMany () (MkColon []) Nothing LF $ + Block [] + (SmallStatement (Indents [i] ()) $ + MkSmallStatement (Pass () []) [] Nothing Nothing Nothing) + [Right . SmallStatement (Indents [i] ()) $ + MkSmallStatement (Pass () []) [] Nothing Nothing Nothing] + res <- shouldBeParseSuccess parseStatement (showStatement e) + res' <- shouldBeParseSuccess parseStatement (showStatement res) + void res === void res' + +prop_syntax_3 :: Property +prop_syntax_3 = + withTests 1 . property $ do + let + s = "@a\ndef a():\n pass\n @a\n class a: return " + e <- shouldBeParseSuccess parseModule s + shouldBeFailure =<< syntaxValidateModule (() <$ e) + +prop_syntax_4 :: Property +prop_syntax_4 = + withTests 1 . property $ do + let + e :: Expr '[] () + e = + String () . pure $ + StringLiteral () + Nothing + ShortString SingleQuote + [Char_lit '\\', Char_lit 'u'] + [] + res <- shouldBeParseSuccess parseExpr (showExpr e) + res' <- shouldBeParseSuccess parseExpr (showExpr res) + void res === void res' + +prop_syntax_5 :: Property +prop_syntax_5 = + withTests 1 . property $ do + let + e :: Expr '[] () + e = + String () . pure $ + StringLiteral () + Nothing + ShortString SingleQuote + [Char_lit '\\', Char_lit 'x'] + [] + res <- shouldBeParseSuccess parseExpr (showExpr e) + res' <- shouldBeParseSuccess parseExpr (showExpr res) + void res === void res' + +prop_syntax_6 :: Property +prop_syntax_6 = + withTests 1 . property $ do + let s= "async def a():\n class a(await None):\n pass" + e <- shouldBeParseSuccess parseModule s + void . shouldBeSuccess =<< syntaxValidateModule (() <$ e) + +prop_syntax_7 :: Property +prop_syntax_7 = + withTests 1 . property $ do + let + s = "def a(b): global b" + e <- shouldBeParseSuccess parseModule s + shouldBeFailure =<< syntaxValidateModule (() <$ e) + +prop_syntax_8 :: Property +prop_syntax_8 = + withTests 1 . property $ do + let + s = "def a(*): pass" + e <- shouldBeParseSuccess parseModule s + shouldBeFailure =<< syntaxValidateModule (() <$ e) + +prop_syntax_9 :: Property +prop_syntax_9 = + withTests 1 . property $ do + let + s = "def a(*, b=None): pass" + e <- shouldBeParseSuccess parseModule s + void . shouldBeSuccess =<< syntaxValidateModule (() <$ e) + +prop_syntax_10 :: Property +prop_syntax_10 = + withTests 1 . property $ do + let e = _Fundef # def_ "a" [s_ "b", s_ "c"] [line_ pass_] + void . shouldBeFailure =<< syntaxValidateStatement e + +prop_syntax_11 :: Property +prop_syntax_11 = + withTests 1 . property $ do + let e = lambda_ [s_ "a", s_ "b"] (var_ "a") + void . shouldBeFailure =<< syntaxValidateExpr e + +prop_syntax_12 :: Property +prop_syntax_12 = + withTests 1 . property $ do + let e = _Fundef # def_ "a" [star_, s_ "b"] [line_ pass_] + void . shouldBeFailure =<< syntaxValidateStatement e + +prop_syntax_13 :: Property +prop_syntax_13 = + withTests 1 . property $ do + let e = lambda_ [star_, s_ "a"] (var_ "b") + void . shouldBeFailure =<< syntaxValidateExpr e + +prop_syntax_14 :: Property +prop_syntax_14 = + withTests 1 . property $ do + let e = _Fundef # def_ "a" [star_, k_ "b" none_, s_ "c"] [line_ pass_] + void . shouldBeFailure =<< syntaxValidateStatement e + +prop_syntax_15 :: Property +prop_syntax_15 = + withTests 1 . property $ do + let e = lambda_ [star_, k_ "a" none_, s_ "b"] (var_ "c") + void . shouldBeFailure =<< syntaxValidateExpr e + +prop_syntax_16 :: Property +prop_syntax_16 = + withTests 1 . property $ do + let e = lambda_ [star_, star_, k_ "a" none_] (var_ "c") + void . shouldBeFailure =<< syntaxValidateExpr e diff --git a/test/files/ansible.py b/test/files/ansible.py new file mode 100644 --- /dev/null +++ b/test/files/ansible.py @@ -0,0 +1,939 @@ +# (c) 2013-2014, Michael DeHaan +# (c) 2015 Toshio Kuratomi +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see . + +# Make coding more python3-ish +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +import ast +import base64 +import datetime +import imp +import json +import os +import shlex +import zipfile +import random +import re +from io import BytesIO + +from ansible.release import __version__, __author__ +from ansible import constants as C +from ansible.errors import AnsibleError +from ansible.module_utils._text import to_bytes, to_text, to_native +from ansible.plugins.loader import module_utils_loader, ps_module_utils_loader +from ansible.plugins.shell.powershell import async_watchdog, async_wrapper, become_wrapper, leaf_exec, exec_wrapper +# Must import strategy and use write_locks from there +# If we import write_locks directly then we end up binding a +# variable to the object and then it never gets updated. +from ansible.executor import action_write_locks + +try: + from __main__ import display +except ImportError: + from ansible.utils.display import Display + display = Display() + + +REPLACER = b"#<>" +REPLACER_VERSION = b"\"<>\"" +REPLACER_COMPLEX = b"\"<>\"" +REPLACER_WINDOWS = b"# POWERSHELL_COMMON" +REPLACER_JSONARGS = b"<>" +REPLACER_SELINUX = b"<>" + +# We could end up writing out parameters with unicode characters so we need to +# specify an encoding for the python source file +ENCODING_STRING = u'# -*- coding: utf-8 -*-' +b_ENCODING_STRING = b'# -*- coding: utf-8 -*-' + +# module_common is relative to module_utils, so fix the path +_MODULE_UTILS_PATH = os.path.join(os.path.dirname(__file__), '..', 'module_utils') + +# ****************************************************************************** + +ANSIBALLZ_TEMPLATE = u'''%(shebang)s +%(coding)s +ANSIBALLZ_WRAPPER = True # For test-module script to tell this is a ANSIBALLZ_WRAPPER +# This code is part of Ansible, but is an independent component. +# The code in this particular templatable string, and this templatable string +# only, is BSD licensed. Modules which end up using this snippet, which is +# dynamically combined together by Ansible still belong to the author of the +# module, and they may assign their own license to the complete work. +# +# Copyright (c), James Cammarata, 2016 +# Copyright (c), Toshio Kuratomi, 2016 +# +# Redistribution and use in source and binary forms, with or without modification, +# are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. +# IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE +# USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +import os +import os.path +import sys +import __main__ + +# For some distros and python versions we pick up this script in the temporary +# directory. This leads to problems when the ansible module masks a python +# library that another import needs. We have not figured out what about the +# specific distros and python versions causes this to behave differently. +# +# Tested distros: +# Fedora23 with python3.4 Works +# Ubuntu15.10 with python2.7 Works +# Ubuntu15.10 with python3.4 Fails without this +# Ubuntu16.04.1 with python3.5 Fails without this +# To test on another platform: +# * use the copy module (since this shadows the stdlib copy module) +# * Turn off pipelining +# * Make sure that the destination file does not exist +# * ansible ubuntu16-test -m copy -a 'src=/etc/motd dest=/var/tmp/m' +# This will traceback in shutil. Looking at the complete traceback will show +# that shutil is importing copy which finds the ansible module instead of the +# stdlib module +scriptdir = None +try: + scriptdir = os.path.dirname(os.path.realpath(__main__.__file__)) +except (AttributeError, OSError): + # Some platforms don't set __file__ when reading from stdin + # OSX raises OSError if using abspath() in a directory we don't have + # permission to read (realpath calls abspath) + pass +if scriptdir is not None: + sys.path = [p for p in sys.path if p != scriptdir] + +import base64 +import shutil +import zipfile +import tempfile +import subprocess + +if sys.version_info < (3,): + bytes = str + PY3 = False +else: + unicode = str + PY3 = True +try: + # Python-2.6+ + from io import BytesIO as IOStream +except ImportError: + # Python < 2.6 + from StringIO import StringIO as IOStream + +ZIPDATA = """%(zipdata)s""" + +def invoke_module(module, modlib_path, json_params): + pythonpath = os.environ.get('PYTHONPATH') + if pythonpath: + os.environ['PYTHONPATH'] = ':'.join((modlib_path, pythonpath)) + else: + os.environ['PYTHONPATH'] = modlib_path + + p = subprocess.Popen([%(interpreter)s, module], env=os.environ, shell=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE) + (stdout, stderr) = p.communicate(json_params) + + if not isinstance(stderr, (bytes, unicode)): + stderr = stderr.read() + if not isinstance(stdout, (bytes, unicode)): + stdout = stdout.read() + if PY3: + sys.stderr.buffer.write(stderr) + sys.stdout.buffer.write(stdout) + else: + sys.stderr.write(stderr) + sys.stdout.write(stdout) + return p.returncode + +def debug(command, zipped_mod, json_params): + # The code here normally doesn't run. It's only used for debugging on the + # remote machine. + # + # The subcommands in this function make it easier to debug ansiballz + # modules. Here's the basic steps: + # + # Run ansible with the environment variable: ANSIBLE_KEEP_REMOTE_FILES=1 and -vvv + # to save the module file remotely:: + # $ ANSIBLE_KEEP_REMOTE_FILES=1 ansible host1 -m ping -a 'data=october' -vvv + # + # Part of the verbose output will tell you where on the remote machine the + # module was written to:: + # [...] + # SSH: EXEC ssh -C -q -o ControlMaster=auto -o ControlPersist=60s -o KbdInteractiveAuthentication=no -o + # PreferredAuthentications=gssapi-with-mic,gssapi-keyex,hostbased,publickey -o PasswordAuthentication=no -o ConnectTimeout=10 -o + # ControlPath=/home/badger/.ansible/cp/ansible-ssh-%%h-%%p-%%r -tt rhel7 '/bin/sh -c '"'"'LANG=en_US.UTF-8 LC_ALL=en_US.UTF-8 + # LC_MESSAGES=en_US.UTF-8 /usr/bin/python /home/badger/.ansible/tmp/ansible-tmp-1461173013.93-9076457629738/ping'"'"'' + # [...] + # + # Login to the remote machine and run the module file via from the previous + # step with the explode subcommand to extract the module payload into + # source files:: + # $ ssh host1 + # $ /usr/bin/python /home/badger/.ansible/tmp/ansible-tmp-1461173013.93-9076457629738/ping explode + # Module expanded into: + # /home/badger/.ansible/tmp/ansible-tmp-1461173408.08-279692652635227/ansible + # + # You can now edit the source files to instrument the code or experiment with + # different parameter values. When you're ready to run the code you've modified + # (instead of the code from the actual zipped module), use the execute subcommand like this:: + # $ /usr/bin/python /home/badger/.ansible/tmp/ansible-tmp-1461173013.93-9076457629738/ping execute + + # Okay to use __file__ here because we're running from a kept file + basedir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'debug_dir') + args_path = os.path.join(basedir, 'args') + script_path = os.path.join(basedir, 'ansible_module_%(ansible_module)s.py') + + if command == 'explode': + # transform the ZIPDATA into an exploded directory of code and then + # print the path to the code. This is an easy way for people to look + # at the code on the remote machine for debugging it in that + # environment + z = zipfile.ZipFile(zipped_mod) + for filename in z.namelist(): + if filename.startswith('/'): + raise Exception('Something wrong with this module zip file: should not contain absolute paths') + + dest_filename = os.path.join(basedir, filename) + if dest_filename.endswith(os.path.sep) and not os.path.exists(dest_filename): + os.makedirs(dest_filename) + else: + directory = os.path.dirname(dest_filename) + if not os.path.exists(directory): + os.makedirs(directory) + f = open(dest_filename, 'wb') + f.write(z.read(filename)) + f.close() + + # write the args file + f = open(args_path, 'wb') + f.write(json_params) + f.close() + + print('Module expanded into:') + print('%%s' %% basedir) + exitcode = 0 + + elif command == 'execute': + # Execute the exploded code instead of executing the module from the + # embedded ZIPDATA. This allows people to easily run their modified + # code on the remote machine to see how changes will affect it. + # This differs slightly from default Ansible execution of Python modules + # as it passes the arguments to the module via a file instead of stdin. + + # Set pythonpath to the debug dir + pythonpath = os.environ.get('PYTHONPATH') + if pythonpath: + os.environ['PYTHONPATH'] = ':'.join((basedir, pythonpath)) + else: + os.environ['PYTHONPATH'] = basedir + + p = subprocess.Popen([%(interpreter)s, script_path, args_path], + env=os.environ, shell=False, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, stdin=subprocess.PIPE) + (stdout, stderr) = p.communicate() + + if not isinstance(stderr, (bytes, unicode)): + stderr = stderr.read() + if not isinstance(stdout, (bytes, unicode)): + stdout = stdout.read() + if PY3: + sys.stderr.buffer.write(stderr) + sys.stdout.buffer.write(stdout) + else: + sys.stderr.write(stderr) + sys.stdout.write(stdout) + return p.returncode + + elif command == 'excommunicate': + # This attempts to run the module in-process (by importing a main + # function and then calling it). It is not the way ansible generally + # invokes the module so it won't work in every case. It is here to + # aid certain debuggers which work better when the code doesn't change + # from one process to another but there may be problems that occur + # when using this that are only artifacts of how we're invoking here, + # not actual bugs (as they don't affect the real way that we invoke + # ansible modules) + + # stub the args and python path + sys.argv = ['%(ansible_module)s', args_path] + sys.path.insert(0, basedir) + + from ansible_module_%(ansible_module)s import main + main() + print('WARNING: Module returned to wrapper instead of exiting') + sys.exit(1) + else: + print('WARNING: Unknown debug command. Doing nothing.') + exitcode = 0 + + return exitcode + +if __name__ == '__main__': + # + # See comments in the debug() method for information on debugging + # + + ANSIBALLZ_PARAMS = %(params)s + if PY3: + ANSIBALLZ_PARAMS = ANSIBALLZ_PARAMS.encode('utf-8') + try: + # There's a race condition with the controller removing the + # remote_tmpdir and this module executing under async. So we cannot + # store this in remote_tmpdir (use system tempdir instead) + temp_path = tempfile.mkdtemp(prefix='ansible_') + + zipped_mod = os.path.join(temp_path, 'ansible_modlib.zip') + modlib = open(zipped_mod, 'wb') + modlib.write(base64.b64decode(ZIPDATA)) + modlib.close() + + if len(sys.argv) == 2: + exitcode = debug(sys.argv[1], zipped_mod, ANSIBALLZ_PARAMS) + else: + z = zipfile.ZipFile(zipped_mod, mode='r') + module = os.path.join(temp_path, 'ansible_module_%(ansible_module)s.py') + f = open(module, 'wb') + f.write(z.read('ansible_module_%(ansible_module)s.py')) + f.close() + + # When installed via setuptools (including python setup.py install), + # ansible may be installed with an easy-install.pth file. That file + # may load the system-wide install of ansible rather than the one in + # the module. sitecustomize is the only way to override that setting. + z = zipfile.ZipFile(zipped_mod, mode='a') + + # py3: zipped_mod will be text, py2: it's bytes. Need bytes at the end + sitecustomize = u'import sys\\nsys.path.insert(0,"%%s")\\n' %% zipped_mod + sitecustomize = sitecustomize.encode('utf-8') + # Use a ZipInfo to work around zipfile limitation on hosts with + # clocks set to a pre-1980 year (for instance, Raspberry Pi) + zinfo = zipfile.ZipInfo() + zinfo.filename = 'sitecustomize.py' + zinfo.date_time = ( %(year)i, %(month)i, %(day)i, %(hour)i, %(minute)i, %(second)i) + z.writestr(zinfo, sitecustomize) + z.close() + + exitcode = invoke_module(module, zipped_mod, ANSIBALLZ_PARAMS) + finally: + try: + shutil.rmtree(temp_path) + except (NameError, OSError): + # tempdir creation probably failed + pass + sys.exit(exitcode) +''' + + +def _strip_comments(source): + # Strip comments and blank lines from the wrapper + buf = [] + for line in source.splitlines(): + l = line.strip() + if not l or l.startswith(u'#'): + continue + buf.append(line) + return u'\n'.join(buf) + + +if C.DEFAULT_KEEP_REMOTE_FILES: + # Keep comments when KEEP_REMOTE_FILES is set. That way users will see + # the comments with some nice usage instructions + ACTIVE_ANSIBALLZ_TEMPLATE = ANSIBALLZ_TEMPLATE +else: + # ANSIBALLZ_TEMPLATE stripped of comments for smaller over the wire size + ACTIVE_ANSIBALLZ_TEMPLATE = _strip_comments(ANSIBALLZ_TEMPLATE) + + +class ModuleDepFinder(ast.NodeVisitor): + # Caveats: + # This code currently does not handle: + # * relative imports from py2.6+ from . import urls + IMPORT_PREFIX_SIZE = len('ansible.module_utils.') + def __init__(self, *args, **kwargs): + """ + Walk the ast tree for the python module. + + Save submodule[.submoduleN][.identifier] into self.submodules + + self.submodules will end up with tuples like: + - ('basic',) + - ('urls', 'fetch_url') + - ('database', 'postgres') + - ('database', 'postgres', 'quote') + + It's up to calling code to determine whether the final element of the + dotted strings are module names or something else (function, class, or + variable names) + """ + super(ModuleDepFinder, self).__init__(*args, **kwargs) + self.submodules = set() + + def visit_Import(self, node): + # import ansible.module_utils.MODLIB[.MODLIBn] [as asname] + for alias in (a for a in node.names if a.name.startswith('ansible.module_utils.')): + py_mod = alias.name[self.IMPORT_PREFIX_SIZE:] + py_mod = tuple(py_mod.split('.')) + self.submodules.add(py_mod) + self.generic_visit(node) + + def visit_ImportFrom(self, node): + # Specialcase: six is a special case because of its + # import logic + if node.names[0].name == '_six': + self.submodules.add(('_six',)) + elif node.module.startswith('ansible.module_utils'): + where_from = node.module[self.IMPORT_PREFIX_SIZE:] + if where_from: + # from ansible.module_utils.MODULE1[.MODULEn] import IDENTIFIER [as asname] + # from ansible.module_utils.MODULE1[.MODULEn] import MODULEn+1 [as asname] + # from ansible.module_utils.MODULE1[.MODULEn] import MODULEn+1 [,IDENTIFIER] [as asname] + py_mod = tuple(where_from.split('.')) + for alias in node.names: + self.submodules.add(py_mod + (alias.name,)) + else: + # from ansible.module_utils import MODLIB [,MODLIB2] [as asname] + for alias in node.names: + self.submodules.add((alias.name,)) + self.generic_visit(node) + + +def _slurp(path): + if not os.path.exists(path): + raise AnsibleError("imported module support code does not exist at %s" % os.path.abspath(path)) + fd = open(path, 'rb') + data = fd.read() + fd.close() + return data + + +def _get_shebang(interpreter, task_vars, templar, args=tuple()): + """ + Note not stellar API: + Returns None instead of always returning a shebang line. Doing it this + way allows the caller to decide to use the shebang it read from the + file rather than trust that we reformatted what they already have + correctly. + """ + interpreter_config = u'ansible_%s_interpreter' % os.path.basename(interpreter).strip() + + if interpreter_config not in task_vars: + return (None, interpreter) + + interpreter = templar.template(task_vars[interpreter_config].strip()) + shebang = u'#!' + interpreter + + if args: + shebang = shebang + u' ' + u' '.join(args) + + return (shebang, interpreter) + + +def recursive_finder(name, data, py_module_names, py_module_cache, zf): + """ + Using ModuleDepFinder, make sure we have all of the module_utils files that + the module its module_utils files needs. + """ + # Parse the module and find the imports of ansible.module_utils + tree = ast.parse(data) + finder = ModuleDepFinder() + finder.visit(tree) + + # + # Determine what imports that we've found are modules (vs class, function. + # variable names) for packages + # + + normalized_modules = set() + # Loop through the imports that we've found to normalize them + # Exclude paths that match with paths we've already processed + # (Have to exclude them a second time once the paths are processed) + + module_utils_paths = [p for p in module_utils_loader._get_paths(subdirs=False) if os.path.isdir(p)] + module_utils_paths.append(_MODULE_UTILS_PATH) + for py_module_name in finder.submodules.difference(py_module_names): + module_info = None + + if py_module_name[0] == 'six': + # Special case the python six library because it messes up the + # import process in an incompatible way + module_info = imp.find_module('six', module_utils_paths) + py_module_name = ('six',) + idx = 0 + elif py_module_name[0] == '_six': + # Special case the python six library because it messes up the + # import process in an incompatible way + module_info = imp.find_module('_six', [os.path.join(p, 'six') for p in module_utils_paths]) + py_module_name = ('six', '_six') + idx = 0 + else: + # Check whether either the last or the second to last identifier is + # a module name + for idx in (1, 2): + if len(py_module_name) < idx: + break + try: + module_info = imp.find_module(py_module_name[-idx], + [os.path.join(p, *py_module_name[:-idx]) for p in module_utils_paths]) + break + except ImportError: + continue + + # Could not find the module. Construct a helpful error message. + if module_info is None: + msg = ['Could not find imported module support code for %s. Looked for' % (name,)] + if idx == 2: + msg.append('either %s.py or %s.py' % (py_module_name[-1], py_module_name[-2])) + else: + msg.append(py_module_name[-1]) + raise AnsibleError(' '.join(msg)) + + # Found a byte compiled file rather than source. We cannot send byte + # compiled over the wire as the python version might be different. + # imp.find_module seems to prefer to return source packages so we just + # error out if imp.find_module returns byte compiled files (This is + # fragile as it depends on undocumented imp.find_module behaviour) + if module_info[2][2] not in (imp.PY_SOURCE, imp.PKG_DIRECTORY): + msg = ['Could not find python source for imported module support code for %s. Looked for' % name] + if idx == 2: + msg.append('either %s.py or %s.py' % (py_module_name[-1], py_module_name[-2])) + else: + msg.append(py_module_name[-1]) + raise AnsibleError(' '.join(msg)) + + if idx == 2: + # We've determined that the last portion was an identifier and + # thus, not part of the module name + py_module_name = py_module_name[:-1] + + # If not already processed then we've got work to do + if py_module_name not in py_module_names: + # If not in the cache, then read the file into the cache + # We already have a file handle for the module open so it makes + # sense to read it now + if py_module_name not in py_module_cache: + if module_info[2][2] == imp.PKG_DIRECTORY: + # Read the __init__.py instead of the module file as this is + # a python package + normalized_name = py_module_name + ('__init__',) + normalized_path = os.path.join(os.path.join(module_info[1], '__init__.py')) + normalized_data = _slurp(normalized_path) + else: + normalized_name = py_module_name + normalized_path = module_info[1] + normalized_data = module_info[0].read() + module_info[0].close() + + py_module_cache[normalized_name] = (normalized_data, normalized_path) + normalized_modules.add(normalized_name) + + # Make sure that all the packages that this module is a part of + # are also added + for i in range(1, len(py_module_name)): + py_pkg_name = py_module_name[:-i] + ('__init__',) + if py_pkg_name not in py_module_names: + pkg_dir_info = imp.find_module(py_pkg_name[-1], + [os.path.join(p, *py_pkg_name[:-1]) for p in module_utils_paths]) + normalized_modules.add(py_pkg_name) + py_module_cache[py_pkg_name] = (_slurp(pkg_dir_info[1]), pkg_dir_info[1]) + + # + # iterate through all of the ansible.module_utils* imports that we haven't + # already checked for new imports + # + + # set of modules that we haven't added to the zipfile + unprocessed_py_module_names = normalized_modules.difference(py_module_names) + + for py_module_name in unprocessed_py_module_names: + py_module_path = os.path.join(*py_module_name) + py_module_file_name = '%s.py' % py_module_path + + zf.writestr(os.path.join("ansible/module_utils", + py_module_file_name), py_module_cache[py_module_name][0]) + display.vvvvv("Using module_utils file %s" % py_module_cache[py_module_name][1]) + + # Add the names of the files we're scheduling to examine in the loop to + # py_module_names so that we don't re-examine them in the next pass + # through recursive_finder() + py_module_names.update(unprocessed_py_module_names) + + for py_module_file in unprocessed_py_module_names: + recursive_finder(py_module_file, py_module_cache[py_module_file][0], py_module_names, py_module_cache, zf) + # Save memory; the file won't have to be read again for this ansible module. + del py_module_cache[py_module_file] + + +def _is_binary(b_module_data): + textchars = bytearray(set([7, 8, 9, 10, 12, 13, 27]) | set(range(0x20, 0x100)) - set([0x7f])) + start = b_module_data[:1024] + return bool(start.translate(None, textchars)) + + +def _find_module_utils(module_name, b_module_data, module_path, module_args, task_vars, templar, module_compression, async_timeout, become, + become_method, become_user, become_password, become_flags, environment): + """ + Given the source of the module, convert it to a Jinja2 template to insert + module code and return whether it's a new or old style module. + """ + module_substyle = module_style = 'old' + + # module_style is something important to calling code (ActionBase). It + # determines how arguments are formatted (json vs k=v) and whether + # a separate arguments file needs to be sent over the wire. + # module_substyle is extra information that's useful internally. It tells + # us what we have to look to substitute in the module files and whether + # we're using module replacer or ansiballz to format the module itself. + if _is_binary(b_module_data): + module_substyle = module_style = 'binary' + elif REPLACER in b_module_data: + # Do REPLACER before from ansible.module_utils because we need make sure + # we substitute "from ansible.module_utils basic" for REPLACER + module_style = 'new' + module_substyle = 'python' + b_module_data = b_module_data.replace(REPLACER, b'from ansible.module_utils.basic import *') + elif b'from ansible.module_utils.' in b_module_data: + module_style = 'new' + module_substyle = 'python' + elif REPLACER_WINDOWS in b_module_data: + module_style = 'new' + module_substyle = 'powershell' + b_module_data = b_module_data.replace(REPLACER_WINDOWS, b'#Requires -Module Ansible.ModuleUtils.Legacy') + elif re.search(b'#Requires -Module', b_module_data, re.IGNORECASE) \ + or re.search(b'#Requires -Version', b_module_data, re.IGNORECASE)\ + or re.search(b'#AnsibleRequires -OSVersion', b_module_data, re.IGNORECASE): + module_style = 'new' + module_substyle = 'powershell' + elif REPLACER_JSONARGS in b_module_data: + module_style = 'new' + module_substyle = 'jsonargs' + elif b'WANT_JSON' in b_module_data: + module_substyle = module_style = 'non_native_want_json' + + shebang = None + # Neither old-style, non_native_want_json nor binary modules should be modified + # except for the shebang line (Done by modify_module) + if module_style in ('old', 'non_native_want_json', 'binary'): + return b_module_data, module_style, shebang + + output = BytesIO() + py_module_names = set() + + if module_substyle == 'python': + params = dict(ANSIBLE_MODULE_ARGS=module_args,) + python_repred_params = repr(json.dumps(params)) + + try: + compression_method = getattr(zipfile, module_compression) + except AttributeError: + display.warning(u'Bad module compression string specified: %s. Using ZIP_STORED (no compression)' % module_compression) + compression_method = zipfile.ZIP_STORED + + lookup_path = os.path.join(C.DEFAULT_LOCAL_TMP, 'ansiballz_cache') + cached_module_filename = os.path.join(lookup_path, "%s-%s" % (module_name, module_compression)) + + zipdata = None + # Optimization -- don't lock if the module has already been cached + if os.path.exists(cached_module_filename): + display.debug('ANSIBALLZ: using cached module: %s' % cached_module_filename) + zipdata = open(cached_module_filename, 'rb').read() + else: + if module_name in action_write_locks.action_write_locks: + display.debug('ANSIBALLZ: Using lock for %s' % module_name) + lock = action_write_locks.action_write_locks[module_name] + else: + # If the action plugin directly invokes the module (instead of + # going through a strategy) then we don't have a cross-process + # Lock specifically for this module. Use the "unexpected + # module" lock instead + display.debug('ANSIBALLZ: Using generic lock for %s' % module_name) + lock = action_write_locks.action_write_locks[None] + + display.debug('ANSIBALLZ: Acquiring lock') + with lock: + display.debug('ANSIBALLZ: Lock acquired: %s' % id(lock)) + # Check that no other process has created this while we were + # waiting for the lock + if not os.path.exists(cached_module_filename): + display.debug('ANSIBALLZ: Creating module') + # Create the module zip data + zipoutput = BytesIO() + zf = zipfile.ZipFile(zipoutput, mode='w', compression=compression_method) + # Note: If we need to import from release.py first, + # remember to catch all exceptions: https://github.com/ansible/ansible/issues/16523 + zf.writestr('ansible/__init__.py', + b'from pkgutil import extend_path\n__path__=extend_path(__path__,__name__)\n__version__="' + + to_bytes(__version__) + b'"\n__author__="' + + to_bytes(__author__) + b'"\n') + zf.writestr('ansible/module_utils/__init__.py', b'from pkgutil import extend_path\n__path__=extend_path(__path__,__name__)\n') + + zf.writestr('ansible_module_%s.py' % module_name, b_module_data) + + py_module_cache = {('__init__',): (b'', '[builtin]')} + recursive_finder(module_name, b_module_data, py_module_names, py_module_cache, zf) + zf.close() + zipdata = base64.b64encode(zipoutput.getvalue()) + + # Write the assembled module to a temp file (write to temp + # so that no one looking for the file reads a partially + # written file) + if not os.path.exists(lookup_path): + # Note -- if we have a global function to setup, that would + # be a better place to run this + os.makedirs(lookup_path) + display.debug('ANSIBALLZ: Writing module') + with open(cached_module_filename + '-part', 'wb') as f: + f.write(zipdata) + + # Rename the file into its final position in the cache so + # future users of this module can read it off the + # filesystem instead of constructing from scratch. + display.debug('ANSIBALLZ: Renaming module') + os.rename(cached_module_filename + '-part', cached_module_filename) + display.debug('ANSIBALLZ: Done creating module') + + if zipdata is None: + display.debug('ANSIBALLZ: Reading module after lock') + # Another process wrote the file while we were waiting for + # the write lock. Go ahead and read the data from disk + # instead of re-creating it. + try: + zipdata = open(cached_module_filename, 'rb').read() + except IOError: + raise AnsibleError('A different worker process failed to create module file. ' + 'Look at traceback for that process for debugging information.') + zipdata = to_text(zipdata, errors='surrogate_or_strict') + + shebang, interpreter = _get_shebang(u'/usr/bin/python', task_vars, templar) + if shebang is None: + shebang = u'#!/usr/bin/python' + + # Enclose the parts of the interpreter in quotes because we're + # substituting it into the template as a Python string + interpreter_parts = interpreter.split(u' ') + interpreter = u"'{0}'".format(u"', '".join(interpreter_parts)) + + now = datetime.datetime.utcnow() + output.write(to_bytes(ACTIVE_ANSIBALLZ_TEMPLATE % dict( + zipdata=zipdata, + ansible_module=module_name, + params=python_repred_params, + shebang=shebang, + interpreter=interpreter, + coding=ENCODING_STRING, + year=now.year, + month=now.month, + day=now.day, + hour=now.hour, + minute=now.minute, + second=now.second, + ))) + b_module_data = output.getvalue() + + elif module_substyle == 'powershell': + # Powershell/winrm don't actually make use of shebang so we can + # safely set this here. If we let the fallback code handle this + # it can fail in the presence of the UTF8 BOM commonly added by + # Windows text editors + shebang = u'#!powershell' + + exec_manifest = dict( + module_entry=to_text(base64.b64encode(b_module_data)), + powershell_modules=dict(), + module_args=module_args, + actions=['exec'], + environment=environment + ) + + exec_manifest['exec'] = to_text(base64.b64encode(to_bytes(leaf_exec))) + + if async_timeout > 0: + exec_manifest["actions"].insert(0, 'async_watchdog') + exec_manifest["async_watchdog"] = to_text(base64.b64encode(to_bytes(async_watchdog))) + exec_manifest["actions"].insert(0, 'async_wrapper') + exec_manifest["async_wrapper"] = to_text(base64.b64encode(to_bytes(async_wrapper))) + exec_manifest["async_jid"] = str(random.randint(0, 999999999999)) + exec_manifest["async_timeout_sec"] = async_timeout + + if become and become_method == 'runas': + exec_manifest["actions"].insert(0, 'become') + exec_manifest["become_user"] = become_user + exec_manifest["become_password"] = become_password + exec_manifest['become_flags'] = become_flags + exec_manifest["become"] = to_text(base64.b64encode(to_bytes(become_wrapper))) + + lines = b_module_data.split(b'\n') + module_names = set() + become_required = False + min_os_version = None + min_ps_version = None + + requires_module_list = re.compile(to_bytes(r'(?i)^#\s*requires\s+\-module(?:s?)\s*(Ansible\.ModuleUtils\..+)')) + requires_ps_version = re.compile(to_bytes(r'(?i)^#requires\s+\-version\s+([0-9]+(\.[0-9]+){0,3})$')) + requires_os_version = re.compile(to_bytes(r'(?i)^#ansiblerequires\s+\-osversion\s+([0-9]+(\.[0-9]+){0,3})$')) + requires_become = re.compile(to_bytes(r'(?i)^#ansiblerequires\s+\-become$')) + + for line in lines: + module_util_line_match = requires_module_list.match(line) + if module_util_line_match: + module_names.add(module_util_line_match.group(1)) + + requires_ps_version_match = requires_ps_version.match(line) + if requires_ps_version_match: + min_ps_version = to_text(requires_ps_version_match.group(1)) + # Powershell cannot cast a string of "1" to version, it must + # have at least the major.minor for it to work so we append 0 + if requires_ps_version_match.group(2) is None: + min_ps_version = "%s.0" % min_ps_version + + requires_os_version_match = requires_os_version.match(line) + if requires_os_version_match: + min_os_version = to_text(requires_os_version_match.group(1)) + if requires_os_version_match.group(2) is None: + min_os_version = "%s.0" % min_os_version + + requires_become_match = requires_become.match(line) + if requires_become_match: + become_required = True + + for m in set(module_names): + m = to_text(m).rstrip() # tolerate windows line endings + mu_path = ps_module_utils_loader.find_plugin(m, ".psm1") + if not mu_path: + raise AnsibleError('Could not find imported module support code for \'%s\'.' % m) + exec_manifest["powershell_modules"][m] = to_text( + base64.b64encode( + to_bytes( + _slurp(mu_path) + ) + ) + ) + + exec_manifest['min_ps_version'] = min_ps_version + exec_manifest['min_os_version'] = min_os_version + if become_required and 'become' not in exec_manifest["actions"]: + exec_manifest["actions"].insert(0, 'become') + exec_manifest["become_user"] = "SYSTEM" + exec_manifest["become_password"] = None + exec_manifest['become_flags'] = None + exec_manifest["become"] = to_text(base64.b64encode(to_bytes(become_wrapper))) + + # FUTURE: smuggle this back as a dict instead of serializing here; the connection plugin may need to modify it + module_json = json.dumps(exec_manifest) + + b_module_data = exec_wrapper.replace(b"$json_raw = ''", b"$json_raw = @'\r\n%s\r\n'@" % to_bytes(module_json)) + + elif module_substyle == 'jsonargs': + module_args_json = to_bytes(json.dumps(module_args)) + + # these strings could be included in a third-party module but + # officially they were included in the 'basic' snippet for new-style + # python modules (which has been replaced with something else in + # ansiballz) If we remove them from jsonargs-style module replacer + # then we can remove them everywhere. + python_repred_args = to_bytes(repr(module_args_json)) + b_module_data = b_module_data.replace(REPLACER_VERSION, to_bytes(repr(__version__))) + b_module_data = b_module_data.replace(REPLACER_COMPLEX, python_repred_args) + b_module_data = b_module_data.replace(REPLACER_SELINUX, to_bytes(','.join(C.DEFAULT_SELINUX_SPECIAL_FS))) + + # The main event -- substitute the JSON args string into the module + b_module_data = b_module_data.replace(REPLACER_JSONARGS, module_args_json) + + facility = b'syslog.' + to_bytes(task_vars.get('ansible_syslog_facility', C.DEFAULT_SYSLOG_FACILITY), errors='surrogate_or_strict') + b_module_data = b_module_data.replace(b'syslog.LOG_USER', facility) + + return (b_module_data, module_style, shebang) + + +def modify_module(module_name, module_path, module_args, templar, task_vars=None, module_compression='ZIP_STORED', async_timeout=0, become=False, + become_method=None, become_user=None, become_password=None, become_flags=None, environment=None): + """ + Used to insert chunks of code into modules before transfer rather than + doing regular python imports. This allows for more efficient transfer in + a non-bootstrapping scenario by not moving extra files over the wire and + also takes care of embedding arguments in the transferred modules. + + This version is done in such a way that local imports can still be + used in the module code, so IDEs don't have to be aware of what is going on. + + Example: + + from ansible.module_utils.basic import * + + ... will result in the insertion of basic.py into the module + from the module_utils/ directory in the source tree. + + For powershell, this code effectively no-ops, as the exec wrapper requires access to a number of + properties not available here. + + """ + task_vars = {} if task_vars is None else task_vars + environment = {} if environment is None else environment + + with open(module_path, 'rb') as f: + + # read in the module source + b_module_data = f.read() + + (b_module_data, module_style, shebang) = _find_module_utils(module_name, b_module_data, module_path, module_args, task_vars, templar, module_compression, + async_timeout=async_timeout, become=become, become_method=become_method, + become_user=become_user, become_password=become_password, become_flags=become_flags, + environment=environment) + + if module_style == 'binary': + return (b_module_data, module_style, to_text(shebang, nonstring='passthru')) + elif shebang is None: + b_lines = b_module_data.split(b"\n", 1) + if b_lines[0].startswith(b"#!"): + b_shebang = b_lines[0].strip() + # shlex.split on python-2.6 needs bytes. On python-3.x it needs text + args = shlex.split(to_native(b_shebang[2:], errors='surrogate_or_strict')) + + # _get_shebang() takes text strings + args = [to_text(a, errors='surrogate_or_strict') for a in args] + interpreter = args[0] + b_new_shebang = to_bytes(_get_shebang(interpreter, task_vars, templar, args[1:])[0], + errors='surrogate_or_strict', nonstring='passthru') + + if b_new_shebang: + b_lines[0] = b_shebang = b_new_shebang + + if os.path.basename(interpreter).startswith(u'python'): + b_lines.insert(1, b_ENCODING_STRING) + + shebang = to_text(b_shebang, nonstring='passthru', errors='surrogate_or_strict') + else: + # No shebang, assume a binary module? + pass + + b_module_data = b"\n".join(b_lines) + + return (b_module_data, module_style, shebang) diff --git a/test/files/asyncstatements.py b/test/files/asyncstatements.py new file mode 100644 --- /dev/null +++ b/test/files/asyncstatements.py @@ -0,0 +1,37 @@ +@a +async def a(): + async with a as b: + await f(1, 2, 3) + + async for x in y: + await (lambda x: x) + +async def a(): + pass + +def a(): + pass + +@a +def a(): + with a as b: + pass + + for x in y: + pass + + +async def a(): + with a as b: + pass + + async for x in y: + pass + + await 3 + +async = 2 +async +await = 2 +await +print(async) diff --git a/test/files/comments.py b/test/files/comments.py new file mode 100644 --- /dev/null +++ b/test/files/comments.py @@ -0,0 +1,5 @@ +# blah +def boo(): + pass + #bing + # bop diff --git a/test/files/decorators.py b/test/files/decorators.py new file mode 100644 --- /dev/null +++ b/test/files/decorators.py @@ -0,0 +1,5 @@ +@a + +@a +def b(): + pass diff --git a/test/files/dictcomp.py b/test/files/dictcomp.py new file mode 100644 --- /dev/null +++ b/test/files/dictcomp.py @@ -0,0 +1,4 @@ +{() for a in ()} # set comprehension +{(): () for a in ()} +{' ' * min_indent for k in keys} # set comprehension +{k: ' ' * min_indent for k in keys} diff --git a/test/files/django.py b/test/files/django.py new file mode 100644 --- /dev/null +++ b/test/files/django.py @@ -0,0 +1,688 @@ +""" +Multi-part parsing for file uploads. + +Exposes one class, ``MultiPartParser``, which feeds chunks of uploaded data to +file upload handlers for processing. +""" +import base64 +import binascii +import cgi +from urllib.parse import unquote + +from django.conf import settings +from django.core.exceptions import ( + RequestDataTooBig, SuspiciousMultipartForm, TooManyFieldsSent, +) +from django.core.files.uploadhandler import ( + SkipFile, StopFutureHandlers, StopUpload, +) +from django.utils.datastructures import MultiValueDict +from django.utils.encoding import force_text +from django.utils.text import unescape_entities + +__all__ = ('MultiPartParser', 'MultiPartParserError', 'InputStreamExhausted') + + +class MultiPartParserError(Exception): + pass + + +class InputStreamExhausted(Exception): + """ + No more reads are allowed from this device. + """ + pass + + +RAW = "raw" +FILE = "file" +FIELD = "field" + + +class MultiPartParser: + """ + A rfc2388 multipart/form-data parser. + + ``MultiValueDict.parse()`` reads the input stream in ``chunk_size`` chunks + and returns a tuple of ``(MultiValueDict(POST), MultiValueDict(FILES))``. + """ + def __init__(self, META, input_data, upload_handlers, encoding=None): + """ + Initialize the MultiPartParser object. + + :META: + The standard ``META`` dictionary in Django request objects. + :input_data: + The raw post data, as a file-like object. + :upload_handlers: + A list of UploadHandler instances that perform operations on the + uploaded data. + :encoding: + The encoding with which to treat the incoming data. + """ + # Content-Type should contain multipart and the boundary information. + content_type = META.get('CONTENT_TYPE', '') + if not content_type.startswith('multipart/'): + raise MultiPartParserError('Invalid Content-Type: %s' % content_type) + + # Parse the header to get the boundary to split the parts. + ctypes, opts = parse_header(content_type.encode('ascii')) + boundary = opts.get('boundary') + if not boundary or not cgi.valid_boundary(boundary): + raise MultiPartParserError('Invalid boundary in multipart: %s' % boundary.decode()) + + # Content-Length should contain the length of the body we are about + # to receive. + try: + content_length = int(META.get('CONTENT_LENGTH', 0)) + except (ValueError, TypeError): + content_length = 0 + + if content_length < 0: + # This means we shouldn't continue...raise an error. + raise MultiPartParserError("Invalid content length: %r" % content_length) + + if isinstance(boundary, str): + boundary = boundary.encode('ascii') + self._boundary = boundary + self._input_data = input_data + + # For compatibility with low-level network APIs (with 32-bit integers), + # the chunk size should be < 2^31, but still divisible by 4. + possible_sizes = [x.chunk_size for x in upload_handlers if x.chunk_size] + self._chunk_size = min([2 ** 31 - 4] + possible_sizes) + + self._meta = META + self._encoding = encoding or settings.DEFAULT_CHARSET + self._content_length = content_length + self._upload_handlers = upload_handlers + + def parse(self): + """ + Parse the POST data and break it into a FILES MultiValueDict and a POST + MultiValueDict. + + Return a tuple containing the POST and FILES dictionary, respectively. + """ + from django.http import QueryDict + + encoding = self._encoding + handlers = self._upload_handlers + + # HTTP spec says that Content-Length >= 0 is valid + # handling content-length == 0 before continuing + if self._content_length == 0: + return QueryDict(encoding=self._encoding), MultiValueDict() + + # See if any of the handlers take care of the parsing. + # This allows overriding everything if need be. + for handler in handlers: + result = handler.handle_raw_input( + self._input_data, + self._meta, + self._content_length, + self._boundary, + encoding, + ) + # Check to see if it was handled + if result is not None: + return result[0], result[1] + + # Create the data structures to be used later. + self._post = QueryDict(mutable=True) + self._files = MultiValueDict() + + # Instantiate the parser and stream: + stream = LazyStream(ChunkIter(self._input_data, self._chunk_size)) + + # Whether or not to signal a file-completion at the beginning of the loop. + old_field_name = None + counters = [0] * len(handlers) + + # Number of bytes that have been read. + num_bytes_read = 0 + # To count the number of keys in the request. + num_post_keys = 0 + # To limit the amount of data read from the request. + read_size = None + + try: + for item_type, meta_data, field_stream in Parser(stream, self._boundary): + if old_field_name: + # We run this at the beginning of the next loop + # since we cannot be sure a file is complete until + # we hit the next boundary/part of the multipart content. + self.handle_file_complete(old_field_name, counters) + old_field_name = None + + try: + disposition = meta_data['content-disposition'][1] + field_name = disposition['name'].strip() + except (KeyError, IndexError, AttributeError): + continue + + transfer_encoding = meta_data.get('content-transfer-encoding') + if transfer_encoding is not None: + transfer_encoding = transfer_encoding[0].strip() + field_name = force_text(field_name, encoding, errors='replace') + + if item_type == FIELD: + # Avoid storing more than DATA_UPLOAD_MAX_NUMBER_FIELDS. + num_post_keys += 1 + if (settings.DATA_UPLOAD_MAX_NUMBER_FIELDS is not None and + settings.DATA_UPLOAD_MAX_NUMBER_FIELDS < num_post_keys): + raise TooManyFieldsSent( + 'The number of GET/POST parameters exceeded ' + 'settings.DATA_UPLOAD_MAX_NUMBER_FIELDS.' + ) + + # Avoid reading more than DATA_UPLOAD_MAX_MEMORY_SIZE. + if settings.DATA_UPLOAD_MAX_MEMORY_SIZE is not None: + read_size = settings.DATA_UPLOAD_MAX_MEMORY_SIZE - num_bytes_read + + # This is a post field, we can just set it in the post + if transfer_encoding == 'base64': + raw_data = field_stream.read(size=read_size) + num_bytes_read += len(raw_data) + try: + data = base64.b64decode(raw_data) + except binascii.Error: + data = raw_data + else: + data = field_stream.read(size=read_size) + num_bytes_read += len(data) + + # Add two here to make the check consistent with the + # x-www-form-urlencoded check that includes '&='. + num_bytes_read += len(field_name) + 2 + if (settings.DATA_UPLOAD_MAX_MEMORY_SIZE is not None and + num_bytes_read > settings.DATA_UPLOAD_MAX_MEMORY_SIZE): + raise RequestDataTooBig('Request body exceeded settings.DATA_UPLOAD_MAX_MEMORY_SIZE.') + + self._post.appendlist(field_name, force_text(data, encoding, errors='replace')) + elif item_type == FILE: + # This is a file, use the handler... + file_name = disposition.get('filename') + if file_name: + file_name = force_text(file_name, encoding, errors='replace') + file_name = self.IE_sanitize(unescape_entities(file_name)) + if not file_name: + continue + + content_type, content_type_extra = meta_data.get('content-type', ('', {})) + content_type = content_type.strip() + charset = content_type_extra.get('charset') + + try: + content_length = int(meta_data.get('content-length')[0]) + except (IndexError, TypeError, ValueError): + content_length = None + + counters = [0] * len(handlers) + try: + for handler in handlers: + try: + handler.new_file( + field_name, file_name, content_type, + content_length, charset, content_type_extra, + ) + except StopFutureHandlers: + break + + for chunk in field_stream: + if transfer_encoding == 'base64': + # We only special-case base64 transfer encoding + # We should always decode base64 chunks by multiple of 4, + # ignoring whitespace. + + stripped_chunk = b"".join(chunk.split()) + + remaining = len(stripped_chunk) % 4 + while remaining != 0: + over_chunk = field_stream.read(4 - remaining) + stripped_chunk += b"".join(over_chunk.split()) + remaining = len(stripped_chunk) % 4 + + try: + chunk = base64.b64decode(stripped_chunk) + except Exception as exc: + # Since this is only a chunk, any error is an unfixable error. + raise MultiPartParserError("Could not decode base64 data.") from exc + + for i, handler in enumerate(handlers): + chunk_length = len(chunk) + chunk = handler.receive_data_chunk(chunk, counters[i]) + counters[i] += chunk_length + if chunk is None: + # Don't continue if the chunk received by + # the handler is None. + break + + except SkipFile: + self._close_files() + # Just use up the rest of this file... + exhaust(field_stream) + else: + # Handle file upload completions on next iteration. + old_field_name = field_name + else: + # If this is neither a FIELD or a FILE, just exhaust the stream. + exhaust(stream) + except StopUpload as e: + self._close_files() + if not e.connection_reset: + exhaust(self._input_data) + else: + # Make sure that the request data is all fed + exhaust(self._input_data) + + # Signal that the upload has completed. + # any() shortcircuits if a handler's upload_complete() returns a value. + any(handler.upload_complete() for handler in handlers) + self._post._mutable = False + return self._post, self._files + + def handle_file_complete(self, old_field_name, counters): + """ + Handle all the signaling that takes place when a file is complete. + """ + for i, handler in enumerate(self._upload_handlers): + file_obj = handler.file_complete(counters[i]) + if file_obj: + # If it returns a file object, then set the files dict. + self._files.appendlist(force_text(old_field_name, self._encoding, errors='replace'), file_obj) + break + + def IE_sanitize(self, filename): + """Cleanup filename from Internet Explorer full paths.""" + return filename and filename[filename.rfind("\\") + 1:].strip() + + def _close_files(self): + # Free up all file handles. + # FIXME: this currently assumes that upload handlers store the file as 'file' + # We should document that... (Maybe add handler.free_file to complement new_file) + for handler in self._upload_handlers: + if hasattr(handler, 'file'): + handler.file.close() + + +class LazyStream: + """ + The LazyStream wrapper allows one to get and "unget" bytes from a stream. + + Given a producer object (an iterator that yields bytestrings), the + LazyStream object will support iteration, reading, and keeping a "look-back" + variable in case you need to "unget" some bytes. + """ + def __init__(self, producer, length=None): + """ + Every LazyStream must have a producer when instantiated. + + A producer is an iterable that returns a string each time it + is called. + """ + self._producer = producer + self._empty = False + self._leftover = b'' + self.length = length + self.position = 0 + self._remaining = length + self._unget_history = [] + + def tell(self): + return self.position + + def read(self, size=None): + def parts(): + remaining = self._remaining if size is None else size + # do the whole thing in one shot if no limit was provided. + if remaining is None: + yield b''.join(self) + return + + # otherwise do some bookkeeping to return exactly enough + # of the stream and stashing any extra content we get from + # the producer + while remaining != 0: + assert remaining > 0, 'remaining bytes to read should never go negative' + + try: + chunk = next(self) + except StopIteration: + return + else: + emitting = chunk[:remaining] + self.unget(chunk[remaining:]) + remaining -= len(emitting) + yield emitting + + out = b''.join(parts()) + return out + + def __next__(self): + """ + Used when the exact number of bytes to read is unimportant. + + Return whatever chunk is conveniently returned from the iterator. + Useful to avoid unnecessary bookkeeping if performance is an issue. + """ + if self._leftover: + output = self._leftover + self._leftover = b'' + else: + output = next(self._producer) + self._unget_history = [] + self.position += len(output) + return output + + def close(self): + """ + Used to invalidate/disable this lazy stream. + + Replace the producer with an empty list. Any leftover bytes that have + already been read will still be reported upon read() and/or next(). + """ + self._producer = [] + + def __iter__(self): + return self + + def unget(self, bytes): + """ + Place bytes back onto the front of the lazy stream. + + Future calls to read() will return those bytes first. The + stream position and thus tell() will be rewound. + """ + if not bytes: + return + self._update_unget_history(len(bytes)) + self.position -= len(bytes) + self._leftover = bytes + self._leftover + + def _update_unget_history(self, num_bytes): + """ + Update the unget history as a sanity check to see if we've pushed + back the same number of bytes in one chunk. If we keep ungetting the + same number of bytes many times (here, 50), we're mostly likely in an + infinite loop of some sort. This is usually caused by a + maliciously-malformed MIME request. + """ + self._unget_history = [num_bytes] + self._unget_history[:49] + number_equal = len([ + current_number for current_number in self._unget_history + if current_number == num_bytes + ]) + + if number_equal > 40: + raise SuspiciousMultipartForm( + "The multipart parser got stuck, which shouldn't happen with" + " normal uploaded files. Check for malicious upload activity;" + " if there is none, report this to the Django developers." + ) + + +class ChunkIter: + """ + An iterable that will yield chunks of data. Given a file-like object as the + constructor, yield chunks of read operations from that object. + """ + def __init__(self, flo, chunk_size=64 * 1024): + self.flo = flo + self.chunk_size = chunk_size + + def __next__(self): + try: + data = self.flo.read(self.chunk_size) + except InputStreamExhausted: + raise StopIteration() + if data: + return data + else: + raise StopIteration() + + def __iter__(self): + return self + + +class InterBoundaryIter: + """ + A Producer that will iterate over boundaries. + """ + def __init__(self, stream, boundary): + self._stream = stream + self._boundary = boundary + + def __iter__(self): + return self + + def __next__(self): + try: + return LazyStream(BoundaryIter(self._stream, self._boundary)) + except InputStreamExhausted: + raise StopIteration() + + +class BoundaryIter: + """ + A Producer that is sensitive to boundaries. + + Will happily yield bytes until a boundary is found. Will yield the bytes + before the boundary, throw away the boundary bytes themselves, and push the + post-boundary bytes back on the stream. + + The future calls to next() after locating the boundary will raise a + StopIteration exception. + """ + + def __init__(self, stream, boundary): + self._stream = stream + self._boundary = boundary + self._done = False + # rollback an additional six bytes because the format is like + # this: CRLF[--CRLF] + self._rollback = len(boundary) + 6 + + # Try to use mx fast string search if available. Otherwise + # use Python find. Wrap the latter for consistency. + unused_char = self._stream.read(1) + if not unused_char: + raise InputStreamExhausted() + self._stream.unget(unused_char) + + def __iter__(self): + return self + + def __next__(self): + if self._done: + raise StopIteration() + + stream = self._stream + rollback = self._rollback + + bytes_read = 0 + chunks = [] + for bytes in stream: + bytes_read += len(bytes) + chunks.append(bytes) + if bytes_read > rollback: + break + if not bytes: + break + else: + self._done = True + + if not chunks: + raise StopIteration() + + chunk = b''.join(chunks) + boundary = self._find_boundary(chunk) + + if boundary: + end, next = boundary + stream.unget(chunk[next:]) + self._done = True + return chunk[:end] + else: + # make sure we don't treat a partial boundary (and + # its separators) as data + if not chunk[:-rollback]: # and len(chunk) >= (len(self._boundary) + 6): + # There's nothing left, we should just return and mark as done. + self._done = True + return chunk + else: + stream.unget(chunk[-rollback:]) + return chunk[:-rollback] + + def _find_boundary(self, data): + """ + Find a multipart boundary in data. + + Should no boundary exist in the data, return None. Otherwise, return + a tuple containing the indices of the following: + * the end of current encapsulation + * the start of the next encapsulation + """ + index = data.find(self._boundary) + if index < 0: + return None + else: + end = index + next = index + len(self._boundary) + # backup over CRLF + last = max(0, end - 1) + if data[last:last + 1] == b'\n': + end -= 1 + last = max(0, end - 1) + if data[last:last + 1] == b'\r': + end -= 1 + return end, next + + +def exhaust(stream_or_iterable): + """Exhaust an iterator or stream.""" + try: + iterator = iter(stream_or_iterable) + except TypeError: + iterator = ChunkIter(stream_or_iterable, 16384) + + for __ in iterator: + pass + + +def parse_boundary_stream(stream, max_header_size): + """ + Parse one and exactly one stream that encapsulates a boundary. + """ + # Stream at beginning of header, look for end of header + # and parse it if found. The header must fit within one + # chunk. + chunk = stream.read(max_header_size) + + # 'find' returns the top of these four bytes, so we'll + # need to munch them later to prevent them from polluting + # the payload. + header_end = chunk.find(b'\r\n\r\n') + + def _parse_header(line): + main_value_pair, params = parse_header(line) + try: + name, value = main_value_pair.split(':', 1) + except ValueError: + raise ValueError("Invalid header: %r" % line) + return name, (value, params) + + if header_end == -1: + # we find no header, so we just mark this fact and pass on + # the stream verbatim + stream.unget(chunk) + return (RAW, {}, stream) + + header = chunk[:header_end] + + # here we place any excess chunk back onto the stream, as + # well as throwing away the CRLFCRLF bytes from above. + stream.unget(chunk[header_end + 4:]) + + TYPE = RAW + outdict = {} + + # Eliminate blank lines + for line in header.split(b'\r\n'): + # This terminology ("main value" and "dictionary of + # parameters") is from the Python docs. + try: + name, (value, params) = _parse_header(line) + except ValueError: + continue + + if name == 'content-disposition': + TYPE = FIELD + if params.get('filename'): + TYPE = FILE + + outdict[name] = value, params + + if TYPE == RAW: + stream.unget(chunk) + + return (TYPE, outdict, stream) + + +class Parser: + def __init__(self, stream, boundary): + self._stream = stream + self._separator = b'--' + boundary + + def __iter__(self): + boundarystream = InterBoundaryIter(self._stream, self._separator) + for sub_stream in boundarystream: + # Iterate over each part + yield parse_boundary_stream(sub_stream, 1024) + + +def parse_header(line): + """ + Parse the header into a key-value. + + Input (line): bytes, output: str for key/name, bytes for values which + will be decoded later. + """ + plist = _parse_header_params(b';' + line) + key = plist.pop(0).lower().decode('ascii') + pdict = {} + for p in plist: + i = p.find(b'=') + if i >= 0: + has_encoding = False + name = p[:i].strip().lower().decode('ascii') + if name.endswith('*'): + # Lang/encoding embedded in the value (like "filename*=UTF-8''file.ext") + # http://tools.ietf.org/html/rfc2231#section-4 + name = name[:-1] + if p.count(b"'") == 2: + has_encoding = True + value = p[i + 1:].strip() + if has_encoding: + encoding, lang, value = value.split(b"'") + value = unquote(value.decode(), encoding=encoding.decode()) + if len(value) >= 2 and value[:1] == value[-1:] == b'"': + value = value[1:-1] + value = value.replace(b'\\\\', b'\\').replace(b'\\"', b'"') + pdict[name] = value + return key, pdict + + +def _parse_header_params(s): + plist = [] + while s[:1] == b';': + s = s[1:] + end = s.find(b';') + while end > 0 and s.count(b'"', 0, end) % 2: + end = s.find(b';', end + 1) + if end < 0: + end = len(s) + f = s[:end] + plist.append(f.strip()) + s = s[end:] + return plist diff --git a/test/files/django2.py b/test/files/django2.py new file mode 100644 --- /dev/null +++ b/test/files/django2.py @@ -0,0 +1,2337 @@ +import collections.abc +import copy +import datetime +import decimal +import operator +import uuid +import warnings +from base64 import b64decode, b64encode +from functools import partialmethod, total_ordering + +from django import forms +from django.apps import apps +from django.conf import settings +from django.core import checks, exceptions, validators +# When the _meta object was formalized, this exception was moved to +# django.core.exceptions. It is retained here for backwards compatibility +# purposes. +from django.core.exceptions import FieldDoesNotExist # NOQA +from django.db import connection, connections, router +from django.db.models.constants import LOOKUP_SEP +from django.db.models.query_utils import DeferredAttribute, RegisterLookupMixin +from django.utils import timezone +from django.utils.datastructures import DictWrapper +from django.utils.dateparse import ( + parse_date, parse_datetime, parse_duration, parse_time, +) +from django.utils.duration import duration_microseconds, duration_string +from django.utils.encoding import force_bytes, smart_text +from django.utils.functional import Promise, cached_property +from django.utils.ipv6 import clean_ipv6_address +from django.utils.itercompat import is_iterable +from django.utils.text import capfirst +from django.utils.translation import gettext_lazy as _ + +__all__ = [ + 'AutoField', 'BLANK_CHOICE_DASH', 'BigAutoField', 'BigIntegerField', + 'BinaryField', 'BooleanField', 'CharField', 'CommaSeparatedIntegerField', + 'DateField', 'DateTimeField', 'DecimalField', 'DurationField', + 'EmailField', 'Empty', 'Field', 'FieldDoesNotExist', 'FilePathField', + 'FloatField', 'GenericIPAddressField', 'IPAddressField', 'IntegerField', + 'NOT_PROVIDED', 'NullBooleanField', 'PositiveIntegerField', + 'PositiveSmallIntegerField', 'SlugField', 'SmallIntegerField', 'TextField', + 'TimeField', 'URLField', 'UUIDField', +] + + +class Empty: + pass + + +class NOT_PROVIDED: + pass + + +# The values to use for "blank" in SelectFields. Will be appended to the start +# of most "choices" lists. +BLANK_CHOICE_DASH = [("", "---------")] + + +def _load_field(app_label, model_name, field_name): + return apps.get_model(app_label, model_name)._meta.get_field(field_name) + + +# A guide to Field parameters: +# +# * name: The name of the field specified in the model. +# * attname: The attribute to use on the model object. This is the same as +# "name", except in the case of ForeignKeys, where "_id" is +# appended. +# * db_column: The db_column specified in the model (or None). +# * column: The database column for this field. This is the same as +# "attname", except if db_column is specified. +# +# Code that introspects values, or does other dynamic things, should use +# attname. For example, this gets the primary key value of object "obj": +# +# getattr(obj, opts.pk.attname) + +def _empty(of_cls): + new = Empty() + new.__class__ = of_cls + return new + + +def return_None(): + return None + + +@total_ordering +class Field(RegisterLookupMixin): + """Base class for all field types""" + + # Designates whether empty strings fundamentally are allowed at the + # database level. + empty_strings_allowed = True + empty_values = list(validators.EMPTY_VALUES) + + # These track each time a Field instance is created. Used to retain order. + # The auto_creation_counter is used for fields that Django implicitly + # creates, creation_counter is used for all user-specified fields. + creation_counter = 0 + auto_creation_counter = -1 + default_validators = [] # Default set of validators + default_error_messages = { + 'invalid_choice': _('Value %(value)r is not a valid choice.'), + 'null': _('This field cannot be null.'), + 'blank': _('This field cannot be blank.'), + 'unique': _('%(model_name)s with this %(field_label)s ' + 'already exists.'), + # Translators: The 'lookup_type' is one of 'date', 'year' or 'month'. + # Eg: "Title must be unique for pub_date year" + 'unique_for_date': _("%(field_label)s must be unique for " + "%(date_field_label)s %(lookup_type)s."), + } + system_check_deprecated_details = None + system_check_removed_details = None + + # Field flags + hidden = False + + many_to_many = None + many_to_one = None + one_to_many = None + one_to_one = None + related_model = None + + # Generic field type description, usually overridden by subclasses + def _description(self): + return _('Field of type: %(field_type)s') % { + 'field_type': self.__class__.__name__ + } + description = property(_description) + + def __init__(self, verbose_name=None, name=None, primary_key=False, + max_length=None, unique=False, blank=False, null=False, + db_index=False, rel=None, default=NOT_PROVIDED, editable=True, + serialize=True, unique_for_date=None, unique_for_month=None, + unique_for_year=None, choices=None, help_text='', db_column=None, + db_tablespace=None, auto_created=False, validators=(), + error_messages=None): + self.name = name + self.verbose_name = verbose_name # May be set by set_attributes_from_name + self._verbose_name = verbose_name # Store original for deconstruction + self.primary_key = primary_key + self.max_length, self._unique = max_length, unique + self.blank, self.null = blank, null + self.remote_field = rel + self.is_relation = self.remote_field is not None + self.default = default + self.editable = editable + self.serialize = serialize + self.unique_for_date = unique_for_date + self.unique_for_month = unique_for_month + self.unique_for_year = unique_for_year + if isinstance(choices, collections.abc.Iterator): + choices = list(choices) + self.choices = choices or [] + self.help_text = help_text + self.db_index = db_index + self.db_column = db_column + self._db_tablespace = db_tablespace + self.auto_created = auto_created + + # Adjust the appropriate creation counter, and save our local copy. + if auto_created: + self.creation_counter = Field.auto_creation_counter + Field.auto_creation_counter -= 1 + else: + self.creation_counter = Field.creation_counter + Field.creation_counter += 1 + + self._validators = list(validators) # Store for deconstruction later + + messages = {} + for c in reversed(self.__class__.__mro__): + messages.update(getattr(c, 'default_error_messages', {})) + messages.update(error_messages or {}) + self._error_messages = error_messages # Store for deconstruction later + self.error_messages = messages + + def __str__(self): + """ + Return "app_label.model_label.field_name" for fields attached to + models. + """ + if not hasattr(self, 'model'): + return super().__str__() + model = self.model + app = model._meta.app_label + return '%s.%s.%s' % (app, model._meta.object_name, self.name) + + def __repr__(self): + """Display the module, class, and name of the field.""" + path = '%s.%s' % (self.__class__.__module__, self.__class__.__qualname__) + name = getattr(self, 'name', None) + if name is not None: + return '<%s: %s>' % (path, name) + return '<%s>' % path + + def check(self, **kwargs): + return [ + *self._check_field_name(), + *self._check_choices(), + *self._check_db_index(), + *self._check_null_allowed_for_primary_keys(), + *self._check_backend_specific_checks(**kwargs), + *self._check_validators(), + *self._check_deprecation_details(), + ] + + def _check_field_name(self): + """ + Check if field name is valid, i.e. 1) does not end with an + underscore, 2) does not contain "__" and 3) is not "pk". + """ + if self.name.endswith('_'): + return [ + checks.Error( + 'Field names must not end with an underscore.', + obj=self, + id='fields.E001', + ) + ] + elif LOOKUP_SEP in self.name: + return [ + checks.Error( + 'Field names must not contain "%s".' % (LOOKUP_SEP,), + obj=self, + id='fields.E002', + ) + ] + elif self.name == 'pk': + return [ + checks.Error( + "'pk' is a reserved word that cannot be used as a field name.", + obj=self, + id='fields.E003', + ) + ] + else: + return [] + + def _check_choices(self): + if not self.choices: + return [] + + def is_value(value, accept_promise=True): + return isinstance(value, (str, Promise) if accept_promise else str) or not is_iterable(value) + + if is_value(self.choices, accept_promise=False): + return [ + checks.Error( + "'choices' must be an iterable (e.g., a list or tuple).", + obj=self, + id='fields.E004', + ) + ] + + # Expect [group_name, [value, display]] + for choices_group in self.choices: + try: + group_name, group_choices = choices_group + except ValueError: + # Containing non-pairs + break + try: + if not all( + is_value(value) and is_value(human_name) + for value, human_name in group_choices + ): + break + except (TypeError, ValueError): + # No groups, choices in the form [value, display] + value, human_name = group_name, group_choices + if not is_value(value) or not is_value(human_name): + break + + # Special case: choices=['ab'] + if isinstance(choices_group, str): + break + else: + return [] + + return [ + checks.Error( + "'choices' must be an iterable containing " + "(actual value, human readable name) tuples.", + obj=self, + id='fields.E005', + ) + ] + + def _check_db_index(self): + if self.db_index not in (None, True, False): + return [ + checks.Error( + "'db_index' must be None, True or False.", + obj=self, + id='fields.E006', + ) + ] + else: + return [] + + def _check_null_allowed_for_primary_keys(self): + if (self.primary_key and self.null and + not connection.features.interprets_empty_strings_as_nulls): + # We cannot reliably check this for backends like Oracle which + # consider NULL and '' to be equal (and thus set up + # character-based fields a little differently). + return [ + checks.Error( + 'Primary keys must not have null=True.', + hint=('Set null=False on the field, or ' + 'remove primary_key=True argument.'), + obj=self, + id='fields.E007', + ) + ] + else: + return [] + + def _check_backend_specific_checks(self, **kwargs): + app_label = self.model._meta.app_label + for db in connections: + if router.allow_migrate(db, app_label, model_name=self.model._meta.model_name): + return connections[db].validation.check_field(self, **kwargs) + return [] + + def _check_validators(self): + errors = [] + for i, validator in enumerate(self.validators): + if not callable(validator): + errors.append( + checks.Error( + "All 'validators' must be callable.", + hint=( + "validators[{i}] ({repr}) isn't a function or " + "instance of a validator class.".format( + i=i, repr=repr(validator), + ) + ), + obj=self, + id='fields.E008', + ) + ) + return errors + + def _check_deprecation_details(self): + if self.system_check_removed_details is not None: + return [ + checks.Error( + self.system_check_removed_details.get( + 'msg', + '%s has been removed except for support in historical ' + 'migrations.' % self.__class__.__name__ + ), + hint=self.system_check_removed_details.get('hint'), + obj=self, + id=self.system_check_removed_details.get('id', 'fields.EXXX'), + ) + ] + elif self.system_check_deprecated_details is not None: + return [ + checks.Warning( + self.system_check_deprecated_details.get( + 'msg', + '%s has been deprecated.' % self.__class__.__name__ + ), + hint=self.system_check_deprecated_details.get('hint'), + obj=self, + id=self.system_check_deprecated_details.get('id', 'fields.WXXX'), + ) + ] + return [] + + def get_col(self, alias, output_field=None): + if output_field is None: + output_field = self + if alias != self.model._meta.db_table or output_field != self: + from django.db.models.expressions import Col + return Col(alias, self, output_field) + else: + return self.cached_col + + @cached_property + def cached_col(self): + from django.db.models.expressions import Col + return Col(self.model._meta.db_table, self) + + def select_format(self, compiler, sql, params): + """ + Custom format for select clauses. For example, GIS columns need to be + selected as AsText(table.col) on MySQL as the table.col data can't be + used by Django. + """ + return sql, params + + def deconstruct(self): + """ + Return enough information to recreate the field as a 4-tuple: + + * The name of the field on the model, if contribute_to_class() has + been run. + * The import path of the field, including the class:e.g. + django.db.models.IntegerField This should be the most portable + version, so less specific may be better. + * A list of positional arguments. + * A dict of keyword arguments. + + Note that the positional or keyword arguments must contain values of + the following types (including inner values of collection types): + + * None, bool, str, int, float, complex, set, frozenset, list, tuple, + dict + * UUID + * datetime.datetime (naive), datetime.date + * top-level classes, top-level functions - will be referenced by their + full import path + * Storage instances - these have their own deconstruct() method + + This is because the values here must be serialized into a text format + (possibly new Python code, possibly JSON) and these are the only types + with encoding handlers defined. + + There's no need to return the exact way the field was instantiated this + time, just ensure that the resulting field is the same - prefer keyword + arguments over positional ones, and omit parameters with their default + values. + """ + # Short-form way of fetching all the default parameters + keywords = {} + possibles = { + "verbose_name": None, + "primary_key": False, + "max_length": None, + "unique": False, + "blank": False, + "null": False, + "db_index": False, + "default": NOT_PROVIDED, + "editable": True, + "serialize": True, + "unique_for_date": None, + "unique_for_month": None, + "unique_for_year": None, + "choices": [], + "help_text": '', + "db_column": None, + "db_tablespace": None, + "auto_created": False, + "validators": [], + "error_messages": None, + } + attr_overrides = { + "unique": "_unique", + "error_messages": "_error_messages", + "validators": "_validators", + "verbose_name": "_verbose_name", + "db_tablespace": "_db_tablespace", + } + equals_comparison = {"choices", "validators"} + for name, default in possibles.items(): + value = getattr(self, attr_overrides.get(name, name)) + # Unroll anything iterable for choices into a concrete list + if name == "choices" and isinstance(value, collections.abc.Iterable): + value = list(value) + # Do correct kind of comparison + if name in equals_comparison: + if value != default: + keywords[name] = value + else: + if value is not default: + keywords[name] = value + # Work out path - we shorten it for known Django core fields + path = "%s.%s" % (self.__class__.__module__, self.__class__.__qualname__) + if path.startswith("django.db.models.fields.related"): + path = path.replace("django.db.models.fields.related", "django.db.models") + if path.startswith("django.db.models.fields.files"): + path = path.replace("django.db.models.fields.files", "django.db.models") + if path.startswith("django.db.models.fields.proxy"): + path = path.replace("django.db.models.fields.proxy", "django.db.models") + if path.startswith("django.db.models.fields"): + path = path.replace("django.db.models.fields", "django.db.models") + # Return basic info - other fields should override this. + return (self.name, path, [], keywords) + + def clone(self): + """ + Uses deconstruct() to clone a new copy of this Field. + Will not preserve any class attachments/attribute names. + """ + name, path, args, kwargs = self.deconstruct() + return self.__class__(*args, **kwargs) + + def __eq__(self, other): + # Needed for @total_ordering + if isinstance(other, Field): + return self.creation_counter == other.creation_counter + return NotImplemented + + def __lt__(self, other): + # This is needed because bisect does not take a comparison function. + if isinstance(other, Field): + return self.creation_counter < other.creation_counter + return NotImplemented + + def __hash__(self): + return hash(self.creation_counter) + + def __deepcopy__(self, memodict): + # We don't have to deepcopy very much here, since most things are not + # intended to be altered after initial creation. + obj = copy.copy(self) + if self.remote_field: + obj.remote_field = copy.copy(self.remote_field) + if hasattr(self.remote_field, 'field') and self.remote_field.field is self: + obj.remote_field.field = obj + memodict[id(self)] = obj + return obj + + def __copy__(self): + # We need to avoid hitting __reduce__, so define this + # slightly weird copy construct. + obj = Empty() + obj.__class__ = self.__class__ + obj.__dict__ = self.__dict__.copy() + return obj + + def __reduce__(self): + """ + Pickling should return the model._meta.fields instance of the field, + not a new copy of that field. So, use the app registry to load the + model and then the field back. + """ + if not hasattr(self, 'model'): + # Fields are sometimes used without attaching them to models (for + # example in aggregation). In this case give back a plain field + # instance. The code below will create a new empty instance of + # class self.__class__, then update its dict with self.__dict__ + # values - so, this is very close to normal pickle. + state = self.__dict__.copy() + # The _get_default cached_property can't be pickled due to lambda + # usage. + state.pop('_get_default', None) + return _empty, (self.__class__,), state + return _load_field, (self.model._meta.app_label, self.model._meta.object_name, + self.name) + + def get_pk_value_on_save(self, instance): + """ + Hook to generate new PK values on save. This method is called when + saving instances with no primary key value set. If this method returns + something else than None, then the returned value is used when saving + the new instance. + """ + if self.default: + return self.get_default() + return None + + def to_python(self, value): + """ + Convert the input value into the expected Python data type, raising + django.core.exceptions.ValidationError if the data can't be converted. + Return the converted value. Subclasses should override this. + """ + return value + + @cached_property + def validators(self): + """ + Some validators can't be created at field initialization time. + This method provides a way to delay their creation until required. + """ + return [*self.default_validators, *self._validators] + + def run_validators(self, value): + if value in self.empty_values: + return + + errors = [] + for v in self.validators: + try: + v(value) + except exceptions.ValidationError as e: + if hasattr(e, 'code') and e.code in self.error_messages: + e.message = self.error_messages[e.code] + errors.extend(e.error_list) + + if errors: + raise exceptions.ValidationError(errors) + + def validate(self, value, model_instance): + """ + Validate value and raise ValidationError if necessary. Subclasses + should override this to provide validation logic. + """ + if not self.editable: + # Skip validation for non-editable fields. + return + + if self.choices and value not in self.empty_values: + for option_key, option_value in self.choices: + if isinstance(option_value, (list, tuple)): + # This is an optgroup, so look inside the group for + # options. + for optgroup_key, optgroup_value in option_value: + if value == optgroup_key: + return + elif value == option_key: + return + raise exceptions.ValidationError( + self.error_messages['invalid_choice'], + code='invalid_choice', + params={'value': value}, + ) + + if value is None and not self.null: + raise exceptions.ValidationError(self.error_messages['null'], code='null') + + if not self.blank and value in self.empty_values: + raise exceptions.ValidationError(self.error_messages['blank'], code='blank') + + def clean(self, value, model_instance): + """ + Convert the value's type and run validation. Validation errors + from to_python() and validate() are propagated. Return the correct + value if no error is raised. + """ + value = self.to_python(value) + self.validate(value, model_instance) + self.run_validators(value) + return value + + def db_type_parameters(self, connection): + return DictWrapper(self.__dict__, connection.ops.quote_name, 'qn_') + + def db_check(self, connection): + """ + Return the database column check constraint for this field, for the + provided connection. Works the same way as db_type() for the case that + get_internal_type() does not map to a preexisting model field. + """ + data = self.db_type_parameters(connection) + try: + return connection.data_type_check_constraints[self.get_internal_type()] % data + except KeyError: + return None + + def db_type(self, connection): + """ + Return the database column data type for this field, for the provided + connection. + """ + # The default implementation of this method looks at the + # backend-specific data_types dictionary, looking up the field by its + # "internal type". + # + # A Field class can implement the get_internal_type() method to specify + # which *preexisting* Django Field class it's most similar to -- i.e., + # a custom field might be represented by a TEXT column type, which is + # the same as the TextField Django field type, which means the custom + # field's get_internal_type() returns 'TextField'. + # + # But the limitation of the get_internal_type() / data_types approach + # is that it cannot handle database column types that aren't already + # mapped to one of the built-in Django field types. In this case, you + # can implement db_type() instead of get_internal_type() to specify + # exactly which wacky database column type you want to use. + data = self.db_type_parameters(connection) + try: + return connection.data_types[self.get_internal_type()] % data + except KeyError: + return None + + def rel_db_type(self, connection): + """ + Return the data type that a related field pointing to this field should + use. For example, this method is called by ForeignKey and OneToOneField + to determine its data type. + """ + return self.db_type(connection) + + def cast_db_type(self, connection): + """Return the data type to use in the Cast() function.""" + db_type = connection.ops.cast_data_types.get(self.get_internal_type()) + if db_type: + return db_type % self.db_type_parameters(connection) + return self.db_type(connection) + + def db_parameters(self, connection): + """ + Extension of db_type(), providing a range of different return values + (type, checks). This will look at db_type(), allowing custom model + fields to override it. + """ + type_string = self.db_type(connection) + check_string = self.db_check(connection) + return { + "type": type_string, + "check": check_string, + } + + def db_type_suffix(self, connection): + return connection.data_types_suffix.get(self.get_internal_type()) + + def get_db_converters(self, connection): + if hasattr(self, 'from_db_value'): + return [self.from_db_value] + return [] + + @property + def unique(self): + return self._unique or self.primary_key + + @property + def db_tablespace(self): + return self._db_tablespace or settings.DEFAULT_INDEX_TABLESPACE + + def set_attributes_from_name(self, name): + self.name = self.name or name + self.attname, self.column = self.get_attname_column() + self.concrete = self.column is not None + if self.verbose_name is None and self.name: + self.verbose_name = self.name.replace('_', ' ') + + def contribute_to_class(self, cls, name, private_only=False): + """ + Register the field with the model class it belongs to. + + If private_only is True, create a separate instance of this field + for every subclass of cls, even if cls is not an abstract model. + """ + self.set_attributes_from_name(name) + self.model = cls + if private_only: + cls._meta.add_field(self, private=True) + else: + cls._meta.add_field(self) + if self.column: + # Don't override classmethods with the descriptor. This means that + # if you have a classmethod and a field with the same name, then + # such fields can't be deferred (we don't have a check for this). + if not getattr(cls, self.attname, None): + setattr(cls, self.attname, DeferredAttribute(self.attname)) + if self.choices: + setattr(cls, 'get_%s_display' % self.name, + partialmethod(cls._get_FIELD_display, field=self)) + + def get_filter_kwargs_for_object(self, obj): + """ + Return a dict that when passed as kwargs to self.model.filter(), would + yield all instances having the same value for this field as obj has. + """ + return {self.name: getattr(obj, self.attname)} + + def get_attname(self): + return self.name + + def get_attname_column(self): + attname = self.get_attname() + column = self.db_column or attname + return attname, column + + def get_internal_type(self): + return self.__class__.__name__ + + def pre_save(self, model_instance, add): + """Return field's value just before saving.""" + return getattr(model_instance, self.attname) + + def get_prep_value(self, value): + """Perform preliminary non-db specific value checks and conversions.""" + if isinstance(value, Promise): + value = value._proxy____cast() + return value + + def get_db_prep_value(self, value, connection, prepared=False): + """ + Return field's value prepared for interacting with the database backend. + + Used by the default implementations of get_db_prep_save(). + """ + if not prepared: + value = self.get_prep_value(value) + return value + + def get_db_prep_save(self, value, connection): + """Return field's value prepared for saving into a database.""" + return self.get_db_prep_value(value, connection=connection, prepared=False) + + def has_default(self): + """Return a boolean of whether this field has a default value.""" + return self.default is not NOT_PROVIDED + + def get_default(self): + """Return the default value for this field.""" + return self._get_default() + + @cached_property + def _get_default(self): + if self.has_default(): + if callable(self.default): + return self.default + return lambda: self.default + + if not self.empty_strings_allowed or self.null and not connection.features.interprets_empty_strings_as_nulls: + return return_None + return str # return empty string + + def get_choices(self, include_blank=True, blank_choice=BLANK_CHOICE_DASH, limit_choices_to=None): + """ + Return choices with a default blank choices included, for use + as